Strict concurrency for NIOResumableUpload (#265)

HTTPResumableUpload contains the core logic. It uses an event loop to
synchronize its state internally. Some methods are safe to call from off
of that event loop and have been moved to a new sendable view. The
HTTPResumableUpload type is marked as explicitly not sendable.

As such, most other types now hold on to the sendable view and use that
as the interface to HTTPResumableUpload.

HTTPResumableUploadChannel must be sendable (it's a Channel) and now
uses safe abstractions (where possible).

Package.swift

@@ -214,7 +214,8 @@ var targets: [PackageDescription.Target] = [
             .product(name: "NIOCore", package: "swift-nio"),
             .product(name: "StructuredFieldValues", package: "swift-http-structured-headers"),
             .product(name: "Atomics", package: "swift-atomics"),
-        ]
+        ],
+        swiftSettings: strictConcurrencySettings
     ),
     .executableTarget(
         name: "NIOResumableUploadDemo",
@@ -224,14 +225,16 @@ var targets: [PackageDescription.Target] = [
             .product(name: "HTTPTypes", package: "swift-http-types"),
             .product(name: "NIOCore", package: "swift-nio"),
             .product(name: "NIOPosix", package: "swift-nio"),
-        ]
+        ],
+        swiftSettings: strictConcurrencySettings
     ),
     .testTarget(
         name: "NIOResumableUploadTests",
         dependencies: [
             "NIOResumableUpload",
             .product(name: "NIOEmbedded", package: "swift-nio"),
-        ]
+        ],
+        swiftSettings: strictConcurrencySettings
     ),
     .target(
         name: "NIOHTTPResponsiveness",

Sources/NIOResumableUpload/HTTPResumableUpload.swift

@@ -63,7 +63,7 @@ final class HTTPResumableUpload {
 
     private func createChannel(handler: HTTPResumableUploadHandler, parent: Channel) -> HTTPResumableUploadChannel {
         let channel = HTTPResumableUploadChannel(
-            upload: self,
+            upload: self.sendableView,
             parent: parent,
             channelConfigurator: self.channelConfigurator
         )
@@ -89,6 +89,86 @@ final class HTTPResumableUpload {
     }
 }
 
+extension HTTPResumableUpload {
+    var sendableView: SendableView {
+        SendableView(handler: self)
+    }
+
+    struct SendableView: Sendable {
+        let eventLoop: any EventLoop
+        let loopBoundUpload: NIOLoopBound<HTTPResumableUpload>
+        let uploadHandlerChannel: NIOLockedValueBox<Channel?>
+
+        init(handler: HTTPResumableUpload) {
+            self.eventLoop = handler.eventLoop
+            self.loopBoundUpload = NIOLoopBound(handler, eventLoop: eventLoop)
+            self.uploadHandlerChannel = handler.uploadHandlerChannel
+        }
+
+        var parentChannel: Channel? {
+            self.uploadHandlerChannel.withLockedValue { $0 }
+        }
+
+        private func withHandlerOnEventLoop(
+            checkHandler: HTTPResumableUploadHandler,
+            _ work: @escaping @Sendable (HTTPResumableUpload) -> Void
+        ) {
+            let id = ObjectIdentifier(checkHandler)
+
+            if self.eventLoop.inEventLoop {
+                let upload = self.loopBoundUpload.value
+                if let handler = upload.uploadHandler, ObjectIdentifier(handler) == id {
+                    work(upload)
+                }
+            } else {
+                self.eventLoop.execute {
+                    let upload = self.loopBoundUpload.value
+                    if let handler = upload.uploadHandler, ObjectIdentifier(handler) == id {
+                        work(upload)
+                    }
+                }
+            }
+        }
+
+        func receive(handler: HTTPResumableUploadHandler, channel: Channel, part: HTTPRequestPart) {
+            self.withHandlerOnEventLoop(checkHandler: handler) { upload in
+                switch part {
+                case .head(let request):
+                    upload.receiveHead(handler: upload.uploadHandler!, channel: channel, request: request)
+                case .body(let body):
+                    upload.receiveBody(handler: upload.uploadHandler!, body: body)
+                case .end(let trailers):
+                    upload.receiveEnd(handler: upload.uploadHandler!, trailers: trailers)
+                }
+            }
+        }
+
+        func receiveComplete(handler: HTTPResumableUploadHandler) {
+            self.withHandlerOnEventLoop(checkHandler: handler) { upload in
+                upload.uploadChannel?.receiveComplete()
+            }
+        }
+
+        func writabilityChanged(handler: HTTPResumableUploadHandler) {
+            self.withHandlerOnEventLoop(checkHandler: handler) { upload in
+                upload.uploadChannel?.writabilityChanged()
+            }
+        }
+
+        func end(handler: HTTPResumableUploadHandler, error: Error?) {
+            self.withHandlerOnEventLoop(checkHandler: handler) { upload in
+                if !upload.uploadComplete && upload.resumePath != nil {
+                    upload.pendingError = error
+                    upload.detachUploadHandler(close: false)
+                } else {
+                    upload.destroyChannel(error: error)
+                    upload.detachUploadHandler(close: false)
+                }
+            }
+        }
+    }
+}
+
 // For `HTTPResumableUploadHandler`.
 extension HTTPResumableUpload {
     /// `HTTPResumableUpload` runs on the same event loop as the initial upload handler that started the upload.
@@ -99,22 +179,6 @@ extension HTTPResumableUpload {
         self.eventLoop = eventLoop
     }
 
-    private func runInEventLoop(_ work: @escaping () -> Void) {
-        if self.eventLoop.inEventLoop {
-            work()
-        } else {
-            self.eventLoop.execute(work)
-        }
-    }
-
-    private func runInEventLoop(checkHandler handler: HTTPResumableUploadHandler, _ work: @escaping () -> Void) {
-        self.runInEventLoop {
-            if self.uploadHandler === handler {
-                work()
-            }
-        }
-    }
-
     func attachUploadHandler(_ handler: HTTPResumableUploadHandler, channel: Channel) {
         self.eventLoop.preconditionInEventLoop()
 
@@ -146,7 +210,8 @@ extension HTTPResumableUpload {
 
             if self.uploadChannel != nil {
                 self.idleTimer?.cancel()
-                self.idleTimer = self.eventLoop.scheduleTask(in: self.context.timeout) {
+                // Unsafe unchecked is fine: there's a precondition on entering this function.
+                self.idleTimer = self.eventLoop.assumeIsolatedUnsafeUnchecked().scheduleTask(in: self.context.timeout) {
                     let error = self.pendingError ?? HTTPResumableUploadError.timeoutWaitingForResumption
                     self.uploadChannel?.end(error: error)
                     self.uploadChannel = nil
@@ -159,15 +224,13 @@ extension HTTPResumableUpload {
         otherHandler: HTTPResumableUploadHandler,
         version: HTTPResumableUploadProtocol.InteropVersion
     ) {
-        self.runInEventLoop {
+        self.detachUploadHandler(close: true)
-            self.detachUploadHandler(close: true)
+        let response = HTTPResumableUploadProtocol.offsetRetrievingResponse(
-            let response = HTTPResumableUploadProtocol.offsetRetrievingResponse(
+            offset: self.offset,
-                offset: self.offset,
+            complete: self.uploadComplete,
-                complete: self.uploadComplete,
+            version: version
-                version: version
+        )
-            )
+        self.respondAndDetach(response, handler: otherHandler)
-            self.respondAndDetach(response, handler: otherHandler)
-        }
     }
 
     private func saveUploadLength(complete: Bool, contentLength: Int64?, uploadLength: Int64?) -> Bool {
@@ -198,40 +261,36 @@ extension HTTPResumableUpload {
         uploadLength: Int64?,
         version: HTTPResumableUploadProtocol.InteropVersion
     ) {
-        self.runInEventLoop {
+        let conflict: Bool
-            let conflict: Bool
+        if self.uploadHandler == nil && self.offset == offset && !self.responseStarted {
-            if self.uploadHandler == nil && self.offset == offset && !self.responseStarted {
+            conflict = !self.saveUploadLength(
-                conflict = !self.saveUploadLength(
+                complete: complete,
-                    complete: complete,
+                contentLength: contentLength,
-                    contentLength: contentLength,
+                uploadLength: uploadLength
-                    uploadLength: uploadLength
+            )
-                )
+        } else {
-            } else {
+            conflict = true
-                conflict = true
-            }
-            guard !conflict else {
-                self.detachUploadHandler(close: true)
-                self.destroyChannel(error: HTTPResumableUploadError.badResumption)
-                let response = HTTPResumableUploadProtocol.conflictResponse(
-                    offset: self.offset,
-                    complete: self.uploadComplete,
-                    version: version
-                )
-                self.respondAndDetach(response, handler: otherHandler)
-                return
-            }
-            self.requestIsCreation = false
-            self.requestIsComplete = complete
-            self.interopVersion = version
-            self.attachUploadHandler(otherHandler, channel: channel)
         }
+        guard !conflict else {
+            self.detachUploadHandler(close: true)
+            self.destroyChannel(error: HTTPResumableUploadError.badResumption)
+            let response = HTTPResumableUploadProtocol.conflictResponse(
+                offset: self.offset,
+                complete: self.uploadComplete,
+                version: version
+            )
+            self.respondAndDetach(response, handler: otherHandler)
+            return
+        }
+        self.requestIsCreation = false
+        self.requestIsComplete = complete
+        self.interopVersion = version
+        self.attachUploadHandler(otherHandler, channel: channel)
     }
 
     private func uploadCancellation() {
-        self.runInEventLoop {
+        self.detachUploadHandler(close: true)
-            self.detachUploadHandler(close: true)
+        self.destroyChannel(error: HTTPResumableUploadError.uploadCancelled)
-            self.destroyChannel(error: HTTPResumableUploadError.uploadCancelled)
-        }
     }
 
     private func receiveHead(handler: HTTPResumableUploadHandler, channel: Channel, request: HTTPRequest) {
@@ -277,7 +336,7 @@ extension HTTPResumableUpload {
                 if let path = request.path, let upload = self.context.findUpload(path: path) {
                     self.uploadHandler = nil
                     self.uploadHandlerChannel.withLockedValue { $0 = nil }
-                    upload.offsetRetrieving(otherHandler: handler, version: version)
+                    upload.loopBoundUpload.value.offsetRetrieving(otherHandler: handler, version: version)
                 } else {
                     let response = HTTPResumableUploadProtocol.notFoundResponse(version: version)
                     self.respondAndDetach(response, handler: handler)
@@ -287,7 +346,7 @@ extension HTTPResumableUpload {
                     handler.upload = upload
                     self.uploadHandler = nil
                     self.uploadHandlerChannel.withLockedValue { $0 = nil }
-                    upload.uploadAppending(
+                    upload.loopBoundUpload.value.uploadAppending(
                         otherHandler: handler,
                         channel: channel,
                         offset: offset,
@@ -302,7 +361,7 @@ extension HTTPResumableUpload {
                 }
             case .uploadCancellation:
                 if let path = request.path, let upload = self.context.findUpload(path: path) {
-                    upload.uploadCancellation()
+                    upload.loopBoundUpload.value.uploadCancellation()
                     let response = HTTPResumableUploadProtocol.cancelledResponse(version: version)
                     self.respondAndDetach(response, handler: handler)
                 } else {
@@ -358,43 +417,6 @@ extension HTTPResumableUpload {
             self.uploadChannel?.receive(.end(trailers))
         }
     }
-
-    func receive(handler: HTTPResumableUploadHandler, channel: Channel, part: HTTPRequestPart) {
-        self.runInEventLoop(checkHandler: handler) {
-            switch part {
-            case .head(let request):
-                self.receiveHead(handler: handler, channel: channel, request: request)
-            case .body(let body):
-                self.receiveBody(handler: handler, body: body)
-            case .end(let trailers):
-                self.receiveEnd(handler: handler, trailers: trailers)
-            }
-        }
-    }
-
-    func receiveComplete(handler: HTTPResumableUploadHandler) {
-        self.runInEventLoop(checkHandler: handler) {
-            self.uploadChannel?.receiveComplete()
-        }
-    }
-
-    func writabilityChanged(handler: HTTPResumableUploadHandler) {
-        self.runInEventLoop(checkHandler: handler) {
-            self.uploadChannel?.writabilityChanged()
-        }
-    }
-
-    func end(handler: HTTPResumableUploadHandler, error: Error?) {
-        self.runInEventLoop(checkHandler: handler) {
-            if !self.uploadComplete && self.resumePath != nil {
-                self.pendingError = error
-                self.detachUploadHandler(close: false)
-            } else {
-                self.destroyChannel(error: error)
-                self.detachUploadHandler(close: false)
-            }
-        }
-    }
 }
 
 // For `HTTPResumableUploadChannel`.
@@ -477,6 +499,9 @@ extension HTTPResumableUpload {
     }
 }
 
+@available(*, unavailable)
+extension HTTPResumableUpload: Sendable {}
+
 /// Errors produced by resumable upload.
 enum HTTPResumableUploadError: Error {
     /// An upload cancelation request received.

Sources/NIOResumableUpload/HTTPResumableUploadChannel.swift

@@ -18,8 +18,10 @@ import NIOHTTPTypes
 
 /// The child channel that persists across upload resumption attempts, delivering data as if it is
 /// a single HTTP upload.
-final class HTTPResumableUploadChannel: Channel, ChannelCore {
+final class HTTPResumableUploadChannel: Channel, ChannelCore, @unchecked Sendable {
-    let upload: HTTPResumableUpload
+    // @unchecked because of '_pipeline' which is an IUO assigned during init.
+
+    let upload: HTTPResumableUpload.SendableView
 
     let allocator: ByteBufferAllocator
 
@@ -63,10 +65,10 @@ final class HTTPResumableUploadChannel: Channel, ChannelCore {
 
     let eventLoop: EventLoop
 
-    private var autoRead: Bool
+    private var autoRead: NIOLoopBound<Bool>
 
     init(
-        upload: HTTPResumableUpload,
+        upload: HTTPResumableUpload.SendableView,
         parent: Channel,
         channelConfigurator: (Channel) -> Void
     ) {
@@ -75,7 +77,8 @@ final class HTTPResumableUploadChannel: Channel, ChannelCore {
         self.closePromise = parent.eventLoop.makePromise()
         self.eventLoop = parent.eventLoop
         // Only support Channels that implement sync options
-        self.autoRead = try! parent.syncOptions!.getOption(ChannelOptions.autoRead)
+        let autoRead = try! parent.syncOptions!.getOption(ChannelOptions.autoRead)
+        self.autoRead = NIOLoopBound(autoRead, eventLoop: eventLoop)
         self._pipeline = ChannelPipeline(channel: self)
         channelConfigurator(self)
     }
@@ -109,7 +112,7 @@ final class HTTPResumableUploadChannel: Channel, ChannelCore {
 
         switch option {
         case _ as ChannelOptions.Types.AutoReadOption:
-            self.autoRead = value as! Bool
+            self.autoRead.value = value as! Bool
         default:
             if let parent = self.parent {
                 // Only support Channels that implement sync options
@@ -125,7 +128,7 @@ final class HTTPResumableUploadChannel: Channel, ChannelCore {
 
         switch option {
         case _ as ChannelOptions.Types.AutoReadOption:
-            return self.autoRead as! Option.Value
+            return self.autoRead.value as! Option.Value
         default:
             if let parent = self.parent {
                 // Only support Channels that implement sync options
@@ -157,23 +160,19 @@ final class HTTPResumableUploadChannel: Channel, ChannelCore {
     }
 
     func write0(_ data: NIOAny, promise: EventLoopPromise<Void>?) {
-        self.eventLoop.preconditionInEventLoop()
+        self.upload.loopBoundUpload.value.write(unwrapData(data), promise: promise)
-        self.upload.write(unwrapData(data), promise: promise)
     }
 
     func flush0() {
-        self.eventLoop.preconditionInEventLoop()
+        self.upload.loopBoundUpload.value.flush()
-        self.upload.flush()
     }
 
     func read0() {
-        self.eventLoop.preconditionInEventLoop()
+        self.upload.loopBoundUpload.value.read()
-        self.upload.read()
     }
 
     func close0(error: Error, mode: CloseMode, promise: EventLoopPromise<Void>?) {
-        self.eventLoop.preconditionInEventLoop()
+        self.upload.loopBoundUpload.value.close(mode: mode, promise: promise)
-        self.upload.close(mode: mode, promise: promise)
     }
 
     func triggerUserOutboundEvent0(_ event: Any, promise: EventLoopPromise<Void>?) {
@@ -228,7 +227,7 @@ extension HTTPResumableUploadChannel {
         self.eventLoop.preconditionInEventLoop()
         self.pipeline.fireChannelReadComplete()
 
-        if self.autoRead {
+        if self.autoRead.value {
             self.pipeline.read()
         }
     }

Sources/NIOResumableUpload/HTTPResumableUploadContext.swift

@@ -16,11 +16,11 @@ import NIOConcurrencyHelpers
 import NIOCore
 
 /// `HTTPResumableUploadContext` manages ongoing uploads.
-public final class HTTPResumableUploadContext {
+public final class HTTPResumableUploadContext: Sendable {
     let origin: String
     let path: String
     let timeout: TimeAmount
-    private let uploads: NIOLockedValueBox<[String: HTTPResumableUpload]> = .init([:])
+    private let uploads: NIOLockedValueBox<[String: HTTPResumableUpload.SendableView]> = .init([:])
 
     /// Create an `HTTPResumableUploadContext` for use with `HTTPResumableUploadHandler`.
     /// - Parameters:
@@ -51,7 +51,7 @@ public final class HTTPResumableUploadContext {
         let token = "\(random.next())-\(random.next())"
         self.uploads.withLockedValue {
             assert($0[token] == nil)
-            $0[token] = upload
+            $0[token] = upload.sendableView
         }
         return self.path(fromToken: token)
     }
@@ -65,7 +65,7 @@ public final class HTTPResumableUploadContext {
         }
     }
 
-    func findUpload(path: String) -> HTTPResumableUpload? {
+    func findUpload(path: String) -> HTTPResumableUpload.SendableView? {
         let token = token(fromPath: path)
         return self.uploads.withLockedValue {
             $0[token]

Sources/NIOResumableUpload/HTTPResumableUploadHandler.swift

@@ -24,7 +24,7 @@ public final class HTTPResumableUploadHandler: ChannelDuplexHandler {
     public typealias OutboundIn = Never
     public typealias OutboundOut = HTTPResponsePart
 
-    var upload: HTTPResumableUpload? = nil
+    var upload: HTTPResumableUpload.SendableView? = nil
     let createUpload: () -> HTTPResumableUpload
     var shouldReset: Bool = false
 
@@ -60,7 +60,7 @@ public final class HTTPResumableUploadHandler: ChannelDuplexHandler {
         self.createUpload = {
             HTTPResumableUpload(context: context) { channel in
                 if !handlers.isEmpty {
-                    _ = channel.pipeline.addHandlers(handlers)
+                    try? channel.pipeline.syncOperations.addHandlers(handlers)
                 }
             }
         }
@@ -73,7 +73,7 @@ public final class HTTPResumableUploadHandler: ChannelDuplexHandler {
         let upload = self.createUpload()
         upload.scheduleOnEventLoop(self.eventLoop)
         upload.attachUploadHandler(self, channel: context.channel)
-        self.upload = upload
+        self.upload = upload.sendableView
         self.shouldReset = false
     }
 
@@ -124,49 +124,32 @@ public final class HTTPResumableUploadHandler: ChannelDuplexHandler {
     }
 }
 
+@available(*, unavailable)
+extension HTTPResumableUploadHandler: Sendable {}
+
 // For `HTTPResumableUpload`.
 extension HTTPResumableUploadHandler {
-    private func runInEventLoop(_ work: @escaping () -> Void) {
-        if self.eventLoop.inEventLoop {
-            work()
-        } else {
-            self.eventLoop.execute(work)
-        }
-    }
-
     func write(_ part: HTTPResponsePart, promise: EventLoopPromise<Void>?) {
-        self.runInEventLoop {
+        self.context?.write(self.wrapOutboundOut(part), promise: promise)
-            self.context?.write(self.wrapOutboundOut(part), promise: promise)
-        }
     }
 
     func flush() {
-        self.runInEventLoop {
+        self.context?.flush()
-            self.context?.flush()
-        }
     }
 
     func writeAndFlush(_ part: HTTPResponsePart, promise: EventLoopPromise<Void>?) {
-        self.runInEventLoop {
+        self.context?.writeAndFlush(self.wrapOutboundOut(part), promise: promise)
-            self.context?.writeAndFlush(self.wrapOutboundOut(part), promise: promise)
-        }
     }
 
     func read() {
-        self.runInEventLoop {
+        self.context?.read()
-            self.context?.read()
-        }
     }
 
     func close(mode: CloseMode, promise: EventLoopPromise<Void>?) {
-        self.runInEventLoop {
+        self.context?.close(mode: mode, promise: promise)
-            self.context?.close(mode: mode, promise: promise)
-        }
     }
 
     func detach() {
-        self.runInEventLoop {
+        self.context = nil
-            self.context = nil
-        }
     }
 }

Tests/NIOResumableUploadTests/NIOResumableUploadTests.swift

@@ -49,7 +49,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         let request = HTTPRequest(method: .get, scheme: "https", authority: "example.com", path: "/")
         try channel.writeInbound(HTTPRequestPart.head(request))
@@ -66,7 +68,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         let request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         try channel.writeInbound(HTTPRequestPart.head(request))
@@ -85,7 +89,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, HTTPResponsePart>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .options, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "6"
@@ -118,7 +124,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "3"
@@ -150,7 +158,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "5"
@@ -182,7 +192,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "6"
@@ -215,7 +227,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "3"
@@ -235,7 +249,7 @@ final class NIOResumableUploadTests: XCTestCase {
         let resumptionPath = try XCTUnwrap(URLComponents(string: location)?.path)
 
         let channel2 = EmbeddedChannel()
-        try channel2.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel2.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request2 = HTTPRequest(method: .head, scheme: "https", authority: "example.com", path: resumptionPath)
         request2.headerFields[.uploadDraftInteropVersion] = "3"
         try channel2.writeInbound(HTTPRequestPart.head(request2))
@@ -251,7 +265,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertTrue(try channel2.finish().isClean)
 
         let channel3 = EmbeddedChannel()
-        try channel3.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel3.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request3 = HTTPRequest(method: .patch, scheme: "https", authority: "example.com", path: resumptionPath)
         request3.headerFields[.uploadDraftInteropVersion] = "3"
         request3.headerFields[.uploadIncomplete] = "?0"
@@ -277,7 +291,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "5"
@@ -298,7 +314,7 @@ final class NIOResumableUploadTests: XCTestCase {
         let resumptionPath = try XCTUnwrap(URLComponents(string: location)?.path)
 
         let channel2 = EmbeddedChannel()
-        try channel2.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel2.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request2 = HTTPRequest(method: .head, scheme: "https", authority: "example.com", path: resumptionPath)
         request2.headerFields[.uploadDraftInteropVersion] = "3"
         try channel2.writeInbound(HTTPRequestPart.head(request2))
@@ -314,7 +330,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertTrue(try channel2.finish().isClean)
 
         let channel3 = EmbeddedChannel()
-        try channel3.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel3.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request3 = HTTPRequest(method: .patch, scheme: "https", authority: "example.com", path: resumptionPath)
         request3.headerFields[.uploadDraftInteropVersion] = "5"
         request3.headerFields[.uploadComplete] = "?1"
@@ -341,7 +357,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "6"
@@ -362,7 +380,7 @@ final class NIOResumableUploadTests: XCTestCase {
         let resumptionPath = try XCTUnwrap(URLComponents(string: location)?.path)
 
         let channel2 = EmbeddedChannel()
-        try channel2.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel2.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request2 = HTTPRequest(method: .head, scheme: "https", authority: "example.com", path: resumptionPath)
         request2.headerFields[.uploadDraftInteropVersion] = "3"
         try channel2.writeInbound(HTTPRequestPart.head(request2))
@@ -378,7 +396,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertTrue(try channel2.finish().isClean)
 
         let channel3 = EmbeddedChannel()
-        try channel3.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel3.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request3 = HTTPRequest(method: .patch, scheme: "https", authority: "example.com", path: resumptionPath)
         request3.headerFields[.uploadDraftInteropVersion] = "6"
         request3.headerFields[.uploadComplete] = "?1"
@@ -406,7 +424,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "3"
@@ -434,7 +454,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertEqual(try channel.readOutbound(as: HTTPResponsePart.self), .end(nil))
 
         let channel2 = EmbeddedChannel()
-        try channel2.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel2.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request2 = HTTPRequest(method: .head, scheme: "https", authority: "example.com", path: resumptionPath)
         request2.headerFields[.uploadDraftInteropVersion] = "3"
         try channel2.writeInbound(HTTPRequestPart.head(request2))
@@ -450,7 +470,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertTrue(try channel2.finish().isClean)
 
         let channel3 = EmbeddedChannel()
-        try channel3.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel3.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request3 = HTTPRequest(method: .patch, scheme: "https", authority: "example.com", path: resumptionPath)
         request3.headerFields[.uploadDraftInteropVersion] = "3"
         request3.headerFields[.uploadIncomplete] = "?0"
@@ -476,7 +496,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "5"
@@ -505,7 +527,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertEqual(try channel.readOutbound(as: HTTPResponsePart.self), .end(nil))
 
         let channel2 = EmbeddedChannel()
-        try channel2.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel2.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request2 = HTTPRequest(method: .head, scheme: "https", authority: "example.com", path: resumptionPath)
         request2.headerFields[.uploadDraftInteropVersion] = "5"
         try channel2.writeInbound(HTTPRequestPart.head(request2))
@@ -521,7 +543,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertTrue(try channel2.finish().isClean)
 
         let channel3 = EmbeddedChannel()
-        try channel3.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel3.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request3 = HTTPRequest(method: .patch, scheme: "https", authority: "example.com", path: resumptionPath)
         request3.headerFields[.uploadDraftInteropVersion] = "5"
         request3.headerFields[.uploadComplete] = "?1"
@@ -548,7 +570,9 @@ final class NIOResumableUploadTests: XCTestCase {
         let recorder = InboundRecorder<HTTPRequestPart, Never>()
 
         let context = HTTPResumableUploadContext(origin: "https://example.com")
-        try channel.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [recorder])).wait()
+        try channel.pipeline.syncOperations.addHandler(
+            HTTPResumableUploadHandler(context: context, handlers: [recorder])
+        )
 
         var request = HTTPRequest(method: .post, scheme: "https", authority: "example.com", path: "/")
         request.headerFields[.uploadDraftInteropVersion] = "6"
@@ -577,7 +601,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertEqual(try channel.readOutbound(as: HTTPResponsePart.self), .end(nil))
 
         let channel2 = EmbeddedChannel()
-        try channel2.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel2.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request2 = HTTPRequest(method: .head, scheme: "https", authority: "example.com", path: resumptionPath)
         request2.headerFields[.uploadDraftInteropVersion] = "6"
         try channel2.writeInbound(HTTPRequestPart.head(request2))
@@ -593,7 +617,7 @@ final class NIOResumableUploadTests: XCTestCase {
         XCTAssertTrue(try channel2.finish().isClean)
 
         let channel3 = EmbeddedChannel()
-        try channel3.pipeline.addHandler(HTTPResumableUploadHandler(context: context, handlers: [])).wait()
+        try channel3.pipeline.syncOperations.addHandler(HTTPResumableUploadHandler(context: context, handlers: []))
         var request3 = HTTPRequest(method: .patch, scheme: "https", authority: "example.com", path: resumptionPath)
         request3.headerFields[.uploadDraftInteropVersion] = "6"
         request3.headerFields[.uploadComplete] = "?1"