From ae87ce50560673dce73a9fc34ed25958a50362fe Mon Sep 17 00:00:00 2001
From: Cory Benfield <lukasa@apple.com>
Date: Mon, 18 Nov 2024 17:40:00 +0000
Subject: [PATCH 01/37] Add netstat -s to nio-diagnose (#2977)

Motivation:

netstat -s can identify many system-wide issues that may be useful extra
data. There's no reason not to collect it.

Modifications:

Add netstat -s to the nio-diagnose script.

Result:

Better diagnostics
---
 scripts/nio-diagnose | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/scripts/nio-diagnose b/scripts/nio-diagnose
index 21e6d40cf2..4a661d3861 100755
--- a/scripts/nio-diagnose
+++ b/scripts/nio-diagnose
@@ -366,6 +366,16 @@ function analyse_process() {
     analyse_syscalls "$pid"
 }
 
+function produce_network_stats() {
+    declare -a command
+
+    command=( netstat -s )
+
+    output "### System-wide network statistics"
+    output
+    run_and_print "${command[@]}"
+}
+
 number_of_nio_pids=0
 
 output "# NIO diagnose ($(shasum "${BASH_SOURCE[0]}"))"
@@ -378,6 +388,7 @@ analyse_system_versions
 analyse_tcp_conns
 analyse_udp
 analyse_uds
+produce_network_stats
 analyse_procs
 analyse_ulimit
 

From af0e612e8f23c14ccf5243b6b738e50245333e0a Mon Sep 17 00:00:00 2001
From: George Barnett <gbarnett@apple.com>
Date: Tue, 19 Nov 2024 11:16:40 +0000
Subject: [PATCH 02/37] Use condition variable in NIOThreadPool (#2964)

Motivation:

The thread performance checker warns that a threads with QoS are waiting
on threads in the NIOThreadPool. We can avoid these warnings by using a
condition variable instead.

Modifications:

- Replace the usage of a semaphore and lock in NIOThreadPool with a
condition lock.
- The condition value indicates whether the threads have work to do
(where work is processing work items or exiting their run loop).

Result:

- Fewer warnings
- Resolves #2960
---
 Sources/NIOPosix/NIOThreadPool.swift | 187 ++++++++++++++++++++-------
 1 file changed, 141 insertions(+), 46 deletions(-)

diff --git a/Sources/NIOPosix/NIOThreadPool.swift b/Sources/NIOPosix/NIOThreadPool.swift
index e60a73812d..7854cbff7b 100644
--- a/Sources/NIOPosix/NIOThreadPool.swift
+++ b/Sources/NIOPosix/NIOThreadPool.swift
@@ -79,9 +79,22 @@ public final class NIOThreadPool {
         /// It should never be "leaked" outside of the lock block.
         case modifying
     }
-    private let semaphore = DispatchSemaphore(value: 0)
-    private let lock = NIOLock()
-    private var threads: [NIOThread]? = nil  // protected by `lock`
+
+    /// Whether threads in the pool have work.
+    private enum WorkState: Hashable {
+        case hasWork
+        case hasNoWork
+    }
+
+    // The condition lock is used in place of a lock and a semaphore to avoid warnings from the
+    // thread performance checker.
+    //
+    // Only the worker threads wait for the condition lock to take a value, no other threads need
+    // to wait for a given value. The value indicates whether the thread has some work to do. Work
+    // in this case can be either processing a work item or exiting the threads processing
+    // loop (i.e. shutting down).
+    private let conditionLock: ConditionLock<WorkState>
+    private var threads: [NIOThread]? = nil  // protected by `conditionLock`
     private var state: State = .stopped
 
     // WorkItems don't have a handle so they can't be cancelled directly. Instead an ID is assigned
@@ -124,7 +137,7 @@ public final class NIOThreadPool {
             return
         }
 
-        let threadsToJoin = self.lock.withLock { () -> [NIOThread] in
+        let threadsToJoin = self.conditionLock.withLock {
             switch self.state {
             case .running(let items):
                 self.state = .modifying
@@ -133,17 +146,17 @@ public final class NIOThreadPool {
                         item.workItem(.cancelled)
                     }
                 }
-                self.state = .shuttingDown(Array(repeating: true, count: numberOfThreads))
-                for _ in (0..<numberOfThreads) {
-                    self.semaphore.signal()
-                }
+                self.state = .shuttingDown(Array(repeating: true, count: self.numberOfThreads))
+
                 let threads = self.threads!
-                defer {
-                    self.threads = nil
-                }
-                return threads
+                self.threads = nil
+
+                // Each thread has work to do: shutdown.
+                return (unlockWith: .hasWork, result: threads)
+
             case .shuttingDown, .stopped:
-                return []
+                return (unlockWith: nil, result: [])
+
             case .modifying:
                 fatalError(".modifying state misuse")
             }
@@ -171,22 +184,33 @@ public final class NIOThreadPool {
     }
 
     private func _submit(id: Int?, _ body: @escaping WorkItem) {
-        let item = self.lock.withLock { () -> WorkItem? in
+        let submitted = self.conditionLock.withLock {
+            let workState: WorkState
+            let submitted: Bool
+
             switch self.state {
             case .running(var items):
                 self.state = .modifying
                 items.append(.init(workItem: body, id: id))
                 self.state = .running(items)
-                self.semaphore.signal()
-                return nil
+                workState = items.isEmpty ? .hasNoWork : .hasWork
+                submitted = true
+
             case .shuttingDown, .stopped:
-                return body
+                workState = .hasNoWork
+                submitted = false
+
             case .modifying:
                 fatalError(".modifying state misuse")
             }
+
+            return (unlockWith: workState, result: submitted)
         }
+
         // if item couldn't be added run it immediately indicating that it couldn't be run
-        item.map { $0(.cancelled) }
+        if !submitted {
+            body(.cancelled)
+        }
     }
 
     /// Initialize a `NIOThreadPool` thread pool with `numberOfThreads` threads.
@@ -209,17 +233,18 @@ public final class NIOThreadPool {
     private init(numberOfThreads: Int, canBeStopped: Bool) {
         self.numberOfThreads = numberOfThreads
         self.canBeStopped = canBeStopped
+        self.conditionLock = ConditionLock(value: .hasNoWork)
     }
 
     private func process(identifier: Int) {
         var itemAndState: (item: WorkItem, state: WorkItemState)? = nil
 
         repeat {
-            // wait until work has become available
-            itemAndState = nil  // ensure previous work item is not retained for duration of semaphore wait
-            self.semaphore.wait()
+            itemAndState = nil  // ensure previous work item is not retained while waiting for the condition
+            itemAndState = self.conditionLock.withLock(when: .hasWork) {
+                let workState: WorkState
+                let result: (WorkItem, WorkItemState)?
 
-            itemAndState = self.lock.withLock { () -> (WorkItem, WorkItemState)? in
                 switch self.state {
                 case .running(var items):
                     self.state = .modifying
@@ -233,18 +258,32 @@ public final class NIOThreadPool {
                     }
 
                     self.state = .running(items)
-                    return (itemAndID.workItem, state)
+
+                    workState = items.isEmpty ? .hasNoWork : .hasWork
+                    result = (itemAndID.workItem, state)
+
                 case .shuttingDown(var aliveStates):
+                    self.state = .modifying
                     assert(aliveStates[identifier])
                     aliveStates[identifier] = false
                     self.state = .shuttingDown(aliveStates)
-                    return nil
+
+                    // Unlock with '.hasWork' to resume any other threads waiting to shutdown.
+                    workState = .hasWork
+                    result = nil
+
                 case .stopped:
-                    return nil
+                    // Unreachable: 'stopped' is the initial state which is left when starting the
+                    // thread pool, and before any thread calls this function.
+                    fatalError("Invalid state")
+
                 case .modifying:
                     fatalError(".modifying state misuse")
                 }
+
+                return (unlockWith: workState, result: result)
             }
+
             // if there was a work item popped, run it
             itemAndState.map { item, state in item(state) }
         } while itemAndState != nil
@@ -256,16 +295,24 @@ public final class NIOThreadPool {
     }
 
     public func _start(threadNamePrefix: String) {
-        let alreadyRunning: Bool = self.lock.withLock {
+        let alreadyRunning = self.conditionLock.withLock {
             switch self.state {
-            case .running(_):
-                return true
-            case .shuttingDown(_):
+            case .running:
+                // Already running, this has no effect on whether there is more work for the
+                // threads to run.
+                return (unlockWith: nil, result: true)
+
+            case .shuttingDown:
                 // This should never happen
                 fatalError("start() called while in shuttingDown")
+
             case .stopped:
                 self.state = .running(Deque(minimumCapacity: 16))
-                return false
+                assert(self.threads == nil)
+                self.threads = []
+                self.threads!.reserveCapacity(self.numberOfThreads)
+                return (unlockWith: .hasNoWork, result: false)
+
             case .modifying:
                 fatalError(".modifying state misuse")
             }
@@ -278,21 +325,34 @@ public final class NIOThreadPool {
         // We use this condition lock as a tricky kind of semaphore.
         // This is done to sidestep the thread performance checker warning
         // that would otherwise be emitted.
-        let cond = ConditionLock(value: 0)
-
-        self.lock.withLock {
-            assert(self.threads == nil)
-            self.threads = []
-            self.threads?.reserveCapacity(self.numberOfThreads)
-        }
-
+        let readyThreads = ConditionLock(value: 0)
         for id in 0..<self.numberOfThreads {
             // We should keep thread names under 16 characters because Linux doesn't allow more.
             NIOThread.spawnAndRun(name: "\(threadNamePrefix)\(id)", detachThread: false) { thread in
-                self.lock.withLock {
-                    self.threads!.append(thread)
-                    cond.lock()
-                    cond.unlock(withValue: self.threads!.count)
+                readyThreads.withLock {
+                    let threadCount = self.conditionLock.withLock {
+                        self.threads!.append(thread)
+                        let workState: WorkState
+
+                        switch self.state {
+                        case .running(let items):
+                            workState = items.isEmpty ? .hasNoWork : .hasWork
+                        case .shuttingDown:
+                            // The thread has work to do: it's shutting down.
+                            workState = .hasWork
+                        case .stopped:
+                            // Unreachable: .stopped always transitions to .running in the function
+                            // and .stopped is never entered again.
+                            fatalError("Invalid state")
+                        case .modifying:
+                            fatalError(".modifying state misuse")
+                        }
+
+                        let threadCount = self.threads!.count
+                        return (unlockWith: workState, result: threadCount)
+                    }
+
+                    return (unlockWith: threadCount, result: ())
                 }
 
                 self.process(identifier: id)
@@ -300,9 +360,15 @@ public final class NIOThreadPool {
             }
         }
 
-        cond.lock(whenValue: self.numberOfThreads)
-        cond.unlock()
-        assert(self.lock.withLock { self.threads?.count ?? -1 } == self.numberOfThreads)
+        readyThreads.lock(whenValue: self.numberOfThreads)
+        readyThreads.unlock()
+
+        func threadCount() -> Int {
+            self.conditionLock.withLock {
+                (unlockWith: nil, result: self.threads?.count ?? -1)
+            }
+        }
+        assert(threadCount() == self.numberOfThreads)
     }
 
     deinit {
@@ -374,8 +440,9 @@ extension NIOThreadPool {
                 }
             }
         } onCancel: {
-            self.lock.withLockVoid {
+            self.conditionLock.withLock {
                 self.cancelledWorkIDs.insert(workID)
+                return (unlockWith: nil, result: ())
             }
         }
     }
@@ -427,3 +494,31 @@ extension NIOThreadPool {
         }
     }
 }
+
+extension ConditionLock {
+    @inlinable
+    func _lock(when value: T?) {
+        if let value = value {
+            self.lock(whenValue: value)
+        } else {
+            self.lock()
+        }
+    }
+
+    @inlinable
+    func _unlock(with value: T?) {
+        if let value = value {
+            self.unlock(withValue: value)
+        } else {
+            self.unlock()
+        }
+    }
+
+    @inlinable
+    func withLock<Result>(when value: T? = nil, _ body: () -> (unlockWith: T?, result: Result)) -> Result {
+        self._lock(when: value)
+        let (unlockValue, result) = body()
+        self._unlock(with: unlockValue)
+        return result
+    }
+}

From 2a3a333d8b35a0decb8980497a604e51313c4b2a Mon Sep 17 00:00:00 2001
From: Cory Benfield <lukasa@apple.com>
Date: Wed, 20 Nov 2024 10:28:14 +0000
Subject: [PATCH 03/37] Better handle ECONNRESET on connected datagram sockets.
 (#2979)

Motivation:

When a connected datagram socket sends a datagram to a port or host that
is not listening, an ICMP Destination Unreachable message may be
returned. That message triggers an ECONNRESET to be produced at the
socket layer.

On Darwin we handled this well, but on Linux it turned out that this
would push us into connection teardown and, eventually, into a crash.
Not so good.

To better handle this, we need to distinguish EPOLLERR from EPOLLHUP on
datagram sockets. In these cases, we should check whether the socket
error was fatal and, if it was not, we should continue our execution
having fired the error down the pipeline.

Modifications:

Modify the selector code to distinguish reset and error. Add support for
our channels to handle errors.
Have most channels handle errors as resets.
Override the logic for datagram channels to duplicate the logic in
readable.
Add a unit test.

Result:

Better datagrams for all.
---
 Sources/NIOPosix/BaseSocketChannel.swift      | 13 +++-
 Sources/NIOPosix/PipeChannel.swift            |  8 +--
 Sources/NIOPosix/SelectableChannel.swift      |  8 +++
 Sources/NIOPosix/SelectableEventLoop.swift    | 18 +++++-
 Sources/NIOPosix/SelectorEpoll.swift          |  5 +-
 Sources/NIOPosix/SelectorGeneric.swift        | 12 +++-
 Sources/NIOPosix/SelectorKqueue.swift         |  2 +-
 Sources/NIOPosix/SocketChannel.swift          | 31 +++++++--
 .../NIOPosixTests/DatagramChannelTests.swift  | 55 ++++++++++++++++
 Tests/NIOPosixTests/SALChannelTests.swift     | 64 +++++++++----------
 Tests/NIOPosixTests/SelectorTest.swift        |  6 +-
 .../SyscallAbstractionLayer.swift             | 12 ++--
 12 files changed, 175 insertions(+), 59 deletions(-)

diff --git a/Sources/NIOPosix/BaseSocketChannel.swift b/Sources/NIOPosix/BaseSocketChannel.swift
index 164d5802bd..50668f35a5 100644
--- a/Sources/NIOPosix/BaseSocketChannel.swift
+++ b/Sources/NIOPosix/BaseSocketChannel.swift
@@ -1091,7 +1091,8 @@ class BaseSocketChannel<SocketType: BaseSocketProtocol>: SelectableChannel, Chan
                 let result: Int32 = try self.socket.getOption(level: .socket, name: .so_error)
                 if result != 0 {
                     // we have a socket error, let's forward
-                    // this path will be executed on Linux (EPOLLERR) & Darwin (ev.fflags != 0)
+                    // this path will be executed on Linux (EPOLLERR) & Darwin (ev.fflags != 0) for
+                    // stream sockets, and most (but not all) errors on datagram sockets
                     error = IOError(errnoCode: result, reason: "connection reset (error set)")
                 } else {
                     // we don't have a socket error, this must be connection reset without an error then
@@ -1209,6 +1210,14 @@ class BaseSocketChannel<SocketType: BaseSocketProtocol>: SelectableChannel, Chan
         true
     }
 
+    /// Handles an error reported by the selector.
+    ///
+    /// Default behaviour is to treat this as if it were a reset.
+    func error() -> ErrorResult {
+        self.reset()
+        return .fatal
+    }
+
     internal final func updateCachedAddressesFromSocket(updateLocal: Bool = true, updateRemote: Bool = true) {
         self.eventLoop.assertInEventLoop()
         assert(updateLocal || updateRemote)
@@ -1331,7 +1340,7 @@ class BaseSocketChannel<SocketType: BaseSocketProtocol>: SelectableChannel, Chan
         // The initial set of interested events must not contain `.readEOF` because when connect doesn't return
         // synchronously, kevent might send us a `readEOF` because the `writable` event that marks the connect as completed.
         // See SocketChannelTest.testServerClosesTheConnectionImmediately for a regression test.
-        try self.safeRegister(interested: [.reset])
+        try self.safeRegister(interested: [.reset, .error])
         self.lifecycleManager.finishRegistration()(nil, self.pipeline)
     }
 
diff --git a/Sources/NIOPosix/PipeChannel.swift b/Sources/NIOPosix/PipeChannel.swift
index a049cf9f91..1a85a30359 100644
--- a/Sources/NIOPosix/PipeChannel.swift
+++ b/Sources/NIOPosix/PipeChannel.swift
@@ -68,7 +68,7 @@ final class PipeChannel: BaseStreamSocketChannel<PipePair> {
         if let inputFD = self.pipePair.inputFD {
             try selector.register(
                 selectable: inputFD,
-                interested: interested.intersection([.read, .reset]),
+                interested: interested.intersection([.read, .reset, .error]),
                 makeRegistration: self.registrationForInput
             )
         }
@@ -76,7 +76,7 @@ final class PipeChannel: BaseStreamSocketChannel<PipePair> {
         if let outputFD = self.pipePair.outputFD {
             try selector.register(
                 selectable: outputFD,
-                interested: interested.intersection([.write, .reset]),
+                interested: interested.intersection([.write, .reset, .error]),
                 makeRegistration: self.registrationForOutput
             )
         }
@@ -95,13 +95,13 @@ final class PipeChannel: BaseStreamSocketChannel<PipePair> {
         if let inputFD = self.pipePair.inputFD, inputFD.isOpen {
             try selector.reregister(
                 selectable: inputFD,
-                interested: interested.intersection([.read, .reset])
+                interested: interested.intersection([.read, .reset, .error])
             )
         }
         if let outputFD = self.pipePair.outputFD, outputFD.isOpen {
             try selector.reregister(
                 selectable: outputFD,
-                interested: interested.intersection([.write, .reset])
+                interested: interested.intersection([.write, .reset, .error])
             )
         }
     }
diff --git a/Sources/NIOPosix/SelectableChannel.swift b/Sources/NIOPosix/SelectableChannel.swift
index 20882d915e..2155a9b6ae 100644
--- a/Sources/NIOPosix/SelectableChannel.swift
+++ b/Sources/NIOPosix/SelectableChannel.swift
@@ -44,9 +44,17 @@ internal protocol SelectableChannel: Channel {
     /// Called when the `SelectableChannel` was reset (ie. is now unusable)
     func reset()
 
+    /// Called when the `SelectableChannel` had an error reported on the selector.
+    func error() -> ErrorResult
+
     func register(selector: Selector<NIORegistration>, interested: SelectorEventSet) throws
 
     func deregister(selector: Selector<NIORegistration>, mode: CloseMode) throws
 
     func reregister(selector: Selector<NIORegistration>, interested: SelectorEventSet) throws
 }
+
+internal enum ErrorResult {
+    case fatal
+    case nonFatal
+}
diff --git a/Sources/NIOPosix/SelectableEventLoop.swift b/Sources/NIOPosix/SelectableEventLoop.swift
index 2ec9c71605..20fe17fc41 100644
--- a/Sources/NIOPosix/SelectableEventLoop.swift
+++ b/Sources/NIOPosix/SelectableEventLoop.swift
@@ -443,6 +443,18 @@ internal final class SelectableEventLoop: EventLoop {
         if ev.contains(.reset) {
             channel.reset()
         } else {
+            if ev.contains(.error) {
+                switch channel.error() {
+                case .fatal:
+                    return
+                case .nonFatal:
+                    break
+                }
+
+                guard channel.isOpen else {
+                    return
+                }
+            }
             if ev.contains(.writeEOF) {
                 channel.writeEOF()
 
@@ -746,10 +758,10 @@ internal final class SelectableEventLoop: EventLoop {
                         self.handleEvent(ev.io, channel: chan)
                     case .pipeChannel(let chan, let direction):
                         var ev = ev
-                        if ev.io.contains(.reset) {
-                            // .reset needs special treatment here because we're dealing with two separate pipes instead
+                        if ev.io.contains(.reset) || ev.io.contains(.error) {
+                            // .reset and .error needs special treatment here because we're dealing with two separate pipes instead
                             // of one socket. So we turn .reset input .readEOF/.writeEOF.
-                            ev.io.subtract([.reset])
+                            ev.io.subtract([.reset, .error])
                             ev.io.formUnion([direction == .input ? .readEOF : .writeEOF])
                         }
                         self.handleEvent(ev.io, channel: chan)
diff --git a/Sources/NIOPosix/SelectorEpoll.swift b/Sources/NIOPosix/SelectorEpoll.swift
index 61a1132a15..208aa3a950 100644
--- a/Sources/NIOPosix/SelectorEpoll.swift
+++ b/Sources/NIOPosix/SelectorEpoll.swift
@@ -90,7 +90,10 @@ extension SelectorEventSet {
         if epollEvent.events & Epoll.EPOLLRDHUP != 0 {
             selectorEventSet.formUnion(.readEOF)
         }
-        if epollEvent.events & Epoll.EPOLLHUP != 0 || epollEvent.events & Epoll.EPOLLERR != 0 {
+        if epollEvent.events & Epoll.EPOLLERR != 0 {
+            selectorEventSet.formUnion(.error)
+        }
+        if epollEvent.events & Epoll.EPOLLHUP != 0 {
             selectorEventSet.formUnion(.reset)
         }
         self = selectorEventSet
diff --git a/Sources/NIOPosix/SelectorGeneric.swift b/Sources/NIOPosix/SelectorGeneric.swift
index db4d5ce5b4..d97496d6a6 100644
--- a/Sources/NIOPosix/SelectorGeneric.swift
+++ b/Sources/NIOPosix/SelectorGeneric.swift
@@ -62,7 +62,7 @@ struct SelectorEventSet: OptionSet, Equatable {
     /// of flags or to compare against spurious wakeups.
     static let _none = SelectorEventSet([])
 
-    /// Connection reset or other errors.
+    /// Connection reset.
     static let reset = SelectorEventSet(rawValue: 1 << 0)
 
     /// EOF at the read/input end of a `Selectable`.
@@ -79,6 +79,9 @@ struct SelectorEventSet: OptionSet, Equatable {
     /// - Note: This is rarely used because in many cases, there is no signal that this happened.
     static let writeEOF = SelectorEventSet(rawValue: 1 << 4)
 
+    /// Error encountered.
+    static let error = SelectorEventSet(rawValue: 1 << 5)
+
     init(rawValue: SelectorEventSet.RawValue) {
         self.rawValue = rawValue
     }
@@ -237,7 +240,7 @@ internal class Selector<R: Registration> {
         makeRegistration: (SelectorEventSet, SelectorRegistrationID) -> R
     ) throws {
         assert(self.myThread == NIOThread.current)
-        assert(interested.contains(.reset))
+        assert(interested.contains([.reset, .error]))
         guard self.lifecycleState == .open else {
             throw IOError(errnoCode: EBADF, reason: "can't register on selector as it's \(self.lifecycleState).")
         }
@@ -265,7 +268,10 @@ internal class Selector<R: Registration> {
         guard self.lifecycleState == .open else {
             throw IOError(errnoCode: EBADF, reason: "can't re-register on selector as it's \(self.lifecycleState).")
         }
-        assert(interested.contains(.reset), "must register for at least .reset but tried registering for \(interested)")
+        assert(
+            interested.contains([.reset, .error]),
+            "must register for at least .reset & .error but tried registering for \(interested)"
+        )
         try selectable.withUnsafeHandle { fd in
             var reg = registrations[Int(fd)]!
             try self.reregister0(
diff --git a/Sources/NIOPosix/SelectorKqueue.swift b/Sources/NIOPosix/SelectorKqueue.swift
index 26d13298c1..49a264d730 100644
--- a/Sources/NIOPosix/SelectorKqueue.swift
+++ b/Sources/NIOPosix/SelectorKqueue.swift
@@ -240,7 +240,7 @@ extension Selector: _SelectorBackendProtocol {
     ) throws {
         try kqueueUpdateEventNotifications(
             selectable: selectable,
-            interested: .reset,
+            interested: [.reset, .error],
             oldInterested: oldInterested,
             registrationID: registrationID
         )
diff --git a/Sources/NIOPosix/SocketChannel.swift b/Sources/NIOPosix/SocketChannel.swift
index 245d880dda..e1a9a473c0 100644
--- a/Sources/NIOPosix/SocketChannel.swift
+++ b/Sources/NIOPosix/SocketChannel.swift
@@ -840,10 +840,8 @@ final class DatagramChannel: BaseSocketChannel<Socket> {
         #endif
     }
 
-    override func shouldCloseOnReadError(_ err: Error) -> Bool {
-        guard let err = err as? IOError else { return true }
-
-        switch err.errnoCode {
+    private func shouldCloseOnErrnoCode(_ errnoCode: CInt) -> Bool {
+        switch errnoCode {
         // ECONNREFUSED can happen on linux if the previous sendto(...) failed.
         // See also:
         // -    https://bugzilla.redhat.com/show_bug.cgi?id=1375
@@ -857,6 +855,31 @@ final class DatagramChannel: BaseSocketChannel<Socket> {
         }
     }
 
+    override func shouldCloseOnReadError(_ err: Error) -> Bool {
+        guard let err = err as? IOError else { return true }
+        return self.shouldCloseOnErrnoCode(err.errnoCode)
+    }
+
+    override func error() -> ErrorResult {
+        // Assume we can get the error from the socket.
+        do {
+            let errnoCode: CInt = try self.socket.getOption(level: .socket, name: .so_error)
+            if self.shouldCloseOnErrnoCode(errnoCode) {
+                self.reset()
+                return .fatal
+            } else {
+                self.pipeline.syncOperations.fireErrorCaught(
+                    IOError(errnoCode: errnoCode, reason: "so_error")
+                )
+                return .nonFatal
+            }
+        } catch {
+            // Unknown error, fatal.
+            self.reset()
+            return .fatal
+        }
+    }
+
     /// Buffer a write in preparation for a flush.
     ///
     /// When the channel is unconnected, `data` _must_ be of type `AddressedEnvelope<ByteBuffer>`.
diff --git a/Tests/NIOPosixTests/DatagramChannelTests.swift b/Tests/NIOPosixTests/DatagramChannelTests.swift
index 2fae280af4..601f5faa26 100644
--- a/Tests/NIOPosixTests/DatagramChannelTests.swift
+++ b/Tests/NIOPosixTests/DatagramChannelTests.swift
@@ -34,6 +34,17 @@ extension Channel {
         }.wait()
     }
 
+    func waitForErrors(count: Int) throws -> [any Error] {
+        try self.pipeline.context(name: "ByteReadRecorder").flatMap { context in
+            if let future = (context.handler as? DatagramReadRecorder<ByteBuffer>)?.notifyForErrors(count) {
+                return future
+            }
+
+            XCTFail("Could not wait for errors")
+            return self.eventLoop.makeSucceededFuture([])
+        }.wait()
+    }
+
     func readCompleteCount() throws -> Int {
         try self.pipeline.context(name: "ByteReadRecorder").map { context in
             (context.handler as! DatagramReadRecorder<ByteBuffer>).readCompleteCount
@@ -66,10 +77,12 @@ final class DatagramReadRecorder<DataType>: ChannelInboundHandler {
     }
 
     var reads: [AddressedEnvelope<DataType>] = []
+    var errors: [any Error] = []
     var loop: EventLoop? = nil
     var state: State = .fresh
 
     var readWaiters: [Int: EventLoopPromise<[AddressedEnvelope<DataType>]>] = [:]
+    var errorWaiters: [Int: EventLoopPromise<[any Error]>] = [:]
     var readCompleteCount = 0
 
     func channelRegistered(context: ChannelHandlerContext) {
@@ -95,6 +108,16 @@ final class DatagramReadRecorder<DataType>: ChannelInboundHandler {
         context.fireChannelRead(Self.wrapInboundOut(data))
     }
 
+    func errorCaught(context: ChannelHandlerContext, error: any Error) {
+        self.errors.append(error)
+
+        if let promise = self.errorWaiters.removeValue(forKey: self.errors.count) {
+            promise.succeed(self.errors)
+        }
+
+        context.fireErrorCaught(error)
+    }
+
     func channelReadComplete(context: ChannelHandlerContext) {
         self.readCompleteCount += 1
         context.fireChannelReadComplete()
@@ -108,6 +131,15 @@ final class DatagramReadRecorder<DataType>: ChannelInboundHandler {
         readWaiters[count] = loop!.makePromise()
         return readWaiters[count]!.futureResult
     }
+
+    func notifyForErrors(_ count: Int) -> EventLoopFuture<[any Error]> {
+        guard self.errors.count < count else {
+            return self.loop!.makeSucceededFuture(.init(self.errors.prefix(count)))
+        }
+
+        self.errorWaiters[count] = self.loop!.makePromise()
+        return self.errorWaiters[count]!.futureResult
+    }
 }
 
 class DatagramChannelTests: XCTestCase {
@@ -1712,6 +1744,29 @@ class DatagramChannelTests: XCTestCase {
         }
     }
 
+    func testShutdownReadOnConnectedUDP() throws {
+        var buffer = self.firstChannel.allocator.buffer(capacity: 256)
+        buffer.writeStaticString("hello, world!")
+
+        // Connect and write
+        XCTAssertNoThrow(try self.firstChannel.connect(to: self.secondChannel.localAddress!).wait())
+
+        let writeData = AddressedEnvelope(remoteAddress: self.secondChannel.localAddress!, data: buffer)
+        XCTAssertNoThrow(try self.firstChannel.writeAndFlush(writeData).wait())
+        _ = try self.secondChannel.waitForDatagrams(count: 1)
+
+        // Ok, close on the second channel.
+        XCTAssertNoThrow(try self.secondChannel.close(mode: .all).wait())
+        print("closed")
+
+        // Write again.
+        XCTAssertNoThrow(try self.firstChannel.writeAndFlush(writeData).wait())
+
+        // This should trigger an error.
+        let errors = try self.firstChannel.waitForErrors(count: 1)
+        XCTAssertEqual((errors[0] as? IOError)?.errnoCode, ECONNREFUSED)
+    }
+
     private func hasGoodGROSupport() throws -> Bool {
         // Source code for UDP_GRO was added in Linux 5.0. However, this support is somewhat limited
         // and some sources indicate support was actually added in 5.10 (perhaps more widely
diff --git a/Tests/NIOPosixTests/SALChannelTests.swift b/Tests/NIOPosixTests/SALChannelTests.swift
index 51802ce3f7..80a2337f8c 100644
--- a/Tests/NIOPosixTests/SALChannelTests.swift
+++ b/Tests/NIOPosixTests/SALChannelTests.swift
@@ -149,7 +149,7 @@ final class SALChannelTest: XCTestCase, SALTest {
                 // Next, we expect a reregistration which adds the `.write` notification
                 try self.assertReregister { selectable, eventSet in
                     XCTAssert(selectable as? Socket === channel.socket)
-                    XCTAssertEqual([.read, .reset, .readEOF, .write], eventSet)
+                    XCTAssertEqual([.read, .reset, .error, .readEOF, .write], eventSet)
                     return true
                 }
 
@@ -201,7 +201,7 @@ final class SALChannelTest: XCTestCase, SALTest {
                 // And lastly, after having written everything, we'd expect to unregister for write
                 try self.assertReregister { selectable, eventSet in
                     XCTAssert(selectable as? Socket === channel.socket)
-                    XCTAssertEqual([.read, .reset, .readEOF], eventSet)
+                    XCTAssertEqual([.read, .reset, .error, .readEOF], eventSet)
                     return true
                 }
 
@@ -315,11 +315,11 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertLocalAddress(address: localAddress)
                 try self.assertRemoteAddress(address: localAddress)
                 try self.assertRegister { selectable, event, Registration in
-                    XCTAssertEqual([.reset], event)
+                    XCTAssertEqual([.reset, .error], event)
                     return true
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
                 try self.assertDeregister { selectable in
@@ -360,11 +360,11 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertLocalAddress(address: localAddress)
                 try self.assertRemoteAddress(address: localAddress)
                 try self.assertRegister { selectable, event, Registration in
-                    XCTAssertEqual([.reset], event)
+                    XCTAssertEqual([.reset, .error], event)
                     return true
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
                 try self.assertDeregister { selectable in
@@ -431,19 +431,19 @@ final class SALChannelTest: XCTestCase, SALTest {
                         XCTAssertEqual(localAddress, channel.localAddress)
                         XCTAssertEqual(remoteAddress, channel.remoteAddress)
                         XCTAssertEqual(eventSet, registrationEventSet)
-                        XCTAssertEqual(.reset, eventSet)
+                        XCTAssertEqual([.reset, .error], eventSet)
                         return true
                     } else {
                         return false
                     }
                 }
                 try self.assertReregister { selectable, eventSet in
-                    XCTAssertEqual([.reset, .readEOF], eventSet)
+                    XCTAssertEqual([.reset, .error, .readEOF], eventSet)
                     return true
                 }
                 // because autoRead is on by default
                 try self.assertReregister { selectable, eventSet in
-                    XCTAssertEqual([.reset, .readEOF, .read], eventSet)
+                    XCTAssertEqual([.reset, .error, .readEOF, .read], eventSet)
                     return true
                 }
 
@@ -504,19 +504,19 @@ final class SALChannelTest: XCTestCase, SALTest {
                         XCTAssertEqual(localAddress, channel.localAddress)
                         XCTAssertEqual(remoteAddress, channel.remoteAddress)
                         XCTAssertEqual(eventSet, registrationEventSet)
-                        XCTAssertEqual(.reset, eventSet)
+                        XCTAssertEqual([.reset, .error], eventSet)
                         return true
                     } else {
                         return false
                     }
                 }
                 try self.assertReregister { selectable, eventSet in
-                    XCTAssertEqual([.reset, .readEOF], eventSet)
+                    XCTAssertEqual([.reset, .error, .readEOF], eventSet)
                     return true
                 }
                 // because autoRead is on by default
                 try self.assertReregister { selectable, eventSet in
-                    XCTAssertEqual([.reset, .readEOF, .read], eventSet)
+                    XCTAssertEqual([.reset, .error, .readEOF, .read], eventSet)
                     return true
                 }
 
@@ -549,11 +549,11 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertConnect(expectedAddress: serverAddress, result: false)
                 try self.assertLocalAddress(address: localAddress)
                 try self.assertRegister { selectable, event, Registration in
-                    XCTAssertEqual([.reset], event)
+                    XCTAssertEqual([.reset, .error], event)
                     return true
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .write], event)
+                    XCTAssertEqual([.reset, .error, .write], event)
                     return true
                 }
 
@@ -570,7 +570,7 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertRemoteAddress(address: serverAddress)
 
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF, .write], event)
+                    XCTAssertEqual([.reset, .error, .readEOF, .write], event)
                     return true
                 }
                 try self.assertWritev(
@@ -621,11 +621,11 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertLocalAddress(address: localAddress)
                 try self.assertRemoteAddress(address: serverAddress)
                 try self.assertRegister { selectable, event, Registration in
-                    XCTAssertEqual([.reset], event)
+                    XCTAssertEqual([.reset, .error], event)
                     return true
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
                 try self.assertWritev(
@@ -676,11 +676,11 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertLocalAddress(address: localAddress)
                 try self.assertRemoteAddress(address: serverAddress)
                 try self.assertRegister { selectable, event, Registration in
-                    XCTAssertEqual([.reset], event)
+                    XCTAssertEqual([.reset, .error], event)
                     return true
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
                 try self.assertWritev(
@@ -690,7 +690,7 @@ final class SALChannelTest: XCTestCase, SALTest {
                 )
                 try self.assertWrite(expectedFD: .max, expectedBytes: secondWrite, return: .wouldBlock(0))
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF, .write], event)
+                    XCTAssertEqual([.reset, .error, .readEOF, .write], event)
                     return true
                 }
 
@@ -741,11 +741,11 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertLocalAddress(address: localAddress)
                 try self.assertRemoteAddress(address: serverAddress)
                 try self.assertRegister { selectable, event, Registration in
-                    XCTAssertEqual([.reset], event)
+                    XCTAssertEqual([.reset, .error], event)
                     return true
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
                 try self.assertWritev(
@@ -815,14 +815,14 @@ final class SALChannelTest: XCTestCase, SALTest {
                         XCTAssertEqual(localAddress, channel.localAddress)
                         XCTAssertEqual(remoteAddress, channel.remoteAddress)
                         XCTAssertEqual(eventSet, registrationEventSet)
-                        XCTAssertEqual(.reset, eventSet)
+                        XCTAssertEqual([.reset, .error], eventSet)
                         return true
                     } else {
                         return false
                     }
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
 
@@ -895,14 +895,14 @@ final class SALChannelTest: XCTestCase, SALTest {
                         XCTAssertEqual(localAddress, channel.localAddress)
                         XCTAssertEqual(remoteAddress, channel.remoteAddress)
                         XCTAssertEqual(eventSet, registrationEventSet)
-                        XCTAssertEqual(.reset, eventSet)
+                        XCTAssertEqual([.reset, .error], eventSet)
                         return true
                     } else {
                         return false
                     }
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
 
@@ -913,7 +913,7 @@ final class SALChannelTest: XCTestCase, SALTest {
                 )
                 try self.assertWrite(expectedFD: .max, expectedBytes: secondWrite, return: .wouldBlock(0))
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF, .write], event)
+                    XCTAssertEqual([.reset, .error, .readEOF, .write], event)
                     return true
                 }
 
@@ -983,14 +983,14 @@ final class SALChannelTest: XCTestCase, SALTest {
                         XCTAssertEqual(localAddress, channel.localAddress)
                         XCTAssertEqual(remoteAddress, channel.remoteAddress)
                         XCTAssertEqual(eventSet, registrationEventSet)
-                        XCTAssertEqual(.reset, eventSet)
+                        XCTAssertEqual([.reset, .error], eventSet)
                         return true
                     } else {
                         return false
                     }
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF], event)
+                    XCTAssertEqual([.reset, .error, .readEOF], event)
                     return true
                 }
 
@@ -1064,11 +1064,11 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertConnect(expectedAddress: serverAddress, result: false)
                 try self.assertLocalAddress(address: localAddress)
                 try self.assertRegister { selectable, event, Registration in
-                    XCTAssertEqual([.reset], event)
+                    XCTAssertEqual([.reset, .error], event)
                     return true
                 }
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .write], event)
+                    XCTAssertEqual([.reset, .error, .write], event)
                     return true
                 }
 
@@ -1085,7 +1085,7 @@ final class SALChannelTest: XCTestCase, SALTest {
                 try self.assertRemoteAddress(address: serverAddress)
 
                 try self.assertReregister { selectable, event in
-                    XCTAssertEqual([.reset, .readEOF, .write], event)
+                    XCTAssertEqual([.reset, .error, .readEOF, .write], event)
                     return true
                 }
                 try self.assertWritev(
diff --git a/Tests/NIOPosixTests/SelectorTest.swift b/Tests/NIOPosixTests/SelectorTest.swift
index e2560b64d3..c98958cf38 100644
--- a/Tests/NIOPosixTests/SelectorTest.swift
+++ b/Tests/NIOPosixTests/SelectorTest.swift
@@ -82,7 +82,7 @@ class SelectorTest: XCTestCase {
         // Register both sockets with .write. This will ensure both are ready when calling selector.whenReady.
         try selector.register(
             selectable: socket1,
-            interested: [.reset, .write],
+            interested: [.reset, .error, .write],
             makeRegistration: { ev, regID in
                 TestRegistration(socket: socket1, interested: ev, registrationID: regID)
             }
@@ -90,7 +90,7 @@ class SelectorTest: XCTestCase {
 
         try selector.register(
             selectable: socket2,
-            interested: [.reset, .write],
+            interested: [.reset, .error, .write],
             makeRegistration: { ev, regID in
                 TestRegistration(socket: socket2, interested: ev, registrationID: regID)
             }
@@ -477,7 +477,7 @@ class SelectorTest: XCTestCase {
                             + " This should really only ever happen in very bizarre conditions."
                     )
                 }
-                channel.interestedEvent = [.readEOF, .reset]
+                channel.interestedEvent = [.readEOF, .reset, .error]
                 func workaroundSR9815() {
                     channel.registerAlreadyConfigured0(promise: nil)
                 }
diff --git a/Tests/NIOPosixTests/SyscallAbstractionLayer.swift b/Tests/NIOPosixTests/SyscallAbstractionLayer.swift
index ab5b075110..6137b43491 100644
--- a/Tests/NIOPosixTests/SyscallAbstractionLayer.swift
+++ b/Tests/NIOPosixTests/SyscallAbstractionLayer.swift
@@ -753,19 +753,19 @@ extension SALTest {
                     XCTAssertEqual(localAddress, channel.localAddress)
                     XCTAssertEqual(remoteAddress, channel.remoteAddress)
                     XCTAssertEqual(eventSet, registrationEventSet)
-                    XCTAssertEqual(.reset, eventSet)
+                    XCTAssertEqual([.reset, .error], eventSet)
                     return true
                 } else {
                     return false
                 }
             }
             try self.assertReregister { selectable, eventSet in
-                XCTAssertEqual([.reset, .readEOF], eventSet)
+                XCTAssertEqual([.reset, .error, .readEOF], eventSet)
                 return true
             }
             // because autoRead is on by default
             try self.assertReregister { selectable, eventSet in
-                XCTAssertEqual([.reset, .readEOF, .read], eventSet)
+                XCTAssertEqual([.reset, .error, .readEOF, .read], eventSet)
                 return true
             }
         } _: {
@@ -795,19 +795,19 @@ extension SALTest {
                     XCTAssertEqual(localAddress, channel.localAddress)
                     XCTAssertEqual(nil, channel.remoteAddress)
                     XCTAssertEqual(eventSet, registrationEventSet)
-                    XCTAssertEqual(.reset, eventSet)
+                    XCTAssertEqual([.reset, .error], eventSet)
                     return true
                 } else {
                     return false
                 }
             }
             try self.assertReregister { selectable, eventSet in
-                XCTAssertEqual([.reset, .readEOF], eventSet)
+                XCTAssertEqual([.reset, .error, .readEOF], eventSet)
                 return true
             }
             // because autoRead is on by default
             try self.assertReregister { selectable, eventSet in
-                XCTAssertEqual([.reset, .readEOF, .read], eventSet)
+                XCTAssertEqual([.reset, .error, .readEOF, .read], eventSet)
                 return true
             }
         } _: {

From 64eb8bfeb066882d7d107aae27516a35abeca7bc Mon Sep 17 00:00:00 2001
From: Si Beaumont <beaumont@apple.com>
Date: Thu, 21 Nov 2024 10:33:24 +0000
Subject: [PATCH 04/37] tests: Remove asserts on the value of local Vsock CID
 (#2975)

### Motivation:

We have some tests for our Vsock support that only run when we detect
the system supports Vsock. These weren't running in CI but, now we've
migrated CI to GitHub Actions, they can. In prototyping this though, I
discovered that one of our tests was too strict, and will fail for VMs
running on Hyper-V, which is the hypervisor used by GitHub actions.

Specifically, we have support for getting the local context ID by a
channel option. This is implemented by an ioctl, but the semantics of
the return value differ between the various Vsock transport kernel
modules:

- `virtio_transport`: returns the guest CID, which is a number greater
than `VMADDR_CID_HOST`, but less than `VMADDR_CID_ANY`, and returns
`VMADDR_CID_ANY` as an error.
- `vsock_loopback`: returns `VMADDR_CID_LOCAL` always.
- `vmci_transport`: returns the guest CID if the guest and host
transport are active; `VMADDR_CID_HOST` if only the host transport is
active; and `VMCI_INVALID_ID` otherwise, which happens to be the same
value as `VMADDR_CID_ANY`.
- `hyperv_transport`: returns `VMADDR_CID_ANY` always.

For this reason, we should probably remove any attempts to interpret the
validity of the value that comes back from the driver and users will
need to know what to do with it.

### Modifications:

- tests: Only run Vsock echo tests on Linux, when `vsock_loopback` is
loaded.
- tests: Remove asserts on the value of local Vsock CID.
- ci: Add pipeline that runs Vsock tests.

### Result:

- Vsock tests will no longer run unless `vsock_loopback` kernel module
is loaded.
- Vsock tests will no longer fail in Hyper-V VMs.
- Vsock tests will now run in CI.
---
 .github/workflows/pull_request.yml             | 18 ++++++++++++++++++
 Sources/NIOPosix/VsockAddress.swift            |  2 +-
 .../AsyncChannelBootstrapTests.swift           |  2 +-
 Tests/NIOPosixTests/EchoServerClientTest.swift |  2 +-
 Tests/NIOPosixTests/TestUtils.swift            | 17 ++++-------------
 Tests/NIOPosixTests/VsockAddressTest.swift     |  8 ++++----
 6 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 81ac778df8..1e5a3b7340 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -41,3 +41,21 @@ jobs:
     with:
       name: "Integration tests"
       matrix_linux_command: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq && ./scripts/integration_tests.sh"
+
+  vsock-tests:
+    name: Vsock tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Load vsock_loopback kernel module
+        run: sudo modprobe vsock_loopback
+      - name: Build package tests
+        run: swift build --build-tests
+      - name: Run Vsock tests
+        shell: bash  # explicitly choose bash, which ensures -o pipefail
+        run: swift test --filter "(?i)vsock" | tee test.out
+      - name: Check for skipped tests
+        run: test -r test.out && ! grep -i skipped test.out
diff --git a/Sources/NIOPosix/VsockAddress.swift b/Sources/NIOPosix/VsockAddress.swift
index 40b33673ae..437bbba0aa 100644
--- a/Sources/NIOPosix/VsockAddress.swift
+++ b/Sources/NIOPosix/VsockAddress.swift
@@ -217,7 +217,7 @@ extension VsockAddress.ContextID {
     /// - On Darwin, the `ioctl()` request operates on a socket.
     /// - On Linux, the `ioctl()` request operates on the `/dev/vsock` device.
     ///
-    /// - Note: On Linux, ``local`` may be a better choice.
+    /// - Note: The semantics of this `ioctl` vary between vsock transports on Linux; ``local`` may be more suitable.
     static func getLocalContextID(_ socketFD: NIOBSDSocket.Handle) throws -> Self {
         #if canImport(Darwin)
         let request = CNIODarwin_IOCTL_VM_SOCKETS_GET_LOCAL_CID
diff --git a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
index 62dfbbc291..e1dc3d215b 100644
--- a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
+++ b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
@@ -1141,7 +1141,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     // MARK: VSock
 
     func testVSock() async throws {
-        try XCTSkipUnless(System.supportsVsock, "No vsock transport available")
+        try XCTSkipUnless(System.supportsVsockLoopback, "No vsock loopback transport available")
         let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
         defer {
             try! eventLoopGroup.syncShutdownGracefully()
diff --git a/Tests/NIOPosixTests/EchoServerClientTest.swift b/Tests/NIOPosixTests/EchoServerClientTest.swift
index f32c15b284..2be0133d9d 100644
--- a/Tests/NIOPosixTests/EchoServerClientTest.swift
+++ b/Tests/NIOPosixTests/EchoServerClientTest.swift
@@ -234,7 +234,7 @@ class EchoServerClientTest: XCTestCase {
     }
 
     func testEchoVsock() throws {
-        try XCTSkipUnless(System.supportsVsock, "No vsock transport available")
+        try XCTSkipUnless(System.supportsVsockLoopback, "No vsock loopback transport available")
         let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
         defer {
             XCTAssertNoThrow(try group.syncShutdownGracefully())
diff --git a/Tests/NIOPosixTests/TestUtils.swift b/Tests/NIOPosixTests/TestUtils.swift
index c646379156..c2bb121903 100644
--- a/Tests/NIOPosixTests/TestUtils.swift
+++ b/Tests/NIOPosixTests/TestUtils.swift
@@ -30,19 +30,10 @@ extension System {
         }
     }
 
-    static var supportsVsock: Bool {
-        #if canImport(Darwin) || os(Linux) || os(Android)
-        guard let socket = try? Socket(protocolFamily: .vsock, type: .stream) else { return false }
-        XCTAssertNoThrow(try socket.close())
-        #if !canImport(Darwin)
-        do {
-            let fd = try Posix.open(file: "/dev/vsock", oFlag: O_RDONLY | O_CLOEXEC)
-            try Posix.close(descriptor: fd)
-        } catch {
-            return false
-        }
-        #endif
-        return true
+    static var supportsVsockLoopback: Bool {
+        #if os(Linux) || os(Android)
+        guard let modules = try? String(contentsOf: URL(fileURLWithPath: "/proc/modules")) else { return false }
+        return modules.split(separator: "\n").compactMap({ $0.split(separator: " ").first }).contains("vsock_loopback")
         #else
         return false
         #endif
diff --git a/Tests/NIOPosixTests/VsockAddressTest.swift b/Tests/NIOPosixTests/VsockAddressTest.swift
index c32924a8bb..6c9f44c101 100644
--- a/Tests/NIOPosixTests/VsockAddressTest.swift
+++ b/Tests/NIOPosixTests/VsockAddressTest.swift
@@ -58,15 +58,15 @@ class VsockAddressTest: XCTestCase {
     }
 
     func testGetLocalCID() throws {
-        try XCTSkipUnless(System.supportsVsock)
+        try XCTSkipUnless(System.supportsVsockLoopback, "No vsock loopback transport available")
 
         let socket = try ServerSocket(protocolFamily: .vsock, setNonBlocking: true)
         defer { try? socket.close() }
 
-        // Check the local CID is valid: higher than reserved values, but not VMADDR_CID_ANY.
+        // Check we can get the local CID using the static property on ContextID.
         let localCID = try socket.withUnsafeHandle(VsockAddress.ContextID.getLocalContextID)
-        XCTAssertNotEqual(localCID, .any)
-        XCTAssertGreaterThan(localCID.rawValue, VsockAddress.ContextID.host.rawValue)
+
+        // Check the local CID from the socket matches.
         XCTAssertEqual(try socket.getLocalVsockContextID(), localCID)
 
         // Check the local CID from the channel option matches.

From 035d3a57c9580e9fc3d333a4e913912d5cb5531f Mon Sep 17 00:00:00 2001
From: Cory Benfield <lukasa@apple.com>
Date: Thu, 21 Nov 2024 16:55:29 +0000
Subject: [PATCH 05/37] Add back alloc limits script with JSON mode (#2987)

Motivation:

We removed the alloc limits script, but it is still very useful. It just
couldn't produce JSON. So now it can!

Modifications:

Re-adds the alloc limits script.
Makes it produce JSON.

Result:

JSON can be produced from the CI output.
---
 dev/alloc-limits-from-test-output | 104 ++++++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100755 dev/alloc-limits-from-test-output

diff --git a/dev/alloc-limits-from-test-output b/dev/alloc-limits-from-test-output
new file mode 100755
index 0000000000..2fa9322ce4
--- /dev/null
+++ b/dev/alloc-limits-from-test-output
@@ -0,0 +1,104 @@
+#!/bin/bash
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2021 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+# This script allows you to consume any Jenkins/alloc counter run output and
+# convert it into the right for for the docker-compose script.
+
+set -eu
+
+mode_flag=${1---docker-compose}
+
+function usage() {
+    echo >&1 "Usage: $0 [--docker-compose|--export|--json]"
+    echo >&1
+    echo >&1 "Example:"
+    echo >&1 "  # copy the output from the Jenkins CI into your clipboard, then"
+    echo >&1 "  pbpaste | $0 --docker-compose"
+}
+
+function die() {
+    echo >&2 "ERROR: $*"
+    exit 1
+}
+
+case "$mode_flag" in
+    --docker-compose)
+        mode=docker
+        ;;
+    --export)
+        mode="export"
+        ;;
+    --json)
+        mode=json
+        ;;
+    *)
+        usage
+        exit 1
+        ;;
+esac
+
+function allow_slack() {
+    raw="$1"
+    if [[ ! "$raw" =~ ^[0-9]+$ ]]; then
+        die "not a malloc count: '$raw'"
+    fi
+    if [[ "$raw" -lt 1000 ]]; then
+        echo "$raw"
+        return
+    fi
+
+    allocs=$raw
+    while true; do
+        allocs=$(( allocs + 1 ))
+        if [[ "$allocs" =~ [0-9]+00$ || "$allocs" =~ [0-9]+50$ ]]; then
+            echo "$allocs"
+            return
+        fi
+    done
+}
+
+json_blob="{"
+
+lines=$(grep -e "total number of mallocs" -e ".total_allocations" -e "export MAX_ALLOCS_ALLOWED_" | \
+    sed -e "s/: total number of mallocs: /=/g" \
+        -e "s/.total_allocations: /=/g" \
+        -e "s/info: /test_/g" \
+        -e "s/export MAX_ALLOCS_ALLOWED_/test_/g" | \
+    grep -Eo 'test_[a-zA-Z0-9_-]+=[0-9]+' | sort | uniq)
+
+while read -r info; do
+    test_name=$(echo "$info" | sed "s/test_//g" | cut -d= -f1 )
+    allocs=$(allow_slack "$(echo "$info" | cut -d= -f2 | sed "s/ //g")")
+    case "$mode" in
+        docker)
+            echo "      - MAX_ALLOCS_ALLOWED_$test_name=$allocs"
+            ;;
+        export)
+            echo "export MAX_ALLOCS_ALLOWED_$test_name=$allocs"
+            ;;
+        json)
+            json_blob="${json_blob}"$'\n'"    \"$test_name\": $allocs,"
+            ;;
+        *)
+            die "Unexpected mode: $mode"
+            ;;
+    esac
+done <<< "$lines"
+
+if [[ "$mode" == "json" ]]; then
+    json_blob=$(sed '$ s/,$//g' <<< "$json_blob")
+    json_blob+=$'\n'"}"
+    echo "$json_blob"
+fi

From 74cf44e2618c5ccd72336a20af1f4ecc0ca018cc Mon Sep 17 00:00:00 2001
From: Adam Fowler <adamfowler71@gmail.com>
Date: Thu, 21 Nov 2024 17:44:42 +0000
Subject: [PATCH 06/37] Add ByteBuffer methods getUTF8ValidatedString and
 readUTF8ValidatedString (#2973)

Add methods to ByteBuffer to read validated UTF8 strings

### Motivation:

The current `readString` and `getString` methods of `ByteBuffer` do not
verify that the string being read is valid UTF8. The Swift 6 standard
library comes with a new initialiser `String(validating:as:)`. This PR
adds alternative methods to ByteBuffer which uses this instead of
`String(decoding:as:)`.

### Modifications:

Added `ByteBuffer.getUTF8ValidatedString(at:length:)`
Added `ByteBuffer.readUTF8ValidatedString(length:)`

### Result:

You can read strings from a ByteBuffer and be certain they are valid
UTF8

---------

Co-authored-by: Cory Benfield <lukasa@apple.com>
---
 Sources/NIOCore/ByteBuffer-aux.swift    | 63 +++++++++++++++++++++++++
 Tests/NIOCoreTests/ByteBufferTest.swift | 47 ++++++++++++++++++
 2 files changed, 110 insertions(+)

diff --git a/Sources/NIOCore/ByteBuffer-aux.swift b/Sources/NIOCore/ByteBuffer-aux.swift
index 799b3f8511..5e07f4af2b 100644
--- a/Sources/NIOCore/ByteBuffer-aux.swift
+++ b/Sources/NIOCore/ByteBuffer-aux.swift
@@ -902,3 +902,66 @@ extension Optional where Wrapped == ByteBuffer {
         }
     }
 }
+
+#if compiler(>=6)
+extension ByteBuffer {
+    /// Get the string at `index` from this `ByteBuffer` decoding using the UTF-8 encoding. Does not move the reader index.
+    /// The selected bytes must be readable or else `nil` will be returned.
+    ///
+    /// This is an alternative to `ByteBuffer.getString(at:length:)` which ensures the returned string is valid UTF8. If the
+    /// string is not valid UTF8 then a `ReadUTF8ValidationError` error is thrown.
+    ///
+    /// - Parameters:
+    ///   - index: The starting index into `ByteBuffer` containing the string of interest.
+    ///   - length: The number of bytes making up the string.
+    /// - Returns: A `String` value containing the UTF-8 decoded selected bytes from this `ByteBuffer` or `nil` if
+    ///            the requested bytes are not readable.
+    @inlinable
+    @available(macOS 15, iOS 18, tvOS 18, watchOS 11, *)
+    public func getUTF8ValidatedString(at index: Int, length: Int) throws -> String? {
+        guard let slice = self.getSlice(at: index, length: length) else {
+            return nil
+        }
+        guard
+            let string = String(
+                validating: slice.readableBytesView,
+                as: Unicode.UTF8.self
+            )
+        else {
+            throw ReadUTF8ValidationError.invalidUTF8
+        }
+        return string
+    }
+
+    /// Read `length` bytes off this `ByteBuffer`, decoding it as `String` using the UTF-8 encoding. Move the reader index
+    /// forward by `length`.
+    ///
+    /// This is an alternative to `ByteBuffer.readString(length:)` which ensures the returned string is valid UTF8. If the
+    /// string is not valid UTF8 then a `ReadUTF8ValidationError` error is thrown and the reader index is not advanced.
+    ///
+    /// - Parameters:
+    ///   - length: The number of bytes making up the string.
+    /// - Returns: A `String` value deserialized from this `ByteBuffer` or `nil` if there aren't at least `length` bytes readable.
+    @inlinable
+    @available(macOS 15, iOS 18, tvOS 18, watchOS 11, *)
+    public mutating func readUTF8ValidatedString(length: Int) throws -> String? {
+        guard let result = try self.getUTF8ValidatedString(at: self.readerIndex, length: length) else {
+            return nil
+        }
+        self.moveReaderIndex(forwardBy: length)
+        return result
+    }
+
+    /// Errors thrown when calling `readUTF8ValidatedString` or `getUTF8ValidatedString`.
+    public struct ReadUTF8ValidationError: Error, Equatable {
+        private enum BaseError: Hashable {
+            case invalidUTF8
+        }
+
+        private var baseError: BaseError
+
+        /// The length of the bytes to copy was negative.
+        public static let invalidUTF8: ReadUTF8ValidationError = .init(baseError: .invalidUTF8)
+    }
+}
+#endif  // compiler(>=6)
diff --git a/Tests/NIOCoreTests/ByteBufferTest.swift b/Tests/NIOCoreTests/ByteBufferTest.swift
index 3826b7e1c4..6be0ce37a6 100644
--- a/Tests/NIOCoreTests/ByteBufferTest.swift
+++ b/Tests/NIOCoreTests/ByteBufferTest.swift
@@ -1315,6 +1315,53 @@ class ByteBufferTest: XCTestCase {
         XCTAssertEqual("a", buf.readString(length: 1))
     }
 
+    @available(macOS 15, iOS 18, tvOS 18, watchOS 11, *)
+    func testReadUTF8ValidatedString() throws {
+        #if compiler(>=6)
+        buf.clear()
+        let expected = "hello"
+        buf.writeString(expected)
+        let actual = try buf.readUTF8ValidatedString(length: expected.utf8.count)
+        XCTAssertEqual(expected, actual)
+        XCTAssertEqual("", try buf.readUTF8ValidatedString(length: 0))
+        XCTAssertNil(try buf.readUTF8ValidatedString(length: 1))
+        #else
+        throw XCTSkip("'readUTF8ValidatedString' is only available in Swift 6 and later")
+        #endif  // compiler(>=6)
+    }
+
+    @available(macOS 15, iOS 18, tvOS 18, watchOS 11, *)
+    func testGetUTF8ValidatedString() throws {
+        #if compiler(>=6)
+        buf.clear()
+        let expected = "hello, goodbye"
+        buf.writeString(expected)
+        let actual = try buf.getUTF8ValidatedString(at: 7, length: 7)
+        XCTAssertEqual("goodbye", actual)
+        #else
+        throw XCTSkip("'getUTF8ValidatedString' is only available in Swift 6 and later")
+        #endif  // compiler(>=6)
+    }
+
+    @available(macOS 15, iOS 18, tvOS 18, watchOS 11, *)
+    func testReadUTF8InvalidString() throws {
+        #if compiler(>=6)
+        buf.clear()
+        buf.writeBytes([UInt8](repeating: 255, count: 16))
+        XCTAssertThrowsError(try buf.readUTF8ValidatedString(length: 16)) { error in
+            switch error {
+            case is ByteBuffer.ReadUTF8ValidationError:
+                break
+            default:
+                XCTFail("Error: \(error)")
+            }
+        }
+        XCTAssertEqual(buf.readableBytes, 16)
+        #else
+        throw XCTSkip("'readUTF8ValidatedString' is only available in Swift 6 and later")
+        #endif  // compiler(>=6)
+    }
+
     func testSetIntegerBeyondCapacity() throws {
         var buf = ByteBufferAllocator().buffer(capacity: 32)
         XCTAssertLessThan(buf.capacity, 200)

From 2a8811acd6916527143962998612c8e07999a7ec Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Thu, 21 Nov 2024 18:22:42 +0000
Subject: [PATCH 07/37] EventLoopFuture.waitSpinningRunLoop() (#2985)

### Motivation:

In some (probably niche) scenarios, especially in pre-Concurrency UI
applications on Darwin, it can be useful to wait for an a value whilst
still running the current `RunLoop`. That allows the UI and other things
to work whilst we're waiting for a future to complete.

### Modifications:

- Add `NIOFoundationCompat.EventLoopFuture.waitSpinningRunLoop()`.

### Result:

Better compatibility with Cocoa.
---
 .../WaitSpinningRunLoop.swift                 | 72 +++++++++++++++++++
 .../WaitSpinningRunLoopTests.swift            | 64 +++++++++++++++++
 2 files changed, 136 insertions(+)
 create mode 100644 Sources/NIOFoundationCompat/WaitSpinningRunLoop.swift
 create mode 100644 Tests/NIOFoundationCompatTests/WaitSpinningRunLoopTests.swift

diff --git a/Sources/NIOFoundationCompat/WaitSpinningRunLoop.swift b/Sources/NIOFoundationCompat/WaitSpinningRunLoop.swift
new file mode 100644
index 0000000000..9b9e16f955
--- /dev/null
+++ b/Sources/NIOFoundationCompat/WaitSpinningRunLoop.swift
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftNIO open source project
+//
+// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Atomics
+import Foundation
+import NIOConcurrencyHelpers
+import NIOCore
+
+extension EventLoopFuture {
+    /// Wait for the resolution of this `EventLoopFuture` by spinning `RunLoop.current` in `mode` until the future
+    /// resolves. The calling thread will be blocked albeit running `RunLoop.current`.
+    ///
+    /// If the `EventLoopFuture` resolves with a value, that value is returned from `waitSpinningRunLoop()`. If
+    /// the `EventLoopFuture` resolves with an error, that error will be thrown instead.
+    /// `waitSpinningRunLoop()` will block whatever thread it is called on, so it must not be called on event loop
+    /// threads: it is primarily useful for testing, or for building interfaces between blocking
+    /// and non-blocking code.
+    ///
+    /// This is also forbidden in async contexts: prefer `EventLoopFuture/get()`.
+    ///
+    /// - Note: The `Value` must be `Sendable` since it is shared outside of the isolation domain of the event loop.
+    ///
+    /// - Returns: The value of the `EventLoopFuture` when it completes.
+    /// - Throws: The error value of the `EventLoopFuture` if it errors.
+    @available(*, noasync, message: "waitSpinningRunLoop() can block indefinitely, prefer get()", renamed: "get()")
+    @inlinable
+    public func waitSpinningRunLoop(
+        inMode mode: RunLoop.Mode = .default,
+        file: StaticString = #file,
+        line: UInt = #line
+    ) throws -> Value where Value: Sendable {
+        try self._blockingWaitForFutureCompletion(mode: mode, file: file, line: line)
+    }
+
+    @inlinable
+    @inline(never)
+    func _blockingWaitForFutureCompletion(
+        mode: RunLoop.Mode,
+        file: StaticString,
+        line: UInt
+    ) throws -> Value where Value: Sendable {
+        self.eventLoop._preconditionSafeToWait(file: file, line: line)
+
+        let runLoop = RunLoop.current
+
+        let value: NIOLockedValueBox<Result<Value, any Error>?> = NIOLockedValueBox(nil)
+        self.whenComplete { result in
+            value.withLockedValue { value in
+                value = result
+            }
+        }
+
+        while value.withLockedValue({ $0 }) == nil {
+            _ = runLoop.run(mode: mode, before: Date().addingTimeInterval(0.01))
+        }
+
+        return try value.withLockedValue { value in
+            try value!.get()
+        }
+    }
+}
diff --git a/Tests/NIOFoundationCompatTests/WaitSpinningRunLoopTests.swift b/Tests/NIOFoundationCompatTests/WaitSpinningRunLoopTests.swift
new file mode 100644
index 0000000000..18307c744f
--- /dev/null
+++ b/Tests/NIOFoundationCompatTests/WaitSpinningRunLoopTests.swift
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftNIO open source project
+//
+// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import NIO
+import NIOFoundationCompat
+import XCTest
+
+final class WaitSpinningRunLoopTests: XCTestCase {
+    private let loop = MultiThreadedEventLoopGroup.singleton.any()
+
+    func testPreFailedWorks() {
+        struct Dummy: Error {}
+        let future: EventLoopFuture<Never> = self.loop.makeFailedFuture(Dummy())
+        XCTAssertThrowsError(try future.waitSpinningRunLoop()) { error in
+            XCTAssert(error is Dummy)
+        }
+    }
+
+    func testPreSucceededWorks() {
+        let future = self.loop.makeSucceededFuture("hello")
+        XCTAssertEqual("hello", try future.waitSpinningRunLoop())
+    }
+
+    func testFailingAfterALittleWhileWorks() {
+        struct Dummy: Error {}
+        let future: EventLoopFuture<Never> = self.loop.scheduleTask(in: .milliseconds(10)) {
+            throw Dummy()
+        }.futureResult
+        XCTAssertThrowsError(try future.waitSpinningRunLoop()) { error in
+            XCTAssert(error is Dummy)
+        }
+    }
+
+    func testSucceedingAfterALittleWhileWorks() {
+        let future = self.loop.scheduleTask(in: .milliseconds(10)) {
+            "hello"
+        }.futureResult
+        XCTAssertEqual("hello", try future.waitSpinningRunLoop())
+    }
+
+    func testWeCanStillUseOurRunLoopWhilstBlocking() {
+        let promise = self.loop.makePromise(of: String.self)
+        let myRunLoop = RunLoop.current
+        let timer = Timer(timeInterval: 0.1, repeats: false) { [loop = self.loop] _ in
+            loop.scheduleTask(in: .microseconds(10)) {
+                promise.succeed("hello")
+            }
+        }
+        myRunLoop.add(timer, forMode: .default)
+        XCTAssertEqual("hello", try promise.futureResult.waitSpinningRunLoop())
+    }
+
+}

From 70dfce82b0c892c06e5730a43dc3fd717d4c1fbd Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Sat, 23 Nov 2024 22:47:34 +0000
Subject: [PATCH 08/37] give common blocking functions a clear name (#2984)

### Motivation:

In many situations, for example continuous profiling with sampling
profilers, it's important to distinguish between on- and off-CPU work.
That's most easily done if there are clear function names/prefixes to
grep for that are common places to just wait (off-CPU) and do no real
work. In SwiftNIO those are chiefly three places:
1. The `EventLoops` waiting for work
2. `The NIOThreadPool` threads waiting for work
3. `EventLoopFuture.wait()`

This patch makes sure that each of those will have a function with
prefix `_blockingWaitFor` in the function name which is easily
greppable.

### Modifications:

- Create some non-inlinable functions with `_blockingWaitFor...` that
just wait for `...`.

### Result:

SwiftNIO makes operational excellence easier and your SRE team more
happy.
---
 Sources/NIOCore/EventLoopFuture.swift         |   7 +-
 Sources/NIOCore/SystemCallHelpers.swift       |  15 +-
 Sources/NIOPosix/BSDSocketAPICommon.swift     |   4 +-
 Sources/NIOPosix/BaseSocket.swift             |   3 +-
 Sources/NIOPosix/IO.swift                     |   3 +-
 Sources/NIOPosix/Linux.swift                  |  36 +++-
 .../MultiThreadedEventLoopGroup.swift         |   6 +-
 Sources/NIOPosix/NIOThreadPool.swift          | 163 ++++++++++--------
 Sources/NIOPosix/SelectableEventLoop.swift    |  31 +++-
 Sources/NIOPosix/SelectorEpoll.swift          |   7 +-
 Sources/NIOPosix/SelectorGeneric.swift        |  51 +++++-
 Sources/NIOPosix/SelectorKqueue.swift         |  15 +-
 Sources/NIOPosix/System.swift                 | 121 ++++++++-----
 Sources/NIOPosix/Thread.swift                 |   3 +-
 14 files changed, 323 insertions(+), 142 deletions(-)

diff --git a/Sources/NIOCore/EventLoopFuture.swift b/Sources/NIOCore/EventLoopFuture.swift
index ea5632e5c1..793d3a7ed5 100644
--- a/Sources/NIOCore/EventLoopFuture.swift
+++ b/Sources/NIOCore/EventLoopFuture.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2020 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -1048,11 +1048,12 @@ extension EventLoopFuture {
     @preconcurrency
     @inlinable
     public func wait(file: StaticString = #file, line: UInt = #line) throws -> Value where Value: Sendable {
-        try self._wait(file: file, line: line)
+        try self._blockingWaitForFutureCompletion(file: file, line: line)
     }
 
     @inlinable
-    func _wait(file: StaticString, line: UInt) throws -> Value where Value: Sendable {
+    @inline(never)
+    func _blockingWaitForFutureCompletion(file: StaticString, line: UInt) throws -> Value where Value: Sendable {
         self.eventLoop._preconditionSafeToWait(file: file, line: line)
 
         let v: UnsafeMutableTransferBox<Result<Value, Error>?> = .init(nil)
diff --git a/Sources/NIOCore/SystemCallHelpers.swift b/Sources/NIOCore/SystemCallHelpers.swift
index 159b2f10b2..710cefb9e1 100644
--- a/Sources/NIOCore/SystemCallHelpers.swift
+++ b/Sources/NIOCore/SystemCallHelpers.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -60,7 +60,8 @@ private let sysGetifaddrs: @convention(c) (UnsafeMutablePointer<UnsafeMutablePoi
 #endif
 #endif
 
-private func isUnacceptableErrno(_ code: Int32) -> Bool {
+@inlinable
+internal func isUnacceptableErrno(_ code: Int32) -> Bool {
     switch code {
     case EFAULT, EBADF:
         return true
@@ -69,7 +70,8 @@ private func isUnacceptableErrno(_ code: Int32) -> Bool {
     }
 }
 
-private func preconditionIsNotUnacceptableErrno(err: CInt, where function: String) {
+@inlinable
+internal func preconditionIsNotUnacceptableErrno(err: CInt, where function: String) {
     // strerror is documented to return "Unknown error: ..." for illegal value so it won't ever fail
     precondition(
         !isUnacceptableErrno(err),
@@ -126,6 +128,7 @@ enum SystemCalls {
     #endif
 
     @inline(never)
+    @usableFromInline
     internal static func close(descriptor: CInt) throws {
         let res = sysClose(descriptor)
         if res == -1 {
@@ -150,6 +153,7 @@ enum SystemCalls {
     }
 
     @inline(never)
+    @usableFromInline
     internal static func open(
         file: UnsafePointer<CChar>,
         oFlag: CInt,
@@ -170,6 +174,7 @@ enum SystemCalls {
 
     @discardableResult
     @inline(never)
+    @usableFromInline
     internal static func lseek(descriptor: CInt, offset: off_t, whence: CInt) throws -> off_t {
         try syscall(blocking: false) {
             sysLseek(descriptor, offset, whence)
@@ -178,6 +183,7 @@ enum SystemCalls {
 
     #if os(Windows)
     @inline(never)
+    @usableFromInline
     internal static func read(
         descriptor: CInt,
         pointer: UnsafeMutableRawPointer,
@@ -189,6 +195,7 @@ enum SystemCalls {
     }
     #elseif !os(WASI)
     @inline(never)
+    @usableFromInline
     internal static func read(
         descriptor: CInt,
         pointer: UnsafeMutableRawPointer,
@@ -202,6 +209,7 @@ enum SystemCalls {
 
     #if !os(WASI)
     @inline(never)
+    @usableFromInline
     internal static func if_nametoindex(_ name: UnsafePointer<CChar>?) throws -> CUnsignedInt {
         try syscall(blocking: false) {
             sysIfNameToIndex(name!)
@@ -210,6 +218,7 @@ enum SystemCalls {
 
     #if !os(Windows)
     @inline(never)
+    @usableFromInline
     internal static func getifaddrs(_ addrs: UnsafeMutablePointer<UnsafeMutablePointer<ifaddrs>?>) throws {
         _ = try syscall(blocking: false) {
             sysGetifaddrs(addrs)
diff --git a/Sources/NIOPosix/BSDSocketAPICommon.swift b/Sources/NIOPosix/BSDSocketAPICommon.swift
index 0465ed54af..37f91f4031 100644
--- a/Sources/NIOPosix/BSDSocketAPICommon.swift
+++ b/Sources/NIOPosix/BSDSocketAPICommon.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2020-2022 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2020-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -31,6 +31,7 @@ protocol _SocketShutdownProtocol {
     var cValue: CInt { get }
 }
 
+@usableFromInline
 internal enum Shutdown: _SocketShutdownProtocol {
     case RD
     case WR
@@ -47,6 +48,7 @@ extension NIOBSDSocket {
 
 extension NIOBSDSocket {
     /// Specifies the type of socket.
+    @usableFromInline
     internal struct SocketType: RawRepresentable {
         public typealias RawValue = CInt
         public var rawValue: RawValue
diff --git a/Sources/NIOPosix/BaseSocket.swift b/Sources/NIOPosix/BaseSocket.swift
index bc6663445f..58e20d3e86 100644
--- a/Sources/NIOPosix/BaseSocket.swift
+++ b/Sources/NIOPosix/BaseSocket.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -22,6 +22,7 @@ import let WinSDK.EBADF
 import struct WinSDK.socklen_t
 #endif
 
+@usableFromInline
 protocol Registration {
     /// The `SelectorEventSet` in which the `Registration` is interested.
     var interested: SelectorEventSet { get set }
diff --git a/Sources/NIOPosix/IO.swift b/Sources/NIOPosix/IO.swift
index 5a9a581583..b44c77cf9b 100644
--- a/Sources/NIOPosix/IO.swift
+++ b/Sources/NIOPosix/IO.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -24,6 +24,7 @@ extension IOResult where T: FixedWidthInteger {
 }
 
 /// An result for an IO operation that was done on a non-blocking resource.
+@usableFromInline
 enum IOResult<T: Equatable>: Equatable {
 
     /// Signals that the IO operation could not be completed as otherwise we would need to block.
diff --git a/Sources/NIOPosix/Linux.swift b/Sources/NIOPosix/Linux.swift
index 07e48815ec..f484768861 100644
--- a/Sources/NIOPosix/Linux.swift
+++ b/Sources/NIOPosix/Linux.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -18,11 +18,13 @@
 #if os(Linux) || os(Android)
 import CNIOLinux
 
+@usableFromInline
 internal enum TimerFd {
     internal static let TFD_CLOEXEC = CNIOLinux.TFD_CLOEXEC
     internal static let TFD_NONBLOCK = CNIOLinux.TFD_NONBLOCK
 
     @inline(never)
+    @usableFromInline
     internal static func timerfd_settime(
         fd: CInt,
         flags: CInt,
@@ -35,6 +37,7 @@ internal enum TimerFd {
     }
 
     @inline(never)
+    @usableFromInline
     internal static func timerfd_create(clockId: CInt, flags: CInt) throws -> CInt {
         try syscall(blocking: false) {
             CNIOLinux.timerfd_create(clockId, flags)
@@ -42,9 +45,13 @@ internal enum TimerFd {
     }
 }
 
+@usableFromInline
 internal enum EventFd {
+    @usableFromInline
     internal static let EFD_CLOEXEC = CNIOLinux.EFD_CLOEXEC
+    @usableFromInline
     internal static let EFD_NONBLOCK = CNIOLinux.EFD_NONBLOCK
+    @usableFromInline
     internal typealias eventfd_t = CNIOLinux.eventfd_t
 
     @inline(never)
@@ -55,6 +62,7 @@ internal enum EventFd {
     }
 
     @inline(never)
+    @usableFromInline
     internal static func eventfd_read(fd: CInt, value: UnsafeMutablePointer<UInt64>) throws -> CInt {
         try syscall(blocking: false) {
             CNIOLinux.eventfd_read(fd, value)
@@ -73,40 +81,65 @@ internal enum EventFd {
     }
 }
 
+@usableFromInline
 internal enum Epoll {
+    @usableFromInline
     internal typealias epoll_event = CNIOLinux.epoll_event
 
+    @usableFromInline
     internal static let EPOLL_CTL_ADD: CInt = numericCast(CNIOLinux.EPOLL_CTL_ADD)
+    @usableFromInline
     internal static let EPOLL_CTL_MOD: CInt = numericCast(CNIOLinux.EPOLL_CTL_MOD)
+    @usableFromInline
     internal static let EPOLL_CTL_DEL: CInt = numericCast(CNIOLinux.EPOLL_CTL_DEL)
 
     #if canImport(Android) || canImport(Musl)
+    @usableFromInline
     internal static let EPOLLIN: CUnsignedInt = numericCast(CNIOLinux.EPOLLIN)
+    @usableFromInline
     internal static let EPOLLOUT: CUnsignedInt = numericCast(CNIOLinux.EPOLLOUT)
+    @usableFromInline
     internal static let EPOLLERR: CUnsignedInt = numericCast(CNIOLinux.EPOLLERR)
+    @usableFromInline
     internal static let EPOLLRDHUP: CUnsignedInt = numericCast(CNIOLinux.EPOLLRDHUP)
+    @usableFromInline
     internal static let EPOLLHUP: CUnsignedInt = numericCast(CNIOLinux.EPOLLHUP)
     #if canImport(Android)
+    @usableFromInline
     internal static let EPOLLET: CUnsignedInt = 2_147_483_648  // C macro not imported by ClangImporter
     #else
+    @usableFromInline
     internal static let EPOLLET: CUnsignedInt = numericCast(CNIOLinux.EPOLLET)
     #endif
     #elseif os(Android)
+    @usableFromInline
     internal static let EPOLLIN: CUnsignedInt = 1  //numericCast(CNIOLinux.EPOLLIN)
+    @usableFromInline
     internal static let EPOLLOUT: CUnsignedInt = 4  //numericCast(CNIOLinux.EPOLLOUT)
+    @usableFromInline
     internal static let EPOLLERR: CUnsignedInt = 8  // numericCast(CNIOLinux.EPOLLERR)
+    @usableFromInline
     internal static let EPOLLRDHUP: CUnsignedInt = 8192  //numericCast(CNIOLinux.EPOLLRDHUP)
+    @usableFromInline
     internal static let EPOLLHUP: CUnsignedInt = 16  //numericCast(CNIOLinux.EPOLLHUP)
+    @usableFromInline
     internal static let EPOLLET: CUnsignedInt = 2_147_483_648  //numericCast(CNIOLinux.EPOLLET)
     #else
+    @usableFromInline
     internal static let EPOLLIN: CUnsignedInt = numericCast(CNIOLinux.EPOLLIN.rawValue)
+    @usableFromInline
     internal static let EPOLLOUT: CUnsignedInt = numericCast(CNIOLinux.EPOLLOUT.rawValue)
+    @usableFromInline
     internal static let EPOLLERR: CUnsignedInt = numericCast(CNIOLinux.EPOLLERR.rawValue)
+    @usableFromInline
     internal static let EPOLLRDHUP: CUnsignedInt = numericCast(CNIOLinux.EPOLLRDHUP.rawValue)
+    @usableFromInline
     internal static let EPOLLHUP: CUnsignedInt = numericCast(CNIOLinux.EPOLLHUP.rawValue)
+    @usableFromInline
     internal static let EPOLLET: CUnsignedInt = numericCast(CNIOLinux.EPOLLET.rawValue)
     #endif
 
+    @usableFromInline
     internal static let ENOENT: CUnsignedInt = numericCast(CNIOLinux.ENOENT)
 
     @inline(never)
@@ -130,6 +163,7 @@ internal enum Epoll {
     }
 
     @inline(never)
+    @usableFromInline
     internal static func epoll_wait(
         epfd: CInt,
         events: UnsafeMutablePointer<epoll_event>,
diff --git a/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift b/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift
index c09f9fe8d3..b92a4bb01b 100644
--- a/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift
+++ b/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -20,6 +20,7 @@ import NIOCore
 import Dispatch
 #endif
 
+@usableFromInline
 struct NIORegistration: Registration {
     enum ChannelType {
         case serverSocketChannel(ServerSocketChannel)
@@ -31,9 +32,11 @@ struct NIORegistration: Registration {
     var channel: ChannelType
 
     /// The `SelectorEventSet` in which this `NIORegistration` is interested in.
+    @usableFromInline
     var interested: SelectorEventSet
 
     /// The registration ID for this `NIORegistration` used by the `Selector`.
+    @usableFromInline
     var registrationID: SelectorRegistrationID
 }
 
@@ -568,6 +571,7 @@ extension ScheduledTask: Comparable {
 }
 
 extension NIODeadline {
+    @inlinable
     func readyIn(_ target: NIODeadline) -> TimeAmount {
         if self < target {
             return .nanoseconds(0)
diff --git a/Sources/NIOPosix/NIOThreadPool.swift b/Sources/NIOPosix/NIOThreadPool.swift
index 7854cbff7b..6d3bc08f73 100644
--- a/Sources/NIOPosix/NIOThreadPool.swift
+++ b/Sources/NIOPosix/NIOThreadPool.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -63,17 +63,22 @@ public final class NIOThreadPool {
     /// The work that should be done by the `NIOThreadPool`.
     public typealias WorkItem = @Sendable (WorkItemState) -> Void
 
-    private struct IdentifiableWorkItem: Sendable {
+    @usableFromInline
+    struct IdentifiableWorkItem: Sendable {
+        @usableFromInline
         var workItem: WorkItem
+
+        @usableFromInline
         var id: Int?
     }
 
-    private enum State {
+    @usableFromInline
+    internal enum State {
         /// The `NIOThreadPool` is already stopped.
         case stopped
         /// The `NIOThreadPool` is shutting down, the array has one boolean entry for each thread indicating if it has shut down already.
         case shuttingDown([Bool])
-        /// The `NIOThreadPool` is up and running, the `CircularBuffer` containing the yet unprocessed `WorkItems`.
+        /// The `NIOThreadPool` is up and running, the `Deque` containing the yet unprocessed `IdentifiableWorkItem`s.
         case running(Deque<IdentifiableWorkItem>)
         /// Temporary state used when mutating the .running(items). Used to avoid CoW copies.
         /// It should never be "leaked" outside of the lock block.
@@ -81,7 +86,8 @@ public final class NIOThreadPool {
     }
 
     /// Whether threads in the pool have work.
-    private enum WorkState: Hashable {
+    @usableFromInline
+    internal enum _WorkState: Hashable {
         case hasWork
         case hasNoWork
     }
@@ -93,9 +99,11 @@ public final class NIOThreadPool {
     // to wait for a given value. The value indicates whether the thread has some work to do. Work
     // in this case can be either processing a work item or exiting the threads processing
     // loop (i.e. shutting down).
-    private let conditionLock: ConditionLock<WorkState>
+    @usableFromInline
+    internal let _conditionLock: ConditionLock<_WorkState>
     private var threads: [NIOThread]? = nil  // protected by `conditionLock`
-    private var state: State = .stopped
+    @usableFromInline
+    internal var _state: State = .stopped
 
     // WorkItems don't have a handle so they can't be cancelled directly. Instead an ID is assigned
     // to each cancellable work item and the IDs of each work item to cancel is stored in this set.
@@ -113,7 +121,8 @@ public final class NIOThreadPool {
     // be removed.
     //
     // Note: protected by 'lock'.
-    private var cancelledWorkIDs: Set<Int> = []
+    @usableFromInline
+    internal var _cancelledWorkIDs: Set<Int> = []
     private let nextWorkID = ManagedAtomic(0)
 
     public let numberOfThreads: Int
@@ -137,16 +146,16 @@ public final class NIOThreadPool {
             return
         }
 
-        let threadsToJoin = self.conditionLock.withLock {
-            switch self.state {
+        let threadsToJoin = self._conditionLock.withLock {
+            switch self._state {
             case .running(let items):
-                self.state = .modifying
+                self._state = .modifying
                 queue.async {
                     for item in items {
                         item.workItem(.cancelled)
                     }
                 }
-                self.state = .shuttingDown(Array(repeating: true, count: self.numberOfThreads))
+                self._state = .shuttingDown(Array(repeating: true, count: self.numberOfThreads))
 
                 let threads = self.threads!
                 self.threads = nil
@@ -184,15 +193,15 @@ public final class NIOThreadPool {
     }
 
     private func _submit(id: Int?, _ body: @escaping WorkItem) {
-        let submitted = self.conditionLock.withLock {
-            let workState: WorkState
+        let submitted = self._conditionLock.withLock {
+            let workState: _WorkState
             let submitted: Bool
 
-            switch self.state {
+            switch self._state {
             case .running(var items):
-                self.state = .modifying
+                self._state = .modifying
                 items.append(.init(workItem: body, id: id))
-                self.state = .running(items)
+                self._state = .running(items)
                 workState = items.isEmpty ? .hasNoWork : .hasWork
                 submitted = true
 
@@ -233,60 +242,74 @@ public final class NIOThreadPool {
     private init(numberOfThreads: Int, canBeStopped: Bool) {
         self.numberOfThreads = numberOfThreads
         self.canBeStopped = canBeStopped
-        self.conditionLock = ConditionLock(value: .hasNoWork)
+        self._conditionLock = ConditionLock(value: .hasNoWork)
     }
 
-    private func process(identifier: Int) {
-        var itemAndState: (item: WorkItem, state: WorkItemState)? = nil
+    // Do not rename or remove this function.
+    //
+    // When doing on-/off-CPU analysis, for example with continuous profiling, it's
+    // important to recognise certain functions that are purely there to wait.
+    //
+    // This function is one of those and giving it a consistent name makes it much easier to remove from the profiles
+    // when only interested in on-CPU work.
+    @inlinable
+    internal func _blockingWaitForWork(identifier: Int) -> (item: WorkItem, state: WorkItemState)? {
+        self._conditionLock.withLock(when: .hasWork) {
+            () -> (unlockWith: _WorkState, result: (WorkItem, WorkItemState)?) in
+            let workState: _WorkState
+            let result: (WorkItem, WorkItemState)?
 
-        repeat {
-            itemAndState = nil  // ensure previous work item is not retained while waiting for the condition
-            itemAndState = self.conditionLock.withLock(when: .hasWork) {
-                let workState: WorkState
-                let result: (WorkItem, WorkItemState)?
-
-                switch self.state {
-                case .running(var items):
-                    self.state = .modifying
-                    let itemAndID = items.removeFirst()
-
-                    let state: WorkItemState
-                    if let id = itemAndID.id, !self.cancelledWorkIDs.isEmpty {
-                        state = self.cancelledWorkIDs.remove(id) == nil ? .active : .cancelled
-                    } else {
-                        state = .active
-                    }
+            switch self._state {
+            case .running(var items):
+                self._state = .modifying
+                let itemAndID = items.removeFirst()
 
-                    self.state = .running(items)
+                let state: WorkItemState
+                if let id = itemAndID.id, !self._cancelledWorkIDs.isEmpty {
+                    state = self._cancelledWorkIDs.remove(id) == nil ? .active : .cancelled
+                } else {
+                    state = .active
+                }
 
-                    workState = items.isEmpty ? .hasNoWork : .hasWork
-                    result = (itemAndID.workItem, state)
+                self._state = .running(items)
 
-                case .shuttingDown(var aliveStates):
-                    self.state = .modifying
-                    assert(aliveStates[identifier])
-                    aliveStates[identifier] = false
-                    self.state = .shuttingDown(aliveStates)
+                workState = items.isEmpty ? .hasNoWork : .hasWork
+                result = (itemAndID.workItem, state)
 
-                    // Unlock with '.hasWork' to resume any other threads waiting to shutdown.
-                    workState = .hasWork
-                    result = nil
+            case .shuttingDown(var aliveStates):
+                self._state = .modifying
+                assert(aliveStates[identifier])
+                aliveStates[identifier] = false
+                self._state = .shuttingDown(aliveStates)
 
-                case .stopped:
-                    // Unreachable: 'stopped' is the initial state which is left when starting the
-                    // thread pool, and before any thread calls this function.
-                    fatalError("Invalid state")
+                // Unlock with '.hasWork' to resume any other threads waiting to shutdown.
+                workState = .hasWork
+                result = nil
 
-                case .modifying:
-                    fatalError(".modifying state misuse")
-                }
+            case .stopped:
+                // Unreachable: 'stopped' is the initial state which is left when starting the
+                // thread pool, and before any thread calls this function.
+                fatalError("Invalid state")
 
-                return (unlockWith: workState, result: result)
+            case .modifying:
+                fatalError(".modifying state misuse")
             }
 
-            // if there was a work item popped, run it
-            itemAndState.map { item, state in item(state) }
-        } while itemAndState != nil
+            return (unlockWith: workState, result: result)
+        }
+    }
+
+    private func process(identifier: Int) {
+        repeat {
+            let itemAndState = self._blockingWaitForWork(identifier: identifier)
+
+            if let (item, state) = itemAndState {
+                // if there was a work item popped, run it
+                item(state)
+            } else {
+                break  // Otherwise, we're done
+            }
+        } while true
     }
 
     /// Start the `NIOThreadPool` if not already started.
@@ -295,8 +318,8 @@ public final class NIOThreadPool {
     }
 
     public func _start(threadNamePrefix: String) {
-        let alreadyRunning = self.conditionLock.withLock {
-            switch self.state {
+        let alreadyRunning = self._conditionLock.withLock {
+            switch self._state {
             case .running:
                 // Already running, this has no effect on whether there is more work for the
                 // threads to run.
@@ -307,7 +330,7 @@ public final class NIOThreadPool {
                 fatalError("start() called while in shuttingDown")
 
             case .stopped:
-                self.state = .running(Deque(minimumCapacity: 16))
+                self._state = .running(Deque(minimumCapacity: 16))
                 assert(self.threads == nil)
                 self.threads = []
                 self.threads!.reserveCapacity(self.numberOfThreads)
@@ -330,11 +353,11 @@ public final class NIOThreadPool {
             // We should keep thread names under 16 characters because Linux doesn't allow more.
             NIOThread.spawnAndRun(name: "\(threadNamePrefix)\(id)", detachThread: false) { thread in
                 readyThreads.withLock {
-                    let threadCount = self.conditionLock.withLock {
+                    let threadCount = self._conditionLock.withLock {
                         self.threads!.append(thread)
-                        let workState: WorkState
+                        let workState: _WorkState
 
-                        switch self.state {
+                        switch self._state {
                         case .running(let items):
                             workState = items.isEmpty ? .hasNoWork : .hasWork
                         case .shuttingDown:
@@ -364,7 +387,7 @@ public final class NIOThreadPool {
         readyThreads.unlock()
 
         func threadCount() -> Int {
-            self.conditionLock.withLock {
+            self._conditionLock.withLock {
                 (unlockWith: nil, result: self.threads?.count ?? -1)
             }
         }
@@ -376,11 +399,11 @@ public final class NIOThreadPool {
             self.canBeStopped,
             "Perpetual NIOThreadPool has been deinited, you must make sure that perpetual pools don't deinit"
         )
-        switch self.state {
+        switch self._state {
         case .stopped, .shuttingDown:
             ()
         default:
-            assertionFailure("wrong state \(self.state)")
+            assertionFailure("wrong state \(self._state)")
         }
     }
 }
@@ -440,8 +463,8 @@ extension NIOThreadPool {
                 }
             }
         } onCancel: {
-            self.conditionLock.withLock {
-                self.cancelledWorkIDs.insert(workID)
+            self._conditionLock.withLock {
+                self._cancelledWorkIDs.insert(workID)
                 return (unlockWith: nil, result: ())
             }
         }
diff --git a/Sources/NIOPosix/SelectableEventLoop.swift b/Sources/NIOPosix/SelectableEventLoop.swift
index 20fe17fc41..89fc4737f9 100644
--- a/Sources/NIOPosix/SelectableEventLoop.swift
+++ b/Sources/NIOPosix/SelectableEventLoop.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -93,6 +93,7 @@ internal final class SelectableEventLoop: EventLoop {
         case exitingThread
     }
 
+    @usableFromInline
     internal let _selector: NIOPosix.Selector<NIORegistration>
     private let thread: NIOThread
     @usableFromInline
@@ -477,7 +478,8 @@ internal final class SelectableEventLoop: EventLoop {
         }
     }
 
-    private func currentSelectorStrategy(nextReadyDeadline: NIODeadline?) -> SelectorStrategy {
+    @inlinable
+    internal func _currentSelectorStrategy(nextReadyDeadline: NIODeadline?) -> SelectorStrategy {
         guard let deadline = nextReadyDeadline else {
             // No tasks to handle so just block. If any tasks were added in the meantime wakeup(...) was called and so this
             // will directly unblock.
@@ -679,6 +681,26 @@ internal final class SelectableEventLoop: EventLoop {
         }
     }
 
+    // Do not rename or remove this function.
+    //
+    // When doing on-/off-CPU analysis, for example with continuous profiling, it's
+    // important to recognise certain functions that are purely there to wait.
+    //
+    // This function is one of those and giving it a consistent name makes it much easier to remove from the profiles
+    // when only interested in on-CPU work.
+    @inline(never)
+    @inlinable
+    internal func _blockingWaitForWork(
+        nextReadyDeadline: NIODeadline?,
+        _ body: (SelectorEvent<NIORegistration>) -> Void
+    ) throws {
+        try self._selector.whenReady(
+            strategy: self._currentSelectorStrategy(nextReadyDeadline: nextReadyDeadline),
+            onLoopBegin: { self._tasksLock.withLock { () -> Void in self._pendingTaskPop = true } },
+            body
+        )
+    }
+
     /// Start processing I/O and tasks for this `SelectableEventLoop`. This method will continue running (and so block) until the `SelectableEventLoop` is closed.
     internal func run() throws {
         self.preconditionInEventLoop()
@@ -745,10 +767,7 @@ internal final class SelectableEventLoop: EventLoop {
             // Block until there are events to handle or the selector was woken up
             // for macOS: in case any calls we make to Foundation put objects into an autoreleasepool
             try withAutoReleasePool {
-                try self._selector.whenReady(
-                    strategy: currentSelectorStrategy(nextReadyDeadline: nextReadyDeadline),
-                    onLoopBegin: { self._tasksLock.withLock { () -> Void in self._pendingTaskPop = true } }
-                ) { ev in
+                try self._blockingWaitForWork(nextReadyDeadline: nextReadyDeadline) { ev in
                     switch ev.registration.channel {
                     case .serverSocketChannel(let chan):
                         self.handleEvent(ev.io, channel: chan)
diff --git a/Sources/NIOPosix/SelectorEpoll.swift b/Sources/NIOPosix/SelectorEpoll.swift
index 208aa3a950..5da3978b8c 100644
--- a/Sources/NIOPosix/SelectorEpoll.swift
+++ b/Sources/NIOPosix/SelectorEpoll.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -17,6 +17,7 @@ import NIOCore
 #if !SWIFTNIO_USE_IO_URING
 
 #if os(Linux) || os(Android)
+import CNIOLinux
 
 /// Represents the `epoll` filters/events we might use:
 ///
@@ -79,7 +80,8 @@ extension SelectorEventSet {
         return filter
     }
 
-    fileprivate init(epollEvent: Epoll.epoll_event) {
+    @inlinable
+    internal init(epollEvent: Epoll.epoll_event) {
         var selectorEventSet: SelectorEventSet = ._none
         if epollEvent.events & Epoll.EPOLLIN != 0 {
             selectorEventSet.formUnion(.read)
@@ -207,6 +209,7 @@ extension Selector: _SelectorBackendProtocol {
     /// - Parameters:
     ///   - strategy: The `SelectorStrategy` to apply
     ///   - body: The function to execute for each `SelectorEvent` that was produced.
+    @inlinable
     func whenReady0(
         strategy: SelectorStrategy,
         onLoopBegin loopStart: () -> Void,
diff --git a/Sources/NIOPosix/SelectorGeneric.swift b/Sources/NIOPosix/SelectorGeneric.swift
index d97496d6a6..a587079e6b 100644
--- a/Sources/NIOPosix/SelectorGeneric.swift
+++ b/Sources/NIOPosix/SelectorGeneric.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -15,6 +15,11 @@
 import NIOConcurrencyHelpers
 import NIOCore
 
+#if os(Linux)
+import CNIOLinux
+#endif
+
+@usableFromInline
 internal enum SelectorLifecycleState {
     case open
     case closing
@@ -22,6 +27,7 @@ internal enum SelectorLifecycleState {
 }
 
 extension Optional {
+    @inlinable
     internal func withUnsafeOptionalPointer<T>(_ body: (UnsafePointer<Wrapped>?) throws -> T) rethrows -> T {
         if var this = self {
             return try withUnsafePointer(to: &this) { x in
@@ -35,6 +41,7 @@ extension Optional {
 
 #if !os(Windows)
 extension timespec {
+    @inlinable
     init(timeAmount amount: TimeAmount) {
         let nsecPerSec: Int64 = 1_000_000_000
         let ns = amount.nanoseconds
@@ -52,36 +59,47 @@ extension timespec {
 /// receives a connection reset, express interest with `[.read, .write, .reset]`.
 /// If then suddenly the socket becomes both readable and writable, the eventing mechanism will tell you about that
 /// fact using `[.read, .write]`.
+@usableFromInline
 struct SelectorEventSet: OptionSet, Equatable {
 
+    @usableFromInline
     typealias RawValue = UInt8
 
+    @usableFromInline
     let rawValue: RawValue
 
     /// It's impossible to actually register for no events, therefore `_none` should only be used to bootstrap a set
     /// of flags or to compare against spurious wakeups.
+    @usableFromInline
     static let _none = SelectorEventSet([])
 
     /// Connection reset.
+    @usableFromInline
     static let reset = SelectorEventSet(rawValue: 1 << 0)
 
     /// EOF at the read/input end of a `Selectable`.
+    @usableFromInline
     static let readEOF = SelectorEventSet(rawValue: 1 << 1)
 
     /// Interest in/availability of data to be read
+    @usableFromInline
     static let read = SelectorEventSet(rawValue: 1 << 2)
 
     /// Interest in/availability of data to be written
+    @usableFromInline
     static let write = SelectorEventSet(rawValue: 1 << 3)
 
     /// EOF at the write/output end of a `Selectable`.
     ///
     /// - Note: This is rarely used because in many cases, there is no signal that this happened.
+    @usableFromInline
     static let writeEOF = SelectorEventSet(rawValue: 1 << 4)
 
     /// Error encountered.
+    @usableFromInline
     static let error = SelectorEventSet(rawValue: 1 << 5)
 
+    @inlinable
     init(rawValue: SelectorEventSet.RawValue) {
         self.rawValue = rawValue
     }
@@ -144,40 +162,59 @@ protocol _SelectorBackendProtocol {
 /// There are specific subclasses  per API type with a shared common superclass providing overall scaffolding.
 
 // this is deliberately not thread-safe, only the wakeup() function may be called unprotectedly
+@usableFromInline
 internal class Selector<R: Registration> {
+    @usableFromInline
     var lifecycleState: SelectorLifecycleState
+    @usableFromInline
     var registrations = [Int: R]()
+    @usableFromInline
     var registrationID: SelectorRegistrationID = .initialRegistrationID
 
+    @usableFromInline
     let myThread: NIOThread
     // The rules for `self.selectorFD`, `self.eventFD`, and `self.timerFD`:
     // reads: `self.externalSelectorFDLock` OR access from the EventLoop thread
     // writes: `self.externalSelectorFDLock` AND access from the EventLoop thread
     let externalSelectorFDLock = NIOLock()
+    @usableFromInline
     var selectorFD: CInt = -1  // -1 == we're closed
 
     // Here we add the stored properties that are used by the specific backends
     #if canImport(Darwin)
+    @usableFromInline
     typealias EventType = kevent
     #elseif os(Linux) || os(Android)
     #if !SWIFTNIO_USE_IO_URING
+    @usableFromInline
     typealias EventType = Epoll.epoll_event
+    @usableFromInline
     var earliestTimer: NIODeadline = .distantFuture
+    @usableFromInline
     var eventFD: CInt = -1  // -1 == we're closed
+    @usableFromInline
     var timerFD: CInt = -1  // -1 == we're closed
     #else
+    @usableFromInline
     typealias EventType = URingEvent
+    @usableFromInline
     var eventFD: CInt = -1  // -1 == we're closed
+    @usableFromInline
     var ring = URing()
+    @usableFromInline
     let multishot = URing.io_uring_use_multishot_poll  // if true, we run with streaming multishot polls
+    @usableFromInline
     let deferReregistrations = true  // if true we only flush once at reentring whenReady() - saves syscalls
+    @usableFromInline
     var deferredReregistrationsPending = false  // true if flush needed when reentring whenReady()
     #endif
     #else
     #error("Unsupported platform, no suitable selector backend (we need kqueue or epoll support)")
     #endif
 
+    @usableFromInline
     var events: UnsafeMutablePointer<EventType>
+    @usableFromInline
     var eventsCapacity = 64
 
     internal func testsOnly_withUnsafeSelectorFD<T>(_ body: (CInt) throws -> T) throws -> T {
@@ -205,17 +242,20 @@ internal class Selector<R: Registration> {
         Selector.deallocateEventsArray(events: events, capacity: eventsCapacity)
     }
 
-    private static func allocateEventsArray(capacity: Int) -> UnsafeMutablePointer<EventType> {
+    @inlinable
+    internal static func allocateEventsArray(capacity: Int) -> UnsafeMutablePointer<EventType> {
         let events: UnsafeMutablePointer<EventType> = UnsafeMutablePointer.allocate(capacity: capacity)
         events.initialize(to: EventType())
         return events
     }
 
-    private static func deallocateEventsArray(events: UnsafeMutablePointer<EventType>, capacity: Int) {
+    @inlinable
+    internal static func deallocateEventsArray(events: UnsafeMutablePointer<EventType>, capacity: Int) {
         events.deinitialize(count: capacity)
         events.deallocate()
     }
 
+    @inlinable
     func growEventArrayIfNeeded(ready: Int) {
         assert(self.myThread == NIOThread.current)
         guard ready == eventsCapacity else {
@@ -317,6 +357,7 @@ internal class Selector<R: Registration> {
     ///   - strategy: The `SelectorStrategy` to apply
     ///   - onLoopBegin: A function executed after the selector returns, just before the main loop begins..
     ///   - body: The function to execute for each `SelectorEvent` that was produced.
+    @inlinable
     func whenReady(
         strategy: SelectorStrategy,
         onLoopBegin loopStart: () -> Void,
@@ -345,6 +386,7 @@ internal class Selector<R: Registration> {
 }
 
 extension Selector: CustomStringConvertible {
+    @usableFromInline
     var description: String {
         func makeDescription() -> String {
             "Selector { descriptor = \(self.selectorFD) }"
@@ -361,6 +403,7 @@ extension Selector: CustomStringConvertible {
 }
 
 /// An event that is triggered once the `Selector` was able to select something.
+@usableFromInline
 struct SelectorEvent<R> {
     public let registration: R
     public var io: SelectorEventSet
@@ -370,6 +413,7 @@ struct SelectorEvent<R> {
     /// - Parameters:
     ///   - io: The `SelectorEventSet` that triggered this event.
     ///   - registration: The registration that belongs to the event.
+    @inlinable
     init(io: SelectorEventSet, registration: R) {
         self.io = io
         self.registration = registration
@@ -426,6 +470,7 @@ extension Selector where R == NIORegistration {
 }
 
 /// The strategy used for the `Selector`.
+@usableFromInline
 enum SelectorStrategy {
     /// Block until there is some IO ready to be processed or the `Selector` is explicitly woken up.
     case block
diff --git a/Sources/NIOPosix/SelectorKqueue.swift b/Sources/NIOPosix/SelectorKqueue.swift
index 49a264d730..2b2b31edac 100644
--- a/Sources/NIOPosix/SelectorKqueue.swift
+++ b/Sources/NIOPosix/SelectorKqueue.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -99,6 +99,7 @@ extension KQueueEventFilterSet {
 }
 
 extension SelectorRegistrationID {
+    @inlinable
     init(kqueueUData: UnsafeMutableRawPointer?) {
         self = .init(rawValue: UInt32(truncatingIfNeeded: UInt(bitPattern: kqueueUData)))
     }
@@ -106,7 +107,8 @@ extension SelectorRegistrationID {
 
 // this is deliberately not thread-safe, only the wakeup() function may be called unprotectedly
 extension Selector: _SelectorBackendProtocol {
-    private static func toKQueueTimeSpec(strategy: SelectorStrategy) -> timespec? {
+    @inlinable
+    internal static func toKQueueTimeSpec(strategy: SelectorStrategy) -> timespec? {
         switch strategy {
         case .block:
             return nil
@@ -251,6 +253,7 @@ extension Selector: _SelectorBackendProtocol {
     /// - Parameters:
     ///   - strategy: The `SelectorStrategy` to apply
     ///   - body: The function to execute for each `SelectorEvent` that was produced.
+    @inlinable
     func whenReady0(
         strategy: SelectorStrategy,
         onLoopBegin loopStart: () -> Void,
@@ -268,8 +271,8 @@ extension Selector: _SelectorBackendProtocol {
                     kq: self.selectorFD,
                     changelist: nil,
                     nchanges: 0,
-                    eventlist: events,
-                    nevents: Int32(eventsCapacity),
+                    eventlist: self.events,
+                    nevents: Int32(self.eventsCapacity),
                     timeout: ts
                 )
             )
@@ -287,7 +290,7 @@ extension Selector: _SelectorBackendProtocol {
                     reason: "kevent returned with EV_ERROR set: \(String(describing: ev))"
                 )
             }
-            guard filter != EVFILT_USER, let registration = registrations[Int(ev.ident)] else {
+            guard filter != EVFILT_USER, let registration = self.registrations[Int(ev.ident)] else {
                 continue
             }
             guard eventRegistrationID == registration.registrationID else {
@@ -327,7 +330,7 @@ extension Selector: _SelectorBackendProtocol {
             try body((SelectorEvent(io: selectorEvent, registration: registration)))
         }
 
-        growEventArrayIfNeeded(ready: ready)
+        self.growEventArrayIfNeeded(ready: ready)
     }
 
     /// Close the `Selector`.
diff --git a/Sources/NIOPosix/System.swift b/Sources/NIOPosix/System.swift
index 33b3be73d3..adb7cb2ffa 100644
--- a/Sources/NIOPosix/System.swift
+++ b/Sources/NIOPosix/System.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -172,7 +172,8 @@ private let sysRecvMmsg = CNIODarwin_recvmmsg
 private let sysIoctl: @convention(c) (CInt, CUnsignedLong, UnsafeMutableRawPointer) -> CInt = ioctl
 #endif  // !os(Windows)
 
-private func isUnacceptableErrno(_ code: Int32) -> Bool {
+@inlinable
+func isUnacceptableErrno(_ code: CInt) -> Bool {
     // On iOS, EBADF is a possible result when a file descriptor has been reaped in the background.
     // In particular, it's possible to get EBADF from accept(), where the underlying accept() FD
     // is valid but the accepted one is not. The right solution here is to perform a check for
@@ -195,7 +196,8 @@ private func isUnacceptableErrno(_ code: Int32) -> Bool {
     #endif
 }
 
-private func isUnacceptableErrnoOnClose(_ code: Int32) -> Bool {
+@inlinable
+public func isUnacceptableErrnoOnClose(_ code: CInt) -> Bool {
     // We treat close() differently to all other FDs: we still want to catch EBADF here.
     switch code {
     case EFAULT, EBADF:
@@ -205,7 +207,8 @@ private func isUnacceptableErrnoOnClose(_ code: Int32) -> Bool {
     }
 }
 
-private func isUnacceptableErrnoForbiddingEINVAL(_ code: Int32) -> Bool {
+@inlinable
+internal func isUnacceptableErrnoForbiddingEINVAL(_ code: CInt) -> Bool {
     // We treat read() and pread() differently since we also want to catch EINVAL.
     #if canImport(Darwin) && !os(macOS)
     switch code {
@@ -225,6 +228,7 @@ private func isUnacceptableErrnoForbiddingEINVAL(_ code: Int32) -> Bool {
 }
 
 #if os(Windows)
+@inlinable
 internal func strerror(_ errno: CInt) -> String {
     withUnsafeTemporaryAllocation(of: CChar.self, capacity: 95) {
         let result = strerror_s($0.baseAddress, $0.count, errno)
@@ -234,7 +238,8 @@ internal func strerror(_ errno: CInt) -> String {
 }
 #endif
 
-private func preconditionIsNotUnacceptableErrno(err: CInt, where function: String) {
+@inlinable
+internal func preconditionIsNotUnacceptableErrno(err: CInt, where function: String) {
     // strerror is documented to return "Unknown error: ..." for illegal value so it won't ever fail
     #if os(Windows)
     precondition(!isUnacceptableErrno(err), "unacceptable errno \(err) \(strerror(err)) in \(function))")
@@ -246,7 +251,8 @@ private func preconditionIsNotUnacceptableErrno(err: CInt, where function: Strin
     #endif
 }
 
-private func preconditionIsNotUnacceptableErrnoOnClose(err: CInt, where function: String) {
+@inlinable
+internal func preconditionIsNotUnacceptableErrnoOnClose(err: CInt, where function: String) {
     // strerror is documented to return "Unknown error: ..." for illegal value so it won't ever fail
     #if os(Windows)
     precondition(!isUnacceptableErrnoOnClose(err), "unacceptable errno \(err) \(strerror(err)) in \(function))")
@@ -258,7 +264,8 @@ private func preconditionIsNotUnacceptableErrnoOnClose(err: CInt, where function
     #endif
 }
 
-private func preconditionIsNotUnacceptableErrnoForbiddingEINVAL(err: CInt, where function: String) {
+@inlinable
+internal func preconditionIsNotUnacceptableErrnoForbiddingEINVAL(err: CInt, where function: String) {
     // strerror is documented to return "Unknown error: ..." for illegal value so it won't ever fail
     #if os(Windows)
     precondition(
@@ -278,6 +285,7 @@ private func preconditionIsNotUnacceptableErrnoForbiddingEINVAL(err: CInt, where
 // take twice the time, ie. we need this exception.
 @inline(__always)
 @discardableResult
+@inlinable
 internal func syscall<T: FixedWidthInteger>(
     blocking: Bool,
     where function: String = #function,
@@ -310,6 +318,7 @@ internal func syscall<T: FixedWidthInteger>(
 
 #if canImport(Darwin)
 @inline(__always)
+@inlinable
 @discardableResult
 internal func syscall<T>(
     where function: String = #function,
@@ -334,6 +343,7 @@ internal func syscall<T>(
 }
 #elseif os(Linux) || os(Android)
 @inline(__always)
+@inlinable
 @discardableResult
 internal func syscall(
     where function: String = #function,
@@ -360,6 +370,7 @@ internal func syscall(
 
 #if !os(Windows)
 @inline(__always)
+@inlinable
 @discardableResult
 internal func syscallOptional<T>(
     where function: String = #function,
@@ -391,6 +402,7 @@ internal func syscallOptional<T>(
 // however we seem to break the inlining threshold which makes a system call
 // take twice the time, ie. we need this exception.
 @inline(__always)
+@inlinable
 @discardableResult
 internal func syscallForbiddingEINVAL<T: FixedWidthInteger>(
     where function: String = #function,
@@ -421,39 +433,60 @@ internal func syscallForbiddingEINVAL<T: FixedWidthInteger>(
     }
 }
 
+@usableFromInline
 internal enum Posix {
     #if canImport(Darwin)
+    @usableFromInline
     static let UIO_MAXIOV: Int = 1024
+    @usableFromInline
     static let SHUT_RD: CInt = CInt(Darwin.SHUT_RD)
+    @usableFromInline
     static let SHUT_WR: CInt = CInt(Darwin.SHUT_WR)
+    @usableFromInline
     static let SHUT_RDWR: CInt = CInt(Darwin.SHUT_RDWR)
     #elseif os(Linux) || os(FreeBSD) || os(Android)
     #if canImport(Glibc)
+    @usableFromInline
     static let UIO_MAXIOV: Int = Int(Glibc.UIO_MAXIOV)
+    @usableFromInline
     static let SHUT_RD: CInt = CInt(Glibc.SHUT_RD)
+    @usableFromInline
     static let SHUT_WR: CInt = CInt(Glibc.SHUT_WR)
+    @usableFromInline
     static let SHUT_RDWR: CInt = CInt(Glibc.SHUT_RDWR)
     #elseif canImport(Musl)
+    @usableFromInline
     static let UIO_MAXIOV: Int = Int(Musl.UIO_MAXIOV)
+    @usableFromInline
     static let SHUT_RD: CInt = CInt(Musl.SHUT_RD)
+    @usableFromInline
     static let SHUT_WR: CInt = CInt(Musl.SHUT_WR)
+    @usableFromInline
     static let SHUT_RDWR: CInt = CInt(Musl.SHUT_RDWR)
     #elseif canImport(Android)
+    @usableFromInline
     static let UIO_MAXIOV: Int = Int(Android.UIO_MAXIOV)
+    @usableFromInline
     static let SHUT_RD: CInt = CInt(Android.SHUT_RD)
+    @usableFromInline
     static let SHUT_WR: CInt = CInt(Android.SHUT_WR)
+    @usableFromInline
     static let SHUT_RDWR: CInt = CInt(Android.SHUT_RDWR)
     #endif
     #else
+    @usableFromInline
     static var UIO_MAXIOV: Int {
         fatalError("unsupported OS")
     }
+    @usableFromInline
     static var SHUT_RD: Int {
         fatalError("unsupported OS")
     }
+    @usableFromInline
     static var SHUT_WR: Int {
         fatalError("unsupported OS")
     }
+    @usableFromInline
     static var SHUT_RDWR: Int {
         fatalError("unsupported OS")
     }
@@ -503,14 +536,14 @@ internal enum Posix {
 
     #if !os(Windows)
     @inline(never)
-    internal static func shutdown(descriptor: CInt, how: Shutdown) throws {
+    public static func shutdown(descriptor: CInt, how: Shutdown) throws {
         _ = try syscall(blocking: false) {
             sysShutdown(descriptor, how.cValue)
         }
     }
 
     @inline(never)
-    internal static func close(descriptor: CInt) throws {
+    public static func close(descriptor: CInt) throws {
         let res = sysClose(descriptor)
         if res == -1 {
             #if os(Windows)
@@ -534,7 +567,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func bind(descriptor: CInt, ptr: UnsafePointer<sockaddr>, bytes: Int) throws {
+    public static func bind(descriptor: CInt, ptr: UnsafePointer<sockaddr>, bytes: Int) throws {
         _ = try syscall(blocking: false) {
             sysBind(descriptor, ptr, socklen_t(bytes))
         }
@@ -542,6 +575,7 @@ internal enum Posix {
 
     @inline(never)
     @discardableResult
+    @usableFromInline
     // TODO: Allow varargs
     internal static func fcntl(descriptor: CInt, command: CInt, value: CInt) throws -> CInt {
         try syscall(blocking: false) {
@@ -550,7 +584,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func socket(
+    public static func socket(
         domain: NIOBSDSocket.ProtocolFamily,
         type: NIOBSDSocket.SocketType,
         protocolSubtype: NIOBSDSocket.ProtocolSubtype
@@ -561,7 +595,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func setsockopt(
+    public static func setsockopt(
         socket: CInt,
         level: CInt,
         optionName: CInt,
@@ -574,7 +608,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func getsockopt(
+    public static func getsockopt(
         socket: CInt,
         level: CInt,
         optionName: CInt,
@@ -587,14 +621,14 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func listen(descriptor: CInt, backlog: CInt) throws {
+    public static func listen(descriptor: CInt, backlog: CInt) throws {
         _ = try syscall(blocking: false) {
             sysListen(descriptor, backlog)
         }
     }
 
     @inline(never)
-    internal static func accept(
+    public static func accept(
         descriptor: CInt,
         addr: UnsafeMutablePointer<sockaddr>?,
         len: UnsafeMutablePointer<socklen_t>?
@@ -611,7 +645,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func connect(descriptor: CInt, addr: UnsafePointer<sockaddr>, size: socklen_t) throws -> Bool {
+    public static func connect(descriptor: CInt, addr: UnsafePointer<sockaddr>, size: socklen_t) throws -> Bool {
         do {
             _ = try syscall(blocking: false) {
                 sysConnect(descriptor, addr, size)
@@ -626,14 +660,14 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func open(file: UnsafePointer<CChar>, oFlag: CInt, mode: mode_t) throws -> CInt {
+    public static func open(file: UnsafePointer<CChar>, oFlag: CInt, mode: mode_t) throws -> CInt {
         try syscall(blocking: false) {
             sysOpenWithMode(file, oFlag, mode)
         }.result
     }
 
     @inline(never)
-    internal static func open(file: UnsafePointer<CChar>, oFlag: CInt) throws -> CInt {
+    public static func open(file: UnsafePointer<CChar>, oFlag: CInt) throws -> CInt {
         try syscall(blocking: false) {
             sysOpen(file, oFlag)
         }.result
@@ -641,21 +675,21 @@ internal enum Posix {
 
     @inline(never)
     @discardableResult
-    internal static func ftruncate(descriptor: CInt, size: off_t) throws -> CInt {
+    public static func ftruncate(descriptor: CInt, size: off_t) throws -> CInt {
         try syscall(blocking: false) {
             sysFtruncate(descriptor, size)
         }.result
     }
 
     @inline(never)
-    internal static func write(descriptor: CInt, pointer: UnsafeRawPointer, size: Int) throws -> IOResult<Int> {
+    public static func write(descriptor: CInt, pointer: UnsafeRawPointer, size: Int) throws -> IOResult<Int> {
         try syscall(blocking: true) {
             sysWrite(descriptor, pointer, size)
         }
     }
 
     @inline(never)
-    internal static func pwrite(
+    public static func pwrite(
         descriptor: CInt,
         pointer: UnsafeRawPointer,
         size: Int,
@@ -668,7 +702,7 @@ internal enum Posix {
 
     #if !os(Windows)
     @inline(never)
-    internal static func writev(descriptor: CInt, iovecs: UnsafeBufferPointer<IOVector>) throws -> IOResult<Int> {
+    public static func writev(descriptor: CInt, iovecs: UnsafeBufferPointer<IOVector>) throws -> IOResult<Int> {
         try syscall(blocking: true) {
             sysWritev(descriptor, iovecs.baseAddress!, CInt(iovecs.count))
         }
@@ -676,7 +710,7 @@ internal enum Posix {
     #endif
 
     @inline(never)
-    internal static func read(
+    public static func read(
         descriptor: CInt,
         pointer: UnsafeMutableRawPointer,
         size: size_t
@@ -687,7 +721,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func pread(
+    public static func pread(
         descriptor: CInt,
         pointer: UnsafeMutableRawPointer,
         size: size_t,
@@ -699,7 +733,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func recvmsg(
+    public static func recvmsg(
         descriptor: CInt,
         msgHdr: UnsafeMutablePointer<msghdr>,
         flags: CInt
@@ -710,7 +744,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func sendmsg(
+    public static func sendmsg(
         descriptor: CInt,
         msgHdr: UnsafePointer<msghdr>,
         flags: CInt
@@ -722,7 +756,7 @@ internal enum Posix {
 
     @discardableResult
     @inline(never)
-    internal static func lseek(descriptor: CInt, offset: off_t, whence: CInt) throws -> off_t {
+    public static func lseek(descriptor: CInt, offset: off_t, whence: CInt) throws -> off_t {
         try syscall(blocking: false) {
             sysLseek(descriptor, offset, whence)
         }.result
@@ -731,7 +765,7 @@ internal enum Posix {
 
     @discardableResult
     @inline(never)
-    internal static func dup(descriptor: CInt) throws -> CInt {
+    public static func dup(descriptor: CInt) throws -> CInt {
         try syscall(blocking: false) {
             sysDup(descriptor)
         }.result
@@ -740,7 +774,7 @@ internal enum Posix {
     #if !os(Windows)
     // It's not really posix but exists on Linux and MacOS / BSD so just put it here for now to keep it simple
     @inline(never)
-    internal static func sendfile(descriptor: CInt, fd: CInt, offset: off_t, count: size_t) throws -> IOResult<Int> {
+    public static func sendfile(descriptor: CInt, fd: CInt, offset: off_t, count: size_t) throws -> IOResult<Int> {
         var written: off_t = 0
         do {
             _ = try syscall(blocking: false) { () -> ssize_t in
@@ -778,7 +812,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func sendmmsg(
+    public static func sendmmsg(
         sockfd: CInt,
         msgvec: UnsafeMutablePointer<MMsgHdr>,
         vlen: CUnsignedInt,
@@ -790,7 +824,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func recvmmsg(
+    public static func recvmmsg(
         sockfd: CInt,
         msgvec: UnsafeMutablePointer<MMsgHdr>,
         vlen: CUnsignedInt,
@@ -803,7 +837,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func getpeername(
+    public static func getpeername(
         socket: CInt,
         address: UnsafeMutablePointer<sockaddr>,
         addressLength: UnsafeMutablePointer<socklen_t>
@@ -814,7 +848,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func getsockname(
+    public static func getsockname(
         socket: CInt,
         address: UnsafeMutablePointer<sockaddr>,
         addressLength: UnsafeMutablePointer<socklen_t>
@@ -826,7 +860,7 @@ internal enum Posix {
     #endif
 
     @inline(never)
-    internal static func if_nametoindex(_ name: UnsafePointer<CChar>?) throws -> CUnsignedInt {
+    public static func if_nametoindex(_ name: UnsafePointer<CChar>?) throws -> CUnsignedInt {
         try syscall(blocking: false) {
             sysIfNameToIndex(name!)
         }.result
@@ -834,28 +868,28 @@ internal enum Posix {
 
     #if !os(Windows)
     @inline(never)
-    internal static func poll(fds: UnsafeMutablePointer<pollfd>, nfds: nfds_t, timeout: CInt) throws -> CInt {
+    public static func poll(fds: UnsafeMutablePointer<pollfd>, nfds: nfds_t, timeout: CInt) throws -> CInt {
         try syscall(blocking: false) {
             sysPoll(fds, nfds, timeout)
         }.result
     }
 
     @inline(never)
-    internal static func fstat(descriptor: CInt, outStat: UnsafeMutablePointer<stat>) throws {
+    public static func fstat(descriptor: CInt, outStat: UnsafeMutablePointer<stat>) throws {
         _ = try syscall(blocking: false) {
             sysFstat(descriptor, outStat)
         }
     }
 
     @inline(never)
-    internal static func stat(pathname: String, outStat: UnsafeMutablePointer<stat>) throws {
+    public static func stat(pathname: String, outStat: UnsafeMutablePointer<stat>) throws {
         _ = try syscall(blocking: false) {
             sysStat(pathname, outStat)
         }
     }
 
     @inline(never)
-    internal static func lstat(pathname: String, outStat: UnsafeMutablePointer<stat>) throws {
+    public static func lstat(pathname: String, outStat: UnsafeMutablePointer<stat>) throws {
         _ = try syscall(blocking: false) {
             sysLstat(pathname, outStat)
         }
@@ -959,7 +993,7 @@ internal enum Posix {
     }
 
     @inline(never)
-    internal static func socketpair(
+    public static func socketpair(
         domain: NIOBSDSocket.ProtocolFamily,
         type: NIOBSDSocket.SocketType,
         protocolSubtype: NIOBSDSocket.ProtocolSubtype,
@@ -972,7 +1006,7 @@ internal enum Posix {
     #endif
     #if !os(Windows)
     @inline(never)
-    internal static func ioctl(fd: CInt, request: CUnsignedLong, ptr: UnsafeMutableRawPointer) throws {
+    public static func ioctl(fd: CInt, request: CUnsignedLong, ptr: UnsafeMutableRawPointer) throws {
         _ = try syscall(blocking: false) {
             /// `numericCast` to support musl which accepts `CInt` (cf. `CUnsignedLong`).
             sysIoctl(fd, numericCast(request), ptr)
@@ -997,7 +1031,7 @@ public struct NIOFailedToSetSocketNonBlockingError: Error {}
 
 #if !os(Windows)
 extension Posix {
-    static func setNonBlocking(socket: CInt) throws {
+    public static func setNonBlocking(socket: CInt) throws {
         let flags = try Posix.fcntl(descriptor: socket, command: F_GETFL, value: 0)
         do {
             let ret = try Posix.fcntl(descriptor: socket, command: F_SETFL, value: flags | O_NONBLOCK)
@@ -1014,12 +1048,13 @@ extension Posix {
 #endif
 
 #if canImport(Darwin)
+@usableFromInline
 internal enum KQueue {
 
     // TODO: Figure out how to specify a typealias to the kevent struct without run into trouble with the swift compiler
 
     @inline(never)
-    internal static func kqueue() throws -> CInt {
+    public static func kqueue() throws -> CInt {
         try syscall(blocking: false) {
             Darwin.kqueue()
         }.result
@@ -1027,7 +1062,7 @@ internal enum KQueue {
 
     @inline(never)
     @discardableResult
-    internal static func kevent(
+    public static func kevent(
         kq: CInt,
         changelist: UnsafePointer<kevent>?,
         nchanges: CInt,
diff --git a/Sources/NIOPosix/Thread.swift b/Sources/NIOPosix/Thread.swift
index f61ab29e1d..565b775ef5 100644
--- a/Sources/NIOPosix/Thread.swift
+++ b/Sources/NIOPosix/Thread.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2014 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -40,6 +40,7 @@ protocol ThreadOps {
 /// A Thread that executes some runnable block.
 ///
 /// All methods exposed are thread-safe.
+@usableFromInline
 final class NIOThread {
     internal typealias ThreadBoxValue = (body: (NIOThread) -> Void, name: String?)
     internal typealias ThreadBox = Box<ThreadBoxValue>

From dd7d4b0fb1d39345b09ba59c2ec49b9742224ca2 Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Mon, 25 Nov 2024 09:58:25 +0000
Subject: [PATCH 09/37] fix warnings: syncShutdownGracefully not available in
 async contexts (#2995)

### Motivation:

Warnings are annoying. Companion of #2994

### Modifications:

Fix all the `syncShutdownGracefully` not being available in `async`
contexts warnings.

### Result:

Everybody happier.
---
 .../AsyncChannelBootstrapTests.swift          | 83 ++++++-------------
 Tests/NIOPosixTests/SerialExecutorTests.swift | 36 +++++---
 2 files changed, 50 insertions(+), 69 deletions(-)

diff --git a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
index e1dc3d215b..847a5ac782 100644
--- a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
+++ b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2023-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -199,6 +199,8 @@ private final class AddressedEnvelopingHandler: ChannelDuplexHandler {
 
 @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
 final class AsyncChannelBootstrapTests: XCTestCase {
+    var group: MultiThreadedEventLoopGroup!
+
     enum NegotiationResult {
         case string(NIOAsyncChannel<String, String>)
         case byte(NIOAsyncChannel<UInt8, UInt8>)
@@ -214,10 +216,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     // MARK: Server/Client Bootstrap
 
     func testServerClientBootstrap_withAsyncChannel_andHostPort() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         let channel = try await ServerBootstrap(group: eventLoopGroup)
             .serverChannelOption(.socketOption(.so_reuseaddr), value: 1)
@@ -273,10 +272,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     func testAsyncChannelProtocolNegotiation() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         let channel: NIOAsyncChannel<EventLoopFuture<NegotiationResult>, Never> = try await ServerBootstrap(
             group: eventLoopGroup
@@ -360,10 +356,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     func testAsyncChannelNestedProtocolNegotiation() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         let channel: NIOAsyncChannel<EventLoopFuture<EventLoopFuture<NegotiationResult>>, Never> =
             try await ServerBootstrap(group: eventLoopGroup)
@@ -497,10 +490,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
             }
         }
 
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
         let channels = NIOLockedValueBox<[Channel]>([Channel]())
 
         let channel: NIOAsyncChannel<EventLoopFuture<NegotiationResult>, Never> = try await ServerBootstrap(
@@ -610,10 +600,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     func testServerClientBootstrap_withAsyncChannel_clientConnectedSocket() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         let channel = try await ServerBootstrap(group: eventLoopGroup)
             .serverChannelOption(.socketOption(.so_reuseaddr), value: 1)
@@ -675,10 +662,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     // MARK: Datagram Bootstrap
 
     func testDatagramBootstrap_withAsyncChannel_andHostPort() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         let serverChannel = try await self.makeUDPServerChannel(eventLoopGroup: eventLoopGroup)
         let clientChannel = try await self.makeUDPClientChannel(
@@ -700,10 +684,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     func testDatagramBootstrap_withProtocolNegotiation_andHostPort() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         // We are creating a channel here to get a random port from the system
         let channel = try await DatagramBootstrap(group: eventLoopGroup)
@@ -785,10 +766,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     // MARK: - Pipe Bootstrap
 
     func testPipeBootstrap() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
         let (pipe1ReadFD, pipe1WriteFD, pipe2ReadFD, pipe2WriteFD) = self.makePipeFileDescriptors()
         let channel: NIOAsyncChannel<ByteBuffer, ByteBuffer>
         let toChannel: NIOAsyncChannel<Never, ByteBuffer>
@@ -861,10 +839,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     func testPipeBootstrap_whenInputNil() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
         let (pipe1ReadFD, pipe1WriteFD) = self.makePipeFileDescriptors()
         let channel: NIOAsyncChannel<ByteBuffer, ByteBuffer>
         let fromChannel: NIOAsyncChannel<ByteBuffer, Never>
@@ -916,10 +891,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     func testPipeBootstrap_whenOutputNil() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
         let (pipe1ReadFD, pipe1WriteFD) = self.makePipeFileDescriptors()
         let channel: NIOAsyncChannel<ByteBuffer, ByteBuffer>
         let toChannel: NIOAsyncChannel<Never, ByteBuffer>
@@ -973,10 +945,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     func testPipeBootstrap_withProtocolNegotiation() async throws {
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
         let (pipe1ReadFD, pipe1WriteFD, pipe2ReadFD, pipe2WriteFD) = self.makePipeFileDescriptors()
         let negotiationResult: EventLoopFuture<NegotiationResult>
         let toChannel: NIOAsyncChannel<Never, ByteBuffer>
@@ -1067,10 +1036,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
 
     func testRawSocketBootstrap() async throws {
         try XCTSkipIfUserHasNotEnoughRightsForRawSocketAPI()
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         let serverChannel = try await self.makeRawSocketServerChannel(eventLoopGroup: eventLoopGroup)
         let clientChannel = try await self.makeRawSocketClientChannel(eventLoopGroup: eventLoopGroup)
@@ -1091,10 +1057,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
 
     func testRawSocketBootstrap_withProtocolNegotiation() async throws {
         try XCTSkipIfUserHasNotEnoughRightsForRawSocketAPI()
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         try await withThrowingTaskGroup(of: EventLoopFuture<NegotiationResult>.self) { group in
             group.addTask {
@@ -1142,10 +1105,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
 
     func testVSock() async throws {
         try XCTSkipUnless(System.supportsVsockLoopback, "No vsock loopback transport available")
-        let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3)
-        defer {
-            try! eventLoopGroup.syncShutdownGracefully()
-        }
+        let eventLoopGroup = self.group!
 
         let port = VsockAddress.Port(1234)
 
@@ -1560,6 +1520,15 @@ final class AsyncChannelBootstrapTests: XCTestCase {
         try channel.pipeline.syncOperations.addHandler(negotiationHandler)
         return negotiationHandler.protocolNegotiationResult
     }
+
+    override func setUp() {
+        self.group = MultiThreadedEventLoopGroup(numberOfThreads: 3)
+    }
+
+    override func tearDown() {
+        XCTAssertNoThrow(try self.group.syncShutdownGracefully())
+        self.group = nil
+    }
 }
 
 @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
diff --git a/Tests/NIOPosixTests/SerialExecutorTests.swift b/Tests/NIOPosixTests/SerialExecutorTests.swift
index 0ec3994f9c..3c69698d0f 100644
--- a/Tests/NIOPosixTests/SerialExecutorTests.swift
+++ b/Tests/NIOPosixTests/SerialExecutorTests.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2023-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -47,6 +47,8 @@ actor EventLoopBoundActor {
 
 @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *)
 final class SerialExecutorTests: XCTestCase {
+    var group: MultiThreadedEventLoopGroup!
+
     private func _testBasicExecutorFitsOnEventLoop(loop1: EventLoop, loop2: EventLoop) async throws {
         let testActor = EventLoopBoundActor(loop: loop1)
         await testActor.assertInLoop(loop1)
@@ -54,23 +56,25 @@ final class SerialExecutorTests: XCTestCase {
     }
 
     func testBasicExecutorFitsOnEventLoop_MTELG() async throws {
-        let group = MultiThreadedEventLoopGroup(numberOfThreads: 2)
-        defer {
-            try! group.syncShutdownGracefully()
-        }
-        let loops = Array(group.makeIterator())
+        let loops = Array(self.group.makeIterator())
         try await self._testBasicExecutorFitsOnEventLoop(loop1: loops[0], loop2: loops[1])
     }
 
     func testBasicExecutorFitsOnEventLoop_AsyncTestingEventLoop() async throws {
         let loop1 = NIOAsyncTestingEventLoop()
         let loop2 = NIOAsyncTestingEventLoop()
-        defer {
-            try? loop1.syncShutdownGracefully()
-            try? loop2.syncShutdownGracefully()
+        func shutdown() async {
+            await loop1.shutdownGracefully()
+            await loop2.shutdownGracefully()
         }
 
-        try await self._testBasicExecutorFitsOnEventLoop(loop1: loop1, loop2: loop2)
+        do {
+            try await self._testBasicExecutorFitsOnEventLoop(loop1: loop1, loop2: loop2)
+            await shutdown()
+        } catch {
+            await shutdown()
+            throw error
+        }
     }
 
     func testAssumeIsolation() async throws {
@@ -78,8 +82,7 @@ final class SerialExecutorTests: XCTestCase {
         throw XCTSkip("Custom executors are only supported in 5.9")
         #else
 
-        let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
-        let el = group.next()
+        let el = self.group.next()
 
         let testActor = EventLoopBoundActor(loop: el)
         let result = try await el.submit {
@@ -88,4 +91,13 @@ final class SerialExecutorTests: XCTestCase {
         XCTAssertEqual(result, 0)
         #endif
     }
+
+    override func setUp() {
+        self.group = MultiThreadedEventLoopGroup(numberOfThreads: 3)
+    }
+
+    override func tearDown() {
+        XCTAssertNoThrow(try self.group.syncShutdownGracefully())
+        self.group = nil
+    }
 }

From ba6608ec10f8dcce1aaae90f5e31f11e39f1bbfc Mon Sep 17 00:00:00 2001
From: George Barnett <gbarnett@apple.com>
Date: Mon, 25 Nov 2024 10:55:27 +0000
Subject: [PATCH 10/37] Only toggle production state when above/below watermark
 (#2996)

Motivation:

The high/low watermark backpressure strategy assumes that a yield can
never happen when there's no outstanding demand. It's reasonably easy
for this to not be a true: a handler decoding messages in a loop, for
example, may not pay attention to `read() calls and produce when there's
no outstanding demand. The channel should of course not read from the
socket, so produced values will stop being produced.

Modifications:

- Alter the high/low watermarks such that demand is only enabled when
the buffer depth is below the low watermark, and only disabled when
above the high watermark.

Result:

Fewer crashes
---
 .../NIOAsyncSequenceProducerStrategies.swift  | 21 ++++++-------------
 ...owWatermarkBackPressureStrategyTests.swift | 11 ++++++++++
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift b/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift
index 6de186197c..abed4fd870 100644
--- a/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift
+++ b/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift
@@ -37,29 +37,20 @@ public enum NIOAsyncSequenceProducerBackPressureStrategies {
 
         public mutating func didYield(bufferDepth: Int) -> Bool {
             // We are demanding more until we reach the high watermark
-            if bufferDepth < self.highWatermark {
-                precondition(self.hasOustandingDemand)
-                return true
-            } else {
+            if bufferDepth >= self.highWatermark {
                 self.hasOustandingDemand = false
-                return false
             }
+
+            return self.hasOustandingDemand
         }
 
         public mutating func didConsume(bufferDepth: Int) -> Bool {
             // We start demanding again once we are below the low watermark
             if bufferDepth < self.lowWatermark {
-                if self.hasOustandingDemand {
-                    // We are below and have outstanding demand
-                    return true
-                } else {
-                    // We are below but don't have outstanding demand but need more
-                    self.hasOustandingDemand = true
-                    return true
-                }
-            } else {
-                return self.hasOustandingDemand
+                self.hasOustandingDemand = true
             }
+
+            return self.hasOustandingDemand
         }
     }
 }
diff --git a/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift b/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift
index f7a6790ecf..cd7c738f98 100644
--- a/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift
+++ b/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift
@@ -57,4 +57,15 @@ final class NIOAsyncSequenceProducerBackPressureStrategiesHighLowWatermarkTests:
     func testDidConsume_whenAtLowWatermark() {
         XCTAssertTrue(self.strategy.didConsume(bufferDepth: 5))
     }
+
+    func testDidYieldWhenNoOutstandingDemand() {
+        // Hit the high watermark
+        XCTAssertFalse(self.strategy.didYield(bufferDepth: 10))
+        // Drop below it, don't read.
+        XCTAssertFalse(self.strategy.didConsume(bufferDepth: 7))
+        // Yield more, still above the low watermark, so don't produce more.
+        XCTAssertFalse(self.strategy.didYield(bufferDepth: 8))
+        // Drop below low watermark to start producing again.
+        XCTAssertTrue(self.strategy.didConsume(bufferDepth: 4))
+    }
 }

From 0ee1657e8a648b2997aef020e0d2b2adbf49dce8 Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Mon, 25 Nov 2024 16:05:30 +0000
Subject: [PATCH 11/37] Make NIOFileDescriptor/FileRegion/IOData Sendable &
 soft-deprecated (#2598)

Motivation:

`IOData` is a legacy but alas also core type that needs to be
`Sendable`. Before this PR however it can't be `Sendable` because it
holds a `FileRegion` which holds a `NIOFileDescriptor`. So let's make
all of these `Sendable` but let's also start the deprecation journey for
the following types:

- `IOData`, now soft-deprecated (no warnings) because on its reliance on
`FileRegion`
- `FileRegion`, now soft-deprecated (no warnings) because on its
reliance on `NIOFileHandle`
- `NIOFileHandle`, now soft-deprecated (warnings on the
`NIOFileHandle(descriptor:)` constructor but with a
`NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor:)` alternative
- `NonBlockingFileIO`, now soft-deprecated (warnings on the `openFile`
functions (but with `_deprecated` alternatives) because of their
reliance on `NIOFileHandle)

Modification:

- Make `NIOFileDescriptor`, `FileRegion` and `IOData` `Sendable` by
tracking the fd number and the usage state in an atomic
- Enforce singular access by making the `withFileDescriptor { fd ... }`
function atomically exchange the fd number for a "I'm busy" sentinel
value
- Start deprecating `IOData`, `NIOFileHandle`, `NonBlockingFileIO`,
`FileRegion`

Result:
- `NIOFileDescriptor`, `FileRegion` and `IOData` can be `Sendable`
---
 Sources/NIOCore/ChannelInvoker.swift          |   2 +-
 Sources/NIOCore/FileHandle.swift              | 289 ++++++++++++++++--
 Sources/NIOCore/FileRegion.swift              |  12 +-
 Sources/NIOCore/IOData.swift                  |  12 +-
 Sources/NIOCore/Linux.swift                   |   2 +-
 Sources/NIOCrashTester/OutputGrepper.swift    |   6 +-
 Sources/NIOHTTP1Server/main.swift             |   2 +-
 Sources/NIOPosix/Bootstrap.swift              |  32 +-
 Sources/NIOPosix/NonBlockingFileIO.swift      | 109 ++++++-
 Sources/NIOPosix/PendingWritesManager.swift   |   7 +-
 Sources/NIOPosix/PipeChannel.swift            |  38 +--
 Sources/NIOPosix/PipePair.swift               |  91 ++++--
 Sources/NIOPosix/SelectorEpoll.swift          |  12 +-
 Sources/NIOPosix/SelectorGeneric.swift        |  20 +-
 Sources/NIOPosix/SelectorKqueue.swift         |  38 ++-
 Sources/NIOPosix/SelectorUring.swift          |  14 +-
 Tests/NIOCoreTests/BaseObjectsTest.swift      |  10 +-
 Tests/NIOCoreTests/NIOAnyDebugTest.swift      |   2 +-
 Tests/NIOCoreTests/XCTest+Extensions.swift    |   4 +-
 .../EmbeddedChannelTest.swift                 |   4 +-
 .../NIOHTTP1Tests/HTTPServerClientTest.swift  |   4 +-
 Tests/NIOPosixTests/BootstrapTest.swift       |  33 +-
 Tests/NIOPosixTests/ChannelPipelineTest.swift |   4 +-
 Tests/NIOPosixTests/ChannelTests.swift        |  16 +-
 Tests/NIOPosixTests/FileRegionTest.swift      |  14 +-
 Tests/NIOPosixTests/NIOFileHandleTest.swift   | 171 +++++++++++
 .../NIOPosixTests/NonBlockingFileIOTest.swift |  62 ++--
 Tests/NIOPosixTests/TestUtils.swift           |  14 +-
 28 files changed, 771 insertions(+), 253 deletions(-)
 create mode 100644 Tests/NIOPosixTests/NIOFileHandleTest.swift

diff --git a/Sources/NIOCore/ChannelInvoker.swift b/Sources/NIOCore/ChannelInvoker.swift
index 8d831cac34..af3051b5f7 100644
--- a/Sources/NIOCore/ChannelInvoker.swift
+++ b/Sources/NIOCore/ChannelInvoker.swift
@@ -195,7 +195,7 @@ extension ChannelOutboundInvoker {
     public func close(mode: CloseMode = .all, file: StaticString = #fileID, line: UInt = #line) -> EventLoopFuture<Void>
     {
         let promise = makePromise(file: file, line: line)
-        close(mode: mode, promise: promise)
+        self.close(mode: mode, promise: promise)
         return promise.futureResult
     }
 
diff --git a/Sources/NIOCore/FileHandle.swift b/Sources/NIOCore/FileHandle.swift
index 733aec483e..62fd100e28 100644
--- a/Sources/NIOCore/FileHandle.swift
+++ b/Sources/NIOCore/FileHandle.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -11,6 +11,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 //===----------------------------------------------------------------------===//
+
+import Atomics
+
 #if os(Windows)
 import ucrt
 #elseif canImport(Darwin)
@@ -34,7 +37,105 @@ public typealias NIOPOSIXFileMode = CInt
 public typealias NIOPOSIXFileMode = mode_t
 #endif
 
-/// A `NIOFileHandle` is a handle to an open file.
+#if arch(x86_64) || arch(arm64)
+// 64 bit architectures
+typealias OneUInt32 = UInt32
+typealias TwoUInt32s = UInt64
+
+// Now we need to make `UInt64` match `DoubleWord`'s API but we can't use a custom
+// type because we need special support by the `swift-atomics` package.
+extension UInt64 {
+    fileprivate init(first: UInt32, second: UInt32) {
+        self = UInt64(first) << 32 | UInt64(second)
+    }
+
+    fileprivate var first: UInt32 {
+        get {
+            UInt32(truncatingIfNeeded: self >> 32)
+        }
+        set {
+            self = (UInt64(newValue) << 32) | UInt64(self.second)
+        }
+    }
+
+    fileprivate var second: UInt32 {
+        get {
+            UInt32(truncatingIfNeeded: self & 0xff_ff_ff_ff)
+        }
+        set {
+            self = (UInt64(self.first) << 32) | UInt64(newValue)
+        }
+    }
+}
+#elseif arch(arm) || arch(i386) || arch(arm64_32)
+// 32 bit architectures
+// Note: for testing purposes you can also use these defines for 64 bit platforms, they'll just consume twice as
+// much space, nothing else will go bad.
+typealias OneUInt32 = UInt
+typealias TwoUInt32s = DoubleWord
+#else
+#error("Unknown architecture")
+#endif
+
+internal struct FileDescriptorState {
+    private static let closedValue: OneUInt32 = 0xdead
+    private static let inUseValue: OneUInt32 = 0xbeef
+    private static let openValue: OneUInt32 = 0xcafe
+    internal var rawValue: TwoUInt32s
+
+    internal init(rawValue: TwoUInt32s) {
+        self.rawValue = rawValue
+    }
+
+    internal init(descriptor: CInt) {
+        self.rawValue = TwoUInt32s(
+            first: .init(truncatingIfNeeded: CUnsignedInt(bitPattern: descriptor)),
+            second: Self.openValue
+        )
+    }
+
+    internal var descriptor: CInt {
+        get {
+            CInt(bitPattern: UInt32(truncatingIfNeeded: self.rawValue.first))
+        }
+        set {
+            self.rawValue.first = .init(truncatingIfNeeded: CUnsignedInt(bitPattern: newValue))
+        }
+    }
+
+    internal var isOpen: Bool {
+        self.rawValue.second == Self.openValue
+    }
+
+    internal var isInUse: Bool {
+        self.rawValue.second == Self.inUseValue
+    }
+
+    internal var isClosed: Bool {
+        self.rawValue.second == Self.closedValue
+    }
+
+    mutating func close() {
+        assert(self.isOpen)
+        self.rawValue.second = Self.closedValue
+    }
+
+    mutating func markInUse() {
+        assert(self.isOpen)
+        self.rawValue.second = Self.inUseValue
+    }
+
+    mutating func markNotInUse() {
+        assert(self.rawValue.second == Self.inUseValue)
+        self.rawValue.second = Self.openValue
+    }
+}
+
+/// Deprecated. `NIOFileHandle` is a handle to an open file descriptor.
+///
+/// - warning: The `NIOFileHandle` API is deprecated, do not use going forward. It's not marked as `deprecated` yet such
+///            that users don't get the deprecation warnings affecting their APIs everywhere. For file I/O, please use
+///            the `NIOFileSystem` API.
 ///
 /// When creating a `NIOFileHandle` it takes ownership of the underlying file descriptor. When a `NIOFileHandle` is no longer
 /// needed you must `close` it or take back ownership of the file descriptor using `takeDescriptorOwnership`.
@@ -43,16 +144,54 @@ public typealias NIOPOSIXFileMode = mode_t
 ///
 /// - warning: Failing to manage the lifetime of a `NIOFileHandle` correctly will result in undefined behaviour.
 ///
-/// - warning: `NIOFileHandle` objects are not thread-safe and are mutable. They also cannot be fully thread-safe as they refer to a global underlying file descriptor.
-public final class NIOFileHandle: FileDescriptor {
-    public private(set) var isOpen: Bool
-    private let descriptor: CInt
+/// - Note: As of SwiftNIO 2.77.0, `NIOFileHandle` objects are are thread-safe and enforce singular access. If you access the same `NIOFileHandle`
+///         multiple times, it will throw `IOError(errorCode: EBUSY)` for the second access.
+public final class NIOFileHandle: FileDescriptor & Sendable {
+    private static let descriptorClosed: CInt = CInt.min
+    private let descriptor: UnsafeAtomic<TwoUInt32s>
+
+    public var isOpen: Bool {
+        FileDescriptorState(
+            rawValue: self.descriptor.load(ordering: .sequentiallyConsistent)
+        ).isOpen
+    }
+
+    private static func interpretDescriptorValueThrowIfInUseOrNotOpen(
+        _ descriptor: TwoUInt32s
+    ) throws -> FileDescriptorState {
+        let descriptorState = FileDescriptorState(rawValue: descriptor)
+        if descriptorState.isOpen {
+            return descriptorState
+        } else if descriptorState.isClosed {
+            throw IOError(errnoCode: EBADF, reason: "can't close file (as it's not open anymore).")
+        } else {
+            throw IOError(errnoCode: EBUSY, reason: "file descriptor currently in use")
+        }
+    }
+
+    private func peekAtDescriptorIfOpen() throws -> FileDescriptorState {
+        let descriptor = self.descriptor.load(ordering: .relaxed)
+        return try Self.interpretDescriptorValueThrowIfInUseOrNotOpen(descriptor)
+    }
+
+    /// Create a `NIOFileHandle` taking ownership of `descriptor`. You must call `NIOFileHandle.close` or `NIOFileHandle.takeDescriptorOwnership` before
+    /// this object can be safely released.
+    @available(
+        *,
+        deprecated,
+        message: """
+            Avoid using NIOFileHandle. The type is difficult to hold correctly, \
+            use NIOFileSystem as a replacement API.
+            """
+    )
+    public convenience init(descriptor: CInt) {
+        self.init(_deprecatedTakingOwnershipOfDescriptor: descriptor)
+    }
 
     /// Create a `NIOFileHandle` taking ownership of `descriptor`. You must call `NIOFileHandle.close` or `NIOFileHandle.takeDescriptorOwnership` before
     /// this object can be safely released.
-    public init(descriptor: CInt) {
-        self.descriptor = descriptor
-        self.isOpen = true
+    public init(_deprecatedTakingOwnershipOfDescriptor descriptor: CInt) {
+        self.descriptor = UnsafeAtomic.create(FileDescriptorState(descriptor: descriptor).rawValue)
     }
 
     deinit {
@@ -60,6 +199,7 @@ public final class NIOFileHandle: FileDescriptor {
             !self.isOpen,
             "leaked open NIOFileHandle(descriptor: \(self.descriptor)). Call `close()` to close or `takeDescriptorOwnership()` to take ownership and close by some other means."
         )
+        self.descriptor.destroy()
     }
 
     #if !os(WASI)
@@ -70,12 +210,64 @@ public final class NIOFileHandle: FileDescriptor {
     ///
     /// - Returns: A new `NIOFileHandle` with a fresh underlying file descriptor but shared seek pointer.
     public func duplicate() throws -> NIOFileHandle {
-        try withUnsafeFileDescriptor { fd in
-            NIOFileHandle(descriptor: try SystemCalls.dup(descriptor: fd))
+        try self.withUnsafeFileDescriptor { fd in
+            NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: try SystemCalls.dup(descriptor: fd))
         }
     }
     #endif
 
+    private func activateDescriptor(as descriptor: CInt) {
+        let desired = FileDescriptorState(descriptor: descriptor)
+        var expected = desired
+        expected.markInUse()
+        let (exchanged, original) = self.descriptor.compareExchange(
+            expected: expected.rawValue,
+            desired: desired.rawValue,
+            ordering: .sequentiallyConsistent
+        )
+        guard exchanged || FileDescriptorState(rawValue: original).isClosed else {
+            fatalError("bug in NIO (please report): NIOFileDescritor activate failed \(original)")
+        }
+    }
+
+    private func deactivateDescriptor(toClosed: Bool) throws -> CInt {
+        let peekedDescriptor = try self.peekAtDescriptorIfOpen()
+        // Don't worry, the above is just opportunistic. If we lose the race, we re-check below --> `!exchanged`
+        assert(peekedDescriptor.isOpen)
+        var desired = peekedDescriptor
+        if toClosed {
+            desired.close()
+        } else {
+            desired.markInUse()
+        }
+        assert(desired.rawValue != peekedDescriptor.rawValue, "\(desired.rawValue) == \(peekedDescriptor.rawValue)")
+        let (exchanged, originalDescriptor) = self.descriptor.compareExchange(
+            expected: peekedDescriptor.rawValue,
+            desired: desired.rawValue,
+            ordering: .sequentiallyConsistent
+        )
+
+        if exchanged {
+            assert(peekedDescriptor.rawValue == originalDescriptor)
+            return peekedDescriptor.descriptor
+        } else {
+            // We lost the race above, so this _will_ throw (as we're not closed).
+            let fauxDescriptor = try Self.interpretDescriptorValueThrowIfInUseOrNotOpen(originalDescriptor)
+            // This is impossible, because there are only 4 options in which the exchange above can fail
+            // 1. Descriptor already closed (would've thrown above)
+            // 2. Descriptor in use (would've thrown above)
+            // 3. Descriptor at illegal negative value (would've crashed above)
+            // 4. Descriptor a different, positive value (this is where we're at) --> memory corruption, let's crash
+            fatalError(
+                """
+                bug in NIO (please report): \
+                NIOFileDescriptor illegal state \
+                (\(peekedDescriptor), \(originalDescriptor), \(fauxDescriptor))")
+                """
+            )
+        }
+    }
+
     /// Take the ownership of the underlying file descriptor. This is similar to `close()` but the underlying file
     /// descriptor remains open. The caller is responsible for closing the file descriptor by some other means.
     ///
@@ -83,27 +275,20 @@ public final class NIOFileHandle: FileDescriptor {
     ///
     /// - Returns: The underlying file descriptor, now owned by the caller.
     public func takeDescriptorOwnership() throws -> CInt {
-        guard self.isOpen else {
-            throw IOError(errnoCode: EBADF, reason: "can't close file (as it's not open anymore).")
-        }
-
-        self.isOpen = false
-        return self.descriptor
+        try self.deactivateDescriptor(toClosed: true)
     }
 
     public func close() throws {
-        try withUnsafeFileDescriptor { fd in
-            try SystemCalls.close(descriptor: fd)
-        }
-
-        self.isOpen = false
+        let descriptor = try self.deactivateDescriptor(toClosed: true)
+        try SystemCalls.close(descriptor: descriptor)
     }
 
     public func withUnsafeFileDescriptor<T>(_ body: (CInt) throws -> T) throws -> T {
-        guard self.isOpen else {
-            throw IOError(errnoCode: EBADF, reason: "file descriptor already closed!")
+        let descriptor = try self.deactivateDescriptor(toClosed: false)
+        defer {
+            self.activateDescriptor(as: descriptor)
         }
-        return try body(self.descriptor)
+        return try body(descriptor)
     }
 }
 
@@ -180,29 +365,75 @@ extension NIOFileHandle {
     ///   - path: The path of the file to open. The ownership of the file descriptor is transferred to this `NIOFileHandle` and so it will be closed once `close` is called.
     ///   - mode: Access mode. Default mode is `.read`.
     ///   - flags: Additional POSIX flags.
-    public convenience init(path: String, mode: Mode = .read, flags: Flags = .default) throws {
+    @available(
+        *,
+        deprecated,
+        message: """
+            Avoid using NIOFileHandle. The type is difficult to hold correctly, \
+            use NIOFileSystem as a replacement API.
+            """
+    )
+    @available(*, noasync, message: "This method may block the calling thread")
+    public convenience init(
+        path: String,
+        mode: Mode = .read,
+        flags: Flags = .default
+    ) throws {
+        try self.init(_deprecatedPath: path, mode: mode, flags: flags)
+    }
+
+    /// Open a new `NIOFileHandle`. This operation is blocking.
+    ///
+    /// - Parameters:
+    ///   - path: The path of the file to open. The ownership of the file descriptor is transferred to this `NIOFileHandle` and so it will be closed once `close` is called.
+    ///   - mode: Access mode. Default mode is `.read`.
+    ///   - flags: Additional POSIX flags.
+    @available(*, noasync, message: "This method may block the calling thread")
+    public convenience init(
+        _deprecatedPath path: String,
+        mode: Mode = .read,
+        flags: Flags = .default
+    ) throws {
         #if os(Windows)
         let fl = mode.posixFlags | flags.posixFlags | _O_NOINHERIT
         #else
         let fl = mode.posixFlags | flags.posixFlags | O_CLOEXEC
         #endif
         let fd = try SystemCalls.open(file: path, oFlag: fl, mode: flags.posixMode)
-        self.init(descriptor: fd)
+        self.init(_deprecatedTakingOwnershipOfDescriptor: fd)
     }
 
     /// Open a new `NIOFileHandle`. This operation is blocking.
     ///
     /// - Parameters:
     ///   - path: The path of the file to open. The ownership of the file descriptor is transferred to this `NIOFileHandle` and so it will be closed once `close` is called.
+    @available(
+        *,
+        deprecated,
+        message: """
+            Avoid using NIOFileHandle. The type is difficult to hold correctly, \
+            use NIOFileSystem as a replacement API.
+            """
+    )
+    @available(*, noasync, message: "This method may block the calling thread")
     public convenience init(path: String) throws {
+        try self.init(_deprecatedPath: path)
+    }
+
+    /// Open a new `NIOFileHandle`. This operation is blocking.
+    ///
+    /// - Parameters:
+    ///   - path: The path of the file to open. The ownership of the file descriptor is transferred to this `NIOFileHandle` and so it will be closed once `close` is called.
+    @available(*, noasync, message: "This method may block the calling thread")
+    public convenience init(_deprecatedPath path: String) throws {
         // This function is here because we had a function like this in NIO 2.0, and the one above doesn't quite match. Sadly we can't
         // really deprecate this either, because it'll be preferred to the one above in many cases.
-        try self.init(path: path, mode: .read, flags: .default)
+        try self.init(_deprecatedPath: path, mode: .read, flags: .default)
     }
 }
 
 extension NIOFileHandle: CustomStringConvertible {
     public var description: String {
-        "FileHandle { descriptor: \(self.descriptor) }"
+        "FileHandle { descriptor: \(FileDescriptorState(rawValue: self.descriptor.load(ordering: .relaxed)).descriptor) }"
     }
 }
diff --git a/Sources/NIOCore/FileRegion.swift b/Sources/NIOCore/FileRegion.swift
index fdeef77844..1b1f1c5b03 100644
--- a/Sources/NIOCore/FileRegion.swift
+++ b/Sources/NIOCore/FileRegion.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -29,6 +29,10 @@ import WASILibc
 
 /// A `FileRegion` represent a readable portion usually created to be sent over the network.
 ///
+/// - warning: The `FileRegion` API is deprecated, do not use going forward. It's not marked as `deprecated` yet such
+///            that users don't get the deprecation warnings affecting their APIs everywhere. For file I/O, please use
+///            the `NIOFileSystem` API.
+///
 /// Usually a `FileRegion` will allow the underlying transport to use `sendfile` to transfer its content and so allows transferring
 /// the file content without copying it into user-space at all. If the actual transport implementation really can make use of sendfile
 /// or if it will need to copy the content to user-space first and use `write` / `writev` is an implementation detail. That said
@@ -37,9 +41,9 @@ import WASILibc
 /// One important note, depending your `ChannelPipeline` setup it may not be possible to use a `FileRegion` as a `ChannelHandler` may
 /// need access to the bytes (in a `ByteBuffer`) to transform these.
 ///
-/// - Note: It is important to manually manage the lifetime of the `NIOFileHandle` used to create a `FileRegion`.
-/// - warning: `FileRegion` objects are not thread-safe and are mutable. They also cannot be fully thread-safe as they refer to a global underlying file descriptor.
-public struct FileRegion {
+/// - Note: It is important to manually manage the lifetime of the ``NIOFileHandle`` used to create a ``FileRegion``.
+/// - Note: As of SwiftNIO 2.77.0, `FileRegion` objects are are thread-safe and the underlying ``NIOFileHandle`` does enforce singular access.
+public struct FileRegion: Sendable {
 
     /// The `NIOFileHandle` that is used by this `FileRegion`.
     public let fileHandle: NIOFileHandle
diff --git a/Sources/NIOCore/IOData.swift b/Sources/NIOCore/IOData.swift
index 293ad9b00b..bad0585b67 100644
--- a/Sources/NIOCore/IOData.swift
+++ b/Sources/NIOCore/IOData.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -14,14 +14,19 @@
 
 /// `IOData` unifies standard SwiftNIO types that are raw bytes of data; currently `ByteBuffer` and `FileRegion`.
 ///
+/// - warning: `IOData` is a legacy API, please avoid using it as much as possible.
+///
 /// Many `ChannelHandler`s receive or emit bytes and in most cases this can be either a `ByteBuffer` or a `FileRegion`
 /// from disk. To still form a well-typed `ChannelPipeline` such handlers should receive and emit value of type `IOData`.
-public enum IOData {
+public enum IOData: Sendable {
     /// A `ByteBuffer`.
     case byteBuffer(ByteBuffer)
 
     /// A `FileRegion`.
     ///
+    /// - warning: `IOData.fileRegion` is a legacy API, please avoid using it. It cannot work with TLS and `FileRegion`
+    ///            and the underlying `NIOFileHandle` objects are very difficult to hold correctly.
+    ///
     /// Sending a `FileRegion` through the `ChannelPipeline` using `write` can be useful because some `Channel`s can
     /// use `sendfile` to send a `FileRegion` more efficiently.
     case fileRegion(FileRegion)
@@ -30,9 +35,6 @@ public enum IOData {
 /// `IOData` objects are comparable just like the values they wrap.
 extension IOData: Equatable {}
 
-@available(*, unavailable)
-extension IOData: Sendable {}
-
 /// `IOData` provide a number of readable bytes.
 extension IOData {
     /// Returns the number of readable bytes in this `IOData`.
diff --git a/Sources/NIOCore/Linux.swift b/Sources/NIOCore/Linux.swift
index 3f944ee726..754a35afca 100644
--- a/Sources/NIOCore/Linux.swift
+++ b/Sources/NIOCore/Linux.swift
@@ -24,7 +24,7 @@ enum Linux {
     static let cfsCpuMaxPath = "/sys/fs/cgroup/cpu.max"
 
     private static func firstLineOfFile(path: String) throws -> Substring {
-        let fh = try NIOFileHandle(path: path)
+        let fh = try NIOFileHandle(_deprecatedPath: path)
         defer { try! fh.close() }
         // linux doesn't properly report /sys/fs/cgroup/* files lengths so we use a reasonable limit
         var buf = ByteBufferAllocator().buffer(capacity: 1024)
diff --git a/Sources/NIOCrashTester/OutputGrepper.swift b/Sources/NIOCrashTester/OutputGrepper.swift
index 069f182fae..45f56d179a 100644
--- a/Sources/NIOCrashTester/OutputGrepper.swift
+++ b/Sources/NIOCrashTester/OutputGrepper.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2020-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2020-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -39,7 +39,9 @@ internal struct OutputGrepper {
                 }
             }
             .takingOwnershipOfDescriptor(input: dup(processToChannel.fileHandleForReading.fileDescriptor))
-        let processOutputPipe = NIOFileHandle(descriptor: dup(processToChannel.fileHandleForWriting.fileDescriptor))
+        let processOutputPipe = NIOFileHandle(
+            _deprecatedTakingOwnershipOfDescriptor: dup(processToChannel.fileHandleForWriting.fileDescriptor)
+        )
         processToChannel.fileHandleForReading.closeFile()
         processToChannel.fileHandleForWriting.closeFile()
         channelFuture.cascadeFailure(to: outputPromise)
diff --git a/Sources/NIOHTTP1Server/main.swift b/Sources/NIOHTTP1Server/main.swift
index 03f755442f..2b662bcc3d 100644
--- a/Sources/NIOHTTP1Server/main.swift
+++ b/Sources/NIOHTTP1Server/main.swift
@@ -422,7 +422,7 @@ private final class HTTPHandler: ChannelInboundHandler {
                 return
             }
             let path = self.htdocsPath + "/" + path
-            let fileHandleAndRegion = self.fileIO.openFile(path: path, eventLoop: context.eventLoop)
+            let fileHandleAndRegion = self.fileIO.openFile(_deprecatedPath: path, eventLoop: context.eventLoop)
             fileHandleAndRegion.whenFailure {
                 sendErrorResponse(request: request, $0)
             }
diff --git a/Sources/NIOPosix/Bootstrap.swift b/Sources/NIOPosix/Bootstrap.swift
index c48043651b..ff2773a2cd 100644
--- a/Sources/NIOPosix/Bootstrap.swift
+++ b/Sources/NIOPosix/Bootstrap.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -2407,8 +2407,8 @@ extension NIOPipeBootstrap {
         let channelOptions = self._channelOptions
 
         let channel: PipeChannel
-        let inputFileHandle: NIOFileHandle?
-        let outputFileHandle: NIOFileHandle?
+        let pipeChannelInput: SelectablePipeHandle?
+        let pipeChannelOutput: SelectablePipeHandle?
         do {
             if let input = input {
                 try self.validateFileDescriptorIsNotAFile(input)
@@ -2417,18 +2417,18 @@ extension NIOPipeBootstrap {
                 try self.validateFileDescriptorIsNotAFile(output)
             }
 
-            inputFileHandle = input.flatMap { NIOFileHandle(descriptor: $0) }
-            outputFileHandle = output.flatMap { NIOFileHandle(descriptor: $0) }
+            pipeChannelInput = input.flatMap { SelectablePipeHandle(takingOwnershipOfDescriptor: $0) }
+            pipeChannelOutput = output.flatMap { SelectablePipeHandle(takingOwnershipOfDescriptor: $0) }
             do {
                 channel = try self.hooks.makePipeChannel(
                     eventLoop: eventLoop as! SelectableEventLoop,
-                    inputPipe: inputFileHandle,
-                    outputPipe: outputFileHandle
+                    input: pipeChannelInput,
+                    output: pipeChannelOutput
                 )
             } catch {
                 // Release file handles back to the caller in case of failure.
-                _ = try? inputFileHandle?.takeDescriptorOwnership()
-                _ = try? outputFileHandle?.takeDescriptorOwnership()
+                _ = try? pipeChannelInput?.takeDescriptorOwnership()
+                _ = try? pipeChannelOutput?.takeDescriptorOwnership()
                 throw error
             }
         } catch {
@@ -2447,10 +2447,10 @@ extension NIOPipeBootstrap {
                 channel.registerAlreadyConfigured0(promise: promise)
                 return promise.futureResult.map { result }
             }.flatMap { result -> EventLoopFuture<ChannelInitializerResult> in
-                if inputFileHandle == nil {
+                if pipeChannelInput == nil {
                     return channel.close(mode: .input).map { result }
                 }
-                if outputFileHandle == nil {
+                if pipeChannelOutput == nil {
                     return channel.close(mode: .output).map { result }
                 }
                 return channel.selectableEventLoop.makeSucceededFuture(result)
@@ -2476,17 +2476,17 @@ extension NIOPipeBootstrap: Sendable {}
 protocol NIOPipeBootstrapHooks {
     func makePipeChannel(
         eventLoop: SelectableEventLoop,
-        inputPipe: NIOFileHandle?,
-        outputPipe: NIOFileHandle?
+        input: SelectablePipeHandle?,
+        output: SelectablePipeHandle?
     ) throws -> PipeChannel
 }
 
 private struct DefaultNIOPipeBootstrapHooks: NIOPipeBootstrapHooks {
     func makePipeChannel(
         eventLoop: SelectableEventLoop,
-        inputPipe: NIOFileHandle?,
-        outputPipe: NIOFileHandle?
+        input: SelectablePipeHandle?,
+        output: SelectablePipeHandle?
     ) throws -> PipeChannel {
-        try PipeChannel(eventLoop: eventLoop, inputPipe: inputPipe, outputPipe: outputPipe)
+        try PipeChannel(eventLoop: eventLoop, input: input, output: output)
     }
 }
diff --git a/Sources/NIOPosix/NonBlockingFileIO.swift b/Sources/NIOPosix/NonBlockingFileIO.swift
index e1bb2eb113..fe3448d11a 100644
--- a/Sources/NIOPosix/NonBlockingFileIO.swift
+++ b/Sources/NIOPosix/NonBlockingFileIO.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -17,6 +17,10 @@ import NIOCore
 
 /// ``NonBlockingFileIO`` is a helper that allows you to read files without blocking the calling thread.
 ///
+/// - warning: The `NonBlockingFileIO` API is deprecated, do not use going forward. It's not marked as `deprecated` yet such
+///            that users don't get the deprecation warnings affecting their APIs everywhere. For file I/O, please use
+///            the `NIOFileSystem` API.
+///
 /// It is worth noting that `kqueue`, `epoll` or `poll` returning claiming a file is readable does not mean that the
 /// data is already available in the kernel's memory. In other words, a `read` from a file can still block even if
 /// reported as readable. This behaviour is also documented behaviour:
@@ -553,9 +557,33 @@ public struct NonBlockingFileIO: Sendable {
     ///   - path: The path of the file to be opened for reading.
     ///   - eventLoop: The `EventLoop` on which the returned `EventLoopFuture` will fire.
     /// - Returns: An `EventLoopFuture` containing the `NIOFileHandle` and the `FileRegion` comprising the whole file.
+    @available(
+        *,
+        deprecated,
+        message:
+            "Avoid using NIOFileHandle. The type is difficult to hold correctly, use NIOFileSystem as a replacement API."
+    )
     public func openFile(path: String, eventLoop: EventLoop) -> EventLoopFuture<(NIOFileHandle, FileRegion)> {
+        self.openFile(_deprecatedPath: path, eventLoop: eventLoop)
+    }
+
+    /// Open the file at `path` for reading on a private thread pool which is separate from any `EventLoop` thread.
+    ///
+    /// This function will return (a future) of the `NIOFileHandle` associated with the file opened and a `FileRegion`
+    /// comprising of the whole file. The caller must close the returned `NIOFileHandle` when it's no longer needed.
+    ///
+    /// - Note: The reason this returns the `NIOFileHandle` and the `FileRegion` is that both the opening of a file as well as the querying of its size are blocking.
+    ///
+    /// - Parameters:
+    ///   - path: The path of the file to be opened for reading.
+    ///   - eventLoop: The `EventLoop` on which the returned `EventLoopFuture` will fire.
+    /// - Returns: An `EventLoopFuture` containing the `NIOFileHandle` and the `FileRegion` comprising the whole file.
+    public func openFile(
+        _deprecatedPath path: String,
+        eventLoop: EventLoop
+    ) -> EventLoopFuture<(NIOFileHandle, FileRegion)> {
         self.threadPool.runIfActive(eventLoop: eventLoop) {
-            let fh = try NIOFileHandle(path: path)
+            let fh = try NIOFileHandle(_deprecatedPath: path)
             do {
                 let fr = try FileRegion(fileHandle: fh)
                 return (fh, fr)
@@ -577,14 +605,40 @@ public struct NonBlockingFileIO: Sendable {
     ///   - flags: Additional POSIX flags.
     ///   - eventLoop: The `EventLoop` on which the returned `EventLoopFuture` will fire.
     /// - Returns: An `EventLoopFuture` containing the `NIOFileHandle`.
+    @available(
+        *,
+        deprecated,
+        message:
+            "Avoid using NonBlockingFileIO. The type is difficult to hold correctly, use NIOFileSystem as a replacement API."
+    )
     public func openFile(
         path: String,
         mode: NIOFileHandle.Mode,
         flags: NIOFileHandle.Flags = .default,
         eventLoop: EventLoop
+    ) -> EventLoopFuture<NIOFileHandle> {
+        self.openFile(_deprecatedPath: path, mode: mode, flags: flags, eventLoop: eventLoop)
+    }
+
+    /// Open the file at `path` with specified access mode and POSIX flags on a private thread pool which is separate from any `EventLoop` thread.
+    ///
+    /// This function will return (a future) of the `NIOFileHandle` associated with the file opened.
+    /// The caller must close the returned `NIOFileHandle` when it's no longer needed.
+    ///
+    /// - Parameters:
+    ///   - path: The path of the file to be opened for writing.
+    ///   - mode: File access mode.
+    ///   - flags: Additional POSIX flags.
+    ///   - eventLoop: The `EventLoop` on which the returned `EventLoopFuture` will fire.
+    /// - Returns: An `EventLoopFuture` containing the `NIOFileHandle`.
+    public func openFile(
+        _deprecatedPath path: String,
+        mode: NIOFileHandle.Mode,
+        flags: NIOFileHandle.Flags = .default,
+        eventLoop: EventLoop
     ) -> EventLoopFuture<NIOFileHandle> {
         self.threadPool.runIfActive(eventLoop: eventLoop) {
-            try NIOFileHandle(path: path, mode: mode, flags: flags)
+            try NIOFileHandle(_deprecatedPath: path, mode: mode, flags: flags)
         }
     }
 
@@ -1009,16 +1063,29 @@ extension NonBlockingFileIO {
     ///   - path: The path of the file to be opened for reading.
     ///   - body: operation to run with file handle and region
     /// - Returns: return value of operation
+    @available(
+        *,
+        deprecated,
+        message:
+            "Avoid using NonBlockingFileIO. The API is difficult to hold correctly, use NIOFileSystem as a replacement API."
+    )
     @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
     public func withFileRegion<Result>(
         path: String,
         _ body: (_ fileRegion: FileRegion) async throws -> Result
+    ) async throws -> Result {
+        try await self.withFileRegion(_deprecatedPath: path, body)
+    }
+
+    @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
+    public func withFileRegion<Result>(
+        _deprecatedPath path: String,
+        _ body: (_ fileRegion: FileRegion) async throws -> Result
     ) async throws -> Result {
         let fileRegion = try await self.threadPool.runIfActive {
-            let fh = try NIOFileHandle(path: path)
+            let fh = try NIOFileHandle(_deprecatedPath: path)
             do {
-                let fr = try FileRegion(fileHandle: fh)
-                return UnsafeTransfer(fr)
+                return try FileRegion(fileHandle: fh)
             } catch {
                 _ = try? fh.close()
                 throw error
@@ -1026,12 +1093,12 @@ extension NonBlockingFileIO {
         }
         let result: Result
         do {
-            result = try await body(fileRegion.wrappedValue)
+            result = try await body(fileRegion)
         } catch {
-            try fileRegion.wrappedValue.fileHandle.close()
+            try fileRegion.fileHandle.close()
             throw error
         }
-        try fileRegion.wrappedValue.fileHandle.close()
+        try fileRegion.fileHandle.close()
         return result
     }
 
@@ -1045,24 +1112,40 @@ extension NonBlockingFileIO {
     ///   - flags: Additional POSIX flags.
     ///   - body: operation to run with the file handle
     /// - Returns: return value of operation
+    @available(
+        *,
+        deprecated,
+        message:
+            "Avoid using NonBlockingFileIO. The API is difficult to hold correctly, use NIOFileSystem as a replacement API."
+    )
     @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
     public func withFileHandle<Result>(
         path: String,
         mode: NIOFileHandle.Mode,
         flags: NIOFileHandle.Flags = .default,
         _ body: (NIOFileHandle) async throws -> Result
+    ) async throws -> Result {
+        try await self.withFileHandle(_deprecatedPath: path, mode: mode, flags: flags, body)
+    }
+
+    @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
+    public func withFileHandle<Result>(
+        _deprecatedPath path: String,
+        mode: NIOFileHandle.Mode,
+        flags: NIOFileHandle.Flags = .default,
+        _ body: (NIOFileHandle) async throws -> Result
     ) async throws -> Result {
         let fileHandle = try await self.threadPool.runIfActive {
-            try UnsafeTransfer(NIOFileHandle(path: path, mode: mode, flags: flags))
+            try NIOFileHandle(_deprecatedPath: path, mode: mode, flags: flags)
         }
         let result: Result
         do {
-            result = try await body(fileHandle.wrappedValue)
+            result = try await body(fileHandle)
         } catch {
-            try fileHandle.wrappedValue.close()
+            try fileHandle.close()
             throw error
         }
-        try fileHandle.wrappedValue.close()
+        try fileHandle.close()
         return result
     }
 
diff --git a/Sources/NIOPosix/PendingWritesManager.swift b/Sources/NIOPosix/PendingWritesManager.swift
index 83323f480f..001b05c215 100644
--- a/Sources/NIOPosix/PendingWritesManager.swift
+++ b/Sources/NIOPosix/PendingWritesManager.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -434,9 +434,10 @@ final class PendingStreamWritesManager: PendingWritesManager {
         case .fileRegion(let file):
             let readerIndex = file.readerIndex
             let endIndex = file.endIndex
-            return try file.fileHandle.withUnsafeFileDescriptor { fd in
-                self.didWrite(itemCount: 1, result: try operation(fd, readerIndex, endIndex))
+            let writeResult = try file.fileHandle.withUnsafeFileDescriptor { fd in
+                try operation(fd, readerIndex, endIndex)
             }
+            return self.didWrite(itemCount: 1, result: writeResult)
         case .byteBuffer:
             preconditionFailure("called \(#function) but first item to write was a ByteBuffer")
         }
diff --git a/Sources/NIOPosix/PipeChannel.swift b/Sources/NIOPosix/PipeChannel.swift
index 1a85a30359..e105a724f1 100644
--- a/Sources/NIOPosix/PipeChannel.swift
+++ b/Sources/NIOPosix/PipeChannel.swift
@@ -23,10 +23,10 @@ final class PipeChannel: BaseStreamSocketChannel<PipePair> {
 
     init(
         eventLoop: SelectableEventLoop,
-        inputPipe: NIOFileHandle?,
-        outputPipe: NIOFileHandle?
+        input: SelectablePipeHandle?,
+        output: SelectablePipeHandle?
     ) throws {
-        self.pipePair = try PipePair(inputFD: inputPipe, outputFD: outputPipe)
+        self.pipePair = try PipePair(input: input, output: output)
         try super.init(
             socket: self.pipePair,
             parent: nil,
@@ -65,17 +65,17 @@ final class PipeChannel: BaseStreamSocketChannel<PipePair> {
     }
 
     override func register(selector: Selector<NIORegistration>, interested: SelectorEventSet) throws {
-        if let inputFD = self.pipePair.inputFD {
+        if let inputSPH = self.pipePair.input {
             try selector.register(
-                selectable: inputFD,
+                selectable: inputSPH,
                 interested: interested.intersection([.read, .reset, .error]),
                 makeRegistration: self.registrationForInput
             )
         }
 
-        if let outputFD = self.pipePair.outputFD {
+        if let outputSPH = self.pipePair.output {
             try selector.register(
-                selectable: outputFD,
+                selectable: outputSPH,
                 interested: interested.intersection([.write, .reset, .error]),
                 makeRegistration: self.registrationForOutput
             )
@@ -83,24 +83,24 @@ final class PipeChannel: BaseStreamSocketChannel<PipePair> {
     }
 
     override func deregister(selector: Selector<NIORegistration>, mode: CloseMode) throws {
-        if let inputFD = self.pipePair.inputFD, (mode == .all || mode == .input) && inputFD.isOpen {
-            try selector.deregister(selectable: inputFD)
+        if let inputSPH = self.pipePair.input, (mode == .all || mode == .input) && inputSPH.isOpen {
+            try selector.deregister(selectable: inputSPH)
         }
-        if let outputFD = self.pipePair.outputFD, (mode == .all || mode == .output) && outputFD.isOpen {
-            try selector.deregister(selectable: outputFD)
+        if let outputSPH = self.pipePair.output, (mode == .all || mode == .output) && outputSPH.isOpen {
+            try selector.deregister(selectable: outputSPH)
         }
     }
 
     override func reregister(selector: Selector<NIORegistration>, interested: SelectorEventSet) throws {
-        if let inputFD = self.pipePair.inputFD, inputFD.isOpen {
+        if let inputSPH = self.pipePair.input, inputSPH.isOpen {
             try selector.reregister(
-                selectable: inputFD,
+                selectable: inputSPH,
                 interested: interested.intersection([.read, .reset, .error])
             )
         }
-        if let outputFD = self.pipePair.outputFD, outputFD.isOpen {
+        if let outputSPH = self.pipePair.output, outputSPH.isOpen {
             try selector.reregister(
-                selectable: outputFD,
+                selectable: outputSPH,
                 interested: interested.intersection([.write, .reset, .error])
             )
         }
@@ -108,19 +108,19 @@ final class PipeChannel: BaseStreamSocketChannel<PipePair> {
 
     override func readEOF() {
         super.readEOF()
-        guard let inputFD = self.pipePair.inputFD, inputFD.isOpen else {
+        guard let inputSPH = self.pipePair.input, inputSPH.isOpen else {
             return
         }
         try! self.selectableEventLoop.deregister(channel: self, mode: .input)
-        try! inputFD.close()
+        try! inputSPH.close()
     }
 
     override func writeEOF() {
-        guard let outputFD = self.pipePair.outputFD, outputFD.isOpen else {
+        guard let outputSPH = self.pipePair.output, outputSPH.isOpen else {
             return
         }
         try! self.selectableEventLoop.deregister(channel: self, mode: .output)
-        try! outputFD.close()
+        try! outputSPH.close()
     }
 
     override func shutdownSocket(mode: CloseMode) throws {
diff --git a/Sources/NIOPosix/PipePair.swift b/Sources/NIOPosix/PipePair.swift
index 76b58c2f25..ffa76a2617 100644
--- a/Sources/NIOPosix/PipePair.swift
+++ b/Sources/NIOPosix/PipePair.swift
@@ -13,47 +13,72 @@
 //===----------------------------------------------------------------------===//
 import NIOCore
 
-struct SelectableFileHandle {
-    var handle: NIOFileHandle
+final class SelectablePipeHandle {
+    var fileDescriptor: CInt
 
     var isOpen: Bool {
-        handle.isOpen
+        self.fileDescriptor >= 0
     }
 
-    init(_ handle: NIOFileHandle) {
-        self.handle = handle
+    init(takingOwnershipOfDescriptor fd: CInt) {
+        precondition(fd >= 0)
+        self.fileDescriptor = fd
     }
 
     func close() throws {
-        try handle.close()
+        let fd = try self.takeDescriptorOwnership()
+        try Posix.close(descriptor: fd)
+    }
+
+    func takeDescriptorOwnership() throws -> CInt {
+        guard self.isOpen else {
+            throw IOError(errnoCode: EBADF, reason: "SelectablePipeHandle already closed [in close]")
+        }
+        defer {
+            self.fileDescriptor = -1
+        }
+        return self.fileDescriptor
+    }
+
+    deinit {
+        assert(!self.isOpen, "leaking \(self)")
     }
 }
 
-extension SelectableFileHandle: Selectable {
+extension SelectablePipeHandle: Selectable {
     func withUnsafeHandle<T>(_ body: (CInt) throws -> T) throws -> T {
-        try self.handle.withUnsafeFileDescriptor(body)
+        guard self.isOpen else {
+            throw IOError(errnoCode: EBADF, reason: "SelectablePipeHandle already closed [in wUH]")
+        }
+        return try body(self.fileDescriptor)
+    }
+}
+
+extension SelectablePipeHandle: CustomStringConvertible {
+    public var description: String {
+        "SelectableFileHandle(isOpen: \(self.isOpen), fd: \(self.fileDescriptor))"
     }
 }
 
 final class PipePair: SocketProtocol {
-    typealias SelectableType = SelectableFileHandle
+    typealias SelectableType = SelectablePipeHandle
 
-    let inputFD: SelectableFileHandle?
-    let outputFD: SelectableFileHandle?
+    let input: SelectablePipeHandle?
+    let output: SelectablePipeHandle?
 
-    init(inputFD: NIOFileHandle?, outputFD: NIOFileHandle?) throws {
-        self.inputFD = inputFD.flatMap { SelectableFileHandle($0) }
-        self.outputFD = outputFD.flatMap { SelectableFileHandle($0) }
+    init(input: SelectablePipeHandle?, output: SelectablePipeHandle?) throws {
+        self.input = input
+        self.output = output
         try self.ignoreSIGPIPE()
-        for fileHandle in [inputFD, outputFD].compactMap({ $0 }) {
-            try fileHandle.withUnsafeFileDescriptor {
-                try NIOFileHandle.setNonBlocking(fileDescriptor: $0)
+        for fh in [input, output].compactMap({ $0 }) {
+            try fh.withUnsafeHandle { fd in
+                try NIOFileHandle.setNonBlocking(fileDescriptor: fd)
             }
         }
     }
 
     func ignoreSIGPIPE() throws {
-        for fileHandle in [self.inputFD, self.outputFD].compactMap({ $0 }) {
+        for fileHandle in [self.input, self.output].compactMap({ $0 }) {
             try fileHandle.withUnsafeHandle {
                 try PipePair.ignoreSIGPIPE(descriptor: $0)
             }
@@ -61,7 +86,7 @@ final class PipePair: SocketProtocol {
     }
 
     var description: String {
-        "PipePair { in=\(String(describing: inputFD)), out=\(String(describing: inputFD)) }"
+        "PipePair { in=\(String(describing: self.input)), out=\(String(describing: self.output)) }"
     }
 
     func connect(to address: SocketAddress) throws -> Bool {
@@ -73,28 +98,28 @@ final class PipePair: SocketProtocol {
     }
 
     func write(pointer: UnsafeRawBufferPointer) throws -> IOResult<Int> {
-        guard let outputFD = self.outputFD else {
-            fatalError("Internal inconsistency inside NIO. Please file a bug")
+        guard let outputSPH = self.output else {
+            fatalError("Internal inconsistency inside NIO: outputSPH closed on write. Please file a bug")
         }
-        return try outputFD.withUnsafeHandle {
+        return try outputSPH.withUnsafeHandle {
             try Posix.write(descriptor: $0, pointer: pointer.baseAddress!, size: pointer.count)
         }
     }
 
     func writev(iovecs: UnsafeBufferPointer<IOVector>) throws -> IOResult<Int> {
-        guard let outputFD = self.outputFD else {
-            fatalError("Internal inconsistency inside NIO. Please file a bug")
+        guard let outputSPH = self.output else {
+            fatalError("Internal inconsistency inside NIO: outputSPH closed on writev. Please file a bug")
         }
-        return try outputFD.withUnsafeHandle {
+        return try outputSPH.withUnsafeHandle {
             try Posix.writev(descriptor: $0, iovecs: iovecs)
         }
     }
 
     func read(pointer: UnsafeMutableRawBufferPointer) throws -> IOResult<Int> {
-        guard let inputFD = self.inputFD else {
-            fatalError("Internal inconsistency inside NIO. Please file a bug")
+        guard let inputSPH = self.input else {
+            fatalError("Internal inconsistency inside NIO: inputSPH closed on read. Please file a bug")
         }
-        return try inputFD.withUnsafeHandle {
+        return try inputSPH.withUnsafeHandle {
             try Posix.read(descriptor: $0, pointer: pointer.baseAddress!, size: pointer.count)
         }
     }
@@ -132,16 +157,16 @@ final class PipePair: SocketProtocol {
     func shutdown(how: Shutdown) throws {
         switch how {
         case .RD:
-            try self.inputFD?.close()
+            try self.input?.close()
         case .WR:
-            try self.outputFD?.close()
+            try self.output?.close()
         case .RDWR:
             try self.close()
         }
     }
 
     var isOpen: Bool {
-        self.inputFD?.isOpen ?? false || self.outputFD?.isOpen ?? false
+        self.input?.isOpen ?? false || self.output?.isOpen ?? false
     }
 
     func close() throws {
@@ -149,12 +174,12 @@ final class PipePair: SocketProtocol {
             throw ChannelError._alreadyClosed
         }
         let r1 = Result {
-            if let inputFD = self.inputFD, inputFD.isOpen {
+            if let inputFD = self.input, inputFD.isOpen {
                 try inputFD.close()
             }
         }
         let r2 = Result {
-            if let outputFD = self.outputFD, outputFD.isOpen {
+            if let outputFD = self.output, outputFD.isOpen {
                 try outputFD.close()
             }
         }
diff --git a/Sources/NIOPosix/SelectorEpoll.swift b/Sources/NIOPosix/SelectorEpoll.swift
index 5da3978b8c..832dc41dff 100644
--- a/Sources/NIOPosix/SelectorEpoll.swift
+++ b/Sources/NIOPosix/SelectorEpoll.swift
@@ -167,8 +167,8 @@ extension Selector: _SelectorBackendProtocol {
         assert(self.timerFD == -1, "self.timerFD == \(self.timerFD) in deinitAssertions0, forgot close?")
     }
 
-    func register0<S: Selectable>(
-        selectable: S,
+    func register0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         interested: SelectorEventSet,
         registrationID: SelectorRegistrationID
@@ -180,8 +180,8 @@ extension Selector: _SelectorBackendProtocol {
         try Epoll.epoll_ctl(epfd: self.selectorFD, op: Epoll.EPOLL_CTL_ADD, fd: fileDescriptor, event: &ev)
     }
 
-    func reregister0<S: Selectable>(
-        selectable: S,
+    func reregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         newInterested: SelectorEventSet,
@@ -194,8 +194,8 @@ extension Selector: _SelectorBackendProtocol {
         _ = try Epoll.epoll_ctl(epfd: self.selectorFD, op: Epoll.EPOLL_CTL_MOD, fd: fileDescriptor, event: &ev)
     }
 
-    func deregister0<S: Selectable>(
-        selectable: S,
+    func deregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         registrationID: SelectorRegistrationID
diff --git a/Sources/NIOPosix/SelectorGeneric.swift b/Sources/NIOPosix/SelectorGeneric.swift
index a587079e6b..cb2626c76c 100644
--- a/Sources/NIOPosix/SelectorGeneric.swift
+++ b/Sources/NIOPosix/SelectorGeneric.swift
@@ -122,27 +122,29 @@ protocol _SelectorBackendProtocol {
     associatedtype R: Registration
     func initialiseState0() throws
     func deinitAssertions0()  // allows actual implementation to run some assertions as part of the class deinit
-    func register0<S: Selectable>(
-        selectable: S,
+    func register0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         interested: SelectorEventSet,
         registrationID: SelectorRegistrationID
     ) throws
-    func reregister0<S: Selectable>(
-        selectable: S,
+    func reregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         newInterested: SelectorEventSet,
         registrationID: SelectorRegistrationID
     ) throws
-    func deregister0<S: Selectable>(
-        selectable: S,
+    func deregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         registrationID: SelectorRegistrationID
     ) throws
+
     // attention, this may (will!) be called from outside the event loop, ie. can't access mutable shared state (such as `self.open`)
     func wakeup0() throws
+
     /// Apply the given `SelectorStrategy` and execute `body` once it's complete (which may produce `SelectorEvent`s to handle).
     ///
     /// - Parameters:
@@ -288,7 +290,7 @@ internal class Selector<R: Registration> {
         try selectable.withUnsafeHandle { fd in
             assert(registrations[Int(fd)] == nil)
             try self.register0(
-                selectable: selectable,
+                selectableFD: fd,
                 fileDescriptor: fd,
                 interested: interested,
                 registrationID: self.registrationID
@@ -315,7 +317,7 @@ internal class Selector<R: Registration> {
         try selectable.withUnsafeHandle { fd in
             var reg = registrations[Int(fd)]!
             try self.reregister0(
-                selectable: selectable,
+                selectableFD: fd,
                 fileDescriptor: fd,
                 oldInterested: reg.interested,
                 newInterested: interested,
@@ -343,7 +345,7 @@ internal class Selector<R: Registration> {
                 return
             }
             try self.deregister0(
-                selectable: selectable,
+                selectableFD: fd,
                 fileDescriptor: fd,
                 oldInterested: reg.interested,
                 registrationID: reg.registrationID
diff --git a/Sources/NIOPosix/SelectorKqueue.swift b/Sources/NIOPosix/SelectorKqueue.swift
index 2b2b31edac..4c45dd1599 100644
--- a/Sources/NIOPosix/SelectorKqueue.swift
+++ b/Sources/NIOPosix/SelectorKqueue.swift
@@ -163,8 +163,8 @@ extension Selector: _SelectorBackendProtocol {
         }
     }
 
-    private func kqueueUpdateEventNotifications<S: Selectable>(
-        selectable: S,
+    private func kqueueUpdateEventNotifications(
+        selectableFD: CInt,
         interested: SelectorEventSet,
         oldInterested: SelectorEventSet?,
         registrationID: SelectorRegistrationID
@@ -175,14 +175,12 @@ extension Selector: _SelectorBackendProtocol {
         assert(interested.contains(.reset))
         assert(oldInterested?.contains(.reset) ?? true)
 
-        try selectable.withUnsafeHandle {
-            try newKQueueFilters.calculateKQueueFilterSetChanges(
-                previousKQueueFilterSet: oldKQueueFilters,
-                fileDescriptor: $0,
-                registrationID: registrationID,
-                kqueueApplyEventChangeSet
-            )
-        }
+        try newKQueueFilters.calculateKQueueFilterSetChanges(
+            previousKQueueFilterSet: oldKQueueFilters,
+            fileDescriptor: selectableFD,
+            registrationID: registrationID,
+            kqueueApplyEventChangeSet
+        )
     }
 
     func initialiseState0() throws {
@@ -205,44 +203,44 @@ extension Selector: _SelectorBackendProtocol {
     func deinitAssertions0() {
     }
 
-    func register0<S: Selectable>(
-        selectable: S,
+    func register0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         interested: SelectorEventSet,
         registrationID: SelectorRegistrationID
     ) throws {
         try kqueueUpdateEventNotifications(
-            selectable: selectable,
+            selectableFD: selectableFD,
             interested: interested,
             oldInterested: nil,
             registrationID: registrationID
         )
     }
 
-    func reregister0<S: Selectable>(
-        selectable: S,
+    func reregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         newInterested: SelectorEventSet,
         registrationID: SelectorRegistrationID
     ) throws {
         try kqueueUpdateEventNotifications(
-            selectable: selectable,
+            selectableFD: selectableFD,
             interested: newInterested,
             oldInterested: oldInterested,
             registrationID: registrationID
         )
     }
 
-    func deregister0<S: Selectable>(
-        selectable: S,
+    func deregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         registrationID: SelectorRegistrationID
     ) throws {
         try kqueueUpdateEventNotifications(
-            selectable: selectable,
-            interested: [.reset, .error],
+            selectableFD: selectableFD,
+            interested: .reset,
             oldInterested: oldInterested,
             registrationID: registrationID
         )
diff --git a/Sources/NIOPosix/SelectorUring.swift b/Sources/NIOPosix/SelectorUring.swift
index 2bc9fdb379..a92608b2dd 100644
--- a/Sources/NIOPosix/SelectorUring.swift
+++ b/Sources/NIOPosix/SelectorUring.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2021-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -131,8 +131,8 @@ extension Selector: _SelectorBackendProtocol {
         assert(self.eventFD == -1, "self.eventFD == \(self.eventFD) on deinitAssertions0 deinit, forgot close?")
     }
 
-    func register0<S: Selectable>(
-        selectable: S,
+    func register0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         interested: SelectorEventSet,
         registrationID: SelectorRegistrationID
@@ -150,8 +150,8 @@ extension Selector: _SelectorBackendProtocol {
         )
     }
 
-    func reregister0<S: Selectable>(
-        selectable: S,
+    func reregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         newInterested: SelectorEventSet,
@@ -190,8 +190,8 @@ extension Selector: _SelectorBackendProtocol {
         }
     }
 
-    func deregister0<S: Selectable>(
-        selectable: S,
+    func deregister0(
+        selectableFD: CInt,
         fileDescriptor: CInt,
         oldInterested: SelectorEventSet,
         registrationID: SelectorRegistrationID
diff --git a/Tests/NIOCoreTests/BaseObjectsTest.swift b/Tests/NIOCoreTests/BaseObjectsTest.swift
index 74f8a91563..cf541487bf 100644
--- a/Tests/NIOCoreTests/BaseObjectsTest.swift
+++ b/Tests/NIOCoreTests/BaseObjectsTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -52,7 +52,7 @@ class BaseObjectTest: XCTestCase {
     }
 
     func testNIOFileRegionConversion() {
-        let handle = NIOFileHandle(descriptor: -1)
+        let handle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
         let expected = FileRegion(fileHandle: handle, readerIndex: 1, endIndex: 2)
         defer {
             // fake descriptor, so shouldn't be closed.
@@ -74,7 +74,7 @@ class BaseObjectTest: XCTestCase {
     }
 
     func testBadConversions() {
-        let handle = NIOFileHandle(descriptor: -1)
+        let handle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
         let bb = ByteBufferAllocator().buffer(capacity: 1024)
         let fr = FileRegion(fileHandle: handle, readerIndex: 1, endIndex: 2)
         defer {
@@ -95,7 +95,7 @@ class BaseObjectTest: XCTestCase {
     }
 
     func testFileRegionFromIOData() {
-        let handle = NIOFileHandle(descriptor: -1)
+        let handle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
         let expected = FileRegion(fileHandle: handle, readerIndex: 1, endIndex: 2)
         defer {
             // fake descriptor, so shouldn't be closed.
@@ -106,7 +106,7 @@ class BaseObjectTest: XCTestCase {
     }
 
     func testIODataEquals() {
-        let handle = NIOFileHandle(descriptor: -1)
+        let handle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
         var bb1 = ByteBufferAllocator().buffer(capacity: 1024)
         let bb2 = ByteBufferAllocator().buffer(capacity: 1024)
         bb1.writeString("hello")
diff --git a/Tests/NIOCoreTests/NIOAnyDebugTest.swift b/Tests/NIOCoreTests/NIOAnyDebugTest.swift
index 8d9b6f4aa3..3aa49362e2 100644
--- a/Tests/NIOCoreTests/NIOAnyDebugTest.swift
+++ b/Tests/NIOCoreTests/NIOAnyDebugTest.swift
@@ -27,7 +27,7 @@ class NIOAnyDebugTest: XCTestCase {
             "ByteBuffer: [627974652062756666657220737472696e67](18 bytes)"
         )
 
-        let fileHandle = NIOFileHandle(descriptor: 1)
+        let fileHandle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: 1)
         defer {
             XCTAssertNoThrow(_ = try fileHandle.takeDescriptorOwnership())
         }
diff --git a/Tests/NIOCoreTests/XCTest+Extensions.swift b/Tests/NIOCoreTests/XCTest+Extensions.swift
index 0ae75cdb49..0fc5c8b7d4 100644
--- a/Tests/NIOCoreTests/XCTest+Extensions.swift
+++ b/Tests/NIOCoreTests/XCTest+Extensions.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -62,7 +62,7 @@ func withTemporaryFile<T>(content: String? = nil, _ body: (NIOCore.NIOFileHandle
         XCTAssertNoThrow(try FileManager.default.removeItem(atPath: temporaryFilePath))
     }
 
-    let fileHandle = try NIOFileHandle(path: temporaryFilePath, mode: [.read, .write])
+    let fileHandle = try NIOFileHandle(_deprecatedPath: temporaryFilePath, mode: [.read, .write])
     defer {
         XCTAssertNoThrow(try fileHandle.close())
     }
diff --git a/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift b/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift
index ce5e150077..56c09b67f8 100644
--- a/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift
+++ b/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift
@@ -232,7 +232,7 @@ class EmbeddedChannelTest: XCTestCase {
 
         let buffer = channel.allocator.buffer(capacity: 0)
         let ioData = IOData.byteBuffer(buffer)
-        let fileHandle = NIOFileHandle(descriptor: -1)
+        let fileHandle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
         let fileRegion = FileRegion(fileHandle: fileHandle, readerIndex: 0, endIndex: 0)
         defer {
             XCTAssertNoThrow(_ = try fileHandle.takeDescriptorOwnership())
@@ -387,7 +387,7 @@ class EmbeddedChannelTest: XCTestCase {
         let channel = EmbeddedChannel()
         let buffer = ByteBufferAllocator().buffer(capacity: 5)
         let socketAddress = try SocketAddress(unixDomainSocketPath: "path")
-        let handle = NIOFileHandle(descriptor: 1)
+        let handle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: 1)
         let fileRegion = FileRegion(fileHandle: handle, readerIndex: 1, endIndex: 2)
         defer {
             // fake descriptor, so shouldn't be closed.
diff --git a/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift b/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift
index a9f56d4adb..28a42923a7 100644
--- a/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift
+++ b/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -98,7 +98,7 @@ class HTTPServerClientTest: XCTestCase {
 
                 let content = buffer.getData(at: 0, length: buffer.readableBytes)!
                 XCTAssertNoThrow(try content.write(to: URL(fileURLWithPath: filePath)))
-                let fh = try! NIOFileHandle(path: filePath)
+                let fh = try! NIOFileHandle(_deprecatedPath: filePath)
                 let region = FileRegion(
                     fileHandle: fh,
                     readerIndex: 0,
diff --git a/Tests/NIOPosixTests/BootstrapTest.swift b/Tests/NIOPosixTests/BootstrapTest.swift
index cd75b45a36..8c64a578e3 100644
--- a/Tests/NIOPosixTests/BootstrapTest.swift
+++ b/Tests/NIOPosixTests/BootstrapTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -451,26 +451,25 @@ class BootstrapTest: XCTestCase {
 
         let eventLoop = self.group.next()
 
-        eventLoop.execute {
-            do {
+        XCTAssertNoThrow(
+            try eventLoop.submit {
                 let pipe = Pipe()
-                let readHandle = NIOFileHandle(descriptor: pipe.fileHandleForReading.fileDescriptor)
-                let writeHandle = NIOFileHandle(descriptor: pipe.fileHandleForWriting.fileDescriptor)
-                _ = NIOPipeBootstrap(group: self.group)
+                defer {
+                    XCTAssertNoThrow(try pipe.fileHandleForReading.close())
+                    XCTAssertNoThrow(try pipe.fileHandleForWriting.close())
+                }
+                return NIOPipeBootstrap(group: self.group)
                     .takingOwnershipOfDescriptors(
-                        input: try readHandle.takeDescriptorOwnership(),
-                        output: try writeHandle.takeDescriptorOwnership()
+                        input: dup(pipe.fileHandleForReading.fileDescriptor),
+                        output: dup(pipe.fileHandleForWriting.fileDescriptor)
                     )
                     .flatMap({ channel in
                         channel.close()
                     }).always({ _ in
                         testGrp.leave()
                     })
-            } catch {
-                XCTFail("Failed to bootstrap pipechannel in eventloop: \(error)")
-                testGrp.leave()
-            }
-        }
+            }.wait()
+        )
         testGrp.wait()
     }
 
@@ -777,9 +776,9 @@ class BootstrapTest: XCTestCase {
         struct NIOPipeBootstrapHooksChannelFail: NIOPipeBootstrapHooks {
             func makePipeChannel(
                 eventLoop: NIOPosix.SelectableEventLoop,
-                inputPipe: NIOCore.NIOFileHandle?,
-                outputPipe: NIOCore.NIOFileHandle?
-            ) throws -> NIOPosix.PipeChannel {
+                input: SelectablePipeHandle?,
+                output: SelectablePipeHandle?
+            ) throws -> PipeChannel {
                 throw IOError(errnoCode: EBADF, reason: "testing")
             }
         }
@@ -790,7 +789,7 @@ class BootstrapTest: XCTestCase {
         }
         let elg = MultiThreadedEventLoopGroup(numberOfThreads: 1)
         defer {
-            try! elg.syncShutdownGracefully()
+            XCTAssertNoThrow(try elg.syncShutdownGracefully())
         }
 
         let bootstrap = NIOPipeBootstrap(validatingGroup: elg, hooks: NIOPipeBootstrapHooksChannelFail())
diff --git a/Tests/NIOPosixTests/ChannelPipelineTest.swift b/Tests/NIOPosixTests/ChannelPipelineTest.swift
index e813328981..9226d7a0a6 100644
--- a/Tests/NIOPosixTests/ChannelPipelineTest.swift
+++ b/Tests/NIOPosixTests/ChannelPipelineTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -320,7 +320,7 @@ class ChannelPipelineTest: XCTestCase {
 
         XCTAssertTrue(loop.inEventLoop)
         do {
-            let handle = NIOFileHandle(descriptor: -1)
+            let handle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
             let fr = FileRegion(fileHandle: handle, readerIndex: 0, endIndex: 0)
             defer {
                 // fake descriptor, so shouldn't be closed.
diff --git a/Tests/NIOPosixTests/ChannelTests.swift b/Tests/NIOPosixTests/ChannelTests.swift
index a1df4bee11..8774c4f6a5 100644
--- a/Tests/NIOPosixTests/ChannelTests.swift
+++ b/Tests/NIOPosixTests/ChannelTests.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -776,7 +776,7 @@ public final class ChannelTests: XCTestCase {
             )
             buffer.clear()
             buffer.writeBytes([UInt8](repeating: 0xff, count: 1))
-            let handle = NIOFileHandle(descriptor: -1)
+            let handle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
             defer {
                 // fake file handle, so don't actually close
                 XCTAssertNoThrow(try handle.takeDescriptorOwnership())
@@ -962,8 +962,8 @@ public final class ChannelTests: XCTestCase {
         try withPendingStreamWritesManager { pwm in
             let ps: [EventLoopPromise<Void>] = (0..<2).map { (_: Int) in el.makePromise() }
 
-            let fh1 = NIOFileHandle(descriptor: -1)
-            let fh2 = NIOFileHandle(descriptor: -2)
+            let fh1 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
+            let fh2 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -2)
             let fr1 = FileRegion(fileHandle: fh1, readerIndex: 12, endIndex: 14)
             let fr2 = FileRegion(fileHandle: fh2, readerIndex: 0, endIndex: 2)
             defer {
@@ -1027,7 +1027,7 @@ public final class ChannelTests: XCTestCase {
         try withPendingStreamWritesManager { pwm in
             let ps: [EventLoopPromise<Void>] = (0..<1).map { (_: Int) in el.makePromise() }
 
-            let fh = NIOFileHandle(descriptor: -1)
+            let fh = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
             let fr = FileRegion(fileHandle: fh, readerIndex: 99, endIndex: 99)
             defer {
                 // fake descriptor, so shouldn't be closed.
@@ -1061,8 +1061,8 @@ public final class ChannelTests: XCTestCase {
         try withPendingStreamWritesManager { pwm in
             let ps: [EventLoopPromise<Void>] = (0..<5).map { (_: Int) in el.makePromise() }
 
-            let fh1 = NIOFileHandle(descriptor: -1)
-            let fh2 = NIOFileHandle(descriptor: -1)
+            let fh1 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
+            let fh2 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
             let fr1 = FileRegion(fileHandle: fh1, readerIndex: 99, endIndex: 99)
             let fr2 = FileRegion(fileHandle: fh1, readerIndex: 0, endIndex: 10)
             defer {
@@ -1320,7 +1320,7 @@ public final class ChannelTests: XCTestCase {
         try withPendingStreamWritesManager { pwm in
             let ps: [EventLoopPromise<Void>] = (0..<1).map { (_: Int) in el.makePromise() }
 
-            let fh = NIOFileHandle(descriptor: -1)
+            let fh = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: -1)
             let fr = FileRegion(fileHandle: fh, readerIndex: 0, endIndex: 8192)
             defer {
                 // fake descriptor, so shouldn't be closed.
diff --git a/Tests/NIOPosixTests/FileRegionTest.swift b/Tests/NIOPosixTests/FileRegionTest.swift
index 764ecfff39..4e06a86ee3 100644
--- a/Tests/NIOPosixTests/FileRegionTest.swift
+++ b/Tests/NIOPosixTests/FileRegionTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -60,7 +60,7 @@ class FileRegionTest: XCTestCase {
         try withTemporaryFile { _, filePath in
             try content.write(toFile: filePath, atomically: false, encoding: .ascii)
             try clientChannel.eventLoop.submit {
-                try NIOFileHandle(path: filePath)
+                try NIOFileHandle(_deprecatedPath: filePath)
             }.flatMap { (handle: NIOFileHandle) in
                 let fr = FileRegion(fileHandle: handle, readerIndex: 0, endIndex: bytes.count)
                 let promise = clientChannel.eventLoop.makePromise(of: Void.self)
@@ -118,7 +118,7 @@ class FileRegionTest: XCTestCase {
             try "".write(toFile: filePath, atomically: false, encoding: .ascii)
 
             try clientChannel.eventLoop.submit {
-                try NIOFileHandle(path: filePath)
+                try NIOFileHandle(_deprecatedPath: filePath)
             }.flatMap { (handle: NIOFileHandle) in
                 let fr = FileRegion(fileHandle: handle, readerIndex: 0, endIndex: 0)
                 var futures: [EventLoopFuture<Void>] = []
@@ -180,8 +180,8 @@ class FileRegionTest: XCTestCase {
             try content.write(toFile: filePath, atomically: false, encoding: .ascii)
 
             let future = clientChannel.eventLoop.submit {
-                let fh1 = try NIOFileHandle(path: filePath)
-                let fh2 = try NIOFileHandle(path: filePath)
+                let fh1 = try NIOFileHandle(_deprecatedPath: filePath)
+                let fh2 = try NIOFileHandle(_deprecatedPath: filePath)
                 return (fh1, fh2)
             }.flatMap { (fh1, fh2) in
                 let fr1 = FileRegion(fileHandle: fh1, readerIndex: 0, endIndex: bytes.count)
@@ -229,7 +229,7 @@ class FileRegionTest: XCTestCase {
 
     func testWholeFileFileRegion() throws {
         try withTemporaryFile(content: "hello") { fd, path in
-            let handle = try NIOFileHandle(path: path)
+            let handle = try NIOFileHandle(_deprecatedPath: path)
             let region = try FileRegion(fileHandle: handle)
             defer {
                 XCTAssertNoThrow(try handle.close())
@@ -242,7 +242,7 @@ class FileRegionTest: XCTestCase {
 
     func testWholeEmptyFileFileRegion() throws {
         try withTemporaryFile(content: "") { _, path in
-            let handle = try NIOFileHandle(path: path)
+            let handle = try NIOFileHandle(_deprecatedPath: path)
             let region = try FileRegion(fileHandle: handle)
             defer {
                 XCTAssertNoThrow(try handle.close())
diff --git a/Tests/NIOPosixTests/NIOFileHandleTest.swift b/Tests/NIOPosixTests/NIOFileHandleTest.swift
new file mode 100644
index 0000000000..6493914c52
--- /dev/null
+++ b/Tests/NIOPosixTests/NIOFileHandleTest.swift
@@ -0,0 +1,171 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftNIO open source project
+//
+// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Dispatch
+import NIOPosix
+import XCTest
+
+@testable import NIOCore
+
+final class NIOFileHandleTest: XCTestCase {
+    func testOpenCloseWorks() throws {
+        let pipeFDs = try Self.makePipe()
+        let fh1 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: pipeFDs.0)
+        XCTAssertTrue(fh1.isOpen)
+        defer {
+            XCTAssertTrue(fh1.isOpen)
+            XCTAssertNoThrow(try fh1.close())
+            XCTAssertFalse(fh1.isOpen)
+        }
+        let fh2 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: pipeFDs.1)
+        XCTAssertTrue(fh2.isOpen)
+        defer {
+            XCTAssertTrue(fh2.isOpen)
+            XCTAssertNoThrow(try fh2.close())
+            XCTAssertFalse(fh2.isOpen)
+        }
+        XCTAssertTrue(fh1.isOpen)
+        XCTAssertTrue(fh2.isOpen)
+    }
+
+    func testCloseStorm() throws {
+        for _ in 0..<1000 {
+            let pipeFDs = try Self.makePipe()
+            let fh1 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: pipeFDs.0)
+            let fh2 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: pipeFDs.1)
+
+            let threads = 32
+            let threadReadySems = (0..<threads).map { _ in
+                DispatchSemaphore(value: 0)
+            }
+            let threadGoSems = (0..<threads).map { _ in
+                DispatchSemaphore(value: 0)
+            }
+            let allDoneGroup = DispatchGroup()
+
+            for threadID in 0..<threads {
+                DispatchQueue.global().async(group: allDoneGroup) {
+                    threadReadySems[threadID].signal()
+                    threadGoSems[threadID].wait()
+
+                    do {
+                        if threadID % 2 == 0 {
+                            try fh1.close()
+                        } else {
+                            try fh2.close()
+                        }
+                    } catch let error as IOError where error.errnoCode == EBADF {
+                        // expected
+                    } catch {
+                        XCTFail("unexpected error \(error)")
+                    }
+                }
+            }
+
+            for threadReadySem in threadReadySems {
+                threadReadySem.wait()
+            }
+            for threadGoSem in threadGoSems {
+                threadGoSem.signal()
+            }
+            allDoneGroup.wait()
+            XCTAssertFalse(fh1.isOpen)
+            XCTAssertFalse(fh2.isOpen)
+        }
+    }
+
+    func testCloseVsUseRace() throws {
+        for _ in 0..<1000 {
+            let pipeFDs = try Self.makePipe()
+            let fh1 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: pipeFDs.0)
+            let fh2 = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: pipeFDs.1)
+
+            let threads = 32
+            let threadReadySems = (0..<threads).map { _ in
+                DispatchSemaphore(value: 0)
+            }
+            let threadGoSems = (0..<threads).map { _ in
+                DispatchSemaphore(value: 0)
+            }
+            let allDoneGroup = DispatchGroup()
+
+            for threadID in 0..<threads {
+                DispatchQueue.global().async(group: allDoneGroup) {
+                    threadReadySems[threadID].signal()
+                    threadGoSems[threadID].wait()
+
+                    do {
+                        switch threadID % 4 {
+                        case 0:
+                            try fh1.close()
+                        case 1:
+                            try fh2.close()
+                        case 2:
+                            try fh1.withUnsafeFileDescriptor { fd in
+                                precondition(fd >= 0)
+                                usleep(.random(in: 0..<10))
+                            }
+                        case 3:
+                            try fh2.withUnsafeFileDescriptor { fd in
+                                precondition(fd >= 0)
+                                usleep(.random(in: 0..<10))
+                            }
+                        default:
+                            fatalError("impossible")
+                        }
+                    } catch let error as IOError where error.errnoCode == EBADF || error.errnoCode == EBUSY {
+                        // expected
+                    } catch {
+                        XCTFail("unexpected error \(error)")
+                    }
+                }
+            }
+
+            for threadReadySem in threadReadySems {
+                threadReadySem.wait()
+            }
+            for threadGoSem in threadGoSems {
+                threadGoSem.signal()
+            }
+            allDoneGroup.wait()
+            for fh in [fh1, fh2] {
+                // They may or may not be closed, depends on races above.
+                do {
+                    try fh.close()
+                } catch let error as IOError where error.errnoCode == EBADF {
+                    // expected
+                }
+            }
+            XCTAssertFalse(fh1.isOpen)
+            XCTAssertFalse(fh2.isOpen)
+        }
+    }
+
+    // MARK: - Helpers
+    struct POSIXError: Error {
+        var what: String
+        var errnoCode: CInt
+    }
+
+    private static func makePipe() throws -> (CInt, CInt) {
+        var pipeFDs: [CInt] = [-1, -1]
+        let err = pipeFDs.withUnsafeMutableBufferPointer { pipePtr in
+            pipe(pipePtr.baseAddress!)
+        }
+        guard err == 0 else {
+            throw POSIXError(what: "pipe", errnoCode: errno)
+        }
+        return (pipeFDs[0], pipeFDs[1])
+    }
+}
diff --git a/Tests/NIOPosixTests/NonBlockingFileIOTest.swift b/Tests/NIOPosixTests/NonBlockingFileIOTest.swift
index 405f7e84b7..26ab80d614 100644
--- a/Tests/NIOPosixTests/NonBlockingFileIOTest.swift
+++ b/Tests/NIOPosixTests/NonBlockingFileIOTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -634,7 +634,7 @@ class NonBlockingFileIOTest: XCTestCase {
     func testFileOpenWorks() throws {
         let content = "123"
         try withTemporaryFile(content: content) { (fileHandle, path) -> Void in
-            try self.fileIO.openFile(path: path, eventLoop: self.eventLoop).flatMapThrowing { vals in
+            try self.fileIO.openFile(_deprecatedPath: path, eventLoop: self.eventLoop).flatMapThrowing { vals in
                 let (fh, fr) = vals
                 try fh.withUnsafeFileDescriptor { fd in
                     XCTAssertGreaterThanOrEqual(fd, 0)
@@ -650,7 +650,7 @@ class NonBlockingFileIOTest: XCTestCase {
     func testFileOpenWorksWithEmptyFile() throws {
         let content = ""
         try withTemporaryFile(content: content) { (fileHandle, path) -> Void in
-            try self.fileIO.openFile(path: path, eventLoop: self.eventLoop).flatMapThrowing { vals in
+            try self.fileIO.openFile(_deprecatedPath: path, eventLoop: self.eventLoop).flatMapThrowing { vals in
                 let (fh, fr) = vals
                 try fh.withUnsafeFileDescriptor { fd in
                     XCTAssertGreaterThanOrEqual(fd, 0)
@@ -666,7 +666,7 @@ class NonBlockingFileIOTest: XCTestCase {
     func testFileOpenFails() throws {
         do {
             try self.fileIO.openFile(
-                path: "/dev/null/this/does/not/exist",
+                _deprecatedPath: "/dev/null/this/does/not/exist",
                 eventLoop: self.eventLoop
             ).map { _ in }.wait()
             XCTFail("should've thrown")
@@ -681,7 +681,7 @@ class NonBlockingFileIOTest: XCTestCase {
         XCTAssertNoThrow(
             try withTemporaryDirectory { dir in
                 try self.fileIO!.openFile(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: .write,
                     flags: .allowFileCreation(),
                     eventLoop: self.eventLoop
@@ -694,7 +694,7 @@ class NonBlockingFileIOTest: XCTestCase {
         XCTAssertThrowsError(
             try withTemporaryDirectory { dir in
                 try self.fileIO!.openFile(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: .write,
                     flags: .default,
                     eventLoop: self.eventLoop
@@ -709,7 +709,7 @@ class NonBlockingFileIOTest: XCTestCase {
         XCTAssertNoThrow(
             try withTemporaryDirectory { dir in
                 let fileHandle = try self.fileIO!.openFile(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: .write,
                     flags: .allowFileCreation(),
                     eventLoop: self.eventLoop
@@ -734,7 +734,7 @@ class NonBlockingFileIOTest: XCTestCase {
         XCTAssertNoThrow(
             try withTemporaryDirectory { dir in
                 let fileHandle = try self.fileIO!.openFile(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: [.write, .read],
                     flags: .allowFileCreation(),
                     eventLoop: self.eventLoop
@@ -761,7 +761,7 @@ class NonBlockingFileIOTest: XCTestCase {
                 // open 1 + write
                 try {
                     let fileHandle = try self.fileIO!.openFile(
-                        path: "\(dir)/file",
+                        _deprecatedPath: "\(dir)/file",
                         mode: [.write, .read],
                         flags: .allowFileCreation(),
                         eventLoop: self.eventLoop
@@ -782,7 +782,7 @@ class NonBlockingFileIOTest: XCTestCase {
                 // open 2 + write again + read
                 try {
                     let fileHandle = try self.fileIO!.openFile(
-                        path: "\(dir)/file",
+                        _deprecatedPath: "\(dir)/file",
                         mode: [.write, .read],
                         flags: .default,
                         eventLoop: self.eventLoop
@@ -826,7 +826,7 @@ class NonBlockingFileIOTest: XCTestCase {
                 // open 1 + write
                 try {
                     let fileHandle = try self.fileIO!.openFile(
-                        path: "\(dir)/file",
+                        _deprecatedPath: "\(dir)/file",
                         mode: [.write, .read],
                         flags: .allowFileCreation(),
                         eventLoop: self.eventLoop
@@ -847,7 +847,7 @@ class NonBlockingFileIOTest: XCTestCase {
                 // open 2 (with truncation) + write again + read
                 try {
                     let fileHandle = try self.fileIO!.openFile(
-                        path: "\(dir)/file",
+                        _deprecatedPath: "\(dir)/file",
                         mode: [.write, .read],
                         flags: .posix(flags: O_TRUNC, mode: 0),
                         eventLoop: self.eventLoop
@@ -1076,7 +1076,7 @@ class NonBlockingFileIOTest: XCTestCase {
             let expectation = XCTestExpectation(description: "Opened file")
             let threadPool = NIOThreadPool(numberOfThreads: 1)
             let fileIO = NonBlockingFileIO(threadPool: threadPool)
-            fileIO.openFile(path: path, eventLoop: eventLoopGroup.next()).whenFailure { (error) in
+            fileIO.openFile(_deprecatedPath: path, eventLoop: eventLoopGroup.next()).whenFailure { (error) in
                 XCTAssertTrue(error is NIOThreadPoolError.ThreadPoolInactive)
                 expectation.fulfill()
             }
@@ -1166,7 +1166,7 @@ class NonBlockingFileIOTest: XCTestCase {
             try withTemporaryDirectory { path in
                 let file = "\(path)/file"
                 let handle = try self.fileIO.openFile(
-                    path: file,
+                    _deprecatedPath: file,
                     mode: .write,
                     flags: .allowFileCreation(),
                     eventLoop: self.eventLoop
@@ -1186,7 +1186,7 @@ class NonBlockingFileIOTest: XCTestCase {
             try withTemporaryDirectory { path in
                 let file = "\(path)/file"
                 let handle = try self.fileIO.openFile(
-                    path: file,
+                    _deprecatedPath: file,
                     mode: .write,
                     flags: .allowFileCreation(),
                     eventLoop: self.eventLoop
@@ -1216,7 +1216,7 @@ class NonBlockingFileIOTest: XCTestCase {
             try withTemporaryDirectory { path in
                 let file = "\(path)/file"
                 let handle = try self.fileIO.openFile(
-                    path: file,
+                    _deprecatedPath: file,
                     mode: .write,
                     flags: .allowFileCreation(),
                     eventLoop: self.eventLoop
@@ -1557,7 +1557,7 @@ extension NonBlockingFileIOTest {
     func testAsyncFileOpenWorks() async throws {
         let content = "123"
         try await withTemporaryFile(content: content) { (fileHandle, path) -> Void in
-            try await self.fileIO.withFileRegion(path: path) { fr in
+            try await self.fileIO.withFileRegion(_deprecatedPath: path) { fr in
                 try fr.fileHandle.withUnsafeFileDescriptor { fd in
                     XCTAssertGreaterThanOrEqual(fd, 0)
                 }
@@ -1571,7 +1571,7 @@ extension NonBlockingFileIOTest {
     func testAsyncFileOpenWorksWithEmptyFile() async throws {
         let content = ""
         try await withTemporaryFile(content: content) { (fileHandle, path) -> Void in
-            try await self.fileIO.withFileRegion(path: path) { fr in
+            try await self.fileIO.withFileRegion(_deprecatedPath: path) { fr in
                 try fr.fileHandle.withUnsafeFileDescriptor { fd in
                     XCTAssertGreaterThanOrEqual(fd, 0)
                 }
@@ -1584,7 +1584,7 @@ extension NonBlockingFileIOTest {
 
     func testAsyncFileOpenFails() async throws {
         do {
-            _ = try await self.fileIO.withFileRegion(path: "/dev/null/this/does/not/exist") { _ in }
+            _ = try await self.fileIO.withFileRegion(_deprecatedPath: "/dev/null/this/does/not/exist") { _ in }
             XCTFail("should've thrown")
         } catch let e as IOError where e.errnoCode == ENOTDIR {
             // OK
@@ -1596,7 +1596,7 @@ extension NonBlockingFileIOTest {
     func testAsyncOpeningFilesForWriting() async throws {
         try await withTemporaryDirectory { dir in
             try await self.fileIO!.withFileHandle(
-                path: "\(dir)/file",
+                _deprecatedPath: "\(dir)/file",
                 mode: .write,
                 flags: .allowFileCreation()
             ) { _ in }
@@ -1607,7 +1607,7 @@ extension NonBlockingFileIOTest {
         do {
             try await withTemporaryDirectory { dir in
                 try await self.fileIO!.withFileHandle(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: .write,
                     flags: .default
                 ) { _ in }
@@ -1621,7 +1621,7 @@ extension NonBlockingFileIOTest {
     func testAsyncOpeningFilesForWritingDoesNotAllowReading() async throws {
         try await withTemporaryDirectory { dir in
             try await self.fileIO!.withFileHandle(
-                path: "\(dir)/file",
+                _deprecatedPath: "\(dir)/file",
                 mode: .write,
                 flags: .allowFileCreation()
             ) { fileHandle in
@@ -1641,7 +1641,7 @@ extension NonBlockingFileIOTest {
     func testAsyncOpeningFilesForWritingAndReading() async throws {
         try await withTemporaryDirectory { dir in
             try await self.fileIO!.withFileHandle(
-                path: "\(dir)/file",
+                _deprecatedPath: "\(dir)/file",
                 mode: [.write, .read],
                 flags: .allowFileCreation()
             ) { fileHandle in
@@ -1663,7 +1663,7 @@ extension NonBlockingFileIOTest {
             // open 1 + write
             do {
                 try await self.fileIO.withFileHandle(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: [.write, .read],
                     flags: .allowFileCreation()
                 ) { fileHandle in
@@ -1682,7 +1682,7 @@ extension NonBlockingFileIOTest {
             // open 2 + write again + read
             do {
                 try await self.fileIO!.withFileHandle(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: [.write, .read],
                     flags: .default
                 ) { fileHandle in
@@ -1721,7 +1721,7 @@ extension NonBlockingFileIOTest {
             // open 1 + write
             do {
                 try await self.fileIO!.withFileHandle(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: [.write, .read],
                     flags: .allowFileCreation()
                 ) { fileHandle in
@@ -1739,7 +1739,7 @@ extension NonBlockingFileIOTest {
             // open 2 (with truncation) + write again + read
             do {
                 try await self.fileIO!.withFileHandle(
-                    path: "\(dir)/file",
+                    _deprecatedPath: "\(dir)/file",
                     mode: [.write, .read],
                     flags: .posix(flags: O_TRUNC, mode: 0)
                 ) { fileHandle in
@@ -1811,7 +1811,7 @@ extension NonBlockingFileIOTest {
             let threadPool = NIOThreadPool(numberOfThreads: 1)
             let fileIO = NonBlockingFileIO(threadPool: threadPool)
             do {
-                try await fileIO.withFileRegion(path: path) { _ in }
+                try await fileIO.withFileRegion(_deprecatedPath: path) { _ in }
                 XCTFail("testAsyncThrowsErrorOnUnstartedPool: openFile should throw an error")
             } catch {
             }
@@ -1873,7 +1873,7 @@ extension NonBlockingFileIOTest {
         try await withTemporaryDirectory { path in
             let file = "\(path)/file"
             try await self.fileIO.withFileHandle(
-                path: file,
+                _deprecatedPath: file,
                 mode: .write,
                 flags: .allowFileCreation()
             ) { handle in
@@ -1887,7 +1887,7 @@ extension NonBlockingFileIOTest {
         try await withTemporaryDirectory { path in
             let file = "\(path)/file"
             try await self.fileIO.withFileHandle(
-                path: file,
+                _deprecatedPath: file,
                 mode: .write,
                 flags: .allowFileCreation()
             ) { handle in
@@ -1914,7 +1914,7 @@ extension NonBlockingFileIOTest {
         try await withTemporaryDirectory { path in
             let file = "\(path)/file"
             try await self.fileIO.withFileHandle(
-                path: file,
+                _deprecatedPath: file,
                 mode: .write,
                 flags: .allowFileCreation()
             ) { handle in
diff --git a/Tests/NIOPosixTests/TestUtils.swift b/Tests/NIOPosixTests/TestUtils.swift
index c2bb121903..c239a09c8a 100644
--- a/Tests/NIOPosixTests/TestUtils.swift
+++ b/Tests/NIOPosixTests/TestUtils.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -45,8 +45,8 @@ func withPipe(_ body: (NIOCore.NIOFileHandle, NIOCore.NIOFileHandle) throws -> [
     fds.withUnsafeMutableBufferPointer { ptr in
         XCTAssertEqual(0, pipe(ptr.baseAddress!))
     }
-    let readFH = NIOFileHandle(descriptor: fds[0])
-    let writeFH = NIOFileHandle(descriptor: fds[1])
+    let readFH = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: fds[0])
+    let writeFH = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: fds[1])
     var toClose: [NIOFileHandle] = [readFH, writeFH]
     var error: Error? = nil
     do {
@@ -70,8 +70,8 @@ func withPipe(
     fds.withUnsafeMutableBufferPointer { ptr in
         XCTAssertEqual(0, pipe(ptr.baseAddress!))
     }
-    let readFH = NIOFileHandle(descriptor: fds[0])
-    let writeFH = NIOFileHandle(descriptor: fds[1])
+    let readFH = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: fds[0])
+    let writeFH = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: fds[1])
     var toClose: [NIOFileHandle] = [readFH, writeFH]
     var error: Error? = nil
     do {
@@ -149,7 +149,7 @@ func withTemporaryUnixDomainSocketPathName<T>(
 
 func withTemporaryFile<T>(content: String? = nil, _ body: (NIOCore.NIOFileHandle, String) throws -> T) rethrows -> T {
     let (fd, path) = openTemporaryFile()
-    let fileHandle = NIOFileHandle(descriptor: fd)
+    let fileHandle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: fd)
     defer {
         XCTAssertNoThrow(try fileHandle.close())
         XCTAssertEqual(0, unlink(path))
@@ -181,7 +181,7 @@ func withTemporaryFile<T>(
     _ body: @escaping @Sendable (NIOCore.NIOFileHandle, String) async throws -> T
 ) async rethrows -> T {
     let (fd, path) = openTemporaryFile()
-    let fileHandle = NIOFileHandle(descriptor: fd)
+    let fileHandle = NIOFileHandle(_deprecatedTakingOwnershipOfDescriptor: fd)
     defer {
         XCTAssertNoThrow(try fileHandle.close())
         XCTAssertEqual(0, unlink(path))

From 1f1e787f0bfa9f5e4f5b54176f9f879ce44b638c Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Mon, 25 Nov 2024 16:21:41 +0000
Subject: [PATCH 12/37] Add Cxx interop swift settings to CI (#2999)

### Motivation:

At the moment the Cxx interoperability CI workflow doesn't actually test
Cxx interoperability.

### Modifications:

The script now looks for the precise way that the `Package.swift`
currently formats the target entry and appends a `swiftSettings` entry
to it.

This approach is pretty brittle but has the advantage that it's the same
on Linux and Darwin. If this formatting changes too often then we could
give up on this and assume the availability of gnu-sed instead (and make
sure it's present on our CI containers).

### Result:

All Cxx interoperability checks will check what they are named for.

---------

Co-authored-by: George Barnett <gbarnett@apple.com>
---
 scripts/check-cxx-interop-compatibility.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scripts/check-cxx-interop-compatibility.sh b/scripts/check-cxx-interop-compatibility.sh
index f393eb4c78..5ad33a609c 100755
--- a/scripts/check-cxx-interop-compatibility.sh
+++ b/scripts/check-cxx-interop-compatibility.sh
@@ -19,7 +19,7 @@ log() { printf -- "** %s\n" "$*" >&2; }
 error() { printf -- "** ERROR: %s\n" "$*" >&2; }
 fatal() { error "$@"; exit 1; }
 
-log "Checking for Cxx interoperability comaptibility..."
+log "Checking for Cxx interoperability compatibility..."
 
 source_dir=$(pwd)
 working_dir=$(mktemp -d)
@@ -30,6 +30,12 @@ package_name=$(swift package dump-package | jq -r '.name')
 
 cd "$working_dir"
 swift package init
+
+{
+  echo "let swiftSettings: [SwiftSetting] = [.interoperabilityMode(.Cxx)]"
+  echo "for target in package.targets { target.swiftSettings = swiftSettings }"
+} >> Package.swift
+
 echo "package.dependencies.append(.package(path: \"$source_dir\"))" >> Package.swift
 
 for product in $library_products; do
@@ -39,4 +45,4 @@ done
 
 swift build
 
-log "✅ Passed the Cxx interoperability tests."
\ No newline at end of file
+log "✅ Passed the Cxx interoperability tests."

From 6ba8f4f04aeefa3db0bea01a339f85c7f7241476 Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Mon, 25 Nov 2024 17:56:44 +0000
Subject: [PATCH 13/37] fix almost all Sendable warnings (#2994)

### Motivation:

Opening the `swift-nio` repository made me warning blind because there
were always so many trivially fixable warnings about things that were
correct but cannot be understood by the compiler.

### Modifications:

Fix all the sendable warnings that popped up, except for one test where
`NIOLockedValueBox<Thread?>` isn't sendable because `Foundation.Thread`
seemingly isn't `Sendable` which is odd. Guessing that'll be fixed on
their end.

### Result:

- Fewer warnings
- Less warning-blindness
- More checks
---
 Sources/NIOCore/ChannelPipeline.swift         | 19 ++++++++++-
 .../Internal/BufferedStream.swift             |  1 -
 .../NIOHTTP1/HTTPServerPipelineHandler.swift  |  6 ++--
 .../NIOHTTP1/HTTPServerUpgradeHandler.swift   | 29 ++++++++++-------
 .../NIOHTTPClientUpgradeHandler.swift         | 22 +++++++------
 .../NIOTypedHTTPClientUpgradeHandler.swift    | 19 +++++++----
 .../NIOTypedHTTPServerUpgradeHandler.swift    | 30 +++++++++++------
 Sources/NIOHTTP1Server/main.swift             |  6 ++--
 Sources/NIOPosix/Bootstrap.swift              | 27 +++++++++++-----
 ...pplicationProtocolNegotiationHandler.swift |  4 ++-
 ...pplicationProtocolNegotiationHandler.swift |  4 ++-
 Sources/NIOTestUtils/NIOHTTP1TestServer.swift |  4 ++-
 .../WebSocketProtocolErrorHandler.swift       |  4 ++-
 .../EmbeddedChannelTest.swift                 | 10 +++---
 .../HTTPClientUpgradeTests.swift              |  8 +++--
 Tests/NIOHTTP1Tests/HTTPDecoderTest.swift     | 10 +++---
 .../NIOHTTP1Tests/HTTPServerClientTest.swift  |  9 ++++++
 .../HTTPServerPipelineHandlerTest.swift       |  5 ++-
 .../ChannelNotificationTest.swift             | 30 ++++++++---------
 Tests/NIOPosixTests/ChannelPipelineTest.swift | 14 ++++++++
 Tests/NIOPosixTests/ChannelTests.swift        | 18 ++++++++---
 .../NIOPosixTests/EchoServerClientTest.swift  | 27 +++++++++++-----
 Tests/NIOPosixTests/EventLoopTest.swift       |  4 ++-
 .../NIOPosixTests/IdleStateHandlerTest.swift  |  5 +--
 Tests/NIOPosixTests/NIOThreadPoolTest.swift   | 20 +++++++-----
 Tests/NIOPosixTests/SocketChannelTest.swift   |  4 ++-
 Tests/NIOPosixTests/StreamChannelsTest.swift  | 32 +++++++++++++------
 27 files changed, 253 insertions(+), 118 deletions(-)

diff --git a/Sources/NIOCore/ChannelPipeline.swift b/Sources/NIOCore/ChannelPipeline.swift
index 3db4a88abd..2aa8d6c624 100644
--- a/Sources/NIOCore/ChannelPipeline.swift
+++ b/Sources/NIOCore/ChannelPipeline.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -2131,6 +2131,23 @@ extension ChannelHandlerContext {
         self.writeAndFlush(data, promise: promise)
         return promise.futureResult
     }
+
+    /// Returns this `ChannelHandlerContext` as a `NIOLoopBound`, bound to `self.eventLoop`.
+    ///
+    /// This is a shorthand for `NIOLoopBound(self, eventLoop: self.eventLoop)`.
+    ///
+    /// Being able to capture `ChannelHandlerContext`s in `EventLoopFuture` callbacks is important in SwiftNIO programs.
+    /// Of course, this is not always safe because the `EventLoopFuture` callbacks may run on other threads. SwiftNIO
+    /// programmers therefore always had to manually arrange for those callbacks to run on the correct `EventLoop`
+    /// (i.e. `context.eventLoop`) which then made that construction safe.
+    ///
+    /// Newer Swift versions contain a static feature to automatically detect data races which of course can't detect
+    /// the only _dynamically_ ``EventLoop`` a ``EventLoopFuture`` callback is running on. ``NIOLoopBound`` can be used
+    /// to prove to the compiler that this is safe and in case it is not, ``NIOLoopBound`` will trap at runtime. This is
+    /// therefore dynamically enforce the correct behaviour.
+    public var loopBound: NIOLoopBound<ChannelHandlerContext> {
+        NIOLoopBound(self, eventLoop: self.eventLoop)
+    }
 }
 
 @available(*, unavailable)
diff --git a/Sources/NIOFileSystem/Internal/BufferedStream.swift b/Sources/NIOFileSystem/Internal/BufferedStream.swift
index d68ed6b61d..d450b00b58 100644
--- a/Sources/NIOFileSystem/Internal/BufferedStream.swift
+++ b/Sources/NIOFileSystem/Internal/BufferedStream.swift
@@ -243,7 +243,6 @@ extension BufferedStream {
         }
 
         /// A type that indicates the result of writing elements to the source.
-        @frozen
         internal enum WriteResult: Sendable {
             /// A token that is returned when the asynchronous stream's backpressure strategy indicated that production should
             /// be suspended. Use this token to enqueue a callback by  calling the ``enqueueCallback(_:)`` method.
diff --git a/Sources/NIOHTTP1/HTTPServerPipelineHandler.swift b/Sources/NIOHTTP1/HTTPServerPipelineHandler.swift
index c6626dd0e7..1272adc1fe 100644
--- a/Sources/NIOHTTP1/HTTPServerPipelineHandler.swift
+++ b/Sources/NIOHTTP1/HTTPServerPipelineHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -439,9 +439,11 @@ public final class HTTPServerPipelineHandler: ChannelDuplexHandler, RemovableCha
                 // we just received the .end that we're missing so we can fall through to closing the connection
                 fallthrough
             case .quiescingLastRequestEndReceived:
+                let loopBoundContext = context.loopBound
                 self.lifecycleState = .quiescingCompleted
                 context.write(data).flatMap {
-                    context.close()
+                    let context = loopBoundContext.value
+                    return context.close()
                 }.cascade(to: promise)
             case .acceptingEvents, .quiescingWaitingForRequestEnd:
                 context.write(data, promise: promise)
diff --git a/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift b/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift
index cadaff37c3..464baeec36 100644
--- a/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift
+++ b/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -180,14 +180,16 @@ public final class HTTPServerUpgradeHandler: ChannelInboundHandler, RemovableCha
         // We'll attempt to upgrade. This may take a while, so while we're waiting more data can come in.
         self.upgradeState = .awaitingUpgrader
 
+        let eventLoop = context.eventLoop
+        let loopBoundContext = context.loopBound
         self.handleUpgrade(context: context, request: request, requestedProtocols: requestedProtocols)
-            .hop(to: context.eventLoop)  // the user might return a future from another EventLoop.
+            .hop(to: eventLoop)  // the user might return a future from another EventLoop.
             .whenSuccess { callback in
-                context.eventLoop.assertInEventLoop()
+                eventLoop.assertInEventLoop()
                 if let callback = callback {
                     self.gotUpgrader(upgrader: callback)
                 } else {
-                    self.notUpgrading(context: context, data: requestPart)
+                    self.notUpgrading(context: loopBoundContext.value, data: requestPart)
                 }
             }
     }
@@ -253,6 +255,8 @@ public final class HTTPServerUpgradeHandler: ChannelInboundHandler, RemovableCha
         }
 
         let responseHeaders = self.buildUpgradeHeaders(protocol: proto)
+        let pipeline = context.pipeline
+        let loopBoundContext = context.loopBound
         return upgrader.buildUpgradeResponse(
             channel: context.channel,
             upgradeRequest: request,
@@ -271,18 +275,20 @@ public final class HTTPServerUpgradeHandler: ChannelInboundHandler, RemovableCha
                 // internal handler, then call the user code, and then finally when the user code is done we do
                 // our final cleanup steps, namely we replay the received data we buffered in the meantime and
                 // then remove ourselves from the pipeline.
-                self.removeExtraHandlers(context: context).flatMap {
+                self.removeExtraHandlers(pipeline: pipeline).flatMap {
                     self.sendUpgradeResponse(
-                        context: context,
+                        context: loopBoundContext.value,
                         upgradeRequest: request,
                         responseHeaders: finalResponseHeaders
                     )
                 }.flatMap {
-                    context.pipeline.syncOperations.removeHandler(self.httpEncoder)
+                    pipeline.syncOperations.removeHandler(self.httpEncoder)
                 }.flatMap { () -> EventLoopFuture<Void> in
+                    let context = loopBoundContext.value
                     self.upgradeCompletionHandler(context)
                     return upgrader.upgrade(context: context, upgradeRequest: request)
                 }.whenComplete { result in
+                    let context = loopBoundContext.value
                     switch result {
                     case .success:
                         context.fireUserInboundEventTriggered(
@@ -300,6 +306,7 @@ public final class HTTPServerUpgradeHandler: ChannelInboundHandler, RemovableCha
             }
         }.flatMapError { error in
             // No upgrade here. We want to fire the error down the pipeline, and then try another loop iteration.
+            let context = loopBoundContext.value
             context.fireErrorCaught(error)
             return self.handleUpgradeForProtocol(
                 context: context,
@@ -366,14 +373,14 @@ public final class HTTPServerUpgradeHandler: ChannelInboundHandler, RemovableCha
     }
 
     /// Removes any extra HTTP-related handlers from the channel pipeline.
-    private func removeExtraHandlers(context: ChannelHandlerContext) -> EventLoopFuture<Void> {
+    private func removeExtraHandlers(pipeline: ChannelPipeline) -> EventLoopFuture<Void> {
         guard self.extraHTTPHandlers.count > 0 else {
-            return context.eventLoop.makeSucceededFuture(())
+            return pipeline.eventLoop.makeSucceededFuture(())
         }
 
         return .andAllSucceed(
-            self.extraHTTPHandlers.map { context.pipeline.removeHandler($0) },
-            on: context.eventLoop
+            self.extraHTTPHandlers.map { pipeline.removeHandler($0) },
+            on: pipeline.eventLoop
         )
     }
 }
diff --git a/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift b/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift
index dbc403f69f..66a7b1a17a 100644
--- a/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift
+++ b/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2019-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -325,19 +325,22 @@ public final class NIOHTTPClientUpgradeHandler: ChannelDuplexHandler, RemovableC
         // Once that's done, we call the internal handler, then call the upgrader code, and then finally when the
         // upgrader code is done, we do our final cleanup steps, namely we replay the received data we
         // buffered in the meantime and then remove ourselves from the pipeline.
-        {
+        let pipeline = context.pipeline
+        let loopBoundContext = context.loopBound
+        return {
             self.upgradeState = .upgrading
 
-            self.removeHTTPHandlers(context: context)
+            self.removeHTTPHandlers(pipeline: pipeline)
                 .map {
                     // Let the other handlers be removed before continuing with upgrade.
-                    self.upgradeCompletionHandler(context)
+                    self.upgradeCompletionHandler(loopBoundContext.value)
                     self.upgradeState = .upgradingAddingHandlers
                 }
                 .flatMap {
-                    upgrader.upgrade(context: context, upgradeResponse: response)
+                    upgrader.upgrade(context: loopBoundContext.value, upgradeResponse: response)
                 }
                 .map {
+                    let context = loopBoundContext.value
                     // We unbuffer any buffered data here.
 
                     // If we received any, we fire readComplete.
@@ -356,19 +359,20 @@ public final class NIOHTTPClientUpgradeHandler: ChannelDuplexHandler, RemovableC
                     self.upgradeState = .upgradeComplete
                 }
                 .whenComplete { _ in
+                    let context = loopBoundContext.value
                     context.pipeline.syncOperations.removeHandler(context: context, promise: nil)
                 }
         }
     }
 
     /// Removes any extra HTTP-related handlers from the channel pipeline.
-    private func removeHTTPHandlers(context: ChannelHandlerContext) -> EventLoopFuture<Void> {
+    private func removeHTTPHandlers(pipeline: ChannelPipeline) -> EventLoopFuture<Void> {
         guard self.httpHandlers.count > 0 else {
-            return context.eventLoop.makeSucceededFuture(())
+            return pipeline.eventLoop.makeSucceededFuture(())
         }
 
-        let removeFutures = self.httpHandlers.map { context.pipeline.removeHandler($0) }
-        return .andAllSucceed(removeFutures, on: context.eventLoop)
+        let removeFutures = self.httpHandlers.map { pipeline.removeHandler($0) }
+        return .andAllSucceed(removeFutures, on: pipeline.eventLoop)
     }
 
     private func gotUpgrader(upgrader: @escaping (() -> Void)) {
diff --git a/Sources/NIOHTTP1/NIOTypedHTTPClientUpgradeHandler.swift b/Sources/NIOHTTP1/NIOTypedHTTPClientUpgradeHandler.swift
index 30f168c4e4..29764820a3 100644
--- a/Sources/NIOHTTP1/NIOTypedHTTPClientUpgradeHandler.swift
+++ b/Sources/NIOHTTP1/NIOTypedHTTPClientUpgradeHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2013 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -191,6 +191,7 @@ public final class NIOTypedHTTPClientUpgradeHandler<UpgradeResult: Sendable>: Ch
     }
 
     private func channelRead(context: ChannelHandlerContext, responsePart: HTTPClientResponsePart) {
+        let loopBoundContext = context.loopBound
         switch self.stateMachine.channelReadResponsePart(responsePart) {
         case .fireErrorCaughtAndRemoveHandler(let error):
             self.upgradeResultPromise.fail(error)
@@ -201,6 +202,7 @@ public final class NIOTypedHTTPClientUpgradeHandler<UpgradeResult: Sendable>: Ch
             self.notUpgradingCompletionHandler(context.channel)
                 .hop(to: context.eventLoop)
                 .whenComplete { result in
+                    let context = loopBoundContext.value
                     self.upgradingHandlerCompleted(context: context, result)
                 }
 
@@ -223,11 +225,14 @@ public final class NIOTypedHTTPClientUpgradeHandler<UpgradeResult: Sendable>: Ch
     ) {
         // Before we start the upgrade we have to remove the HTTPEncoder and HTTPDecoder handlers from the
         // pipeline, to prevent them parsing any more data. We'll buffer the incoming data until that completes.
-        self.removeHTTPHandlers(context: context)
+        let channel = context.channel
+        let loopBoundContext = context.loopBound
+        self.removeHTTPHandlers(pipeline: context.pipeline)
             .flatMap {
-                upgrader.upgrade(channel: context.channel, upgradeResponse: responseHead)
+                upgrader.upgrade(channel: channel, upgradeResponse: responseHead)
             }.hop(to: context.eventLoop)
             .whenComplete { result in
+                let context = loopBoundContext.value
                 self.upgradingHandlerCompleted(context: context, result)
             }
     }
@@ -275,13 +280,13 @@ public final class NIOTypedHTTPClientUpgradeHandler<UpgradeResult: Sendable>: Ch
     }
 
     /// Removes any extra HTTP-related handlers from the channel pipeline.
-    private func removeHTTPHandlers(context: ChannelHandlerContext) -> EventLoopFuture<Void> {
+    private func removeHTTPHandlers(pipeline: ChannelPipeline) -> EventLoopFuture<Void> {
         guard self.httpHandlers.count > 0 else {
-            return context.eventLoop.makeSucceededFuture(())
+            return pipeline.eventLoop.makeSucceededFuture(())
         }
 
-        let removeFutures = self.httpHandlers.map { context.pipeline.removeHandler($0) }
-        return .andAllSucceed(removeFutures, on: context.eventLoop)
+        let removeFutures = self.httpHandlers.map { pipeline.removeHandler($0) }
+        return .andAllSucceed(removeFutures, on: pipeline.eventLoop)
     }
 }
 #endif
diff --git a/Sources/NIOHTTP1/NIOTypedHTTPServerUpgradeHandler.swift b/Sources/NIOHTTP1/NIOTypedHTTPServerUpgradeHandler.swift
index b430cf1be4..77fb961b0d 100644
--- a/Sources/NIOHTTP1/NIOTypedHTTPServerUpgradeHandler.swift
+++ b/Sources/NIOHTTP1/NIOTypedHTTPServerUpgradeHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2023-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -174,6 +174,7 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
     }
 
     private func channelRead(context: ChannelHandlerContext, requestPart: HTTPServerRequestPart) {
+        let loopBoundContext = context.loopBound
         switch self.stateMachine.channelReadRequestPart(requestPart) {
         case .failUpgradePromise(let error):
             self.upgradeResultPromise.fail(error)
@@ -182,6 +183,7 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
             self.notUpgradingCompletionHandler(context.channel)
                 .hop(to: context.eventLoop)
                 .whenComplete { result in
+                    let context = loopBoundContext.value
                     self.upgradingHandlerCompleted(context: context, result, requestHeadAndProtocol: nil)
                 }
 
@@ -194,6 +196,7 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
                 allHeaderNames: allHeaderNames,
                 connectionHeader: connectionHeader
             ).whenComplete { result in
+                let context = loopBoundContext.value
                 context.eventLoop.assertInEventLoop()
                 self.findingUpgradeCompleted(context: context, requestHead: head, result)
             }
@@ -297,6 +300,7 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
             )
         }
 
+        let loopBoundContext = context.loopBound
         let responseHeaders = self.buildUpgradeHeaders(protocol: proto)
         return upgrader.buildUpgradeResponse(
             channel: context.channel,
@@ -307,6 +311,7 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
         .map { (upgrader, $0, proto) }
         .flatMapError { error in
             // No upgrade here. We want to fire the error down the pipeline, and then try another loop iteration.
+            let context = loopBoundContext.value
             context.fireErrorCaught(error)
             return self.handleUpgradeForProtocol(
                 context: context,
@@ -339,9 +344,11 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
             )
 
         case .runNotUpgradingInitializer:
+            let loopBoundContext = context.loopBound
             self.notUpgradingCompletionHandler(context.channel)
                 .hop(to: context.eventLoop)
                 .whenComplete { result in
+                    let context = loopBoundContext.value
                     self.upgradingHandlerCompleted(context: context, result, requestHeadAndProtocol: nil)
                 }
 
@@ -376,14 +383,19 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
         // internal handler, then call the user code, and then finally when the user code is done we do
         // our final cleanup steps, namely we replay the received data we buffered in the meantime and
         // then remove ourselves from the pipeline.
-        self.removeExtraHandlers(context: context).flatMap {
-            self.sendUpgradeResponse(context: context, responseHeaders: responseHeaders)
+        let channel = context.channel
+        let pipeline = context.pipeline
+        let loopBoundContext = context.loopBound
+        self.removeExtraHandlers(pipeline: pipeline).flatMap {
+            let context = loopBoundContext.value
+            return self.sendUpgradeResponse(context: context, responseHeaders: responseHeaders)
         }.flatMap {
-            context.pipeline.syncOperations.removeHandler(self.httpEncoder)
+            pipeline.syncOperations.removeHandler(self.httpEncoder)
         }.flatMap { () -> EventLoopFuture<UpgradeResult> in
-            upgrader.upgrade(channel: context.channel, upgradeRequest: requestHead)
+            upgrader.upgrade(channel: channel, upgradeRequest: requestHead)
         }.hop(to: context.eventLoop)
             .whenComplete { result in
+                let context = loopBoundContext.value
                 self.upgradingHandlerCompleted(context: context, result, requestHeadAndProtocol: (requestHead, proto))
             }
     }
@@ -404,14 +416,14 @@ public final class NIOTypedHTTPServerUpgradeHandler<UpgradeResult: Sendable>: Ch
     }
 
     /// Removes any extra HTTP-related handlers from the channel pipeline.
-    private func removeExtraHandlers(context: ChannelHandlerContext) -> EventLoopFuture<Void> {
+    private func removeExtraHandlers(pipeline: ChannelPipeline) -> EventLoopFuture<Void> {
         guard self.extraHTTPHandlers.count > 0 else {
-            return context.eventLoop.makeSucceededFuture(())
+            return pipeline.eventLoop.makeSucceededFuture(())
         }
 
         return .andAllSucceed(
-            self.extraHTTPHandlers.map { context.pipeline.removeHandler($0) },
-            on: context.eventLoop
+            self.extraHTTPHandlers.map { pipeline.removeHandler($0) },
+            on: pipeline.eventLoop
         )
     }
 
diff --git a/Sources/NIOHTTP1Server/main.swift b/Sources/NIOHTTP1Server/main.swift
index 2b662bcc3d..d3b6ce133e 100644
--- a/Sources/NIOHTTP1Server/main.swift
+++ b/Sources/NIOHTTP1Server/main.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -207,7 +207,7 @@ private final class HTTPHandler: ChannelInboundHandler {
             ()
         case .end:
             self.state.requestComplete()
-            let loopBoundContext = NIOLoopBound(context, eventLoop: context.eventLoop)
+            let loopBoundContext = context.loopBound
             let loopBoundSelf = NIOLoopBound(self, eventLoop: context.eventLoop)
             context.eventLoop.scheduleTask(in: delay) { () -> Void in
                 let `self` = loopBoundSelf.value
@@ -501,7 +501,7 @@ private final class HTTPHandler: ChannelInboundHandler {
         promise: EventLoopPromise<Void>?
     ) {
         self.state.responseComplete()
-        let loopBoundContext = NIOLoopBound(context, eventLoop: context.eventLoop)
+        let loopBoundContext = context.loopBound
 
         let promise = self.keepAlive ? promise : (promise ?? context.eventLoop.makePromise())
         if !self.keepAlive {
diff --git a/Sources/NIOPosix/Bootstrap.swift b/Sources/NIOPosix/Bootstrap.swift
index ff2773a2cd..fa117e8576 100644
--- a/Sources/NIOPosix/Bootstrap.swift
+++ b/Sources/NIOPosix/Bootstrap.swift
@@ -431,6 +431,7 @@ public final class ServerBootstrap {
 
     final class AcceptHandler: ChannelInboundHandler {
         public typealias InboundIn = SocketChannel
+        public typealias InboundOut = SocketChannel
 
         private let childChannelInit: ((Channel) -> EventLoopFuture<Void>)?
         private let childChannelOptions: ChannelOptions.Storage
@@ -445,7 +446,9 @@ public final class ServerBootstrap {
 
         func userInboundEventTriggered(context: ChannelHandlerContext, event: Any) {
             if event is ChannelShouldQuiesceEvent {
+                let loopBoundContext = context.loopBound
                 context.channel.close().whenFailure { error in
+                    let context = loopBoundContext.value
                     context.fireErrorCaught(error)
                 }
             }
@@ -467,28 +470,33 @@ public final class ServerBootstrap {
             }
 
             @inline(__always)
-            func fireThroughPipeline(_ future: EventLoopFuture<Void>) {
+            func fireThroughPipeline(_ future: EventLoopFuture<Void>, context: ChannelHandlerContext) {
                 ctxEventLoop.assertInEventLoop()
+                assert(ctxEventLoop === context.eventLoop)
+                let loopBoundContext = context.loopBound
                 future.flatMap { (_) -> EventLoopFuture<Void> in
+                    let context = loopBoundContext.value
                     ctxEventLoop.assertInEventLoop()
                     guard context.channel.isActive else {
-                        return context.eventLoop.makeFailedFuture(ChannelError._ioOnClosedChannel)
+                        return ctxEventLoop.makeFailedFuture(ChannelError._ioOnClosedChannel)
                     }
-                    context.fireChannelRead(data)
+                    context.fireChannelRead(Self.wrapInboundOut(accepted))
                     return context.eventLoop.makeSucceededFuture(())
                 }.whenFailure { error in
+                    let context = loopBoundContext.value
                     ctxEventLoop.assertInEventLoop()
                     self.closeAndFire(context: context, accepted: accepted, err: error)
                 }
             }
 
             if childEventLoop === ctxEventLoop {
-                fireThroughPipeline(setupChildChannel())
+                fireThroughPipeline(setupChildChannel(), context: context)
             } else {
                 fireThroughPipeline(
                     childEventLoop.flatSubmit {
                         setupChildChannel()
-                    }.hop(to: ctxEventLoop)
+                    }.hop(to: ctxEventLoop),
+                    context: context
                 )
             }
         }
@@ -498,7 +506,9 @@ public final class ServerBootstrap {
             if context.eventLoop.inEventLoop {
                 context.fireErrorCaught(err)
             } else {
+                let loopBoundContext = context.loopBound
                 context.eventLoop.execute {
+                    let context = loopBoundContext.value
                     context.fireErrorCaught(err)
                 }
             }
@@ -998,7 +1008,8 @@ public final class ClientBootstrap: NIOClientTCPBootstrapProtocol {
         let connectPromise = channel.eventLoop.makePromise(of: Void.self)
         channel.connect(to: address, promise: connectPromise)
         let cancelTask = channel.eventLoop.scheduleTask(in: self.connectTimeout) {
-            connectPromise.fail(ChannelError.connectTimeout(self.connectTimeout))
+            [connectTimeout = self.connectTimeout] in
+            connectPromise.fail(ChannelError.connectTimeout(connectTimeout))
             channel.close(promise: nil)
         }
 
@@ -1147,8 +1158,8 @@ public final class ClientBootstrap: NIOClientTCPBootstrapProtocol {
         @inline(__always)
         func setupChannel() -> EventLoopFuture<Channel> {
             eventLoop.assertInEventLoop()
-            return channelOptions.applyAllChannelOptions(to: channel).flatMap {
-                if let bindTarget = self.bindTarget {
+            return channelOptions.applyAllChannelOptions(to: channel).flatMap { [bindTarget = self.bindTarget] in
+                if let bindTarget = bindTarget {
                     return channel.bind(to: bindTarget).flatMap {
                         channelInitializer(channel)
                     }
diff --git a/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift b/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift
index 3d6859dec9..71186337f2 100644
--- a/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift
+++ b/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -119,10 +119,12 @@ public final class ApplicationProtocolNegotiationHandler: ChannelInboundHandler,
 
     private func invokeUserClosure(context: ChannelHandlerContext, result: ALPNResult) {
         let switchFuture = self.completionHandler(result, context.channel)
+        let loopBoundSelfAndContext = NIOLoopBound((self, context), eventLoop: context.eventLoop)
 
         switchFuture
             .hop(to: context.eventLoop)
             .whenComplete { result in
+                let (`self`, context) = loopBoundSelfAndContext.value
                 self.userFutureCompleted(context: context, result: result)
             }
     }
diff --git a/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift b/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift
index 4749be12ec..75d95c5a45 100644
--- a/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift
+++ b/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2023-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -125,10 +125,12 @@ public final class NIOTypedApplicationProtocolNegotiationHandler<NegotiationResu
 
     private func invokeUserClosure(context: ChannelHandlerContext, result: ALPNResult) {
         let switchFuture = self.completionHandler(result, context.channel)
+        let loopBoundSelfAndContext = NIOLoopBound((self, context), eventLoop: context.eventLoop)
 
         switchFuture
             .hop(to: context.eventLoop)
             .whenComplete { result in
+                let (`self`, context) = loopBoundSelfAndContext.value
                 self.userFutureCompleted(context: context, result: result)
             }
     }
diff --git a/Sources/NIOTestUtils/NIOHTTP1TestServer.swift b/Sources/NIOTestUtils/NIOHTTP1TestServer.swift
index d3dbb15f6d..6687d6e883 100644
--- a/Sources/NIOTestUtils/NIOHTTP1TestServer.swift
+++ b/Sources/NIOTestUtils/NIOHTTP1TestServer.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2019-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -118,6 +118,7 @@ private final class WebServerHandler: ChannelDuplexHandler {
     }
 
     func write(context: ChannelHandlerContext, data: NIOAny, promise: EventLoopPromise<Void>?) {
+        let loopBoundContext = context.loopBound
         switch Self.unwrapOutboundIn(data) {
         case .head(var head):
             head.headers.replaceOrAdd(name: "connection", value: "close")
@@ -127,6 +128,7 @@ private final class WebServerHandler: ChannelDuplexHandler {
             context.write(data, promise: promise)
         case .end:
             context.write(data).map {
+                let context = loopBoundContext.value
                 context.close(promise: nil)
             }.cascade(to: promise)
         }
diff --git a/Sources/NIOWebSocket/WebSocketProtocolErrorHandler.swift b/Sources/NIOWebSocket/WebSocketProtocolErrorHandler.swift
index 89714f4c90..e367184b62 100644
--- a/Sources/NIOWebSocket/WebSocketProtocolErrorHandler.swift
+++ b/Sources/NIOWebSocket/WebSocketProtocolErrorHandler.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -25,6 +25,7 @@ public final class WebSocketProtocolErrorHandler: ChannelInboundHandler {
     public init() {}
 
     public func errorCaught(context: ChannelHandlerContext, error: Error) {
+        let loopBoundContext = context.loopBound
         if let error = error as? NIOWebSocketError {
             var data = context.channel.allocator.buffer(capacity: 2)
             data.write(webSocketErrorCode: WebSocketErrorCode(error))
@@ -34,6 +35,7 @@ public final class WebSocketProtocolErrorHandler: ChannelInboundHandler {
                 data: data
             )
             context.writeAndFlush(Self.wrapOutboundOut(frame)).whenComplete { (_: Result<Void, Error>) in
+                let context = loopBoundContext.value
                 context.close(promise: nil)
             }
         }
diff --git a/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift b/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift
index 56c09b67f8..3c77ca624a 100644
--- a/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift
+++ b/Tests/NIOEmbeddedTests/EmbeddedChannelTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -478,7 +478,7 @@ class EmbeddedChannelTest: XCTestCase {
 
     func testFinishWithRecursivelyScheduledTasks() throws {
         let channel = EmbeddedChannel()
-        var tasks: [Scheduled<Void>] = []
+        let tasks: NIOLoopBoundBox<[Scheduled<Void>]> = NIOLoopBoundBox([], eventLoop: channel.eventLoop)
         var invocations = 0
 
         func recursivelyScheduleAndIncrement() {
@@ -486,7 +486,7 @@ class EmbeddedChannelTest: XCTestCase {
                 invocations += 1
                 recursivelyScheduleAndIncrement()
             }
-            tasks.append(task)
+            tasks.value.append(task)
         }
 
         recursivelyScheduleAndIncrement()
@@ -497,11 +497,11 @@ class EmbeddedChannelTest: XCTestCase {
         XCTAssertEqual(invocations, 0)
 
         // Because the root task didn't run, it should be the onnly one scheduled.
-        XCTAssertEqual(tasks.count, 1)
+        XCTAssertEqual(tasks.value.count, 1)
 
         // Check the task was failed with cancelled error.
         let taskChecked = expectation(description: "task future fulfilled")
-        tasks.first?.futureResult.whenComplete { result in
+        tasks.value.first?.futureResult.whenComplete { result in
             switch result {
             case .success: XCTFail("Expected task to be cancelled, not run.")
             case .failure(let error): XCTAssertEqual(error as? EventLoopError, .cancelled)
diff --git a/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift b/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift
index 537fa0740a..43cfbb4c6b 100644
--- a/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift
+++ b/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2019-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -200,8 +200,8 @@ private final class UpgradeDelayClientUpgrader: TypedAndUntypedHTTPClientProtoco
     fileprivate func upgrade(context: ChannelHandlerContext, upgradeResponse: HTTPResponseHead) -> EventLoopFuture<Void>
     {
         self.upgradePromise = context.eventLoop.makePromise()
-        return self.upgradePromise!.futureResult.flatMap {
-            context.pipeline.addHandler(self.upgradedHandler)
+        return self.upgradePromise!.futureResult.flatMap { [pipeline = context.pipeline] in
+            pipeline.addHandler(self.upgradedHandler)
         }
     }
 
@@ -1104,10 +1104,12 @@ final class TypedHTTPClientUpgradeTestCase: HTTPClientUpgradeTestCase {
             handlerType: NIOTypedHTTPClientUpgradeHandler<Bool>.self
         )
 
+        let loopBoundContext = context.loopBound
         try channel.connect(to: SocketAddress(ipAddress: "127.0.0.1", port: 0))
             .wait()
         upgradeResult.whenSuccess { result in
             if result {
+                let context = loopBoundContext.value
                 upgradeCompletionHandler(context)
             }
         }
diff --git a/Tests/NIOHTTP1Tests/HTTPDecoderTest.swift b/Tests/NIOHTTP1Tests/HTTPDecoderTest.swift
index da74c7685d..ffb144e5c5 100644
--- a/Tests/NIOHTTP1Tests/HTTPDecoderTest.swift
+++ b/Tests/NIOHTTP1Tests/HTTPDecoderTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -251,8 +251,8 @@ class HTTPDecoderTest: XCTestCase {
                 let part = Self.unwrapInboundIn(data)
                 switch part {
                 case .end:
-                    _ = context.pipeline.removeHandler(self).flatMap { _ in
-                        context.pipeline.addHandler(self.collector)
+                    _ = context.pipeline.removeHandler(self).flatMap { [pipeline = context.pipeline] _ in
+                        pipeline.addHandler(self.collector)
                     }
                 default:
                     // ignore
@@ -324,8 +324,8 @@ class HTTPDecoderTest: XCTestCase {
                 let part = Self.unwrapInboundIn(data)
                 switch part {
                 case .end:
-                    _ = context.pipeline.removeHandler(self).flatMap { _ in
-                        context.pipeline.addHandler(ByteCollector())
+                    _ = context.pipeline.removeHandler(self).flatMap { [pipeline = context.pipeline] _ in
+                        pipeline.addHandler(ByteCollector())
                     }
                     break
                 default:
diff --git a/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift b/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift
index 28a42923a7..c42e7c4068 100644
--- a/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift
+++ b/Tests/NIOHTTP1Tests/HTTPServerClientTest.swift
@@ -109,6 +109,7 @@ class HTTPServerClientTest: XCTestCase {
         }
 
         public func channelRead(context: ChannelHandlerContext, data: NIOAny) {
+            let loopBoundContext = context.loopBound
             switch Self.unwrapInboundIn(data) {
             case .head(let req):
                 switch req.uri {
@@ -129,6 +130,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(nil))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
@@ -154,6 +156,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(nil))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
@@ -184,6 +187,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(trailers))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
@@ -208,6 +212,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(nil))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
@@ -221,6 +226,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(nil))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
@@ -233,6 +239,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(nil))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
@@ -251,6 +258,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(nil))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
@@ -271,6 +279,7 @@ class HTTPServerClientTest: XCTestCase {
                     context.write(Self.wrapOutboundOut(.end(nil))).recover { error in
                         XCTFail("unexpected error \(error)")
                     }.whenComplete { (_: Result<Void, Error>) in
+                        let context = loopBoundContext.value
                         self.sentEnd = true
                         self.maybeClose(context: context)
                     }
diff --git a/Tests/NIOHTTP1Tests/HTTPServerPipelineHandlerTest.swift b/Tests/NIOHTTP1Tests/HTTPServerPipelineHandlerTest.swift
index 15144e8f2b..f78cdff130 100644
--- a/Tests/NIOHTTP1Tests/HTTPServerPipelineHandlerTest.swift
+++ b/Tests/NIOHTTP1Tests/HTTPServerPipelineHandlerTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -912,11 +912,13 @@ class HTTPServerPipelineHandlerTest: XCTestCase {
             }
 
             func channelRead(context: ChannelHandlerContext, data: NIOAny) {
+                let loopBoundContext = context.loopBound
                 switch Self.unwrapInboundIn(data) {
                 case .head:
                     // We dispatch this to the event loop so that it doesn't happen immediately but rather can be
                     // run from the driving test code whenever it wants by running the EmbeddedEventLoop.
                     context.eventLoop.execute {
+                        let context = loopBoundContext.value
                         context.writeAndFlush(
                             Self.wrapOutboundOut(
                                 .head(
@@ -937,6 +939,7 @@ class HTTPServerPipelineHandlerTest: XCTestCase {
                     // We dispatch this to the event loop so that it doesn't happen immediately but rather can be
                     // run from the driving test code whenever it wants by running the EmbeddedEventLoop.
                     context.eventLoop.execute {
+                        let context = loopBoundContext.value
                         context.writeAndFlush(Self.wrapOutboundOut(.end(nil)), promise: nil)
                     }
                     XCTAssertEqual(.reqEndExpected, self.state)
diff --git a/Tests/NIOPosixTests/ChannelNotificationTest.swift b/Tests/NIOPosixTests/ChannelNotificationTest.swift
index cf0b963d5d..fe6fdf4729 100644
--- a/Tests/NIOPosixTests/ChannelNotificationTest.swift
+++ b/Tests/NIOPosixTests/ChannelNotificationTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -138,8 +138,8 @@ class ChannelNotificationTest: XCTestCase {
             XCTAssertNil(self.connectPromise)
             XCTAssertNil(self.closePromise)
 
-            promise!.futureResult.whenSuccess {
-                XCTAssertFalse(context.channel.isActive)
+            promise!.futureResult.whenSuccess { [channel = context.channel] in
+                XCTAssertFalse(channel.isActive)
             }
 
             self.registerPromise = promise
@@ -157,8 +157,8 @@ class ChannelNotificationTest: XCTestCase {
             XCTAssertNil(self.connectPromise)
             XCTAssertNil(self.closePromise)
 
-            promise!.futureResult.whenSuccess {
-                XCTAssertTrue(context.channel.isActive)
+            promise!.futureResult.whenSuccess { [channel = context.channel] in
+                XCTAssertTrue(channel.isActive)
             }
 
             self.connectPromise = promise
@@ -170,8 +170,8 @@ class ChannelNotificationTest: XCTestCase {
             XCTAssertNotNil(self.connectPromise)
             XCTAssertNil(self.closePromise)
 
-            promise!.futureResult.whenSuccess {
-                XCTAssertFalse(context.channel.isActive)
+            promise!.futureResult.whenSuccess { [channel = context.channel] in
+                XCTAssertFalse(channel.isActive)
             }
 
             self.closePromise = promise
@@ -248,8 +248,8 @@ class ChannelNotificationTest: XCTestCase {
             XCTAssertNil(self.registerPromise)
 
             let p = promise ?? context.eventLoop.makePromise()
-            p.futureResult.whenSuccess {
-                XCTAssertFalse(context.channel.isActive)
+            p.futureResult.whenSuccess { [channel = context.channel] in
+                XCTAssertFalse(channel.isActive)
             }
 
             self.registerPromise = p
@@ -354,8 +354,8 @@ class ChannelNotificationTest: XCTestCase {
             XCTAssertNil(self.closePromise)
 
             let p = promise ?? context.eventLoop.makePromise()
-            p.futureResult.whenSuccess {
-                XCTAssertFalse(context.channel.isActive)
+            p.futureResult.whenSuccess { [channel = context.channel] in
+                XCTAssertFalse(channel.isActive)
             }
 
             self.registerPromise = p
@@ -373,8 +373,8 @@ class ChannelNotificationTest: XCTestCase {
             XCTAssertNil(self.bindPromise)
             XCTAssertNil(self.closePromise)
 
-            promise?.futureResult.whenSuccess {
-                XCTAssertTrue(context.channel.isActive)
+            promise?.futureResult.whenSuccess { [channel = context.channel] in
+                XCTAssertTrue(channel.isActive)
             }
 
             self.bindPromise = promise
@@ -387,8 +387,8 @@ class ChannelNotificationTest: XCTestCase {
             XCTAssertNil(self.closePromise)
 
             let p = promise ?? context.eventLoop.makePromise()
-            p.futureResult.whenSuccess {
-                XCTAssertFalse(context.channel.isActive)
+            p.futureResult.whenSuccess { [channel = context.channel] in
+                XCTAssertFalse(channel.isActive)
             }
 
             self.closePromise = p
diff --git a/Tests/NIOPosixTests/ChannelPipelineTest.swift b/Tests/NIOPosixTests/ChannelPipelineTest.swift
index 9226d7a0a6..130e39fd2c 100644
--- a/Tests/NIOPosixTests/ChannelPipelineTest.swift
+++ b/Tests/NIOPosixTests/ChannelPipelineTest.swift
@@ -803,7 +803,9 @@ class ChannelPipelineTest: XCTestCase {
         buffer.writeStaticString("Hello, world!")
 
         let removalPromise = channel.eventLoop.makePromise(of: Void.self)
+        let loopBoundContext = context.loopBound
         removalPromise.futureResult.whenSuccess {
+            let context = loopBoundContext.value
             context.writeAndFlush(NIOAny(buffer), promise: nil)
             context.fireErrorCaught(DummyError())
         }
@@ -845,7 +847,9 @@ class ChannelPipelineTest: XCTestCase {
 
         XCTAssertNoThrow(XCTAssertNil(try channel.readOutbound()))
         XCTAssertNoThrow(try channel.throwIfErrorCaught())
+        let loopBoundContext = context.loopBound
         channel.pipeline.syncOperations.removeHandler(context: context).whenSuccess {
+            let context = loopBoundContext.value
             context.writeAndFlush(NIOAny(buffer), promise: nil)
             context.fireErrorCaught(DummyError())
         }
@@ -873,7 +877,9 @@ class ChannelPipelineTest: XCTestCase {
         buffer.writeStaticString("Hello, world!")
 
         let removalPromise = channel.eventLoop.makePromise(of: Void.self)
+        let loopBoundContext = context.loopBound
         removalPromise.futureResult.map {
+            let context = loopBoundContext._value
             context.writeAndFlush(NIOAny(buffer), promise: nil)
             context.fireErrorCaught(DummyError())
         }.whenFailure {
@@ -912,7 +918,9 @@ class ChannelPipelineTest: XCTestCase {
 
         XCTAssertNoThrow(XCTAssertNil(try channel.readOutbound()))
         XCTAssertNoThrow(try channel.throwIfErrorCaught())
+        let loopBoundContext = context.loopBound
         channel.pipeline.removeHandler(name: "TestHandler").whenSuccess {
+            let context = loopBoundContext.value
             context.writeAndFlush(NIOAny(buffer), promise: nil)
             context.fireErrorCaught(DummyError())
         }
@@ -941,7 +949,9 @@ class ChannelPipelineTest: XCTestCase {
         buffer.writeStaticString("Hello, world!")
 
         let removalPromise = channel.eventLoop.makePromise(of: Void.self)
+        let loopBoundContext = context.loopBound
         removalPromise.futureResult.whenSuccess {
+            let context = loopBoundContext.value
             context.writeAndFlush(NIOAny(buffer), promise: nil)
             context.fireErrorCaught(DummyError())
         }
@@ -979,7 +989,9 @@ class ChannelPipelineTest: XCTestCase {
 
         XCTAssertNoThrow(XCTAssertNil(try channel.readOutbound()))
         XCTAssertNoThrow(try channel.throwIfErrorCaught())
+        let loopBoundContext = context.loopBound
         channel.pipeline.removeHandler(handler).whenSuccess {
+            let context = loopBoundContext.value
             context.writeAndFlush(NIOAny(buffer), promise: nil)
             context.fireErrorCaught(DummyError())
         }
@@ -1403,7 +1415,9 @@ class ChannelPipelineTest: XCTestCase {
                 self.removeHandlerCalls += 1
                 XCTAssertEqual(1, self.removeHandlerCalls)
                 self.removalTriggeredPromise.succeed(())
+                let loopBoundContext = context.loopBound
                 self.continueRemovalFuture.whenSuccess {
+                    let context = loopBoundContext.value
                     context.leavePipeline(removalToken: removalToken)
                 }
             }
diff --git a/Tests/NIOPosixTests/ChannelTests.swift b/Tests/NIOPosixTests/ChannelTests.swift
index 8774c4f6a5..362260ebe0 100644
--- a/Tests/NIOPosixTests/ChannelTests.swift
+++ b/Tests/NIOPosixTests/ChannelTests.swift
@@ -2953,7 +2953,9 @@ public final class ChannelTests: XCTestCase {
             func channelActive(context: ChannelHandlerContext) {
                 var buffer = context.channel.allocator.buffer(capacity: 1)
                 buffer.writeStaticString("X")
-                context.channel.writeAndFlush(buffer).map { context.channel }.cascade(
+                context.channel.writeAndFlush(buffer).map { [channel = context.channel] in
+                    channel
+                }.cascade(
                     to: self.channelAvailablePromise
                 )
             }
@@ -3003,7 +3005,8 @@ public final class ChannelTests: XCTestCase {
 
             func channelActive(context: ChannelHandlerContext) {
                 XCTAssert(serverChannel.eventLoop === context.eventLoop)
-                self.serverChannel.whenSuccess { serverChannel in
+                let loopBoundContext = context.loopBound
+                self.serverChannel.whenSuccess { [channel = context.channel] serverChannel in
                     // all of the following futures need to complete synchronously for this test to test the correct
                     // thing. Therefore we keep track if we're still on the same stack frame.
                     var inSameStackFrame = true
@@ -3014,9 +3017,10 @@ public final class ChannelTests: XCTestCase {
                     XCTAssertTrue(serverChannel.isActive)
                     // we allow auto-read again to make sure that the socket buffer is drained on write error
                     // (cf. https://github.com/apple/swift-nio/issues/593)
-                    context.channel.setOption(.autoRead, value: true).flatMap {
+                    channel.setOption(.autoRead, value: true).flatMap {
+                        let context = loopBoundContext.value
                         // let's trigger the write error
-                        var buffer = context.channel.allocator.buffer(capacity: 16)
+                        var buffer = channel.allocator.buffer(capacity: 16)
                         buffer.writeStaticString("THIS WILL FAIL ANYWAY")
 
                         // this needs to be in a function as otherwise the Swift compiler believes this is throwing
@@ -3025,7 +3029,7 @@ public final class ChannelTests: XCTestCase {
                             // arrived at the time the write fails. So this is a hack that makes sure they do have arrived.
                             // (https://github.com/apple/swift-nio/issues/657)
                             XCTAssertNoThrow(
-                                try self.veryNasty_blockUntilReadBufferIsNonEmpty(channel: context.channel)
+                                try self.veryNasty_blockUntilReadBufferIsNonEmpty(channel: channel)
                             )
                         }
                         workaroundSR487()
@@ -3489,8 +3493,10 @@ private final class FailRegistrationAndDelayCloseHandler: ChannelOutboundHandler
     }
 
     func close(context: ChannelHandlerContext, mode: CloseMode, promise: EventLoopPromise<Void>?) {
+        let loopBoundContext = context.loopBound
         // for extra nastiness, let's delay close. This makes sure the ChannelPipeline correctly retains the Channel
         _ = context.eventLoop.scheduleTask(in: .milliseconds(10)) {
+            let context = loopBoundContext.value
             context.close(mode: mode, promise: promise)
         }
     }
@@ -3559,7 +3565,9 @@ final class ReentrantWritabilityChangingHandler: ChannelInboundHandler {
         // emitted. The flush for that write should result in the writability flipping back
         // again.
         let b1 = context.channel.allocator.buffer(repeating: 0, count: 50)
+        let loopBoundContext = context.loopBound
         context.write(Self.wrapOutboundOut(b1)).whenSuccess { _ in
+            let context = loopBoundContext.value
             // We should still be writable.
             XCTAssertTrue(context.channel.isWritable)
             XCTAssertEqual(self.isNotWritableCount, 0)
diff --git a/Tests/NIOPosixTests/EchoServerClientTest.swift b/Tests/NIOPosixTests/EchoServerClientTest.swift
index 2be0133d9d..678f6da805 100644
--- a/Tests/NIOPosixTests/EchoServerClientTest.swift
+++ b/Tests/NIOPosixTests/EchoServerClientTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -382,6 +382,7 @@ class EchoServerClientTest: XCTestCase {
 
     private final class EchoAndEchoAgainAfterSomeTimeServer: ChannelInboundHandler {
         typealias InboundIn = ByteBuffer
+        typealias InboundOut = ByteBuffer
         typealias OutboundOut = ByteBuffer
 
         private let timeAmount: TimeAmount
@@ -398,10 +399,13 @@ class EchoServerClientTest: XCTestCase {
         }
 
         func channelRead(context: ChannelHandlerContext, data: NIOAny) {
+            let bytes = Self.unwrapInboundIn(data)
             self.numberOfReads += 1
             precondition(self.numberOfReads == 1, "\(self) is only ever allowed to read once")
+            let loopBoundContext = context.loopBound
             _ = context.eventLoop.scheduleTask(in: self.timeAmount) {
-                context.writeAndFlush(data, promise: nil)
+                let context = loopBoundContext.value
+                context.writeAndFlush(Self.wrapInboundOut(bytes), promise: nil)
                 self.group.leave()
             }.futureResult.recover { e in
                 XCTFail("we failed to schedule the task: \(e)")
@@ -753,8 +757,10 @@ class EchoServerClientTest: XCTestCase {
             }
 
             private func writeUntilFailed(_ context: ChannelHandlerContext, _ buffer: ByteBuffer) {
-                context.writeAndFlush(NIOAny(buffer)).whenSuccess {
-                    context.eventLoop.execute {
+                let loopBoundContext = context.loopBound
+                context.writeAndFlush(NIOAny(buffer)).whenSuccess { [eventLoop = context.eventLoop] in
+                    eventLoop.execute {
+                        let context = loopBoundContext.value
                         self.writeUntilFailed(context, buffer)
                     }
                 }
@@ -776,14 +782,19 @@ class EchoServerClientTest: XCTestCase {
                 let buffer = context.channel.allocator.buffer(string: str)
 
                 // write it four times and then close the connect.
+                let loopBoundContext = context.loopBound
                 context.writeAndFlush(NIOAny(buffer)).flatMap {
-                    context.writeAndFlush(NIOAny(buffer))
+                    let context = loopBoundContext.value
+                    return context.writeAndFlush(NIOAny(buffer))
                 }.flatMap {
-                    context.writeAndFlush(NIOAny(buffer))
+                    let context = loopBoundContext.value
+                    return context.writeAndFlush(NIOAny(buffer))
                 }.flatMap {
-                    context.writeAndFlush(NIOAny(buffer))
+                    let context = loopBoundContext.value
+                    return context.writeAndFlush(NIOAny(buffer))
                 }.flatMap {
-                    context.close()
+                    let context = loopBoundContext.value
+                    return context.close()
                 }.whenComplete { (_: Result<Void, Error>) in
                     self.dpGroup.leave()
                 }
diff --git a/Tests/NIOPosixTests/EventLoopTest.swift b/Tests/NIOPosixTests/EventLoopTest.swift
index 4b0abd9530..f14ba2dc4f 100644
--- a/Tests/NIOPosixTests/EventLoopTest.swift
+++ b/Tests/NIOPosixTests/EventLoopTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -542,7 +542,9 @@ public final class EventLoopTest: XCTestCase {
                 }
                 XCTAssertTrue(context.channel.isActive)
                 self.closePromise = context.eventLoop.makePromise()
+                let loopBoundContext = context.loopBound
                 self.closePromise!.futureResult.whenSuccess {
+                    let context = loopBoundContext.value
                     context.close(mode: mode, promise: promise)
                 }
                 promiseRegisterCallback(self.closePromise!)
diff --git a/Tests/NIOPosixTests/IdleStateHandlerTest.swift b/Tests/NIOPosixTests/IdleStateHandlerTest.swift
index 87ae3d8cd7..04b81173cf 100644
--- a/Tests/NIOPosixTests/IdleStateHandlerTest.swift
+++ b/Tests/NIOPosixTests/IdleStateHandlerTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -37,7 +37,7 @@ class IdleStateHandlerTest: XCTestCase {
     }
 
     private func testIdle(
-        _ handler: IdleStateHandler,
+        _ handler: @escaping @Sendable @autoclosure () -> IdleStateHandler,
         _ writeToChannel: Bool,
         _ assertEventFn: @escaping (IdleStateHandler.IdleStateEvent) -> Bool
     ) throws {
@@ -86,6 +86,7 @@ class IdleStateHandlerTest: XCTestCase {
                 .serverChannelOption(.socketOption(.so_reuseaddr), value: 1)
                 .childChannelInitializer { channel in
                     channel.eventLoop.makeCompletedFuture {
+                        let handler = handler()
                         try channel.pipeline.syncOperations.addHandler(handler)
                         try channel.pipeline.syncOperations.addHandler(TestWriteHandler(writeToChannel, assertEventFn))
                     }
diff --git a/Tests/NIOPosixTests/NIOThreadPoolTest.swift b/Tests/NIOPosixTests/NIOThreadPoolTest.swift
index a1969f6204..ed39352df0 100644
--- a/Tests/NIOPosixTests/NIOThreadPoolTest.swift
+++ b/Tests/NIOPosixTests/NIOThreadPoolTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2020-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2020-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -79,15 +79,17 @@ class NIOThreadPoolTest: XCTestCase {
         // The lock here is arguably redundant with the dispatchgroup, but let's make
         // this test thread-safe even if I screw up.
         let lock = NIOLock()
-        var threadOne = Thread?.none
-        var threadTwo = Thread?.none
+        let threadOne: NIOLockedValueBox<Thread?> = NIOLockedValueBox(Thread?.none)
+        let threadTwo: NIOLockedValueBox<Thread?> = NIOLockedValueBox(Thread?.none)
 
         completionGroup.enter()
         pool.submit { s in
             precondition(s == .active)
             lock.withLock { () -> Void in
-                XCTAssertEqual(threadOne, nil)
-                threadOne = Thread.current
+                threadOne.withLockedValue { threadOne in
+                    XCTAssertEqual(threadOne, nil)
+                    threadOne = Thread.current
+                }
             }
             completionGroup.leave()
         }
@@ -98,8 +100,10 @@ class NIOThreadPoolTest: XCTestCase {
         pool.submit { s in
             precondition(s == .active)
             lock.withLock { () -> Void in
-                XCTAssertEqual(threadTwo, nil)
-                threadTwo = Thread.current
+                threadTwo.withLockedValue { threadTwo in
+                    XCTAssertEqual(threadTwo, nil)
+                    threadTwo = Thread.current
+                }
             }
             completionGroup.leave()
         }
@@ -109,7 +113,7 @@ class NIOThreadPoolTest: XCTestCase {
         lock.withLock { () -> Void in
             XCTAssertNotNil(threadOne)
             XCTAssertNotNil(threadTwo)
-            XCTAssertEqual(threadOne, threadTwo)
+            XCTAssertEqual(threadOne.withLockedValue { $0 }, threadTwo.withLockedValue { $0 })
         }
     }
 
diff --git a/Tests/NIOPosixTests/SocketChannelTest.swift b/Tests/NIOPosixTests/SocketChannelTest.swift
index ebd4787600..cb8161a7e5 100644
--- a/Tests/NIOPosixTests/SocketChannelTest.swift
+++ b/Tests/NIOPosixTests/SocketChannelTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -550,7 +550,9 @@ public final class SocketChannelTest: XCTestCase {
                 XCTAssertEqual(.inactive, state)
                 state = .removed
 
+                let loopBoundContext = context.loopBound
                 context.channel.closeFuture.whenComplete { (_: Result<Void, Error>) in
+                    let context = loopBoundContext.value
                     XCTAssertNil(context.localAddress)
                     XCTAssertNil(context.remoteAddress)
 
diff --git a/Tests/NIOPosixTests/StreamChannelsTest.swift b/Tests/NIOPosixTests/StreamChannelsTest.swift
index 277b88594c..072f35d41b 100644
--- a/Tests/NIOPosixTests/StreamChannelsTest.swift
+++ b/Tests/NIOPosixTests/StreamChannelsTest.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2019-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2019-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -486,9 +486,11 @@ class StreamChannelTest: XCTestCase {
                     // raise the high water mark so we don't get another call straight away.
                     var buffer = context.channel.allocator.buffer(capacity: 5)
                     buffer.writeString("hello")
+                    let loopBoundContext = context.loopBound
                     context.channel.setOption(.writeBufferWaterMark, value: .init(low: 1024, high: 1024))
                         .flatMap {
-                            context.writeAndFlush(Self.wrapOutboundOut(buffer))
+                            let context = loopBoundContext.value
+                            return context.writeAndFlush(Self.wrapOutboundOut(buffer))
                         }.whenFailure { error in
                             XCTFail("unexpected error: \(error)")
                         }
@@ -638,7 +640,9 @@ class StreamChannelTest: XCTestCase {
                     XCTFail("unexpected error \(error)")
                 }
 
+                let loopBoundContext = context.loopBound
                 context.eventLoop.execute {
+                    let context = loopBoundContext.value
                     self.kickOff(context: context)
                 }
             }
@@ -648,26 +652,34 @@ class StreamChannelTest: XCTestCase {
             }
 
             func kickOff(context: ChannelHandlerContext) {
-                var buffer = context.channel.allocator.buffer(capacity: self.chunkSize)
-                buffer.writeBytes(Array(repeating: UInt8(ascii: "0"), count: chunkSize))
+                let buffer = NIOLoopBoundBox(
+                    context.channel.allocator.buffer(capacity: self.chunkSize),
+                    eventLoop: context.eventLoop
+                )
+                buffer.value.writeBytes(Array(repeating: UInt8(ascii: "0"), count: chunkSize))
 
+                let loopBoundContext = context.loopBound
+                let loopBoundSelf = NIOLoopBound(self, eventLoop: context.eventLoop)
                 func writeOneMore() {
-                    self.bytesWritten += buffer.readableBytes
-                    context.writeAndFlush(Self.wrapOutboundOut(buffer)).whenFailure { error in
+                    let context = loopBoundContext.value
+                    self.bytesWritten += buffer.value.readableBytes
+                    context.writeAndFlush(Self.wrapOutboundOut(buffer.value)).whenFailure { error in
                         XCTFail("unexpected error \(error)")
                     }
                     context.eventLoop.scheduleTask(in: .microseconds(100)) {
+                        let context = loopBoundContext.value
                         switch self.state {
                         case .writingUntilFull:
                             // We're just enqueuing another chunk.
                             writeOneMore()
                         case .writeSentinel:
+                            let `self` = loopBoundSelf.value
                             // We've seen the notification that the channel is unwritable, let's write one more byte.
-                            buffer.clear()
-                            buffer.writeString("1")
+                            buffer.value.clear()
+                            buffer.value.writeString("1")
                             self.state = .done
                             self.bytesWritten += 1
-                            context.writeAndFlush(Self.wrapOutboundOut(buffer)).whenFailure { error in
+                            context.writeAndFlush(Self.wrapOutboundOut(buffer.value)).whenFailure { error in
                                 XCTFail("unexpected error \(error)")
                             }
                             self.wroteEnoughToBeStuckPromise.succeed(self.bytesWritten)
@@ -692,7 +704,9 @@ class StreamChannelTest: XCTestCase {
                     ()  // ignored, we're done
                 }
                 context.fireChannelWritabilityChanged()
+                let loopBoundContext = context.loopBound
                 self.wroteEnoughToBeStuckPromise.futureResult.whenSuccess { _ in
+                    let context = loopBoundContext.value
                     context.pipeline.removeHandler(self).whenFailure { error in
                         XCTFail("unexpected error \(error)")
                     }

From 848e42899027d1b47219bf513cf771fcfd14e4eb Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Mon, 25 Nov 2024 18:15:18 +0000
Subject: [PATCH 14/37] deprecate NIOEventLoopGroupProvider.createNew (#2480)

### Motivation:

`NIOEventLoopGroupProvider.createNew` was probably never a good idea
because it creates shutdown issues for any library that uses it. Given
that we now have singleton (#2471) `EventLoopGroup`s, we can solve this
issue by just not having event loop group providers.

Users can just use `group: any EventLoopGroup` and optionally `group:
any EventLoopGroup = MultiThreadedEventLoopGroup.singleton`.

### Modifications:

- deprecate `NIOEventLoopGroupProvider.createNew`
- soft-deprecate (document as deprecated but don't mark
`@available(deprecated)`) `NIOEventLoopGroupProvider`

### Result:

- Libraries becomes easier to write and maintain.
- Fixes #2142
---
 Sources/NIOCore/Docs.docc/index.md |  1 -
 Sources/NIOCore/EventLoop.swift    | 21 +++++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/Sources/NIOCore/Docs.docc/index.md b/Sources/NIOCore/Docs.docc/index.md
index 13d76894ec..5b437b073e 100644
--- a/Sources/NIOCore/Docs.docc/index.md
+++ b/Sources/NIOCore/Docs.docc/index.md
@@ -21,7 +21,6 @@ More specialized modules provide concrete implementations of many of the abstrac
 
 - ``EventLoopGroup``
 - ``EventLoop``
-- ``NIOEventLoopGroupProvider``
 - ``EventLoopIterator``
 - ``Scheduled``
 - ``RepeatedTask``
diff --git a/Sources/NIOCore/EventLoop.swift b/Sources/NIOCore/EventLoop.swift
index 047c06e67e..376f06d8fb 100644
--- a/Sources/NIOCore/EventLoop.swift
+++ b/Sources/NIOCore/EventLoop.swift
@@ -1246,16 +1246,33 @@ extension EventLoopGroup {
 }
 #endif
 
-/// This type is intended to be used by libraries which use NIO, and offer their users either the option
+/// Deprecated.
+///
+/// This type was intended to be used by libraries which use NIO, and offer their users either the option
 /// to `.share` an existing event loop group or create (and manage) a new one (`.createNew`) and let it be
 /// managed by given library and its lifecycle.
+///
+/// Please use a `group: any EventLoopGroup` parameter instead. If you want to default to a global
+/// singleton group instead, consider group: any EventLoopGroup = MultiThreadedEventLoopGroup.singleton` or
+/// similar.
+///
+/// - See also: https://github.com/apple/swift-nio/issues/2142
 public enum NIOEventLoopGroupProvider {
     /// Use an `EventLoopGroup` provided by the user.
     /// The owner of this group is responsible for its lifecycle.
     case shared(EventLoopGroup)
-    /// Create a new `EventLoopGroup` when necessary.
+
+    /// Deprecated. Create a new `EventLoopGroup` when necessary.
     /// The library which accepts this provider takes ownership of the created event loop group,
     /// and must ensure its proper shutdown when the library is being shut down.
+    @available(
+        *,
+        deprecated,
+        message: """
+            Please use `.shared(existingGroup)` or use the singleton via \
+            `.shared(MultiThreadedEventLoopGroup.singleton)` or similar
+            """
+    )
     case createNew
 }
 

From d6be946626e395deb740ca83f99f5d2ec752f0e3 Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Tue, 26 Nov 2024 11:36:30 +0000
Subject: [PATCH 15/37] ByteBuffer: one fewer allocs to go to Data (#1839)

Motivation:

When getting a `Data` from a `ByteBuffer` we currently allocate twice
(`__DataStorage`) and the closure for `Data.Deallocator`.

Modifications:

We can optimise that by making the closure capture exactly one
`AnyObject` which is already a reference counted object. The compiler
realises that (on Linux) and saves us an alloc.

Thanks @lukasa for the suggestion here:
https://github.com/apple/swift-nio/pull/1836#issuecomment-830044155

Result:

Fewer allocs.
---
 Sources/NIOFoundationCompat/ByteBuffer-foundation.swift | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift b/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift
index 8b221d75cb..cf1a25830a 100644
--- a/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift
+++ b/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift
@@ -2,7 +2,7 @@
 //
 // This source file is part of the SwiftNIO open source project
 //
-// Copyright (c) 2017-2021 Apple Inc. and the SwiftNIO project authors
+// Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors
 // Licensed under Apache License v2.0
 //
 // See LICENSE.txt for license information
@@ -126,11 +126,11 @@ extension ByteBuffer {
                     count: Int(length)
                 )
             } else {
-                _ = storageRef.retain()
+                let storage = storageRef.takeUnretainedValue()
                 return Data(
                     bytesNoCopy: UnsafeMutableRawPointer(mutating: ptr.baseAddress!.advanced(by: index)),
                     count: Int(length),
-                    deallocator: .custom { _, _ in storageRef.release() }
+                    deallocator: .custom { _, _ in withExtendedLifetime(storage) {} }
                 )
             }
         }

From 4d2fb577b9c0430d1c56478a1bf6f9f4c73942e4 Mon Sep 17 00:00:00 2001
From: Michael Gecht <github@gecht.net>
Date: Tue, 26 Nov 2024 14:48:29 +0000
Subject: [PATCH 16/37] Update documentation comments (#2998)

Updates to documentation comments in `NIOFileSystem`.

### Motivation:

I've been reviewing some parts of `NIOFileSystem` and came across a
variety of formatting of documentation comments.

### Modifications:

I've adopted a stricter 100-character limit (arbitrary, really; it could
be longer or shorter and is totally not consistent with the rest of the
repository). Also updated a bunch of grammar in comments. Fixed a few
typos along the way.

### Result:

- Updated comments have a better flow while reading, and aren't as
abrupt as before.
- Fewer typos in comments

---------

Co-authored-by: Cory Benfield <lukasa@apple.com>
---
 CONTRIBUTING.md                               |   7 +-
 Package.swift                                 |   2 +-
 README.md                                     |   6 +-
 Sources/NIOFileSystem/CopyStrategy.swift      |  51 +++---
 Sources/NIOFileSystem/FileSystem.swift        | 168 +++++++++---------
 .../Internal/ParallelDirCopy.swift            |  63 +++----
 Sources/NIOPosix/NIOThreadPool.swift          |   4 +-
 7 files changed, 150 insertions(+), 151 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7d1311924f..05b102e75d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,7 +5,6 @@ your contribution to Apple and the community, and agree by submitting the patch
 that your contributions are licensed under the Apache 2.0 license (see
 `LICENSE.txt`).
 
-
 ## How to submit a bug report
 
 Please ensure to specify the following:
@@ -20,7 +19,6 @@ Please ensure to specify the following:
   * OS version and the output of `uname -a`
   * Network configuration
 
-
 ### Example
 
 ```
@@ -89,7 +87,7 @@ SwiftNIO has been created to be high performance.  The integration tests cover s
 
 ### Formatting
 
-Try to keep your lines less than 120 characters long so github can correctly display your changes.
+Try to keep your lines less than 120 characters long so GitHub can correctly display your changes.
 
 SwiftNIO uses the [swift-format](https://github.com/swiftlang/swift-format) tool to bring consistency to code formatting.  There is a specific [.swift-format](./.swift-format) configuration file.  This will be checked and enforced on PRs.  Note that the check will run on the current most recent stable version target which may not match that in your own local development environment.
 
@@ -106,7 +104,6 @@ act --container-architecture linux/amd64 --action-offline-mode --bind workflow_c
 
 This will run the format checks, binding to your local checkout so the edits made are to your own source.
 
-
 ### Extensibility
 
 Try to make sure your code is robust to future extensions.  The public interface is very hard to change after release - please refer to the [API guidelines](./docs/public-api.md)
@@ -115,4 +112,4 @@ Try to make sure your code is robust to future extensions.  The public interface
 
 Please open a pull request at https://github.com/apple/swift-nio. Make sure the CI passes, and then wait for code review.
 
-After review you may be asked to make changes.  When you are ready, use the request re-review feature of github or mention the reviewers by name in a comment.
+After review you may be asked to make changes.  When you are ready, use the request re-review feature of GitHub or mention the reviewers by name in a comment.
diff --git a/Package.swift b/Package.swift
index 4bbbe48e72..021ebf8e50 100644
--- a/Package.swift
+++ b/Package.swift
@@ -23,7 +23,7 @@ let swiftAtomics: PackageDescription.Target.Dependency = .product(name: "Atomics
 let swiftCollections: PackageDescription.Target.Dependency = .product(name: "DequeModule", package: "swift-collections")
 let swiftSystem: PackageDescription.Target.Dependency = .product(name: "SystemPackage", package: "swift-system")
 
-// These platforms require a depdency on `NIOPosix` from `NIOHTTP1` to maintain backward
+// These platforms require a dependency on `NIOPosix` from `NIOHTTP1` to maintain backward
 // compatibility with previous NIO versions.
 let historicalNIOPosixDependencyRequired: [Platform] = [.macOS, .iOS, .tvOS, .watchOS, .linux, .android]
 
diff --git a/README.md b/README.md
index 055c5359c3..48d8b24974 100644
--- a/README.md
+++ b/README.md
@@ -86,7 +86,7 @@ SwiftNIO            | Minimum Swift Version
 `2.43.0 ..< 2.51.0` | 5.5.2
 `2.51.0 ..< 2.60.0` | 5.6
 `2.60.0 ..< 2.65.0` | 5.7
-`2.65.0 ..< 2.76.0  | 5.8
+`2.65.0 ..< 2.76.0` | 5.8
 `2.76.0 ...`        | 5.9
 
 ### SwiftNIO 1
@@ -116,7 +116,7 @@ SemVer and SwiftNIO's Public API guarantees should result in a working program w
 
 SwiftNIO is fundamentally a low-level tool for building high-performance networking applications in Swift. It particularly targets those use-cases where using a "thread-per-connection" model of concurrency is inefficient or untenable. This is a common limitation when building servers that use a large number of relatively low-utilization connections, such as HTTP servers.
 
-To achieve its goals SwiftNIO extensively uses "non-blocking I/O": hence the name! Non-blocking I/O differs from the more common blocking I/O model because the application does not wait for data to be sent to or received from the network: instead, SwiftNIO asks for the kernel to notify it when I/O operations can be performed without waiting.
+To achieve its goals, SwiftNIO extensively uses "non-blocking I/O": hence the name! Non-blocking I/O differs from the more common blocking I/O model because the application does not wait for data to be sent to or received from the network: instead, SwiftNIO asks for the kernel to notify it when I/O operations can be performed without waiting.
 
 SwiftNIO does not aim to provide high-level solutions like, for example, web frameworks do. Instead, SwiftNIO is focused on providing the low-level building blocks for these higher-level applications. When it comes to building a web application, most users will not want to use SwiftNIO directly: instead, they'll want to use one of the many great web frameworks available in the Swift ecosystem. Those web frameworks, however, may choose to use SwiftNIO under the covers to provide their networking support.
 
@@ -139,7 +139,7 @@ All SwiftNIO applications are ultimately constructed of these various components
 
 #### EventLoops and EventLoopGroups
 
-The basic I/O primitive of SwiftNIO is the event loop. The event loop is an object that waits for events (usually I/O related events, such as "data received") to happen and then fires some kind of callback when they do. In almost all SwiftNIO applications there will be relatively few event loops: usually only one or two per CPU core the application wants to use. Generally speaking event loops run for the entire lifetime of your application, spinning in an endless loop dispatching events.
+The basic I/O primitive of SwiftNIO is the event loop. The event loop is an object that waits for events (usually I/O related events, such as "data received") to happen and then fires some kind of callback when they do. In almost all SwiftNIO applications there will be relatively few event loops: usually only one or two per CPU core the application wants to use. Generally speaking, event loops run for the entire lifetime of your application, spinning in an endless loop dispatching events.
 
 Event loops are gathered together into event loop *groups*. These groups provide a mechanism to distribute work around the event loops. For example, when listening for inbound connections the listening socket will be registered on one event loop. However, we don't want all connections that are accepted on that listening socket to be registered with the same event loop, as that would potentially overload one event loop while leaving the others empty. For that reason, the event loop group provides the ability to spread load across multiple event loops.
 
diff --git a/Sources/NIOFileSystem/CopyStrategy.swift b/Sources/NIOFileSystem/CopyStrategy.swift
index 790206daef..ae8b4770e1 100644
--- a/Sources/NIOFileSystem/CopyStrategy.swift
+++ b/Sources/NIOFileSystem/CopyStrategy.swift
@@ -13,16 +13,16 @@
 //===----------------------------------------------------------------------===//
 
 /// How to perform copies. Currently only relevant to directory level copies when using
-/// ``FileSystemProtocol/copyItem(at:to:strategy:shouldProceedAfterError:shouldCopyItem:)``
-/// or  other overloads that use the default behaviour
+/// ``FileSystemProtocol/copyItem(at:to:strategy:shouldProceedAfterError:shouldCopyItem:)`` or other
+/// overloads that use the default behaviour.
 public struct CopyStrategy: Hashable, Sendable {
-    // Avoid exposing to prevent alterations being breaking changes
+    // Avoid exposing to prevent breaking changes
     internal enum Wrapped: Hashable, Sendable {
         // platformDefault is reified into one of the concrete options below:
 
         case sequential
-        // Constraints on this value are enforced only on making `CopyStrategy`,
-        // the early error check there is desirable over validating on downstream use.
+        // Constraints on this value are enforced only on creation of `CopyStrategy`. The early
+        // error check is desirable over validating on downstream use.
         case parallel(_ maxDescriptors: Int)
     }
 
@@ -33,14 +33,13 @@ public struct CopyStrategy: Hashable, Sendable {
 
     // These selections are relatively arbitrary but the rationale is as follows:
     //
-    // - Never exceed the default OS limits even if 4 such operations
-    //   were happening at once
+    // - Never exceed the default OS limits even if 4 such operations were happening at once.
     // - Sufficient to enable significant speed up from parallelism
-    // - Not wasting effort by pushing contention to the underlying storage device
-    //   Further we assume an SSD or similar underlying storage tech.
-    //   Users on spinning rust need to account for that themselves anyway
+    // - Not wasting effort by pushing contention to the underlying storage device. Further we
+    //   assume an SSD or similar underlying storage tech. Users on spinning rust need to account
+    //   for that themselves anyway.
     //
-    // That said, empirical testing for this has not been performed, suggestions welcome
+    // That said, empirical testing for this has not been performed, suggestions welcome.
     //
     // Note: for now we model the directory scan as needing two handles because, during the creation
     // of the destination directory we hold the handle for a while copying attributes
@@ -48,20 +47,22 @@ public struct CopyStrategy: Hashable, Sendable {
     // This may not result in a faster copy though so things are left simple
     internal static func determinePlatformDefault() -> Wrapped {
         #if os(macOS) || os(Linux) || os(Windows)
-        // 4 concurrent file copies/directory scans.
-        // Avoiding storage system contention is the dominant aspect here.
-        // Empirical testing on an SSD copying to the same volume with a dense directory of small
-        // files and sub directories of similar shape totalling 12GB showed improvements in elapsed
-        // time for (expected) increases in CPU time up to parallel(8), beyond this the increases
-        // in CPU came with only moderate gains.
+        // 4 concurrent file copies/directory scans. Avoiding storage system contention is of utmost
+        // importance.
+        //
+        // Testing was performed on an SSD, while copying objects (a dense directory of small files
+        // and subdirectories of similar shape) to the same volume, totalling 12GB. Results showed
+        // improvements in elapsed time for (expected) increases in CPU time up to parallel(8).
+        // Beyond this, the increases in CPU led to only moderate gains.
+        //
         // Anyone tuning this is encouraged to cover worst case scenarios.
         return .parallel(8)
         #elseif os(iOS) || os(tvOS) || os(watchOS) || os(Android)
-        // Reduced maximum descriptors in embedded world
-        // This is chosen based on biasing to safety, not empirical testing.
+        // Reduced maximum descriptors in embedded world. This is chosen based on biasing towards
+        // safety, not empirical testing.
         return .parallel(4)
         #else
-        // Safety first, if we have no view on it keep it simple.
+        // Safety first. If we do not know what system we run on, we keep it simple.
         return .sequential
         #endif
     }
@@ -69,19 +70,19 @@ public struct CopyStrategy: Hashable, Sendable {
 
 extension CopyStrategy {
     // A copy fundamentally can't work without two descriptors unless you copy
-    // everything into memory which is infeasible/inefficeint for large copies.
+    // everything into memory which is infeasible/inefficient for large copies.
     private static let minDescriptorsAllowed = 2
 
-    /// Operate in whatever manner is deemed a reasonable default for the platform.
-    /// This will limit the maximum file descriptors usage based on 'reasonable' defaults.
+    /// Operate in whatever manner is deemed a reasonable default for the platform. This will limit
+    /// the maximum file descriptors usage based on reasonable defaults.
     ///
     /// Current assumptions (which are subject to change):
     /// - Only one copy operation would be performed at once
     /// - The copy operation is not intended to be the primary activity on the device
     public static let platformDefault: Self = Self(Self.determinePlatformDefault())
 
-    /// The copy is done asynchronously, but only one operation will occur at a time.
-    /// This is the only way to guarantee only one callback to the `shouldCopyItem` will happen at a time
+    /// The copy is done asynchronously, but only one operation will occur at a time. This is the
+    /// only way to guarantee only one callback to the `shouldCopyItem` will happen at a time.
     public static let sequential: Self = Self(.sequential)
 
     /// Allow multiple IO operations to run concurrently, including file copies/directory creation and scanning
diff --git a/Sources/NIOFileSystem/FileSystem.swift b/Sources/NIOFileSystem/FileSystem.swift
index 76a1bcb8a9..33bf8c605e 100644
--- a/Sources/NIOFileSystem/FileSystem.swift
+++ b/Sources/NIOFileSystem/FileSystem.swift
@@ -32,12 +32,11 @@ import Bionic
 ///
 /// ### Creating a `FileSystem`
 ///
-/// You should prefer using the `shared` instance of the file system. The
-/// `shared` instance uses two threads unless the `SWIFT_FILE_SYSTEM_THREAD_COUNT`
-/// environment variable is set.
+/// You should prefer using the `shared` instance of the file system. The `shared` instance uses two
+/// threads unless the `SWIFT_FILE_SYSTEM_THREAD_COUNT` environment variable is set.
 ///
-/// If you require more granular control you can create a ``FileSystem`` with the required number
-/// of threads by calling ``withFileSystem(numberOfThreads:_:)`` or by using ``init(threadPool:)``.
+/// If you require more granular control you can create a ``FileSystem`` with the required number of
+/// threads by calling ``withFileSystem(numberOfThreads:_:)`` or by using ``init(threadPool:)``.
 ///
 /// ### Errors
 ///
@@ -54,8 +53,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     /// Returns a shared global instance of the ``FileSystem``.
     ///
     /// The file system executes blocking work in a thread pool which defaults to having two
-    /// threads. This can be modified by `blockingPoolThreadCountSuggestion` or by
-    /// setting the `NIO_SINGLETON_BLOCKING_POOL_THREAD_COUNT` environment variable.
+    /// threads. This can be modified by `blockingPoolThreadCountSuggestion` or by setting the
+    /// `NIO_SINGLETON_BLOCKING_POOL_THREAD_COUNT` environment variable.
     public static var shared: FileSystem { globalFileSystem }
 
     private let threadPool: NIOThreadPool
@@ -70,8 +69,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     /// Creates a new ``FileSystem`` using the provided thread pool.
     ///
     /// - Parameter threadPool: A started thread pool to execute blocking system calls on. The
-    ///     ``FileSystem`` doesn't take ownership of the thread pool and you remain responsible
-    ///     for shutting it down when necessary.
+    ///     ``FileSystem`` doesn't take ownership of the thread pool and you remain responsible for
+    ///     shutting it down when necessary.
     public init(threadPool: NIOThreadPool) {
         self.init(threadPool: threadPool, ownsThreadPool: false)
     }
@@ -217,8 +216,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     /// Error codes thrown include:
     /// - ``FileSystemError/Code-swift.struct/fileAlreadyExists`` if a file or directory already
     ///   exists .
-    /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if a component in the `path`
-    ///   prefix does not exist and `createIntermediateDirectories` is `false`.
+    /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if a component in the `path` prefix
+    ///   does not exist and `createIntermediateDirectories` is `false`.
     ///
     /// #### Implementation details
     ///
@@ -248,10 +247,10 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     /// #### Errors
     ///
     /// Error codes thrown include:
-    /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if the template doesn't end
-    ///   in at least 3 'X's.
-    /// - ``FileSystemError/Code-swift.struct/permissionDenied`` if the user doesn't have
-    ///  permission to create a directory at the path specified in the template.
+    /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if the template doesn't end in at
+    ///   least 3 'X's.
+    /// - ``FileSystemError/Code-swift.struct/permissionDenied`` if the user doesn't have permission
+    ///  to create a directory at the path specified in the template.
     ///
     /// #### Implementation details
     ///
@@ -277,9 +276,9 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     ///
     /// - Parameters:
     ///    - path: The path of the file.
-    ///    - infoAboutSymbolicLink: If the file is a symbolic link and this parameter is `true` then information
-    ///        about the link will be returned, otherwise information about the destination of the
-    ///        symbolic link is returned.
+    ///    - infoAboutSymbolicLink: If the file is a symbolic link and this parameter is `true`,
+    ///        then information about the link will be returned. Otherwise, information about the
+    ///        destination of the symbolic link is returned.
     /// - Returns: Information about the file at the given path or `nil` if no file exists.
     public func info(
         forFileAt path: FilePath,
@@ -299,18 +298,18 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     /// - symbolic link, or
     /// - directory.
     ///
-    /// But `shouldCopyItem` can be used to ignore things outside this supported set.
+    /// `shouldCopyItem` can be used to ignore objects not part of this set.
     ///
     /// #### Errors
     ///
     /// In addition to the already documented errors these may be thrown
-    /// - ``FileSystemError/Code-swift.struct/unsupported`` if an item to be copied is not a
-    ///   regular file, symbolic link or directory.
+    /// - ``FileSystemError/Code-swift.struct/unsupported`` if an item to be copied is not a regular
+    ///   file, symbolic link or directory.
     ///
     /// #### Implementation details
     ///
-    /// This function is platform dependent. On Darwin the `copyfile(2)` system call is
-    /// used and items are cloned where possible. On Linux the `sendfile(2)` system call is used.
+    /// This function is platform dependent. On Darwin the `copyfile(2)` system call is used and
+    /// items are cloned where possible. On Linux the `sendfile(2)` system call is used.
     public func copyItem(
         at sourcePath: FilePath,
         to destinationPath: FilePath,
@@ -334,8 +333,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
             )
         }
 
-        // By doing this before looking at the type we allow callers to decide whether
-        // unanticipated kinds of entries can be safely ignored without needing changes upstream
+        // By doing this before looking at the type, we allow callers to decide whether
+        // unanticipated kinds of entries can be safely ignored without needing changes upstream.
         if await shouldCopyItem(.init(path: sourcePath, type: info.type)!, destinationPath) {
             switch info.type {
             case .regular:
@@ -400,7 +399,7 @@ public struct FileSystem: Sendable, FileSystemProtocol {
 
         switch result {
         case .success:
-            // Great; we removed 1 whole item.
+            // Great; we removed an entire item.
             return 1
 
         case .failure(.noSuchFileOrDirectory):
@@ -413,8 +412,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
                     code: .notEmpty,
                     message: """
                         Can't remove directory at path '\(path)', it isn't empty and \
-                        'removeItemRecursively' is false. Remove items from the directory first \
-                        or set 'removeItemRecursively' to true when calling \
+                        'removeItemRecursively' is false. Remove items from the directory first or \
+                        set 'removeItemRecursively' to true when calling \
                         'removeItem(at:recursively:)'.
                         """,
                     cause: nil,
@@ -674,8 +673,8 @@ private let globalFileSystem: FileSystem = {
 extension NIOSingletons {
     /// Returns a shared global instance of the ``FileSystem``.
     ///
-    /// The file system executes blocking work in a thread pool see `blockingPoolThreadCountSuggestion`
-    /// for the default behaviour and ways to control it.
+    /// The file system executes blocking work in a thread pool. See
+    /// `blockingPoolThreadCountSuggestion` for the default behaviour and ways to control it.
     public static var fileSystem: FileSystem { globalFileSystem }
 }
 
@@ -737,7 +736,8 @@ extension FileSystem {
         }
     }
 
-    /// Opens `path` for reading and writing and returns ``ReadWriteFileHandle`` or ``FileSystemError``.
+    /// Opens `path` for reading and writing and returns ``ReadWriteFileHandle`` or
+    /// ``FileSystemError``.
     private func _openFile(
         forReadingAndWritingAt path: FilePath,
         options: OpenOptions.Write
@@ -777,15 +777,16 @@ extension FileSystem {
         withIntermediateDirectories createIntermediateDirectories: Bool,
         permissions: FilePermissions
     ) -> Result<Void, FileSystemError> {
-        // Logic, assuming we are creating intermediate directories: try creating the directory,
-        // if it fails with ENOENT (no such file or directory) then drop the last component and
-        // append it to a buffer. Repeat until the path is empty meaning we cannot create the
-        // directory, or we succeed in which case we can append build up our original path
-        // creating directories one at a time.
+        // We assume that we will be creating intermediate directories:
+        // - Try creating the directory. If it fails with ENOENT (no such file or directory), then
+        //   drop the last component and append it to a buffer.
+        // - Repeat until the path is empty. This means we cannot create the directory or we
+        //   succeed, in which case we can build up our original path and create directories one at
+        //   a time.
         var droppedComponents: [FilePath.Component] = []
         var path = fullPath
 
-        // Normalize the path to remove any '..' which may not be necessary.
+        // Normalize the path to remove any superflous '..'.
         path.lexicallyNormalize()
 
         if path.isEmpty {
@@ -857,28 +858,24 @@ extension FileSystem {
         }
     }
 
-    /// Represents an item in a directory that needs copying, or
-    /// an explicit indication of the end of items.
-    /// The provision of the ``endOfDir`` case significantly simplifies the parallel code
+    /// Represents an item in a directory that needs copying, or an explicit indication of the end
+    /// of items. The provision of the ``endOfDir`` case significantly simplifies the parallel code
     enum DirCopyItem: Hashable, Sendable {
         case endOfDir
         case toCopy(from: DirectoryEntry, to: FilePath)
     }
 
-    /// Creates the directory ``destinationPath`` based on the directory at ``sourcePath``
-    /// including any permissions/attributes.
-    /// It does not copy the contents but does indicate the items directly within ``sourcePath`` which
-    /// should be copied
-    ///
-    /// This is a little cumbersome because it is used by ``copyDirectorySequential`` and
-    /// ``copyDirectoryParallel``.
-    /// It is desirable to use the file descriptor for the directory itself for as little time as possible
-    /// (certainly not across async invocations).
-    /// The down stream paths in the parallel and sequential paths are very different
-    /// - Returns:
-    ///     An array of `DirCopyItem` which have passed the ``shouldCopyItem```filter
-    ///     The target file paths will all be in ``destinationPath``
-    ///     The array will always finish with an ``DirCopyItem.endOfDir``
+    /// Creates the directory ``destinationPath`` based on the directory at ``sourcePath`` including
+    /// any permissions/attributes. It does not copy the contents but indicates the items within
+    /// ``sourcePath`` which should be copied.
+    ///
+    /// This is a little cumbersome, because it is used by ``copyDirectorySequential`` and
+    /// ``copyDirectoryParallel``. It is desirable to use the directories' file descriptor for as
+    /// little time as possible, and certainly not across asynchronous invocations. The downstream
+    /// paths in the parallel and sequential paths are very different
+    /// - Returns: An array of `DirCopyItem` which have passed the ``shouldCopyItem``` filter. The
+    ///     target file paths will all be in ``destinationPath``. The array will always finish with
+    ///     an ``DirCopyItem.endOfDir``.
     private func prepareDirectoryForRecusiveCopy(
         from sourcePath: FilePath,
         to destinationPath: FilePath,
@@ -917,26 +914,24 @@ extension FileSystem {
                     }
                 }
             } catch let error as FileSystemError where error.code == .unsupported {
-                // Not all file systems support extended attributes. Swallow errors which indicate
-                // that is the case.
+                // Not all file systems support extended attributes. Swallow errors indicating this.
                 ()
             }
             #endif
-            // Build a list of items the caller needs to deal with,
-            // they then do any further work after closing the current directory
+            // Build a list of items the caller needs to deal with, then do any further work after
+            // closing the current directory.
             var contentsToCopy = [DirCopyItem]()
 
             for try await batch in dir.listContents().batched() {
                 for entry in batch {
-                    // Any further work is pointless, we are under no obligation to cleanup
-                    // so exit as fast and cleanly as possible.
+                    // Any further work is pointless. We are under no obligation to cleanup. Exit as
+                    // fast and cleanly as possible.
                     try Task.checkCancellation()
                     let entryDestination = destinationPath.appending(entry.name)
 
                     if await shouldCopyItem(entry, entryDestination) {
-                        // Assume there's a good chance of everything in the batch
-                        // being included in the common case.
-                        // Let geometric growth go from this point though.
+                        // Assume there's a good chance of everything in the batch being included in
+                        // the common case. Let geometric growth go from this point though.
                         if contentsToCopy.isEmpty {
                             // Reserve space for the endOfDir entry too.
                             contentsToCopy.reserveCapacity(batch.count + 1)
@@ -967,8 +962,8 @@ extension FileSystem {
         }
     }
 
-    /// This could be achieved through quite complicated special casing of the parallel copy.
-    /// The resulting code is far harder to read and debug though so this is kept as a special case
+    /// This could be achieved through quite complicated special casing of the parallel copy. The
+    /// resulting code is far harder to read and debug, so this is kept as a special case.
     private func copyDirectorySequential(
         from sourcePath: FilePath,
         to destinationPath: FilePath,
@@ -981,9 +976,9 @@ extension FileSystem {
             _ destination: FilePath
         ) async -> Bool
     ) async throws {
-        // Strategy: find all needed items to copy/recurse into while the directory is open;
-        // defer actual copying and recursion until after the source directory has been closed
-        // to avoid consuming too many file descriptors.
+        // Strategy: find all needed items to copy/recurse into while the directory is open; defer
+        // actual copying and recursion until after the source directory has been closed to avoid
+        // consuming too many file descriptors.
         let toCopy = try await self.prepareDirectoryForRecusiveCopy(
             from: sourcePath,
             to: destinationPath,
@@ -997,9 +992,9 @@ extension FileSystem {
                 // Sequential cases doesn't need to worry about this, it uses simple recursion.
                 continue
             case let .toCopy(source, destination):
-                // Note: The entry type could have changed between finding it and acting on it.
-                // This is inherent in file systems, just more likely in an async environment
-                // we just accept those coming through as regular errors.
+                // Note: The entry type could have changed between finding it and acting on it. This
+                // is inherent in file systems, just more likely in an asynchronous environment. We
+                // just accept those coming through as regular errors.
                 switch source.type {
                 case .regular:
                     do {
@@ -1069,12 +1064,14 @@ extension FileSystem {
                 shouldCopyItem: shouldCopyItem
             )
         case let .parallel(maxDescriptors):
-            // Note that maxDescriptors was validated on construction of CopyStrategy.
-            // See notes on CopyStrategy about assumptions on descriptor use.
-            // For now we take the worst case peak for every operation, which is 2 descriptors,
-            // this keeps the downstream limiting code simple
-            // We do not preclude the use of more granular limiting in future (e.g. a directory
-            // scan only requires 1), for now we just drop any excess remainder entirely.
+            // Note that maxDescriptors was validated on construction of CopyStrategy. See notes on
+            // CopyStrategy about assumptions on descriptor use. For now, we take the worst case
+            // peak for every operation, which is two file descriptors. This keeps the downstream
+            // limiting code simple.
+            //
+            // We do not preclude the use of more granular limiting in the future (e.g. a directory
+            // scan requires only a single file descriptor). For now we just drop any excess
+            // remainder entirely.
             let limitValue = maxDescriptors / 2
             return try await self.copyDirectoryParallel(
                 from: sourcePath,
@@ -1086,9 +1083,9 @@ extension FileSystem {
         }
     }
 
-    /// Building block of the parallel directory copy implementation
-    /// Each invovation of this is allowed to consume two file descriptors,
-    /// any further work (if any) should be sent to `yield` for future processing
+    /// Building block of the parallel directory copy implementation. Each invocation of this is
+    /// allowed to consume two file descriptors. Any further work (if any) should be sent to `yield`
+    /// for future processing.
     func copySelfAndEnqueueChildren(
         from: DirectoryEntry,
         to: FilePath,
@@ -1344,7 +1341,7 @@ extension FileSystem {
         // Check that the destination doesn't exist. 'rename' will remove it otherwise!
         switch self._info(forFileAt: destinationPath, infoAboutSymbolicLink: true) {
         case .success(.none):
-            // Doens't exist: continue
+            // Doesn't exist: continue
             ()
 
         case .success(.some):
@@ -1375,8 +1372,7 @@ extension FileSystem {
             return .success(.moved)
 
         case .failure(.improperLink):
-            // The two paths are on different logical devices; copy and then remove the
-            // original.
+            // The two paths are on different logical devices; copy and then remove the original.
             return .success(.differentLogicalDevices)
 
         case let .failure(errno):
@@ -1410,7 +1406,8 @@ extension FileSystem {
 
         let lastComponent = lastComponentPath.string
 
-        // Finding the index of the last non-'X' character in `lastComponent.string` and advancing it by one.
+        // Finding the index of the last non-'X' character in `lastComponent.string` and advancing
+        // it by one.
         let prefix: String
         var index = lastComponent.lastIndex(where: { $0 != "X" })
         if index != nil {
@@ -1484,7 +1481,8 @@ extension FileSystem {
             case let .failure(error):
                 if let systemCallError = error.cause as? FileSystemError.SystemCallError {
                     switch systemCallError.errno {
-                    // If the file at the generated path already exists, we generate a new file path.
+                    // If the file at the generated path already exists, we generate a new file
+                    // path.
                     case .fileExists, .isDirectory:
                         break
                     default:
diff --git a/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift b/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift
index 8f1d6b3813..ce793264a1 100644
--- a/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift
+++ b/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift
@@ -16,16 +16,18 @@ import NIOCore
 
 @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *)
 extension FileSystem {
-
-    /// Iterative implementation of a recursive parallel copy of the directory from `sourcePath` to `destinationPath`.
+    /// Iterative implementation of a recursive parallel copy of the directory from `sourcePath` to
+    /// `destinationPath`.
+    ///
+    /// The parallelism is solely at the level of individual items (files, symbolic links and
+    /// directories). A larger file is not 'split' into concurrent reads or writes.
     ///
-    /// The parallelism is solely at the level of individual items (so files, symbolic links and directories), a larger file
-    /// is not considered for being 'split' into concurrent reads or wites.
-    /// If any symbolic link is encountered then only the link is copied.
-    /// The copied items will preserve permissions and any extended attributes (if supported by the file system).
+    /// If a symbolic link is encountered, then only the link is copied. If supported by the file
+    /// system, the copied items will preserve permissions and any extended attributes.
     ///
-    /// Note: `maxConcurrentOperations` is used as a hard (conservative) limit on the number of open file descriptors at any point.
-    /// Operations are assume to consume 2 descriptors so maximum open descriptors is `maxConcurrentOperations * 2`
+    /// Note: `maxConcurrentOperations` is used as a hard (conservative) limit on the number of open
+    /// file descriptors at any point. Operations are assumed to consume 2 descriptors, so the
+    /// maximum open descriptors are `maxConcurrentOperations * 2`
     @usableFromInline
     func copyDirectoryParallel(
         from sourcePath: FilePath,
@@ -40,9 +42,9 @@ extension FileSystem {
             _ destination: FilePath
         ) async -> Bool
     ) async throws {
-        // Implemented with NIOAsyncSequenceProducer rather than AsyncStream.
-        // It is approximately the same speed in the best case but has significantly less variance.
-        // NIOAsyncSequenceProducer also enforces a multi producer single consumer access pattern.
+        // Implemented with NIOAsyncSequenceProducer rather than AsyncStream. It is approximately
+        // the same speed in the best case, but has significantly less variance.
+        // NIOAsyncSequenceProducer also enforces a multi-producer, single-consumer access pattern.
         let copyRequiredQueue = NIOAsyncSequenceProducer.makeSequence(
             elementType: DirCopyItem.self,
             backPressureStrategy: NoBackPressureStrategy(),
@@ -50,33 +52,35 @@ extension FileSystem {
             delegate: DirCopyDelegate()
         )
 
-        // We ignore the result of yield in all cases because we are not implementing back pressure
-        // and cancellation is dealt with separately.
+        // We ignore the result of yield in all cases, because we are not implementing back
+        // pressure, and cancellation is dealt with separately.
         @Sendable func yield(_ contentsOf: [DirCopyItem]) {
             _ = copyRequiredQueue.source.yield(contentsOf: contentsOf)
         }
 
-        // Kick start the procees by enqueuing the root entry,
-        // the calling function already validated the root needed copying.
+        // Kick-start the procees by enqueuing the root entry. The calling function already
+        // validated the root needed copying, so it is safe to force unwrap the value.
         _ = copyRequiredQueue.source.yield(
             .toCopy(from: .init(path: sourcePath, type: .directory)!, to: destinationPath)
         )
 
-        // The processing of the very first item (the root) will increment this,
-        // after then when it hits zero we've finished.
+        // The processing of the very first item (the root) will increment this counter. Processing
+        // will finish when the counter hits zero again.
+        //
         // This does not need to be a ManagedAtomic or similar because:
-        // - All maintenance of state is done in the withThrowingTaskGroup callback
+        // - All state maintenance is done within the withThrowingTaskGroup closure
         // - All actual file system work is done by tasks created on the `taskGroup`
         var activeDirCount = 0
 
-        // Despite there being no 'result' of each operation we cannot use a discarding task group
+        // Despite there being no 'result' for each operation, we cannot use a discarding task group,
         // because we use the 'drain results' queue as a concurrency limiting side effect.
         try await withThrowingTaskGroup(of: Void.self) { taskGroup in
-
-            // Code handling each item to process on the current task
+            // Process each item in the current task.
+            //
             // Side Effects:
             // - Updates activeDirCount and finishes the stream if required.
-            // - Either adds a single task on `taskGroup` or none.
+            // - Might add a task to `taskGroup`.
+            //
             // Returns true if it added a task, false otherwise.
             func onNextItem(_ item: DirCopyItem) -> Bool {
                 switch item {
@@ -105,7 +109,7 @@ extension FileSystem {
 
             let iter = copyRequiredQueue.sequence.makeAsyncIterator()
 
-            // inProgress counts the number of tasks we have added to the task group
+            // inProgress counts the number of tasks we have added to the task group.
             // Get up to the maximum concurrency first.
             // We haven't started monitoring for task completion, so inProgress is 'worst case'.
             var inProgress = 0
@@ -116,16 +120,15 @@ extension FileSystem {
                         inProgress += 1
                     }
                 } else {
-                    // Either we completed things before we hit the limit or we were cancelled.
-                    // In the latter case we choose to propagate the cancel clearly.
-                    // This makes testing for the cancellation more reliable.
+                    // Either we completed things before we hit the limit or we were cancelled. In
+                    // the latter case we choose to propagate the cancellation clearly. This makes
+                    // testing for it more reliable.
                     try Task.checkCancellation()
                     return
                 }
             }
 
-            // Then operate one in (finish) -> one out (start another),
-            // but only for items that trigger a task.
+            // One in (finish) -> one out (start another), but only for items that trigger a task.
             while let _ = try await taskGroup.next() {
                 var keepConsuming = true
                 while keepConsuming {
@@ -133,7 +136,7 @@ extension FileSystem {
                     if let item = item {
                         keepConsuming = !onNextItem(item)
                     } else {
-                        // To accurately propagate the cancellation we must check here too
+                        // We must check here, to accurately propagate the cancellation.
                         try Task.checkCancellation()
                         keepConsuming = false
                     }
@@ -143,7 +146,7 @@ extension FileSystem {
     }
 }
 
-/// An 'always ask for more' no  back-pressure strategy for a ``NIOAsyncSequenceProducer``.
+/// An 'always ask for more' no back-pressure strategy for a ``NIOAsyncSequenceProducer``.
 @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
 private struct NoBackPressureStrategy: NIOAsyncSequenceProducerBackPressureStrategy {
     mutating func didYield(bufferDepth: Int) -> Bool { true }
diff --git a/Sources/NIOPosix/NIOThreadPool.swift b/Sources/NIOPosix/NIOThreadPool.swift
index 6d3bc08f73..75dfd08749 100644
--- a/Sources/NIOPosix/NIOThreadPool.swift
+++ b/Sources/NIOPosix/NIOThreadPool.swift
@@ -441,11 +441,11 @@ extension NIOThreadPool {
     }
 
     /// Runs the submitted closure if the thread pool is still active, otherwise throw an error.
-    /// The closure will be run on the thread pool so can do blocking work.
+    /// The closure will be run on the thread pool, such that we can do blocking work.
     ///
     /// - Parameters:
     ///   - body: The closure which performs some blocking work to be done on the thread pool.
-    /// - Returns: result of the passed closure.
+    /// - Returns: Result of the passed closure.
     @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
     public func runIfActive<T: Sendable>(_ body: @escaping @Sendable () throws -> T) async throws -> T {
         let workID = self.nextWorkID.loadThenWrappingIncrement(ordering: .relaxed)

From d2cc4d7f94bbc705c6a4a6fbb37ad56caa814b2c Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Tue, 26 Nov 2024 15:42:10 +0000
Subject: [PATCH 17/37] Cxx interop CI appends swiftSettings (#3002)

Cxx interop CI appends `swiftSettings` to any existing settings rather
than overriding them.

### Motivation:

This better accommodates workflows for example which use Swift 5
language mode on a per-target basis.

### Modifications:
Cxx interop CI appends `swiftSettings` to any existing settings rather
than overriding them.

### Result:

Cxx interop CI jobs don't cause unexpected build configurations.
---
 scripts/check-cxx-interop-compatibility.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/check-cxx-interop-compatibility.sh b/scripts/check-cxx-interop-compatibility.sh
index 5ad33a609c..9e868e94b5 100755
--- a/scripts/check-cxx-interop-compatibility.sh
+++ b/scripts/check-cxx-interop-compatibility.sh
@@ -33,7 +33,7 @@ swift package init
 
 {
   echo "let swiftSettings: [SwiftSetting] = [.interoperabilityMode(.Cxx)]"
-  echo "for target in package.targets { target.swiftSettings = swiftSettings }"
+  echo "for target in package.targets { target.swiftSettings = (target.swiftSettings ?? []) + swiftSettings }"
 } >> Package.swift
 
 echo "package.dependencies.append(.package(path: \"$source_dir\"))" >> Package.swift

From 49b9d9725ef1dc20d01970fa9e9334e1d16c19ca Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Tue, 26 Nov 2024 16:02:36 +0000
Subject: [PATCH 18/37] fix remaining warnings & enable -warnings-as-errors in
 CI (#3000)

### Motivation:

Warnings are annoying.

### Modifications:

- Remove unnecessary use of `Foundation.Thread` which isn't Sendable.
- Remove now unnecessary `@retroactive`s.
- Enable `-warnings-as-errors` in CI

### Result:

- Warnings can't sneak in as easily anymore
- No more warnings left in `swift-nio`

```
$ rm -rf .build/arm64-apple-macosx/ && swift build --build-tests -Xswiftc -warnings-as-errors > /dev/null
echo $?
$ echo $?
0
```

Co-authored-by: Cory Benfield <lukasa@apple.com>
---
 .github/workflows/main.yml                    |  4 +--
 .../ByteBuffer-foundation.swift               |  6 ----
 .../NIOPosixTests/EchoServerClientTest.swift  |  2 +-
 Tests/NIOPosixTests/NIOThreadPoolTest.swift   | 29 +++++++------------
 Tests/NIOPosixTests/SALEventLoopTests.swift   |  6 +---
 Tests/NIOPosixTests/TestUtils.swift           |  4 +--
 6 files changed, 17 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 13420a4d87..f6fd16711c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -14,8 +14,8 @@ jobs:
     with:
       linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
-      linux_6_0_arguments_override: "--explicit-target-dependency-import-check error"
-      linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error"
+      linux_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
+      linux_nightly_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error"
 
   cxx-interop:
diff --git a/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift b/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift
index cf1a25830a..c4240d4ed0 100644
--- a/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift
+++ b/Sources/NIOFoundationCompat/ByteBuffer-foundation.swift
@@ -377,15 +377,9 @@ extension ByteBufferAllocator {
 }
 
 // MARK: - Conformances
-#if compiler(>=6.0)
-extension ByteBufferView: @retroactive ContiguousBytes {}
-extension ByteBufferView: @retroactive DataProtocol {}
-extension ByteBufferView: @retroactive MutableDataProtocol {}
-#else
 extension ByteBufferView: ContiguousBytes {}
 extension ByteBufferView: DataProtocol {}
 extension ByteBufferView: MutableDataProtocol {}
-#endif
 
 extension ByteBufferView {
     public typealias Regions = CollectionOfOne<ByteBufferView>
diff --git a/Tests/NIOPosixTests/EchoServerClientTest.swift b/Tests/NIOPosixTests/EchoServerClientTest.swift
index 678f6da805..9675e19467 100644
--- a/Tests/NIOPosixTests/EchoServerClientTest.swift
+++ b/Tests/NIOPosixTests/EchoServerClientTest.swift
@@ -224,7 +224,7 @@ class EchoServerClientTest: XCTestCase {
 
         try withTemporaryUnixDomainSocketPathName { udsPath in
             // Bootstrap should not overwrite an existing file unless it is a socket
-            FileManager.default.createFile(atPath: udsPath, contents: nil, attributes: nil)
+            _ = FileManager.default.createFile(atPath: udsPath, contents: nil, attributes: nil)
             let bootstrap = ServerBootstrap(group: group)
 
             XCTAssertThrowsError(
diff --git a/Tests/NIOPosixTests/NIOThreadPoolTest.swift b/Tests/NIOPosixTests/NIOThreadPoolTest.swift
index ed39352df0..21eda2b446 100644
--- a/Tests/NIOPosixTests/NIOThreadPoolTest.swift
+++ b/Tests/NIOPosixTests/NIOThreadPoolTest.swift
@@ -78,18 +78,15 @@ class NIOThreadPoolTest: XCTestCase {
 
         // The lock here is arguably redundant with the dispatchgroup, but let's make
         // this test thread-safe even if I screw up.
-        let lock = NIOLock()
-        let threadOne: NIOLockedValueBox<Thread?> = NIOLockedValueBox(Thread?.none)
-        let threadTwo: NIOLockedValueBox<Thread?> = NIOLockedValueBox(Thread?.none)
+        let threadOne: NIOLockedValueBox<NIOThread?> = NIOLockedValueBox(NIOThread?.none)
+        let threadTwo: NIOLockedValueBox<NIOThread?> = NIOLockedValueBox(NIOThread?.none)
 
         completionGroup.enter()
         pool.submit { s in
             precondition(s == .active)
-            lock.withLock { () -> Void in
-                threadOne.withLockedValue { threadOne in
-                    XCTAssertEqual(threadOne, nil)
-                    threadOne = Thread.current
-                }
+            threadOne.withLockedValue { threadOne in
+                XCTAssertEqual(threadOne, nil)
+                threadOne = NIOThread.current
             }
             completionGroup.leave()
         }
@@ -99,22 +96,18 @@ class NIOThreadPoolTest: XCTestCase {
         completionGroup.enter()
         pool.submit { s in
             precondition(s == .active)
-            lock.withLock { () -> Void in
-                threadTwo.withLockedValue { threadTwo in
-                    XCTAssertEqual(threadTwo, nil)
-                    threadTwo = Thread.current
-                }
+            threadTwo.withLockedValue { threadTwo in
+                XCTAssertEqual(threadTwo, nil)
+                threadTwo = NIOThread.current
             }
             completionGroup.leave()
         }
 
         completionGroup.wait()
 
-        lock.withLock { () -> Void in
-            XCTAssertNotNil(threadOne)
-            XCTAssertNotNil(threadTwo)
-            XCTAssertEqual(threadOne.withLockedValue { $0 }, threadTwo.withLockedValue { $0 })
-        }
+        XCTAssertNotNil(threadOne.withLockedValue { $0 })
+        XCTAssertNotNil(threadTwo.withLockedValue { $0 })
+        XCTAssertEqual(threadOne.withLockedValue { $0 }, threadTwo.withLockedValue { $0 })
     }
 
     func testAsyncThreadPool() async throws {
diff --git a/Tests/NIOPosixTests/SALEventLoopTests.swift b/Tests/NIOPosixTests/SALEventLoopTests.swift
index 6cc81f5c86..ad3656475d 100644
--- a/Tests/NIOPosixTests/SALEventLoopTests.swift
+++ b/Tests/NIOPosixTests/SALEventLoopTests.swift
@@ -59,16 +59,12 @@ final class SALEventLoopTests: XCTestCase, SALTest {
                 }
 
                 // Now execute 10 tasks.
-                var i = 0
                 for _ in 0..<10 {
-                    thisLoop.execute {
-                        i &+= 1
-                    }
+                    thisLoop.execute {}
                 }
 
                 // Now enqueue a "last" task.
                 thisLoop.execute {
-                    i &+= 1
                     promise.succeed(())
                 }
 
diff --git a/Tests/NIOPosixTests/TestUtils.swift b/Tests/NIOPosixTests/TestUtils.swift
index c239a09c8a..ab5a11891e 100644
--- a/Tests/NIOPosixTests/TestUtils.swift
+++ b/Tests/NIOPosixTests/TestUtils.swift
@@ -127,7 +127,7 @@ func withTemporaryUnixDomainSocketPathName<T>(
         shortEnoughPath = path
         restoreSavedCWD = false
     } catch SocketAddressError.unixDomainSocketPathTooLong {
-        FileManager.default.changeCurrentDirectoryPath(
+        _ = FileManager.default.changeCurrentDirectoryPath(
             URL(fileURLWithPath: path).deletingLastPathComponent().absoluteString
         )
         shortEnoughPath = URL(fileURLWithPath: path).lastPathComponent
@@ -141,7 +141,7 @@ func withTemporaryUnixDomainSocketPathName<T>(
             try? FileManager.default.removeItem(atPath: path)
         }
         if restoreSavedCWD {
-            FileManager.default.changeCurrentDirectoryPath(saveCurrentDirectory)
+            _ = FileManager.default.changeCurrentDirectoryPath(saveCurrentDirectory)
         }
     }
     return try body(shortEnoughPath)

From 16f19c0c645014a41456097f1848018efca41d7e Mon Sep 17 00:00:00 2001
From: Cory Benfield <lukasa@apple.com>
Date: Tue, 26 Nov 2024 16:19:33 +0000
Subject: [PATCH 19/37] Prevent crash in Happy Eyeballs Resolver (#3003)

Motivation:

In vanishingly rare situations it is possible for the AAAA results to
come in on the same tick as the resolution delay timer completes. In
those cases, depending on the ordering of the tasks, we can get
situations where the resolution delay timer completion causes a crash.

Modifications:

Tolerate receiving the resolution delay timer after resolution
completes.

Result:

Fewer crashes
---
 Sources/NIOPosix/HappyEyeballs.swift        |  7 ++-
 Tests/NIOPosixTests/HappyEyeballsTest.swift | 47 +++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/Sources/NIOPosix/HappyEyeballs.swift b/Sources/NIOPosix/HappyEyeballs.swift
index 015a8b4af6..25071ec32a 100644
--- a/Sources/NIOPosix/HappyEyeballs.swift
+++ b/Sources/NIOPosix/HappyEyeballs.swift
@@ -464,7 +464,12 @@ internal final class HappyEyeballsConnector<ChannelBuilderResult> {
         // notifications, and can also get late scheduled task callbacks. We want to just quietly
         // ignore these, as our transition into the complete state should have already sent
         // cleanup messages to all of these things.
-        case (.complete, .resolverACompleted),
+        //
+        // We can also get the resolutionDelayElapsed after allResolved, as it's possible that
+        // callback was already dequeued in the same tick as the cancellation. That's also fine:
+        // the resolution delay isn't interesting.
+        case (.allResolved, .resolutionDelayElapsed),
+            (.complete, .resolverACompleted),
             (.complete, .resolverAAAACompleted),
             (.complete, .connectSuccess),
             (.complete, .connectFailed),
diff --git a/Tests/NIOPosixTests/HappyEyeballsTest.swift b/Tests/NIOPosixTests/HappyEyeballsTest.swift
index 66f0dec09d..29f35820f8 100644
--- a/Tests/NIOPosixTests/HappyEyeballsTest.swift
+++ b/Tests/NIOPosixTests/HappyEyeballsTest.swift
@@ -1308,4 +1308,51 @@ public final class HappyEyeballsTest: XCTestCase {
 
         XCTAssertNoThrow(try client.close().wait())
     }
+
+    func testResolutionTimeoutAndResolutionInSameTick() throws {
+        var channels: [Channel] = []
+        let (eyeballer, resolver, loop) = buildEyeballer(host: "example.com", port: 80) {
+            let channelFuture = defaultChannelBuilder(loop: $0, family: $1)
+            channelFuture.whenSuccess { channel in
+                try! channel.pipeline.addHandler(ConnectionDelayer(), name: CONNECT_DELAYER, position: .first).wait()
+                channels.append(channel)
+            }
+            return channelFuture
+        }
+        let targetFuture = eyeballer.resolveAndConnect().flatMapThrowing { (channel) -> String? in
+            let target = channel.connectTarget()
+            _ = try (channel as! EmbeddedChannel).finish()
+            return target
+        }
+        loop.run()
+
+        // Then, queue a task to resolve the v6 promise after 50ms.
+        // Why 50ms? This is the same time as the resolution delay.
+        let promise = resolver.v6Promise
+        loop.scheduleTask(in: .milliseconds(50)) {
+            promise.fail(DummyError())
+        }
+
+        // Kick off the IPv4 resolution. This triggers the timer for the resolution delay.
+        resolver.v4Promise.succeed(SINGLE_IPv4_RESULT)
+        loop.run()
+
+        // Advance time 50ms.
+        loop.advanceTime(by: .milliseconds(50))
+
+        // Then complete the connection future.
+        XCTAssertEqual(channels.count, 1)
+        channels.first!.succeedConnection()
+
+        // Should be done.
+        let target = try targetFuture.wait()
+        XCTAssertEqual(target!, "10.0.0.1")
+
+        // We should have had queries for AAAA and A.
+        let expectedQueries: [DummyResolver.Event] = [
+            .aaaa(host: "example.com", port: 80),
+            .a(host: "example.com", port: 80),
+        ]
+        XCTAssertEqual(resolver.events, expectedQueries)
+    }
 }

From ca55b0e50febfc597537a820048ccc5cdbd4f5bf Mon Sep 17 00:00:00 2001
From: George Barnett <gbarnett@apple.com>
Date: Wed, 27 Nov 2024 11:21:34 +0000
Subject: [PATCH 20/37] Remove noasync from NIOFileHandle (#3001)

Motivation:

In #2598, `NIOFileHandle` was annotated with `noasync` in a few places.
This is, unfortunately, a breaking change.

Modifications:

- Remove the `noasync` annotations

Result:

Adopters aren't broken

---------

Co-authored-by: Cory Benfield <lukasa@apple.com>
---
 Sources/NIOCore/FileHandle.swift | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Sources/NIOCore/FileHandle.swift b/Sources/NIOCore/FileHandle.swift
index 62fd100e28..6c7dc08125 100644
--- a/Sources/NIOCore/FileHandle.swift
+++ b/Sources/NIOCore/FileHandle.swift
@@ -373,7 +373,6 @@ extension NIOFileHandle {
             use NIOFileSystem as a replacement API.
             """
     )
-    @available(*, noasync, message: "This method may block the calling thread")
     public convenience init(
         path: String,
         mode: Mode = .read,
@@ -415,7 +414,6 @@ extension NIOFileHandle {
             use NIOFileSystem as a replacement API.
             """
     )
-    @available(*, noasync, message: "This method may block the calling thread")
     public convenience init(path: String) throws {
         try self.init(_deprecatedPath: path)
     }

From 8126cba71251682a15a7878f67ab0ad5659f2a98 Mon Sep 17 00:00:00 2001
From: Cory Benfield <lukasa@apple.com>
Date: Thu, 28 Nov 2024 11:30:54 +0000
Subject: [PATCH 21/37] Avoid converting array holding existentials (#3006)

Motivation:

When converting an Array that holds existentials, it is necessary for
Swift to allocate a new Array and copy the elements into it, so that it
can fix up their existential boxes. We forced this to happen with taking
a `[ChannelHandler]` to `[ChannelHandler & Sendable]`, without rewriting
the other functions.

The result of that was that the other functions needed to convert the
channel handler types, causing extra allocations in this path.

Modifications:

Propagate the constraint down to the point where we iterate the Array.

Result:

Allocations reduced.
---
 .../tests_04_performance/Thresholds/5.10.json | 76 +++++++++---------
 .../tests_04_performance/Thresholds/5.9.json  | 77 ++++++++++---------
 .../tests_04_performance/Thresholds/6.0.json  | 76 +++++++++---------
 .../Thresholds/nightly-6.0.json               | 76 +++++++++---------
 .../Thresholds/nightly-main.json              | 76 +++++++++---------
 Sources/NIOCore/ChannelPipeline.swift         | 58 +++++++++++++-
 6 files changed, 246 insertions(+), 193 deletions(-)

diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.10.json b/IntegrationTests/tests_04_performance/Thresholds/5.10.json
index 1fb41c2193..9f75ed151e 100644
--- a/IntegrationTests/tests_04_performance/Thresholds/5.10.json
+++ b/IntegrationTests/tests_04_performance/Thresholds/5.10.json
@@ -1,50 +1,50 @@
 {
     "10000000_asyncsequenceproducer": 19,
-    "1000000_asyncwriter": 1000013,
-    "1000_addHandlers": 44000,
-    "1000_addHandlers_sync": 36000,
-    "1000_addRemoveHandlers_handlercontext": 8032,
-    "1000_addRemoveHandlers_handlername": 8032,
-    "1000_addRemoveHandlers_handlertype": 8032,
-    "1000_autoReadGetAndSet": 18000,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 43050,
+    "1000_addHandlers_sync": 36050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
     "1000_autoReadGetAndSet_sync": 0,
-    "1000_copying_bytebufferview_to_array": 1000,
-    "1000_copying_circularbuffer_to_array": 1000,
-    "1000_getHandlers": 8034,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
     "1000_getHandlers_sync": 34,
-    "1000_reqs_1_conn": 26363,
-    "1000_rst_connections": 145004,
-    "1000_tcpbootstraps": 3001,
-    "1000_tcpconnections": 152002,
-    "1000_udp_reqs": 6014,
-    "1000_udpbootstraps": 2000,
-    "1000_udpconnections": 75008,
-    "1_reqs_1000_conn": 389000,
-    "bytebuffer_lots_of_rw": 2005,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 145050,
+    "1000_tcpbootstraps": 3050,
+    "1000_tcpconnections": 152050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 389050,
+    "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
-    "decode_1000_ws_frames": 2000,
+    "decode_1000_ws_frames": 2050,
     "encode_1000_ws_frames_holding_buffer": 3,
-    "encode_1000_ws_frames_holding_buffer_with_mask": 2003,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
     "encode_1000_ws_frames_holding_buffer_with_space": 3,
-    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003,
-    "encode_1000_ws_frames_new_buffer": 3000,
-    "encode_1000_ws_frames_new_buffer_with_mask": 5000,
-    "encode_1000_ws_frames_new_buffer_with_space": 3000,
-    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
-    "future_erase_result": 4001,
-    "future_lots_of_callbacks": 53001,
-    "get_100000_headers_canonical_form": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
     "modifying_1000_circular_buffer_elements": 0,
-    "modifying_byte_buffer_view": 6000,
+    "modifying_byte_buffer_view": 6050,
     "ping_pong_1000_reqs_1_conn": 319,
-    "read_10000_chunks_from_file": 110009,
-    "schedule_10000_tasks": 40084,
-    "schedule_and_run_10000_tasks": 50010,
+    "read_10000_chunks_from_file": 110050,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
     "scheduling_10000_executions": 89,
-    "udp_1000_reqs_1_conn": 6156,
-    "udp_1_reqs_1000_conn": 162000
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
 }
diff --git a/IntegrationTests/tests_04_performance/Thresholds/5.9.json b/IntegrationTests/tests_04_performance/Thresholds/5.9.json
index b77ad41e47..2edeea7c88 100644
--- a/IntegrationTests/tests_04_performance/Thresholds/5.9.json
+++ b/IntegrationTests/tests_04_performance/Thresholds/5.9.json
@@ -1,50 +1,51 @@
 {
     "10000000_asyncsequenceproducer": 19,
-    "1000000_asyncwriter": 1000013,
-    "1000_addHandlers": 44000,
-    "1000_addHandlers_sync": 36000,
-    "1000_addRemoveHandlers_handlercontext": 8032,
-    "1000_addRemoveHandlers_handlername": 8032,
-    "1000_addRemoveHandlers_handlertype": 8032,
-    "1000_autoReadGetAndSet": 18000,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 43050,
+    "1000_addHandlers_sync": 36050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
     "1000_autoReadGetAndSet_sync": 0,
-    "1000_copying_bytebufferview_to_array": 1000,
-    "1000_copying_circularbuffer_to_array": 1000,
-    "1000_getHandlers": 8034,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
     "1000_getHandlers_sync": 34,
-    "1000_reqs_1_conn": 26372,
-    "1000_rst_connections": 147004,
-    "1000_tcpbootstraps": 4001,
-    "1000_tcpconnections": 154002,
-    "1000_udp_reqs": 6014,
-    "1000_udpbootstraps": 2000,
-    "1000_udpconnections": 75008,
-    "1_reqs_1000_conn": 398000,
-    "bytebuffer_lots_of_rw": 2005,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 147050,
+    "1000_tcpbootstraps": 4050,
+    "1000_tcpconnections": 154050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 398050,
+    "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
-    "decode_1000_ws_frames": 2000,
+    "decode_1000_ws_frames": 2050,
     "encode_1000_ws_frames_holding_buffer": 3,
-    "encode_1000_ws_frames_holding_buffer_with_mask": 2003,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
     "encode_1000_ws_frames_holding_buffer_with_space": 3,
-    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003,
-    "encode_1000_ws_frames_new_buffer": 3000,
-    "encode_1000_ws_frames_new_buffer_with_mask": 5000,
-    "encode_1000_ws_frames_new_buffer_with_space": 3000,
-    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
-    "future_erase_result": 4001,
-    "future_lots_of_callbacks": 53001,
-    "get_100000_headers_canonical_form": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
     "modifying_1000_circular_buffer_elements": 0,
-    "modifying_byte_buffer_view": 6000,
+    "modifying_byte_buffer_view": 6050,
     "ping_pong_1000_reqs_1_conn": 334,
-    "read_10000_chunks_from_file": 110009,
-    "schedule_10000_tasks": 40084,
-    "schedule_and_run_10000_tasks": 50010,
+    "read_10000_chunks_from_file": 110050,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
     "scheduling_10000_executions": 89,
-    "udp_1000_reqs_1_conn": 6156,
-    "udp_1_reqs_1000_conn": 162000
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
 }
+
diff --git a/IntegrationTests/tests_04_performance/Thresholds/6.0.json b/IntegrationTests/tests_04_performance/Thresholds/6.0.json
index 1fb41c2193..9f75ed151e 100644
--- a/IntegrationTests/tests_04_performance/Thresholds/6.0.json
+++ b/IntegrationTests/tests_04_performance/Thresholds/6.0.json
@@ -1,50 +1,50 @@
 {
     "10000000_asyncsequenceproducer": 19,
-    "1000000_asyncwriter": 1000013,
-    "1000_addHandlers": 44000,
-    "1000_addHandlers_sync": 36000,
-    "1000_addRemoveHandlers_handlercontext": 8032,
-    "1000_addRemoveHandlers_handlername": 8032,
-    "1000_addRemoveHandlers_handlertype": 8032,
-    "1000_autoReadGetAndSet": 18000,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 43050,
+    "1000_addHandlers_sync": 36050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
     "1000_autoReadGetAndSet_sync": 0,
-    "1000_copying_bytebufferview_to_array": 1000,
-    "1000_copying_circularbuffer_to_array": 1000,
-    "1000_getHandlers": 8034,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
     "1000_getHandlers_sync": 34,
-    "1000_reqs_1_conn": 26363,
-    "1000_rst_connections": 145004,
-    "1000_tcpbootstraps": 3001,
-    "1000_tcpconnections": 152002,
-    "1000_udp_reqs": 6014,
-    "1000_udpbootstraps": 2000,
-    "1000_udpconnections": 75008,
-    "1_reqs_1000_conn": 389000,
-    "bytebuffer_lots_of_rw": 2005,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 145050,
+    "1000_tcpbootstraps": 3050,
+    "1000_tcpconnections": 152050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 389050,
+    "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
-    "decode_1000_ws_frames": 2000,
+    "decode_1000_ws_frames": 2050,
     "encode_1000_ws_frames_holding_buffer": 3,
-    "encode_1000_ws_frames_holding_buffer_with_mask": 2003,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
     "encode_1000_ws_frames_holding_buffer_with_space": 3,
-    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003,
-    "encode_1000_ws_frames_new_buffer": 3000,
-    "encode_1000_ws_frames_new_buffer_with_mask": 5000,
-    "encode_1000_ws_frames_new_buffer_with_space": 3000,
-    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
-    "future_erase_result": 4001,
-    "future_lots_of_callbacks": 53001,
-    "get_100000_headers_canonical_form": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
     "modifying_1000_circular_buffer_elements": 0,
-    "modifying_byte_buffer_view": 6000,
+    "modifying_byte_buffer_view": 6050,
     "ping_pong_1000_reqs_1_conn": 319,
-    "read_10000_chunks_from_file": 110009,
-    "schedule_10000_tasks": 40084,
-    "schedule_and_run_10000_tasks": 50010,
+    "read_10000_chunks_from_file": 110050,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
     "scheduling_10000_executions": 89,
-    "udp_1000_reqs_1_conn": 6156,
-    "udp_1_reqs_1000_conn": 162000
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
 }
diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json
index 1fb41c2193..9f75ed151e 100644
--- a/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json
+++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json
@@ -1,50 +1,50 @@
 {
     "10000000_asyncsequenceproducer": 19,
-    "1000000_asyncwriter": 1000013,
-    "1000_addHandlers": 44000,
-    "1000_addHandlers_sync": 36000,
-    "1000_addRemoveHandlers_handlercontext": 8032,
-    "1000_addRemoveHandlers_handlername": 8032,
-    "1000_addRemoveHandlers_handlertype": 8032,
-    "1000_autoReadGetAndSet": 18000,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 43050,
+    "1000_addHandlers_sync": 36050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
     "1000_autoReadGetAndSet_sync": 0,
-    "1000_copying_bytebufferview_to_array": 1000,
-    "1000_copying_circularbuffer_to_array": 1000,
-    "1000_getHandlers": 8034,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
     "1000_getHandlers_sync": 34,
-    "1000_reqs_1_conn": 26363,
-    "1000_rst_connections": 145004,
-    "1000_tcpbootstraps": 3001,
-    "1000_tcpconnections": 152002,
-    "1000_udp_reqs": 6014,
-    "1000_udpbootstraps": 2000,
-    "1000_udpconnections": 75008,
-    "1_reqs_1000_conn": 389000,
-    "bytebuffer_lots_of_rw": 2005,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 145050,
+    "1000_tcpbootstraps": 3050,
+    "1000_tcpconnections": 152050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 389050,
+    "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
-    "decode_1000_ws_frames": 2000,
+    "decode_1000_ws_frames": 2050,
     "encode_1000_ws_frames_holding_buffer": 3,
-    "encode_1000_ws_frames_holding_buffer_with_mask": 2003,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
     "encode_1000_ws_frames_holding_buffer_with_space": 3,
-    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003,
-    "encode_1000_ws_frames_new_buffer": 3000,
-    "encode_1000_ws_frames_new_buffer_with_mask": 5000,
-    "encode_1000_ws_frames_new_buffer_with_space": 3000,
-    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
-    "future_erase_result": 4001,
-    "future_lots_of_callbacks": 53001,
-    "get_100000_headers_canonical_form": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
     "modifying_1000_circular_buffer_elements": 0,
-    "modifying_byte_buffer_view": 6000,
+    "modifying_byte_buffer_view": 6050,
     "ping_pong_1000_reqs_1_conn": 319,
-    "read_10000_chunks_from_file": 110009,
-    "schedule_10000_tasks": 40084,
-    "schedule_and_run_10000_tasks": 50010,
+    "read_10000_chunks_from_file": 110050,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
     "scheduling_10000_executions": 89,
-    "udp_1000_reqs_1_conn": 6156,
-    "udp_1_reqs_1000_conn": 162000
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
 }
diff --git a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json
index 1fb41c2193..9f75ed151e 100644
--- a/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json
+++ b/IntegrationTests/tests_04_performance/Thresholds/nightly-main.json
@@ -1,50 +1,50 @@
 {
     "10000000_asyncsequenceproducer": 19,
-    "1000000_asyncwriter": 1000013,
-    "1000_addHandlers": 44000,
-    "1000_addHandlers_sync": 36000,
-    "1000_addRemoveHandlers_handlercontext": 8032,
-    "1000_addRemoveHandlers_handlername": 8032,
-    "1000_addRemoveHandlers_handlertype": 8032,
-    "1000_autoReadGetAndSet": 18000,
+    "1000000_asyncwriter": 1000050,
+    "1000_addHandlers": 43050,
+    "1000_addHandlers_sync": 36050,
+    "1000_addRemoveHandlers_handlercontext": 8050,
+    "1000_addRemoveHandlers_handlername": 8050,
+    "1000_addRemoveHandlers_handlertype": 8050,
+    "1000_autoReadGetAndSet": 18050,
     "1000_autoReadGetAndSet_sync": 0,
-    "1000_copying_bytebufferview_to_array": 1000,
-    "1000_copying_circularbuffer_to_array": 1000,
-    "1000_getHandlers": 8034,
+    "1000_copying_bytebufferview_to_array": 1050,
+    "1000_copying_circularbuffer_to_array": 1050,
+    "1000_getHandlers": 8050,
     "1000_getHandlers_sync": 34,
-    "1000_reqs_1_conn": 26363,
-    "1000_rst_connections": 145004,
-    "1000_tcpbootstraps": 3001,
-    "1000_tcpconnections": 152002,
-    "1000_udp_reqs": 6014,
-    "1000_udpbootstraps": 2000,
-    "1000_udpconnections": 75008,
-    "1_reqs_1000_conn": 389000,
-    "bytebuffer_lots_of_rw": 2005,
+    "1000_reqs_1_conn": 26400,
+    "1000_rst_connections": 145050,
+    "1000_tcpbootstraps": 3050,
+    "1000_tcpconnections": 152050,
+    "1000_udp_reqs": 6050,
+    "1000_udpbootstraps": 2050,
+    "1000_udpconnections": 75050,
+    "1_reqs_1000_conn": 389050,
+    "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
-    "decode_1000_ws_frames": 2000,
+    "decode_1000_ws_frames": 2050,
     "encode_1000_ws_frames_holding_buffer": 3,
-    "encode_1000_ws_frames_holding_buffer_with_mask": 2003,
+    "encode_1000_ws_frames_holding_buffer_with_mask": 2050,
     "encode_1000_ws_frames_holding_buffer_with_space": 3,
-    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2003,
-    "encode_1000_ws_frames_new_buffer": 3000,
-    "encode_1000_ws_frames_new_buffer_with_mask": 5000,
-    "encode_1000_ws_frames_new_buffer_with_space": 3000,
-    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5000,
+    "encode_1000_ws_frames_holding_buffer_with_space_with_mask": 2050,
+    "encode_1000_ws_frames_new_buffer": 3050,
+    "encode_1000_ws_frames_new_buffer_with_mask": 5050,
+    "encode_1000_ws_frames_new_buffer_with_space": 3050,
+    "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
-    "future_erase_result": 4001,
-    "future_lots_of_callbacks": 53001,
-    "get_100000_headers_canonical_form": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700000,
-    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700000,
+    "future_erase_result": 4050,
+    "future_lots_of_callbacks": 53050,
+    "get_100000_headers_canonical_form": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_long_string": 700050,
+    "get_100000_headers_canonical_form_trimming_whitespace_from_short_string": 700050,
     "modifying_1000_circular_buffer_elements": 0,
-    "modifying_byte_buffer_view": 6000,
+    "modifying_byte_buffer_view": 6050,
     "ping_pong_1000_reqs_1_conn": 319,
-    "read_10000_chunks_from_file": 110009,
-    "schedule_10000_tasks": 40084,
-    "schedule_and_run_10000_tasks": 50010,
+    "read_10000_chunks_from_file": 110050,
+    "schedule_10000_tasks": 40100,
+    "schedule_and_run_10000_tasks": 50050,
     "scheduling_10000_executions": 89,
-    "udp_1000_reqs_1_conn": 6156,
-    "udp_1_reqs_1000_conn": 162000
+    "udp_1000_reqs_1_conn": 6200,
+    "udp_1_reqs_1000_conn": 162050
 }
diff --git a/Sources/NIOCore/ChannelPipeline.swift b/Sources/NIOCore/ChannelPipeline.swift
index 2aa8d6c624..6920a2b0fd 100644
--- a/Sources/NIOCore/ChannelPipeline.swift
+++ b/Sources/NIOCore/ChannelPipeline.swift
@@ -1119,7 +1119,7 @@ extension ChannelPipeline {
     ///   - position: The position in the `ChannelPipeline` to add the handlers.
     /// - Returns: A result representing whether the handlers were added or not.
     fileprivate func addHandlersSync(
-        _ handlers: [ChannelHandler],
+        _ handlers: [ChannelHandler & Sendable],
         position: ChannelPipeline.Position
     ) -> Result<Void, Error> {
         switch position {
@@ -1130,6 +1130,29 @@ extension ChannelPipeline {
         }
     }
 
+    /// Synchronously adds the provided `ChannelHandler`s to the pipeline in the order given, taking
+    /// account of the behaviour of `ChannelHandler.add(first:)`.
+    ///
+    /// This duplicate of the above method exists to avoid needing to rebox the array of existentials
+    /// from any (ChannelHandler & Sendable) to any ChannelHandler.
+    ///
+    /// - Important: Must be called on the `EventLoop`.
+    /// - Parameters:
+    ///   - handlers: The array of `ChannelHandler`s to add.
+    ///   - position: The position in the `ChannelPipeline` to add the handlers.
+    /// - Returns: A result representing whether the handlers were added or not.
+    fileprivate func addHandlersSyncNotSendable(
+        _ handlers: [ChannelHandler],
+        position: ChannelPipeline.Position
+    ) -> Result<Void, Error> {
+        switch position {
+        case .first, .after:
+            return self._addHandlersSyncNotSendable(handlers.reversed(), position: position)
+        case .last, .before:
+            return self._addHandlersSyncNotSendable(handlers, position: position)
+        }
+    }
+
     /// Synchronously adds a sequence of `ChannelHandlers` to the pipeline at the given position.
     ///
     /// - Important: Must be called on the `EventLoop`.
@@ -1140,6 +1163,35 @@ extension ChannelPipeline {
     private func _addHandlersSync<Handlers: Sequence>(
         _ handlers: Handlers,
         position: ChannelPipeline.Position
+    ) -> Result<Void, Error> where Handlers.Element == ChannelHandler & Sendable {
+        self.eventLoop.assertInEventLoop()
+
+        for handler in handlers {
+            let result = self.addHandlerSync(handler, position: position)
+            switch result {
+            case .success:
+                ()
+            case .failure:
+                return result
+            }
+        }
+
+        return .success(())
+    }
+
+    /// Synchronously adds a sequence of `ChannelHandlers` to the pipeline at the given position.
+    ///
+    /// This duplicate of the above method exists to avoid needing to rebox the array of existentials
+    /// from any (ChannelHandler & Sendable) to any ChannelHandler.
+    ///
+    /// - Important: Must be called on the `EventLoop`.
+    /// - Parameters:
+    ///   - handlers: A sequence of handlers to add.
+    ///   - position: The position in the `ChannelPipeline` to add the handlers.
+    /// - Returns: A result representing whether the handlers were added or not.
+    private func _addHandlersSyncNotSendable<Handlers: Sequence>(
+        _ handlers: Handlers,
+        position: ChannelPipeline.Position
     ) -> Result<Void, Error> where Handlers.Element == ChannelHandler {
         self.eventLoop.assertInEventLoop()
 
@@ -1201,7 +1253,7 @@ extension ChannelPipeline {
             _ handlers: [ChannelHandler],
             position: ChannelPipeline.Position = .last
         ) throws {
-            try self._pipeline.addHandlersSync(handlers, position: position).get()
+            try self._pipeline.addHandlersSyncNotSendable(handlers, position: position).get()
         }
 
         /// Add one or more handlers to the pipeline.
@@ -1214,7 +1266,7 @@ extension ChannelPipeline {
             _ handlers: ChannelHandler...,
             position: ChannelPipeline.Position = .last
         ) throws {
-            try self._pipeline.addHandlersSync(handlers, position: position).get()
+            try self._pipeline.addHandlersSyncNotSendable(handlers, position: position).get()
         }
 
         /// Remove a `ChannelHandler` from the `ChannelPipeline`.

From 8a1523fb1a7e04c41346e375905d510eb45df55d Mon Sep 17 00:00:00 2001
From: Franz Busch <f.busch@apple.com>
Date: Thu, 28 Nov 2024 15:20:37 +0100
Subject: [PATCH 22/37] Aligning semantic version label check name (#3007)

---
 .github/workflows/pull_request_label.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pull_request_label.yml b/.github/workflows/pull_request_label.yml
index fbc8618639..c78fe14fae 100644
--- a/.github/workflows/pull_request_label.yml
+++ b/.github/workflows/pull_request_label.yml
@@ -6,7 +6,7 @@ on:
 
 jobs:
     semver-label-check:
-        name: Semantic Version label check
+        name: Semantic version label check
         runs-on: ubuntu-latest
         timeout-minutes: 1
         steps:

From 33b8a4801fbef544301c82a62564c0071322dda5 Mon Sep 17 00:00:00 2001
From: finagolfin <finagolfin@tuta.io>
Date: Sun, 1 Dec 2024 23:27:08 +0530
Subject: [PATCH 23/37] Use the new Android overlay in the tests and update
 some Bionic declarations (#3009)

### Motivation:

Get this repo building again for Android with NDK 27

### Modifications:

- Update some networking declarations for newly added nullability
annotations
- Import the new Android overlay instead in some tests
- Add two force-unwraps on all platforms, that are needed for Android

### Result:

This repo and its tests build for Android again

I've been [using these patches on my Android
CI](https://github.com/finagolfin/swift-android-sdk/blob/main/swift-nio-ndk27.patch)
and natively on Android for a couple months now. I didn't bother keeping
this patch building for Android with Swift 5 anymore, as my Android CI
no longer tests Swift 5.

I built this pull and ran the tests on linux x86_64 to make sure there
was no regression.
---
 .../Internal/System Calls/Syscalls.swift       | 11 +----------
 Sources/NIOPosix/System.swift                  | 18 ++++++++++++++----
 .../NIOConcurrencyHelpersTests.swift           |  4 ++--
 Tests/NIOCoreTests/XCTest+Extensions.swift     |  4 ++++
 Tests/NIOEmbeddedTests/TestUtils.swift         |  4 ++++
 Tests/NIOFileSystemTests/FileInfoTests.swift   | 10 +++++-----
 Tests/NIOFileSystemTests/FileTypeTests.swift   |  2 ++
 7 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift b/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift
index dcff578018..e2f5efd326 100644
--- a/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift	
+++ b/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift	
@@ -452,21 +452,12 @@ internal func libc_confstr(
 #endif
 
 /// fts(3)
-#if os(Android)
-internal func libc_fts_open(
-    _ path: [UnsafeMutablePointer<CInterop.PlatformChar>],
-    _ options: CInt
-) -> UnsafeMutablePointer<CInterop.FTS> {
-    fts_open(path, options, nil)!
-}
-#else
 internal func libc_fts_open(
     _ path: [UnsafeMutablePointer<CInterop.PlatformChar>?],
     _ options: CInt
 ) -> UnsafeMutablePointer<CInterop.FTS> {
-    fts_open(path, options, nil)
+    fts_open(path, options, nil)!
 }
-#endif
 
 /// fts(3)
 internal func libc_fts_read(
diff --git a/Sources/NIOPosix/System.swift b/Sources/NIOPosix/System.swift
index adb7cb2ffa..c221bd40d2 100644
--- a/Sources/NIOPosix/System.swift
+++ b/Sources/NIOPosix/System.swift
@@ -124,12 +124,20 @@ private let sysWritev = sysWritev_wrapper
 #elseif !os(Windows)
 private let sysWritev: @convention(c) (Int32, UnsafePointer<iovec>?, CInt) -> CLong = writev
 #endif
-#if !os(Windows)
+#if canImport(Android)
+private let sysRecvMsg: @convention(c) (CInt, UnsafeMutablePointer<msghdr>, CInt) -> ssize_t = recvmsg
+private let sysSendMsg: @convention(c) (CInt, UnsafePointer<msghdr>, CInt) -> ssize_t = sendmsg
+#elseif !os(Windows)
 private let sysRecvMsg: @convention(c) (CInt, UnsafeMutablePointer<msghdr>?, CInt) -> ssize_t = recvmsg
 private let sysSendMsg: @convention(c) (CInt, UnsafePointer<msghdr>?, CInt) -> ssize_t = sendmsg
 #endif
 private let sysDup: @convention(c) (CInt) -> CInt = dup
-#if !os(Windows)
+#if canImport(Android)
+private let sysGetpeername:
+    @convention(c) (CInt, UnsafeMutablePointer<sockaddr>, UnsafeMutablePointer<socklen_t>) -> CInt = getpeername
+private let sysGetsockname:
+    @convention(c) (CInt, UnsafeMutablePointer<sockaddr>, UnsafeMutablePointer<socklen_t>) -> CInt = getsockname
+#elseif !os(Windows)
 private let sysGetpeername:
     @convention(c) (CInt, UnsafeMutablePointer<sockaddr>?, UnsafeMutablePointer<socklen_t>?) -> CInt = getpeername
 private let sysGetsockname:
@@ -141,7 +149,9 @@ private let sysIfNameToIndex: @convention(c) (UnsafePointer<CChar>) -> CUnsigned
 #else
 private let sysIfNameToIndex: @convention(c) (UnsafePointer<CChar>?) -> CUnsignedInt = if_nametoindex
 #endif
-#if !os(Windows)
+#if canImport(Android)
+private let sysSocketpair: @convention(c) (CInt, CInt, CInt, UnsafeMutablePointer<CInt>) -> CInt = socketpair
+#elseif !os(Windows)
 private let sysSocketpair: @convention(c) (CInt, CInt, CInt, UnsafeMutablePointer<CInt>?) -> CInt = socketpair
 #endif
 
@@ -1000,7 +1010,7 @@ internal enum Posix {
         socketVector: UnsafeMutablePointer<CInt>?
     ) throws {
         _ = try syscall(blocking: false) {
-            sysSocketpair(domain.rawValue, type.rawValue, protocolSubtype.rawValue, socketVector)
+            sysSocketpair(domain.rawValue, type.rawValue, protocolSubtype.rawValue, socketVector!)
         }
     }
     #endif
diff --git a/Tests/NIOConcurrencyHelpersTests/NIOConcurrencyHelpersTests.swift b/Tests/NIOConcurrencyHelpersTests/NIOConcurrencyHelpersTests.swift
index 53864db0f7..bb0622262f 100644
--- a/Tests/NIOConcurrencyHelpersTests/NIOConcurrencyHelpersTests.swift
+++ b/Tests/NIOConcurrencyHelpersTests/NIOConcurrencyHelpersTests.swift
@@ -22,8 +22,8 @@ import XCTest
 import Darwin
 #elseif canImport(Glibc)
 import Glibc
-#elseif canImport(Bionic)
-import Bionic
+#elseif canImport(Android)
+import Android
 #else
 #error("The Concurrency helpers test module was unable to identify your C library.")
 #endif
diff --git a/Tests/NIOCoreTests/XCTest+Extensions.swift b/Tests/NIOCoreTests/XCTest+Extensions.swift
index 0fc5c8b7d4..2970c65511 100644
--- a/Tests/NIOCoreTests/XCTest+Extensions.swift
+++ b/Tests/NIOCoreTests/XCTest+Extensions.swift
@@ -15,6 +15,10 @@
 import NIOCore
 import XCTest
 
+#if canImport(Android)
+import Android
+#endif
+
 func assert(
     _ condition: @autoclosure () -> Bool,
     within time: TimeAmount,
diff --git a/Tests/NIOEmbeddedTests/TestUtils.swift b/Tests/NIOEmbeddedTests/TestUtils.swift
index 02fe152e7d..c25bc44a1b 100644
--- a/Tests/NIOEmbeddedTests/TestUtils.swift
+++ b/Tests/NIOEmbeddedTests/TestUtils.swift
@@ -17,6 +17,10 @@ import NIOConcurrencyHelpers
 import NIOCore
 import XCTest
 
+#if canImport(Android)
+import Android
+#endif
+
 // FIXME: Duplicated with NIO
 func assert(
     _ condition: @autoclosure () -> Bool,
diff --git a/Tests/NIOFileSystemTests/FileInfoTests.swift b/Tests/NIOFileSystemTests/FileInfoTests.swift
index 2e679d9a90..ae06ab5e29 100644
--- a/Tests/NIOFileSystemTests/FileInfoTests.swift
+++ b/Tests/NIOFileSystemTests/FileInfoTests.swift
@@ -19,8 +19,8 @@ import _NIOFileSystem
 import Darwin
 #elseif canImport(Glibc)
 import Glibc
-#elseif canImport(Bionic)
-import Bionic
+#elseif canImport(Android)
+import Android
 #endif
 
 final class FileInfoTests: XCTestCase {
@@ -44,7 +44,7 @@ final class FileInfoTests: XCTestCase {
         status.st_birthtimespec = timespec(tv_sec: 3, tv_nsec: 0)
         status.st_flags = 11
         status.st_gen = 12
-        #elseif canImport(Glibc) || canImport(Bionic)
+        #elseif canImport(Glibc) || canImport(Android)
         status.st_atim = timespec(tv_sec: 0, tv_nsec: 0)
         status.st_mtim = timespec(tv_sec: 1, tv_nsec: 0)
         status.st_ctim = timespec(tv_sec: 2, tv_nsec: 0)
@@ -98,7 +98,7 @@ final class FileInfoTests: XCTestCase {
         assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_birthtimespec.tv_sec += 1 }
         assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_flags += 1 }
         assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_gen += 1 }
-        #elseif canImport(Glibc) || canImport(Bionic)
+        #elseif canImport(Glibc) || canImport(Android)
         assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_atim.tv_sec += 1 }
         assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_mtim.tv_sec += 1 }
         assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_ctim.tv_sec += 1 }
@@ -151,7 +151,7 @@ final class FileInfoTests: XCTestCase {
         }
         assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_flags += 1 }
         assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_gen += 1 }
-        #elseif canImport(Glibc) || canImport(Bionic)
+        #elseif canImport(Glibc) || canImport(Android)
         assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_atim.tv_sec += 1 }
         assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_mtim.tv_sec += 1 }
         assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_ctim.tv_sec += 1 }
diff --git a/Tests/NIOFileSystemTests/FileTypeTests.swift b/Tests/NIOFileSystemTests/FileTypeTests.swift
index 0655d32a59..b25e2170f9 100644
--- a/Tests/NIOFileSystemTests/FileTypeTests.swift
+++ b/Tests/NIOFileSystemTests/FileTypeTests.swift
@@ -19,6 +19,8 @@ import XCTest
 import Darwin
 #elseif canImport(Glibc)
 import Glibc
+#elseif canImport(Android)
+import Android
 #endif
 
 final class FileTypeTests: XCTestCase {

From 876fbf694b2f27a9783a8e944dddd37bb1710d3e Mon Sep 17 00:00:00 2001
From: Clinton Nkwocha <32041805+clintonpi@users.noreply.github.com>
Date: Tue, 3 Dec 2024 15:29:47 +0000
Subject: [PATCH 24/37] Revert `fastRebase` implementation (#3014)

Motivation:

`UnsafeRawBufferPointer.init(fastRebase:)` and
`UnsafeMutableRawBufferPointer.init(fastRebase:)` were shimmed into NIO
in https://github.com/apple/swift-nio/pull/1696. The shim is no longer
necessary.

Modifications:

- Revert the use of the `fastRebase` inits to the native
`.init(rebasing:)`.

Result:

Use of native APIs instead.
---
 Sources/NIOCore/ByteBuffer-aux.swift  | 13 ++++----
 Sources/NIOCore/ByteBuffer-core.swift | 16 +++++-----
 Sources/NIOCore/ByteBuffer-int.swift  |  2 +-
 Sources/NIOCore/PointerHelpers.swift  | 43 ---------------------------
 Sources/NIOPosix/ControlMessage.swift | 10 +++++--
 Sources/NIOPosix/PointerHelpers.swift | 41 -------------------------
 6 files changed, 25 insertions(+), 100 deletions(-)
 delete mode 100644 Sources/NIOCore/PointerHelpers.swift
 delete mode 100644 Sources/NIOPosix/PointerHelpers.swift

diff --git a/Sources/NIOCore/ByteBuffer-aux.swift b/Sources/NIOCore/ByteBuffer-aux.swift
index 5e07f4af2b..232dffd874 100644
--- a/Sources/NIOCore/ByteBuffer-aux.swift
+++ b/Sources/NIOCore/ByteBuffer-aux.swift
@@ -39,7 +39,7 @@ extension ByteBuffer {
             // this is not technically correct because we shouldn't just bind
             // the memory to `UInt8` but it's not a real issue either and we
             // need to work around https://bugs.swift.org/browse/SR-9604
-            [UInt8](UnsafeRawBufferPointer(fastRebase: ptr[range]).bindMemory(to: UInt8.self))
+            [UInt8](UnsafeRawBufferPointer(rebasing: ptr[range]).bindMemory(to: UInt8.self))
         }
     }
 
@@ -205,7 +205,10 @@ extension ByteBuffer {
         }
         return self.withUnsafeReadableBytes { pointer in
             assert(range.lowerBound >= 0 && (range.upperBound - range.lowerBound) <= pointer.count)
-            return String(decoding: UnsafeRawBufferPointer(fastRebase: pointer[range]), as: Unicode.UTF8.self)
+            return String(
+                decoding: UnsafeRawBufferPointer(rebasing: pointer[range]),
+                as: Unicode.UTF8.self
+            )
         }
     }
 
@@ -329,7 +332,7 @@ extension ByteBuffer {
         self.withVeryUnsafeMutableBytes { destCompleteStorage in
             assert(destCompleteStorage.count >= index + allBytesCount)
             let dest = destCompleteStorage[index..<index + allBytesCount]
-            dispatchData.copyBytes(to: .init(fastRebase: dest), count: dest.count)
+            dispatchData.copyBytes(to: .init(rebasing: dest), count: dest.count)
         }
         return allBytesCount
     }
@@ -348,7 +351,7 @@ extension ByteBuffer {
             return nil
         }
         return self.withUnsafeReadableBytes { pointer in
-            DispatchData(bytes: UnsafeRawBufferPointer(fastRebase: pointer[range]))
+            DispatchData(bytes: UnsafeRawBufferPointer(rebasing: pointer[range]))
         }
     }
 
@@ -497,7 +500,7 @@ extension ByteBuffer {
         precondition(count >= 0, "Can't write fewer than 0 bytes")
         self.reserveCapacity(index + count)
         self.withVeryUnsafeMutableBytes { pointer in
-            let dest = UnsafeMutableRawBufferPointer(fastRebase: pointer[index..<index + count])
+            let dest = UnsafeMutableRawBufferPointer(rebasing: pointer[index..<index + count])
             _ = dest.initializeMemory(as: UInt8.self, repeating: byte)
         }
         return count
diff --git a/Sources/NIOCore/ByteBuffer-core.swift b/Sources/NIOCore/ByteBuffer-core.swift
index f85928451e..812a8cc386 100644
--- a/Sources/NIOCore/ByteBuffer-core.swift
+++ b/Sources/NIOCore/ByteBuffer-core.swift
@@ -503,7 +503,9 @@ public struct ByteBuffer {
 
     @inlinable
     mutating func _setBytesAssumingUniqueBufferAccess(_ bytes: UnsafeRawBufferPointer, at index: _Index) {
-        let targetPtr = UnsafeMutableRawBufferPointer(fastRebase: self._slicedStorageBuffer.dropFirst(Int(index)))
+        let targetPtr = UnsafeMutableRawBufferPointer(
+            rebasing: self._slicedStorageBuffer.dropFirst(Int(index))
+        )
         targetPtr.copyMemory(from: bytes)
     }
 
@@ -515,7 +517,7 @@ public struct ByteBuffer {
         func ensureCapacityAndReturnStorageBase(capacity: Int) -> UnsafeMutablePointer<UInt8> {
             self._ensureAvailableCapacity(_Capacity(capacity), at: index)
             let newBytesPtr = UnsafeMutableRawBufferPointer(
-                fastRebase: self._slicedStorageBuffer[Int(index)..<Int(index) + Int(capacity)]
+                rebasing: self._slicedStorageBuffer[Int(index)..<Int(index) + Int(capacity)]
             )
             return newBytesPtr.bindMemory(to: UInt8.self).baseAddress!
         }
@@ -659,7 +661,7 @@ public struct ByteBuffer {
         self._copyStorageAndRebaseIfNeeded()
         // this is safe because we always know that readerIndex >= writerIndex
         let range = Range<Int>(uncheckedBounds: (lower: self.readerIndex, upper: self.writerIndex))
-        return try body(.init(fastRebase: self._slicedStorageBuffer[range]))
+        return try body(.init(rebasing: self._slicedStorageBuffer[range]))
     }
 
     /// Yields the bytes currently writable (`bytesWritable` = `capacity` - `writerIndex`). Before reading those bytes you must first
@@ -677,7 +679,7 @@ public struct ByteBuffer {
         _ body: (UnsafeMutableRawBufferPointer) throws -> T
     ) rethrows -> T {
         self._copyStorageAndRebaseIfNeeded()
-        return try body(.init(fastRebase: self._slicedStorageBuffer.dropFirst(self.writerIndex)))
+        return try body(.init(rebasing: self._slicedStorageBuffer.dropFirst(self.writerIndex)))
     }
 
     /// This vends a pointer of the `ByteBuffer` at the `writerIndex` after ensuring that the buffer has at least `minimumWritableBytes` of writable bytes available.
@@ -748,7 +750,7 @@ public struct ByteBuffer {
     public func withUnsafeReadableBytes<T>(_ body: (UnsafeRawBufferPointer) throws -> T) rethrows -> T {
         // This is safe, writerIndex >= readerIndex
         let range = Range<Int>(uncheckedBounds: (lower: self.readerIndex, upper: self.writerIndex))
-        return try body(.init(fastRebase: self._slicedStorageBuffer[range]))
+        return try body(.init(rebasing: self._slicedStorageBuffer[range]))
     }
 
     /// Yields a buffer pointer containing this `ByteBuffer`'s readable bytes. You may hold a pointer to those bytes
@@ -769,7 +771,7 @@ public struct ByteBuffer {
         let storageReference: Unmanaged<AnyObject> = Unmanaged.passUnretained(self._storage)
         // This is safe, writerIndex >= readerIndex
         let range = Range<Int>(uncheckedBounds: (lower: self.readerIndex, upper: self.writerIndex))
-        return try body(.init(fastRebase: self._slicedStorageBuffer[range]), storageReference)
+        return try body(.init(rebasing: self._slicedStorageBuffer[range]), storageReference)
     }
 
     /// See `withUnsafeReadableBytesWithStorageManagement` and `withVeryUnsafeBytes`.
@@ -1120,7 +1122,7 @@ extension ByteBuffer {
         self._ensureAvailableCapacity(_Capacity(length), at: _toIndex(toIndex))
         self.withVeryUnsafeMutableBytes { ptr in
             let srcPtr = UnsafeRawBufferPointer(start: ptr.baseAddress!.advanced(by: fromIndex), count: length)
-            let targetPtr = UnsafeMutableRawBufferPointer(fastRebase: ptr.dropFirst(toIndex))
+            let targetPtr = UnsafeMutableRawBufferPointer(rebasing: ptr.dropFirst(toIndex))
             targetPtr.copyMemory(from: srcPtr)
         }
 
diff --git a/Sources/NIOCore/ByteBuffer-int.swift b/Sources/NIOCore/ByteBuffer-int.swift
index d0fb026901..87e0c9abb5 100644
--- a/Sources/NIOCore/ByteBuffer-int.swift
+++ b/Sources/NIOCore/ByteBuffer-int.swift
@@ -67,7 +67,7 @@ extension ByteBuffer {
         return self.withUnsafeReadableBytes { ptr in
             var value: T = 0
             withUnsafeMutableBytes(of: &value) { valuePtr in
-                valuePtr.copyMemory(from: UnsafeRawBufferPointer(fastRebase: ptr[range]))
+                valuePtr.copyMemory(from: UnsafeRawBufferPointer(rebasing: ptr[range]))
             }
             return _toEndianness(value: value, endianness: endianness)
         }
diff --git a/Sources/NIOCore/PointerHelpers.swift b/Sources/NIOCore/PointerHelpers.swift
deleted file mode 100644
index 0acc2f2c2d..0000000000
--- a/Sources/NIOCore/PointerHelpers.swift
+++ /dev/null
@@ -1,43 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// This source file is part of the SwiftNIO open source project
-//
-// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
-// Licensed under Apache License v2.0
-//
-// See LICENSE.txt for license information
-// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-//===----------------------------------------------------------------------===//
-
-// MARK: Rebasing shims
-
-// FIXME: Duplicated in NIO.
-
-// These methods are shimmed in to NIO until https://github.com/apple/swift/pull/34879 is resolved.
-// They address the fact that the current rebasing initializers are surprisingly expensive and do excessive
-// checked arithmetic. This expense forces them to often be outlined, reducing the ability to optimise out
-// further preconditions and branches.
-extension UnsafeRawBufferPointer {
-    @inlinable
-    init(fastRebase slice: Slice<UnsafeRawBufferPointer>) {
-        let base = slice.base.baseAddress?.advanced(by: slice.startIndex)
-        self.init(start: base, count: slice.endIndex &- slice.startIndex)
-    }
-
-    @inlinable
-    init(fastRebase slice: Slice<UnsafeMutableRawBufferPointer>) {
-        let base = slice.base.baseAddress?.advanced(by: slice.startIndex)
-        self.init(start: base, count: slice.endIndex &- slice.startIndex)
-    }
-}
-
-extension UnsafeMutableRawBufferPointer {
-    @inlinable
-    init(fastRebase slice: Slice<UnsafeMutableRawBufferPointer>) {
-        let base = slice.base.baseAddress?.advanced(by: slice.startIndex)
-        self.init(start: base, count: slice.endIndex &- slice.startIndex)
-    }
-}
diff --git a/Sources/NIOPosix/ControlMessage.swift b/Sources/NIOPosix/ControlMessage.swift
index 79cf861e58..6e6338e6b6 100644
--- a/Sources/NIOPosix/ControlMessage.swift
+++ b/Sources/NIOPosix/ControlMessage.swift
@@ -79,7 +79,9 @@ struct UnsafeControlMessageStorage: Collection {
     /// Get the part of the buffer for use with a message.
     public subscript(position: Int) -> UnsafeMutableRawBufferPointer {
         UnsafeMutableRawBufferPointer(
-            fastRebase: self.buffer[(position * self.bytesPerMessage)..<((position + 1) * self.bytesPerMessage)]
+            rebasing: self.buffer[
+                (position * self.bytesPerMessage)..<((position + 1) * self.bytesPerMessage)
+            ]
         )
     }
 
@@ -316,7 +318,9 @@ struct UnsafeOutboundControlBytes {
         type: CInt,
         payload: PayloadType
     ) {
-        let writableBuffer = UnsafeMutableRawBufferPointer(fastRebase: self.controlBytes[writePosition...])
+        let writableBuffer = UnsafeMutableRawBufferPointer(
+            rebasing: self.controlBytes[writePosition...]
+        )
 
         let requiredSize = NIOBSDSocketControlMessage.space(payloadSize: MemoryLayout.stride(ofValue: payload))
         precondition(writableBuffer.count >= requiredSize, "Insufficient size for cmsghdr and data")
@@ -342,7 +346,7 @@ struct UnsafeOutboundControlBytes {
         if writePosition == 0 {
             return UnsafeMutableRawBufferPointer(start: nil, count: 0)
         }
-        return UnsafeMutableRawBufferPointer(fastRebase: self.controlBytes[0..<self.writePosition])
+        return UnsafeMutableRawBufferPointer(rebasing: self.controlBytes[0..<self.writePosition])
     }
 
 }
diff --git a/Sources/NIOPosix/PointerHelpers.swift b/Sources/NIOPosix/PointerHelpers.swift
deleted file mode 100644
index 918fbf6dda..0000000000
--- a/Sources/NIOPosix/PointerHelpers.swift
+++ /dev/null
@@ -1,41 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// This source file is part of the SwiftNIO open source project
-//
-// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
-// Licensed under Apache License v2.0
-//
-// See LICENSE.txt for license information
-// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-//===----------------------------------------------------------------------===//
-
-// MARK: Rebasing shims
-
-// These methods are shimmed in to NIO until https://github.com/apple/swift/pull/34879 is resolved.
-// They address the fact that the current rebasing initializers are surprisingly expensive and do excessive
-// checked arithmetic. This expense forces them to often be outlined, reducing the ability to optimise out
-// further preconditions and branches.
-extension UnsafeRawBufferPointer {
-    @inlinable
-    init(fastRebase slice: Slice<UnsafeRawBufferPointer>) {
-        let base = slice.base.baseAddress?.advanced(by: slice.startIndex)
-        self.init(start: base, count: slice.endIndex &- slice.startIndex)
-    }
-
-    @inlinable
-    init(fastRebase slice: Slice<UnsafeMutableRawBufferPointer>) {
-        let base = slice.base.baseAddress?.advanced(by: slice.startIndex)
-        self.init(start: base, count: slice.endIndex &- slice.startIndex)
-    }
-}
-
-extension UnsafeMutableRawBufferPointer {
-    @inlinable
-    init(fastRebase slice: Slice<UnsafeMutableRawBufferPointer>) {
-        let base = slice.base.baseAddress?.advanced(by: slice.startIndex)
-        self.init(start: base, count: slice.endIndex &- slice.startIndex)
-    }
-}

From d3e8c8c67d11db89607bb8dbff098f8b12d946de Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Thu, 5 Dec 2024 08:23:51 +0000
Subject: [PATCH 25/37] Introduce JSON matrix workflow (#3013)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Introduce and adopt a new method for defining test matrices,
`swift_test_matrix.yml`.

⚠️ Any external adopters of the unit tests, Cxx interop and benchmarks
workflows are automatically opted in to use the new infrastructure.

### Motivation:

* The current matrix workflow has the limitation that it only supports
pre-defined sets of variables which are explored in the test matrix. At
the moment this is a pre-defined set of Swift versions on Linux and
Windows.
* Adding more means hard-coding them at multiple levels of the workflow
hierarchy.
* Currently skipped Windows matrix jobs show up as successes in the
GitHub UI leading to a misleading impression of good coverage.

### Modifications:

Introduce and adopt a new method for defining test matrices,
`swift_test_matrix.yml`. The new method is based around the approach of
defining the test matrix via a JSON object which may be supplied via an
input string from another workflow or from a file on-disk in a
repository.

Taking this approach means that we have the ability to add new targets
to the matrix simply by adding new elements to the JSON array,
increasing flexibility and the scope for future growth.

The unit tests, Cxx interop and benchmarks workflows are all modified to
use the new approach, this opts-in all downstream adopters. This should
be transparent to all downstream adopters.

In order to unify the Linux and Windows jobs I removed the use of the
`container:` GitHub Actions affordance in the Linux jobs which
transparently means all steps are executed within the nested container.
Instead we must manually call in to the docker container which
complicates scripting a little. I tested to see if doing this slowed
down the jobs (perhaps GitHub was caching the docker images more
intelligently) but it does not.

This approach follows the pattern of @FranzBusch 's open PR
https://github.com/apple/swift-nio/pull/2942

### Result:

* More flexible test matrix definitions
* No more false-passes for disabled Windows targets
---
 .github/workflows/benchmarks.yml        |  34 ++++--
 .github/workflows/cxx_interop.yml       |  40 +++++--
 .github/workflows/main.yml              |  29 ++++-
 .github/workflows/pull_request.yml      |  33 +++--
 .github/workflows/swift_test_matrix.yml |  93 ++++++++++++++
 .github/workflows/unit_tests.yml        |  54 ++++++---
 scripts/generate_matrix.sh              | 153 ++++++++++++++++++++++++
 7 files changed, 380 insertions(+), 56 deletions(-)
 create mode 100644 .github/workflows/swift_test_matrix.yml
 create mode 100755 scripts/generate_matrix.sh

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 2b2e4eb38b..f039ba7c55 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -45,18 +45,32 @@ on:
         default: false
 
 jobs:
+  construct-matrix:
+    name: Construct Benchmarks matrix
+    runs-on: ubuntu-latest
+    outputs:
+      benchmarks-matrix: '${{ steps.generate-matrix.outputs.benchmarks-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "benchmarks-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_COMMAND: "swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev"
+          MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }}
+          MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }}
+          MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_6_0_ENABLED: ${{ inputs.linux_nightly_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_MAIN_ENABLED: ${{ inputs.linux_nightly_main_enabled }}
+
   benchmarks:
     name: Benchmarks
+    needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Benchmarks"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev && swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
-      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
-      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
-      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
-      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
-      matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }}
-      matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }}
-      matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }}
+      matrix_string: '${{ needs.construct-matrix.outputs.benchmarks-matrix }}'
diff --git a/.github/workflows/cxx_interop.yml b/.github/workflows/cxx_interop.yml
index 3e66426076..545d596380 100644
--- a/.github/workflows/cxx_interop.yml
+++ b/.github/workflows/cxx_interop.yml
@@ -26,30 +26,44 @@ on:
 
       windows_6_0_enabled:
         type: boolean
-        description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to true."
+        description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to false. Currently has no effect!"  # TODO: implement Windows Cxx compat checking
         default: false
       windows_nightly_6_0_enabled:
         type: boolean
-        description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to true."
+        description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to false. Currently has no effect!"  # TODO: implement Windows Cxx compat checking
         default: false
       windows_nightly_main_enabled:
         type: boolean
-        description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to true."
+        description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to false. Currently has no effect!"  # TODO: implement Windows Cxx compat checking
         default: false
 
 jobs:
+  construct-matrix:
+    name: Construct Cxx interop matrix
+    runs-on: ubuntu-latest
+    outputs:
+      cxx-interop-matrix: '${{ steps.generate-matrix.outputs.cxx-interop-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "cxx-interop-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_COMMAND: "curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q curl jq"
+          MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }}
+          MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }}
+          MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_6_0_ENABLED: ${{ inputs.linux_nightly_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_MAIN_ENABLED: ${{ inputs.linux_nightly_main_enabled }}
+
   cxx-interop:
     name: Cxx interop
+    needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Cxx interop"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q jq && curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
-      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
-      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
-      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
-      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
-      matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }}
-      matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }}
-      matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }}
+      matrix_string: '${{ needs.construct-matrix.outputs.cxx-interop-matrix }}'
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f6fd16711c..b7ff327793 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -10,7 +10,7 @@ jobs:
   unit-tests:
     name: Unit tests
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/unit_tests.yml@main
+    uses: apple/swift-nio/.github/workflows/unit_tests.yml@matrix_file  # TODO: replace with @main
     with:
       linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
@@ -21,19 +21,36 @@ jobs:
   cxx-interop:
     name: Cxx interop
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main
+    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@matrix_file  # TODO: replace with @main
 
   benchmarks:
     name: Benchmarks
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
+    uses: apple/swift-nio/.github/workflows/benchmarks.yml@matrix_file  # TODO: replace with @main
     with:
       benchmark_package_path: "Benchmarks"
 
+  construct-integration-test-matrix:
+    name: Construct integration test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      integration-test-matrix: '${{ steps.generate-matrix.outputs.integration-test-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "integration-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq"
+          MATRIX_LINUX_COMMAND: "./scripts/integration_tests.sh"
+
   integration-tests:
-    name: Integration Tests
+    name: Integration tests
+    needs: construct-integration-test-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Integration tests"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq && ./scripts/integration_tests.sh"
+      matrix_string: '${{ needs.construct-integration-test-matrix.outputs.integration-test-matrix }}'
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 1e5a3b7340..b20fa6ff5a 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -14,33 +14,50 @@ jobs:
   unit-tests:
     name: Unit tests
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/unit_tests.yml@main
+    uses: apple/swift-nio/.github/workflows/unit_tests.yml@matrix_file  # TODO: replace with @main
     with:
       linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
-      linux_6_0_arguments_override: "--explicit-target-dependency-import-check error"
-      linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error"
+      linux_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
+      linux_nightly_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error"
 
   benchmarks:
     name: Benchmarks
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
+    uses: apple/swift-nio/.github/workflows/benchmarks.yml@matrix_file  # TODO: replace with @main
     with:
       benchmark_package_path: "Benchmarks"
 
   cxx-interop:
     name: Cxx interop
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main
+    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@matrix_file  # TODO: replace with @main
+
+  construct-integration-test-matrix:
+    name: Construct integration test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      integration-test-matrix: '${{ steps.generate-matrix.outputs.integration-test-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "integration-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq"
+          MATRIX_LINUX_COMMAND: "./scripts/integration_tests.sh"
 
   integration-tests:
-    name: Integration Tests
+    name: Integration tests
+    needs: construct-integration-test-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Integration tests"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq && ./scripts/integration_tests.sh"
+      matrix_string: '${{ needs.construct-integration-test-matrix.outputs.integration-test-matrix }}'
 
   vsock-tests:
     name: Vsock tests
diff --git a/.github/workflows/swift_test_matrix.yml b/.github/workflows/swift_test_matrix.yml
new file mode 100644
index 0000000000..675bf09157
--- /dev/null
+++ b/.github/workflows/swift_test_matrix.yml
@@ -0,0 +1,93 @@
+name: Matrix
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        type: string
+        description: "The name of the workflow used for the concurrency group."
+        required: true
+      matrix_path:
+        type: string
+        description: "The path of the test matrix definition."
+        default: ""
+      matrix_string:
+        type: string
+        description: "The test matrix definition."
+        default: ""
+
+# We will cancel previously triggered workflow runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }}
+  cancel-in-progress: true
+
+jobs:
+  generate-matrix:
+    name: Prepare matrices
+    runs-on: ubuntu-latest
+    outputs:
+      swift-matrix: ${{ steps.load-matrix.outputs.swift-matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Mark the workspace as safe
+        # https://github.com/actions/checkout/issues/766
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - id: load-matrix
+        run: |
+          if [ -n '${{ inputs.matrix_string }}' ]; then
+            printf "swift-matrix=%s" "$(echo '${{ inputs.matrix_string }}' | jq -c '.')" >> "$GITHUB_OUTPUT"
+          else
+            printf "swift-matrix=%s" "$(jq -c '.' ${{ inputs.matrix_path }})" >> "$GITHUB_OUTPUT"
+          fi
+
+  execute-matrix:
+    name: ${{ matrix.swift.platform }} (${{ matrix.swift.name }})
+    needs: generate-matrix
+    runs-on: ${{ matrix.swift.runner }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.swift-matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          submodules: true
+      - name: Pull Docker image
+        run: docker pull ${{ matrix.swift.image }}
+      - name: Run matrix job
+        if: ${{ matrix.swift.platform != 'Windows' }}
+        run: |
+          if [[ -n "${{ matrix.swift.setup_command }}" ]]; then
+            setup_command_expression="${{ matrix.swift.setup_command }} &&"
+          else
+            setup_command_expression=""
+          fi
+          workspace="/$(basename ${{ github.workspace }})"
+          docker run -v ${{ github.workspace }}:"$workspace" \
+            -w "$workspace" \
+            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" \
+            -e setup_command_expression="$setup_command_expression" \
+            -e workspace="$workspace" \
+            ${{ matrix.swift.image }} \
+            bash -c "swift --version && git config --global --add safe.directory \"$workspace\" && $setup_command_expression ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"
+      - name: Run matrix job (Windows)
+        if: ${{ matrix.swift.platform == 'Windows' }}
+        run: |
+          if (-not [string]::IsNullOrEmpty("${{ matrix.swift.setup_command }}")) {
+              $setup_command_expression = "${{ matrix.swift.setup_command }} &"
+          } else {
+              $setup_command_expression = ""
+          }
+          $workspace = "C:\" + (Split-Path ${{ github.workspace }} -Leaf)
+          docker run -v ${{ github.workspace }}:$($workspace) `
+            -w $($workspace) `
+            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" `
+            -e setup_command_expression=%setup_command_expression% `
+            ${{ matrix.swift.image }} `
+            cmd /s /c  "swift --version & powershell Invoke-Expression ""$($setup_command_expression) ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"""
+    env:
+      SWIFT_VERSION: ${{ matrix.swift.swift_version }}
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index 8a70518bb6..a0be8a1076 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -70,27 +70,43 @@ on:
         default: ""
 
 jobs:
+  construct-matrix:
+    name: Construct unit test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      unit-test-matrix: '${{ steps.generate-matrix.outputs.unit-test-matrix }}'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - id: generate-matrix
+        run: echo "unit-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          MATRIX_LINUX_COMMAND: "swift test"
+          MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }}
+          MATRIX_LINUX_5_9_COMMAND_ARGUMENTS: ${{ inputs.linux_5_9_arguments_override }}
+          MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }}
+          MATRIX_LINUX_5_10_COMMAND_ARGUMENTS: ${{ inputs.linux_5_10_arguments_override }}
+          MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }}
+          MATRIX_LINUX_6_0_COMMAND_ARGUMENTS: ${{ inputs.linux_6_0_arguments_override }}
+          MATRIX_LINUX_NIGHTLY_6_0_ENABLED: ${{ inputs.linux_nightly_6_0_enabled }}
+          MATRIX_LINUX_NIGHTLY_6_0_COMMAND_ARGUMENTS: ${{ inputs.linux_nightly_6_0_arguments_override }}
+          MATRIX_LINUX_NIGHTLY_MAIN_ENABLED: ${{ inputs.linux_nightly_main_enabled }}
+          MATRIX_LINUX_NIGHTLY_MAIN_COMMAND_ARGUMENTS: ${{ inputs.linux_nightly_main_arguments_override }}
+          MATRIX_WINDOWS_COMMAND: "swift test"
+          MATRIX_WINDOWS_6_0_ENABLED: ${{ inputs.windows_6_0_enabled }}
+          MATRIX_WINDOWS_6_0_COMMAND_ARGUMENTS: ${{ inputs.windows_6_0_arguments_override }}
+          MATRIX_WINDOWS_NIGHTLY_6_0_ENABLED: ${{ inputs.windows_nightly_6_0_enabled }}
+          MATRIX_WINDOWS_NIGHTLY_6_0_COMMAND_ARGUMENTS: ${{ inputs.windows_nightly_6_0_arguments_override }}
+          MATRIX_WINDOWS_NIGHTLY_MAIN_ENABLED: ${{ inputs.windows_nightly_main_enabled }}
+          MATRIX_WINDOWS_NIGHTLY_MAIN_COMMAND_ARGUMENTS: ${{ inputs.windows_nightly_main_arguments_override }}
+
   unit-tests:
     name: Unit tests
+    needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
     with:
       name: "Unit tests"
-      matrix_linux_command: "swift test"
-      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
-      matrix_linux_5_9_command_override: "swift test ${{ inputs.linux_5_9_arguments_override }}"
-      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
-      matrix_linux_5_10_command_override: "swift test ${{ inputs.linux_5_10_arguments_override }}"
-      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
-      matrix_linux_6_0_command_override: "swift test ${{ inputs.linux_6_0_arguments_override }}"
-      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-      matrix_linux_nightly_6_0_command_override: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}"
-      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
-      matrix_linux_nightly_main_command_override: "swift test ${{ inputs.linux_nightly_main_arguments_override }}"
-      matrix_windows_command: "swift test"
-      matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }}
-      matrix_windows_6_0_command_override: "swift test ${{ inputs.windows_6_0_arguments_override }}"
-      matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }}
-      matrix_windows_nightly_6_0_command_override: "swift test ${{ inputs.windows_nightly_6_0_arguments_override }}"
-      matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }}
-      matrix_windows_nightly_main_command_override: "swift test ${{ inputs.windows_nightly_main_arguments_override }}"
+      matrix_string: '${{ needs.construct-matrix.outputs.unit-test-matrix }}'
diff --git a/scripts/generate_matrix.sh b/scripts/generate_matrix.sh
new file mode 100755
index 0000000000..dbab4f69a3
--- /dev/null
+++ b/scripts/generate_matrix.sh
@@ -0,0 +1,153 @@
+#!/bin/bash
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+# Parameters
+linux_command="$MATRIX_LINUX_COMMAND"  # required if any Linux pipeline is enabled
+linux_setup_command="$MATRIX_LINUX_SETUP_COMMAND"
+linux_5_9_enabled="${MATRIX_LINUX_5_9_ENABLED:=true}"
+linux_5_9_command_arguments="$MATRIX_LINUX_5_9_COMMAND_ARGUMENTS"
+linux_5_10_enabled="${MATRIX_LINUX_5_10_ENABLED:=true}"
+linux_5_10_command_arguments="$MATRIX_LINUX_5_10_COMMAND_ARGUMENTS"
+linux_6_0_enabled="${MATRIX_LINUX_6_0_ENABLED:=true}"
+linux_6_0_command_arguments="$MATRIX_LINUX_6_0_COMMAND_ARGUMENTS"
+linux_nightly_6_0_enabled="${MATRIX_LINUX_NIGHTLY_6_0_ENABLED:=true}"
+linux_nightly_6_0_command_arguments="$MATRIX_LINUX_NIGHTLY_6_0_COMMAND_ARGUMENTS"
+linux_nightly_main_enabled="${MATRIX_LINUX_NIGHTLY_MAIN_ENABLED:=true}"
+linux_nightly_main_command_arguments="$MATRIX_LINUX_NIGHTLY_MAIN_COMMAND_ARGUMENTS"
+
+windows_command="$MATRIX_WINDOWS_COMMAND"  # required if any Windows pipeline is enabled
+windows_setup_command="$MATRIX_WINDOWS_SETUP_COMMAND"
+windows_6_0_enabled="${MATRIX_WINDOWS_6_0_ENABLED:=false}"
+windows_6_0_command_arguments="$MATRIX_WINDOWS_6_0_COMMAND_ARGUMENTS"
+windows_nightly_6_0_enabled="${MATRIX_WINDOWS_NIGHTLY_6_0_ENABLED:=false}"
+windows_nightly_6_0_command_arguments="$MATRIX_WINDOWS_NIGHTLY_6_0_COMMAND_ARGUMENTS"
+windows_nightly_main_enabled="${MATRIX_WINDOWS_NIGHTLY_MAIN_ENABLED:=false}"
+windows_nightly_main_command_arguments="$MATRIX_WINDOWS_NIGHTLY_MAIN_COMMAND_ARGUMENTS"
+
+# Defaults
+linux_runner="ubuntu-latest"
+linux_5_9_container_image="swift:5.9-jammy"
+linux_5_10_container_image="swift:5.10-jammy"
+linux_6_0_container_image="swift:6.0-jammy"
+linux_nightly_6_0_container_image="swiftlang/swift:nightly-6.0-jammy"
+linux_nightly_main_container_image="swiftlang/swift:nightly-main-jammy"
+
+windows_6_0_runner="windows-2022"
+windows_6_0_container_image="swift:6.0-windowsservercore-ltsc2022"
+windows_nightly_6_0_runner="windows-2019"
+windows_nightly_6_0_container_image="swiftlang/swift:nightly-6.0-windowsservercore-1809"
+windows_nightly_main_runner="windows-2019"
+windows_nightly_main_container_image="swiftlang/swift:nightly-main-windowsservercore-1809"
+
+# Create matrix from inputs
+matrix='{"swift": []}'
+
+## Linux
+if [[ "$linux_5_9_enabled" == "true" || "$linux_5_10_enabled" == "true" || "$linux_6_0_enabled" == "true" || \
+  "$linux_nightly_6_0_enabled" == "true" || "$linux_nightly_main_enabled" == "true" ]]; then
+  if [[ -z "$linux_command" ]]; then
+    echo "No linux command defined"; exit 1
+  fi
+fi
+
+
+if [[ "$linux_5_9_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$linux_setup_command"  \
+    --arg command "$linux_command"  \
+    --arg command_arguments "$linux_5_9_command_arguments" \
+    --arg container_image "$linux_5_9_container_image" \
+    --arg runner "$linux_runner" \
+    '.swift[.swift| length] |= . + { "name": "5.9", "image": $container_image, "swift_version": "5.9", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+fi
+
+if [[ "$linux_5_10_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$linux_setup_command"  \
+    --arg command "$linux_command"  \
+    --arg command_arguments "$linux_5_10_command_arguments" \
+    --arg container_image "$linux_5_10_container_image" \
+    --arg runner "$linux_runner" \
+    '.swift[.swift| length] |= . + { "name": "5.10", "image": $container_image, "swift_version": "5.10", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+fi
+
+if [[ "$linux_6_0_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$linux_setup_command"  \
+    --arg command "$linux_command"  \
+    --arg command_arguments "$linux_6_0_command_arguments" \
+    --arg container_image "$linux_6_0_container_image" \
+    --arg runner "$linux_runner" \
+    '.swift[.swift| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+fi
+
+if [[ "$linux_nightly_6_0_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$linux_setup_command"  \
+    --arg command "$linux_command"  \
+    --arg command_arguments "$linux_nightly_6_0_command_arguments" \
+    --arg container_image "$linux_nightly_6_0_container_image" \
+    --arg runner "$linux_runner" \
+    '.swift[.swift| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+fi
+
+if [[ "$linux_nightly_main_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$linux_setup_command"  \
+    --arg command "$linux_command"  \
+    --arg command_arguments "$linux_nightly_main_command_arguments" \
+    --arg container_image "$linux_nightly_main_container_image" \
+    --arg runner "$linux_runner" \
+    '.swift[.swift| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+fi
+
+## Windows
+if [[ "$windows_6_0_enabled" == "true" || "$windows_nightly_6_0_enabled" == "true" || "$windows_nightly_main_enabled" == "true" ]]; then
+  if [[ -z "$windows_command" ]]; then
+    echo "No windows command defined"; exit 1
+  fi
+fi
+
+if [[ "$windows_6_0_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$windows_setup_command"  \
+    --arg command "$windows_command"  \
+    --arg command_arguments "$windows_6_0_command_arguments" \
+    --arg container_image "$windows_6_0_container_image" \
+    --arg runner "$windows_6_0_runner" \
+    '.swift[.swift| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
+fi
+
+if [[ "$windows_nightly_6_0_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$windows_setup_command"  \
+    --arg command "$windows_command"  \
+    --arg command_arguments "$windows_nightly_6_0_command_arguments" \
+    --arg container_image "$windows_nightly_6_0_container_image" \
+    --arg runner "$windows_nightly_6_0_runner" \
+    '.swift[.swift| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
+fi
+
+if [[ "$windows_nightly_main_enabled" == "true" ]]; then
+  matrix=$(echo "$matrix" | jq -c \
+    --arg setup_command "$windows_setup_command"  \
+    --arg command "$windows_command"  \
+    --arg command_arguments "$windows_nightly_main_command_arguments" \
+    --arg container_image "$windows_nightly_main_container_image" \
+    --arg runner "$windows_nightly_main_runner" \
+    '.swift[.swift| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
+fi
+
+echo "$matrix" | jq -c
\ No newline at end of file

From 692cf0a7b82ae1d669cf6ca1b8c7a1f83650a688 Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Thu, 5 Dec 2024 08:47:27 +0000
Subject: [PATCH 26/37] Matrix workflows refer to main branch (#3016)

Matrix workflows refer to main branch

### Motivation:

Now that the workflow files are committed we can refer to them on the
main branch

### Modifications:

Replace @matrix_file with @main

### Result:

More stable references in workflows.
---
 .github/workflows/benchmarks.yml   | 2 +-
 .github/workflows/cxx_interop.yml  | 2 +-
 .github/workflows/main.yml         | 8 ++++----
 .github/workflows/pull_request.yml | 8 ++++----
 .github/workflows/unit_tests.yml   | 2 +-
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index f039ba7c55..87c22373eb 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -70,7 +70,7 @@ jobs:
     name: Benchmarks
     needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@main
     with:
       name: "Benchmarks"
       matrix_string: '${{ needs.construct-matrix.outputs.benchmarks-matrix }}'
diff --git a/.github/workflows/cxx_interop.yml b/.github/workflows/cxx_interop.yml
index 545d596380..95e3b872ea 100644
--- a/.github/workflows/cxx_interop.yml
+++ b/.github/workflows/cxx_interop.yml
@@ -63,7 +63,7 @@ jobs:
     name: Cxx interop
     needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@main
     with:
       name: "Cxx interop"
       matrix_string: '${{ needs.construct-matrix.outputs.cxx-interop-matrix }}'
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b7ff327793..d578891140 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -10,7 +10,7 @@ jobs:
   unit-tests:
     name: Unit tests
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/unit_tests.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/unit_tests.yml@main
     with:
       linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
@@ -21,12 +21,12 @@ jobs:
   cxx-interop:
     name: Cxx interop
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main
 
   benchmarks:
     name: Benchmarks
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/benchmarks.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
     with:
       benchmark_package_path: "Benchmarks"
 
@@ -50,7 +50,7 @@ jobs:
     name: Integration tests
     needs: construct-integration-test-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@main
     with:
       name: "Integration tests"
       matrix_string: '${{ needs.construct-integration-test-matrix.outputs.integration-test-matrix }}'
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index b20fa6ff5a..2ad5d6c365 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -14,7 +14,7 @@ jobs:
   unit-tests:
     name: Unit tests
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/unit_tests.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/unit_tests.yml@main
     with:
       linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
       linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error"
@@ -25,14 +25,14 @@ jobs:
   benchmarks:
     name: Benchmarks
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/benchmarks.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/benchmarks.yml@main
     with:
       benchmark_package_path: "Benchmarks"
 
   cxx-interop:
     name: Cxx interop
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main
 
   construct-integration-test-matrix:
     name: Construct integration test matrix
@@ -54,7 +54,7 @@ jobs:
     name: Integration tests
     needs: construct-integration-test-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@main
     with:
       name: "Integration tests"
       matrix_string: '${{ needs.construct-integration-test-matrix.outputs.integration-test-matrix }}'
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index a0be8a1076..2b7a82cc64 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -106,7 +106,7 @@ jobs:
     name: Unit tests
     needs: construct-matrix
     # Workaround https://github.com/nektos/act/issues/1875
-    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@matrix_file  # TODO: replace with @main
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@main
     with:
       name: "Unit tests"
       matrix_string: '${{ needs.construct-matrix.outputs.unit-test-matrix }}'

From 234ea1051fec6a2c8ac312e77d4313ee1e134033 Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Thu, 5 Dec 2024 09:22:20 +0000
Subject: [PATCH 27/37] Re-usable workflows curl scripts (#3017)

Re-usable workflows curl scripts

### Motivation:

Other workflows re-use the NIO workflows they don't do a full git
checkout so they don't have access to the scripts, we need to curl them.

### Modifications:

curl the scripts

### Result:

Downstream repositories should be able to re-use these workflows
---
 .github/workflows/benchmarks.yml   | 2 +-
 .github/workflows/cxx_interop.yml  | 2 +-
 .github/workflows/main.yml         | 2 +-
 .github/workflows/pull_request.yml | 2 +-
 .github/workflows/unit_tests.yml   | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 87c22373eb..4fd97e040d 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -56,7 +56,7 @@ jobs:
         with:
           persist-credentials: false
       - id: generate-matrix
-        run: echo "benchmarks-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        run: echo "benchmarks-matrix=$(curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT"
         env:
           MATRIX_LINUX_COMMAND: "swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
           MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev"
diff --git a/.github/workflows/cxx_interop.yml b/.github/workflows/cxx_interop.yml
index 95e3b872ea..3e0dda32e2 100644
--- a/.github/workflows/cxx_interop.yml
+++ b/.github/workflows/cxx_interop.yml
@@ -49,7 +49,7 @@ jobs:
         with:
           persist-credentials: false
       - id: generate-matrix
-        run: echo "cxx-interop-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        run: echo "cxx-interop-matrix=$(curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT"
         env:
           MATRIX_LINUX_COMMAND: "curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
           MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q curl jq"
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d578891140..71ced6aa14 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -41,7 +41,7 @@ jobs:
         with:
           persist-credentials: false
       - id: generate-matrix
-        run: echo "integration-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        run: echo "integration-test-matrix=$(curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT"
         env:
           MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq"
           MATRIX_LINUX_COMMAND: "./scripts/integration_tests.sh"
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 2ad5d6c365..0336d05c6f 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -45,7 +45,7 @@ jobs:
         with:
           persist-credentials: false
       - id: generate-matrix
-        run: echo "integration-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        run: echo "integration-test-matrix=$(curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT"
         env:
           MATRIX_LINUX_SETUP_COMMAND: "apt-get update -y -q && apt-get install -y -q lsof dnsutils netcat-openbsd net-tools curl jq"
           MATRIX_LINUX_COMMAND: "./scripts/integration_tests.sh"
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index 2b7a82cc64..8d064fec5f 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -81,7 +81,7 @@ jobs:
         with:
           persist-credentials: false
       - id: generate-matrix
-        run: echo "unit-test-matrix=$(./scripts/generate_matrix.sh)" >> "$GITHUB_OUTPUT"
+        run: echo "unit-test-matrix=$(curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/generate_matrix.sh | bash)" >> "$GITHUB_OUTPUT"
         env:
           MATRIX_LINUX_COMMAND: "swift test"
           MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }}

From 74f767454f648a43176f0161784557898dcb2d7a Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Thu, 5 Dec 2024 15:03:53 +0000
Subject: [PATCH 28/37] GHA: Separate matrix input handling (#3018)

### Motivation:

The new test matrix workflow is quite verbose in the most common case,
unit tests, outputting two workflow entries for handling and preparing
the matrix inputs.

In addition to this, it appeared that when running in `act` the matrix
preparation step would run multiple times.

### Modifications:

* Prioritize the current most common case, string inputs, and skip the
matrix preparation step in this case reducing the number of workflows
run.
* Separate the path for loading the JSON matrix from a file into its own
workflow.

### Result:

* Less noise in the output
* More reliable and efficient local matrix runs
---
 .github/workflows/swift_load_test_matrix.yml | 40 ++++++++++++++++++++
 .github/workflows/swift_test_matrix.yml      | 30 +--------------
 2 files changed, 42 insertions(+), 28 deletions(-)
 create mode 100644 .github/workflows/swift_load_test_matrix.yml

diff --git a/.github/workflows/swift_load_test_matrix.yml b/.github/workflows/swift_load_test_matrix.yml
new file mode 100644
index 0000000000..f0c34c2da5
--- /dev/null
+++ b/.github/workflows/swift_load_test_matrix.yml
@@ -0,0 +1,40 @@
+name: Matrix Load
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        type: string
+        description: "The name of the workflow used for the concurrency group."
+        required: true
+      matrix_path:
+        type: string
+        description: "The path of the test matrix definition."
+        default: ""
+
+jobs:
+  load-matrix:
+    name: Prepare matrices
+    runs-on: ubuntu-latest
+    outputs:
+      swift-matrix: ${{ steps.load-matrix.outputs.swift-matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Mark the workspace as safe
+        # https://github.com/actions/checkout/issues/766
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - id: load-matrix
+        run: |
+            printf "swift-matrix=%s" "$(jq -c '.' ${{ inputs.matrix_path }})" >> "$GITHUB_OUTPUT"
+
+  execute-matrix:
+    name: Execute matrix
+    needs: load-matrix
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: apple/swift-nio/.github/workflows/swift_test_matrix.yml@main
+    with:
+      name: ${{ inputs.name }}
+      matrix_string: '${{ needs.load-matrix.outputs.swift-matrix }}'
diff --git a/.github/workflows/swift_test_matrix.yml b/.github/workflows/swift_test_matrix.yml
index 675bf09157..cec7fd70f5 100644
--- a/.github/workflows/swift_test_matrix.yml
+++ b/.github/workflows/swift_test_matrix.yml
@@ -7,14 +7,10 @@ on:
         type: string
         description: "The name of the workflow used for the concurrency group."
         required: true
-      matrix_path:
-        type: string
-        description: "The path of the test matrix definition."
-        default: ""
       matrix_string:
         type: string
         description: "The test matrix definition."
-        default: ""
+        required: true
 
 # We will cancel previously triggered workflow runs
 concurrency:
@@ -22,34 +18,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  generate-matrix:
-    name: Prepare matrices
-    runs-on: ubuntu-latest
-    outputs:
-      swift-matrix: ${{ steps.load-matrix.outputs.swift-matrix }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: Mark the workspace as safe
-        # https://github.com/actions/checkout/issues/766
-        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
-      - id: load-matrix
-        run: |
-          if [ -n '${{ inputs.matrix_string }}' ]; then
-            printf "swift-matrix=%s" "$(echo '${{ inputs.matrix_string }}' | jq -c '.')" >> "$GITHUB_OUTPUT"
-          else
-            printf "swift-matrix=%s" "$(jq -c '.' ${{ inputs.matrix_path }})" >> "$GITHUB_OUTPUT"
-          fi
-
   execute-matrix:
     name: ${{ matrix.swift.platform }} (${{ matrix.swift.name }})
-    needs: generate-matrix
     runs-on: ${{ matrix.swift.runner }}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs.swift-matrix) }}
+      matrix: ${{ fromJson(inputs.matrix_string) }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From c3a8d18219e32c1eb218c96243eefa130c05cf9f Mon Sep 17 00:00:00 2001
From: Si Beaumont <beaumont@apple.com>
Date: Fri, 6 Dec 2024 17:08:56 +0000
Subject: [PATCH 29/37] Add EventLoop.now API for getting the current time
 (#3015)

### Motivation

Code that tries to work with `NIODeadline` can be challenging to test
using `EmbeddedEventLoop` and `NIOAsyncTestingEventLoop` because they
have their own, fake clock.

These testing event loops do implement `scheduleTask(in:_)` to submit
work in the future, relative to the event loop clock, but there is no
way to get the event loop's current notion of "now", as a `NIODeadline`,
and users sometimes find that `NIODeadline.now`, which returns the time
of the real clock, to be surprising.

### Modifications

Add `EventLoop.now`  to get the current time of the event loop clock.

### Result

New APIs to support writing code that's easier to test.
---
 Sources/NIOCore/EventLoop.swift               |  8 +++++
 .../NIOEmbedded/AsyncTestingEventLoop.swift   |  4 ++-
 Sources/NIOEmbedded/Embedded.swift            |  3 +-
 Sources/NIOPosix/SelectableEventLoop.swift    |  6 ++++
 .../AsyncTestingEventLoopTests.swift          | 10 ++++++
 .../EmbeddedEventLoopTest.swift               | 33 ++++++++++++++++++-
 Tests/NIOPosixTests/EventLoopTest.swift       |  8 +++++
 7 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/Sources/NIOCore/EventLoop.swift b/Sources/NIOCore/EventLoop.swift
index 376f06d8fb..021725af97 100644
--- a/Sources/NIOCore/EventLoop.swift
+++ b/Sources/NIOCore/EventLoop.swift
@@ -269,6 +269,9 @@ public protocol EventLoop: EventLoopGroup {
     @preconcurrency
     func submit<T>(_ task: @escaping @Sendable () throws -> T) -> EventLoopFuture<T>
 
+    /// The current time of the event loop.
+    var now: NIODeadline { get }
+
     /// Schedule a `task` that is executed by this `EventLoop` at the given time.
     ///
     /// - Parameters:
@@ -394,6 +397,11 @@ public protocol EventLoop: EventLoopGroup {
     func cancelScheduledCallback(_ scheduledCallback: NIOScheduledCallback)
 }
 
+extension EventLoop {
+    /// Default implementation of `now`: Returns `NIODeadline.now()`.
+    public var now: NIODeadline { .now() }
+}
+
 extension EventLoop {
     /// Default implementation of `makeSucceededVoidFuture`: Return a fresh future (which will allocate).
     public func makeSucceededVoidFuture() -> EventLoopFuture<Void> {
diff --git a/Sources/NIOEmbedded/AsyncTestingEventLoop.swift b/Sources/NIOEmbedded/AsyncTestingEventLoop.swift
index 14e5242df0..7997d38e36 100644
--- a/Sources/NIOEmbedded/AsyncTestingEventLoop.swift
+++ b/Sources/NIOEmbedded/AsyncTestingEventLoop.swift
@@ -63,7 +63,9 @@ public final class NIOAsyncTestingEventLoop: EventLoop, @unchecked Sendable {
     /// The current "time" for this event loop. This is an amount in nanoseconds.
     /// As we need to access this from any thread, we store this as an atomic.
     private let _now = ManagedAtomic<UInt64>(0)
-    internal var now: NIODeadline {
+
+    /// The current "time" for this event loop. This is an amount in nanoseconds.
+    public var now: NIODeadline {
         NIODeadline.uptimeNanoseconds(self._now.load(ordering: .relaxed))
     }
 
diff --git a/Sources/NIOEmbedded/Embedded.swift b/Sources/NIOEmbedded/Embedded.swift
index c509a8ffa3..70148c4163 100644
--- a/Sources/NIOEmbedded/Embedded.swift
+++ b/Sources/NIOEmbedded/Embedded.swift
@@ -94,8 +94,9 @@ extension EmbeddedScheduledTask: Comparable {
 ///     responsible for ensuring they never call into the `EmbeddedEventLoop` in an
 ///     unsynchronized fashion.
 public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible {
+    private var _now: NIODeadline = .uptimeNanoseconds(0)
     /// The current "time" for this event loop. This is an amount in nanoseconds.
-    internal var _now: NIODeadline = .uptimeNanoseconds(0)
+    public var now: NIODeadline { _now }
 
     private enum State { case open, closing, closed }
     private var state: State = .open
diff --git a/Sources/NIOPosix/SelectableEventLoop.swift b/Sources/NIOPosix/SelectableEventLoop.swift
index 89fc4737f9..9449d62978 100644
--- a/Sources/NIOPosix/SelectableEventLoop.swift
+++ b/Sources/NIOPosix/SelectableEventLoop.swift
@@ -298,6 +298,12 @@ internal final class SelectableEventLoop: EventLoop {
         thread.isCurrent
     }
 
+    /// - see: `EventLoop.now`
+    @usableFromInline
+    internal var now: NIODeadline {
+        .now()
+    }
+
     /// - see: `EventLoop.scheduleTask(deadline:_:)`
     @inlinable
     internal func scheduleTask<T>(deadline: NIODeadline, _ task: @escaping () throws -> T) -> Scheduled<T> {
diff --git a/Tests/NIOEmbeddedTests/AsyncTestingEventLoopTests.swift b/Tests/NIOEmbeddedTests/AsyncTestingEventLoopTests.swift
index 08cee06cb4..66c40c7414 100644
--- a/Tests/NIOEmbeddedTests/AsyncTestingEventLoopTests.swift
+++ b/Tests/NIOEmbeddedTests/AsyncTestingEventLoopTests.swift
@@ -619,4 +619,14 @@ final class NIOAsyncTestingEventLoopTests: XCTestCase {
         await eventLoop.advanceTime(by: .seconds(1))
         XCTAssertEqual(counter.load(ordering: .relaxed), 3)
     }
+
+    func testCurrentTime() async {
+        let eventLoop = NIOAsyncTestingEventLoop()
+
+        await eventLoop.advanceTime(to: .uptimeNanoseconds(42))
+        XCTAssertEqual(eventLoop.now, .uptimeNanoseconds(42))
+
+        await eventLoop.advanceTime(by: .nanoseconds(42))
+        XCTAssertEqual(eventLoop.now, .uptimeNanoseconds(84))
+    }
 }
diff --git a/Tests/NIOEmbeddedTests/EmbeddedEventLoopTest.swift b/Tests/NIOEmbeddedTests/EmbeddedEventLoopTest.swift
index a23665ec25..b1690f7f86 100644
--- a/Tests/NIOEmbeddedTests/EmbeddedEventLoopTest.swift
+++ b/Tests/NIOEmbeddedTests/EmbeddedEventLoopTest.swift
@@ -396,7 +396,7 @@ public final class EmbeddedEventLoopTest: XCTestCase {
                 try eventLoop.syncShutdownGracefully()
                 childTasks.append(
                     scheduleRecursiveTask(
-                        at: eventLoop._now + childTaskStartDelay,
+                        at: eventLoop.now + childTaskStartDelay,
                         andChildTaskAfter: childTaskStartDelay
                     )
                 )
@@ -497,4 +497,35 @@ public final class EmbeddedEventLoopTest: XCTestCase {
         eventLoop.advanceTime(by: .seconds(1))
         XCTAssertEqual(counter, 3)
     }
+
+    func testCurrentTime() {
+        let eventLoop = EmbeddedEventLoop()
+
+        eventLoop.advanceTime(to: .uptimeNanoseconds(42))
+        XCTAssertEqual(eventLoop.now, .uptimeNanoseconds(42))
+
+        eventLoop.advanceTime(by: .nanoseconds(42))
+        XCTAssertEqual(eventLoop.now, .uptimeNanoseconds(84))
+    }
+
+    func testScheduleRepeatedTask() {
+        let eventLoop = EmbeddedEventLoop()
+
+        var counter = 0
+        eventLoop.scheduleRepeatedTask(initialDelay: .seconds(1), delay: .seconds(1)) { repeatedTask in
+            guard counter < 10 else {
+                repeatedTask.cancel()
+                return
+            }
+            counter += 1
+        }
+
+        XCTAssertEqual(counter, 0)
+
+        eventLoop.advanceTime(by: .seconds(1))
+        XCTAssertEqual(counter, 1)
+
+        eventLoop.advanceTime(by: .seconds(9))
+        XCTAssertEqual(counter, 10)
+    }
 }
diff --git a/Tests/NIOPosixTests/EventLoopTest.swift b/Tests/NIOPosixTests/EventLoopTest.swift
index f14ba2dc4f..947d916be0 100644
--- a/Tests/NIOPosixTests/EventLoopTest.swift
+++ b/Tests/NIOPosixTests/EventLoopTest.swift
@@ -1962,6 +1962,10 @@ private class EventLoopWithPreSucceededFuture: EventLoop {
         preconditionFailure("not implemented")
     }
 
+    var now: NIODeadline {
+        preconditionFailure("not implemented")
+    }
+
     @discardableResult
     func scheduleTask<T>(deadline: NIODeadline, _ task: @escaping () throws -> T) -> Scheduled<T> {
         preconditionFailure("not implemented")
@@ -2013,6 +2017,10 @@ private class EventLoopWithoutPreSucceededFuture: EventLoop {
         preconditionFailure("not implemented")
     }
 
+    var now: NIODeadline {
+        preconditionFailure("not implemented")
+    }
+
     @discardableResult
     func scheduleTask<T>(deadline: NIODeadline, _ task: @escaping () throws -> T) -> Scheduled<T> {
         preconditionFailure("not implemented")

From 29c65edb970945a8eea0ef1f654a97ba4e907495 Mon Sep 17 00:00:00 2001
From: Michael Gecht <gecht@apple.com>
Date: Tue, 10 Dec 2024 17:43:29 +0000
Subject: [PATCH 30/37] Add parallel removal of items (#3008)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Introduce parallel removal of items, instead of only doing a sequential
removal (and discovery) one-by-one

### Motivation:

#2933 mentioned this function call appears to be slow. If we attempt to
delete a directory with lots of subdirectories, we end up discovering
each subdirectory one by one on a single thread.

### Modifications:

This PR refactors the logic to remove items. Instead of sequentially
discovering and deleting items one by one, we traverse all directories
up to a maximum concurrency, and issue file deletions in parallel within
each directory.

### Result:

The new concurrent implementation was measured to be ~1.4x faster than
`FileManager`, ~1.5x faster than `rm -rf` and ~3x faster than the
current sequential implementation on `main`. This new concurrent
implementation does use the most CPU out of all the other approaches.

Results are broken down into two sections. We delete a copy of the
`swift-nio` repository, where the `.build/` folder has grown to a large
size, providing lots of additional files and subdirectories.

We produce a single CLI using `--configration=release` that can switch
between using the `.sequential` mode (what is currently available on the
`main` branch), `.parallel` (the new concurrent implementation in this
PR) and `filemanager` (uses `Foundation.FileManager` as a comparison).

Resolves https://github.com/apple/swift-nio/issues/2933

#### Normal size

**Description:** The `swift-nio` library as found when building the CLI
used for benchmarking.
**Benchmark run with:** 3579 directories, 11657 files

```
$ hyperfine --runs 10 --warmup 2 --prepare "rm -rf /tmp/swift-nio && cp -r $(pwd) /tmp/swift-nio" ".build/release/NIOTestCLI --sequential /tmp/swift-nio" ".build/release/NIOTestCLI --parallel /tmp/swift-nio" ".build/release/NIOTestCLI --filemanager /tmp/swift-nio" "rm -rf /tmp/swift-nio"
Benchmark 1: .build/release/NIOTestCLI --sequential /tmp/swift-nio
  Time (mean ± σ):      2.712 s ±  0.053 s    [User: 0.526 s, System: 8.632 s]
  Range (min … max):    2.625 s …  2.781 s    10 runs

Benchmark 2: .build/release/NIOTestCLI --parallel /tmp/swift-nio
  Time (mean ± σ):     896.7 ms ±  25.6 ms    [User: 302.9 ms, System: 5313.5 ms]
  Range (min … max):   853.2 ms … 942.1 ms    10 runs

Benchmark 3: .build/release/NIOTestCLI --filemanager /tmp/swift-nio
  Time (mean ± σ):      1.337 s ±  0.076 s    [User: 0.022 s, System: 1.253 s]
  Range (min … max):    1.252 s …  1.508 s    10 runs

Benchmark 4: rm -rf /tmp/swift-nio
  Time (mean ± σ):      1.483 s ±  0.037 s    [User: 0.015 s, System: 1.416 s]
  Range (min … max):    1.413 s …  1.541 s    10 runs

Summary
  .build/release/NIOTestCLI --parallel /tmp/swift-nio ran
    1.49 ± 0.10 times faster than .build/release/NIOTestCLI --filemanager /tmp/swift-nio
    1.65 ± 0.06 times faster than rm -rf /tmp/swift-nio
    3.02 ± 0.10 times faster than .build/release/NIOTestCLI --sequential /tmp/swift-nio
```

#### Large size

**Description:** The `swift-nio` library as found after re-running the
`.github` workflow many times locally. I'm unable to reproduce this
directory size, but have benchmarked against it.
**Benchmark run with:** 15260 directories, 46842 files

```
$ hyperfine --runs 10 --warmup 2 --prepare "rm -rf /tmp/swift-nio && cp -r $(pwd) /tmp/swift-nio" ".build/release/NIOTestCLI --sequential /tmp/swift-nio" ".build/release/NIOTestCLI --parallel /tmp/swift-nio" ".build/release/NIOTestCLI --filemanager /tmp/swift-nio" "rm -rf /tmp/swift-nio"
Benchmark 1: .build/release/NIOTestCLI --sequential /tmp/swift-nio
  Time (mean ± σ):     12.082 s ±  0.354 s    [User: 2.180 s, System: 41.547 s]
  Range (min … max):   11.717 s … 12.923 s    10 runs

Benchmark 2: .build/release/NIOTestCLI --parallel /tmp/swift-nio
  Time (mean ± σ):      4.770 s ±  0.102 s    [User: 1.357 s, System: 25.785 s]
  Range (min … max):    4.570 s …  4.883 s    10 runs

Benchmark 3: .build/release/NIOTestCLI --filemanager /tmp/swift-nio
  Time (mean ± σ):      6.503 s ±  0.115 s    [User: 0.081 s, System: 5.427 s]
  Range (min … max):    6.262 s …  6.687 s    10 runs

Benchmark 4: rm -rf /tmp/swift-nio
  Time (mean ± σ):      7.056 s ±  0.246 s    [User: 0.064 s, System: 6.119 s]
  Range (min … max):    6.435 s …  7.233 s    10 runs

Summary
  .build/release/NIOTestCLI --parallel /tmp/swift-nio ran
    1.36 ± 0.04 times faster than .build/release/NIOTestCLI --filemanager /tmp/swift-nio
    1.48 ± 0.06 times faster than rm -rf /tmp/swift-nio
    2.53 ± 0.09 times faster than .build/release/NIOTestCLI --sequential /tmp/swift-nio
```

---------

Co-authored-by: George Barnett <gbarnett@apple.com>
---
 NOTICE.txt                                    |   9 +
 Sources/NIOFileSystem/CopyStrategy.swift      | 121 ------------
 Sources/NIOFileSystem/FileSystem.swift        |  80 +++++---
 .../NIOFileSystem/FileSystemProtocol.swift    |  76 +++++++-
 Sources/NIOFileSystem/IOStrategy.swift        | 184 ++++++++++++++++++
 .../Concurrency Primitives/TokenBucket.swift  |  88 +++++++++
 .../Internal/ParallelRemoval.swift            |  72 +++++++
 .../FileSystemTests.swift                     |  49 +++--
 8 files changed, 509 insertions(+), 170 deletions(-)
 delete mode 100644 Sources/NIOFileSystem/CopyStrategy.swift
 create mode 100644 Sources/NIOFileSystem/IOStrategy.swift
 create mode 100644 Sources/NIOFileSystem/Internal/Concurrency Primitives/TokenBucket.swift
 create mode 100644 Sources/NIOFileSystem/Internal/ParallelRemoval.swift

diff --git a/NOTICE.txt b/NOTICE.txt
index cbcc2e69bb..d79250ea55 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -95,3 +95,12 @@ This product contains a derivation of the mocking infrastructure from Swift Syst
     * https://www.apache.org/licenses/LICENSE-2.0
   * HOMEPAGE:
     * https://github.com/apple/swift-system
+
+---
+
+This product contains a derivation of "TokenBucket.swift" from Swift Package Manager.
+
+  * LICENSE (Apache License 2.0):
+    * https://www.apache.org/licenses/LICENSE-2.0
+  * HOMEPAGE:
+    * https://github.com/swiftlang/swift-package-manager
diff --git a/Sources/NIOFileSystem/CopyStrategy.swift b/Sources/NIOFileSystem/CopyStrategy.swift
deleted file mode 100644
index ae8b4770e1..0000000000
--- a/Sources/NIOFileSystem/CopyStrategy.swift
+++ /dev/null
@@ -1,121 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// This source file is part of the SwiftNIO open source project
-//
-// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
-// Licensed under Apache License v2.0
-//
-// See LICENSE.txt for license information
-// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-//===----------------------------------------------------------------------===//
-
-/// How to perform copies. Currently only relevant to directory level copies when using
-/// ``FileSystemProtocol/copyItem(at:to:strategy:shouldProceedAfterError:shouldCopyItem:)`` or other
-/// overloads that use the default behaviour.
-public struct CopyStrategy: Hashable, Sendable {
-    // Avoid exposing to prevent breaking changes
-    internal enum Wrapped: Hashable, Sendable {
-        // platformDefault is reified into one of the concrete options below:
-
-        case sequential
-        // Constraints on this value are enforced only on creation of `CopyStrategy`. The early
-        // error check is desirable over validating on downstream use.
-        case parallel(_ maxDescriptors: Int)
-    }
-
-    internal let wrapped: Wrapped
-    private init(_ wrapped: Wrapped) {
-        self.wrapped = wrapped
-    }
-
-    // These selections are relatively arbitrary but the rationale is as follows:
-    //
-    // - Never exceed the default OS limits even if 4 such operations were happening at once.
-    // - Sufficient to enable significant speed up from parallelism
-    // - Not wasting effort by pushing contention to the underlying storage device. Further we
-    //   assume an SSD or similar underlying storage tech. Users on spinning rust need to account
-    //   for that themselves anyway.
-    //
-    // That said, empirical testing for this has not been performed, suggestions welcome.
-    //
-    // Note: for now we model the directory scan as needing two handles because, during the creation
-    // of the destination directory we hold the handle for a while copying attributes
-    // a much more complex internal state machine could allow doing two of these if desired
-    // This may not result in a faster copy though so things are left simple
-    internal static func determinePlatformDefault() -> Wrapped {
-        #if os(macOS) || os(Linux) || os(Windows)
-        // 4 concurrent file copies/directory scans. Avoiding storage system contention is of utmost
-        // importance.
-        //
-        // Testing was performed on an SSD, while copying objects (a dense directory of small files
-        // and subdirectories of similar shape) to the same volume, totalling 12GB. Results showed
-        // improvements in elapsed time for (expected) increases in CPU time up to parallel(8).
-        // Beyond this, the increases in CPU led to only moderate gains.
-        //
-        // Anyone tuning this is encouraged to cover worst case scenarios.
-        return .parallel(8)
-        #elseif os(iOS) || os(tvOS) || os(watchOS) || os(Android)
-        // Reduced maximum descriptors in embedded world. This is chosen based on biasing towards
-        // safety, not empirical testing.
-        return .parallel(4)
-        #else
-        // Safety first. If we do not know what system we run on, we keep it simple.
-        return .sequential
-        #endif
-    }
-}
-
-extension CopyStrategy {
-    // A copy fundamentally can't work without two descriptors unless you copy
-    // everything into memory which is infeasible/inefficient for large copies.
-    private static let minDescriptorsAllowed = 2
-
-    /// Operate in whatever manner is deemed a reasonable default for the platform. This will limit
-    /// the maximum file descriptors usage based on reasonable defaults.
-    ///
-    /// Current assumptions (which are subject to change):
-    /// - Only one copy operation would be performed at once
-    /// - The copy operation is not intended to be the primary activity on the device
-    public static let platformDefault: Self = Self(Self.determinePlatformDefault())
-
-    /// The copy is done asynchronously, but only one operation will occur at a time. This is the
-    /// only way to guarantee only one callback to the `shouldCopyItem` will happen at a time.
-    public static let sequential: Self = Self(.sequential)
-
-    /// Allow multiple IO operations to run concurrently, including file copies/directory creation and scanning
-    ///
-    /// - Parameter maxDescriptors: a conservative limit on the number of concurrently open
-    ///     file descriptors involved in the copy. This number must be >= 2 though, if you are using a value that low
-    ///     you should use ``sequential``
-    ///
-    /// - Throws: ``FileSystemError/Code-swift.struct/invalidArgument`` if `maxDescriptors`
-    /// is less than 2.
-    ///
-    public static func parallel(maxDescriptors: Int) throws -> Self {
-        guard maxDescriptors >= Self.minDescriptorsAllowed else {
-            // 2 is not quite the same as sequential, you could have two concurrent directory listings for example
-            // less than 2 and you can't actually do a _copy_ though so it's non-sensical.
-            throw FileSystemError(
-                code: .invalidArgument,
-                message: "Can't do a copy operation without at least 2 file descriptors '\(maxDescriptors)' is illegal",
-                cause: nil,
-                location: .here()
-            )
-        }
-        return .init(.parallel(maxDescriptors))
-    }
-}
-
-extension CopyStrategy: CustomStringConvertible {
-    public var description: String {
-        switch self.wrapped {
-        case .sequential:
-            return "sequential"
-        case let .parallel(maxDescriptors):
-            return "parallel with max \(maxDescriptors) descriptors"
-        }
-    }
-}
diff --git a/Sources/NIOFileSystem/FileSystem.swift b/Sources/NIOFileSystem/FileSystem.swift
index 33bf8c605e..8349949254 100644
--- a/Sources/NIOFileSystem/FileSystem.swift
+++ b/Sources/NIOFileSystem/FileSystem.swift
@@ -366,6 +366,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         }
     }
 
+    /// See ``FileSystemProtocol/removeItem(at:strategy:recursively:)``
+    ///
     /// Deletes the file or directory (and its contents) at `path`.
     ///
     /// Only regular files, symbolic links and directories may be removed. If the file at `path` is
@@ -387,11 +389,14 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     ///
     /// - Parameters:
     ///   - path: The path to delete.
+    ///   - removalStrategy: Whether to delete files sequentially (one-by-one), or perform a
+    ///       concurrent scan of the tree at `path` and delete files when they are found.
     ///   - removeItemRecursively: Whether or not to remove items recursively.
     /// - Returns: The number of deleted items which may be zero if `path` did not exist.
     @discardableResult
     public func removeItem(
         at path: FilePath,
+        strategy removalStrategy: RemovalStrategy,
         recursively removeItemRecursively: Bool
     ) async throws -> Int {
         // Try to remove the item: we might just get lucky.
@@ -421,39 +426,60 @@ public struct FileSystem: Sendable, FileSystemProtocol {
                 )
             }
 
-            var (subdirectories, filesRemoved) = try await self.withDirectoryHandle(
-                atPath: path
-            ) { directory in
-                var subdirectories = [FilePath]()
-                var filesRemoved = 0
+            switch removalStrategy.wrapped {
+            case .sequential:
+                return try await self.removeItemSequentially(at: path)
+            case let .parallel(maxDescriptors):
+                return try await self.removeConcurrently(at: path, maxDescriptors)
+            }
 
-                for try await batch in directory.listContents().batched() {
-                    for entry in batch {
-                        switch entry.type {
-                        case .directory:
-                            subdirectories.append(entry.path)
+        case let .failure(errno):
+            throw FileSystemError.remove(errno: errno, path: path, location: .here())
+        }
+    }
 
-                        default:
-                            filesRemoved += try await self.removeOneItem(at: entry.path)
-                        }
+    @discardableResult
+    private func removeItemSequentially(
+        at path: FilePath
+    ) async throws -> Int {
+        var (subdirectories, filesRemoved) = try await self.withDirectoryHandle(
+            atPath: path
+        ) { directory in
+            var subdirectories = [FilePath]()
+            var filesRemoved = 0
+
+            for try await batch in directory.listContents().batched() {
+                for entry in batch {
+                    switch entry.type {
+                    case .directory:
+                        subdirectories.append(entry.path)
+
+                    default:
+                        filesRemoved += try await self.removeOneItem(at: entry.path)
                     }
                 }
-
-                return (subdirectories, filesRemoved)
             }
 
-            for subdirectory in subdirectories {
-                filesRemoved += try await self.removeItem(at: subdirectory)
-            }
+            return (subdirectories, filesRemoved)
+        }
 
-            // The directory should be empty now. Remove ourself.
-            filesRemoved += try await self.removeOneItem(at: path)
+        for subdirectory in subdirectories {
+            filesRemoved += try await self.removeItemSequentially(at: subdirectory)
+        }
 
-            return filesRemoved
+        // The directory should be empty now. Remove ourself.
+        filesRemoved += try await self.removeOneItem(at: path)
 
-        case let .failure(errno):
-            throw FileSystemError.remove(errno: errno, path: path, location: .here())
-        }
+        return filesRemoved
+
+    }
+
+    private func removeConcurrently(
+        at path: FilePath,
+        _ maxDescriptors: Int
+    ) async throws -> Int {
+        let bucket: TokenBucket = .init(tokens: maxDescriptors)
+        return try await self.discoverAndRemoveItemsInTree(at: path, bucket)
     }
 
     /// Moves the named file or directory to a new location.
@@ -490,7 +516,7 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         case .differentLogicalDevices:
             // Fall back to copy and remove.
             try await self.copyItem(at: sourcePath, to: destinationPath)
-            try await self.removeItem(at: sourcePath)
+            try await self.removeItem(at: sourcePath, strategy: .platformDefault)
         }
     }
 
@@ -518,9 +544,9 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         withItemAt existingPath: FilePath
     ) async throws {
         do {
-            try await self.removeItem(at: destinationPath)
+            try await self.removeItem(at: destinationPath, strategy: .platformDefault)
             try await self.moveItem(at: existingPath, to: destinationPath)
-            try await self.removeItem(at: existingPath)
+            try await self.removeItem(at: existingPath, strategy: .platformDefault)
         } catch let error as FileSystemError {
             throw FileSystemError(
                 message: "Can't replace '\(destinationPath)' with '\(existingPath)'.",
diff --git a/Sources/NIOFileSystem/FileSystemProtocol.swift b/Sources/NIOFileSystem/FileSystemProtocol.swift
index b5bf701435..934220762c 100644
--- a/Sources/NIOFileSystem/FileSystemProtocol.swift
+++ b/Sources/NIOFileSystem/FileSystemProtocol.swift
@@ -249,19 +249,23 @@ public protocol FileSystemProtocol: Sendable {
     /// at the given path then this function returns zero.
     ///
     /// If the item at the `path` is a directory and `removeItemRecursively` is `true` then the
-    /// contents of all of its subdirectories will be removed recursively before the directory
-    /// at `path`. Symbolic links are removed (but their targets are not deleted).
+    /// contents of all of its subdirectories will be removed recursively before the directory at
+    /// `path`. Symbolic links are removed (but their targets are not deleted).
     ///
     /// - Parameters:
     ///   - path: The path to delete.
-    ///   - removeItemRecursively: If the item being removed is a directory, remove it by
-    ///       recursively removing its children. Setting this to `true` is synonymous with
-    ///       calling `rm -r`, setting this false is synonymous to calling `rmdir`. Ignored if
+    ///   - removalStrategy: Whether to delete files sequentially (one-by-one), or perform a
+    ///       concurrent scan of the tree at `path` and delete files when they are found. Ignored if
     ///       the item being removed isn't a directory.
+    ///   - removeItemRecursively: If the item being removed is a directory, remove it by
+    ///       recursively removing its children. Setting this to `true` is synonymous with calling
+    ///       `rm -r`, setting this false is synonymous to calling `rmdir`. Ignored if the item
+    ///       being removed isn't a directory.
     /// - Returns: The number of deleted items which may be zero if `path` did not exist.
     @discardableResult
     func removeItem(
         at path: FilePath,
+        strategy removalStrategy: RemovalStrategy,
         recursively removeItemRecursively: Bool
     ) async throws -> Int
 
@@ -588,9 +592,12 @@ extension FileSystemProtocol {
     /// The item to be removed must be a regular file, symbolic link or directory. If no file exists
     /// at the given path then this function returns zero.
     ///
-    /// If the item at the `path` is a directory then the contents of all of its subdirectories
-    /// will be removed recursively before the directory at `path`. Symbolic links are removed (but
-    /// their targets are not deleted).
+    /// If the item at the `path` is a directory then the contents of all of its subdirectories will
+    /// be removed recursively before the directory at `path`. Symbolic links are removed (but their
+    /// targets are not deleted).
+    ///
+    /// The strategy for deletion will be determined automatically depending on the discovered
+    /// platform.
     ///
     /// - Parameters:
     ///   - path: The path to delete.
@@ -599,7 +606,56 @@ extension FileSystemProtocol {
     public func removeItem(
         at path: FilePath
     ) async throws -> Int {
-        try await self.removeItem(at: path, recursively: true)
+        try await self.removeItem(at: path, strategy: .platformDefault, recursively: true)
+    }
+
+    /// Deletes the file or directory (and its contents) at `path`.
+    ///
+    /// The item to be removed must be a regular file, symbolic link or directory. If no file exists
+    /// at the given path then this function returns zero.
+    ///
+    /// If the item at the `path` is a directory then the contents of all of its subdirectories will
+    /// be removed recursively before the directory at `path`. Symbolic links are removed (but their
+    /// targets are not deleted).
+    ///
+    /// The strategy for deletion will be determined automatically depending on the discovered
+    /// platform.
+    ///
+    /// - Parameters:
+    ///   - path: The path to delete.
+    ///   - removeItemRecursively: If the item being removed is a directory, remove it by
+    ///       recursively removing its children. Setting this to `true` is synonymous with calling
+    ///       `rm -r`, setting this false is synonymous to calling `rmdir`. Ignored if the item
+    ///       being removed isn't a directory.
+    /// - Returns: The number of deleted items which may be zero if `path` did not exist.
+    @discardableResult
+    public func removeItem(
+        at path: FilePath,
+        recursively removeItemRecursively: Bool
+    ) async throws -> Int {
+        try await self.removeItem(at: path, strategy: .platformDefault, recursively: removeItemRecursively)
+    }
+
+    /// Deletes the file or directory (and its contents) at `path`.
+    ///
+    /// The item to be removed must be a regular file, symbolic link or directory. If no file exists
+    /// at the given path then this function returns zero.
+    ///
+    /// If the item at the `path` is a directory then the contents of all of its subdirectories will
+    /// be removed recursively before the directory at `path`. Symbolic links are removed (but their
+    /// targets are not deleted).
+    ///
+    /// - Parameters:
+    ///   - path: The path to delete.
+    ///   - removalStrategy: Whether to delete files sequentially (one-by-one), or perform a
+    ///       concurrent scan of the tree at `path` and delete files when they are found.
+    /// - Returns: The number of deleted items which may be zero if `path` did not exist.
+    @discardableResult
+    public func removeItem(
+        at path: FilePath,
+        strategy removalStrategy: RemovalStrategy
+    ) async throws -> Int {
+        try await self.removeItem(at: path, strategy: removalStrategy, recursively: true)
     }
 
     /// Create a directory at the given path.
@@ -659,7 +715,7 @@ extension FileSystemProtocol {
                 try await execute(handle, directory)
             }
         } tearDown: { _ in
-            try await self.removeItem(at: directory, recursively: true)
+            try await self.removeItem(at: directory, strategy: .platformDefault, recursively: true)
         }
     }
 }
diff --git a/Sources/NIOFileSystem/IOStrategy.swift b/Sources/NIOFileSystem/IOStrategy.swift
new file mode 100644
index 0000000000..fcb7fb432a
--- /dev/null
+++ b/Sources/NIOFileSystem/IOStrategy.swift
@@ -0,0 +1,184 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftNIO open source project
+//
+// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+/// How many file descriptors to open when performing I/O operations.
+enum IOStrategy: Hashable, Sendable {
+    // platformDefault is reified into one of the concrete options below:
+    case sequential
+    case parallel(_ maxDescriptors: Int)
+
+    // These selections are relatively arbitrary but the rationale is as follows:
+    //
+    // - Never exceed the default OS limits even if 4 such operations were happening at once.
+    // - Sufficient to enable significant speed up from parallelism
+    // - Not wasting effort by pushing contention to the underlying storage device. Further we
+    //   assume an SSD or similar underlying storage tech. Users on spinning rust need to account
+    //   for that themselves anyway.
+    //
+    // That said, empirical testing for this has not been performed, suggestions welcome.
+    //
+    // Note: The directory scan is modelled after a copy strategy needing two handles: during the
+    // creation of the destination directory we hold the handle while copying attributes. A much
+    // more complex internal state machine could allow doing two of these if desired. This may not
+    // result in a faster copy though so things are left simple.
+    internal static func determinePlatformDefault() -> Self {
+        #if os(macOS) || os(Linux) || os(Windows)
+        // Eight file descriptors allow for four concurrent file copies/directory scans. Avoiding
+        // storage system contention is of utmost importance.
+        //
+        // Testing was performed on an SSD, while copying objects (a dense directory of small files
+        // and subdirectories of similar shape) to the same volume, totalling 12GB. Results showed
+        // improvements in elapsed time for (expected) increases in CPU time up to parallel(8).
+        // Beyond this, the increases in CPU led to only moderate gains.
+        //
+        // Anyone tuning this is encouraged to cover worst case scenarios.
+        return .parallel(8)
+        #elseif os(iOS) || os(tvOS) || os(watchOS) || os(Android)
+        // Reduced maximum descriptors in embedded world. This is chosen based on biasing towards
+        // safety, not empirical testing.
+        return .parallel(4)
+        #else
+        // Safety first. If we do not know what system we run on, we keep it simple.
+        return .sequential
+        #endif
+    }
+}
+
+/// How to perform copies. Currently only relevant to directory level copies when using
+/// ``FileSystemProtocol/copyItem(at:to:strategy:shouldProceedAfterError:shouldCopyItem:)`` or other
+/// overloads that use the default behaviour.
+public struct CopyStrategy: Hashable, Sendable {
+    internal let wrapped: IOStrategy
+    private init(_ strategy: IOStrategy) {
+        switch strategy {
+        case .sequential:
+            self.wrapped = .sequential
+        case let .parallel(maxDescriptors):
+            self.wrapped = .parallel(maxDescriptors)
+        }
+    }
+
+    // A copy fundamentally can't work without two descriptors unless you copy everything into
+    // memory which is infeasible/inefficient for large copies.
+    private static let minDescriptorsAllowed = 2
+
+    /// Operate in whatever manner is deemed a reasonable default for the platform. This will limit
+    /// the maximum file descriptors usage based on reasonable defaults.
+    ///
+    /// Current assumptions (which are subject to change):
+    /// - Only one copy operation would be performed at once
+    /// - The copy operation is not intended to be the primary activity on the device
+    public static let platformDefault: Self = Self(IOStrategy.determinePlatformDefault())
+
+    /// The copy is done asynchronously, but only one operation will occur at a time. This is the
+    /// only way to guarantee only one callback to the `shouldCopyItem` will happen at a time.
+    public static let sequential: Self = Self(.sequential)
+
+    /// Allow multiple I/O operations to run concurrently, including file copies/directory creation
+    /// and scanning.
+    ///
+    /// - Parameter maxDescriptors: a conservative limit on the number of concurrently open file
+    ///     descriptors involved in the copy. This number must be >= 2 though, if you are using a
+    ///     value that low you should use ``sequential``
+    ///
+    /// - Throws: ``FileSystemError/Code-swift.struct/invalidArgument`` if `maxDescriptors` is less
+    ///     than 2.
+    ///
+    public static func parallel(maxDescriptors: Int) throws -> Self {
+        guard maxDescriptors >= Self.minDescriptorsAllowed else {
+            // 2 is not quite the same as sequential, you could have two concurrent directory
+            // listings for example less than 2 and you can't actually do a _copy_ though so it's
+            // non-sensical.
+            throw FileSystemError(
+                code: .invalidArgument,
+                message: "Can't do a copy operation without at least 2 file descriptors '\(maxDescriptors)' is illegal",
+                cause: nil,
+                location: .here()
+            )
+        }
+        return .init(.parallel(maxDescriptors))
+    }
+}
+
+extension CopyStrategy: CustomStringConvertible {
+    public var description: String {
+        switch self.wrapped {
+        case .sequential:
+            return "sequential"
+        case let .parallel(maxDescriptors):
+            return "parallel with max \(maxDescriptors) descriptors"
+        }
+    }
+}
+
+/// How to perform file deletions. Currently only relevant to directory level deletions when using
+/// ``FileSystemProtocol/removeItem(at:strategy:recursively:)`` or other overloads that use the
+/// default behaviour.
+public struct RemovalStrategy: Hashable, Sendable {
+    internal let wrapped: IOStrategy
+    private init(_ strategy: IOStrategy) {
+        switch strategy {
+        case .sequential:
+            self.wrapped = .sequential
+        case let .parallel(maxDescriptors):
+            self.wrapped = .parallel(maxDescriptors)
+        }
+    }
+
+    // A deletion requires no file descriptors. We only consume a file descriptor while scanning the
+    // contents of a directory, so the minimum is 1.
+    private static let minRequiredDescriptors = 1
+
+    /// Operate in whatever manner is deemed a reasonable default for the platform. This will limit
+    /// the maximum file descriptors usage based on reasonable defaults.
+    ///
+    /// Current assumptions (which are subject to change):
+    /// - Only one delete operation would be performed at once
+    /// - The delete operation is not intended to be the primary activity on the device
+    public static let platformDefault: Self = Self(IOStrategy.determinePlatformDefault())
+
+    /// Traversal of directories and removal of files will be done sequentially without any
+    /// parallelization.
+    public static let sequential: Self = Self(.sequential)
+
+    /// Allow for one or more directory scans to run at the same time. Removal of files will happen
+    /// on asynchronous tasks in parallel.
+    ///
+    /// Setting `maxDescriptors` to 1, will limit the speed of directory discovery. Deletion of
+    /// files within that directory will run in parallel, but discovery of subdirectories will be
+    /// limited to one at a time.
+    public static func parallel(maxDescriptors: Int) throws -> Self {
+        guard maxDescriptors >= Self.minRequiredDescriptors else {
+            throw FileSystemError(
+                code: .invalidArgument,
+                message:
+                    "Can't do a remove operation without at least one file descriptor '\(maxDescriptors)' is illegal",
+                cause: nil,
+                location: .here()
+            )
+        }
+        return .init(.parallel(maxDescriptors))
+    }
+}
+
+extension RemovalStrategy: CustomStringConvertible {
+    public var description: String {
+        switch self.wrapped {
+        case .sequential:
+            return "sequential"
+        case let .parallel(maxDescriptors):
+            return "parallel with max \(maxDescriptors) descriptors"
+        }
+    }
+}
diff --git a/Sources/NIOFileSystem/Internal/Concurrency Primitives/TokenBucket.swift b/Sources/NIOFileSystem/Internal/Concurrency Primitives/TokenBucket.swift
new file mode 100644
index 0000000000..3a998f5b2a
--- /dev/null
+++ b/Sources/NIOFileSystem/Internal/Concurrency Primitives/TokenBucket.swift	
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftNIO open source project
+//
+// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import DequeModule
+import NIOConcurrencyHelpers
+
+/// Type modeled after a "token bucket" pattern, which is similar to a semaphore, but is built with
+/// Swift Concurrency primitives.
+///
+/// This is an adaptation of the TokenBucket found in Swift Package Manager.
+/// Instead of using an ``actor``, we define a class and limit access through
+/// ``NIOLock``.
+@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *)
+final class TokenBucket: @unchecked Sendable {
+    private var tokens: Int
+    private var waiters: Deque<CheckedContinuation<Void, Never>>
+    private let lock: NIOLock
+
+    init(tokens: Int) {
+        precondition(tokens >= 1, "Need at least one token!")
+        self.tokens = tokens
+        self.waiters = Deque()
+        self.lock = NIOLock()
+    }
+
+    /// Executes an `async` closure immediately when a token is available.
+    /// Only the same number of closures will be executed concurrently as the number
+    /// of `tokens` passed to ``TokenBucket/init(tokens:)``, all subsequent
+    /// invocations of `withToken` will suspend until a "free" token is available.
+    /// - Parameter body: The closure to invoke when a token is available.
+    /// - Returns: Resulting value returned by `body`.
+    func withToken<ReturnType>(
+        _ body: @Sendable () async throws -> ReturnType
+    ) async rethrows -> ReturnType {
+        await self.getToken()
+        defer { self.returnToken() }
+        return try await body()
+    }
+
+    private func getToken() async {
+        self.lock.lock()
+        if self.tokens > 0 {
+            self.tokens -= 1
+            self.lock.unlock()
+            return
+        }
+
+        await withCheckedContinuation {
+            self.waiters.append($0)
+            self.lock.unlock()
+        }
+    }
+
+    private func returnToken() {
+        if let waiter = self.lock.withLock({ () -> CheckedContinuation<Void, Never>? in
+            if let nextWaiter = self.waiters.popFirst() {
+                return nextWaiter
+            }
+
+            self.tokens += 1
+            return nil
+        }) {
+            waiter.resume()
+        }
+    }
+}
diff --git a/Sources/NIOFileSystem/Internal/ParallelRemoval.swift b/Sources/NIOFileSystem/Internal/ParallelRemoval.swift
new file mode 100644
index 0000000000..cd8b6702d5
--- /dev/null
+++ b/Sources/NIOFileSystem/Internal/ParallelRemoval.swift
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftNIO open source project
+//
+// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import NIOCore
+
+@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *)
+extension FileSystem {
+    /// Recursively walk all objects found in `path`. Call ourselves recursively
+    /// on each directory that we find, as soon as the file descriptor for
+    /// `path` has been closed; also delete all files that we come across.
+    func discoverAndRemoveItemsInTree(
+        at path: FilePath,
+        _ bucket: TokenBucket
+    ) async throws -> Int {
+        // Discover current directory and find all files/directories. Free up
+        // the handle as fast as possible.
+        let (directoriesToRecurseInto, itemsToDelete) = try await bucket.withToken {
+            try await self.withDirectoryHandle(atPath: path) { directory in
+                var subdirectories: [FilePath] = []
+                var itemsInDirectory: [FilePath] = []
+
+                for try await batch in directory.listContents().batched() {
+                    for entry in batch {
+                        switch entry.type {
+                        case .directory:
+                            subdirectories.append(entry.path)
+                        default:
+                            itemsInDirectory.append(entry.path)
+                        }
+                    }
+                }
+
+                return (subdirectories, itemsInDirectory)
+            }
+        }
+
+        return try await withThrowingTaskGroup(of: Int.self) { group in
+            // Delete all items we found in the current directory.
+            for item in itemsToDelete {
+                group.addTask {
+                    try await self.removeOneItem(at: item)
+                }
+            }
+
+            // Recurse into all newly found subdirectories.
+            for directory in directoriesToRecurseInto {
+                group.addTask {
+                    try await self.discoverAndRemoveItemsInTree(at: directory, bucket)
+                }
+            }
+
+            // Await task groups to finish and sum all items deleted so far.
+            var numberOfDeletedItems = try await group.reduce(0, +)
+
+            // Remove top level directory.
+            numberOfDeletedItems += try await self.removeOneItem(at: path)
+
+            return numberOfDeletedItems
+        }
+    }
+}
diff --git a/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift
index 5de5846603..e262147fa2 100644
--- a/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift
+++ b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift
@@ -222,7 +222,7 @@ final class FileSystemTests: XCTestCase {
             // Avoid dirtying the current working directory.
             if path.isRelative {
                 self.addTeardownBlock { [fileSystem = self.fs] in
-                    try await fileSystem.removeItem(at: path)
+                    try await fileSystem.removeItem(at: path, strategy: .platformDefault)
                 }
             }
 
@@ -351,7 +351,7 @@ final class FileSystemTests: XCTestCase {
             // Avoid dirtying the current working directory.
             if directoryPath.isRelative {
                 self.addTeardownBlock { [fileSystem = self.fs] in
-                    try await fileSystem.removeItem(at: directoryPath)
+                    try await fileSystem.removeItem(at: directoryPath, strategy: .platformDefault)
                 }
             }
 
@@ -399,7 +399,7 @@ final class FileSystemTests: XCTestCase {
 
             if directoryPath.isRelative {
                 self.addTeardownBlock { [fileSystem = self.fs] in
-                    try await fileSystem.removeItem(at: directoryPath, recursively: true)
+                    try await fileSystem.removeItem(at: directoryPath, strategy: .platformDefault, recursively: true)
                 }
             }
 
@@ -586,8 +586,8 @@ final class FileSystemTests: XCTestCase {
         let sourcePath = try await self.fs.temporaryFilePath()
         let destPath = try await self.fs.temporaryFilePath()
         self.addTeardownBlock {
-            _ = try? await self.fs.removeItem(at: sourcePath)
-            _ = try? await self.fs.removeItem(at: destPath)
+            _ = try? await self.fs.removeItem(at: sourcePath, strategy: .platformDefault)
+            _ = try? await self.fs.removeItem(at: destPath, strategy: .platformDefault)
         }
 
         let sourceInfo = try await self.fs.withFileHandle(
@@ -991,7 +991,7 @@ final class FileSystemTests: XCTestCase {
         let infoAfterCreation = try await self.fs.info(forFileAt: path)
         XCTAssertNotNil(infoAfterCreation)
 
-        let removed = try await self.fs.removeItem(at: path)
+        let removed = try await self.fs.removeItem(at: path, strategy: .platformDefault)
         XCTAssertEqual(removed, 1)
 
         let infoAfterRemoval = try await self.fs.info(forFileAt: path)
@@ -1002,11 +1002,11 @@ final class FileSystemTests: XCTestCase {
         let path = try await self.fs.temporaryFilePath()
         let info = try await self.fs.info(forFileAt: path)
         XCTAssertNil(info)
-        let removed = try await self.fs.removeItem(at: path)
+        let removed = try await self.fs.removeItem(at: path, strategy: .platformDefault)
         XCTAssertEqual(removed, 0)
     }
 
-    func testRemoveDirectory() async throws {
+    func testRemoveDirectorySequentially() async throws {
         let path = try await self.fs.temporaryFilePath()
         let created = try await self.generateDirectoryStructure(
             root: path,
@@ -1019,12 +1019,37 @@ final class FileSystemTests: XCTestCase {
 
         // Removing a non-empty directory recursively should throw 'notEmpty'
         await XCTAssertThrowsFileSystemErrorAsync {
-            try await self.fs.removeItem(at: path, recursively: false)
+            try await self.fs.removeItem(at: path, strategy: .sequential, recursively: false)
         } onError: { error in
             XCTAssertEqual(error.code, .notEmpty)
         }
 
-        let removed = try await self.fs.removeItem(at: path)
+        let removed = try await self.fs.removeItem(at: path, strategy: .sequential)
+        XCTAssertEqual(created, removed)
+
+        let infoAfterRemoval = try await self.fs.info(forFileAt: path)
+        XCTAssertNil(infoAfterRemoval)
+    }
+
+    func testRemoveDirectoryConcurrently() async throws {
+        let path = try await self.fs.temporaryFilePath()
+        let created = try await self.generateDirectoryStructure(
+            root: path,
+            maxDepth: 3,
+            maxFilesPerDirectory: 10
+        )
+
+        let infoAfterCreation = try await self.fs.info(forFileAt: path)
+        XCTAssertNotNil(infoAfterCreation)
+
+        // Removing a non-empty directory recursively should throw 'notEmpty'
+        await XCTAssertThrowsFileSystemErrorAsync {
+            try await self.fs.removeItem(at: path, strategy: .parallel(maxDescriptors: 2), recursively: false)
+        } onError: { error in
+            XCTAssertEqual(error.code, .notEmpty)
+        }
+
+        let removed = try await self.fs.removeItem(at: path, strategy: .parallel(maxDescriptors: 2))
         XCTAssertEqual(created, removed)
 
         let infoAfterRemoval = try await self.fs.info(forFileAt: path)
@@ -1602,7 +1627,7 @@ extension FileSystemTests {
 
             // Clean up after ourselves.
             self.addTeardownBlock { [fileSystem = self.fs] in
-                try await fileSystem.removeItem(at: temporaryDirectoryPath)
+                try await fileSystem.removeItem(at: temporaryDirectoryPath, strategy: .platformDefault)
             }
 
             guard let info = try await self.fs.info(forFileAt: temporaryDirectoryPath) else {
@@ -1640,7 +1665,7 @@ extension FileSystemTests {
         let temporaryDirectoryPath = try await self.fs.createTemporaryDirectory(template: template)
 
         self.addTeardownBlock { [fileSystem = self.fs] in
-            try await fileSystem.removeItem(at: templateRoot, recursively: true)
+            try await fileSystem.removeItem(at: templateRoot, strategy: .platformDefault, recursively: true)
         }
 
         guard

From b2356d9e33af062d043d485a3c323bb907053f66 Mon Sep 17 00:00:00 2001
From: Rauhul Varma <rauhul@apple.com>
Date: Wed, 11 Dec 2024 08:31:10 -0800
Subject: [PATCH 31/37] Remove outdated documentation (#3023)

Removes an outdated recommendation about shutting down
`MultiThreadedEventLoopGroup`s before the program exits. It's now
considered best practice to just exit the application and allow the host
OS to do resource clean up.
---
 Sources/NIOPosix/MultiThreadedEventLoopGroup.swift | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift b/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift
index b92a4bb01b..0361e1a197 100644
--- a/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift
+++ b/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift
@@ -51,9 +51,6 @@ typealias ThreadInitializer = (NIOThread) -> Void
 /// all run their own `EventLoop`. Those threads will not be shut down until `shutdownGracefully` or
 /// `syncShutdownGracefully` is called.
 ///
-/// - Note: It's good style to call `MultiThreadedEventLoopGroup.shutdownGracefully` or
-///         `MultiThreadedEventLoopGroup.syncShutdownGracefully` when you no longer need this `EventLoopGroup`. In
-///         many cases that is just before your program exits.
 /// - warning: Unit tests often spawn one `MultiThreadedEventLoopGroup` per unit test to force isolation between the
 ///            tests. In those cases it's important to shut the `MultiThreadedEventLoopGroup` down at the end of the
 ///            test. A good place to start a `MultiThreadedEventLoopGroup` is the `setUp` method of your `XCTestCase`

From 8bf40344623bad55bef36d8c4b50a44ea1dd062e Mon Sep 17 00:00:00 2001
From: George Barnett <gbarnett@apple.com>
Date: Thu, 12 Dec 2024 10:29:03 +0000
Subject: [PATCH 32/37] Deprecate not-actually-public Base64 APIs (#3022)

Motivation:

The not-actually-public `_NIOBase64` module has public extensions on
`String`. These are visible when transitively depending on `_NIOBase64`
but shouldn't be.

Modifications:

- Add underscored variants
- Deprecate public variants

Result:

Stricter API
---
 Sources/NIOCore/ByteBuffer-aux.swift          |  4 ++--
 .../NIOWebSocketClientUpgrader.swift          |  4 ++--
 .../NIOWebSocketServerUpgrader.swift          |  3 ++-
 Sources/_NIOBase64/Base64.swift               | 16 +++++++++++--
 Tests/NIOBase64Tests/Base64Test.swift         | 23 +++++++++----------
 5 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/Sources/NIOCore/ByteBuffer-aux.swift b/Sources/NIOCore/ByteBuffer-aux.swift
index 232dffd874..1ec8e9cf3b 100644
--- a/Sources/NIOCore/ByteBuffer-aux.swift
+++ b/Sources/NIOCore/ByteBuffer-aux.swift
@@ -733,13 +733,13 @@ extension ByteBuffer: Codable {
     public init(from decoder: Decoder) throws {
         let container = try decoder.singleValueContainer()
         let base64String = try container.decode(String.self)
-        self = try ByteBuffer(bytes: base64String.base64Decoded())
+        self = try ByteBuffer(bytes: base64String._base64Decoded())
     }
 
     /// Encodes this buffer as a base64 string in a single value container.
     public func encode(to encoder: Encoder) throws {
         var container = encoder.singleValueContainer()
-        let base64String = String(base64Encoding: self.readableBytesView)
+        let base64String = String(_base64Encoding: self.readableBytesView)
         try container.encode(base64String)
     }
 }
diff --git a/Sources/NIOWebSocket/NIOWebSocketClientUpgrader.swift b/Sources/NIOWebSocket/NIOWebSocketClientUpgrader.swift
index a25e69df90..cc5fa16e33 100644
--- a/Sources/NIOWebSocket/NIOWebSocketClientUpgrader.swift
+++ b/Sources/NIOWebSocket/NIOWebSocketClientUpgrader.swift
@@ -149,7 +149,7 @@ extension NIOWebSocketClientUpgrader {
             UInt64.random(in: UInt64.min...UInt64.max, using: &generator),
             UInt64.random(in: UInt64.min...UInt64.max, using: &generator)
         )
-        return String(base64Encoding: buffer.readableBytesView)
+        return String(_base64Encoding: buffer.readableBytesView)
     }
     /// Generates a random WebSocket Request Key by generating 16 bytes randomly using the `SystemRandomNumberGenerator` and encoding them as a base64 string as defined in RFC6455 https://tools.ietf.org/html/rfc6455#section-4.1.
     /// - Returns: base64 encoded request key
@@ -179,7 +179,7 @@ private func _shouldAllowUpgrade(upgradeResponse: HTTPResponseHead, requestKey:
     var hasher = SHA1()
     hasher.update(string: requestKey)
     hasher.update(string: magicWebSocketGUID)
-    let expectedAcceptValue = String(base64Encoding: hasher.finish())
+    let expectedAcceptValue = String(_base64Encoding: hasher.finish())
 
     return expectedAcceptValue == acceptValueHeader[0]
 }
diff --git a/Sources/NIOWebSocket/NIOWebSocketServerUpgrader.swift b/Sources/NIOWebSocket/NIOWebSocketServerUpgrader.swift
index 8aa16eec7c..4f888a3e34 100644
--- a/Sources/NIOWebSocket/NIOWebSocketServerUpgrader.swift
+++ b/Sources/NIOWebSocket/NIOWebSocketServerUpgrader.swift
@@ -15,6 +15,7 @@
 import CNIOSHA1
 import NIOCore
 import NIOHTTP1
+import _NIOBase64
 
 let magicWebSocketGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
 
@@ -303,7 +304,7 @@ private func _buildUpgradeResponse(
                 var hasher = SHA1()
                 hasher.update(string: key)
                 hasher.update(string: magicWebSocketGUID)
-                acceptValue = String(base64Encoding: hasher.finish())
+                acceptValue = String(_base64Encoding: hasher.finish())
             }
 
             extraHeaders.replaceOrAdd(name: "Upgrade", value: "websocket")
diff --git a/Sources/_NIOBase64/Base64.swift b/Sources/_NIOBase64/Base64.swift
index dc001f85b4..15bf79e3be 100644
--- a/Sources/_NIOBase64/Base64.swift
+++ b/Sources/_NIOBase64/Base64.swift
@@ -16,15 +16,27 @@
 // https://github.com/fabianfett/swift-base64-kit
 
 extension String {
-
     /// Base64 encode a collection of UInt8 to a string, without the use of Foundation.
+    @available(*, deprecated, message: "This API was unintentionally made public.")
     @inlinable
     public init<Buffer: Collection>(base64Encoding bytes: Buffer) where Buffer.Element == UInt8 {
-        self = Base64.encode(bytes: bytes)
+        self.init(_base64Encoding: bytes)
     }
 
+    @available(*, deprecated, message: "This API was unintentionally made public.")
     @inlinable
     public func base64Decoded() throws -> [UInt8] {
+        try self._base64Decoded()
+    }
+
+    /// Base64 encode a collection of UInt8 to a string, without the use of Foundation.
+    @inlinable
+    public init<Buffer: Collection>(_base64Encoding bytes: Buffer) where Buffer.Element == UInt8 {
+        self = Base64.encode(bytes: bytes)
+    }
+
+    @inlinable
+    public func _base64Decoded() throws -> [UInt8] {
         try Base64.decode(string: self)
     }
 }
diff --git a/Tests/NIOBase64Tests/Base64Test.swift b/Tests/NIOBase64Tests/Base64Test.swift
index dd6262a9d8..5c320706bf 100644
--- a/Tests/NIOBase64Tests/Base64Test.swift
+++ b/Tests/NIOBase64Tests/Base64Test.swift
@@ -13,59 +13,58 @@
 //===----------------------------------------------------------------------===//
 
 import XCTest
-
-@testable import _NIOBase64
+import _NIOBase64
 
 class Base64Test: XCTestCase {
     func testEncodeEmptyData() throws {
         let data = [UInt8]()
-        let encodedData = String(base64Encoding: data)
+        let encodedData = String(_base64Encoding: data)
         XCTAssertEqual(encodedData.count, 0)
     }
 
     func testBase64EncodingOfEmptyString() throws {
         let string = ""
-        let encoded = String(base64Encoding: string.utf8)
+        let encoded = String(_base64Encoding: string.utf8)
         XCTAssertEqual(encoded, "")
     }
 
     func testBase64DecodingOfEmptyString() throws {
         let encoded = ""
         XCTAssertNoThrow {
-            let decoded = try encoded.base64Decoded()
+            let decoded = try encoded._base64Decoded()
             XCTAssertEqual(decoded, [UInt8]())
         }
     }
 
     func testBase64EncodingArrayOfNulls() throws {
         let data = Array(repeating: UInt8(0), count: 10)
-        let encodedData = String(base64Encoding: data)
+        let encodedData = String(_base64Encoding: data)
         XCTAssertEqual(encodedData, "AAAAAAAAAAAAAA==")
     }
 
     func testBase64DecodeArrayOfNulls() throws {
         let encoded = "AAAAAAAAAAAAAA=="
-        let decoded = try! encoded.base64Decoded()
+        let decoded = try! encoded._base64Decoded()
         let expected = Array(repeating: UInt8(0), count: 10)
         XCTAssertEqual(decoded, expected)
     }
 
     func testBase64EncodeingHelloWorld() throws {
         let string = "Hello, world!"
-        let encoded = String(base64Encoding: string.utf8)
+        let encoded = String(_base64Encoding: string.utf8)
         let expected = "SGVsbG8sIHdvcmxkIQ=="
         XCTAssertEqual(encoded, expected)
     }
 
     func testBase64DecodeHelloWorld() throws {
         let encoded = "SGVsbG8sIHdvcmxkIQ=="
-        let decoded = try! encoded.base64Decoded()
+        let decoded = try! encoded._base64Decoded()
         XCTAssertEqual(decoded, "Hello, world!".utf8.map { UInt8($0) })
     }
 
     func testBase64EncodingAllTheBytesSequentially() throws {
         let data = Array(UInt8(0)...UInt8(255))
-        let encodedData = String(base64Encoding: data)
+        let encodedData = String(_base64Encoding: data)
         XCTAssertEqual(
             encodedData,
             "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w=="
@@ -74,14 +73,14 @@ class Base64Test: XCTestCase {
 
     func testBase64DecodingWithInvalidLength() {
         let encoded = "dGVzbA!=="
-        XCTAssertThrowsError(try encoded.base64Decoded()) { error in
+        XCTAssertThrowsError(try encoded._base64Decoded()) { error in
             XCTAssertEqual(error as? Base64Error, .invalidLength)
         }
     }
 
     func testBase64DecodeWithInvalidCharacter() throws {
         let encoded = "SGVsbG8sI_dvcmxkIQ=="
-        XCTAssertThrowsError(try encoded.base64Decoded()) { error in
+        XCTAssertThrowsError(try encoded._base64Decoded()) { error in
             XCTAssertEqual(error as? Base64Error, .invalidCharacter)
         }
     }

From 969b06b844f092627c8b888be32d636a5a112166 Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Fri, 13 Dec 2024 13:50:27 +0000
Subject: [PATCH 33/37] rename top matrix workflow object 'swift'->'config'
 (#3024)

### Motivation:

The use of 'swift' here is very arbitrary and adds no value. The object
is only there at all because GitHub Actions seems to require it.

### Modifications:

Rename the top-level object from 'swift' to 'config' to maybe add value
to the intent of the object and hopefully be less cryptic.

### Result:

Everything should just keep working.
---
 .github/workflows/swift_test_matrix.yml | 32 ++++++++++++-------------
 scripts/generate_matrix.sh              | 18 +++++++-------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/swift_test_matrix.yml b/.github/workflows/swift_test_matrix.yml
index cec7fd70f5..9b4a36e935 100644
--- a/.github/workflows/swift_test_matrix.yml
+++ b/.github/workflows/swift_test_matrix.yml
@@ -19,8 +19,8 @@ concurrency:
 
 jobs:
   execute-matrix:
-    name: ${{ matrix.swift.platform }} (${{ matrix.swift.name }})
-    runs-on: ${{ matrix.swift.runner }}
+    name: ${{ matrix.config.platform }} (${{ matrix.config.name }})
+    runs-on: ${{ matrix.config.runner }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(inputs.matrix_string) }}
@@ -31,37 +31,37 @@ jobs:
           persist-credentials: false
           submodules: true
       - name: Pull Docker image
-        run: docker pull ${{ matrix.swift.image }}
+        run: docker pull ${{ matrix.config.image }}
       - name: Run matrix job
-        if: ${{ matrix.swift.platform != 'Windows' }}
+        if: ${{ matrix.config.platform != 'Windows' }}
         run: |
-          if [[ -n "${{ matrix.swift.setup_command }}" ]]; then
-            setup_command_expression="${{ matrix.swift.setup_command }} &&"
+          if [[ -n "${{ matrix.config.setup_command }}" ]]; then
+            setup_command_expression="${{ matrix.config.setup_command }} &&"
           else
             setup_command_expression=""
           fi
           workspace="/$(basename ${{ github.workspace }})"
           docker run -v ${{ github.workspace }}:"$workspace" \
             -w "$workspace" \
-            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" \
+            -e SWIFT_VERSION="${{ matrix.config.swift_version }}" \
             -e setup_command_expression="$setup_command_expression" \
             -e workspace="$workspace" \
-            ${{ matrix.swift.image }} \
-            bash -c "swift --version && git config --global --add safe.directory \"$workspace\" && $setup_command_expression ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"
+            ${{ matrix.config.image }} \
+            bash -c "swift --version && git config --global --add safe.directory \"$workspace\" && $setup_command_expression ${{ matrix.config.command }} ${{ matrix.config.command_arguments }}"
       - name: Run matrix job (Windows)
-        if: ${{ matrix.swift.platform == 'Windows' }}
+        if: ${{ matrix.config.platform == 'Windows' }}
         run: |
-          if (-not [string]::IsNullOrEmpty("${{ matrix.swift.setup_command }}")) {
-              $setup_command_expression = "${{ matrix.swift.setup_command }} &"
+          if (-not [string]::IsNullOrEmpty("${{ matrix.config.setup_command }}")) {
+              $setup_command_expression = "${{ matrix.config.setup_command }} &"
           } else {
               $setup_command_expression = ""
           }
           $workspace = "C:\" + (Split-Path ${{ github.workspace }} -Leaf)
           docker run -v ${{ github.workspace }}:$($workspace) `
             -w $($workspace) `
-            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" `
+            -e SWIFT_VERSION="${{ matrix.config.swift_version }}" `
             -e setup_command_expression=%setup_command_expression% `
-            ${{ matrix.swift.image }} `
-            cmd /s /c  "swift --version & powershell Invoke-Expression ""$($setup_command_expression) ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"""
+            ${{ matrix.config.image }} `
+            cmd /s /c  "swift --version & powershell Invoke-Expression ""$($setup_command_expression) ${{ matrix.config.command }} ${{ matrix.config.command_arguments }}"""
     env:
-      SWIFT_VERSION: ${{ matrix.swift.swift_version }}
+      SWIFT_VERSION: ${{ matrix.config.swift_version }}
diff --git a/scripts/generate_matrix.sh b/scripts/generate_matrix.sh
index dbab4f69a3..c7915b4783 100755
--- a/scripts/generate_matrix.sh
+++ b/scripts/generate_matrix.sh
@@ -52,7 +52,7 @@ windows_nightly_main_runner="windows-2019"
 windows_nightly_main_container_image="swiftlang/swift:nightly-main-windowsservercore-1809"
 
 # Create matrix from inputs
-matrix='{"swift": []}'
+matrix='{"config": []}'
 
 ## Linux
 if [[ "$linux_5_9_enabled" == "true" || "$linux_5_10_enabled" == "true" || "$linux_6_0_enabled" == "true" || \
@@ -70,7 +70,7 @@ if [[ "$linux_5_9_enabled" == "true" ]]; then
     --arg command_arguments "$linux_5_9_command_arguments" \
     --arg container_image "$linux_5_9_container_image" \
     --arg runner "$linux_runner" \
-    '.swift[.swift| length] |= . + { "name": "5.9", "image": $container_image, "swift_version": "5.9", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+    '.config[.config| length] |= . + { "name": "5.9", "image": $container_image, "swift_version": "5.9", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
 fi
 
 if [[ "$linux_5_10_enabled" == "true" ]]; then
@@ -80,7 +80,7 @@ if [[ "$linux_5_10_enabled" == "true" ]]; then
     --arg command_arguments "$linux_5_10_command_arguments" \
     --arg container_image "$linux_5_10_container_image" \
     --arg runner "$linux_runner" \
-    '.swift[.swift| length] |= . + { "name": "5.10", "image": $container_image, "swift_version": "5.10", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+    '.config[.config| length] |= . + { "name": "5.10", "image": $container_image, "swift_version": "5.10", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
 fi
 
 if [[ "$linux_6_0_enabled" == "true" ]]; then
@@ -90,7 +90,7 @@ if [[ "$linux_6_0_enabled" == "true" ]]; then
     --arg command_arguments "$linux_6_0_command_arguments" \
     --arg container_image "$linux_6_0_container_image" \
     --arg runner "$linux_runner" \
-    '.swift[.swift| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+    '.config[.config| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
 fi
 
 if [[ "$linux_nightly_6_0_enabled" == "true" ]]; then
@@ -100,7 +100,7 @@ if [[ "$linux_nightly_6_0_enabled" == "true" ]]; then
     --arg command_arguments "$linux_nightly_6_0_command_arguments" \
     --arg container_image "$linux_nightly_6_0_container_image" \
     --arg runner "$linux_runner" \
-    '.swift[.swift| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+    '.config[.config| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
 fi
 
 if [[ "$linux_nightly_main_enabled" == "true" ]]; then
@@ -110,7 +110,7 @@ if [[ "$linux_nightly_main_enabled" == "true" ]]; then
     --arg command_arguments "$linux_nightly_main_command_arguments" \
     --arg container_image "$linux_nightly_main_container_image" \
     --arg runner "$linux_runner" \
-    '.swift[.swift| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
+    '.config[.config| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "platform": "Linux", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}')
 fi
 
 ## Windows
@@ -127,7 +127,7 @@ if [[ "$windows_6_0_enabled" == "true" ]]; then
     --arg command_arguments "$windows_6_0_command_arguments" \
     --arg container_image "$windows_6_0_container_image" \
     --arg runner "$windows_6_0_runner" \
-    '.swift[.swift| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
+    '.config[.config| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
 fi
 
 if [[ "$windows_nightly_6_0_enabled" == "true" ]]; then
@@ -137,7 +137,7 @@ if [[ "$windows_nightly_6_0_enabled" == "true" ]]; then
     --arg command_arguments "$windows_nightly_6_0_command_arguments" \
     --arg container_image "$windows_nightly_6_0_container_image" \
     --arg runner "$windows_nightly_6_0_runner" \
-    '.swift[.swift| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
+    '.config[.config| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
 fi
 
 if [[ "$windows_nightly_main_enabled" == "true" ]]; then
@@ -147,7 +147,7 @@ if [[ "$windows_nightly_main_enabled" == "true" ]]; then
     --arg command_arguments "$windows_nightly_main_command_arguments" \
     --arg container_image "$windows_nightly_main_container_image" \
     --arg runner "$windows_nightly_main_runner" \
-    '.swift[.swift| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
+    '.config[.config| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "platform": "Windows", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }')
 fi
 
 echo "$matrix" | jq -c
\ No newline at end of file

From 742ae54a2f60ce5a1acf2fd2d2b3c8e4c59cb82a Mon Sep 17 00:00:00 2001
From: Cory Benfield <lukasa@apple.com>
Date: Fri, 13 Dec 2024 16:52:38 +0000
Subject: [PATCH 34/37] Fix new warnings (#3026)

Motivation:

6.0.3 added new warnings about Sendable issues, so let's fix those to
keep things warnings free.

Modifications:

A bunch of Sendable usage fixes.

Result:

Warnings free builds again!
---
 .../NIOAsyncAwaitDemo/AsyncChannelIO.swift    |  7 +-
 Sources/NIOPosix/Bootstrap.swift              |  8 +-
 .../AsyncTestingChannelTests.swift            | 16 ++--
 .../HTTPClientUpgradeTests.swift              |  4 +-
 .../HTTPServerUpgradeTests.swift              |  4 +-
 .../AsyncChannelBootstrapTests.swift          | 30 ++++----
 .../NIOScheduledCallbackTests.swift           | 76 +++++++++----------
 .../RawSocketBootstrapTests.swift             | 36 +++++++--
 Tests/NIOPosixTests/SocketChannelTest.swift   |  3 +-
 .../UniversalBootstrapSupportTest.swift       |  8 +-
 .../WebSocketClientEndToEndTests.swift        | 53 +++++++++----
 11 files changed, 150 insertions(+), 95 deletions(-)

diff --git a/Sources/NIOAsyncAwaitDemo/AsyncChannelIO.swift b/Sources/NIOAsyncAwaitDemo/AsyncChannelIO.swift
index e5612e1f69..5e6f04a38d 100644
--- a/Sources/NIOAsyncAwaitDemo/AsyncChannelIO.swift
+++ b/Sources/NIOAsyncAwaitDemo/AsyncChannelIO.swift
@@ -24,8 +24,11 @@ struct AsyncChannelIO<Request, Response> {
     }
 
     func start() async throws -> AsyncChannelIO<Request, Response> {
-        try await channel.pipeline.addHandler(RequestResponseHandler<HTTPRequestHead, NIOHTTPClientResponseFull>())
-            .get()
+        try await channel.eventLoop.submit {
+            try channel.pipeline.syncOperations.addHandler(
+                RequestResponseHandler<HTTPRequestHead, NIOHTTPClientResponseFull>()
+            )
+        }.get()
         return self
     }
 
diff --git a/Sources/NIOPosix/Bootstrap.swift b/Sources/NIOPosix/Bootstrap.swift
index fa117e8576..e3b5710ffe 100644
--- a/Sources/NIOPosix/Bootstrap.swift
+++ b/Sources/NIOPosix/Bootstrap.swift
@@ -2420,6 +2420,8 @@ extension NIOPipeBootstrap {
         let channel: PipeChannel
         let pipeChannelInput: SelectablePipeHandle?
         let pipeChannelOutput: SelectablePipeHandle?
+        let hasNoInputPipe: Bool
+        let hasNoOutputPipe: Bool
         do {
             if let input = input {
                 try self.validateFileDescriptorIsNotAFile(input)
@@ -2430,6 +2432,8 @@ extension NIOPipeBootstrap {
 
             pipeChannelInput = input.flatMap { SelectablePipeHandle(takingOwnershipOfDescriptor: $0) }
             pipeChannelOutput = output.flatMap { SelectablePipeHandle(takingOwnershipOfDescriptor: $0) }
+            hasNoInputPipe = pipeChannelInput == nil
+            hasNoOutputPipe = pipeChannelOutput == nil
             do {
                 channel = try self.hooks.makePipeChannel(
                     eventLoop: eventLoop as! SelectableEventLoop,
@@ -2458,10 +2462,10 @@ extension NIOPipeBootstrap {
                 channel.registerAlreadyConfigured0(promise: promise)
                 return promise.futureResult.map { result }
             }.flatMap { result -> EventLoopFuture<ChannelInitializerResult> in
-                if pipeChannelInput == nil {
+                if hasNoInputPipe {
                     return channel.close(mode: .input).map { result }
                 }
-                if pipeChannelOutput == nil {
+                if hasNoOutputPipe {
                     return channel.close(mode: .output).map { result }
                 }
                 return channel.selectableEventLoop.makeSucceededFuture(result)
diff --git a/Tests/NIOEmbeddedTests/AsyncTestingChannelTests.swift b/Tests/NIOEmbeddedTests/AsyncTestingChannelTests.swift
index 1619f95638..5fa112f6d1 100644
--- a/Tests/NIOEmbeddedTests/AsyncTestingChannelTests.swift
+++ b/Tests/NIOEmbeddedTests/AsyncTestingChannelTests.swift
@@ -21,7 +21,7 @@ import XCTest
 @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
 class AsyncTestingChannelTests: XCTestCase {
     func testSingleHandlerInit() async throws {
-        class Handler: ChannelInboundHandler {
+        final class Handler: ChannelInboundHandler, Sendable {
             typealias InboundIn = Never
         }
 
@@ -43,7 +43,7 @@ class AsyncTestingChannelTests: XCTestCase {
     }
 
     func testMultipleHandlerInit() async throws {
-        class Handler: ChannelInboundHandler, RemovableChannelHandler {
+        final class Handler: ChannelInboundHandler, RemovableChannelHandler, Sendable {
             typealias InboundIn = Never
             let identifier: String
 
@@ -334,7 +334,7 @@ class AsyncTestingChannelTests: XCTestCase {
         try await XCTAsyncAssertTrue(await channel.finish().isClean)
 
         // channelInactive should fire only once.
-        XCTAssertEqual(inactiveHandler.inactiveNotifications, 1)
+        XCTAssertEqual(inactiveHandler.inactiveNotifications.load(ordering: .sequentiallyConsistent), 1)
     }
 
     func testEmbeddedLifecycle() async throws {
@@ -355,7 +355,7 @@ class AsyncTestingChannelTests: XCTestCase {
         XCTAssertFalse(channel.isActive)
     }
 
-    private final class ExceptionThrowingInboundHandler: ChannelInboundHandler {
+    private final class ExceptionThrowingInboundHandler: ChannelInboundHandler, Sendable {
         typealias InboundIn = String
 
         public func channelRead(context: ChannelHandlerContext, data: NIOAny) {
@@ -363,7 +363,7 @@ class AsyncTestingChannelTests: XCTestCase {
         }
     }
 
-    private final class ExceptionThrowingOutboundHandler: ChannelOutboundHandler {
+    private final class ExceptionThrowingOutboundHandler: ChannelOutboundHandler, Sendable {
         typealias OutboundIn = String
         typealias OutboundOut = Never
 
@@ -372,12 +372,12 @@ class AsyncTestingChannelTests: XCTestCase {
         }
     }
 
-    private final class CloseInChannelInactiveHandler: ChannelInboundHandler {
+    private final class CloseInChannelInactiveHandler: ChannelInboundHandler, Sendable {
         typealias InboundIn = ByteBuffer
-        public var inactiveNotifications = 0
+        public let inactiveNotifications = ManagedAtomic(0)
 
         public func channelInactive(context: ChannelHandlerContext) {
-            inactiveNotifications += 1
+            inactiveNotifications.wrappingIncrement(by: 1, ordering: .sequentiallyConsistent)
             context.close(promise: nil)
         }
     }
diff --git a/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift b/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift
index 43cfbb4c6b..97d9df956a 100644
--- a/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift
+++ b/Tests/NIOHTTP1Tests/HTTPClientUpgradeTests.swift
@@ -318,7 +318,7 @@ private func assertPipelineContainsUpgradeHandler(channel: Channel) {
 @available(macOS 13, iOS 16, tvOS 16, watchOS 9, *)
 class HTTPClientUpgradeTestCase: XCTestCase {
     func setUpClientChannel(
-        clientHTTPHandler: RemovableChannelHandler,
+        clientHTTPHandler: RemovableChannelHandler & Sendable,
         clientUpgraders: [any TypedAndUntypedHTTPClientProtocolUpgrader],
         _ upgradeCompletionHandler: @escaping (ChannelHandlerContext) -> Void
     ) throws -> EmbeddedChannel {
@@ -1063,7 +1063,7 @@ class HTTPClientUpgradeTestCase: XCTestCase {
 @available(macOS 13, iOS 16, tvOS 16, watchOS 9, *)
 final class TypedHTTPClientUpgradeTestCase: HTTPClientUpgradeTestCase {
     override func setUpClientChannel(
-        clientHTTPHandler: RemovableChannelHandler,
+        clientHTTPHandler: RemovableChannelHandler & Sendable,
         clientUpgraders: [any TypedAndUntypedHTTPClientProtocolUpgrader],
         _ upgradeCompletionHandler: @escaping (ChannelHandlerContext) -> Void
     ) throws -> EmbeddedChannel {
diff --git a/Tests/NIOHTTP1Tests/HTTPServerUpgradeTests.swift b/Tests/NIOHTTP1Tests/HTTPServerUpgradeTests.swift
index 3a69f806e3..9049c5122d 100644
--- a/Tests/NIOHTTP1Tests/HTTPServerUpgradeTests.swift
+++ b/Tests/NIOHTTP1Tests/HTTPServerUpgradeTests.swift
@@ -1388,7 +1388,7 @@ class HTTPServerUpgradeTestCase: XCTestCase {
             XCTAssertNil(upgradeRequest.wrappedValue)
             upgradeHandlerCbFired.wrappedValue = true
 
-            _ = context.channel.pipeline.addHandler(
+            try! context.channel.pipeline.syncOperations.addHandler(
                 CheckWeReadInlineAndExtraData(
                     firstByteDonePromise: firstByteDonePromise,
                     secondByteDonePromise: secondByteDonePromise,
@@ -2145,7 +2145,7 @@ final class TypedHTTPServerUpgradeTestCase: HTTPServerUpgradeTestCase {
             XCTAssertNotNil(upgradeRequest.wrappedValue)
             upgradeHandlerCbFired.wrappedValue = true
 
-            _ = context.channel.pipeline.addHandler(
+            try! context.channel.pipeline.syncOperations.addHandler(
                 CheckWeReadInlineAndExtraData(
                     firstByteDonePromise: firstByteDonePromise,
                     secondByteDonePromise: secondByteDonePromise,
diff --git a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
index 847a5ac782..68604ef0f4 100644
--- a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
+++ b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift
@@ -284,7 +284,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
             port: 0
         ) { channel in
             channel.eventLoop.makeCompletedFuture {
-                try self.configureProtocolNegotiationHandlers(channel: channel)
+                try Self.configureProtocolNegotiationHandlers(channel: channel)
             }
         }
 
@@ -366,7 +366,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                 port: 0
             ) { channel in
                 channel.eventLoop.makeCompletedFuture {
-                    try self.configureNestedProtocolNegotiationHandlers(channel: channel)
+                    try Self.configureNestedProtocolNegotiationHandlers(channel: channel)
                 }
             }
 
@@ -508,7 +508,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
             port: 0
         ) { channel in
             channel.eventLoop.makeCompletedFuture {
-                try self.configureProtocolNegotiationHandlers(channel: channel)
+                try Self.configureProtocolNegotiationHandlers(channel: channel)
             }
         }
 
@@ -958,7 +958,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                     output: pipe2WriteFD
                 ) { channel in
                     channel.eventLoop.makeCompletedFuture {
-                        try self.configureProtocolNegotiationHandlers(channel: channel)
+                        try Self.configureProtocolNegotiationHandlers(channel: channel)
                     }
                 }
         } catch {
@@ -1251,7 +1251,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                     try channel.pipeline.syncOperations.addHandler(
                         AddressedEnvelopingHandler(remoteAddress: SocketAddress(ipAddress: "127.0.0.1", port: 0))
                     )
-                    return try self.configureProtocolNegotiationHandlers(
+                    return try Self.configureProtocolNegotiationHandlers(
                         channel: channel,
                         proposedALPN: nil,
                         inboundID: 1,
@@ -1275,7 +1275,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                     try channel.pipeline.syncOperations.addHandler(
                         AddressedEnvelopingHandler(remoteAddress: SocketAddress(ipAddress: "127.0.0.1", port: 0))
                     )
-                    return try self.configureProtocolNegotiationHandlers(
+                    return try Self.configureProtocolNegotiationHandlers(
                         channel: channel,
                         proposedALPN: proposedALPN,
                         inboundID: 2,
@@ -1329,7 +1329,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                 to: .init(ipAddress: "127.0.0.1", port: port)
             ) { channel in
                 channel.eventLoop.makeCompletedFuture {
-                    try self.configureProtocolNegotiationHandlers(channel: channel, proposedALPN: proposedALPN)
+                    try Self.configureProtocolNegotiationHandlers(channel: channel, proposedALPN: proposedALPN)
                 }
             }
     }
@@ -1345,7 +1345,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                 to: .init(ipAddress: "127.0.0.1", port: port)
             ) { channel in
                 channel.eventLoop.makeCompletedFuture {
-                    try self.configureNestedProtocolNegotiationHandlers(
+                    try Self.configureNestedProtocolNegotiationHandlers(
                         channel: channel,
                         proposedOuterALPN: proposedOuterALPN,
                         proposedInnerALPN: proposedInnerALPN
@@ -1382,7 +1382,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
             ) { channel in
                 channel.eventLoop.makeCompletedFuture {
                     try channel.pipeline.syncOperations.addHandler(AddressedEnvelopingHandler())
-                    return try self.configureProtocolNegotiationHandlers(channel: channel, proposedALPN: proposedALPN)
+                    return try Self.configureProtocolNegotiationHandlers(channel: channel, proposedALPN: proposedALPN)
                 }
             }
     }
@@ -1418,13 +1418,13 @@ final class AsyncChannelBootstrapTests: XCTestCase {
             ) { channel in
                 channel.eventLoop.makeCompletedFuture {
                     try channel.pipeline.syncOperations.addHandler(AddressedEnvelopingHandler())
-                    return try self.configureProtocolNegotiationHandlers(channel: channel, proposedALPN: proposedALPN)
+                    return try Self.configureProtocolNegotiationHandlers(channel: channel, proposedALPN: proposedALPN)
                 }
             }
     }
 
     @discardableResult
-    private func configureProtocolNegotiationHandlers(
+    private static func configureProtocolNegotiationHandlers(
         channel: Channel,
         proposedALPN: TLSUserEventHandler.ALPN? = nil,
         inboundID: UInt8? = nil,
@@ -1437,7 +1437,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     @discardableResult
-    private func configureNestedProtocolNegotiationHandlers(
+    private static func configureNestedProtocolNegotiationHandlers(
         channel: Channel,
         proposedOuterALPN: TLSUserEventHandler.ALPN? = nil,
         proposedInnerALPN: TLSUserEventHandler.ALPN? = nil
@@ -1456,7 +1456,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                         try channel.pipeline.syncOperations.addHandler(
                             TLSUserEventHandler(proposedALPN: proposedInnerALPN)
                         )
-                        let negotiationFuture = try self.addTypedApplicationProtocolNegotiationHandler(to: channel)
+                        let negotiationFuture = try Self.addTypedApplicationProtocolNegotiationHandler(to: channel)
 
                         return negotiationFuture
                     }
@@ -1465,7 +1465,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
                         try channel.pipeline.syncOperations.addHandler(
                             TLSUserEventHandler(proposedALPN: proposedInnerALPN)
                         )
-                        let negotiationHandler = try self.addTypedApplicationProtocolNegotiationHandler(to: channel)
+                        let negotiationHandler = try Self.addTypedApplicationProtocolNegotiationHandler(to: channel)
 
                         return negotiationHandler
                     }
@@ -1481,7 +1481,7 @@ final class AsyncChannelBootstrapTests: XCTestCase {
     }
 
     @discardableResult
-    private func addTypedApplicationProtocolNegotiationHandler(
+    private static func addTypedApplicationProtocolNegotiationHandler(
         to channel: Channel
     ) throws -> EventLoopFuture<NegotiationResult> {
         let negotiationHandler = NIOTypedApplicationProtocolNegotiationHandler<NegotiationResult> {
diff --git a/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift b/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift
index b115ac2f8f..9940b74e12 100644
--- a/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift
+++ b/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift
@@ -12,6 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import Atomics
 import NIOCore
 import NIOEmbedded
 import NIOPosix
@@ -26,9 +27,6 @@ protocol ScheduledCallbackTestRequirements {
 
     // ELG-backed ELs need to be shutdown via the ELG.
     func shutdownEventLoop() async throws
-
-    // This is here for NIOAsyncTestingEventLoop only.
-    func maybeInContext<R: Sendable>(_ body: @escaping @Sendable () throws -> R) async throws -> R
 }
 
 final class MTELGScheduledCallbackTests: _BaseScheduledCallbackTests {
@@ -43,10 +41,6 @@ final class MTELGScheduledCallbackTests: _BaseScheduledCallbackTests {
         func shutdownEventLoop() async throws {
             try await self.group.shutdownGracefully()
         }
-
-        func maybeInContext<R: Sendable>(_ body: @escaping @Sendable () throws -> R) async throws -> R {
-            try body()
-        }
     }
 
     override func setUp() async throws {
@@ -66,10 +60,6 @@ final class NIOAsyncTestingEventLoopScheduledCallbackTests: _BaseScheduledCallba
         func shutdownEventLoop() async throws {
             await self._loop.shutdownGracefully()
         }
-
-        func maybeInContext<R: Sendable>(_ body: @escaping @Sendable () throws -> R) async throws -> R {
-            try await self._loop.executeInContext(body)
-        }
     }
 
     override func setUp() async throws {
@@ -98,10 +88,6 @@ extension _BaseScheduledCallbackTests {
     func shutdownEventLoop() async throws {
         try await self.requirements.shutdownEventLoop()
     }
-
-    func maybeInContext<R: Sendable>(_ body: @escaping @Sendable () throws -> R) async throws -> R {
-        try await self.requirements.maybeInContext(body)
-    }
 }
 
 // The tests, abstracted over any of the event loops.
@@ -111,10 +97,10 @@ extension _BaseScheduledCallbackTests {
         let handler = MockScheduledCallbackHandler()
 
         _ = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 0) }
+        handler.assert(callbackCount: 0, cancelCount: 0)
 
         try await self.advanceTime(by: .microseconds(1))
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 0) }
+        handler.assert(callbackCount: 0, cancelCount: 0)
     }
 
     func testSheduledCallbackExecutedAtDeadline() async throws {
@@ -123,7 +109,7 @@ extension _BaseScheduledCallbackTests {
         _ = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
         try await self.advanceTime(by: .milliseconds(1))
         try await handler.waitForCallback(timeout: .seconds(1))
-        try await self.maybeInContext { handler.assert(callbackCount: 1, cancelCount: 0) }
+        handler.assert(callbackCount: 1, cancelCount: 0)
     }
 
     func testMultipleSheduledCallbacksUsingSameHandler() async throws {
@@ -135,7 +121,7 @@ extension _BaseScheduledCallbackTests {
         try await self.advanceTime(by: .milliseconds(1))
         try await handler.waitForCallback(timeout: .seconds(1))
         try await handler.waitForCallback(timeout: .seconds(1))
-        try await self.maybeInContext { handler.assert(callbackCount: 2, cancelCount: 0) }
+        handler.assert(callbackCount: 2, cancelCount: 0)
 
         _ = try self.loop.scheduleCallback(in: .milliseconds(2), handler: handler)
         _ = try self.loop.scheduleCallback(in: .milliseconds(3), handler: handler)
@@ -143,7 +129,7 @@ extension _BaseScheduledCallbackTests {
         try await self.advanceTime(by: .milliseconds(3))
         try await handler.waitForCallback(timeout: .seconds(1))
         try await handler.waitForCallback(timeout: .seconds(1))
-        try await self.maybeInContext { handler.assert(callbackCount: 4, cancelCount: 0) }
+        handler.assert(callbackCount: 4, cancelCount: 0)
     }
 
     func testMultipleSheduledCallbacksUsingDifferentHandlers() async throws {
@@ -156,8 +142,8 @@ extension _BaseScheduledCallbackTests {
         try await self.advanceTime(by: .milliseconds(1))
         try await handlerA.waitForCallback(timeout: .seconds(1))
         try await handlerB.waitForCallback(timeout: .seconds(1))
-        try await self.maybeInContext { handlerA.assert(callbackCount: 1, cancelCount: 0) }
-        try await self.maybeInContext { handlerB.assert(callbackCount: 1, cancelCount: 0) }
+        handlerA.assert(callbackCount: 1, cancelCount: 0)
+        handlerB.assert(callbackCount: 1, cancelCount: 0)
     }
 
     func testCancelExecutesCancellationCallback() async throws {
@@ -165,7 +151,7 @@ extension _BaseScheduledCallbackTests {
 
         let scheduledCallback = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
         scheduledCallback.cancel()
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 1) }
+        handler.assert(callbackCount: 0, cancelCount: 1)
     }
 
     func testCancelAfterDeadlineDoesNotExecutesCancellationCallback() async throws {
@@ -175,7 +161,7 @@ extension _BaseScheduledCallbackTests {
         try await self.advanceTime(by: .milliseconds(1))
         try await handler.waitForCallback(timeout: .seconds(1))
         scheduledCallback.cancel()
-        try await self.maybeInContext { handler.assert(callbackCount: 1, cancelCount: 0) }
+        handler.assert(callbackCount: 1, cancelCount: 0)
     }
 
     func testCancelAfterCancelDoesNotCallCancellationCallbackAgain() async throws {
@@ -184,7 +170,7 @@ extension _BaseScheduledCallbackTests {
         let scheduledCallback = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
         scheduledCallback.cancel()
         scheduledCallback.cancel()
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 1) }
+        handler.assert(callbackCount: 0, cancelCount: 1)
     }
 
     func testCancelAfterShutdownDoesNotCallCancellationCallbackAgain() async throws {
@@ -192,10 +178,10 @@ extension _BaseScheduledCallbackTests {
 
         let scheduledCallback = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
         try await self.shutdownEventLoop()
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 1) }
+        handler.assert(callbackCount: 0, cancelCount: 1)
 
         scheduledCallback.cancel()
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 1) }
+        handler.assert(callbackCount: 0, cancelCount: 1)
     }
 
     func testShutdownCancelsOutstandingScheduledCallbacks() async throws {
@@ -203,7 +189,7 @@ extension _BaseScheduledCallbackTests {
 
         _ = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
         try await self.shutdownEventLoop()
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 1) }
+        handler.assert(callbackCount: 0, cancelCount: 1)
     }
 
     func testShutdownDoesNotCancelCancelledCallbacksAgain() async throws {
@@ -211,10 +197,10 @@ extension _BaseScheduledCallbackTests {
 
         let handle = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
         handle.cancel()
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 1) }
+        handler.assert(callbackCount: 0, cancelCount: 1)
 
         try await self.shutdownEventLoop()
-        try await self.maybeInContext { handler.assert(callbackCount: 0, cancelCount: 1) }
+        handler.assert(callbackCount: 0, cancelCount: 1)
     }
 
     func testShutdownDoesNotCancelPastCallbacks() async throws {
@@ -223,16 +209,16 @@ extension _BaseScheduledCallbackTests {
         _ = try self.loop.scheduleCallback(in: .milliseconds(1), handler: handler)
         try await self.advanceTime(by: .milliseconds(1))
         try await handler.waitForCallback(timeout: .seconds(1))
-        try await self.maybeInContext { handler.assert(callbackCount: 1, cancelCount: 0) }
+        handler.assert(callbackCount: 1, cancelCount: 0)
 
         try await self.shutdownEventLoop()
-        try await self.maybeInContext { handler.assert(callbackCount: 1, cancelCount: 0) }
+        handler.assert(callbackCount: 1, cancelCount: 0)
     }
 }
 
-private final class MockScheduledCallbackHandler: NIOScheduledCallbackHandler {
-    var callbackCount = 0
-    var cancelCount = 0
+private final class MockScheduledCallbackHandler: NIOScheduledCallbackHandler, Sendable {
+    let callbackCount = ManagedAtomic(0)
+    let cancelCount = ManagedAtomic(0)
 
     let callbackStream: AsyncStream<Void>
     private let callbackStreamContinuation: AsyncStream<Void>.Continuation
@@ -246,17 +232,29 @@ private final class MockScheduledCallbackHandler: NIOScheduledCallbackHandler {
     }
 
     func handleScheduledCallback(eventLoop: some EventLoop) {
-        self.callbackCount += 1
+        self.callbackCount.wrappingIncrement(by: 1, ordering: .sequentiallyConsistent)
         self.callbackStreamContinuation.yield()
     }
 
     func didCancelScheduledCallback(eventLoop: some EventLoop) {
-        self.cancelCount += 1
+        self.cancelCount.wrappingIncrement(by: 1, ordering: .sequentiallyConsistent)
     }
 
     func assert(callbackCount: Int, cancelCount: Int, file: StaticString = #file, line: UInt = #line) {
-        XCTAssertEqual(self.callbackCount, callbackCount, "Unexpected callback count", file: file, line: line)
-        XCTAssertEqual(self.cancelCount, cancelCount, "Unexpected cancel count", file: file, line: line)
+        XCTAssertEqual(
+            self.callbackCount.load(ordering: .sequentiallyConsistent),
+            callbackCount,
+            "Unexpected callback count",
+            file: file,
+            line: line
+        )
+        XCTAssertEqual(
+            self.cancelCount.load(ordering: .sequentiallyConsistent),
+            cancelCount,
+            "Unexpected cancel count",
+            file: file,
+            line: line
+        )
     }
 
     func waitForCallback(timeout: TimeAmount, file: StaticString = #file, line: UInt = #line) async throws {
diff --git a/Tests/NIOPosixTests/RawSocketBootstrapTests.swift b/Tests/NIOPosixTests/RawSocketBootstrapTests.swift
index a65ad1ae91..a7c42c3c82 100644
--- a/Tests/NIOPosixTests/RawSocketBootstrapTests.swift
+++ b/Tests/NIOPosixTests/RawSocketBootstrapTests.swift
@@ -47,8 +47,13 @@ final class RawSocketBootstrapTests: XCTestCase {
         let elg = MultiThreadedEventLoopGroup(numberOfThreads: 1)
         defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) }
         let channel = try NIORawSocketBootstrap(group: elg)
-            .channelInitializer {
-                $0.pipeline.addHandler(DatagramReadRecorder<ByteBuffer>(), name: "ByteReadRecorder")
+            .channelInitializer { channel in
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(
+                        DatagramReadRecorder<ByteBuffer>(),
+                        name: "ByteReadRecorder"
+                    )
+                }
             }
             .bind(host: "127.0.0.1", ipProtocol: .reservedForTesting).wait()
         defer { XCTAssertNoThrow(try channel.close().wait()) }
@@ -93,15 +98,25 @@ final class RawSocketBootstrapTests: XCTestCase {
         let elg = MultiThreadedEventLoopGroup(numberOfThreads: 1)
         defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) }
         let readChannel = try NIORawSocketBootstrap(group: elg)
-            .channelInitializer {
-                $0.pipeline.addHandler(DatagramReadRecorder<ByteBuffer>(), name: "ByteReadRecorder")
+            .channelInitializer { channel in
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(
+                        DatagramReadRecorder<ByteBuffer>(),
+                        name: "ByteReadRecorder"
+                    )
+                }
             }
             .bind(host: "127.0.0.1", ipProtocol: .reservedForTesting).wait()
         defer { XCTAssertNoThrow(try readChannel.close().wait()) }
 
         let writeChannel = try NIORawSocketBootstrap(group: elg)
-            .channelInitializer {
-                $0.pipeline.addHandler(DatagramReadRecorder<ByteBuffer>(), name: "ByteReadRecorder")
+            .channelInitializer { channel in
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(
+                        DatagramReadRecorder<ByteBuffer>(),
+                        name: "ByteReadRecorder"
+                    )
+                }
             }
             .bind(host: "127.0.0.1", ipProtocol: .reservedForTesting).wait()
         defer { XCTAssertNoThrow(try writeChannel.close().wait()) }
@@ -147,8 +162,13 @@ final class RawSocketBootstrapTests: XCTestCase {
         defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) }
         let channel = try NIORawSocketBootstrap(group: elg)
             .channelOption(.ipOption(.ip_hdrincl), value: 1)
-            .channelInitializer {
-                $0.pipeline.addHandler(DatagramReadRecorder<ByteBuffer>(), name: "ByteReadRecorder")
+            .channelInitializer { channel in
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(
+                        DatagramReadRecorder<ByteBuffer>(),
+                        name: "ByteReadRecorder"
+                    )
+                }
             }
             .bind(host: "127.0.0.1", ipProtocol: .reservedForTesting).wait()
         defer { XCTAssertNoThrow(try channel.close().wait()) }
diff --git a/Tests/NIOPosixTests/SocketChannelTest.swift b/Tests/NIOPosixTests/SocketChannelTest.swift
index cb8161a7e5..a42eefb0fc 100644
--- a/Tests/NIOPosixTests/SocketChannelTest.swift
+++ b/Tests/NIOPosixTests/SocketChannelTest.swift
@@ -1138,11 +1138,12 @@ class DropAllReadsOnTheFloorHandler: ChannelDuplexHandler {
             // What we're trying to do here is forcing a close without calling `close`. We know that the other side of
             // the connection is fully closed but because we support half-closure, we need to write to 'learn' that the
             // other side has actually fully closed the socket.
+            let promise = self.waitUntilWriteFailedPromise
             func writeUntilError() {
                 context.writeAndFlush(Self.wrapOutboundOut(buffer)).map {
                     writeUntilError()
                 }.whenFailure { (_: Error) in
-                    self.waitUntilWriteFailedPromise.succeed(())
+                    promise.succeed(())
                 }
             }
             writeUntilError()
diff --git a/Tests/NIOPosixTests/UniversalBootstrapSupportTest.swift b/Tests/NIOPosixTests/UniversalBootstrapSupportTest.swift
index 8b671fce63..9d729c5532 100644
--- a/Tests/NIOPosixTests/UniversalBootstrapSupportTest.swift
+++ b/Tests/NIOPosixTests/UniversalBootstrapSupportTest.swift
@@ -83,7 +83,13 @@ class UniversalBootstrapSupportTest: XCTestCase {
 
                 let client = try NIOClientTCPBootstrap(ClientBootstrap(group: group), tls: DummyTLSProvider())
                     .channelInitializer { channel in
-                        channel.pipeline.addHandlers(counter1, DropChannelReadsHandler(), counter2)
+                        channel.eventLoop.makeCompletedFuture {
+                            try channel.pipeline.syncOperations.addHandlers(
+                                counter1,
+                                DropChannelReadsHandler(),
+                                counter2
+                            )
+                        }
                     }
                     .channelOption(.autoRead, value: false)
                     .connectTimeout(.hours(1))
diff --git a/Tests/NIOWebSocketTests/WebSocketClientEndToEndTests.swift b/Tests/NIOWebSocketTests/WebSocketClientEndToEndTests.swift
index 23202c2c4b..71a86bdc1e 100644
--- a/Tests/NIOWebSocketTests/WebSocketClientEndToEndTests.swift
+++ b/Tests/NIOWebSocketTests/WebSocketClientEndToEndTests.swift
@@ -170,7 +170,9 @@ class WebSocketClientEndToEndTests: XCTestCase {
         let basicUpgrader = NIOWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -244,7 +246,9 @@ class WebSocketClientEndToEndTests: XCTestCase {
         let basicUpgrader = NIOWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -275,7 +279,9 @@ class WebSocketClientEndToEndTests: XCTestCase {
         let basicUpgrader = NIOWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -307,7 +313,9 @@ class WebSocketClientEndToEndTests: XCTestCase {
         let basicUpgrader = NIOWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -332,13 +340,12 @@ class WebSocketClientEndToEndTests: XCTestCase {
     }
 
     fileprivate func runSuccessfulUpgrade() throws -> (EmbeddedChannel, WebSocketRecorderHandler) {
-
-        let handler = WebSocketRecorderHandler()
-
         let basicUpgrader = NIOWebSocketClientUpgrader(
             requestKey: "OfS0wDaT5NoxF2gqm7Zj2YtetzM=",
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(handler)
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -361,6 +368,10 @@ class WebSocketClientEndToEndTests: XCTestCase {
 
         clientChannel.embeddedEventLoop.run()
 
+        // Ok, now grab the handler. We can do this with sync operations, because this is an
+        // EmbeddedChannel.
+        let handler = try clientChannel.pipeline.syncOperations.handler(type: WebSocketRecorderHandler.self)
+
         return (clientChannel, handler)
 
     }
@@ -501,7 +512,9 @@ final class TypedWebSocketClientEndToEndTests: WebSocketClientEndToEndTests {
         let basicUpgrader = NIOTypedWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -570,7 +583,9 @@ final class TypedWebSocketClientEndToEndTests: WebSocketClientEndToEndTests {
         let basicUpgrader = NIOTypedWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -603,7 +618,9 @@ final class TypedWebSocketClientEndToEndTests: WebSocketClientEndToEndTests {
         let basicUpgrader = NIOTypedWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -637,7 +654,9 @@ final class TypedWebSocketClientEndToEndTests: WebSocketClientEndToEndTests {
         let basicUpgrader = NIOTypedWebSocketClientUpgrader(
             requestKey: requestKey,
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(WebSocketRecorderHandler())
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -665,12 +684,12 @@ final class TypedWebSocketClientEndToEndTests: WebSocketClientEndToEndTests {
     }
 
     override fileprivate func runSuccessfulUpgrade() throws -> (EmbeddedChannel, WebSocketRecorderHandler) {
-        let handler = WebSocketRecorderHandler()
-
         let basicUpgrader = NIOTypedWebSocketClientUpgrader(
             requestKey: "OfS0wDaT5NoxF2gqm7Zj2YtetzM=",
             upgradePipelineHandler: { (channel: Channel, _: HTTPResponseHead) in
-                channel.pipeline.addHandler(handler)
+                channel.eventLoop.makeCompletedFuture {
+                    try channel.pipeline.syncOperations.addHandler(WebSocketRecorderHandler())
+                }
             }
         )
 
@@ -695,6 +714,10 @@ final class TypedWebSocketClientEndToEndTests: WebSocketClientEndToEndTests {
 
         try upgradeResult.wait()
 
+        // Ok, now grab the handler. We can do this with sync operations, because this is an
+        // EmbeddedChannel.
+        let handler = try clientChannel.pipeline.syncOperations.handler(type: WebSocketRecorderHandler.self)
+
         return (clientChannel, handler)
     }
 }

From 47785adb51d724a2d19c7d042a89bcb1b65036b9 Mon Sep 17 00:00:00 2001
From: finagolfin <finagolfin@tuta.io>
Date: Sun, 15 Dec 2024 17:07:57 +0530
Subject: [PATCH 35/37] Remove now unneeded fts_open bitcast for Android
 (#3025)

I added this changed function call for Android in #2660 earlier this
year, because of [an incorrect Bionic function signature as explained
recently](https://github.com/apple/swift-nio/pull/3009#discussion_r1864807270),
but now that it has been worked around through a new API note as
mentioned there, this change is no longer needed.

I simply did not notice this earlier because it still compiles and
merely gives a warning, which removing this call now silences.
---
 Sources/NIOFileSystem/Internal/System Calls/Syscall.swift | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift b/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift
index de53259404..fe0318707a 100644
--- a/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift	
+++ b/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift	
@@ -400,14 +400,7 @@ public enum Libc {
         return valueOrErrno {
             pathBytes.withUnsafeMutableBufferPointer { pointer in
                 // The array must be terminated with a nil.
-                #if os(Android)
-                libc_fts_open(
-                    [pointer.baseAddress!, unsafeBitCast(0, to: UnsafeMutablePointer<CInterop.PlatformChar>.self)],
-                    options.rawValue
-                )
-                #else
                 libc_fts_open([pointer.baseAddress, nil], options.rawValue)
-                #endif
             }
         }
     }

From ecfaa2c6ff3234bdf7909eba0da01c245922e471 Mon Sep 17 00:00:00 2001
From: Johannes Weiss <johannesweiss@apple.com>
Date: Mon, 16 Dec 2024 08:32:23 +0000
Subject: [PATCH 36/37] fix bogus "unleakable promise leaked" error message
 (#2855) (#3027)

### Motivation:

SwiftNIO contained a bogus error message:

```
BUG in SwiftNIO (please report), unleakable promise leaked.:486: Fatal error: leaking promise created at (file: "BUG in SwiftNIO (please report), unleakable promise leaked.", line: 486)
```

The actual meaning is that the ELG was shut down prematurely.

### Modifications:

- Replace the message with an accurate one

### Result:

- Users less confused
- fixes #2855
- fixes #2201
---
 Sources/NIOCore/EventLoopFuture.swift | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Sources/NIOCore/EventLoopFuture.swift b/Sources/NIOCore/EventLoopFuture.swift
index 793d3a7ed5..4579576ed6 100644
--- a/Sources/NIOCore/EventLoopFuture.swift
+++ b/Sources/NIOCore/EventLoopFuture.swift
@@ -163,7 +163,12 @@ public struct EventLoopPromise<Value> {
     internal static func makeUnleakablePromise(eventLoop: EventLoop, line: UInt = #line) -> EventLoopPromise<Value> {
         EventLoopPromise<Value>(
             eventLoop: eventLoop,
-            file: "BUG in SwiftNIO (please report), unleakable promise leaked.",
+            file: """
+                EventLoopGroup shut down with unfulfilled promises remaining. \
+                This suggests that the EventLoopGroup was shut down with unfinished work outstanding which is \
+                illegal. Either switch to using the singleton EventLoopGroups or fix the issue by only shutting down \
+                the EventLoopGroups when all the work associated with them has finished.
+                """,
             line: line
         )
     }

From 55d0f72b309a9217405c9c7fc0163af521ee0db3 Mon Sep 17 00:00:00 2001
From: Rick Newton-Rogers <rnro@apple.com>
Date: Mon, 16 Dec 2024 11:17:18 +0000
Subject: [PATCH 37/37] Enable MemberImportVisibility check on all targets
 (#3021)

Enable MemberImportVisibility check on all targets. Use a standard
string header and footer to bracket the new block for ease of updating
in the future with scripts.

Co-authored-by: George Barnett <gbarnett@apple.com>
---
 Package.swift | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Package.swift b/Package.swift
index 021ebf8e50..b1fcd3cc5f 100644
--- a/Package.swift
+++ b/Package.swift
@@ -566,3 +566,14 @@ if Context.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
         .package(path: "../swift-system"),
     ]
 }
+
+// ---    STANDARD CROSS-REPO SETTINGS DO NOT EDIT   --- //
+for target in package.targets {
+    if target.type != .plugin {
+        var settings = target.swiftSettings ?? []
+        // https://github.com/swiftlang/swift-evolution/blob/main/proposals/0444-member-import-visibility.md
+        settings.append(.enableUpcomingFeature("MemberImportVisibility"))
+        target.swiftSettings = settings
+    }
+}
+// --- END: STANDARD CROSS-REPO SETTINGS DO NOT EDIT --- //