From f80c69f84b90874432aafd76c240fef1aaa01272 Mon Sep 17 00:00:00 2001 From: George Barnett Date: Tue, 16 Jan 2024 11:30:26 +0000 Subject: [PATCH] Add NIOFilesystem (#2615) Motivation: I/O is typically blocking, but options for interacting with the filesystem using `async` APIs are limited. This change adds a new module to NIO, `NIOFileSystem`. At the moment this is exposed as the `_NIOFileSystem` product while users get a chance to try it out and provide feedback. During this time the API of `NIOFileSystem` is subject to change. Breaking changes will come in _minor_ NIO releases and additive changes may come in _patch_ releases of NIO. Modifications: - Add `NIOFileSystem`, `NIOFileSystemFoundationCompat`, `NIOFileSystemTests`, `NIOFileSystemIntegrationTests`, and `NIOFileSystemFoundationCompatTests`. - Update docs - Update CNIOLinux and CNIODarwin shims Result: File system can be accessed via `async` APIs --- .spi.yml | 2 +- NOTICE.txt | 9 + Package.swift | 58 + README.md | 7 +- Sources/CNIODarwin/include/CNIODarwin.h | 6 + Sources/CNIODarwin/shim.c | 5 + Sources/CNIOLinux/include/CNIOLinux.h | 16 + Sources/CNIOLinux/shim.c | 13 + Sources/NIO/Docs.docc/index.md | 2 + Sources/NIOFileSystem/BufferedReader.swift | 208 ++ Sources/NIOFileSystem/BufferedWriter.swift | 187 ++ .../NIOFileSystem/ByteBuffer+FileSystem.swift | 97 + Sources/NIOFileSystem/ByteCount.swift | 79 + Sources/NIOFileSystem/Convenience.swift | 202 ++ Sources/NIOFileSystem/DirectoryEntries.swift | 654 +++++++ Sources/NIOFileSystem/DirectoryEntry.swift | 47 + .../Extensions/DirectoryFileHandleProtocol.md | 27 + .../Extensions/FileHandleProtocol.md | 27 + .../Extensions/FileSystemProtocol.md | 45 + .../Extensions/ReadableFileHandleProtocol.md | 19 + .../Extensions/WritableFileHandleProtocol.md | 11 + .../NIOFileSystem/Docs.docc/NIOFileSystem.md | 174 ++ Sources/NIOFileSystem/Exports.swift | 21 + Sources/NIOFileSystem/FileChunks.swift | 337 ++++ Sources/NIOFileSystem/FileHandle.swift | 307 +++ .../NIOFileSystem/FileHandleProtocol.swift | 688 +++++++ Sources/NIOFileSystem/FileInfo.swift | 248 +++ Sources/NIOFileSystem/FileSystem.swift | 1435 ++++++++++++++ .../FileSystemError+Syscall.swift | 1026 ++++++++++ Sources/NIOFileSystem/FileSystemError.swift | 333 ++++ .../NIOFileSystem/FileSystemProtocol.swift | 478 +++++ Sources/NIOFileSystem/FileType.swift | 167 ++ .../Internal/BufferedOrAnyStream.swift | 90 + .../Internal/BufferedStream.swift | 1733 +++++++++++++++++ .../NIOFileSystem/Internal/Cancellation.swift | 53 + .../Concurrency Primitives/IOExecutor.swift | 421 ++++ .../Concurrency Primitives/Thread.swift | 97 + .../Concurrency Primitives/ThreadPosix.swift | 141 ++ .../UnsafeTransfer.swift | 24 + .../String+UnsafeUnititializedCapacity.swift | 61 + .../Internal/System Calls/CInterop.swift | 63 + .../Internal/System Calls/Errno.swift | 132 ++ .../FileDescriptor+Syscalls.swift | 320 +++ .../Internal/System Calls/Mocking.swift | 359 ++++ .../Internal/System Calls/Syscall.swift | 378 ++++ .../Internal/System Calls/Syscalls.swift | 412 ++++ .../Internal/SystemFileHandle.swift | 1303 +++++++++++++ .../NIOFileSystem/Internal/Utilities.swift | 45 + Sources/NIOFileSystem/OpenOptions.swift | 297 +++ .../Date+FileInfo.swift | 31 + .../FileSystemFoundationCompatTests.swift | 35 + .../BufferedReaderTests.swift | 261 +++ .../BufferedWriterTests.swift | 143 ++ .../ConvenienceTests.swift | 73 + .../FileHandleTests.swift | 1105 +++++++++++ .../FileSystemTests+SPI.swift | 27 + .../FileSystemTests.swift | 1430 ++++++++++++++ .../Test Data/Foo.symlink | 1 + .../Test Data/Foo/README.txt | 1 + .../Test Data/README.md | 5 + .../Test Data/README.md.symlink | 1 + .../XCTestExtensions.swift | 68 + Tests/NIOFileSystemTests/ByteCountTests.swift | 60 + .../DirectoryEntriesTests.swift | 72 + .../NIOFileSystemTests/FileChunksTests.swift | 40 + .../NIOFileSystemTests/FileHandleTests.swift | 269 +++ Tests/NIOFileSystemTests/FileInfoTests.swift | 158 ++ .../FileOpenOptionsTests.swift | 100 + .../FileSystemErrorTests.swift | 607 ++++++ Tests/NIOFileSystemTests/FileTypeTests.swift | 81 + .../Internal/CancellationTests.swift | 89 + .../BufferedStreamTests.swift | 1136 +++++++++++ .../IOExecutorTests.swift | 172 ++ .../Internal/MockingInfrastructure.swift | 254 +++ .../Internal/SyscallTests.swift | 470 +++++ .../NIOFileSystemTests/XCTestExtensions.swift | 68 + 76 files changed, 19617 insertions(+), 4 deletions(-) create mode 100644 Sources/NIOFileSystem/BufferedReader.swift create mode 100644 Sources/NIOFileSystem/BufferedWriter.swift create mode 100644 Sources/NIOFileSystem/ByteBuffer+FileSystem.swift create mode 100644 Sources/NIOFileSystem/ByteCount.swift create mode 100644 Sources/NIOFileSystem/Convenience.swift create mode 100644 Sources/NIOFileSystem/DirectoryEntries.swift create mode 100644 Sources/NIOFileSystem/DirectoryEntry.swift create mode 100644 Sources/NIOFileSystem/Docs.docc/Extensions/DirectoryFileHandleProtocol.md create mode 100644 Sources/NIOFileSystem/Docs.docc/Extensions/FileHandleProtocol.md create mode 100644 Sources/NIOFileSystem/Docs.docc/Extensions/FileSystemProtocol.md create mode 100644 Sources/NIOFileSystem/Docs.docc/Extensions/ReadableFileHandleProtocol.md create mode 100644 Sources/NIOFileSystem/Docs.docc/Extensions/WritableFileHandleProtocol.md create mode 100644 Sources/NIOFileSystem/Docs.docc/NIOFileSystem.md create mode 100644 Sources/NIOFileSystem/Exports.swift create mode 100644 Sources/NIOFileSystem/FileChunks.swift create mode 100644 Sources/NIOFileSystem/FileHandle.swift create mode 100644 Sources/NIOFileSystem/FileHandleProtocol.swift create mode 100644 Sources/NIOFileSystem/FileInfo.swift create mode 100644 Sources/NIOFileSystem/FileSystem.swift create mode 100644 Sources/NIOFileSystem/FileSystemError+Syscall.swift create mode 100644 Sources/NIOFileSystem/FileSystemError.swift create mode 100644 Sources/NIOFileSystem/FileSystemProtocol.swift create mode 100644 Sources/NIOFileSystem/FileType.swift create mode 100644 Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift create mode 100644 Sources/NIOFileSystem/Internal/BufferedStream.swift create mode 100644 Sources/NIOFileSystem/Internal/Cancellation.swift create mode 100644 Sources/NIOFileSystem/Internal/Concurrency Primitives/IOExecutor.swift create mode 100644 Sources/NIOFileSystem/Internal/Concurrency Primitives/Thread.swift create mode 100644 Sources/NIOFileSystem/Internal/Concurrency Primitives/ThreadPosix.swift create mode 100644 Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift create mode 100644 Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift create mode 100644 Sources/NIOFileSystem/Internal/System Calls/CInterop.swift create mode 100644 Sources/NIOFileSystem/Internal/System Calls/Errno.swift create mode 100644 Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift create mode 100644 Sources/NIOFileSystem/Internal/System Calls/Mocking.swift create mode 100644 Sources/NIOFileSystem/Internal/System Calls/Syscall.swift create mode 100644 Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift create mode 100644 Sources/NIOFileSystem/Internal/SystemFileHandle.swift create mode 100644 Sources/NIOFileSystem/Internal/Utilities.swift create mode 100644 Sources/NIOFileSystem/OpenOptions.swift create mode 100644 Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift create mode 100644 Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift create mode 100644 Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift create mode 100644 Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift create mode 100644 Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift create mode 100644 Tests/NIOFileSystemIntegrationTests/FileHandleTests.swift create mode 100644 Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift create mode 100644 Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift create mode 120000 Tests/NIOFileSystemIntegrationTests/Test Data/Foo.symlink create mode 100644 Tests/NIOFileSystemIntegrationTests/Test Data/Foo/README.txt create mode 100644 Tests/NIOFileSystemIntegrationTests/Test Data/README.md create mode 120000 Tests/NIOFileSystemIntegrationTests/Test Data/README.md.symlink create mode 100644 Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift create mode 100644 Tests/NIOFileSystemTests/ByteCountTests.swift create mode 100644 Tests/NIOFileSystemTests/DirectoryEntriesTests.swift create mode 100644 Tests/NIOFileSystemTests/FileChunksTests.swift create mode 100644 Tests/NIOFileSystemTests/FileHandleTests.swift create mode 100644 Tests/NIOFileSystemTests/FileInfoTests.swift create mode 100644 Tests/NIOFileSystemTests/FileOpenOptionsTests.swift create mode 100644 Tests/NIOFileSystemTests/FileSystemErrorTests.swift create mode 100644 Tests/NIOFileSystemTests/FileTypeTests.swift create mode 100644 Tests/NIOFileSystemTests/Internal/CancellationTests.swift create mode 100644 Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift create mode 100644 Tests/NIOFileSystemTests/Internal/Concurrency Primitives/IOExecutorTests.swift create mode 100644 Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift create mode 100644 Tests/NIOFileSystemTests/Internal/SyscallTests.swift create mode 100644 Tests/NIOFileSystemTests/XCTestExtensions.swift diff --git a/.spi.yml b/.spi.yml index 1a80188e62..54aa41a049 100644 --- a/.spi.yml +++ b/.spi.yml @@ -1,4 +1,4 @@ version: 1 builder: configs: - - documentation_targets: [NIO, NIOConcurrencyHelpers, NIOCore, NIOEmbedded, NIOFoundationCompat, NIOHTTP1, NIOPosix, NIOTLS, NIOWebSocket, NIOTestUtils] + - documentation_targets: [NIO, NIOConcurrencyHelpers, NIOCore, NIOEmbedded, NIOFoundationCompat, NIOHTTP1, NIOPosix, NIOTLS, NIOWebSocket, NIOTestUtils, NIOFileSystem] diff --git a/NOTICE.txt b/NOTICE.txt index e977e8e139..6caf903044 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -96,3 +96,12 @@ This product contains a derivation of "_TinyArray.swift" from SwiftCertificates. * https://www.apache.org/licenses/LICENSE-2.0 * HOMEPAGE: * https://github.com/apple/swift-certificates + +--- + +This product contains a derivation of the mocking infrastructure from Swift System. + + * LICENSE (Apache License 2.0): + * https://www.apache.org/licenses/LICENSE-2.0 + * HOMEPAGE: + * https://github.com/apple/swift-system diff --git a/Package.swift b/Package.swift index ac54050d6d..f63902b99b 100644 --- a/Package.swift +++ b/Package.swift @@ -17,6 +17,7 @@ import PackageDescription let swiftAtomics: PackageDescription.Target.Dependency = .product(name: "Atomics", package: "swift-atomics") let swiftCollections: PackageDescription.Target.Dependency = .product(name: "DequeModule", package: "swift-collections") +let swiftSystem: PackageDescription.Target.Dependency = .product(name: "SystemPackage", package: "swift-system") let package = Package( @@ -33,10 +34,13 @@ let package = Package( .library(name: "NIOFoundationCompat", targets: ["NIOFoundationCompat"]), .library(name: "NIOWebSocket", targets: ["NIOWebSocket"]), .library(name: "NIOTestUtils", targets: ["NIOTestUtils"]), + .library(name: "_NIOFileSystem", targets: ["NIOFileSystem"]), + .library(name: "_NIOFileSystemFoundationCompat", targets: ["NIOFileSystemFoundationCompat"]), ], dependencies: [ .package(url: "https://github.com/apple/swift-atomics.git", from: "1.1.0"), .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.2"), + .package(url: "https://github.com/apple/swift-system.git", from: "1.2.0"), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"), ], targets: [ @@ -185,6 +189,26 @@ let package = Package( swiftAtomics, ] ), + .target( + name: "NIOFileSystem", + dependencies: [ + "NIOCore", + "CNIOLinux", + "CNIODarwin", + swiftAtomics, + swiftCollections, + swiftSystem, + ], + swiftSettings: [ + .define("ENABLE_MOCKING", .when(configuration: .debug)) + ] + ), + .target( + name: "NIOFileSystemFoundationCompat", + dependencies: [ + "NIOFileSystem", + ] + ), // MARK: - Examples @@ -434,5 +458,39 @@ let package = Package( name: "NIOSingletonsTests", dependencies: ["NIOCore", "NIOPosix"] ), + .testTarget( + name: "NIOFileSystemTests", + dependencies: [ + "NIOCore", + "NIOFileSystem", + swiftAtomics, + swiftCollections, + swiftSystem, + ], + swiftSettings: [ + .define("ENABLE_MOCKING", .when(configuration: .debug)) + ] + ), + .testTarget( + name: "NIOFileSystemIntegrationTests", + dependencies: [ + "NIOCore", + "NIOFileSystem", + "NIOFoundationCompat", + ], + exclude: [ + // Contains known files and directory structures used + // for the integration tests. Exclude the whole tree from + // the build. + "Test Data", + ] + ), + .testTarget( + name: "NIOFileSystemFoundationCompatTests", + dependencies: [ + "NIOFileSystem", + "NIOFileSystemFoundationCompat", + ] + ) ] ) diff --git a/README.md b/README.md index 725f5cd30c..b93d416abd 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Within this repository we have a number of products that provide different funct - `NIOHTTP1`. This provides a low-level HTTP/1.1 protocol implementation. - `NIOWebSocket`. This provides a low-level WebSocket protocol implementation. - `NIOTestUtils`. This provides a number of helpers for testing projects that use SwiftNIO. +- `NIOFileSystem`. This provides `async` APIs for interacting with the file system. ### Protocol Implementations @@ -354,16 +355,16 @@ dnf install swift-lang /usr/bin/nc /usr/bin/lsof /usr/bin/shasum ### Benchmarks -Benchmarks for `swift-nio` are in a separate Swift Package in the `Benchmarks` subfolder of this repository. +Benchmarks for `swift-nio` are in a separate Swift Package in the `Benchmarks` subfolder of this repository. They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin. Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics. -An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. +An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking: ``` swift package benchmark ``` -For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). +For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). [ch]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channelhandler [c]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channel diff --git a/Sources/CNIODarwin/include/CNIODarwin.h b/Sources/CNIODarwin/include/CNIODarwin.h index 1b7fd468b2..a4eac11522 100644 --- a/Sources/CNIODarwin/include/CNIODarwin.h +++ b/Sources/CNIODarwin/include/CNIODarwin.h @@ -19,6 +19,10 @@ #include #include #include +#include +#include +#include +#include // Darwin platforms do not have a sendmmsg implementation available to them. This C module // provides a shim that implements sendmmsg on top of sendmsg. It also provides a shim for @@ -54,5 +58,7 @@ size_t CNIODarwin_CMSG_SPACE(size_t); extern const unsigned long CNIODarwin_IOCTL_VM_SOCKETS_GET_LOCAL_CID; +const char* CNIODarwin_dirent_dname(struct dirent* ent); + #endif // __APPLE__ #endif // C_NIO_DARWIN_H diff --git a/Sources/CNIODarwin/shim.c b/Sources/CNIODarwin/shim.c index 46c5815360..30b81bcb67 100644 --- a/Sources/CNIODarwin/shim.c +++ b/Sources/CNIODarwin/shim.c @@ -21,6 +21,7 @@ #include #include #include +#include "dirent.h" int CNIODarwin_sendmmsg(int sockfd, CNIODarwin_mmsghdr *msgvec, unsigned int vlen, int flags) { // Some quick error checking. If vlen can't fit into int, we bail. @@ -92,4 +93,8 @@ const int CNIODarwin_IPV6_PKTINFO = IPV6_PKTINFO; const unsigned long CNIODarwin_IOCTL_VM_SOCKETS_GET_LOCAL_CID = IOCTL_VM_SOCKETS_GET_LOCAL_CID; +const char* CNIODarwin_dirent_dname(struct dirent* ent) { + return ent->d_name; +} + #endif // __APPLE__ diff --git a/Sources/CNIOLinux/include/CNIOLinux.h b/Sources/CNIOLinux/include/CNIOLinux.h index 108301ad3b..a75e07f63a 100644 --- a/Sources/CNIOLinux/include/CNIOLinux.h +++ b/Sources/CNIOLinux/include/CNIOLinux.h @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -30,6 +33,10 @@ #include #include "liburing_nio.h" #include +#include +#include +#include +#include #if __has_include() #include @@ -117,5 +124,14 @@ int CNIOLinux_system_info(struct utsname* uname_data); extern const unsigned long CNIOLinux_IOCTL_VM_SOCKETS_GET_LOCAL_CID; +const char* CNIOLinux_dirent_dname(struct dirent* ent); + +int CNIOLinux_renameat2(int oldfd, const char* old, int newfd, const char* newName, unsigned int flags); + +extern const int CNIOLinux_O_TMPFILE; +extern const int CNIOLinux_AT_EMPTY_PATH; +extern const unsigned int CNIOLinux_RENAME_NOREPLACE; +extern const unsigned int CNIOLinux_RENAME_EXCHANGE; + #endif #endif diff --git a/Sources/CNIOLinux/shim.c b/Sources/CNIOLinux/shim.c index d56c3bba69..f1642a3ef0 100644 --- a/Sources/CNIOLinux/shim.c +++ b/Sources/CNIOLinux/shim.c @@ -188,4 +188,17 @@ int CNIOLinux_system_info(struct utsname* uname_data) { const unsigned long CNIOLinux_IOCTL_VM_SOCKETS_GET_LOCAL_CID = IOCTL_VM_SOCKETS_GET_LOCAL_CID; +const char* CNIOLinux_dirent_dname(struct dirent* ent) { + return ent->d_name; +} + +int CNIOLinux_renameat2(int oldfd, const char* old, int newfd, const char* newName, unsigned int flags) { + return renameat2(oldfd, old, newfd, newName, flags); +} + +const int CNIOLinux_O_TMPFILE = O_TMPFILE; +const unsigned int CNIOLinux_RENAME_NOREPLACE = RENAME_NOREPLACE; +const unsigned int CNIOLinux_RENAME_EXCHANGE = RENAME_EXCHANGE; +const int CNIOLinux_AT_EMPTY_PATH = AT_EMPTY_PATH; + #endif diff --git a/Sources/NIO/Docs.docc/index.md b/Sources/NIO/Docs.docc/index.md index fadd6fe85b..f3f864855e 100644 --- a/Sources/NIO/Docs.docc/index.md +++ b/Sources/NIO/Docs.docc/index.md @@ -35,6 +35,7 @@ SwiftNIO has a number of products that provide different functionality. This pac - [NIOHTTP1][module-http1]. This provides a low-level HTTP/1.1 protocol implementation. - [NIOWebSocket][module-websocket]. This provides a low-level WebSocket protocol implementation. - [NIOTestUtils][module-test-utilities]. This provides a number of helpers for testing projects that use SwiftNIO. +- [NIOFileSystem][module-filesystem]. This provides `async` APIs for interacting with the file system. ### Conceptual Overview @@ -160,6 +161,7 @@ The core SwiftNIO repository will contain a few extremely important protocol imp [module-tls]: ./NIOTLS [module-websocket]: ./NIOWebSocket [module-test-utilities]: ./NIOTestUtils +[module-filesystem]: ./NIOFileSystem [ch]: ./NIOCore/ChannelHandler [c]: ./NIOCore/Channel diff --git a/Sources/NIOFileSystem/BufferedReader.swift b/Sources/NIOFileSystem/BufferedReader.swift new file mode 100644 index 0000000000..1fa2235b9f --- /dev/null +++ b/Sources/NIOFileSystem/BufferedReader.swift @@ -0,0 +1,208 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import DequeModule +import NIOCore + +/// A reader which maintains a buffer of bytes read from the file. +/// +/// You can create a reader from a ``ReadableFileHandleProtocol`` by calling +/// ``ReadableFileHandleProtocol/bufferedReader(startingAtAbsoluteOffset:capacity:)``. Call +/// ``read(_:)`` to read a fixed number of bytes from the file or ``read(while:)`` to read +/// from the file while the bytes match a predicate. +/// +/// You can also read bytes without returning them to caller by calling ``drop(_:)`` and +/// ``drop(while:)``. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct BufferedReader { + /// The handle to read from. + private let handle: Handle + + /// The offset for the next read from the file. + private var offset: Int64 + + /// Whether the reader has read to the end of the file. + private var readEOF = false + + /// A buffer containing the read bytes. + private var buffer: ByteBuffer + + /// The capacity of the buffer. + public let capacity: Int + + /// The number of bytes currently in the buffer. + public var count: Int { + return self.buffer.readableBytes + } + + internal init(wrapping readableHandle: Handle, initialOffset: Int64, capacity: Int) { + precondition( + initialOffset >= 0, + "initialOffset (\(initialOffset)) must be greater than or equal to zero" + ) + precondition(capacity > 0, "capacity (\(capacity)) must be greater than zero") + self.handle = readableHandle + self.offset = initialOffset + self.capacity = capacity + self.buffer = ByteBuffer() + } + + private mutating func readFromFile(_ count: Int) async throws -> ByteBuffer { + let bytes = try await self.handle.readChunk( + fromAbsoluteOffset: self.offset, + length: .bytes(Int64(count)) + ) + // Reading short means reading end-of-file. + self.readEOF = bytes.readableBytes < count + self.offset += Int64(bytes.readableBytes) + return bytes + } + + /// Read at most `count` bytes from the file; reads short if not enough bytes are available. + /// + /// - Parameters: + /// - count: The number of bytes to read. + /// - Returns: The bytes read from the buffer. + public mutating func read(_ count: ByteCount) async throws -> ByteBuffer { + let byteCount = Int(count.bytes) + guard byteCount > 0 else { return ByteBuffer() } + + if let bytes = self.buffer.readSlice(length: byteCount) { + return bytes + } else { + // Not enough bytes: read enough for the caller and to fill the buffer back to capacity. + var buffer = self.buffer + self.buffer = ByteBuffer() + + let bytesFromChunk = byteCount &- buffer.readableBytes + let bytesToRead = bytesFromChunk + self.capacity + + let chunk = try await self.readFromFile(bytesToRead) + self.buffer.writeImmutableBuffer(chunk) + + if let readBytes = self.buffer.readSlice(length: bytesFromChunk) { + buffer.writeImmutableBuffer(readBytes) + } + + return buffer + } + } + + /// Reads from the current position in the file until `predicate` returns `false` and returns + /// the read bytes. + /// + /// - Parameters: + /// - predicate: A predicate which evaluates to `true` for all bytes returned. + /// - Returns: The bytes read from the file. + public mutating func read( + while predicate: (UInt8) -> Bool + ) async throws -> ByteBuffer { + // Check if the required bytes are in the buffer already. + let view = self.buffer.readableBytesView + + if let index = view.firstIndex(where: { !predicate($0) }) { + // Got an index; slice off the front of the buffer. + let prefix = view[.. self.buffer.readableBytes { + self.offset += Int64(count &- self.buffer.readableBytes) + self.buffer.clear() + } else { + self.buffer.moveReaderIndex(forwardBy: count) + } + } + + /// Reads and discards bytes until `predicate` returns `false.` + /// + /// - Parameters: + /// - predicate: A predicate which evaluates to `true` for all dropped bytes. + public mutating func drop(while predicate: (UInt8) -> Bool) async throws { + let view = self.buffer.readableBytesView + + if let index = view.firstIndex(where: { !predicate($0) }) { + let slice = view[.. BufferedReader { + return BufferedReader(wrapping: self, initialOffset: 0, capacity: Int(capacity.bytes)) + } +} diff --git a/Sources/NIOFileSystem/BufferedWriter.swift b/Sources/NIOFileSystem/BufferedWriter.swift new file mode 100644 index 0000000000..cc1a702412 --- /dev/null +++ b/Sources/NIOFileSystem/BufferedWriter.swift @@ -0,0 +1,187 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/// A writer which buffers bytes in memory before writing them to the file system. +/// +/// You can create a ``BufferedWriter`` by calling +/// ``WritableFileHandleProtocol/bufferedWriter(startingAtAbsoluteOffset:capacity:)`` on +/// ``WritableFileHandleProtocol`` and write bytes to it with one of the following methods: +/// - ``BufferedWriter/write(contentsOf:)-8dhyg`` +/// - ``BufferedWriter/write(contentsOf:)-2i7d9`` +/// - ``BufferedWriter/write(contentsOf:)-8ukvd` +/// +/// If a call to one of the write functions reaches the buffers ``BufferedWriter/capacity`` the +/// buffer automatically writes its contents to the file. +/// +/// - Remark: The writer reclaims the buffer's memory when it grows to more than twice the +/// configured size. +/// +/// To write the bytes in the buffer to the file system before the buffer is full +/// use ``BufferedWriter/flush()``. +/// +/// - Important: You should you call ``BufferedWriter/flush()`` when you have finished appending +/// to write any remaining data to the file system. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct BufferedWriter { + private let handle: Handle + /// Offset for the next write. + private var offset: Int64 + /// A buffer of bytes to write. + private var buffer: [UInt8] = [] + + /// The maximum number of bytes to buffer before the buffer is automatically flushed. + public let capacity: Int + + /// The number of bytes in the buffer. + /// + /// You can flush the buffer manually by calling ``flush()``. + public var bufferedBytes: Int { + return self.buffer.count + } + + /// The capacity of the buffer. + @_spi(Testing) + public var bufferCapacity: Int { + return self.buffer.capacity + } + + internal init(wrapping writableHandle: Handle, initialOffset: Int64, capacity: Int) { + precondition( + initialOffset >= 0, + "initialOffset (\(initialOffset)) must be greater than or equal to zero" + ) + precondition(capacity > 0, "capacity (\(capacity)) must be greater than zero") + self.handle = writableHandle + self.offset = initialOffset + self.capacity = capacity + } + + /// Write the contents of the collection of bytes to the buffer. + /// + /// If the number of bytes in the buffer exceeds the size of the buffer then they're + /// automatically written to the file system. + /// + /// - Remark: The writer reclaims the buffer's memory when it grows to more than twice the + /// configured size. + /// + /// To manually flush bytes use ``flush()``. + /// + /// - Parameter bytes: The bytes to write to the buffer. + /// - Returns: The number of bytes written into the buffered writer. + @discardableResult + public mutating func write(contentsOf bytes: some Sequence) async throws -> Int64 { + let bufferSize = Int64(self.buffer.count) + self.buffer.append(contentsOf: bytes) + let bytesWritten = Int64(self.buffer.count) &- bufferSize + + if self.buffer.count >= self.capacity { + try await self.flush() + } + + return bytesWritten + } + + /// Write the contents of the `AsyncSequence` of byte chunks to the buffer. + /// + /// If appending a chunk to the buffer causes it to exceed the capacity of the buffer then the + /// contents of the buffer are automatically written to the file system. + /// + /// - Remark: The writer reclaims the buffer's memory when it grows to more than twice the + /// configured size. + /// + /// To manually flush bytes use ``flush()``. + /// + /// - Parameter bytes: The `AsyncSequence` of byte chunks to write to the buffer. + /// - Returns: The number of bytes written into the buffered writer. + @discardableResult + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public mutating func write( + contentsOf chunks: Chunks + ) async throws -> Int64 where Chunks.Element: Sequence { + var bytesWritten: Int64 = 0 + do { + for try await chunk in chunks { + bytesWritten += try await self.write(contentsOf: chunk) + } + } catch let error as FileSystemError { + // From call to 'write'. + throw error + } catch let error { + // From iterating the async sequence. + throw FileSystemError( + code: .unknown, + message: "AsyncSequence of bytes threw error while writing to the buffered writer.", + cause: error, + location: .here() + ) + } + return bytesWritten + } + + /// Write the contents of the `AsyncSequence` of bytes the buffer. + /// + /// If appending a byte to the buffer causes it to exceed the capacity of the buffer then the + /// contents of the buffer are automatically written to the file system. + /// + /// - Remark: The writer reclaims the buffer's memory when it grows to more than twice the + /// configured size. + /// + /// To manually flush bytes use ``flush()``. + /// + /// - Parameter bytes: The `AsyncSequence` of bytes to write to the buffer. + @discardableResult + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public mutating func write( + contentsOf bytes: Bytes + ) async throws -> Int64 where Bytes.Element == UInt8 { + try await self.write(contentsOf: bytes.map { CollectionOfOne($0) }) + } + + /// Flush any buffered bytes to the file system. + /// + /// - Important: You should you call ``flush()`` when you have finished writing to ensure the + /// buffered writer writes any remaining data to the file system. + public mutating func flush() async throws { + if self.buffer.isEmpty { return } + + try await self.handle.write(contentsOf: self.buffer, toAbsoluteOffset: self.offset) + self.offset += Int64(self.buffer.count) + + // The buffer may grow beyond the specified buffer size. Keep the capacity if it's less than + // double the intended size, otherwise reclaim the memory. + let keepCapacity = self.buffer.capacity <= (self.capacity * 2) + self.buffer.removeAll(keepingCapacity: keepCapacity) + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension WritableFileHandleProtocol { + /// Creates a new ``BufferedWriter`` for this file handle. + /// + /// - Parameters: + /// - initialOffset: The offset to begin writing at, defaults to zero. + /// - capacity: The capacity of the buffer in bytes, as a ``ByteCount``. The writer writes the contents of its + /// buffer to the file system when it exceeds this capacity. Defaults to 512 KiB. + /// - Returns: A ``BufferedWriter``. + public func bufferedWriter( + startingAtAbsoluteOffset initialOffset: Int64 = 0, + capacity: ByteCount = .kibibytes(512) + ) -> BufferedWriter { + return BufferedWriter( + wrapping: self, + initialOffset: initialOffset, + capacity: Int(capacity.bytes) + ) + } +} diff --git a/Sources/NIOFileSystem/ByteBuffer+FileSystem.swift b/Sources/NIOFileSystem/ByteBuffer+FileSystem.swift new file mode 100644 index 0000000000..6c5be612e4 --- /dev/null +++ b/Sources/NIOFileSystem/ByteBuffer+FileSystem.swift @@ -0,0 +1,97 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension ByteBuffer { + /// Reads the contents of the file at the path. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, in bytes, as a ``ByteCount``. + /// - fileSystem: The ``FileSystemProtocol`` instance to use to read the file. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount, + fileSystem: some FileSystemProtocol + ) async throws { + self = try await fileSystem.withFileHandle(forReadingAt: path) { handle in + try await handle.readToEnd( + fromAbsoluteOffset: 0, + maximumSizeAllowed: maximumSizeAllowed + ) + } + } + + /// Reads the contents of the file at the path using ``FileSystem``. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, as a ``ByteCount``. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount + ) async throws { + self = try await Self( + contentsOf: path, + maximumSizeAllowed: maximumSizeAllowed, + fileSystem: .shared + ) + } + + /// Writes the readable bytes of the ``ByteBuffer`` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - fileSystem: The ``FileSystemProtocol`` instance to use. + /// - Returns: The number of bytes written to the file. + @discardableResult + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false), + fileSystem: some FileSystemProtocol + ) async throws -> Int64 { + try await self.readableBytesView.write( + toFileAt: path, + absoluteOffset: offset, + options: options, + fileSystem: fileSystem + ) + } + + /// Writes the readable bytes of the ``ByteBuffer`` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - Returns: The number of bytes written to the file. + @discardableResult + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false) + ) async throws -> Int64 { + try await self.write( + toFileAt: path, + absoluteOffset: offset, + options: options, + fileSystem: .shared + ) + } +} diff --git a/Sources/NIOFileSystem/ByteCount.swift b/Sources/NIOFileSystem/ByteCount.swift new file mode 100644 index 0000000000..5ba29a0a47 --- /dev/null +++ b/Sources/NIOFileSystem/ByteCount.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/// Represents the number of bytes. +public struct ByteCount: Hashable, Sendable { + /// The number of bytes + public var bytes: Int64 + + /// Returns a ``ByteCount`` with a given number of bytes + /// - Parameter count: The number of bytes + public static func bytes(_ count: Int64) -> ByteCount { + return ByteCount(bytes: count) + } + + /// Returns a ``ByteCount`` with a given number of kilobytes + /// + /// One kilobyte is 1000 bytes. + /// + /// - Parameter count: The number of kilobytes + public static func kilobytes(_ count: Int64) -> ByteCount { + return ByteCount(bytes: 1000 * count) + } + + /// Returns a ``ByteCount`` with a given number of megabytes + /// + /// One megabyte is 1,000,000 bytes. + /// + /// - Parameter count: The number of megabytes + public static func megabytes(_ count: Int64) -> ByteCount { + return ByteCount(bytes: 1000 * 1000 * count) + } + + /// Returns a ``ByteCount`` with a given number of gigabytes + /// + /// One gigabyte is 1,000,000,000 bytes. + /// + /// - Parameter count: The number of gigabytes + public static func gigabytes(_ count: Int64) -> ByteCount { + return ByteCount(bytes: 1000 * 1000 * 1000 * count) + } + + /// Returns a ``ByteCount`` with a given number of kibibytes + /// + /// One kibibyte is 1024 bytes. + /// + /// - Parameter count: The number of kibibytes + public static func kibibytes(_ count: Int64) -> ByteCount { + return ByteCount(bytes: 1024 * count) + } + + /// Returns a ``ByteCount`` with a given number of mebibytes + /// + /// One mebibyte is 10,485,760 bytes. + /// + /// - Parameter count: The number of mebibytes + public static func mebibytes(_ count: Int64) -> ByteCount { + return ByteCount(bytes: 1024 * 1024 * count) + } + + /// Returns a ``ByteCount`` with a given number of gibibytes + /// + /// One gibibyte is 10,737,418,240 bytes. + /// + /// - Parameter count: The number of gibibytes + public static func gibibytes(_ count: Int64) -> ByteCount { + return ByteCount(bytes: 1024 * 1024 * 1024 * count) + } +} diff --git a/Sources/NIOFileSystem/Convenience.swift b/Sources/NIOFileSystem/Convenience.swift new file mode 100644 index 0000000000..54f0a56441 --- /dev/null +++ b/Sources/NIOFileSystem/Convenience.swift @@ -0,0 +1,202 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +extension String { + /// Writes the UTF8 encoded `String` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the `String` to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - fileSystem: The ``FileSystemProtocol`` instance to use. + /// - Returns: The number of bytes written to the file. + @discardableResult + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false), + fileSystem: some FileSystemProtocol + ) async throws -> Int64 { + try await self.utf8.write( + toFileAt: path, + absoluteOffset: offset, + options: options, + fileSystem: fileSystem + ) + } + + /// Writes the UTF8 encoded `String` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the `String` to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - Returns: The number of bytes written to the file. + @discardableResult + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false) + ) async throws -> Int64 { + try await self.write( + toFileAt: path, + absoluteOffset: offset, + options: options, + fileSystem: .shared + ) + } +} + +extension Sequence where Self: Sendable { + /// Writes the contents of the `Sequence` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - fileSystem: The ``FileSystemProtocol`` instance to use. + /// - Returns: The number of bytes written to the file. + @discardableResult + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false), + fileSystem: some FileSystemProtocol + ) async throws -> Int64 { + try await fileSystem.withFileHandle(forWritingAt: path, options: options) { handle in + try await handle.write(contentsOf: self, toAbsoluteOffset: offset) + } + } + + /// Writes the contents of the `Sequence` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - Returns: The number of bytes written to the file. + @discardableResult + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false) + ) async throws -> Int64 { + try await self.write( + toFileAt: path, + absoluteOffset: offset, + options: options, + fileSystem: .shared + ) + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension AsyncSequence where Self.Element: Sequence, Self: Sendable { + /// Writes the contents of the `AsyncSequence` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - fileSystem: The ``FileSystemProtocol`` instance to use. + /// - Returns: The number of bytes written to the file. + @discardableResult + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false), + fileSystem: some FileSystemProtocol + ) async throws -> Int64 { + try await fileSystem.withFileHandle(forWritingAt: path, options: options) { handle in + var writer = handle.bufferedWriter(startingAtAbsoluteOffset: offset) + let bytesWritten = try await writer.write(contentsOf: self) + try await writer.flush() + return bytesWritten + } + } + + /// Writes the contents of the `AsyncSequence` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - Returns: The number of bytes written to the file. + @discardableResult + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false) + ) async throws -> Int64 { + try await self.write( + toFileAt: path, + absoluteOffset: offset, + options: options, + fileSystem: .shared + ) + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension AsyncSequence where Self.Element == UInt8, Self: Sendable { + /// Writes the contents of the `AsyncSequence` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - fileSystem: The ``FileSystemProtocol`` instance to use. + /// - Returns: The number of bytes written to the file. + @discardableResult + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false), + fileSystem: some FileSystemProtocol + ) async throws -> Int64 { + try await fileSystem.withFileHandle(forWritingAt: path, options: options) { handle in + var writer = handle.bufferedWriter(startingAtAbsoluteOffset: offset) + let bytesWritten = try await writer.write(contentsOf: self) + try await writer.flush() + return bytesWritten + } + } + + /// Writes the contents of the `AsyncSequence` to a file. + /// + /// - Parameters: + /// - path: The path of the file to write the contents of the sequence to. + /// - offset: The offset into the file to write to, defaults to zero. + /// - options: Options for opening the file, defaults to creating a new file. + /// - Returns: The number of bytes written to the file. + @discardableResult + public func write( + toFileAt path: FilePath, + absoluteOffset offset: Int64 = 0, + options: OpenOptions.Write = .newFile(replaceExisting: false) + ) async throws -> Int64 { + try await self.write( + toFileAt: path, + absoluteOffset: offset, + options: options, + fileSystem: .shared + ) + } +} diff --git a/Sources/NIOFileSystem/DirectoryEntries.swift b/Sources/NIOFileSystem/DirectoryEntries.swift new file mode 100644 index 0000000000..33c9275a02 --- /dev/null +++ b/Sources/NIOFileSystem/DirectoryEntries.swift @@ -0,0 +1,654 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import CNIODarwin +import CNIOLinux +import NIOConcurrencyHelpers +@preconcurrency import SystemPackage + +/// An `AsyncSequence` of entries in a directory. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct DirectoryEntries: AsyncSequence { + public typealias AsyncIterator = DirectoryIterator + public typealias Element = DirectoryEntry + + /// The underlying sequence. + private let batchedSequence: Batched + + /// Creates a new ``DirectoryEntries`` sequence. + internal init(handle: SystemFileHandle, recursive: Bool) { + self.batchedSequence = Batched(handle: handle, recursive: recursive) + } + + /// Creates a ``DirectoryEntries`` sequence by wrapping an `AsyncSequence` of _batches_ of + /// directory entries. + public init(wrapping sequence: S) where S.Element == Batched.Element { + self.batchedSequence = Batched(wrapping: sequence) + } + + public func makeAsyncIterator() -> DirectoryIterator { + return DirectoryIterator(iterator: self.batchedSequence.makeAsyncIterator()) + } + + /// Returns a sequence of directory entry batches. + /// + /// The batched sequence has its element type as `Array` rather + /// than `DirectoryEntry`. This can enable better performance by reducing the number of + /// executor hops. + public func batched() -> Batched { + return self.batchedSequence + } + + /// An `AsyncIteratorProtocol` of `DirectoryEntry`. + public struct DirectoryIterator: AsyncIteratorProtocol { + /// The batched iterator to consume from. + private var iterator: Batched.AsyncIterator + /// A slice of the current batch being iterated. + private var currentBatch: ArraySlice + + init(iterator: Batched.AsyncIterator) { + self.iterator = iterator + self.currentBatch = [] + } + + public mutating func next() async throws -> DirectoryEntry? { + if self.currentBatch.isEmpty { + let batch = try await self.iterator.next() + self.currentBatch = (batch ?? [])[...] + } + + return self.currentBatch.popFirst() + } + } +} + +@available(*, unavailable) +extension DirectoryEntries.AsyncIterator: Sendable {} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension DirectoryEntries { + /// An `AsyncSequence` of batches of directory entries. + /// + /// The ``Batched`` sequence uses `Array` as its element type rather + /// than `DirectoryEntry`. This can enable better performance by reducing the number of + /// executor hops at the cost of ease-of-use. + public struct Batched: AsyncSequence { + public typealias AsyncIterator = BatchedIterator + public typealias Element = [DirectoryEntry] + + private let stream: BufferedOrAnyStream<[DirectoryEntry]> + + /// Creates a ``DirectoryEntries/Batched`` sequence by wrapping an `AsyncSequence` + /// of directory entry batches. + public init(wrapping sequence: S) where S.Element == Element { + self.stream = BufferedOrAnyStream(wrapping: sequence) + } + + fileprivate init(handle: SystemFileHandle, recursive: Bool) { + // Expanding the batches yields watermarks of 256 and 512 directory entries. + let stream = BufferedStream.makeBatchedDirectoryEntryStream( + handle: handle, + recursive: recursive, + entriesPerBatch: 64, + lowWatermark: 4, + highWatermark: 8 + ) + + self.stream = BufferedOrAnyStream(wrapping: stream) + } + + public func makeAsyncIterator() -> BatchedIterator { + return BatchedIterator(wrapping: self.stream.makeAsyncIterator()) + } + + /// An `AsyncIteratorProtocol` of `Array`. + public struct BatchedIterator: AsyncIteratorProtocol { + private var iterator: BufferedOrAnyStream<[DirectoryEntry]>.AsyncIterator + + init(wrapping iterator: BufferedOrAnyStream<[DirectoryEntry]>.AsyncIterator) { + self.iterator = iterator + } + + public mutating func next() async throws -> [DirectoryEntry]? { + try await self.iterator.next() + } + } + } +} + +@available(*, unavailable) +extension DirectoryEntries.Batched.AsyncIterator: Sendable {} + +// MARK: - Internal + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream where Element == [DirectoryEntry] { + fileprivate static func makeBatchedDirectoryEntryStream( + handle: SystemFileHandle, + recursive: Bool, + entriesPerBatch: Int, + lowWatermark: Int, + highWatermark: Int + ) -> BufferedStream<[DirectoryEntry]> { + let state = DirectoryEnumerator(handle: handle, recursive: recursive) + let protectedState = NIOLockedValueBox(state) + + var (stream, source) = BufferedStream.makeStream( + of: [DirectoryEntry].self, + backPressureStrategy: .watermark(low: lowWatermark, high: highWatermark) + ) + + source.onTermination = { + guard let executor = protectedState.withLockedValue({ $0.executorForClosing() }) else { + return + } + + executor.execute { + protectedState.withLockedValue { state in + state.closeIfNecessary() + } + } onCompletion: { _ in + // Ignore the result. + } + } + + let producer = DirectoryEntryProducer( + state: protectedState, + source: source, + entriesPerBatch: entriesPerBatch + ) + // Start producing immediately. + producer.produceMore() + + return stream + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +private struct DirectoryEntryProducer { + let state: NIOLockedValueBox + let source: BufferedStream<[DirectoryEntry]>.Source + let entriesPerBatch: Int + + /// The 'entry point' for producing elements. + /// + /// Calling this function will start producing directory entries asynchronously by dispatching + /// work to the IO executor and feeding the result back to the stream source. On yielding to the + /// source it will either produce more or be scheduled to produce more. Stopping production + /// is signalled via the stream's 'onTermination' handler. + func produceMore() { + let executor = self.state.withLockedValue { state in + state.produceMore() + } + + // No executor means we're done. + guard let executor = executor else { return } + + executor.execute { + try self.nextBatch() + } onCompletion: { result in + self.onNextBatchResult(result) + } + } + + private func nextBatch() throws -> [DirectoryEntry] { + return try self.state.withLockedValue { state in + try state.next(self.entriesPerBatch) + } + } + + private func onNextBatchResult(_ result: Result<[DirectoryEntry], Error>) { + switch result { + case let .success(entries): + self.onNextBatch(entries) + case let .failure(error): + // Failed to read more entries: close and notify the stream so consumers receive the + // error. + self.close() + self.source.finish(throwing: error) + } + } + + private func onNextBatch(_ entries: [DirectoryEntry]) { + // No entries were read: this must be the end (as the batch size must be greater than zero). + if entries.isEmpty { + self.source.finish(throwing: nil) + return + } + + // Reading short means reading EOF. The enumerator closes itself in that case. + let readEOF = entries.count < self.entriesPerBatch + + // Entries were produced: yield them and maybe produce more. + do { + let writeResult = try self.source.write(contentsOf: CollectionOfOne(entries)) + // Exit early if EOF was read; no use in trying to produce more. + if readEOF { + self.source.finish(throwing: nil) + return + } + + switch writeResult { + case .produceMore: + self.produceMore() + case let .enqueueCallback(token): + self.source.enqueueCallback(callbackToken: token) { + switch $0 { + case .success: + self.produceMore() + case .failure: + self.close() + } + } + } + } catch { + // Failure to write means the source is already done, that's okay we just need to + // update our state and stop producing. + self.close() + } + } + + private func close() { + guard let executor = self.state.withLockedValue({ $0.executorForClosing() }) else { + return + } + + executor.execute { + self.state.withLockedValue { state in + state.closeIfNecessary() + } + } onCompletion: { _ in + // Ignore. + } + } +} + +/// Enumerates a directory in batches. +/// +/// Note that this is not a `Sequence` because we allow for errors to be thrown on `next()`. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +private struct DirectoryEnumerator: Sendable { + private enum State: @unchecked Sendable { + case modifying + case idle(SystemFileHandle.SendableView, recursive: Bool) + case open(IOExecutor, Source, [DirectoryEntry]) + case done + } + + /// The source of directory entries. + private enum Source { + case readdir(CInterop.DirPointer) + case fts(CInterop.FTSPointer) + } + + /// The current state of enumeration. + private var state: State + + /// The path to the directory being enumerated. + private let path: FilePath + + /// Information about an entry returned by FTS. See 'fts(3)'. + private enum FTSInfo: Hashable, Sendable { + case directoryPreOrder + case directoryCausingCycle + case ftsDefault + case directoryUnreadable + case dotFile + case directoryPostOrder + case error + case regularFile + case noStatInfoAvailable + case noStatInfoRequested + case symbolicLink + case symbolicLinkToNonExistentTarget + + init?(rawValue: UInt16) { + switch Int32(rawValue) { + case FTS_D: + self = .directoryPreOrder + case FTS_DC: + self = .directoryCausingCycle + case FTS_DEFAULT: + self = .ftsDefault + case FTS_DNR: + self = .directoryUnreadable + case FTS_DOT: + self = .dotFile + case FTS_DP: + self = .directoryPostOrder + case FTS_ERR: + self = .error + case FTS_F: + self = .regularFile + case FTS_NS: + self = .noStatInfoAvailable + case FTS_NSOK: + self = .noStatInfoRequested + case FTS_SL: + self = .symbolicLink + case FTS_SLNONE: + self = .symbolicLinkToNonExistentTarget + default: + return nil + } + } + } + + internal init(handle: SystemFileHandle, recursive: Bool) { + self.state = .idle(handle.sendableView, recursive: recursive) + self.path = handle.path + } + + internal func produceMore() -> IOExecutor? { + switch self.state { + case let .idle(handle, _): + return handle.executor + case let .open(executor, _, _): + return executor + case .done: + return nil + case .modifying: + fatalError() + } + } + + internal func executorForClosing() -> IOExecutor? { + switch self.state { + case let .open(executor, _, _): + return executor + case .idle, .done: + // Don't need to close in the idle state: we don't own the handle. + return nil + case .modifying: + fatalError() + } + } + + /// Returns the next batch of directory entries. + internal mutating func next(_ count: Int) throws -> [DirectoryEntry] { + while true { + switch self.process(count) { + case let .yield(result): + return try result.get() + case .continue: + () + } + } + } + + /// Closes the descriptor, if necessary. + internal mutating func closeIfNecessary() { + switch self.state { + case .idle: + // We don't own the handle so don't close it. + self.state = .done + + case let .open(_, mode, _): + self.state = .done + switch mode { + case .readdir(let dir): + _ = Libc.closedir(dir) + case .fts(let fts): + _ = Libc.ftsClose(fts) + } + + case .done: + () + + case .modifying: + fatalError() + } + } + + private enum ProcessResult { + case yield(Result<[DirectoryEntry], FileSystemError>) + case `continue` + } + + private mutating func makeReaddirSource( + _ handle: SystemFileHandle.SendableView + ) -> Result { + return handle._duplicate().mapError { dupError in + FileSystemError( + message: "Unable to open directory stream for '\(handle.path)'.", + wrapping: dupError + ) + }.flatMap { descriptor in + // We own the descriptor and cede ownership if 'opendir' succeeds; if it doesn't we need + // to close it. + descriptor.opendir().mapError { errno in + // Close the descriptor on error. + try? descriptor.close() + return FileSystemError.fdopendir(errno: errno, path: handle.path, location: .here()) + } + }.map { + .readdir($0) + } + } + + private mutating func makeFTSSource( + _ handle: SystemFileHandle.SendableView + ) -> Result { + return Libc.ftsOpen(handle.path, options: [.noChangeDir, .physical]).mapError { errno in + FileSystemError.open("fts_open", error: errno, path: handle.path, location: .here()) + }.map { + .fts($0) + } + } + + private mutating func processOpenState( + executor: IOExecutor, + dir: CInterop.DirPointer, + entries: inout [DirectoryEntry], + count: Int + ) -> (State, ProcessResult) { + entries.removeAll(keepingCapacity: true) + entries.reserveCapacity(count) + + while entries.count < count { + switch Libc.readdir(dir) { + case let .success(.some(entry)): + // Skip "." and ".." (and empty paths) + if self.isThisOrParentDirectory(entry.pointee) { + continue + } + + let fileType = FileType(direntType: entry.pointee.d_type) + let name: FilePath.Component + #if canImport(Darwin) + // Safe to force unwrap: may be nil if empty, a root, or more than one component. + // Empty is checked for above, root can't exist within a directory, and directory + // items must be a single path component. + name = FilePath.Component(platformString: CNIODarwin_dirent_dname(entry))! + #else + name = FilePath.Component(platformString: CNIOLinux_dirent_dname(entry))! + #endif + + let fullPath = self.path.appending(name) + // '!' is okay here: the init returns nil if there is an empty path which we know + // isn't the case as 'self.path' is non-empty. + entries.append(DirectoryEntry(path: fullPath, type: fileType)!) + + case .success(.none): + // Nothing we can do on failure so ignore the result. + _ = Libc.closedir(dir) + return (.done, .yield(.success(entries))) + + case let .failure(errno): + // Nothing we can do on failure so ignore the result. + _ = Libc.closedir(dir) + let error = FileSystemError.readdir( + errno: errno, + path: self.path, + location: .here() + ) + return (.done, .yield(.failure(error))) + } + } + + // We must have hit our 'count' limit. + return (.open(executor, .readdir(dir), entries), .yield(.success(entries))) + } + + private mutating func processOpenState( + executor: IOExecutor, + fts: CInterop.FTSPointer, + entries: inout [DirectoryEntry], + count: Int + ) -> (State, ProcessResult) { + entries.removeAll(keepingCapacity: true) + entries.reserveCapacity(count) + + while entries.count < count { + switch Libc.ftsRead(fts) { + case .success(.some(let entry)): + let info = FTSInfo(rawValue: entry.pointee.fts_info) + switch info { + case .directoryPreOrder: + let entry = DirectoryEntry(path: entry.path, type: .directory)! + entries.append(entry) + + case .directoryPostOrder: + () // Don't visit directories twice. + + case .regularFile: + let entry = DirectoryEntry(path: entry.path, type: .regular)! + entries.append(entry) + + case .symbolicLink, .symbolicLinkToNonExistentTarget: + let entry = DirectoryEntry(path: entry.path, type: .symlink)! + entries.append(entry) + + case .ftsDefault: + // File type is unknown. + let entry = DirectoryEntry(path: entry.path, type: .unknown)! + entries.append(entry) + + case .error: + let errno = Errno(rawValue: entry.pointee.fts_errno) + let error = FileSystemError( + code: .unknown, + message: "Can't read file system tree.", + cause: FileSystemError.SystemCallError(systemCall: "fts_read", errno: errno), + location: .here() + ) + _ = Libc.ftsClose(fts) + return (.done, .yield(.failure(error))) + + case .directoryCausingCycle: + () // Cycle found, ignore it and continue. + case .directoryUnreadable: + () // Can't read directory, ignore it and continue iterating. + case .dotFile: + () // Ignore "." and ".." + case .noStatInfoAvailable: + () // No stat info available so we can't list the entry, ignore it. + case .noStatInfoRequested: + () // Shouldn't happen. + + case nil: + () // Unknown, ignore. + } + + case .success(.none): + // No entries left to iterate. + _ = Libc.ftsClose(fts) + return (.done, .yield(.success(entries))) + + case .failure(let errno): + // Nothing we can do on failure so ignore the result. + _ = Libc.ftsClose(fts) + let error = FileSystemError.ftsRead( + errno: errno, + path: self.path, + location: .here() + ) + return (.done, .yield(.failure(error))) + } + } + + // We must have hit our 'count' limit. + return (.open(executor, .fts(fts), entries), .yield(.success(entries))) + } + + private mutating func process(_ count: Int) -> ProcessResult { + switch self.state { + case let .idle(handle, recursive): + let result: Result + + if recursive { + result = self.makeFTSSource(handle) + } else { + result = self.makeReaddirSource(handle) + } + + switch result { + case let .success(source): + self.state = .open(handle.executor, source, []) + return .continue + + case let .failure(error): + self.state = .done + return .yield(.failure(error)) + } + + case .open(let executor, let mode, var entries): + self.state = .modifying + + switch mode { + case .readdir(let dir): + let (state, result) = self.processOpenState( + executor: executor, + dir: dir, + entries: &entries, + count: count + ) + self.state = state + return result + + case .fts(let fts): + let (state, result) = self.processOpenState( + executor: executor, + fts: fts, + entries: &entries, + count: count + ) + self.state = state + return result + } + + case .done: + return .yield(.success([])) + + case .modifying: + fatalError() + } + } + + private func isThisOrParentDirectory(_ entry: CInterop.DirEnt) -> Bool { + let dot = CChar(bitPattern: UInt8(ascii: ".")) + switch (entry.d_name.0, entry.d_name.1, entry.d_name.2) { + case (0, _, _), (dot, 0, _), (dot, dot, 0): + return true + default: + return false + } + } +} + +extension UnsafeMutablePointer { + fileprivate var path: FilePath { + return FilePath(platformString: self.pointee.fts_path) + } +} diff --git a/Sources/NIOFileSystem/DirectoryEntry.swift b/Sources/NIOFileSystem/DirectoryEntry.swift new file mode 100644 index 0000000000..c6945eac73 --- /dev/null +++ b/Sources/NIOFileSystem/DirectoryEntry.swift @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import SystemPackage + +/// Information about an item within a directory. +public struct DirectoryEntry: Sendable, Hashable, Equatable { + /// The path of the directory entry. + /// + /// - Precondition: The path must have at least one component. + public let path: FilePath + + /// The name of the entry; the final component of the ``path``. + /// + /// If `path` is "/Users/tim/path-to-4T.key" then `name` will be "path-to-4T.key". + public var name: FilePath.Component { + self.path.lastComponent! + } + + /// The type of entry. + public var type: FileType + + /// Creates a directory entry; returns `nil` if `path` has no components. + /// + /// - Parameters: + /// - path: The path of the directory entry which must contain at least one component. + /// - type: The type of entry. + public init?(path: FilePath, type: FileType) { + if path.components.isEmpty { + return nil + } + + self.path = path + self.type = type + } +} diff --git a/Sources/NIOFileSystem/Docs.docc/Extensions/DirectoryFileHandleProtocol.md b/Sources/NIOFileSystem/Docs.docc/Extensions/DirectoryFileHandleProtocol.md new file mode 100644 index 0000000000..7ea77068ea --- /dev/null +++ b/Sources/NIOFileSystem/Docs.docc/Extensions/DirectoryFileHandleProtocol.md @@ -0,0 +1,27 @@ +# ``NIOFileSystem/DirectoryFileHandleProtocol`` + +## Topics + +### Iterating a directory + +- ``listContents()`` + +### Opening files with managed lifecycles + +Open files and directories at paths relative to the directory handle. These methods manage +the lifecycle of the handles by closing them when the `execute` closure returns. + +- ``withFileHandle(forReadingAt:options:execute:)`` +- ``withFileHandle(forWritingAt:options:execute:)`` +- ``withFileHandle(forReadingAndWritingAt:options:execute:)`` +- ``withDirectoryHandle(atPath:options:execute:)`` + +### Opening files + +Open files and directories at paths relative to the directory handle. These methods return +the handle to the caller who is responsible for closing it. + +- ``openFile(forReadingAt:options:)`` +- ``openFile(forWritingAt:options:)`` +- ``openFile(forReadingAndWritingAt:options:)`` +- ``openDirectory(atPath:options:)`` diff --git a/Sources/NIOFileSystem/Docs.docc/Extensions/FileHandleProtocol.md b/Sources/NIOFileSystem/Docs.docc/Extensions/FileHandleProtocol.md new file mode 100644 index 0000000000..bc38349115 --- /dev/null +++ b/Sources/NIOFileSystem/Docs.docc/Extensions/FileHandleProtocol.md @@ -0,0 +1,27 @@ +# ``NIOFileSystem/FileHandleProtocol`` + +## Topics + +### File Information + +- ``info()`` + +### Permissions + +- ``replacePermissions(_:)`` +- ``addPermissions(_:)`` +- ``removePermissions(_:)`` + +### Extended Attributes + +- ``attributeNames()`` +- ``valueForAttribute(_:)`` +- ``updateValueForAttribute(_:attribute:)`` +- ``removeValueForAttribute(_:)`` + +### Descriptor Management + +- ``synchronize()`` +- ``withUnsafeDescriptor(_:)`` +- ``detachUnsafeFileDescriptor()`` +- ``close()`` diff --git a/Sources/NIOFileSystem/Docs.docc/Extensions/FileSystemProtocol.md b/Sources/NIOFileSystem/Docs.docc/Extensions/FileSystemProtocol.md new file mode 100644 index 0000000000..52440f36f9 --- /dev/null +++ b/Sources/NIOFileSystem/Docs.docc/Extensions/FileSystemProtocol.md @@ -0,0 +1,45 @@ +# ``NIOFileSystem/FileSystemProtocol`` + +## Topics + +### Opening files with managed lifecycles + +Files and directories can be opened and have their lifecycles managed by using the +following methods: + +- ``withFileHandle(forReadingAt:options:execute:)`` +- ``withFileHandle(forWritingAt:options:execute:)`` +- ``withFileHandle(forReadingAndWritingAt:options:execute:)`` +- ``withDirectoryHandle(atPath:options:execute:)`` + +### Opening files + +Files and directories can be opened using the following methods. The caller is responsible for +closing it to avoid leaking resources. + +- ``openFile(forReadingAt:)`` +- ``openFile(forWritingAt:options:)`` +- ``openFile(forReadingAndWritingAt:options:)`` +- ``openDirectory(atPath:options:)`` + +### File information + +- ``info(forFileAt:infoAboutSymbolicLink:)`` + +### Symbolic links + +- ``createSymbolicLink(at:withDestination:)`` +- ``destinationOfSymbolicLink(at:)`` + +### Managing files + +- ``copyItem(at:to:shouldProceedAfterError:shouldCopyFile:)`` +- ``removeItem(at:)`` +- ``moveItem(at:to:)`` +- ``replaceItem(at:withItemAt:)`` +- ``createDirectory(at:withIntermediateDirectories:permissions:)`` + +### System directories + +- ``currentWorkingDirectory`` +- ``temporaryDirectory`` diff --git a/Sources/NIOFileSystem/Docs.docc/Extensions/ReadableFileHandleProtocol.md b/Sources/NIOFileSystem/Docs.docc/Extensions/ReadableFileHandleProtocol.md new file mode 100644 index 0000000000..7ad9711f76 --- /dev/null +++ b/Sources/NIOFileSystem/Docs.docc/Extensions/ReadableFileHandleProtocol.md @@ -0,0 +1,19 @@ +# ``NIOFileSystem/ReadableFileHandleProtocol`` + +## Topics + +### Read the contents of a file + +- ``bufferedReader(startingAtAbsoluteOffset:capacity:)`` +- ``readChunks(chunkLength:)`` +- ``readChunks(in:chunkLength:)-8of2k`` +- ``readChunks(in:chunkLength:)-4l2zx`` +- ``readChunks(in:chunkLength:)-8xhv9`` +- ``readChunks(in:chunkLength:)-ecdr`` +- ``readChunks(in:chunkLength:)-ecdr`` +- ``readChunks(in:chunkLength:)-8ty3f`` +- ``readToEnd(fromAbsoluteOffset:maximumSizeAllowed:)`` + +### Read part of a file + +- ``readChunk(fromAbsoluteOffset:length:)`` diff --git a/Sources/NIOFileSystem/Docs.docc/Extensions/WritableFileHandleProtocol.md b/Sources/NIOFileSystem/Docs.docc/Extensions/WritableFileHandleProtocol.md new file mode 100644 index 0000000000..8d1b2630c2 --- /dev/null +++ b/Sources/NIOFileSystem/Docs.docc/Extensions/WritableFileHandleProtocol.md @@ -0,0 +1,11 @@ +# ``NIOFileSystem/WritableFileHandleProtocol`` + +## Topics + +### Write bytes to a file + +- ``write(contentsOf:toAbsoluteOffset:)`` +- ``bufferedWriter(startingAtAbsoluteOffset:capacity:)`` + +### Resize a file +- ``resize(to:)`` diff --git a/Sources/NIOFileSystem/Docs.docc/NIOFileSystem.md b/Sources/NIOFileSystem/Docs.docc/NIOFileSystem.md new file mode 100644 index 0000000000..f68428d2d1 --- /dev/null +++ b/Sources/NIOFileSystem/Docs.docc/NIOFileSystem.md @@ -0,0 +1,174 @@ +# ``NIOFileSystem`` + +A file system library for Swift. + +## Overview + +This module implements a file system library for Swift, providing ways to interact with and manage +files. It provides a concrete ``FileSystem`` for interacting with the local file system in addition +to a set of protocols for creating other file system implementations. + +``NIOFileSystem`` is cross-platform with the following caveats: +- _Platforms don't have feature parity or system-level API parity._ Where this is the case these + implementation details are documented. One example is copying files, on Apple platforms files are + cloned if possible. +- _Features may be disabled on some systems._ One example is extended attributes. +- _Some types have platform specific representations._ These include the following: + - File paths on Apple platforms and Linux (e.g. `"/Users/hal9000/"`) are different to paths on + Windows (`"C:\Users\hal9000"`). + - Information about files is different on different platforms. See ``FileInfo`` for further + details. + +## A Brief Tour + +The following sample code demonstrates a number of the APIs offered by this module: + +```swift +import NIOFileSystem + +// NIOFileSystem provides access to the local file system via the FileSystem +// type which is available as a global shared instance. +let fileSystem = FileSystem.shared + +// Files can be inspected by using 'info': +if let info = try await fileSystem.info(forFileAt: "/Users/hal9000/demise-of-dave.txt") { + print("demise-of-dave.txt has type '\(info.type)'") +} else { + print("demise-of-dave.txt doesn't exist") +} + +// Let's find out what's in that file. +do { + // Reading a whole file requires a limit. If the file is larger than the limit + // then an error is thrown. This avoids accidentally consuming too much memory + // if the file is larger than expected. + let plan = try await ByteBuffer( + contentsOf: "/Users/hal9000/demise-of-dave.txt", + maximumSizeAllowed: .mebibytes(1) + ) + print("Plan for Dave's demise:", String(decoding: plan, as: UTF8.self)) +} catch let error as FileSystemError where error.code == .notFound { + // All errors thrown by the module have type FileSystemError (or + // Swift.CancellationError). It looks like the file doesn't exist. Let's + // create it now. + // + // The code above for reading the file is shorthand for opening the file in + // read-only mode and then reading its contents. The FileSystemProtocol + // has a few different 'withFileHandle' methods for opening a file in different + // modes. Let's open a file for writing, creating it at the same time. + try await fileSystem.withFileHandle( + forWritingAt: "/Users/hal9000/demise-of-dave.txt", + options: .newFile(replaceExisting: false) + ) { file in + let plan = ByteBuffer(string: "TODO...") + try await file.write(contentsOf: plan.readableBytesView, toAbsoluteOffset: 0) + } +} + +// Directories can be opened like regular files but they cannot be read from or +// written to. However, their contents can be listed: +let path: FilePath? = try await fileSystem.withDirectoryHandle(atPath: "/Users/hal9000/Music") { directory in + for try await entry in directory.listContents() { + if entry.name.extension == "mp3", entry.name.stem.contains("daisy") { + // Found it! + return entry.path + } + } + // No luck. + return nil +} + +if let path = path { + print("Found file at '\(path)'") +} + +// The file system can also be used to perform the following operations on files +// and directories: +// - copy, +// - remove, +// - rename, and +// - replace. +// +// Here's an example of copying a directory: +try await fileSystem.copyItem(at: "/Users/hal9000/Music", to: "/Volumes/Tardis/Music") + +// Symbolic links can also be created (and read with 'destinationOfSymbolicLink(at:)'). +try await fileSystem.createSymbolicLink(at: "/Users/hal9000/Backup", withDestination: "/Volumes/Tardis") + +// Opening a symbolic link opens its destination so in most cases there's no +// need to read the destination of a symbolic link: +try await fileSystem.withDirectoryHandle(atPath: "/Users/hal9000/Backup") { directory in + // Beyond listing the contents of a directory, the directory handle provides a + // number of other functions, many of which are also available on regular file + // handles. + // + // This includes getting information about a file, such as its permissions, last access time, + // and last modification time: + let info = try await directory.info() + print("The directory has permissions '\(info.permissions)'") + + // Where supported, the extended attributes of a file can also be accessed, read, and modified: + for attribute in try await directory.attributeNames() { + let value = try await directory.valueForAttribute(attribute) + print("Extended attribute '\(attribute)' has value '\(value)'") + } + + // Once this closure returns the file system will close the directory handle freeing + // any resources required to access it such as file descriptors. Handles can also be opened + // with the 'openFile' and 'openDirectory' APIs but that places the onus you to close the + // handle at an appropriate time to avoid leaking resources. +} +``` + +In depth documentation can be found in the following sections. + +## Topics + +### Interacting with the Local File System + +- ``FileSystem`` +- ``FileHandle`` +- ``ReadFileHandle`` +- ``WriteFileHandle`` +- ``ReadWriteFileHandle`` +- ``DirectoryFileHandle`` +- ``withFileSystem(numberOfThreads:_:)`` + +### File and Directory Information + +- ``FileInfo`` +- ``FileType`` + +### Reading Files + +- ``FileChunks`` +- ``BufferedReader`` + +### Writing Files + +- ``BufferedWriter`` + +### Listing Directories + +- ``DirectoryEntry`` +- ``DirectoryEntries`` + +### Errors + +``FileSystemError`` is the only top-level error type thrown by the package (apart from Swift's +`CancellationError`). + +- ``FileSystemError`` +- ``FileSystemError/SystemCallError`` + +### Creating a File System + +Custom file system's can be created by implementing ``FileSystemProtocol`` which depends on a number +of other protocols. These include the following: + +- ``FileSystemProtocol`` +- ``FileHandleProtocol`` +- ``ReadableFileHandleProtocol`` +- ``WritableFileHandleProtocol`` +- ``ReadableAndWritableFileHandleProtocol`` +- ``DirectoryFileHandleProtocol`` diff --git a/Sources/NIOFileSystem/Exports.swift b/Sources/NIOFileSystem/Exports.swift new file mode 100644 index 0000000000..2b73a58783 --- /dev/null +++ b/Sources/NIOFileSystem/Exports.swift @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +// These types are used in our public API; expose them to make +// life easier for users. +@_exported import enum SystemPackage.CInterop +@_exported import struct SystemPackage.Errno +@_exported import struct SystemPackage.FileDescriptor +@_exported import struct SystemPackage.FilePath +@_exported import struct SystemPackage.FilePermissions diff --git a/Sources/NIOFileSystem/FileChunks.swift b/Sources/NIOFileSystem/FileChunks.swift new file mode 100644 index 0000000000..d8f4d9b94c --- /dev/null +++ b/Sources/NIOFileSystem/FileChunks.swift @@ -0,0 +1,337 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOConcurrencyHelpers +import NIOCore +@preconcurrency import SystemPackage + +/// An `AsyncSequence` of ordered chunks read from a file. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct FileChunks: AsyncSequence { + enum ChunkRange { + case entireFile + case partial(Range) + } + + public typealias Element = ByteBuffer + + /// The underlying buffered stream. + private let stream: BufferedOrAnyStream + + /// Create a ``FileChunks`` sequence backed by wrapping an `AsyncSequence`. + public init(wrapping sequence: S) where S.Element == ByteBuffer { + self.stream = BufferedOrAnyStream(wrapping: sequence) + } + + internal init( + handle: SystemFileHandle, + chunkLength: ByteCount, + range: Range + ) { + let chunkRange: ChunkRange + if range.lowerBound == 0, range.upperBound == .max { + chunkRange = .entireFile + } else { + chunkRange = .partial(range) + } + + // TODO: choose reasonable watermarks; this should likely be at least somewhat dependent + // on the chunk size. + let stream = BufferedStream.makeFileChunksStream( + of: ByteBuffer.self, + handle: handle, + chunkLength: chunkLength.bytes, + range: chunkRange, + lowWatermark: 4, + highWatermark: 8 + ) + + self.stream = BufferedOrAnyStream(wrapping: stream) + } + + public func makeAsyncIterator() -> FileChunkIterator { + return FileChunkIterator(wrapping: self.stream.makeAsyncIterator()) + } + + public struct FileChunkIterator: AsyncIteratorProtocol { + private var iterator: BufferedOrAnyStream.AsyncIterator + + fileprivate init(wrapping iterator: BufferedOrAnyStream.AsyncIterator) { + self.iterator = iterator + } + + public mutating func next() async throws -> ByteBuffer? { + try await self.iterator.next() + } + } +} + +@available(*, unavailable) +extension FileChunks.FileChunkIterator: Sendable {} + +// MARK: - Internal + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream where Element == ByteBuffer { + static func makeFileChunksStream( + of: Element.Type = Element.self, + handle: SystemFileHandle, + chunkLength: Int64, + range: FileChunks.ChunkRange, + lowWatermark: Int, + highWatermark: Int + ) -> BufferedStream { + let state: ProducerState + switch range { + case .entireFile: + state = ProducerState(handle: handle, range: nil) + case .partial(let partialRange): + state = ProducerState(handle: handle, range: partialRange) + } + let protectedState = NIOLockedValueBox(state) + + var (stream, source) = BufferedStream.makeStream( + of: ByteBuffer.self, + backPressureStrategy: .watermark(low: lowWatermark, high: highWatermark) + ) + + source.onTermination = { + protectedState.withLockedValue { state in + state.done() + } + } + + // Start producing immediately. + let producer = FileChunkProducer( + state: protectedState, + source: source, + path: handle.path, + length: chunkLength + ) + producer.produceMore() + + return stream + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +private struct FileChunkProducer: Sendable { + let state: NIOLockedValueBox + let source: BufferedStream.Source + let path: FilePath + let length: Int64 + + /// The 'entry point' for producing elements. + /// + /// Calling this function will start producing file chunks asynchronously by dispatching work + /// to the IO executor and feeding the result back to the stream source. On yielding to the + /// source it will either produce more or be scheduled to produce more. Stopping production + /// is signalled via the stream's 'onTermination' handler. + func produceMore() { + let executor = self.state.withLockedValue { state in + state.shouldProduceMore() + } + + // No executor means we're done. + guard let executor = executor else { return } + + executor.execute { + try self.readNextChunk() + } onCompletion: { result in + self.onReadNextChunkResult(result) + } + } + + private func readNextChunk() throws -> ByteBuffer { + return try self.state.withLockedValue { state in + state.produceMore() + }.flatMap { + if let (descriptor, range) = $0 { + if let range { + let remainingBytes = range.upperBound - range.lowerBound + let clampedLength = min(self.length, remainingBytes) + return descriptor.readChunk( + fromAbsoluteOffset: range.lowerBound, + length: clampedLength + ).mapError { error in + .read(usingSyscall: .pread, error: error, path: self.path, location: .here()) + } + } else { + return descriptor.readChunk(length: self.length).mapError { error in + .read(usingSyscall: .read, error: error, path: self.path, location: .here()) + } + } + } else { + // nil means done: return empty to indicate the stream should finish. + return .success(ByteBuffer()) + } + }.get() + } + + private func onReadNextChunkResult(_ result: Result) { + switch result { + case let .success(bytes): + self.onReadNextChunk(bytes) + case let .failure(error): + // Failed to read: update our state then notify the stream so consumers receive the + // error. + self.state.withLockedValue { state in state.done() } + self.source.finish(throwing: error) + } + } + + private func onReadNextChunk(_ bytes: ByteBuffer) { + // Reading short means EOF. + let readEOF = bytes.readableBytes < self.length + assert(bytes.readableBytes <= self.length) + + self.state.withLockedValue { state in + if readEOF { + state.readEnd() + } else { + state.readBytes(bytes.readableBytes) + } + } + + // No bytes were read: this must be the end as length is required to be greater than zero. + if bytes.readableBytes == 0 { + assert(readEOF, "read zero bytes but did not read EOF") + self.source.finish(throwing: nil) + return + } + + // Bytes were produced: yield them and maybe produce more. + do { + let writeResult = try self.source.write(contentsOf: CollectionOfOne(bytes)) + // Exit early if EOF was read; no use in trying to produce more. + if readEOF { + self.source.finish(throwing: nil) + return + } + + switch writeResult { + case .produceMore: + self.produceMore() + case let .enqueueCallback(token): + self.source.enqueueCallback(callbackToken: token) { + switch $0 { + case .success: + self.produceMore() + case .failure: + // The source is finished; mark ourselves as done. + self.state.withLockedValue { state in state.done() } + } + } + } + } catch { + // Failure to write means the source is already done, that's okay we just need to + // update our state and stop producing. + self.state.withLockedValue { state in state.done() } + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +private struct ProducerState: Sendable { + private struct Producing { + /// The handle to read from. + var handle: SystemFileHandle.SendableView + + /// An optional range containing the offsets to read from and up to (exclusive) + /// The lower bound should be updated after each successful read. + /// The upper bound should be used to check if producer should stop. + /// If no range is present, then read entire file. + var range: Range? + } + + private enum State { + /// Can potentially produce values (if the handle is not closed). + case producing(Producing) + /// Done producing values either by reaching EOF, some error or the stream terminating. + case done + } + + private var state: State + + init(handle: SystemFileHandle, range: Range?) { + if let range, range.isEmpty { + self.state = .done + } else { + self.state = .producing(.init(handle: handle.sendableView, range: range)) + } + } + + mutating func shouldProduceMore() -> IOExecutor? { + switch self.state { + case let .producing(state): + return state.handle.executor + case .done: + return nil + } + } + + mutating func produceMore() -> Result<(FileDescriptor, Range?)?, FileSystemError> { + switch self.state { + case let .producing(state): + if let descriptor = state.handle.descriptorIfAvailable() { + return .success((descriptor, state.range)) + } else { + let error = FileSystemError( + code: .closed, + message: "Cannot read from closed file ('\(state.handle.path)').", + cause: nil, + location: .here() + ) + return .failure(error) + } + case .done: + return .success(nil) + } + } + + mutating func readBytes(_ count: Int) { + switch self.state { + case var .producing(state): + if let currentRange = state.range { + let newRange = (currentRange.lowerBound + Int64(count))..= newRange.upperBound { + assert(newRange.lowerBound == newRange.upperBound) + self.state = .done + } else { + state.range = newRange + self.state = .producing(state) + } + } else { + if count == 0 { + self.state = .done + } + } + case .done: + () + } + } + + mutating func readEnd() { + switch self.state { + case .producing: + self.state = .done + case .done: + () + } + } + + mutating func done() { + self.state = .done + } +} diff --git a/Sources/NIOFileSystem/FileHandle.swift b/Sources/NIOFileSystem/FileHandle.swift new file mode 100644 index 0000000000..1bf7682423 --- /dev/null +++ b/Sources/NIOFileSystem/FileHandle.swift @@ -0,0 +1,307 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore + +/// Provides a ``FileHandle``. +/// +/// Users should not implement or rely on this protocol; its purpose is to reduce boilerplate +/// by providing a default implementation of ``FileHandleProtocol`` for types which hold +/// a ``FileHandle``. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public protocol _HasFileHandle: FileHandleProtocol { + var fileHandle: FileHandle { get } +} + +// Provides an implementation of `FileHandleProtocol` by calling through to `FileHandle`. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension _HasFileHandle { + public func info() async throws -> FileInfo { + try await self.fileHandle.info() + } + + public func replacePermissions(_ permissions: FilePermissions) async throws { + try await self.fileHandle.replacePermissions(permissions) + } + + public func addPermissions(_ permissions: FilePermissions) async throws -> FilePermissions { + try await self.fileHandle.addPermissions(permissions) + } + + public func removePermissions(_ permissions: FilePermissions) async throws -> FilePermissions { + try await self.fileHandle.removePermissions(permissions) + } + + public func attributeNames() async throws -> [String] { + try await self.fileHandle.attributeNames() + } + + public func valueForAttribute(_ name: String) async throws -> [UInt8] { + try await self.fileHandle.valueForAttribute(name) + } + + public func updateValueForAttribute( + _ bytes: some (Sendable & RandomAccessCollection), + attribute name: String + ) async throws { + try await self.fileHandle.updateValueForAttribute(bytes, attribute: name) + } + + public func removeValueForAttribute(_ name: String) async throws { + try await self.fileHandle.removeValueForAttribute(name) + } + + public func synchronize() async throws { + try await self.fileHandle.synchronize() + } + + public func withUnsafeDescriptor( + _ execute: @Sendable @escaping (FileDescriptor) throws -> R + ) async throws -> R { + try await self.fileHandle.withUnsafeDescriptor { + try execute($0) + } + } + + public func detachUnsafeFileDescriptor() throws -> FileDescriptor { + try self.fileHandle.detachUnsafeFileDescriptor() + } + + public func close() async throws { + try await self.fileHandle.close() + } +} + +/// Implements ``FileHandleProtocol`` by making system calls to interact with the local file system. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct FileHandle: FileHandleProtocol { + internal let systemFileHandle: SystemFileHandle + + internal init(wrapping handle: SystemFileHandle) { + self.systemFileHandle = handle + } + + public func info() async throws -> FileInfo { + try await self.systemFileHandle.info() + } + + public func replacePermissions(_ permissions: FilePermissions) async throws { + try await self.systemFileHandle.replacePermissions(permissions) + } + + public func addPermissions(_ permissions: FilePermissions) async throws -> FilePermissions { + try await self.systemFileHandle.addPermissions(permissions) + } + + public func removePermissions(_ permissions: FilePermissions) async throws -> FilePermissions { + try await self.systemFileHandle.removePermissions(permissions) + } + + public func attributeNames() async throws -> [String] { + try await self.systemFileHandle.attributeNames() + } + + public func valueForAttribute(_ name: String) async throws -> [UInt8] { + try await self.systemFileHandle.valueForAttribute(name) + } + + public func updateValueForAttribute( + _ bytes: some (Sendable & RandomAccessCollection), + attribute name: String + ) async throws { + try await self.systemFileHandle.updateValueForAttribute(bytes, attribute: name) + } + + public func removeValueForAttribute(_ name: String) async throws { + try await self.systemFileHandle.removeValueForAttribute(name) + } + + public func synchronize() async throws { + try await self.systemFileHandle.synchronize() + } + + public func withUnsafeDescriptor( + _ execute: @Sendable @escaping (FileDescriptor) throws -> R + ) async throws -> R { + try await self.systemFileHandle.withUnsafeDescriptor { + try execute($0) + } + } + + public func detachUnsafeFileDescriptor() throws -> FileDescriptor { + try self.systemFileHandle.detachUnsafeFileDescriptor() + } + + public func close() async throws { + try await self.systemFileHandle.close() + } +} + +/// Implements ``ReadableFileHandleProtocol`` by making system calls to interact with the local +/// file system. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct ReadFileHandle: ReadableFileHandleProtocol, _HasFileHandle { + public let fileHandle: FileHandle + + internal init(wrapping systemFileHandle: SystemFileHandle) { + self.fileHandle = FileHandle(wrapping: systemFileHandle) + } + + public func readChunk( + fromAbsoluteOffset offset: Int64, + length: ByteCount + ) async throws -> ByteBuffer { + try await self.fileHandle.systemFileHandle.readChunk( + fromAbsoluteOffset: offset, + length: length + ) + } + + public func readChunks(in range: Range, chunkLength: ByteCount) -> FileChunks { + self.fileHandle.systemFileHandle.readChunks(in: range, chunkLength: chunkLength) + } +} + +/// Implements ``WritableFileHandleProtocol`` by making system calls to interact with the local +/// file system. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct WriteFileHandle: WritableFileHandleProtocol, _HasFileHandle { + public let fileHandle: FileHandle + + internal init(wrapping systemFileHandle: SystemFileHandle) { + self.fileHandle = FileHandle(wrapping: systemFileHandle) + } + + @discardableResult + public func write( + contentsOf bytes: some (Sequence & Sendable), + toAbsoluteOffset offset: Int64 + ) async throws -> Int64 { + try await self.fileHandle.systemFileHandle.write( + contentsOf: bytes, + toAbsoluteOffset: offset + ) + } + + public func resize(to size: ByteCount) async throws { + try await self.fileHandle.systemFileHandle.resize(to: size) + } + + public func close(makeChangesVisible: Bool) async throws { + try await self.fileHandle.systemFileHandle.close(makeChangesVisible: makeChangesVisible) + } +} + +/// Implements ``ReadableAndWritableFileHandleProtocol`` by making system calls to interact with the +/// local file system. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct ReadWriteFileHandle: ReadableAndWritableFileHandleProtocol, _HasFileHandle { + public let fileHandle: FileHandle + + internal init(wrapping systemFileHandle: SystemFileHandle) { + self.fileHandle = FileHandle(wrapping: systemFileHandle) + } + + public func readChunk( + fromAbsoluteOffset offset: Int64, + length: ByteCount + ) async throws -> ByteBuffer { + try await self.fileHandle.systemFileHandle.readChunk( + fromAbsoluteOffset: offset, + length: length + ) + } + + public func readChunks(in offset: Range, chunkLength: ByteCount) -> FileChunks { + self.fileHandle.systemFileHandle.readChunks(in: offset, chunkLength: chunkLength) + } + + @discardableResult + public func write( + contentsOf bytes: some (Sequence & Sendable), + toAbsoluteOffset offset: Int64 + ) async throws -> Int64 { + try await self.fileHandle.systemFileHandle.write( + contentsOf: bytes, + toAbsoluteOffset: offset + ) + } + + public func resize(to size: ByteCount) async throws { + try await self.fileHandle.systemFileHandle.resize(to: size) + } + + public func close(makeChangesVisible: Bool) async throws { + try await self.fileHandle.systemFileHandle.close(makeChangesVisible: makeChangesVisible) + } +} + +/// Implements ``DirectoryFileHandleProtocol`` by making system calls to interact with the local +/// file system. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct DirectoryFileHandle: DirectoryFileHandleProtocol, _HasFileHandle { + public let fileHandle: FileHandle + + internal init(wrapping systemFileHandle: SystemFileHandle) { + self.fileHandle = FileHandle(wrapping: systemFileHandle) + } + + public func listContents(recursive: Bool) -> DirectoryEntries { + self.fileHandle.systemFileHandle.listContents(recursive: recursive) + } + + public func openFile( + forReadingAt path: FilePath, + options: OpenOptions.Read + ) async throws -> ReadFileHandle { + let systemFileHandle = try await self.fileHandle.systemFileHandle.openFile( + forReadingAt: path, + options: options + ) + return ReadFileHandle(wrapping: systemFileHandle) + } + + public func openFile( + forWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> WriteFileHandle { + let systemFileHandle = try await self.fileHandle.systemFileHandle.openFile( + forWritingAt: path, + options: options + ) + return WriteFileHandle(wrapping: systemFileHandle) + } + + public func openFile( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> ReadWriteFileHandle { + let systemFileHandle = try await self.fileHandle.systemFileHandle.openFile( + forReadingAndWritingAt: path, + options: options + ) + return ReadWriteFileHandle(wrapping: systemFileHandle) + } + + public func openDirectory( + atPath path: FilePath, + options: OpenOptions.Directory + ) async throws -> DirectoryFileHandle { + let systemFileHandle = try await self.fileHandle.systemFileHandle.openDirectory( + atPath: path, + options: options + ) + return DirectoryFileHandle(wrapping: systemFileHandle) + } +} diff --git a/Sources/NIOFileSystem/FileHandleProtocol.swift b/Sources/NIOFileSystem/FileHandleProtocol.swift new file mode 100644 index 0000000000..09463d354a --- /dev/null +++ b/Sources/NIOFileSystem/FileHandleProtocol.swift @@ -0,0 +1,688 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +import SystemPackage + +/// A handle for a file system object. +/// +/// There is a hierarchy of file handle protocols which allow for different functionality. All +/// file handle protocols refine the base ``FileHandleProtocol`` protocol. +/// +/// ``` +/// ┌────────────────────┐ +/// │ FileHandleProtocol │ +/// │ [Protocol] │ +/// └────────────────────┘ +/// ▲ +/// ┌──────────────────────────────┼────────────────────────────────┐ +/// │ │ │ +/// ┌────────────────────────────┐ ┌────────────────────────────┐ ┌─────────────────────────────┐ +/// │ ReadableFileHandleProtocol │ │ WritableFileHandleProtocol │ │ DirectoryFileHandleProtocol │ +/// │ [Protocol] │ │ [Protocol] │ │ [Protocol] │ +/// └────────────────────────────┘ └────────────────────────────┘ └─────────────────────────────┘ +/// ``` +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public protocol FileHandleProtocol { + /// Returns information about the file. + /// + /// Information is typically gathered by calling `fstat(2)` on the file. + /// + /// - Returns: Information about the open file. + func info() async throws -> FileInfo + + /// Replaces the permissions set on the file. + /// + /// Permissions are typically set using `fchmod(2)`. + /// + /// - Parameters: + /// - permissions: The permissions to set on the file. + func replacePermissions(_ permissions: FilePermissions) async throws + + /// Adds permissions to the existing permissions set for the file. + /// + /// This is equivalent to retrieving the permissions, merging them with the provided + /// permissions and then replacing the permissions on the file. + /// + /// - Parameters: + /// - permissions: The permissions to add to the file. + /// - Returns: The updated permissions. + @discardableResult + func addPermissions(_ permissions: FilePermissions) async throws -> FilePermissions + + /// Remove permissions from the existing permissions set for the file. + /// + /// This is equivalent to retrieving the permissions, subtracting any of the provided + /// permissions and then replacing the permissions on the file. + /// + /// - Parameters: + /// - permissions: The permissions to remove from the file. + /// - Returns: The updated permissions. + @discardableResult + func removePermissions(_ permissions: FilePermissions) async throws -> FilePermissions + + /// Returns an array containing the names of all extended attributes set on the file. + /// + /// Attributes names are typically fetched using `flistxattr(2)`. + func attributeNames() async throws -> [String] + + /// Returns the value for the named attribute if it exists; `nil` otherwise. + /// + /// Attribute values are typically fetched using `fgetxattr(2)`. + /// + /// - Parameters: + /// - name: The name of the attribute. + /// - Returns: The bytes of the value set for the attribute. If no value is set an empty array + /// is returned. + func valueForAttribute(_ name: String) async throws -> [UInt8] + + /// Replaces the value for the named attribute, creating it if it didn't already exist. + /// + /// Attribute values are typically replaced using `fsetxattr(2)`. + /// + /// - Parameters: + /// - bytes: The bytes to set as the value for the attribute. + /// - name: The name of the attribute. + func updateValueForAttribute( + _ bytes: some (Sendable & RandomAccessCollection), + attribute name: String + ) async throws + + /// Removes the value for the named attribute if it exists. + /// + /// Attribute values are typically removed using `fremovexattr(2)`. + /// + /// - Parameter name: The name of the attribute to remove. + func removeValueForAttribute(_ name: String) async throws + + /// Synchronize modified data and metadata to a permanent storage device. + /// + /// This is typically achieved using `fsync(2)`. + func synchronize() async throws + + /// Runs the provided callback with the file descriptor for this handle. + /// + /// This function should be used with caution: the `FileDescriptor` must not be escaped from + /// the closure nor should it be closed. Where possible make use of the methods defined + /// on ``FileHandleProtocol`` instead; this function is intended as an escape hatch. + /// + /// Note that `execute` is not run if the handle has already been closed. + /// + /// - Parameter execute: A closure to run. + /// - Returns: The result of the closure. + func withUnsafeDescriptor( + _ execute: @Sendable @escaping (FileDescriptor) throws -> R + ) async throws -> R + + /// Detaches and returns the file descriptor from the handle. + /// + /// After detaching the file descriptor the handle is rendered invalid. All methods will throw + /// an appropriate error if called. Detaching the descriptor yields ownerships to the caller. + /// + /// - Returns: The detached `FileDescriptor` + func detachUnsafeFileDescriptor() throws -> FileDescriptor + + /// Closes the file handle. + /// + /// It is important to close a handle once it has been finished with to avoid leaking + /// resources. Prefer using APIs which provided scoped access to a file handle which + /// manage lifecycles on your behalf. Note that if the handle has been detached via + /// ``detachUnsafeFileDescriptor()`` then it is not necessary to close the handle. + /// + /// After closing the handle calls to other functions will throw an appropriate error. + func close() async throws +} + +// MARK: - Readable + +/// A handle for reading data from an open file. +/// +/// ``ReadableFileHandleProtocol`` refines ``FileHandleProtocol`` to add requirements for reading data from a file. +/// +/// There are two requirements for implementing this protocol: +/// 1. ``readChunk(fromAbsoluteOffset:length:)``, and +/// 2. ``readChunks(in:chunkLength:)-8of2k`` +/// +/// A number of overloads are provided which provide sensible defaults. +/// +/// Conformance to ``ReadableFileHandleProtocol`` also provides +/// ``readToEnd(fromAbsoluteOffset:maximumSizeAllowed:)`` (and various overloads with sensible +/// defaults) for reading the contents of a file into memory. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public protocol ReadableFileHandleProtocol: FileHandleProtocol { + /// Returns a slice of bytes read from the file. + /// + /// The length of the slice to read indicates the largest size in bytes that the returned slice + /// may be. The slice may be shorter than the given `length` if there are fewer bytes from the + /// `offset` to the end of the file. + /// + /// - Parameters: + /// - offset: The absolute offset into the file to read from. + /// - length: The maximum number of bytes to read as a ``ByteCount``. + /// - Returns: The bytes read from the file. + func readChunk(fromAbsoluteOffset offset: Int64, length: ByteCount) async throws -> ByteBuffer + + /// Returns an asynchronous sequence of chunks read from the file. + /// + /// - Parameters: + /// - range: The absolute offsets into the file to read. + /// - chunkLength: The maximum length of the chunk to read as a ``ByteCount``. + /// - Returns: A sequence of chunks read from the file. + func readChunks(in range: Range, chunkLength: ByteCount) -> FileChunks +} + +// MARK: - Read chunks with default chunk length + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension ReadableFileHandleProtocol { + /// Returns an asynchronous sequence of chunks read from the file. + /// + /// - Parameters: + /// - range: A range of offsets in the file to read. + /// - chunkLength: The length of chunks to read, defaults to 128 KiB. + /// - as: Type of chunk to read. + /// - SeeAlso: ``ReadableFileHandleProtocol/readChunks(in:chunkLength:)-5ljvn``. + /// - Returns: An `AsyncSequence` of chunks read from the file. + public func readChunks( + in range: ClosedRange, + chunkLength: ByteCount = .kibibytes(128) + ) -> FileChunks { + return self.readChunks(in: Range(range), chunkLength: chunkLength) + } + + /// Returns an asynchronous sequence of chunks read from the file. + /// + /// - Parameters: + /// - range: A range of offsets in the file to read. + /// - chunkLength: The length of chunks to read, defaults to 128 KiB. + /// - as: Type of chunk to read. + /// - SeeAlso: ``ReadableFileHandleProtocol/readChunks(in:chunkLength:)-5ljvn``. + /// - Returns: An `AsyncSequence` of chunks read from the file. + public func readChunks( + in range: Range, + chunkLength: ByteCount = .kibibytes(128) + ) -> FileChunks { + return self.readChunks(in: range, chunkLength: chunkLength) + } + + /// Returns an asynchronous sequence of chunks read from the file. + /// + /// - Parameters: + /// - range: A range of offsets in the file to read. + /// - chunkLength: The length of chunks to read, defaults to 128 KiB. + /// - as: Type of chunk to read. + /// - SeeAlso: ``ReadableFileHandleProtocol/readChunks(in:chunkLength:)-5ljvn``. + /// - Returns: An `AsyncSequence` of chunks read from the file. + public func readChunks( + in range: PartialRangeFrom, + chunkLength: ByteCount = .kibibytes(128) + ) -> FileChunks { + let newRange = range.lowerBound.., + chunkLength: ByteCount = .kibibytes(128) + ) -> FileChunks { + let newRange = 0...range.upperBound + return self.readChunks(in: newRange, chunkLength: chunkLength) + } + + /// Returns an asynchronous sequence of chunks read from the file. + /// + /// - Parameters: + /// - range: A range of offsets in the file to read. + /// - chunkLength: The length of chunks to read, defaults to 128 KiB. + /// - as: Type of chunk to read. + /// - SeeAlso: ``ReadableFileHandleProtocol/readChunks(in:chunkLength:)-5ljvn``. + /// - Returns: An `AsyncSequence` of chunks read from the file. + public func readChunks( + in range: PartialRangeUpTo, + chunkLength: ByteCount = .kibibytes(128) + ) -> FileChunks { + let newRange = 0.. FileChunks { + return self.readChunks(in: 0.. FileChunks { + return self.readChunks(in: ..., chunkLength: chunkLength) + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension ReadableFileHandleProtocol { + /// Returns the contents of a file by loading it into memory. + /// + /// - Important: This method checks whether the file is seekable or not (i.e., whether it's a socket, + /// pipe or FIFO), and will throw ``FileSystemError/Code-swift.struct/unsupported`` if + /// an offset other than zero is passed. + /// + /// - Parameters: + /// - offset: The absolute offset into the file to read from. Defaults to zero. + /// - maximumSizeAllowed: The maximum size of file to read, as a ``ByteCount``. + /// - Returns: The bytes read from the file. + /// - Throws: ``FileSystemError`` with code ``FileSystemError/Code-swift.struct/resourceExhausted`` if there + /// are more bytes to read than `maximumBytesAllowed`. + /// ``FileSystemError/Code-swift.struct/unsupported`` if file is unseekable and + /// `offset` is not 0. + public func readToEnd( + fromAbsoluteOffset offset: Int64 = 0, + maximumSizeAllowed: ByteCount + ) async throws -> ByteBuffer { + let info = try await self.info() + let fileSize = Int64(info.size) + let readSize = max(Int(fileSize - offset), 0) + + if readSize > maximumSizeAllowed.bytes { + throw FileSystemError( + code: .resourceExhausted, + message: """ + There are more bytes to read (\(readSize)) than the maximum size allowed \ + (\(maximumSizeAllowed)). Read the file in chunks or increase the maximum size \ + allowed. + """, + cause: nil, + location: .here() + ) + } + + let isSeekable = !(info.type == .fifo || info.type == .socket) + if isSeekable { + // Limit the size of single shot reads. If the system is busy then avoid slowing down other + // work by blocking an I/O executor thread while doing a large read. The limit is somewhat + // arbitrary. + let singleShotReadLimit = 64 * 1024 * 1024 + + // If the file size isn't 0 but the read size is, then it means that + // we are intending to read an empty fragment: we can just return + // fast and skip any reads. + // If the file size is 0, we can't conclude anything about the size + // of the file, as `stat` will return 0 for unbounded files. + // If this happens, just read in chunks to make sure the whole file + // is read (up to `maximumSizeAllowed` bytes). + var forceChunkedRead = false + if fileSize > 0 { + if readSize == 0 { + return ByteBuffer() + } + } else { + forceChunkedRead = true + } + + if !forceChunkedRead, readSize <= singleShotReadLimit { + return try await self.readChunk( + fromAbsoluteOffset: offset, + length: .bytes(Int64(readSize)) + ) + } else { + var accumulator = ByteBuffer() + accumulator.reserveCapacity(readSize) + + for try await chunk in self.readChunks(in: offset..., chunkLength: .mebibytes(8)) { + accumulator.writeImmutableBuffer(chunk) + if accumulator.readableBytes > maximumSizeAllowed.bytes { + throw FileSystemError( + code: .resourceExhausted, + message: """ + There are more bytes to read than the maximum size allowed \ + (\(maximumSizeAllowed)). Read the file in chunks or increase the maximum size \ + allowed. + """, + cause: nil, + location: .here() + ) + } + } + + return accumulator + } + } else { + guard offset == 0 else { + throw FileSystemError( + code: .unsupported, + message: "File is unseekable.", + cause: nil, + location: .here() + ) + } + var accumulator = ByteBuffer() + accumulator.reserveCapacity(readSize) + + for try await chunk in self.readChunks(in: ..., chunkLength: .mebibytes(8)) { + accumulator.writeImmutableBuffer(chunk) + if accumulator.readableBytes > maximumSizeAllowed.bytes { + throw FileSystemError( + code: .resourceExhausted, + message: """ + There are more bytes to read than the maximum size allowed \ + (\(maximumSizeAllowed)). Read the file in chunks or increase the maximum size \ + allowed. + """, + cause: nil, + location: .here() + ) + } + } + + return accumulator + } + } +} + +// MARK: - Writable + +/// A file handle suitable for writing. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public protocol WritableFileHandleProtocol: FileHandleProtocol { + /// Write the given bytes to the open file. + /// + /// - Important: This method checks whether the file is seekable or not (i.e., whether it's a socket, + /// pipe or FIFO), and will throw ``FileSystemError/Code-swift.struct/unsupported`` + /// if an offset other than zero is passed. + /// + /// - Parameters: + /// - bytes: The bytes to write. + /// - offset: The absolute offset into the file to write the bytes. + /// - Returns: The number of bytes written. + /// - Throws: ``FileSystemError/Code-swift.struct/unsupported`` if file is + /// unseekable and `offset` is not 0. + @discardableResult + func write( + contentsOf bytes: some (Sequence & Sendable), + toAbsoluteOffset offset: Int64 + ) async throws -> Int64 + + /// Resizes a file to the given size. + /// + /// - Parameters: + /// - size: The number of bytes to resize the file to as a ``ByteCount``. + func resize( + to size: ByteCount + ) async throws + + /// Closes the file handle. + /// + /// It is important to close a handle once it has been finished with to avoid leaking + /// resources. Prefer using APIs which provided scoped access to a file handle which + /// manage lifecycles on your behalf. Note that if the handle has been detached via + /// ``FileHandleProtocol/detachUnsafeFileDescriptor()`` then it is not necessary to close + /// the handle. + /// + /// After closing the handle calls to other functions will throw an appropriate error. + /// + /// - Parameters: + /// - makeChangesVisible: Whether the changes made to the file will be made visibile. This + /// parameter is ignored unless ``OpenOptions/NewFile/transactionalCreation`` was + /// set to `true`. When `makeChangesVisible` is `true`, the file will be created on the + /// filesystem with the expected name, otherwise no file will be created or the original + /// file won't be modified (if one existed). + func close(makeChangesVisible: Bool) async throws +} + +/// A file handle which is suitable for reading and writing. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public typealias ReadableAndWritableFileHandleProtocol = ReadableFileHandleProtocol + & WritableFileHandleProtocol + +// MARK: - Directory + +/// A handle suitable for directories. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public protocol DirectoryFileHandleProtocol: FileHandleProtocol { + /// The type of ``ReadableFileHandleProtocol`` to return when opening files for reading. + associatedtype ReadFileHandle: ReadableFileHandleProtocol + + /// The type of ``WritableFileHandleProtocol`` to return when opening files for writing. + associatedtype WriteFileHandle: WritableFileHandleProtocol + + /// The type of ``ReadableAndWritableFileHandleProtocol`` to return when opening files for reading and writing. + associatedtype ReadWriteFileHandle: ReadableAndWritableFileHandleProtocol + + /// Returns an `AsyncSequence` of entries in the open directory. + /// + /// You can recurse into and list the contents of any subdirectories by setting `recursive` + /// to `true`. The current (".") and parent ("..") directory entries are not included. The order + /// of entries is arbitrary and shouldn't be relied upon. + /// + /// - Parameter recursive: Whether subdirectories should be recursively visited. + /// - Returns: An `AsyncSequence` of directory entries. + func listContents(recursive: Bool) -> DirectoryEntries + + /// Opens the file at `path` for reading and returns a handle to it. + /// + /// If `path` is a relative path then it is opened relative to the handle. The file being + /// opened must already exist otherwise this function will throw a ``FileSystemError`` with + /// code ``FileSystemError/Code-swift.struct/notFound``. + /// + /// - Parameters: + /// - path: The path of the directory to open relative to the open file. + /// - options: How the file should be opened. + /// - Returns: A read-only handle to the opened file. + func openFile( + forReadingAt path: FilePath, + options: OpenOptions.Read + ) async throws -> ReadFileHandle + + /// Opens the file at `path` for writing and returns a handle to it. + /// + /// If `path` is a relative path then it is opened relative to the handle. + /// + /// - Parameters: + /// - path: The path of the file to open relative to the open file. + /// - options: How the file should be opened. + /// - Returns: A write-only handle to the opened file. + func openFile( + forWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> WriteFileHandle + + /// Opens the file at `path` for reading and writing and returns a handle to it. + /// + /// If `path` is a relative path then it is opened relative to the handle. + /// + /// - Parameters: + /// - path: The path of the file to open relative to the open file. + /// - options: How the file should be opened. + func openFile( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> ReadWriteFileHandle + + /// Opens the directory at `path` and returns a handle to it. + /// + /// The directory being opened must already exist otherwise this function will throw an error. + /// If `path` is a relative path then it is opened relative to the handle. + /// + /// - Parameters: + /// - path: The path of the directory to open. + /// - options: How the file should be opened. + /// - Returns: A handle to the opened directory. + func openDirectory( + atPath path: FilePath, + options: OpenOptions.Directory + ) async throws -> Self +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension DirectoryFileHandleProtocol { + /// Returns an `AsyncSequence` of entries in the open directory. + /// + /// The current (".") and parent ("..") directory entries are not included. The order of entries + /// is arbitrary and should not be relied upon. + public func listContents() -> DirectoryEntries { + return self.listContents(recursive: false) + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension DirectoryFileHandleProtocol { + /// Opens the file at the given path and provides scoped read access to it. + /// + /// The file remains open during lifetime of the `execute` block and will be closed + /// automatically before the call returns. Files may also be opened in write-only and read-write + /// mode by calling ``withFileHandle(forWritingAt:options:execute:)`` + /// and ``withFileHandle(forReadingAndWritingAt:options:execute:)``, + /// respectively. + /// + /// If `path` is a relative path then it is opened relative to the handle. The file being + /// opened must already exist otherwise this function will throw a ``FileSystemError`` with + /// code ``FileSystemError/Code-swift.struct/notFound``. + /// + /// - Parameters: + /// - path: The path of the file to open for reading. + /// - options: How the file should be opened. + /// - body: A closure which provides read-only access to the open file. The file is closed + /// automatically after the closure exits. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withFileHandle( + forReadingAt path: FilePath, + options: OpenOptions.Read = OpenOptions.Read(), + execute body: (_ read: ReadFileHandle) async throws -> R + ) async throws -> R { + let handle = try await self.openFile(forReadingAt: path, options: options) + + return try await withUncancellableTearDown { + return try await body(handle) + } tearDown: { _ in + try await handle.close() + } + } + + /// Opens the file at the given path and provides scoped write access to it. + /// + /// The file remains open during lifetime of the `execute` block and will be closed + /// automatically before the call returns. Files may also be opened in read-only or read-write + /// mode by calling ``withFileHandle(forReadingAt:options:execute:)`` and + /// ``withFileHandle(forReadingAndWritingAt:options:execute:)``, + /// respectively. + /// + /// If `path` is a relative path then it is opened relative to the handle. + /// + /// - Parameters: + /// - path: The path of the file to open for reading. + /// - options: How the file should be opened. + /// - body: A closure which provides write-only access to the open file. The file is closed + /// automatically after the closure exits. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withFileHandle( + forWritingAt path: FilePath, + options: OpenOptions.Write = .newFile(replaceExisting: false), + execute body: (_ write: WriteFileHandle) async throws -> R + ) async throws -> R { + let handle = try await self.openFile(forWritingAt: path, options: options) + + return try await withUncancellableTearDown { + return try await body(handle) + } tearDown: { result in + switch result { + case .success: + try await handle.close(makeChangesVisible: true) + case .failure: + try await handle.close(makeChangesVisible: false) + } + } + } + + /// Opens the file at the given path and provides scoped read-write access to it. + /// + /// The file remains open during lifetime of the `execute` block and will be closed + /// automatically before the call returns. Files may also be opened in read-only or write-only + /// mode by calling ``withFileHandle(forReadingAt:options:execute:)`` and + /// ``withFileHandle(forWritingAt:options:execute:)``, respectively. + /// + /// If `path` is a relative path then it is opened relative to the handle. + /// + /// - Parameters: + /// - path: The path of the file to open for reading and writing. + /// - options: How the file should be opened. + /// - body: A closure which provides access to the open file. The file is closed + /// automatically after the closure exits. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withFileHandle( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write = .newFile(replaceExisting: false), + execute body: (_ readWrite: ReadWriteFileHandle) async throws -> R + ) async throws -> R { + let handle = try await self.openFile(forReadingAndWritingAt: path, options: options) + + return try await withUncancellableTearDown { + return try await body(handle) + } tearDown: { result in + switch result { + case .success: + try await handle.close(makeChangesVisible: true) + case .failure: + try await handle.close(makeChangesVisible: false) + } + } + } + + /// Opens the directory at the given path and provides scoped access to it. + /// + /// - Parameters: + /// - path: The path of the directory to open. + /// - body: A closure which provides access to the directory. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withDirectoryHandle( + atPath path: FilePath, + options: OpenOptions.Directory = OpenOptions.Directory(), + execute body: (_ directory: Self) async throws -> R + ) async throws -> R { + let handle = try await self.openDirectory(atPath: path, options: options) + + return try await withUncancellableTearDown { + return try await body(handle) + } tearDown: { _ in + try await handle.close() + } + } +} diff --git a/Sources/NIOFileSystem/FileInfo.swift b/Sources/NIOFileSystem/FileInfo.swift new file mode 100644 index 0000000000..ea67f4e98b --- /dev/null +++ b/Sources/NIOFileSystem/FileInfo.swift @@ -0,0 +1,248 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +/// Information about a file system object. +/// +/// The information available for a file depends on the platform, ``FileInfo`` provides +/// convenient access to a common subset of properties. Using these properties ensures that +/// code is portable. If available, the platform specific information is made available via +/// ``FileInfo/platformSpecificStatus``. However users should take care to ensure their +/// code uses the correct platform checks when using it to ensure their code is portable. +public struct FileInfo: Hashable, Sendable { + /// Wraps `CInterop.Stat` providing `Hashable` and `Equatable` conformance. + private var _platformSpecificStatus: Stat? + + /// The information about the file returned from the filesystem, if available. + /// + /// This value is platform specific: you should be careful when using + /// it as some of its fields vary across platforms. In most cases prefer + /// using other properties on this type instead. + /// + /// See also: the manual pages for 'stat' (`man 2 stat`) + public var platformSpecificStatus: CInterop.Stat? { + get { self._platformSpecificStatus?.stat } + set { self._platformSpecificStatus = newValue.map { Stat($0) } } + } + + /// The type of the file. + public var type: FileType + + /// Permissions currently set on the file. + public var permissions: FilePermissions + + /// The size of the file in bytes. + public var size: Int64 + + /// User ID of the file. + public var userID: UserID + + /// Group ID of the file. + public var groupID: GroupID + + /// The last time the file was accessed. + public var lastAccessTime: Timespec + + /// The last time the files data was last changed. + public var lastDataModificationTime: Timespec + + /// The last time the status of the file was changed. + public var lastStatusChangeTime: Timespec + + /// Creates a ``FileInfo`` by deriving values from a platform-specific value. + public init(platformSpecificStatus: CInterop.Stat) { + self._platformSpecificStatus = Stat(platformSpecificStatus) + self.type = FileType(platformSpecificMode: platformSpecificStatus.st_mode) + self.permissions = FilePermissions(masking: platformSpecificStatus.st_mode) + self.size = Int64(platformSpecificStatus.st_size) + self.userID = UserID(rawValue: platformSpecificStatus.st_uid) + self.groupID = GroupID(rawValue: platformSpecificStatus.st_gid) + + #if canImport(Darwin) + self.lastAccessTime = Timespec(platformSpecificStatus.st_atimespec) + self.lastDataModificationTime = Timespec(platformSpecificStatus.st_mtimespec) + self.lastStatusChangeTime = Timespec(platformSpecificStatus.st_ctimespec) + #elseif canImport(Glibc) + self.lastAccessTime = Timespec(platformSpecificStatus.st_atim) + self.lastDataModificationTime = Timespec(platformSpecificStatus.st_mtim) + self.lastStatusChangeTime = Timespec(platformSpecificStatus.st_ctim) + #endif + } + + /// Creates a ``FileInfo`` from the provided values. + /// + /// If you have a platform specific status value prefer calling + /// ``init(platformSpecificStatus:)``. + public init( + type: FileType, + permissions: FilePermissions, + size: Int64, + userID: UserID, + groupID: GroupID, + lastAccessTime: Timespec, + lastDataModificationTime: Timespec, + lastStatusChangeTime: Timespec + ) { + self._platformSpecificStatus = nil + self.type = type + self.permissions = permissions + self.size = size + self.userID = userID + self.groupID = groupID + self.lastAccessTime = lastAccessTime + self.lastDataModificationTime = lastDataModificationTime + self.lastStatusChangeTime = lastStatusChangeTime + } +} + +extension FileInfo { + /// The numeric ID of a user. + public struct UserID: Hashable, Sendable, CustomStringConvertible { + public var rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public var description: String { + String(describing: self.rawValue) + } + } + + /// The numeric ID of a group. + public struct GroupID: Hashable, Sendable, CustomStringConvertible { + public var rawValue: UInt32 + + public init(rawValue: UInt32) { + self.rawValue = rawValue + } + + public var description: String { + String(describing: self.rawValue) + } + } + + /// A time interval consisting of whole seconds and nanoseconds. + public struct Timespec: Hashable, Sendable { + /// The number of seconds. + public var seconds: Int + + /// The number of nanoseconds. + public var nanoseconds: Int + + init(_ timespec: timespec) { + self.seconds = timespec.tv_sec + self.nanoseconds = timespec.tv_nsec + } + + public init(seconds: Int, nanoseconds: Int) { + self.seconds = seconds + self.nanoseconds = nanoseconds + } + } +} + +/// A wrapper providing `Hashable` and `Equatable` conformance for `CInterop.Stat`. +private struct Stat: Hashable { + var stat: CInterop.Stat + + init(_ stat: CInterop.Stat) { + self.stat = stat + } + + func hash(into hasher: inout Hasher) { + let stat = self.stat + // Different platforms have different underlying values; these are + // common between Darwin and Glibc. + hasher.combine(stat.st_dev) + hasher.combine(stat.st_mode) + hasher.combine(stat.st_nlink) + hasher.combine(stat.st_ino) + hasher.combine(stat.st_uid) + hasher.combine(stat.st_gid) + hasher.combine(stat.st_rdev) + hasher.combine(stat.st_size) + hasher.combine(stat.st_blocks) + hasher.combine(stat.st_blksize) + + #if canImport(Darwin) + hasher.combine(FileInfo.Timespec(stat.st_atimespec)) + hasher.combine(FileInfo.Timespec(stat.st_mtimespec)) + hasher.combine(FileInfo.Timespec(stat.st_ctimespec)) + hasher.combine(FileInfo.Timespec(stat.st_birthtimespec)) + hasher.combine(stat.st_flags) + hasher.combine(stat.st_gen) + #elseif canImport(Glibc) + hasher.combine(FileInfo.Timespec(stat.st_atim)) + hasher.combine(FileInfo.Timespec(stat.st_mtim)) + hasher.combine(FileInfo.Timespec(stat.st_ctim)) + #endif + + } + + static func == (lhs: Stat, rhs: Stat) -> Bool { + let lStat = lhs.stat + let rStat = rhs.stat + + // Different platforms have different underlying values; these are + // common between Darwin and Glibc. + var isEqual = lStat.st_dev == rStat.st_dev + isEqual = isEqual && lStat.st_mode == rStat.st_mode + isEqual = isEqual && lStat.st_nlink == rStat.st_nlink + isEqual = isEqual && lStat.st_ino == rStat.st_ino + isEqual = isEqual && lStat.st_uid == rStat.st_uid + isEqual = isEqual && lStat.st_gid == rStat.st_gid + isEqual = isEqual && lStat.st_rdev == rStat.st_rdev + isEqual = isEqual && lStat.st_size == rStat.st_size + isEqual = isEqual && lStat.st_blocks == rStat.st_blocks + isEqual = isEqual && lStat.st_blksize == rStat.st_blksize + + #if canImport(Darwin) + isEqual = + isEqual + && FileInfo.Timespec(lStat.st_atimespec) == FileInfo.Timespec(rStat.st_atimespec) + isEqual = + isEqual + && FileInfo.Timespec(lStat.st_mtimespec) == FileInfo.Timespec(rStat.st_mtimespec) + isEqual = + isEqual + && FileInfo.Timespec(lStat.st_ctimespec) == FileInfo.Timespec(rStat.st_ctimespec) + isEqual = + isEqual + && FileInfo.Timespec(lStat.st_birthtimespec) + == FileInfo.Timespec(rStat.st_birthtimespec) + isEqual = isEqual && lStat.st_flags == rStat.st_flags + isEqual = isEqual && lStat.st_gen == rStat.st_gen + #elseif canImport(Glibc) + isEqual = isEqual && FileInfo.Timespec(lStat.st_atim) == FileInfo.Timespec(rStat.st_atim) + isEqual = isEqual && FileInfo.Timespec(lStat.st_mtim) == FileInfo.Timespec(rStat.st_mtim) + isEqual = isEqual && FileInfo.Timespec(lStat.st_ctim) == FileInfo.Timespec(rStat.st_ctim) + #endif + + return isEqual + } +} + +extension FilePermissions { + internal init(masking rawValue: CInterop.Mode) { + self = .init(rawValue: rawValue & ~S_IFMT) + } +} diff --git a/Sources/NIOFileSystem/FileSystem.swift b/Sources/NIOFileSystem/FileSystem.swift new file mode 100644 index 0000000000..8d9f17d361 --- /dev/null +++ b/Sources/NIOFileSystem/FileSystem.swift @@ -0,0 +1,1435 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Atomics +import NIOCore +@preconcurrency import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +/// A file system which interacts with the local system. The file system uses a thread pool to +/// perform system calls. +/// +/// ### Creating a `FileSystem` +/// +/// You should prefer using the `shared` instance of the file system. The +/// `shared` instance uses two threads unless the `SWIFT_FILE_SYSTEM_THREAD_COUNT` +/// environment variable is set. +/// +/// If you require more granular control you can create a ``FileSystem`` with the required number +/// of threads by calling ``withFileSystem(numberOfThreads:_:)``. +/// +/// ### Errors +/// +/// Errors thrown by ``FileSystem`` will be of type: +/// - ``FileSystemError`` if it wasn't possible to complete the operation, or +/// - `CancellationError` if the `Task` was cancelled. +/// +/// ``FileSystemError`` implements `CustomStringConvertible`. The output from the `description` +/// contains basic information including the error code, message and underlying error. You can get +/// more information about the error by calling ``FileSystemError/detailedDescription()`` which +/// returns a structured multi-line string containing information about the error. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public struct FileSystem: Sendable, FileSystemProtocol { + /// Returns a shared global instance of the ``FileSystem``. + /// + /// The file system executes blocking work in a thread pool which defaults to having two + /// threads. This can be modified by `fileSystemThreadCountSuggestion` or by + /// setting the `NIO_SINGLETON_FILESYSTEM_THREAD_COUNT` environment variable. + public static var shared: FileSystem { globalFileSystem } + + private let executor: IOExecutor + + fileprivate func shutdown() async { + await self.executor.drain() + } + + fileprivate init(executor: IOExecutor) { + self.executor = executor + } + + fileprivate init(numberOfThreads: Int) async { + let executor = await IOExecutor.running(numberOfThreads: numberOfThreads) + self.init(executor: executor) + } + + /// Open the file at `path` for reading. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/notFound`` if `path` doesn't exist. + /// + /// #### Implementation details + /// + /// Uses the `open(2)` system call. + /// + /// - Parameters: + /// - path: The path of the file to open. + /// - options: How the file should be opened. + /// - Returns: A readable handle to the opened file. + public func openFile( + forReadingAt path: FilePath, + options: OpenOptions.Read + ) async throws -> ReadFileHandle { + let handle = try await self.executor.execute { + let handle = try self._openFile(forReadingAt: path, options: options).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } + + /// Open the file at `path` for writing. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/permissionDenied`` if you have insufficient + /// permissions to create the file. + /// - ``FileSystemError/Code-swift.struct/notFound`` if `path` doesn't exist and `options` + /// weren't set to create a file. + /// + /// #### Implementation details + /// + /// Uses the `open(2)` system call. + /// + /// - Parameters: + /// - path: The path of the file to open. + /// - options: How the file should be opened. + /// - Returns: A writable handle to the opened file. + public func openFile( + forWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> WriteFileHandle { + let handle = try await self.executor.execute { + let handle = try self._openFile(forWritingAt: path, options: options).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } + + /// Open the file at `path` for reading and writing. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/permissionDenied`` if you have insufficient + /// permissions to create the file. + /// - ``FileSystemError/Code-swift.struct/notFound`` if `path` doesn't exist and `options` + /// weren't set to create a file. + /// + /// #### Implementation details + /// + /// Uses the `open(2)` system call. + /// + /// - Parameters: + /// - path: The path of the file to open. + /// - options: How the file should be opened. + /// - Returns: A readable and writable handle to the opened file. + public func openFile( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> ReadWriteFileHandle { + let handle = try await self.executor.execute { + let handle = try self._openFile(forReadingAndWritingAt: path, options: options).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } + + /// Open the directory at `path`. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/notFound`` if `path` doesn't exist. + /// + /// #### Implementation details + /// + /// Uses the `open(2)` system call. + /// + /// - Parameters: + /// - path: The path of the directory to open. + /// - Returns: A handle to the opened directory. + public func openDirectory( + atPath path: FilePath, + options: OpenOptions.Directory + ) async throws -> DirectoryFileHandle { + let handle = try await self.executor.execute { + let handle = try self._openDirectory(at: path, options: options).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } + + /// Create a directory at the given path. + /// + /// If a directory (or file) already exists at `path` a ``FileSystemError`` with code + /// ``FileSystemError/Code-swift.struct/fileAlreadyExists`` is thrown. + /// + /// If the parent directory of the directory to created does not exist a ``FileSystemError`` + /// with ``FileSystemError/Code-swift.struct/invalidArgument`` is thrown. Missing directories + /// can be created by passing `true` to `createIntermediateDirectories`. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/fileAlreadyExists`` if a file or directory already + /// exists . + /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if a component in the `path` + /// prefix does not exist and `createIntermediateDirectories` is `false`. + /// + /// #### Implementation details + /// + /// Uses the `mkdir(2)` system call. + /// + /// - Parameters: + /// - path: The directory to create. + /// - createIntermediateDirectories: Whether intermediate directories should be created. + /// - permissions: The permissions to set on the new directory; default permissions will be + /// used if not specified. + public func createDirectory( + at path: FilePath, + withIntermediateDirectories createIntermediateDirectories: Bool, + permissions: FilePermissions? + ) async throws { + try await self.executor.execute { + try self._createDirectory( + at: path, + withIntermediateDirectories: createIntermediateDirectories, + permissions: permissions ?? .defaultsForDirectory + ).get() + } + } + + /// Create a temporary directory at the given path, using a template. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if the template doesn't end + /// in at least 3 'X's. + /// - ``FileSystemError/Code-swift.struct/permissionDenied`` if the user doesn't have + /// permission to create a directory at the path specified in the template. + /// + /// #### Implementation details + /// + /// Uses the `mkdir(2)` system call. + /// + /// - Parameters: + /// - template: The template for the path of the temporary directory. + /// - Returns: + /// - The path to the new temporary directory. + public func createTemporaryDirectory( + template: FilePath + ) async throws -> FilePath { + return try await self.executor.execute { + try self._createTemporaryDirectory(template: template).get() + } + } + + /// Returns information about the file at `path` if it exists; nil otherwise. + /// + /// #### Implementation details + /// + /// Uses `lstat(2)` if `infoAboutSymbolicLink` is `true`, `stat(2)` otherwise. + /// + /// - Parameters: + /// - path: The path of the file. + /// - infoAboutSymbolicLink: If the file is a symbolic link and this parameter is `true` then information + /// about the link will be returned, otherwise information about the destination of the + /// symbolic link is returned. + /// - Returns: Information about the file at the given path or `nil` if no file exists. + public func info( + forFileAt path: FilePath, + infoAboutSymbolicLink: Bool + ) async throws -> FileInfo? { + return try await self.executor.execute { + try self._info(forFileAt: path, infoAboutSymbolicLink: infoAboutSymbolicLink).get() + } + } + + // MARK: - File copying, removal, and moving + + /// Copies the item at the specified path to a new location. + /// + /// The item to be copied must be a: + /// - regular file, + /// - symbolic link, or + /// - directory. + /// + /// If `sourcePath` is a symbolic link then only the link is copied. The copied file will + /// preserve permissions and any extended attributes (if supported by the file system). + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/notFound`` if `sourcePath` doesn't exist. + /// - ``FileSystemError/Code-swift.struct/fileAlreadyExists`` if `destinationPath` exists. + /// - ``FileSystemError/Code-swift.struct/unsupported`` if an item to be copied is not a + /// regular file, symbolic link or directory. + /// + /// #### Implementation details + /// + /// This function is platform dependent. On Darwin the `copyfile(2)` system call is + /// used and items are cloned where possible. On Linux the `sendfile(2)` system call is used. + /// + /// - Parameters: + /// - sourcePath: The path to the item to copy. + /// - destinationPath: The path at which to place the copy. + /// - shouldProceedAfterError: Determines whether to continue copying files if an error is + /// thrown during the operation. This error does not have to match the error passed + /// to the closure. + /// - shouldCopyFile: A closure which is executed before each file to determine whether the + /// file should be copied. + public func copyItem( + at sourcePath: FilePath, + to destinationPath: FilePath, + shouldProceedAfterError: @escaping @Sendable ( + _ entry: DirectoryEntry, + _ error: Error + ) async throws -> Void, + shouldCopyFile: @escaping @Sendable ( + _ source: FilePath, + _ destination: FilePath + ) async -> Bool + ) async throws { + guard let info = try await self.info(forFileAt: sourcePath, infoAboutSymbolicLink: true) + else { + throw FileSystemError( + code: .notFound, + message: "Unable to copy '\(sourcePath)', it does not exist.", + cause: nil, + location: .here() + ) + } + + switch info.type { + case .regular: + if await shouldCopyFile(sourcePath, destinationPath) { + try await self.copyRegularFile(from: sourcePath, to: destinationPath) + } + + case .symlink: + if await shouldCopyFile(sourcePath, destinationPath) { + try await self.copySymbolicLink(from: sourcePath, to: destinationPath) + } + + case .directory: + if await shouldCopyFile(sourcePath, destinationPath) { + try await self.copyDirectory( + from: sourcePath, + to: destinationPath, + shouldProceedAfterError: shouldProceedAfterError, + shouldCopyFile: shouldCopyFile + ) + } + + default: + throw FileSystemError( + code: .unsupported, + message: """ + Can't copy '\(sourcePath)' of type '\(info.type)'; only regular files, \ + symbolic links and directories can be copied. + """, + cause: nil, + location: .here() + ) + } + } + + /// Deletes the file or directory (and its contents) at `path`. + /// + /// Only regular files, symbolic links and directories may be removed. If the file at `path` is + /// a directory then its contents and all of its subdirectories will be removed recursively. + /// Symbolic links are also removed (but their targets are not deleted). If no file exists at + /// `path` this function returns zero. + /// + /// #### Errors + /// + /// Errors codes thrown by this function include: + /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if the item is not a regular file, + /// symbolic link or directory. This also applies to items within the directory being removed. + /// - ``FileSystemError/Code-swift.struct/notFound`` if the item being removed is a directory + /// which isn't empty and `removeItemRecursively` is false. + /// + /// #### Implementation details + /// + /// Uses the `remove(3)` system call. + /// + /// - Parameters: + /// - path: The path to delete. + /// - Returns: The number of deleted items which may be zero if `path` did not exist. + @discardableResult + public func removeItem( + at path: FilePath, + recursively removeItemRecursively: Bool + ) async throws -> Int { + // Try to remove the item: we might just get lucky. + let result = try await self.executor.execute { Libc.remove(path) } + + switch result { + case .success: + // Great; we removed 1 whole item. + return 1 + + case .failure(.noSuchFileOrDirectory): + // Nothing to delete. + return 0 + + case .failure(.directoryNotEmpty): + guard removeItemRecursively else { + throw FileSystemError( + code: .notEmpty, + message: """ + Can't remove directory at path '\(path)', it isn't empty and \ + 'removeItemRecursively' is false. Remove items from the directory first \ + or set 'removeItemRecursively' to true when calling \ + 'removeItem(at:recursively:)'. + """, + cause: nil, + location: .here() + ) + } + + var (subdirectories, filesRemoved) = try await self.withDirectoryHandle( + atPath: path + ) { directory in + var subdirectories = [FilePath]() + var filesRemoved = 0 + + for try await batch in directory.listContents().batched() { + for entry in batch { + switch entry.type { + case .directory: + subdirectories.append(entry.path) + + default: + filesRemoved += try await self.removeOneItem(at: entry.path) + } + } + } + + return (subdirectories, filesRemoved) + } + + for subdirectory in subdirectories { + filesRemoved += try await self.removeItem(at: subdirectory) + } + + // The directory should be empty now. Remove ourself. + filesRemoved += try await self.removeOneItem(at: path) + + return filesRemoved + + case let .failure(errno): + throw FileSystemError.remove(errno: errno, path: path, location: .here()) + } + } + + /// Moves the named file or directory to a new location. + /// + /// Only regular files, symbolic links and directories may be moved. If the item to be is a + /// symbolic link then only the link is moved; the target of the link is not moved. + /// + /// If the file is moved within to a different logical partition then the file is copied to the + /// new partition before being removed from old partition. If removing the item fails the copied + /// file will not be removed. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/notFound`` if the item at `sourcePath` does not exist, + /// - ``FileSystemError/Code-swift.struct/fileAlreadyExists`` if an item at `destinationPath` + /// already exists. + /// + /// #### Implementation details + /// + /// Uses the `rename(2)` system call. + /// + /// - Parameters: + /// - sourcePath: The path to the item to move. + /// - destinationPath: The path at which to place the item. + public func moveItem(at sourcePath: FilePath, to destinationPath: FilePath) async throws { + let result = try await self.executor.execute { + try self._moveItem(at: sourcePath, to: destinationPath).get() + } + + switch result { + case .moved: + () + case .differentLogicalDevices: + // Fall back to copy and remove. + try await self.copyItem(at: sourcePath, to: destinationPath) + try await self.removeItem(at: sourcePath) + } + } + + /// Replaces the item at `destinationPath` with the item at `existingPath`. + /// + /// Only regular files, symbolic links and directories may replace the item at the existing + /// path. The file at the destination path isn't required to exist. If it does exist it does not + /// have to match the type of the file it is being replaced with. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/notFound`` if the item at `existingPath` does not + /// exist. + /// + /// #### Implementation details + /// + /// Uses the `rename(2)` system call. + /// + /// - Parameters: + /// - destinationPath: The path of the file or directory to replace. + /// - existingPath: The path of the existing file or directory. + public func replaceItem( + at destinationPath: FilePath, + withItemAt existingPath: FilePath + ) async throws { + do { + try await self.removeItem(at: destinationPath) + try await self.moveItem(at: existingPath, to: destinationPath) + try await self.removeItem(at: existingPath) + } catch let error as FileSystemError { + throw FileSystemError( + message: "Can't replace '\(destinationPath)' with '\(existingPath)'.", + wrapping: error + ) + } + } + + // MARK: - Symbolic links + + /// Creates a symbolic link between two files. + /// + /// A link is created at `linkPath` which points to `destinationPath`. The destination of a + /// symbolic link can be read with ``destinationOfSymbolicLink(at:)``. + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/fileAlreadyExists`` if a file exists at + /// `destinationPath`. + /// + /// #### Implementation details + /// + /// Uses the `link(2)` system call. + /// + /// - Parameters: + /// - path: The path at which to create the symbolic link. + /// - destinationPath: The path that contains the item that the symbolic link points to.` + public func createSymbolicLink( + at linkPath: FilePath, + withDestination destinationPath: FilePath + ) async throws { + return try await self.executor.execute { + try self._createSymbolicLink(at: linkPath, withDestination: destinationPath).get() + } + } + + /// Returns the path of the item pointed to by a symbolic link. + /// + /// The destination of the symbolic link is not guaranteed to be a valid path, nor is it + /// guaranteed to be an absolute path. If you need to open a file which is the destination of a + /// symbolic link then the appropriate `open` function: + /// - ``openFile(forReadingAt:)`` + /// - ``openFile(forWritingAt:options:)`` + /// - ``openFile(forReadingAndWritingAt:options:)`` + /// - ``openDirectory(atPath:options:)`` + /// + /// #### Errors + /// + /// Error codes thrown include: + /// - ``FileSystemError/Code-swift.struct/notFound`` if `path` does not exist. + /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if the file at `path` is not a + /// symbolic link. + /// + /// #### Implementation details + /// + /// Uses the `readlink(2)` system call. + /// + /// - Parameter path: The path of a file or directory. + /// - Returns: The path of the file or directory to which the symbolic link points to. + public func destinationOfSymbolicLink( + at path: FilePath + ) async throws -> FilePath { + return try await self.executor.execute { + try self._destinationOfSymbolicLink(at: path).get() + } + } + + /// Returns the path of the current working directory. + /// + /// #### Implementation details + /// + /// Uses the `getcwd(2)` system call. + /// + /// - Returns: The path to the current working directory. + public var currentWorkingDirectory: FilePath { + get async throws { + try await self.executor.execute { + try Libc.getcwd().mapError { errno in + FileSystemError.getcwd(errno: errno, location: .here()) + }.get() + } + } + } + + /// Returns a path to a temporary directory. + /// + /// #### Implementation details + /// + /// On Darwin this function uses `confstr(3)` and gets the value of `_CS_DARWIN_USER_TEMP_DIR`; + /// the users temporary directory. Typically items are removed after three days if they are not + /// accessed. + /// + /// On Linux this returns "/tmp". + /// + /// - Returns: The path to a temporary directory. + public var temporaryDirectory: FilePath { + get async throws { + #if canImport(Darwin) + return try await self.executor.execute { + return try Libc.constr(_CS_DARWIN_USER_TEMP_DIR).map { path in + FilePath(path) + }.mapError { errno in + FileSystemError.confstr( + name: "_CS_DARWIN_USER_TEMP_DIR", + errno: errno, + location: .here() + ) + }.get() + } + #else + return "/tmp" + #endif + } + } +} + +// MARK: - Creating FileSystems + +extension NIOSingletons { + /// A suggestion of how many threads the global singleton ``FileSystem`` uses for blocking I/O. + /// + /// The thread count is ``System/coreCount`` unless the environment variable + /// `NIO_SINGLETON_FILESYSTEM_THREAD_COUNT` is set or this value was set manually by the user. + /// + /// - note: This value must be set _before_ any singletons are used and must only be set once. + public static var fileSystemThreadCountSuggestion: Int { + set { + Self.userSetSingletonThreadCount(rawStorage: globalRawSuggestedFileSystemThreadCount, userValue: newValue) + } + + get { + return Self.getTrustworthyThreadCount( + rawStorage: globalRawSuggestedFileSystemThreadCount, + environmentVariable: "NIO_SINGLETON_FILESYSTEM_THREAD_COUNT" + ) + } + } + + // Copied from NIOCore/GlobalSingletons.swift + private static func userSetSingletonThreadCount(rawStorage: ManagedAtomic, userValue: Int) { + precondition(userValue > 0, "illegal value: needs to be strictly positive") + + // The user is trying to set it. We can only do this if the value is at 0 and we will set the + // negative value. So if the user wants `5`, we will set `-5`. Once it's used (set getter), it'll be upped + // to 5. + let (exchanged, _) = rawStorage.compareExchange(expected: 0, desired: -userValue, ordering: .relaxed) + guard exchanged else { + fatalError( + """ + Bug in user code: Global singleton suggested loop/thread count has been changed after \ + user or has been changed more than once. Either is an error, you must set this value very early \ + and only once. + """ + ) + } + } + + // Copied from NIOCore/GlobalSingletons.swift + private static func validateTrustedThreadCount(_ threadCount: Int) { + assert( + threadCount > 0, + "BUG IN NIO, please report: negative suggested loop/thread count: \(threadCount)" + ) + assert( + threadCount <= 1024, + "BUG IN NIO, please report: overly big suggested loop/thread count: \(threadCount)" + ) + } + + // Copied from NIOCore/GlobalSingletons.swift + private static func getTrustworthyThreadCount(rawStorage: ManagedAtomic, environmentVariable: String) -> Int { + let returnedValueUnchecked: Int + + let rawSuggestion = rawStorage.load(ordering: .relaxed) + switch rawSuggestion { + case 0: // == 0 + // Not set by user, not yet finalised, let's try to get it from the env var and fall back to + // `System.coreCount`. + let envVarString = getenv(environmentVariable).map { String(cString: $0) } + returnedValueUnchecked = envVarString.flatMap(Int.init) ?? System.coreCount + case .min..<0: // < 0 + // Untrusted and unchecked user value. Let's invert and then sanitise/check. + returnedValueUnchecked = -rawSuggestion + case 1 ... .max: // > 0 + // Trustworthy value that has been evaluated and sanitised before. + let returnValue = rawSuggestion + Self.validateTrustedThreadCount(returnValue) + return returnValue + default: + // Unreachable + preconditionFailure() + } + + // Can't have fewer than 1, don't want more than 1024. + let returnValue = max(1, min(1024, returnedValueUnchecked)) + Self.validateTrustedThreadCount(returnValue) + + // Store it for next time. + let (exchanged, _) = rawStorage.compareExchange( + expected: rawSuggestion, + desired: returnValue, + ordering: .relaxed + ) + if !exchanged { + // We lost the race, this must mean it has been concurrently set correctly so we can safely recurse + // and try again. + return Self.getTrustworthyThreadCount(rawStorage: rawStorage, environmentVariable: environmentVariable) + } + return returnValue + } +} + +// DO NOT TOUCH THIS DIRECTLY, use `userSetSingletonThreadCount` and `getTrustworthyThreadCount`. +private let globalRawSuggestedFileSystemThreadCount = ManagedAtomic(0) + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +private let globalFileSystem: FileSystem = { + guard NIOSingletons.singletonsEnabledSuggestion else { + fatalError( + """ + Cannot create global singleton FileSystem thread pool because the global singletons \ + have been disabled by setting `NIOSingletons.singletonsEnabledSuggestion = false` + """ + ) + } + + let threadCount = NIOSingletons.fileSystemThreadCountSuggestion + return FileSystem(executor: .runningAsync(numberOfThreads: threadCount)) +}() + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension NIOSingletons { + /// Returns a shared global instance of the ``FileSystem``. + /// + /// The file system executes blocking work in a thread pool which defaults to having two + /// threads. This can be modified by `fileSystemThreadCountSuggestion` or by + /// setting the `NIO_SINGLETON_FILESYSTEM_THREAD_COUNT` environment variable. + public static var fileSystem: FileSystem { globalFileSystem } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension FileSystemProtocol where Self == FileSystem { + /// A global shared instance of ``FileSystem``. + public static var shared: FileSystem { + return FileSystem.shared + } +} + +/// Provides temporary scoped access to a ``FileSystem`` with the given number of threads. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public func withFileSystem( + numberOfThreads: Int, + _ body: (FileSystem) async throws -> R +) async throws -> R { + let fileSystem = await FileSystem(numberOfThreads: numberOfThreads) + return try await withUncancellableTearDown { + try await body(fileSystem) + } tearDown: { _ in + await fileSystem.shutdown() + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension FileSystem { + /// Opens `path` for reading and returns ``ReadFileHandle`` or ``FileSystemError``. + private func _openFile( + forReadingAt path: FilePath, + options: OpenOptions.Read + ) -> Result { + return SystemFileHandle.syncOpen( + atPath: path, + mode: .readOnly, + options: options.descriptorOptions, + permissions: nil, + transactionalIfPossible: false, + executor: self.executor + ).map { + ReadFileHandle(wrapping: $0) + } + } + + /// Opens `path` for writing and returns ``WriteFileHandle`` or ``FileSystemError``. + private func _openFile( + forWritingAt path: FilePath, + options: OpenOptions.Write + ) -> Result { + return SystemFileHandle.syncOpen( + atPath: path, + mode: .writeOnly, + options: options.descriptorOptions, + permissions: options.permissionsForRegularFile, + transactionalIfPossible: options.newFile?.transactionalCreation ?? false, + executor: self.executor + ).map { + WriteFileHandle(wrapping: $0) + } + } + + /// Opens `path` for reading and writing and returns ``ReadWriteFileHandle`` or ``FileSystemError``. + private func _openFile( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write + ) -> Result { + return SystemFileHandle.syncOpen( + atPath: path, + mode: .readWrite, + options: options.descriptorOptions, + permissions: options.permissionsForRegularFile, + transactionalIfPossible: options.newFile?.transactionalCreation ?? false, + executor: self.executor + ).map { + ReadWriteFileHandle(wrapping: $0) + } + } + + /// Opens the directory at `path` and returns ``DirectoryFileHandle`` or ``FileSystemError``. + private func _openDirectory( + at path: FilePath, + options: OpenOptions.Directory + ) -> Result { + return SystemFileHandle.syncOpen( + atPath: path, + mode: .readOnly, + options: options.descriptorOptions, + permissions: nil, + transactionalIfPossible: false, + executor: self.executor + ).map { + DirectoryFileHandle(wrapping: $0) + } + } + + /// Creates a directory at `fullPath`, potentially creating other directories along the way. + private func _createDirectory( + at fullPath: FilePath, + withIntermediateDirectories createIntermediateDirectories: Bool, + permissions: FilePermissions + ) -> Result { + // Logic, assuming we are creating intermediate directories: try creating the directory, + // if it fails with ENOENT (no such file or directory) then drop the last component and + // append it to a buffer. Repeat until the path is empty meaning we cannot create the + // directory, or we succeed in which case we can append build up our original path + // creating directories one at a time. + var droppedComponents: [FilePath.Component] = [] + var path = fullPath + + // Normalize the path to remove any '..' which may not be necessary. + path.lexicallyNormalize() + + if path.isEmpty { + let error = FileSystemError( + code: .invalidArgument, + message: "Path of directory to create must not be empty.", + cause: nil, + location: .here() + ) + return .failure(error) + } + + loop: while true { + switch Syscall.mkdir(at: path, permissions: permissions) { + case .success: + break loop + + case let .failure(errno): + guard createIntermediateDirectories, errno == .noSuchFileOrDirectory else { + return .failure(.mkdir(errno: errno, path: path, location: .here())) + } + + // Drop the last component and loop around. + if let component = path.lastComponent { + path.removeLastComponent() + droppedComponents.append(component) + } else { + // Should only happen if the path is empty or contains just the root. + return .failure(.mkdir(errno: errno, path: path, location: .here())) + } + } + } + + // Successfully made a directory, construct its children. + while let subdirectory = droppedComponents.popLast() { + path.append(subdirectory) + switch Syscall.mkdir(at: path, permissions: permissions) { + case .success: + continue + case let .failure(errno): + return .failure(.mkdir(errno: errno, path: path, location: .here())) + } + } + + return .success(()) + } + + /// Returns info about the file at `path`. + private func _info( + forFileAt path: FilePath, + infoAboutSymbolicLink: Bool + ) -> Result { + let result: Result + if infoAboutSymbolicLink { + result = Syscall.lstat(path: path) + } else { + result = Syscall.stat(path: path) + } + + return result.map { + FileInfo(platformSpecificStatus: $0) + }.flatMapError { errno in + if errno == .noSuchFileOrDirectory { + return .success(nil) + } else { + let name = infoAboutSymbolicLink ? "lstat" : "stat" + return .failure(.stat(name, errno: errno, path: path, location: .here())) + } + } + } + + /// Copies the directory from `sourcePath` to `destinationPath`. + private func copyDirectory( + from sourcePath: FilePath, + to destinationPath: FilePath, + shouldProceedAfterError: @escaping @Sendable ( + _ entry: DirectoryEntry, + _ error: Error + ) async throws -> Void, + shouldCopyFile: @escaping @Sendable ( + _ source: FilePath, + _ destination: FilePath + ) async -> Bool + ) async throws { + // Strategy: copy regular files and symbolic links while the directory is open; defer + // copying directories until after the source directory has been closed to avoid consuming + // too many file descriptors. + let directoriesToCopy = try await self.withDirectoryHandle(atPath: sourcePath) { dir in + // Grab the directory info to copy permissions. + let info = try await dir.info() + try await self.createDirectory( + at: destinationPath, + withIntermediateDirectories: false, + permissions: info.permissions + ) + + // Copy over extended attributes, if any exist. + do { + let attributes = try await dir.attributeNames() + + if !attributes.isEmpty { + try await self.withDirectoryHandle(atPath: destinationPath) { destinationDir in + for attribute in attributes { + let value = try await dir.valueForAttribute(attribute) + try await destinationDir.updateValueForAttribute( + value, + attribute: attribute + ) + } + } + } + } catch let error as FileSystemError where error.code == .unsupported { + // Not all file systems support extended attributes. Swallow errors which indicate + // that is the case. + () + } + + // Build a list of directories to copy over. Do this after closing the current + // directory to avoid using too many descriptors. + var directoriesToCopy = [(from: FilePath, to: FilePath)]() + + for try await batch in dir.listContents().batched() { + for entry in batch { + let entrySource = entry.path + let entryDestination = destinationPath.appending(entry.name) + + switch entry.type { + case .regular: + if await shouldCopyFile(entrySource, entryDestination) { + do { + try await self.copyRegularFile( + from: entry.path, + to: destinationPath.appending(entry.name) + ) + } catch { + try await shouldProceedAfterError(entry, error) + } + } + + case .symlink: + if await shouldCopyFile(entrySource, entryDestination) { + do { + try await self.copySymbolicLink( + from: entry.path, + to: destinationPath.appending(entry.name) + ) + } catch { + try await shouldProceedAfterError(entry, error) + } + } + + case .directory: + directoriesToCopy.append((entrySource, entryDestination)) + + default: + let error = FileSystemError( + code: .unsupported, + message: """ + Can't copy '\(entrySource)' of type '\(entry.type)'; only regular \ + files, symbolic links and directories can be copied. + """, + cause: nil, + location: .here() + ) + + try await shouldProceedAfterError(entry, error) + } + } + } + + return directoriesToCopy + } + + for entry in directoriesToCopy { + if await shouldCopyFile(entry.from, entry.to) { + try await self.copyDirectory( + from: entry.from, + to: entry.to, + shouldProceedAfterError: shouldProceedAfterError, + shouldCopyFile: shouldCopyFile + ) + } + } + } + + private func copyRegularFile( + from sourcePath: FilePath, + to destinationPath: FilePath + ) async throws { + try await self.executor.execute { + try self._copyRegularFile(from: sourcePath, to: destinationPath).get() + } + } + + private func _copyRegularFile( + from sourcePath: FilePath, + to destinationPath: FilePath + ) -> Result { + func makeOnUnavailableError( + path: FilePath, + location: FileSystemError.SourceLocation + ) -> FileSystemError { + return FileSystemError( + code: .closed, + message: "Can't copy '\(sourcePath)' to '\(destinationPath)', '\(path)' is closed.", + cause: nil, + location: location + ) + } + + let openSourceResult = self._openFile( + forReadingAt: sourcePath, + options: OpenOptions.Read(followSymbolicLinks: true) + ).mapError { + FileSystemError( + message: "Can't copy '\(sourcePath)', it couldn't be opened.", + wrapping: $0 + ) + } + + let source: ReadFileHandle + switch openSourceResult { + case let .success(handle): + source = handle + case let .failure(error): + return .failure(error) + } + + defer { + _ = source.fileHandle.systemFileHandle.sendableView._close(materialize: true) + } + + let sourceInfo: FileInfo + switch source.fileHandle.systemFileHandle.sendableView._info() { + case let .success(info): + sourceInfo = info + case let .failure(error): + return .failure(error) + } + + let options = OpenOptions.Write( + existingFile: .none, + newFile: OpenOptions.NewFile( + permissions: sourceInfo.permissions, + transactionalCreation: false + ) + ) + + let openDestinationResult = self._openFile( + forWritingAt: destinationPath, + options: options + ).mapError { + FileSystemError( + message: "Can't copy '\(sourcePath)' as '\(destinationPath)' couldn't be opened.", + wrapping: $0 + ) + } + + let destination: WriteFileHandle + switch openDestinationResult { + case let .success(handle): + destination = handle + case let .failure(error): + return .failure(error) + } + + let copyResult: Result + copyResult = source.fileHandle.systemFileHandle.sendableView._withUnsafeDescriptorResult { sourceFD in + destination.fileHandle.systemFileHandle.sendableView._withUnsafeDescriptorResult { destinationFD in + #if canImport(Darwin) + // COPYFILE_CLONE clones the file if possible and will fallback to doing a copy. + // COPYFILE_ALL is shorthand for: + // COPYFILE_STAT | COPYFILE_ACL | COPYFILE_XATTR | COPYFILE_DATA + let flags = copyfile_flags_t(COPYFILE_CLONE) | copyfile_flags_t(COPYFILE_ALL) + return Libc.fcopyfile( + from: sourceFD, + to: destinationFD, + state: nil, + flags: flags + ).mapError { errno in + FileSystemError.fcopyfile( + errno: errno, + from: sourcePath, + to: destinationPath, + location: .here() + ) + } + #elseif canImport(Glibc) + var offset = 0 + + while offset < sourceInfo.size { + // sendfile(2) limits writes to 0x7ffff000 in size + let size = min(Int(sourceInfo.size) - offset, 0x7fff_f000) + let result = Syscall.sendfile( + to: destinationFD, + from: sourceFD, + offset: offset, + size: size + ).mapError { errno in + FileSystemError.sendfile( + errno: errno, + from: sourcePath, + to: destinationPath, + location: .here() + ) + } + + switch result { + case let .success(bytesCopied): + offset += bytesCopied + case let .failure(error): + return .failure(error) + } + } + return .success(()) + #endif + } onUnavailable: { + makeOnUnavailableError(path: destinationPath, location: .here()) + } + } onUnavailable: { + makeOnUnavailableError(path: sourcePath, location: .here()) + } + + let closeResult = destination.fileHandle.systemFileHandle.sendableView._close(materialize: true) + return copyResult.flatMap { closeResult } + } + + private func copySymbolicLink( + from sourcePath: FilePath, + to destinationPath: FilePath + ) async throws { + try await self.executor.execute { + try self._copySymbolicLink(from: sourcePath, to: destinationPath).get() + } + } + + private func _copySymbolicLink( + from sourcePath: FilePath, + to destinationPath: FilePath + ) -> Result { + self._destinationOfSymbolicLink(at: sourcePath).flatMap { linkDestination in + self._createSymbolicLink(at: destinationPath, withDestination: linkDestination) + } + } + + @_spi(Testing) + public func removeOneItem( + at path: FilePath, + function: String = #function, + file: String = #fileID, + line: Int = #line + ) async throws -> Int { + try await self.executor.execute { + switch Libc.remove(path) { + case .success: + return 1 + case .failure(.noSuchFileOrDirectory): + return 0 + case .failure(let errno): + throw FileSystemError.remove( + errno: errno, + path: path, + location: .init(function: function, file: file, line: line) + ) + } + } + } + + private enum MoveResult { + case moved + case differentLogicalDevices + } + + private func _moveItem( + at sourcePath: FilePath, + to destinationPath: FilePath + ) -> Result { + // Check that the destination doesn't exist. 'rename' will remove it otherwise! + switch self._info(forFileAt: destinationPath, infoAboutSymbolicLink: true) { + case .success(.none): + // Doens't exist: continue + () + + case .success(.some): + let fileSystemError = FileSystemError( + code: .fileAlreadyExists, + message: """ + Unable to move '\(sourcePath)' to '\(destinationPath)', the destination path \ + already exists. + """, + cause: nil, + location: .here() + ) + return .failure(fileSystemError) + + case let .failure(error): + let fileSystemError = FileSystemError( + message: """ + Unable to move '\(sourcePath)' to '\(destinationPath)', could not determine \ + whether the destination path exists or not. + """, + wrapping: error + ) + return .failure(fileSystemError) + } + + switch Syscall.rename(from: sourcePath, to: destinationPath) { + case .success: + return .success(.moved) + + case .failure(.improperLink): + // The two paths are on different logical devices; copy and then remove the + // original. + return .success(.differentLogicalDevices) + + case let .failure(errno): + let error = FileSystemError.rename( + errno: errno, + oldName: sourcePath, + newName: destinationPath, + location: .here() + ) + return .failure(error) + } + } + + private func parseTemporaryDirectoryTemplate( + _ template: FilePath + ) -> Result<(FilePath, String, Int), FileSystemError> { + // Check whether template is valid (i.e. has a `lastComponent`). + guard let lastComponentPath = template.lastComponent else { + let fileSystemError = FileSystemError( + code: .invalidArgument, + message: """ + Can't create temporary directory, the template ('\(template)') is invalid. \ + The template should be a file path ending in at least three 'X's. + """, + cause: nil, + location: .here() + ) + return .failure(fileSystemError) + } + + let lastComponent = lastComponentPath.string + + // Finding the index of the last non-'X' character in `lastComponent.string` and advancing it by one. + let prefix: String + var index = lastComponent.lastIndex(where: { $0 != "X" }) + if index != nil { + lastComponent.formIndex(after: &(index!)) + prefix = String(lastComponent[..= 3 else { + let fileSystemError = FileSystemError( + code: .invalidArgument, + message: """ + Can't create temporary directory, the template ('\(template)') is invalid. \ + The template must end in at least three 'X's. + """, + cause: nil, + location: .here() + ) + return .failure(fileSystemError) + } + + return .success((template.removingLastComponent(), prefix, suffixLength)) + } + + private func _createTemporaryDirectory( + template: FilePath + ) -> Result { + let prefix: String + let root: FilePath + let suffixLength: Int + + let parseResult = self.parseTemporaryDirectoryTemplate(template) + switch parseResult { + case let .success((parseRoot, parsePrefix, parseSuffixLength)): + root = parseRoot + prefix = parsePrefix + suffixLength = parseSuffixLength + case let .failure(error): + return .failure(error) + } + + for _ in 1...16 { + let name = prefix + String(randomAlphaNumericOfLength: suffixLength) + + // Trying to create the directory. + let finalPath = root.appending(name) + let createDirectoriesResult = self._createDirectory( + at: finalPath, + withIntermediateDirectories: true, + permissions: FilePermissions.defaultsForDirectory + ) + switch createDirectoriesResult { + case .success: + return .success(finalPath) + case let .failure(error): + if let systemCallError = error.cause as? FileSystemError.SystemCallError { + switch systemCallError.errno { + // If the file at the generated path already exists, we generate a new file path. + case .fileExists, .isDirectory: + break + default: + let fileSystemError = FileSystemError( + message: "Unable to create temporary directory '\(template)'.", + wrapping: error + ) + return .failure(fileSystemError) + } + } else { + let fileSystemError = FileSystemError( + message: "Unable to create temporary directory '\(template)'.", + wrapping: error + ) + return .failure(fileSystemError) + } + } + } + let fileSystemError = FileSystemError( + code: .unknown, + message: """ + Could not create a temporary directory from the provided template ('\(template)'). \ + Try adding more 'X's at the end of the template. + """, + cause: nil, + location: .here() + ) + return .failure(fileSystemError) + } + + func _createSymbolicLink( + at linkPath: FilePath, + withDestination destinationPath: FilePath + ) -> Result { + return Syscall.symlink(to: destinationPath, from: linkPath).mapError { errno in + FileSystemError.symlink( + errno: errno, + link: linkPath, + target: destinationPath, + location: .here() + ) + } + } + + func _destinationOfSymbolicLink(at path: FilePath) -> Result { + Syscall.readlink(at: path).mapError { errno in + FileSystemError.readlink( + errno: errno, + path: path, + location: .here() + ) + } + } +} diff --git a/Sources/NIOFileSystem/FileSystemError+Syscall.swift b/Sources/NIOFileSystem/FileSystemError+Syscall.swift new file mode 100644 index 0000000000..fcc0a87ad6 --- /dev/null +++ b/Sources/NIOFileSystem/FileSystemError+Syscall.swift @@ -0,0 +1,1026 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +extension FileSystemError { + /// Creates a ``FileSystemError`` by constructing a ``SystemCallError`` as the cause. + internal init( + code: Code, + message: String, + systemCall: String, + errno: Errno, + location: SourceLocation + ) { + self.init( + code: code, + message: message, + cause: SystemCallError(systemCall: systemCall, errno: errno), + location: location + ) + } +} + +extension FileSystemError { + /// Create a file system error appropriate for the `stat`/`lstat`/`fstat` system calls. + @_spi(Testing) + public static func stat( + _ name: String, + errno: Errno, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + + // See: 'man 2 fstat' + switch errno { + case .badFileDescriptor: + code = .closed + message = "Unable to get information about '\(path)', the file is closed." + default: + code = .unknown + message = "Unable to get information about '\(path)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: name, + errno: errno, + location: location + ) + } + + @_spi(Testing) + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public static func fchmod( + operation: SystemFileHandle.UpdatePermissionsOperation, + operand: FilePermissions, + permissions: FilePermissions, + errno: Errno, + path: FilePath, + location: SourceLocation + ) -> Self { + let message: String + let code: FileSystemError.Code + + // See: 'man 2 fchmod' + switch errno { + case .badFileDescriptor: + code = .closed + message = "Could not \(operation) permissions '\(operand)', '\(path)' is closed." + + case .invalidArgument: + // Permissions are invalid so display the raw value in octal. + let rawPermissions = String(permissions.rawValue, radix: 0o10) + let op: String + switch operation { + case .set: + op = "set" + case .add: + op = "added" + case .remove: + op = "removed" + } + code = .invalidArgument + message = """ + Invalid permissions ('\(rawPermissions)') could not be \(op) for file '\(path)'. + """ + + case .notPermitted: + code = .permissionDenied + message = """ + Not permitted to \(operation) permissions '\(operand)' for file '\(path)', \ + the effective user ID does not match the owner of the file and the effective \ + user ID is not the super-user. + """ + + default: + code = .unknown + message = "Could not \(operation) permissions '\(operand)' to '\(path)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "fchmod", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func flistxattr(errno: Errno, path: FilePath, location: SourceLocation) -> Self { + let code: FileSystemError.Code + let message: String + + switch errno { + case .badFileDescriptor: + code = .closed + message = "Could not list extended attributes, '\(path)' is closed." + + case .notSupported: + code = .unsupported + message = "Extended attributes are disabled or not supported by the filesystem." + + case .notPermitted: + code = .unsupported + message = "Extended attributes are not supported by '\(path)'." + + case .permissionDenied: + code = .permissionDenied + message = "Not permitted to list extended attributes for '\(path)'." + + default: + code = .unknown + message = "Could not to list extended attributes for '\(path)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "flistxattr", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func fgetxattr( + attribute name: String, + errno: Errno, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + + switch errno { + case .badFileDescriptor: + code = .closed + message = """ + Could not get value for extended attribute ('\(name)'), '\(path)' is closed. + """ + case .notSupported: + code = .unsupported + message = "Extended attributes are disabled or not supported by the filesystem." + #if canImport(Darwin) + case .fileNameTooLong: + code = .invalidArgument + message = """ + Length of UTF-8 extended attribute name (\(name.utf8.count)) is greater \ + than the limit (\(XATTR_MAXNAMELEN)). Use a shorter attribute name. + """ + #endif + default: + code = .unknown + message = "Could not get value for extended attribute ('\(name)') for '\(path)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "fgetxattr", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func fsetxattr( + attribute name: String, + errno: Errno, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + + // See: 'man 2 fsetxattr' + switch errno { + case .badFileDescriptor: + code = .closed + message = """ + Could not set value for extended attribute ('\(name)'), '\(path)' is closed. + """ + + case .notSupported: + code = .unsupported + message = """ + Extended attributes are disabled or not supported by the filesystem. + """ + + #if canImport(Darwin) + case .fileNameTooLong: + code = .invalidArgument + message = """ + Length of UTF-8 extended attribute name (\(name.utf8.count)) is greater \ + than the limit limit (\(XATTR_MAXNAMELEN)). Use a shorter attribute \ + name. + """ + #endif + + case .invalidArgument: + code = .invalidArgument + message = """ + Extended attribute name ('\(name)') must be a valid UTF-8 string. + """ + default: + code = .unknown + message = """ + Could not set value for extended attribute ('\(name)') for '\(path)'. + """ + } + + return FileSystemError( + code: code, + message: message, + systemCall: "fsetxattr", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func fremovexattr( + attribute name: String, + errno: Errno, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + + // See: 'man 2 fremovexattr' + switch errno { + case .badFileDescriptor: + code = .closed + message = "Could not remove extended attribute ('\(name)'), '\(path)' is closed." + + case .notSupported: + code = .unsupported + message = "Extended attributes are disabled or not supported by the filesystem." + + #if canImport(Darwin) + case .fileNameTooLong: + code = .invalidArgument + message = """ + Length of UTF-8 extended attribute name (\(name.utf8.count)) is greater \ + than the limit (\(XATTR_MAXNAMELEN)). Use a shorter attribute name. + """ + #endif + + default: + code = .unknown + message = "Could not remove extended attribute ('\(name)') from '\(path)'" + } + + return FileSystemError( + code: code, + message: message, + systemCall: "fremovexattr", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func fsync( + errno: Errno, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + + switch errno { + case .badFileDescriptor: + code = .closed + message = "Could not synchronize file, '\(path)' is closed." + case .ioError: + code = .io + message = "An I/O error occurred while synchronizing '\(path)'." + default: + code = .unknown + message = "Could not synchronize file '\(path)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "fsync", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func dup(error: Error, path: FilePath, location: SourceLocation) -> Self { + let code: FileSystemError.Code + let message: String + let cause: Error + + if let errno = error as? Errno { + switch errno { + case .badFileDescriptor: + code = .closed + message = "Unable to duplicate descriptor of closed handle for '\(path)'." + default: + code = .unknown + message = "Unable to duplicate descriptor of handle for '\(path)'." + } + cause = SystemCallError(systemCall: "dup", errno: errno) + } else { + code = .unknown + message = "Unable to duplicate descriptor of handle for '\(path)'." + cause = error + } + + return FileSystemError( + code: code, + message: message, + cause: cause, + location: location + ) + } + + @_spi(Testing) + public static func ftruncate(error: Error, path: FilePath, location: SourceLocation) -> Self { + let code: FileSystemError.Code + let message: String + let cause: Error + + if let errno = error as? Errno { + switch errno { + case .badFileDescriptor: + code = .closed + message = "Can't resize '\(path)', it's closed." + case .fileTooLarge: + code = .invalidArgument + message = "The requested size for '\(path)' is too large." + case .invalidArgument: + code = .invalidArgument + message = "The requested size for '\(path)' is negative, therefore invalid." + default: + code = .unknown + message = "Unable to resize '\(path)'." + } + cause = SystemCallError(systemCall: "ftruncate", errno: errno) + } else { + code = .unknown + message = "Unable to resize '\(path)'." + cause = error + } + + return FileSystemError( + code: code, + message: message, + cause: cause, + location: location + ) + } + + @_spi(Testing) + public static func close(error: Error, path: FilePath, location: SourceLocation) -> Self { + let code: FileSystemError.Code + let message: String + let cause: Error + + // See: 'man 2 close' + if let errno = error as? Errno { + switch errno { + case .badFileDescriptor: + code = .closed + message = "File already closed or file descriptor was invalid ('\(path)')." + case .ioError: + code = .io + message = "I/O error during close, some writes to '\(path)' may have failed." + default: + code = .unknown + message = "Error closing file '\(path)'." + } + cause = SystemCallError(systemCall: "close", errno: errno) + } else { + code = .unknown + message = "Error closing file '\(path)'." + cause = error + } + + return FileSystemError( + code: code, + message: message, + cause: cause, + location: location + ) + } + + @_spi(Testing) + public enum ReadSyscall: String { + case read + case pread + } + + @_spi(Testing) + public static func read( + usingSyscall syscall: ReadSyscall, + error: Error, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + let cause: Error + + // We expect an Errno as 'swift-system' uses result types under-the-hood, + // but don't require that in case something changes. + if let errno = error as? Errno { + switch errno { + case .badFileDescriptor: + code = .closed + message = "Could not read from closed file '\(path)'." + case .ioError: + code = .io + message = """ + Could not read from file ('\(path)'); an I/O error occurred while reading \ + from the file system. + """ + case .illegalSeek: + code = .unsupported + message = "File is not seekable: '\(path)'." + default: + code = .unknown + message = "Could not read from file '\(path)'." + } + cause = SystemCallError(systemCall: syscall.rawValue, errno: errno) + } else { + code = .unknown + message = "Could not read from file '\(path)'." + cause = error + } + + return FileSystemError( + code: code, + message: message, + cause: cause, + location: location + ) + } + + @_spi(Testing) + public enum WriteSyscall: String { + case write + case pwrite + } + + @_spi(Testing) + public static func write( + usingSyscall syscall: WriteSyscall, + error: Error, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + let cause: Error + + // We expect an Errno as 'swift-system' uses result types under-the-hood, + // but don't require that in case something changes. + if let errno = error as? Errno { + switch errno { + case .badFileDescriptor: + code = .closed + message = "Could not write to closed file '\(path)'." + case .ioError: + code = .io + message = """ + Could not write to file ('\(path)'); an I/O error occurred while writing to \ + the file system. + """ + case .illegalSeek: + code = .unsupported + message = "File is not seekable: '\(path)'." + default: + code = .unknown + message = "Could not write to file." + } + cause = SystemCallError(systemCall: syscall.rawValue, errno: errno) + } else { + code = .unknown + message = "Could not write to file." + cause = error + } + + return FileSystemError( + code: code, + message: message, + cause: cause, + location: location + ) + } + + @_spi(Testing) + public static func fdopendir(errno: Errno, path: FilePath, location: SourceLocation) -> Self { + return FileSystemError( + code: .unknown, + message: "Unable to open directory stream for '\(path)'.", + systemCall: "fdopendir", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func readdir(errno: Errno, path: FilePath, location: SourceLocation) -> Self { + return FileSystemError( + code: .unknown, + message: "Unable to read directory stream for '\(path)'.", + systemCall: "readdir", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func ftsRead(errno: Errno, path: FilePath, location: SourceLocation) -> Self { + return FileSystemError( + code: .unknown, + message: "Unable to read FTS stream for '\(path)'.", + systemCall: "fts_read", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func open( + _ name: String, + error: Error, + path: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + let cause: Error + + if let errno = error as? Errno { + switch errno { + case .badFileDescriptor: + code = .closed + message = "Unable to open file at path '\(path)', the descriptor is closed." + case .permissionDenied: + code = .permissionDenied + message = "Unable to open file at path '\(path)', permissions denied." + case .fileExists: + code = .fileAlreadyExists + message = """ + Unable to create file at path '\(path)', no existing file options were set \ + which implies that no file should exist but a file already exists at the \ + specified path. + """ + case .ioError: + code = .io + message = """ + Unable to create file at path '\(path)', an I/O error occurred while \ + creating the file. + """ + case .tooManyOpenFiles: + code = .unavailable + message = """ + Unable to open file at path '\(path)', too many file descriptors are open. + """ + case .noSuchFileOrDirectory: + code = .notFound + message = """ + Unable to open or create file at path '\(path)', either a component of the \ + path did not exist or the named file to be opened did not exist. + """ + case .notDirectory: + code = .notFound + message = """ + Unable to open or create file at path '\(path)', an intermediate component of \ + the path was not a directory. + """ + case .tooManySymbolicLinkLevels: + code = .invalidArgument + message = """ + Can't open file at path '\(path)', the target is a symbolic link and \ + 'followSymbolicLinks' was set to 'false'. + """ + default: + code = .unknown + message = "Unable to open file at path '\(path)'." + } + cause = SystemCallError(systemCall: name, errno: errno) + } else { + code = .unknown + message = "Unable to open file at path '\(path)'." + cause = error + } + + return FileSystemError(code: code, message: message, cause: cause, location: location) + } + + @_spi(Testing) + public static func mkdir(errno: Errno, path: FilePath, location: SourceLocation) -> Self { + let code: Code + let message: String + + switch errno { + case .permissionDenied: + code = .permissionDenied + message = """ + Insufficient permissions to create a directory at '\(path)'. Search permissions \ + denied for a component of the path or write permission denied for the parent \ + directory. + """ + case .isDirectory: + code = .invalidArgument + message = "Can't create directory, '\(path)' is the root directory." + case .fileExists: + code = .fileAlreadyExists + message = "Can't create directory, the pathname '\(path)' already exists." + case .notDirectory: + code = .invalidArgument + message = "Can't create directory, a component of '\(path)' is not a directory." + case .noSuchFileOrDirectory: + code = .invalidArgument + message = """ + Can't create directory, a component of '\(path)' does not exist. Ensure all \ + parent directories exist or set 'withIntermediateDirectories' to 'true' when \ + calling 'createDirectory(at:withIntermediateDirectories:permissions)'. + """ + case .ioError: + code = .io + message = "An I/O error occurred when the directory at '\(path)'." + default: + code = .unknown + message = "" + } + + return FileSystemError( + code: code, + message: message, + systemCall: "mkdir", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func rename( + errno: Errno, + oldName: FilePath, + newName: FilePath, + location: SourceLocation + ) -> Self { + let code: Code + let message: String + + switch errno { + case .permissionDenied: + code = .permissionDenied + message = """ + Insufficient permissions to rename '\(oldName)' to '\(newName)'. Search \ + permissions were denied on a component of either path, or write permissions were \ + denied on the parent directory of either path. + """ + case .fileExists: + code = .fileAlreadyExists + message = "Can't rename '\(oldName)' to '\(newName)' as it already exists." + case .invalidArgument: + code = .invalidArgument + if oldName == "." || oldName == ".." { + message = """ + Can't rename '\(oldName)' to '\(newName)', '.' and '..' can't be renamed. + """ + } else { + message = """ + Can't rename '\(oldName)', it may be a parent directory of '\(newName)'. + """ + } + case .noSuchFileOrDirectory: + code = .notFound + message = """ + Can't rename '\(oldName)' to '\(newName)', a component of '\(oldName)' does \ + not exist. + """ + case .ioError: + code = .io + message = """ + Can't rename '\(oldName)' to '\(newName)', an I/O error occurred while making \ + or updating a directory entry. + """ + default: + code = .unknown + message = "Can't rename '\(oldName)' to '\(newName)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "rename", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func remove(errno: Errno, path: FilePath, location: SourceLocation) -> Self { + let code: Code + let message: String + + switch errno { + case .permissionDenied: + code = .permissionDenied + message = """ + Insufficient permissions to remove '\(path)'. Search permissions denied \ + on a component of the path or write permission denied on the directory \ + containing the item to be removed. + """ + case .notPermitted: + code = .permissionDenied + message = """ + Insufficient permission to remove '\(path)', the effective user ID of the \ + process is not permitted to remove the file. + """ + case .resourceBusy: + code = .unavailable + message = """ + Can't remove '\(path)', it may be being used by another process or is the mount \ + point for a mounted file system. + """ + case .ioError: + code = .io + message = """ + Can't remove '\(path)', an I/O error occurred while deleting its directory entry. + """ + default: + code = .unknown + message = "Can't remove '\(path)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "remove", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func symlink( + errno: Errno, + link: FilePath, + target: FilePath, + location: SourceLocation + ) -> Self { + let code: Code + let message: String + + switch errno { + case .permissionDenied: + code = .permissionDenied + message = """ + Can't create symbolic link '\(link)' to '\(target)', write access to the \ + directory containing the link was denied or one of the directories in its path \ + denied search permissions. + """ + case .notPermitted: + code = .permissionDenied + message = """ + Can't create symbolic link '\(link)' to '\(target)', the file system \ + containing '\(link)' doesn't support the creation of symbolic links. + """ + case .fileExists: + code = .fileAlreadyExists + message = """ + Can't create symbolic link '\(link)' to '\(target)', '\(link)' already exists. + """ + case .noSuchFileOrDirectory: + code = .invalidArgument + message = """ + Can't create symbolic link '\(link)' to '\(target)', a component of '\(link)' \ + does not exist or is a dangling symbolic link. + """ + case .notDirectory: + code = .invalidArgument + message = """ + Can't create symbolic link '\(link)' to '\(target)', a component of '\(link)' \ + is not a directory. + """ + case .ioError: + code = .io + message = """ + Can't create symbolic link '\(link)' to '\(target)', an I/O error occurred. + """ + default: + code = .unknown + message = "Can't create symbolic link '\(link)' to '\(target)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "symlink", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func readlink(errno: Errno, path: FilePath, location: SourceLocation) -> Self { + let code: Code + let message: String + + switch errno { + case .permissionDenied: + code = .permissionDenied + message = """ + Can't read symbolic link at '\(path)'; search permission was denied for a \ + component in its prefix. + """ + case .invalidArgument: + code = .invalidArgument + message = """ + Can't read '\(path)'; it is not a symbolic link. + """ + case .ioError: + code = .io + message = """ + Can't read symbolic link at '\(path)'; an I/O error occurred. + """ + case .noSuchFileOrDirectory: + code = .notFound + message = """ + Can't read symbolic link, no file exists at '\(path)'. + """ + default: + code = .unknown + message = "Can't read symbolic link at '\(path)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "readlink", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func link( + errno: Errno, + from sourcePath: FilePath, + to destinationPath: FilePath, + location: SourceLocation + ) -> Self { + let code: FileSystemError.Code + let message: String + + // See: 'man 2 close' + switch errno { + case .fileExists: + code = .fileAlreadyExists + message = """ + Can't link '\(sourcePath)' to '\(destinationPath)', a file already exists \ + at '\(destinationPath)'. + """ + case .ioError: + code = .io + message = "I/O error while linking '\(sourcePath)' to '\(destinationPath)'." + default: + code = .unknown + message = "Error linking '\(sourcePath)' to '\(destinationPath)'." + } + + return FileSystemError( + code: code, + message: message, + cause: SystemCallError(systemCall: "linkat", errno: errno), + location: location + ) + } + + @_spi(Testing) + public static func getcwd(errno: Errno, location: SourceLocation) -> Self { + return FileSystemError( + code: .unavailable, + message: "Can't get current working directory.", + systemCall: "getcwd", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func confstr(name: String, errno: Errno, location: SourceLocation) -> Self { + return FileSystemError( + code: .unavailable, + message: "Can't get configuration value for '\(name)'.", + systemCall: "confstr", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func fcopyfile( + errno: Errno, + from sourcePath: FilePath, + to destinationPath: FilePath, + location: SourceLocation + ) -> Self { + let code: Code + let message: String + + switch errno { + case .notSupported: + code = .invalidArgument + message = """ + Can't copy file from '\(sourcePath)' to '\(destinationPath)', the item to copy is \ + not a directory, symbolic link or regular file. + """ + case .permissionDenied: + code = .permissionDenied + message = """ + Can't copy file, search permission was denied for a component of the path \ + prefix for the source ('\(sourcePath)') or destination ('\(destinationPath)'), \ + or write permission was denied for a component of the path prefix for the source. + """ + case .invalidArgument: + code = .invalidArgument + message = """ + Can't copy file from '\(sourcePath)' to '\(destinationPath)', the destination \ + path already exists. + """ + default: + code = .unknown + message = "Can't copy file from '\(sourcePath)' to '\(destinationPath)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "fcopyfile", + errno: errno, + location: location + ) + } + + @_spi(Testing) + public static func sendfile( + errno: Errno, + from sourcePath: FilePath, + to destinationPath: FilePath, + location: SourceLocation + ) -> FileSystemError { + let code: FileSystemError.Code + let message: String + + switch errno { + case .ioError: + code = .io + message = """ + An I/O error occurred while reading from '\(sourcePath)', can't copy to \ + '\(destinationPath)'. + """ + case .noMemory: + code = .io + message = """ + Insufficient memory to read from '\(sourcePath)', can't copy to \ + '\(destinationPath)'. + """ + default: + code = .unknown + message = "Can't copy file from '\(sourcePath)' to '\(destinationPath)'." + } + + return FileSystemError( + code: code, + message: message, + systemCall: "sendfile", + errno: errno, + location: location + ) + } +} diff --git a/Sources/NIOFileSystem/FileSystemError.swift b/Sources/NIOFileSystem/FileSystemError.swift new file mode 100644 index 0000000000..2ad674ebb4 --- /dev/null +++ b/Sources/NIOFileSystem/FileSystemError.swift @@ -0,0 +1,333 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +/// An error thrown as a result of interaction with the file system. +/// +/// All errors have a high-level ``FileSystemError/Code-swift.struct`` which identifies the domain +/// of the error. For example an operation performed on a ``FileHandleProtocol`` which has already been +/// closed will result in a ``FileSystemError/Code-swift.struct/closed`` error code. Errors also +/// include a message describing what went wrong and how to remedy it (if applicable). The +/// ``FileSystemError/message`` is not static and may include dynamic information such as the path +/// of the file for which the operation failed, for example. +/// +/// Errors may have a ``FileSystemError/cause``, an underlying error which caused the operation to +/// fail which may be platform specific. +public struct FileSystemError: Error, @unchecked Sendable { + // Note: @unchecked because we use a backing class for storage. + + private var storage: Storage + private mutating func ensureStorageIsUnique() { + if !isKnownUniquelyReferenced(&self.storage) { + self.storage = self.storage.copy() + } + } + + private final class Storage { + var code: Code + var message: String + var cause: Error? + var location: SourceLocation + + init(code: Code, message: String, cause: Error?, location: SourceLocation) { + self.code = code + self.message = message + self.cause = cause + self.location = location + } + + func copy() -> Self { + return Self( + code: self.code, + message: self.message, + cause: self.cause, + location: self.location + ) + } + } + + /// A high-level error code to provide broad a classification. + public var code: Code { + get { self.storage.code } + set { + self.ensureStorageIsUnique() + self.storage.code = newValue + } + } + + /// A message describing what went wrong and how it may be remedied. + public var message: String { + get { self.storage.message } + set { + self.ensureStorageIsUnique() + self.storage.message = newValue + } + } + + /// An underlying error which caused the operation to fail. This may include additional details + /// about the root cause of the failure. + public var cause: Error? { + get { self.storage.cause } + set { + self.ensureStorageIsUnique() + self.storage.cause = newValue + } + } + + /// The location from which this error was thrown. + public var location: SourceLocation { + get { self.storage.location } + set { + self.ensureStorageIsUnique() + self.storage.location = newValue + } + } + + public init( + code: Code, + message: String, + cause: Error?, + location: SourceLocation + ) { + self.storage = Storage(code: code, message: message, cause: cause, location: location) + } + + /// Creates a ``FileSystemError`` by wrapping the given `cause` and its location and code. + internal init(message: String, wrapping cause: FileSystemError) { + self.init(code: cause.code, message: message, cause: cause, location: cause.location) + } +} + +extension FileSystemError: CustomStringConvertible { + public var description: String { + if let cause = self.cause { + return "\(self.code): \(self.message) (\(cause))" + } else { + return "\(self.code): \(self.message)" + } + } +} + +extension FileSystemError: CustomDebugStringConvertible { + public var debugDescription: String { + if let cause = self.cause { + return """ + \(String(reflecting: self.code)): \(String(reflecting: self.message)) \ + (\(String(reflecting: cause))) + """ + } else { + return "\(String(reflecting: self.code)): \(String(reflecting: self.message))" + } + } +} + +extension FileSystemError { + private func detailedDescriptionLines() -> [String] { + // Build up a tree-like description of the error. This allows nested causes to be formatted + // correctly, especially when they are also FileSystemErrors. + // + // An example is: + // + // FileSystemError: Closed + // ├─ Reason: Unable to open file at path 'foo.swift', the descriptor is closed. + // ├─ Cause: 'openat' system call failed with '(9) Bad file descriptor'. + // └─ Source location: openFile(forReadingAt:_:) (FileSystem.swift:314) + var lines = [ + "FileSystemError: \(self.code)", + "├─ Reason: \(self.message)", + ] + + if let error = self.cause as? FileSystemError { + lines.append("├─ Cause:") + let causeLines = error.detailedDescriptionLines() + // We know this will never be empty. + lines.append("│ └─ \(causeLines.first!)") + lines.append(contentsOf: causeLines.dropFirst().map { "│ \($0)" }) + } else if let error = self.cause { + lines.append("├─ Cause: \(String(reflecting: error))") + } + + lines.append( + "└─ Source location: \(self.location.function) (\(self.location.file):\(self.location.line))" + ) + + return lines + } + + /// A detailed multi-line description of the error. + /// + /// - Returns: A multi-line description of the error. + public func detailedDescription() -> String { + return self.detailedDescriptionLines().joined(separator: "\n") + } +} + +extension FileSystemError { + /// A high level indication of the kind of error being thrown. + public struct Code: Hashable, Sendable, CustomStringConvertible { + private enum Wrapped: Hashable, Sendable, CustomStringConvertible { + case closed + case invalidArgument + case io + case permissionDenied + case notEmpty + case notFound + case resourceExhausted + case unavailable + case unknown + case unsupported + case fileAlreadyExists + + var description: String { + switch self { + case .closed: + return "Closed" + case .invalidArgument: + return "Invalid argument" + case .io: + return "I/O error" + case .permissionDenied: + return "Permission denied" + case .resourceExhausted: + return "Resource exhausted" + case .notEmpty: + return "Not empty" + case .notFound: + return "Not found" + case .unavailable: + return "Unavailable" + case .unknown: + return "Unknown" + case .unsupported: + return "Unsupported" + case .fileAlreadyExists: + return "File already exists" + } + } + } + + public var description: String { + String(describing: self.code) + } + + private var code: Wrapped + private init(_ code: Wrapped) { + self.code = code + } + + /// An operation on the file could not be performed because the file is closed + /// (or detached). + public static var closed: Self { + Self(.closed) + } + + /// A provided argument was not valid for the operation. + public static var invalidArgument: Self { + Self(.invalidArgument) + } + + /// An I/O error occurred. + public static var io: Self { + Self(.io) + } + + /// The caller did not have sufficient permission to perform the operation. + public static var permissionDenied: Self { + Self(.permissionDenied) + } + + /// A required resource was exhausted. + public static var resourceExhausted: Self { + Self(.resourceExhausted) + } + + /// The directory wasn't empty. + public static var notEmpty: Self { + Self(.notEmpty) + } + + /// The file could not be found. + public static var notFound: Self { + Self(.notFound) + } + + /// The file system is not currently available, for example if the underlying executor + /// is not running. + public static var unavailable: Self { + Self(.unavailable) + } + + /// The error is not known or may not have an appropriate classification. See + /// ``FileSystemError/cause`` for more information about the error. + public static var unknown: Self { + Self(.unknown) + } + + /// The operation is not supported or is not enabled. + public static var unsupported: Self { + Self(.unsupported) + } + + /// The file already exists. + public static var fileAlreadyExists: Self { + Self(.fileAlreadyExists) + } + } + + /// A location within source code. + public struct SourceLocation: Sendable, Hashable { + /// The function in which the error was thrown. + public var function: String + + /// The file in which the error was thrown. + public var file: String + + /// The line on which the error was thrown. + public var line: Int + + public init(function: String, file: String, line: Int) { + self.function = function + self.file = file + self.line = line + } + + internal static func here( + function: String = #function, + file: String = #fileID, + line: Int = #line + ) -> Self { + return SourceLocation(function: function, file: file, line: line) + } + } +} + +extension FileSystemError { + /// An error resulting from a system call. + public struct SystemCallError: Error, Hashable, CustomStringConvertible { + /// The name of the system call which produced the error. + public var systemCall: String + /// The errno set by the system call. + public var errno: Errno + + public init(systemCall: String, errno: Errno) { + self.systemCall = systemCall + self.errno = errno + } + + public var description: String { + "'\(self.systemCall)' system call failed with '(\(self.errno.rawValue)) \(self.errno)'." + } + } +} diff --git a/Sources/NIOFileSystem/FileSystemProtocol.swift b/Sources/NIOFileSystem/FileSystemProtocol.swift new file mode 100644 index 0000000000..8c5111a2c7 --- /dev/null +++ b/Sources/NIOFileSystem/FileSystemProtocol.swift @@ -0,0 +1,478 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +/// The interface for interacting with a file system. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public protocol FileSystemProtocol: Sendable { + /// The type of ``ReadableFileHandleProtocol`` to return when opening files for reading. + associatedtype ReadFileHandle: ReadableFileHandleProtocol + + /// The type of ``WritableFileHandleProtocol`` to return when opening files for writing. + associatedtype WriteFileHandle: WritableFileHandleProtocol + + /// The type of ``ReadableAndWritableFileHandleProtocol`` to return when opening files for reading and writing. + associatedtype ReadWriteFileHandle: ReadableAndWritableFileHandleProtocol + + /// The type of ``DirectoryFileHandleProtocol`` to return when opening directories. + associatedtype DirectoryFileHandle: DirectoryFileHandleProtocol + where + DirectoryFileHandle.ReadFileHandle == ReadFileHandle, + DirectoryFileHandle.ReadWriteFileHandle == ReadWriteFileHandle, + DirectoryFileHandle.WriteFileHandle == WriteFileHandle + + // MARK: - File access + + /// Opens the file at `path` for reading and returns a handle to it. + /// + /// The file being opened must exist otherwise this function will throw a ``FileSystemError`` + /// with code ``FileSystemError/Code-swift.struct/notFound``. + /// + /// - Parameters: + /// - path: The path of the file to open. + /// - options: How the file should be opened. + /// - Returns: A readable handle to the opened file. + func openFile( + forReadingAt path: FilePath, + options: OpenOptions.Read + ) async throws -> ReadFileHandle + + /// Opens the file at `path` for writing and returns a handle to it. + /// + /// - Parameters: + /// - path: The path of the file to open relative to the open file. + /// - options: How the file should be opened. + /// - Returns: A writable handle to the opened file. + func openFile( + forWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> WriteFileHandle + + /// Opens the file at `path` for reading and writing and returns a handle to it. + /// + /// - Parameters: + /// - path: The path of the file to open relative to the open file. + /// - options: How the file should be opened. + func openFile( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> ReadWriteFileHandle + + /// Opens the directory at `path` and returns a handle to it. + /// + /// The directory being opened must already exist otherwise this function will throw an error. + /// Use ``createDirectory(at:withIntermediateDirectories:permissions:)`` to create directories. + /// + /// - Parameters: + /// - path: The path of the directory to open. + /// - Returns: A handle to the opened directory. + func openDirectory( + atPath path: FilePath, + options: OpenOptions.Directory + ) async throws -> DirectoryFileHandle + + /// Create a directory at the given path. + /// + /// If a directory (or file) already exists at `path` then an error will be thrown. If + /// `createIntermediateDirectories` is `false` then the full prefix of `path` must already + /// exist. If set to `true` then all intermediate directories will be created. + /// + /// Related system calls: `mkdir(2)`. + /// + /// - Parameters: + /// - path: The directory to create. + /// - createIntermediateDirectories: Whether intermediate directories should be created. + /// - permissions: The permissions to set on the new directory; default permissions will be + /// used if not specified. + func createDirectory( + at path: FilePath, + withIntermediateDirectories createIntermediateDirectories: Bool, + permissions: FilePermissions? + ) async throws + + // MARK: - Common directories + + /// Returns the current working directory. + var currentWorkingDirectory: FilePath { get async throws } + + /// Returns the path of the temporary directory. + var temporaryDirectory: FilePath { get async throws } + + /// Create a temporary directory at the given path, from a template. + /// + /// The template for the path of the temporary directory must end in at least + /// three 'X's, which will be replaced with a unique alphanumeric combination. + /// The template can contain intermediary directories which will be created + /// if they do not exist already. + /// + /// Related system calls: `mkdir(2)`. + /// + /// - Parameters: + /// - template: The template for the path of the temporary directory. + /// - Returns: + /// - The path to the new temporary directory. + func createTemporaryDirectory( + template: FilePath + ) async throws -> FilePath + + // MARK: - File information + + /// Returns information about the file at the given path, if it exists; nil otherwise. + /// + /// - Parameters: + /// - path: The path to get information about. + /// - infoAboutSymbolicLink: If the file is a symbolic link and this parameter is `true` then + /// information about the link will be returned, otherwise information about the + /// destination of the symbolic link is returned. + /// - Returns: Information about the file at the given path or `nil` if no file exists. + func info( + forFileAt path: FilePath, + infoAboutSymbolicLink: Bool + ) async throws -> FileInfo? + + // MARK: - Symbolic links + + /// Creates a symbolic link that points to the destination. + /// + /// If a file or directory exists at `path` then an error is thrown. + /// + /// - Parameters: + /// - path: The path at which to create the symbolic link. + /// - destinationPath: The path that contains the item that the symbolic link points to.` + func createSymbolicLink( + at path: FilePath, + withDestination destinationPath: FilePath + ) async throws + + /// Returns the path of the item pointed to by a symbolic link. + /// + /// - Parameter path: The path of a file or directory. + /// - Returns: The path of the file or directory to which the symbolic link points to. + func destinationOfSymbolicLink( + at path: FilePath + ) async throws -> FilePath + + // MARK: - File copying, removal, and moving + + /// Copies the item at the specified path to a new location. + /// + /// The following error codes may be thrown: + /// - ``FileSystemError/Code-swift.struct/notFound`` if the item at `sourcePath` does not exist, + /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if an item at `destinationPath` + /// exists prior to the copy or its parent directory does not exist. + /// + /// Note that other errors may also be thrown. + /// + /// If the file at `sourcePath` is a symbolic link then only the link is copied to the new path. + /// + /// - Parameters: + /// - sourcePath: The path to the item to copy. + /// - destinationPath: The path at which to place the copy. + /// - shouldProceedAfterError: A closure which is executed to determine whether to continue + /// copying files if an error is encountered during the operation. + /// - shouldCopyFile: A closure which is executed before each copy to determine whether each + /// file should be copied. + func copyItem( + at sourcePath: FilePath, + to destinationPath: FilePath, + shouldProceedAfterError: @escaping @Sendable ( + _ path: DirectoryEntry, + _ error: Error + ) async throws -> Void, + shouldCopyFile: @escaping @Sendable ( + _ source: FilePath, + _ destination: FilePath + ) async -> Bool + ) async throws + + /// Deletes the file or directory (and its contents) at `path`. + /// + /// The item to be removed must be a regular file, symbolic link or directory. If no file exists + /// at the given path then this function returns zero. + /// + /// If the item at the `path` is a directory and `removeItemRecursively` is `true` then the + /// contents of all of its subdirectories will be removed recursively before the directory + /// at `path`. Symbolic links are removed (but their targets are not deleted). + /// + /// - Parameters: + /// - path: The path to delete. + /// - removeItemRecursively: If the item being removed is a directory, remove it by + /// recursively removing its children. Setting this to `true` is synonymous with + /// calling `rm -r`, setting this false is synonymous to calling `rmdir`. Ignored if + /// the item being removed isn't a directory. + /// - Returns: The number of deleted items which may be zero if `path` did not exist. + @discardableResult + func removeItem( + at path: FilePath, + recursively removeItemRecursively: Bool + ) async throws -> Int + + /// Moves the file or directory at the specified path to a new location. + /// + /// The following error codes may be thrown: + /// - ``FileSystemError/Code-swift.struct/notFound`` if the item at `sourcePath` does not exist, + /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if an item at `destinationPath` + /// exists prior to the copy or its parent directory does not exist. + /// + /// Note that other errors may also be thrown. + /// + /// If the file at `sourcePath` is a symbolic link then only the link is moved to the new path. + /// + /// - Parameters: + /// - sourcePath: The path to the item to move. + /// - destinationPath: The path at which to place the item. + func moveItem(at sourcePath: FilePath, to destinationPath: FilePath) async throws + + /// Replaces the item at `destinationPath` with the item at `existingPath`. + /// + /// The following error codes may be thrown: + /// - ``FileSystemError/Code-swift.struct/notFound`` if the item at `existingPath` does + /// not exist, + /// - ``FileSystemError/Code-swift.struct/io`` if the file at `existingPath` was successfully + /// copied to `destinationPath` but an error occurred while removing it from `existingPath.` + /// + /// Note that other errors may also be thrown. + /// + /// The item at `destinationPath` is not required to exist. Note that it is possible to replace + /// a file with a directory and vice versa. After the file or directory at `destinationPath` + /// has been replaced, the item at `existingPath` will be removed. + /// + /// - Parameters: + /// - destinationPath: The path of the file or directory to replace. + /// - existingPath: The path of the existing file or directory. + func replaceItem(at destinationPath: FilePath, withItemAt existingPath: FilePath) async throws +} + +// MARK: - Open existing files/directories + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension FileSystemProtocol { + /// Opens the file at the given path and provides scoped read-only access to it. + /// + /// The file remains open during lifetime of the `execute` block and will be closed + /// automatically before the call returns. Files may also be opened in read-write or write-only + /// mode by calling ``withFileHandle(forReadingAndWritingAt:options:execute:)`` and + /// ``withFileHandle(forWritingAt:options:execute:)``. + /// + /// - Parameters: + /// - path: The path of the file to open for reading. + /// - options: How the file should be opened. + /// - execute: A closure which provides read-only access to the open file. The file is closed + /// automatically after the closure exits. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withFileHandle( + forReadingAt path: FilePath, + options: OpenOptions.Read = OpenOptions.Read(), + execute: (_ read: ReadFileHandle) async throws -> R + ) async throws -> R { + let handle = try await self.openFile(forReadingAt: path, options: options) + return try await withUncancellableTearDown { + return try await execute(handle) + } tearDown: { _ in + try await handle.close() + } + } + + /// Opens the file at the given path and provides scoped write-only access to it. + /// + /// The file remains open during lifetime of the `execute` block and will be closed + /// automatically before the call returns. Files may also be opened in read-write or read-only + /// mode by calling ``withFileHandle(forReadingAndWritingAt:options:execute:)`` and + /// ``withFileHandle(forReadingAt:options:execute:)``. + /// + /// - Parameters: + /// - path: The path of the file to open for reading. + /// - options: How the file should be opened. + /// - execute: A closure which provides write-only access to the open file. The file is closed + /// automatically after the closure exits. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withFileHandle( + forWritingAt path: FilePath, + options: OpenOptions.Write = .newFile(replaceExisting: false), + execute: (_ write: WriteFileHandle) async throws -> R + ) async throws -> R { + let handle = try await self.openFile(forWritingAt: path, options: options) + return try await withUncancellableTearDown { + return try await execute(handle) + } tearDown: { result in + switch result { + case .success: + try await handle.close() + case .failure: + try await handle.close(makeChangesVisible: false) + } + } + } + + /// Opens the file at the given path and provides scoped read-write access to it. + /// + /// The file remains open during lifetime of the `execute` block and will be closed + /// automatically before the function returns. Files may also be opened in read-only or + /// write-only mode by with ``withFileHandle(forReadingAt:options:execute:)`` and + /// ``withFileHandle(forWritingAt:options:execute:)``. + /// + /// - Parameters: + /// - path: The path of the file to open for reading and writing. + /// - options: How the file should be opened. + /// - execute: A closure which provides access to the open file. The file is closed + /// automatically after the closure exits. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withFileHandle( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write = .newFile(replaceExisting: false), + execute: (_ readWrite: ReadWriteFileHandle) async throws -> R + ) async throws -> R { + let handle = try await self.openFile(forReadingAndWritingAt: path, options: options) + return try await withUncancellableTearDown { + return try await execute(handle) + } tearDown: { _ in + try await handle.close() + } + } + + /// Opens the directory at the given path and provides scoped access to it. + /// + /// - Parameters: + /// - path: The path of the directory to open. + /// - options: How the file should be opened. + /// - execute: A closure which provides access to the directory. + /// - Important: The handle passed to `execute` must not escape the closure. + /// - Returns: The result of the `execute` closure. + public func withDirectoryHandle( + atPath path: FilePath, + options: OpenOptions.Directory = OpenOptions.Directory(), + execute: (_ directory: DirectoryFileHandle) async throws -> R + ) async throws -> R { + let handle = try await self.openDirectory(atPath: path, options: options) + return try await withUncancellableTearDown { + return try await execute(handle) + } tearDown: { _ in + try await handle.close() + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension FileSystemProtocol { + /// Opens the file at `path` for reading and returns a handle to it. + /// + /// The file being opened must exist otherwise this function will throw a ``FileSystemError`` + /// with code ``FileSystemError/Code-swift.struct/notFound``. + /// + /// - Parameters: + /// - path: The path of the file to open. + /// - Returns: A readable handle to the opened file. + public func openFile( + forReadingAt path: FilePath + ) async throws -> ReadFileHandle { + try await self.openFile(forReadingAt: path, options: OpenOptions.Read()) + } + + /// Opens the directory at `path` and returns a handle to it. + /// + /// The directory being opened must already exist otherwise this function will throw an error. + /// Use ``createDirectory(at:withIntermediateDirectories:permissions:)`` to create directories. + /// + /// - Parameters: + /// - path: The path of the directory to open. + /// - Returns: A handle to the opened directory. + public func openDirectory( + atPath path: FilePath + ) async throws -> DirectoryFileHandle { + try await self.openDirectory(atPath: path, options: OpenOptions.Directory()) + } + + /// Returns information about the file at the given path, if it exists; nil otherwise. + /// + /// Calls ``info(forFileAt:infoAboutSymbolicLink:)`` setting `infoAboutSymbolicLink` to `false`. + /// + /// - Parameters: + /// - path: The path to get information about. + /// - Returns: Information about the file at the given path or `nil` if no file exists. + public func info(forFileAt path: FilePath) async throws -> FileInfo? { + return try await self.info(forFileAt: path, infoAboutSymbolicLink: false) + } + + /// Copies the item at the specified path to a new location. + /// + /// The following error codes may be thrown: + /// - ``FileSystemError/Code-swift.struct/notFound`` if the item at `sourcePath` does not exist, + /// - ``FileSystemError/Code-swift.struct/invalidArgument`` if an item at `destinationPath` + /// exists prior to the copy or its parent directory does not exist. + /// + /// Note that other errors may also be thrown. If any error is encountered during the copy + /// then the copy is aborted. You can modify the behaviour with the `shouldProceedAfterError` + /// parameter of ``copyItem(at:to:shouldProceedAfterError:shouldCopyFile:)``. + /// + /// If the file at `sourcePath` is a symbolic link then only the link is copied to the new path. + /// + /// - Parameters: + /// - sourcePath: The path to the item to copy. + /// - destinationPath: The path at which to place the copy. + public func copyItem(at sourcePath: FilePath, to destinationPath: FilePath) async throws { + try await self.copyItem(at: sourcePath, to: destinationPath) { entry, error in + throw error + } shouldCopyFile: { source, destination in + return true + } + } + + /// Deletes the file or directory (and its contents) at `path`. + /// + /// The item to be removed must be a regular file, symbolic link or directory. If no file exists + /// at the given path then this function returns zero. + /// + /// If the item at the `path` is a directory then the contents of all of its subdirectories + /// will be removed recursively before the directory at `path`. Symbolic links are removed (but + /// their targets are not deleted). + /// + /// - Parameters: + /// - path: The path to delete. + /// - Returns: The number of deleted items which may be zero if `path` did not exist. + @discardableResult + public func removeItem( + at path: FilePath + ) async throws -> Int { + try await self.removeItem(at: path, recursively: true) + } + + /// Create a directory at the given path. + /// + /// If a directory (or file) already exists at `path` then an error will be thrown. If + /// `createIntermediateDirectories` is `false` then the full prefix of `path` must already + /// exist. If set to `true` then all intermediate directories will be created. + /// + /// New directories will be given read-write-execute owner permissions and read-execute group + /// and other permissions. + /// + /// Related system calls: `mkdir(2)`. + /// + /// - Parameters: + /// - path: The directory to create. + /// - createIntermediateDirectories: Whether intermediate directories should be created. + public func createDirectory( + at path: FilePath, + withIntermediateDirectories createIntermediateDirectories: Bool + ) async throws { + try await self.createDirectory( + at: path, + withIntermediateDirectories: createIntermediateDirectories, + permissions: .defaultsForDirectory + ) + } +} diff --git a/Sources/NIOFileSystem/FileType.swift b/Sources/NIOFileSystem/FileType.swift new file mode 100644 index 0000000000..b34de332a8 --- /dev/null +++ b/Sources/NIOFileSystem/FileType.swift @@ -0,0 +1,167 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +/// The type of a file system object. +public struct FileType: Hashable, Sendable { + internal enum Wrapped: Hashable, Sendable, CaseIterable { + case regular + case block + case character + case fifo + case directory + case symlink + case socket + case whiteout + case unknown + } + + internal let wrapped: Wrapped + private init(_ wrapped: Wrapped) { + self.wrapped = wrapped + } +} + +extension FileType { + /// Regular file. + public static var regular: Self { Self(.regular) } + + /// Directory. + public static var directory: Self { Self(.directory) } + + /// Symbolic link. + public static var symlink: Self { Self(.symlink) } + + /// Hardware block device. + public static var block: Self { Self(.block) } + + /// Hardware character device. + public static var character: Self { Self(.character) } + + /// FIFO (or named pipe). + public static var fifo: Self { Self(.fifo) } + + /// Socket. + public static var socket: Self { Self(.socket) } + + /// Whiteout file. + public static var whiteout: Self { Self(.whiteout) } + + /// A file of unknown type. + public static var unknown: Self { Self(.unknown) } +} + +extension FileType: CustomStringConvertible { + public var description: String { + switch self.wrapped { + case .regular: + return "regular" + case .block: + return "block" + case .character: + return "character" + case .fifo: + return "fifo" + case .directory: + return "directory" + case .symlink: + return "symlink" + case .socket: + return "socket" + case .whiteout: + return "whiteout" + case .unknown: + return "unknown" + } + } +} + +extension FileType: CaseIterable { + public static var allCases: [FileType] { + Self.Wrapped.allCases.map { FileType($0) } + } +} + +extension FileType { + /// Initializes a file type from a `CInterop.Mode`. + /// + /// Note: an appropriate mask is applied to `mode`. + public init(platformSpecificMode: CInterop.Mode) { + // See: `man 2 stat` + switch platformSpecificMode & S_IFMT { + case S_IFIFO: + self = .fifo + case S_IFCHR: + self = .character + case S_IFDIR: + self = .directory + case S_IFBLK: + self = .block + case S_IFREG: + self = .regular + case S_IFLNK: + self = .symlink + case S_IFSOCK: + self = .socket + #if canImport(Darwin) + case S_IFWHT: + self = .whiteout + #endif + default: + self = .unknown + } + } + + /// Initializes a file type from the `d_type` from `dirent`. + @_spi(Testing) + public init(direntType: UInt8) { + #if canImport(Darwin) + let value = Int32(direntType) + #elseif canImport(Glibc) + let value = Int(direntType) + #endif + + switch value { + case DT_FIFO: + self = .fifo + case DT_CHR: + self = .character + case DT_DIR: + self = .directory + case DT_BLK: + self = .block + case DT_REG: + self = .regular + case DT_LNK: + self = .symlink + case DT_SOCK: + self = .socket + #if canImport(Darwin) + case DT_WHT: + self = .whiteout + #endif + case DT_UNKNOWN: + self = .unknown + default: + self = .unknown + } + } +} diff --git a/Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift b/Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift new file mode 100644 index 0000000000..ed55ae18a3 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift @@ -0,0 +1,90 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/// Wraps a ``BufferedStream`` or ``AnyAsyncSequence``. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +internal enum BufferedOrAnyStream { + case bufferedStream(BufferedStream) + case anyAsyncSequence(AnyAsyncSequence) + + internal init(wrapping stream: BufferedStream) { + self = .bufferedStream(stream) + } + + internal init(wrapping stream: S) where S.Element == Element { + self = .anyAsyncSequence(AnyAsyncSequence(wrapping: stream)) + } + + internal func makeAsyncIterator() -> AsyncIterator { + switch self { + case let .bufferedStream(stream): + return AsyncIterator(wrapping: stream.makeAsyncIterator()) + case let .anyAsyncSequence(stream): + return AsyncIterator(wrapping: stream.makeAsyncIterator()) + } + } + + internal enum AsyncIterator: AsyncIteratorProtocol { + case bufferedStream(BufferedStream.AsyncIterator) + case anyAsyncSequence(AnyAsyncSequence.AsyncIterator) + + internal mutating func next() async throws -> Element? { + let element: Element? + switch self { + case var .bufferedStream(iterator): + defer { self = .bufferedStream(iterator) } + element = try await iterator.next() + case var .anyAsyncSequence(iterator): + defer { self = .anyAsyncSequence(iterator) } + element = try await iterator.next() + } + return element + } + + internal init(wrapping iterator: BufferedStream.AsyncIterator) { + self = .bufferedStream(iterator) + } + + internal init(wrapping iterator: AnyAsyncSequence.AsyncIterator) { + self = .anyAsyncSequence(iterator) + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +internal struct AnyAsyncSequence: AsyncSequence { + private let _makeAsyncIterator: () -> AsyncIterator + + internal init(wrapping sequence: S) where S.Element == Element { + self._makeAsyncIterator = { + AsyncIterator(wrapping: sequence.makeAsyncIterator()) + } + } + + internal func makeAsyncIterator() -> AsyncIterator { + return self._makeAsyncIterator() + } + + internal struct AsyncIterator: AsyncIteratorProtocol { + private var iterator: any AsyncIteratorProtocol + + init(wrapping iterator: I) where I.Element == Element { + self.iterator = iterator + } + + internal mutating func next() async throws -> Element? { + return try await self.iterator.next() as? Element + } + } +} diff --git a/Sources/NIOFileSystem/Internal/BufferedStream.swift b/Sources/NIOFileSystem/Internal/BufferedStream.swift new file mode 100644 index 0000000000..2c2605b367 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/BufferedStream.swift @@ -0,0 +1,1733 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import DequeModule +import NIOConcurrencyHelpers + +/// An asynchronous sequence generated from an error-throwing closure that +/// calls a continuation to produce new elements. +/// +/// `BufferedStream` conforms to `AsyncSequence`, providing a convenient +/// way to create an asynchronous sequence without manually implementing an +/// asynchronous iterator. In particular, an asynchronous stream is well-suited +/// to adapt callback- or delegation-based APIs to participate with +/// `async`-`await`. +/// +/// In contrast to `AsyncStream`, this type can throw an error from the awaited +/// `next()`, which terminates the stream with the thrown error. +/// +/// You initialize an `BufferedStream` with a closure that receives an +/// `BufferedStream.Continuation`. Produce elements in this closure, then +/// provide them to the stream by calling the continuation's `yield(_:)` method. +/// When there are no further elements to produce, call the continuation's +/// `finish()` method. This causes the sequence iterator to produce a `nil`, +/// which terminates the sequence. If an error occurs, call the continuation's +/// `finish(throwing:)` method, which causes the iterator's `next()` method to +/// throw the error to the awaiting call point. The continuation is `Sendable`, +/// which permits calling it from concurrent contexts external to the iteration +/// of the `BufferedStream`. +/// +/// An arbitrary source of elements can produce elements faster than they are +/// consumed by a caller iterating over them. Because of this, `BufferedStream` +/// defines a buffering behavior, allowing the stream to buffer a specific +/// number of oldest or newest elements. By default, the buffer limit is +/// `Int.max`, which means it's unbounded. +/// +/// ### Adapting Existing Code to Use Streams +/// +/// To adapt existing callback code to use `async`-`await`, use the callbacks +/// to provide values to the stream, by using the continuation's `yield(_:)` +/// method. +/// +/// Consider a hypothetical `QuakeMonitor` type that provides callers with +/// `Quake` instances every time it detects an earthquake. To receive callbacks, +/// callers set a custom closure as the value of the monitor's +/// `quakeHandler` property, which the monitor calls back as necessary. Callers +/// can also set an `errorHandler` to receive asynchronous error notifications, +/// such as the monitor service suddenly becoming unavailable. +/// +/// class QuakeMonitor { +/// var quakeHandler: ((Quake) -> Void)? +/// var errorHandler: ((Error) -> Void)? +/// +/// func startMonitoring() {…} +/// func stopMonitoring() {…} +/// } +/// +/// To adapt this to use `async`-`await`, extend the `QuakeMonitor` to add a +/// `quakes` property, of type `BufferedStream`. In the getter for +/// this property, return an `BufferedStream`, whose `build` closure -- +/// called at runtime to create the stream -- uses the continuation to +/// perform the following steps: +/// +/// 1. Creates a `QuakeMonitor` instance. +/// 2. Sets the monitor's `quakeHandler` property to a closure that receives +/// each `Quake` instance and forwards it to the stream by calling the +/// continuation's `yield(_:)` method. +/// 3. Sets the monitor's `errorHandler` property to a closure that receives +/// any error from the monitor and forwards it to the stream by calling the +/// continuation's `finish(throwing:)` method. This causes the stream's +/// iterator to throw the error and terminate the stream. +/// 4. Sets the continuation's `onTermination` property to a closure that +/// calls `stopMonitoring()` on the monitor. +/// 5. Calls `startMonitoring` on the `QuakeMonitor`. +/// +/// ``` +/// extension QuakeMonitor { +/// +/// static var throwingQuakes: BufferedStream { +/// BufferedStream { continuation in +/// let monitor = QuakeMonitor() +/// monitor.quakeHandler = { quake in +/// continuation.yield(quake) +/// } +/// monitor.errorHandler = { error in +/// continuation.finish(throwing: error) +/// } +/// continuation.onTermination = { @Sendable _ in +/// monitor.stopMonitoring() +/// } +/// monitor.startMonitoring() +/// } +/// } +/// } +/// ``` +/// +/// +/// Because the stream is an `AsyncSequence`, the call point uses the +/// `for`-`await`-`in` syntax to process each `Quake` instance as produced by the stream: +/// +/// do { +/// for try await quake in quakeStream { +/// print("Quake: \(quake.date)") +/// } +/// print("Stream done.") +/// } catch { +/// print("Error: \(error)") +/// } +/// +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +internal struct BufferedStream { + final class _Backing: Sendable { + let storage: _BackPressuredStorage + + init(storage: _BackPressuredStorage) { + self.storage = storage + } + + deinit { + self.storage.sequenceDeinitialized() + } + } + + enum _Implementation: Sendable { + /// This is the implementation with backpressure based on the Source + case backpressured(_Backing) + } + + let implementation: _Implementation +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream: AsyncSequence { + /// The asynchronous iterator for iterating an asynchronous stream. + /// + /// This type is not `Sendable`. Don't use it from multiple + /// concurrent contexts. It is a programmer error to invoke `next()` from a + /// concurrent context that contends with another such call, which + /// results in a call to `fatalError()`. + internal struct Iterator: AsyncIteratorProtocol { + final class _Backing { + let storage: _BackPressuredStorage + + init(storage: _BackPressuredStorage) { + self.storage = storage + self.storage.iteratorInitialized() + } + + deinit { + self.storage.iteratorDeinitialized() + } + } + enum _Implementation { + /// This is the implementation with backpressure based on the Source + case backpressured(_Backing) + } + + var implementation: _Implementation + + /// The next value from the asynchronous stream. + /// + /// When `next()` returns `nil`, this signifies the end of the + /// `BufferedStream`. + /// + /// It is a programmer error to invoke `next()` from a concurrent context + /// that contends with another such call, which results in a call to + /// `fatalError()`. + /// + /// If you cancel the task this iterator is running in while `next()` is + /// awaiting a value, the `BufferedStream` terminates. In this case, + /// `next()` may return `nil` immediately, or else return `nil` on + /// subsequent calls. + internal mutating func next() async throws -> Element? { + switch self.implementation { + case .backpressured(let backing): + return try await backing.storage.next() + } + } + } + + /// Creates the asynchronous iterator that produces elements of this + /// asynchronous sequence. + internal func makeAsyncIterator() -> Iterator { + switch self.implementation { + case .backpressured(let backing): + return Iterator(implementation: .backpressured(.init(storage: backing.storage))) + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream: Sendable where Element: Sendable {} + +internal struct _ManagedCriticalState: @unchecked Sendable { + let lock: NIOLockedValueBox + + internal init(_ initial: State) { + self.lock = .init(initial) + } + + internal func withCriticalRegion( + _ critical: (inout State) throws -> R + ) rethrows -> R { + try self.lock.withLockedValue(critical) + } +} + +internal struct AlreadyFinishedError: Error {} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream { + /// A mechanism to interface between producer code and an asynchronous stream. + /// + /// Use this source to provide elements to the stream by calling one of the `write` methods, then terminate the stream normally + /// by calling the `finish()` method. You can also use the source's `finish(throwing:)` method to terminate the stream by + /// throwing an error. + internal struct Source: Sendable { + /// A strategy that handles the backpressure of the asynchronous stream. + internal struct BackPressureStrategy: Sendable { + /// When the high watermark is reached producers will be suspended. All producers will be resumed again once + /// the low watermark is reached. + internal static func watermark(low: Int, high: Int) -> BackPressureStrategy { + BackPressureStrategy( + internalBackPressureStrategy: .watermark(.init(low: low, high: high)) + ) + } + + private init(internalBackPressureStrategy: _InternalBackPressureStrategy) { + self._internalBackPressureStrategy = internalBackPressureStrategy + } + + fileprivate let _internalBackPressureStrategy: _InternalBackPressureStrategy + } + + /// A type that indicates the result of writing elements to the source. + @frozen + internal enum WriteResult: Sendable { + /// A token that is returned when the asynchronous stream's backpressure strategy indicated that production should + /// be suspended. Use this token to enqueue a callback by calling the ``enqueueCallback(_:)`` method. + internal struct CallbackToken: Sendable { + let id: UInt + } + + /// Indicates that more elements should be produced and written to the source. + case produceMore + + /// Indicates that a callback should be enqueued. + /// + /// The associated token should be passed to the ``enqueueCallback(_:)`` method. + case enqueueCallback(CallbackToken) + } + + /// Backing class for the source used to hook a deinit. + final class _Backing: Sendable { + let storage: _BackPressuredStorage + + init(storage: _BackPressuredStorage) { + self.storage = storage + } + + deinit { + self.storage.sourceDeinitialized() + } + } + + /// A callback to invoke when the stream finished. + /// + /// The stream finishes and calls this closure in the following cases: + /// - No iterator was created and the sequence was deinited + /// - An iterator was created and deinited + /// - After ``finish(throwing:)`` was called and all elements have been consumed + /// - The consuming task got cancelled + internal var onTermination: (@Sendable () -> Void)? { + set { + self._backing.storage.onTermination = newValue + } + get { + self._backing.storage.onTermination + } + } + + private var _backing: _Backing + + internal init(storage: _BackPressuredStorage) { + self._backing = .init(storage: storage) + } + + /// Writes new elements to the asynchronous stream. + /// + /// If there is a task consuming the stream and awaiting the next element then the task will get resumed with the + /// first element of the provided sequence. If the asynchronous stream already terminated then this method will throw an error + /// indicating the failure. + /// + /// - Parameter sequence: The elements to write to the asynchronous stream. + /// - Returns: The result that indicates if more elements should be produced at this time. + internal func write(contentsOf sequence: S) throws -> WriteResult + where Element == S.Element, S: Sequence { + try self._backing.storage.write(contentsOf: sequence) + } + + /// Write the element to the asynchronous stream. + /// + /// If there is a task consuming the stream and awaiting the next element then the task will get resumed with the + /// provided element. If the asynchronous stream already terminated then this method will throw an error + /// indicating the failure. + /// + /// - Parameter element: The element to write to the asynchronous stream. + /// - Returns: The result that indicates if more elements should be produced at this time. + internal func write(_ element: Element) throws -> WriteResult { + try self._backing.storage.write(contentsOf: CollectionOfOne(element)) + } + + /// Enqueues a callback that will be invoked once more elements should be produced. + /// + /// Call this method after ``write(contentsOf:)`` or ``write(:)`` returned ``WriteResult/enqueueCallback(_:)``. + /// + /// - Important: Enqueueing the same token multiple times is not allowed. + /// + /// - Parameters: + /// - callbackToken: The callback token. + /// - onProduceMore: The callback which gets invoked once more elements should be produced. + internal func enqueueCallback( + callbackToken: WriteResult.CallbackToken, + onProduceMore: @escaping @Sendable (Result) -> Void + ) { + self._backing.storage.enqueueProducer( + callbackToken: callbackToken, + onProduceMore: onProduceMore + ) + } + + /// Cancel an enqueued callback. + /// + /// Call this method to cancel a callback enqueued by the ``enqueueCallback(callbackToken:onProduceMore:)`` method. + /// + /// - Note: This methods supports being called before ``enqueueCallback(callbackToken:onProduceMore:)`` is called and + /// will mark the passed `callbackToken` as cancelled. + /// + /// - Parameter callbackToken: The callback token. + internal func cancelCallback(callbackToken: WriteResult.CallbackToken) { + self._backing.storage.cancelProducer(callbackToken: callbackToken) + } + + /// Write new elements to the asynchronous stream and provide a callback which will be invoked once more elements should be produced. + /// + /// If there is a task consuming the stream and awaiting the next element then the task will get resumed with the + /// first element of the provided sequence. If the asynchronous stream already terminated then `onProduceMore` will be invoked with + /// a `Result.failure`. + /// + /// - Parameters: + /// - sequence: The elements to write to the asynchronous stream. + /// - onProduceMore: The callback which gets invoked once more elements should be produced. This callback might be + /// invoked during the call to ``write(contentsOf:onProduceMore:)``. + internal func write( + contentsOf sequence: S, + onProduceMore: @escaping @Sendable (Result) -> Void + ) where Element == S.Element, S: Sequence { + do { + let writeResult = try self.write(contentsOf: sequence) + + switch writeResult { + case .produceMore: + onProduceMore(Result.success(())) + + case .enqueueCallback(let callbackToken): + self.enqueueCallback(callbackToken: callbackToken, onProduceMore: onProduceMore) + } + } catch { + onProduceMore(.failure(error)) + } + } + + /// Writes the element to the asynchronous stream. + /// + /// If there is a task consuming the stream and awaiting the next element then the task will get resumed with the + /// provided element. If the asynchronous stream already terminated then `onProduceMore` will be invoked with + /// a `Result.failure`. + /// + /// - Parameters: + /// - sequence: The element to write to the asynchronous stream. + /// - onProduceMore: The callback which gets invoked once more elements should be produced. This callback might be + /// invoked during the call to ``write(_:onProduceMore:)``. + internal func write( + _ element: Element, + onProduceMore: @escaping @Sendable (Result) -> Void + ) { + self.write(contentsOf: CollectionOfOne(element), onProduceMore: onProduceMore) + } + + /// Write new elements to the asynchronous stream. + /// + /// If there is a task consuming the stream and awaiting the next element then the task will get resumed with the + /// first element of the provided sequence. If the asynchronous stream already terminated then this method will throw an error + /// indicating the failure. + /// + /// This method returns once more elements should be produced. + /// + /// - Parameters: + /// - sequence: The elements to write to the asynchronous stream. + internal func write(contentsOf sequence: S) async throws + where Element == S.Element, S: Sequence { + let writeResult = try { try self.write(contentsOf: sequence) }() + + switch writeResult { + case .produceMore: + return + + case .enqueueCallback(let callbackToken): + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + self.enqueueCallback( + callbackToken: callbackToken, + onProduceMore: { result in + switch result { + case .success(): + continuation.resume(returning: ()) + case .failure(let error): + continuation.resume(throwing: error) + } + } + ) + } + } onCancel: { + self.cancelCallback(callbackToken: callbackToken) + } + } + } + + /// Write new element to the asynchronous stream. + /// + /// If there is a task consuming the stream and awaiting the next element then the task will get resumed with the + /// provided element. If the asynchronous stream already terminated then this method will throw an error + /// indicating the failure. + /// + /// This method returns once more elements should be produced. + /// + /// - Parameters: + /// - sequence: The element to write to the asynchronous stream. + internal func write(_ element: Element) async throws { + try await self.write(contentsOf: CollectionOfOne(element)) + } + + /// Write the elements of the asynchronous sequence to the asynchronous stream. + /// + /// This method returns once the provided asynchronous sequence or the the asynchronous stream finished. + /// + /// - Important: This method does not finish the source if consuming the upstream sequence terminated. + /// + /// - Parameters: + /// - sequence: The elements to write to the asynchronous stream. + internal func write(contentsOf sequence: S) async throws + where Element == S.Element, S: AsyncSequence { + for try await element in sequence { + try await self.write(contentsOf: CollectionOfOne(element)) + } + } + + /// Indicates that the production terminated. + /// + /// After all buffered elements are consumed the next iteration point will return `nil` or throw an error. + /// + /// Calling this function more than once has no effect. After calling finish, the stream enters a terminal state and doesn't accept + /// new elements. + /// + /// - Parameters: + /// - error: The error to throw, or `nil`, to finish normally. + internal func finish(throwing error: Error?) { + self._backing.storage.finish(error) + } + } + + /// Initializes a new ``BufferedStream`` and an ``BufferedStream/Source``. + /// + /// - Parameters: + /// - elementType: The element type of the stream. + /// - failureType: The failure type of the stream. + /// - backPressureStrategy: The backpressure strategy that the stream should use. + /// - Returns: A tuple containing the stream and its source. The source should be passed to the + /// producer while the stream should be passed to the consumer. + internal static func makeStream( + of elementType: Element.Type = Element.self, + throwing failureType: Error.Type = Error.self, + backPressureStrategy: Source.BackPressureStrategy + ) -> (`Self`, Source) where Error == Error { + let storage = _BackPressuredStorage( + backPressureStrategy: backPressureStrategy._internalBackPressureStrategy + ) + let source = Source(storage: storage) + + return (.init(storage: storage), source) + } + + init(storage: _BackPressuredStorage) { + self.implementation = .backpressured(.init(storage: storage)) + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream { + struct _WatermarkBackPressureStrategy { + /// The low watermark where demand should start. + private let _low: Int + /// The high watermark where demand should be stopped. + private let _high: Int + + /// Initializes a new ``_WatermarkBackPressureStrategy``. + /// + /// - Parameters: + /// - low: The low watermark where demand should start. + /// - high: The high watermark where demand should be stopped. + init(low: Int, high: Int) { + precondition(low <= high) + self._low = low + self._high = high + } + + func didYield(bufferDepth: Int) -> Bool { + // We are demanding more until we reach the high watermark + return bufferDepth < self._high + } + + func didConsume(bufferDepth: Int) -> Bool { + // We start demanding again once we are below the low watermark + return bufferDepth < self._low + } + } + + enum _InternalBackPressureStrategy { + case watermark(_WatermarkBackPressureStrategy) + + mutating func didYield(bufferDepth: Int) -> Bool { + switch self { + case .watermark(let strategy): + return strategy.didYield(bufferDepth: bufferDepth) + } + } + + mutating func didConsume(bufferDepth: Int) -> Bool { + switch self { + case .watermark(let strategy): + return strategy.didConsume(bufferDepth: bufferDepth) + } + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream { + // We are unchecked Sendable since we are protecting our state with a lock. + final class _BackPressuredStorage: Sendable { + /// The state machine + let _stateMachine: _ManagedCriticalState<_StateMachine> + + var onTermination: (@Sendable () -> Void)? { + set { + self._stateMachine.withCriticalRegion { + $0._onTermination = newValue + } + } + get { + self._stateMachine.withCriticalRegion { + $0._onTermination + } + } + } + + init( + backPressureStrategy: _InternalBackPressureStrategy + ) { + self._stateMachine = .init(.init(backPressureStrategy: backPressureStrategy)) + } + + func sequenceDeinitialized() { + let action = self._stateMachine.withCriticalRegion { + $0.sequenceDeinitialized() + } + + switch action { + case .callOnTermination(let onTermination): + onTermination?() + + case .failProducersAndCallOnTermination(let producerContinuations, let onTermination): + for producerContinuation in producerContinuations { + producerContinuation(.failure(AlreadyFinishedError())) + } + onTermination?() + + case .none: + break + } + } + + func iteratorInitialized() { + self._stateMachine.withCriticalRegion { + $0.iteratorInitialized() + } + } + + func iteratorDeinitialized() { + let action = self._stateMachine.withCriticalRegion { + $0.iteratorDeinitialized() + } + + switch action { + case .callOnTermination(let onTermination): + onTermination?() + + case .failProducersAndCallOnTermination(let producerContinuations, let onTermination): + for producerContinuation in producerContinuations { + producerContinuation(.failure(AlreadyFinishedError())) + } + onTermination?() + + case .none: + break + } + } + + func sourceDeinitialized() { + let action = self._stateMachine.withCriticalRegion { + $0.sourceDeinitialized() + } + + switch action { + case .callOnTermination(let onTermination): + onTermination?() + + case .failProducersAndCallOnTermination(let producerContinuations, let onTermination): + for producerContinuation in producerContinuations { + producerContinuation(.failure(AlreadyFinishedError())) + } + onTermination?() + + case .failProducers(let producerContinuations): + for producerContinuation in producerContinuations { + producerContinuation(.failure(AlreadyFinishedError())) + } + + case .none: + break + } + } + + func write( + contentsOf sequence: some Sequence + ) throws -> Source.WriteResult { + let action = self._stateMachine.withCriticalRegion { + return $0.write(sequence) + } + + switch action { + case .returnProduceMore: + return .produceMore + + case .returnEnqueue(let callbackToken): + return .enqueueCallback(callbackToken) + + case .resumeConsumerAndReturnProduceMore(let continuation, let element): + continuation.resume(returning: element) + return .produceMore + + case .resumeConsumerAndReturnEnqueue(let continuation, let element, let callbackToken): + continuation.resume(returning: element) + return .enqueueCallback(callbackToken) + + case .throwFinishedError: + throw AlreadyFinishedError() + } + } + + func enqueueProducer( + callbackToken: Source.WriteResult.CallbackToken, + onProduceMore: @escaping @Sendable (Result) -> Void + ) { + let action = self._stateMachine.withCriticalRegion { + $0.enqueueProducer(callbackToken: callbackToken, onProduceMore: onProduceMore) + } + + switch action { + case .resumeProducer(let onProduceMore): + onProduceMore(Result.success(())) + + case .resumeProducerWithError(let onProduceMore, let error): + onProduceMore(Result.failure(error)) + + case .none: + break + } + } + + func cancelProducer(callbackToken: Source.WriteResult.CallbackToken) { + let action = self._stateMachine.withCriticalRegion { + $0.cancelProducer(callbackToken: callbackToken) + } + + switch action { + case .resumeProducerWithCancellationError(let onProduceMore): + onProduceMore(Result.failure(CancellationError())) + + case .none: + break + } + } + + func finish(_ failure: Error?) { + let action = self._stateMachine.withCriticalRegion { + $0.finish(failure) + } + + switch action { + case .callOnTermination(let onTermination): + onTermination?() + + case .resumeConsumerAndCallOnTermination( + let consumerContinuation, + let failure, + let onTermination + ): + switch failure { + case .some(let error): + consumerContinuation.resume(throwing: error) + case .none: + consumerContinuation.resume(returning: nil) + } + + onTermination?() + + case .resumeProducers(let producerContinuations): + for producerContinuation in producerContinuations { + producerContinuation(.failure(AlreadyFinishedError())) + } + + case .none: + break + } + } + + func next() async throws -> Element? { + let action = self._stateMachine.withCriticalRegion { + $0.next() + } + + switch action { + case .returnElement(let element): + return element + + case .returnElementAndResumeProducers(let element, let producerContinuations): + for producerContinuation in producerContinuations { + producerContinuation(Result.success(())) + } + + return element + + case .returnErrorAndCallOnTermination(let failure, let onTermination): + onTermination?() + switch failure { + case .some(let error): + throw error + + case .none: + return nil + } + + case .returnNil: + return nil + + case .suspendTask: + return try await self.suspendNext() + } + } + + func suspendNext() async throws -> Element? { + return try await withTaskCancellationHandler { + return try await withCheckedThrowingContinuation { continuation in + let action = self._stateMachine.withCriticalRegion { + $0.suspendNext(continuation: continuation) + } + + switch action { + case .resumeConsumerWithElement(let continuation, let element): + continuation.resume(returning: element) + + case .resumeConsumerWithElementAndProducers( + let continuation, + let element, + let producerContinuations + ): + continuation.resume(returning: element) + for producerContinuation in producerContinuations { + producerContinuation(Result.success(())) + } + + case .resumeConsumerWithErrorAndCallOnTermination( + let continuation, + let failure, + let onTermination + ): + switch failure { + case .some(let error): + continuation.resume(throwing: error) + + case .none: + continuation.resume(returning: nil) + } + onTermination?() + + case .resumeConsumerWithNil(let continuation): + continuation.resume(returning: nil) + + case .none: + break + } + } + } onCancel: { + let action = self._stateMachine.withCriticalRegion { + $0.cancelNext() + } + + switch action { + case .resumeConsumerWithCancellationErrorAndCallOnTermination( + let continuation, + let onTermination + ): + continuation.resume(throwing: CancellationError()) + onTermination?() + + case .failProducersAndCallOnTermination( + let producerContinuations, + let onTermination + ): + for producerContinuation in producerContinuations { + producerContinuation(.failure(AlreadyFinishedError())) + } + onTermination?() + + case .none: + break + } + } + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension BufferedStream { + /// The state machine of the backpressured async stream. + struct _StateMachine { + enum _State { + struct Initial { + /// The backpressure strategy. + var backPressureStrategy: _InternalBackPressureStrategy + /// Indicates if the iterator was initialized. + var iteratorInitialized: Bool + /// The onTermination callback. + var onTermination: (@Sendable () -> Void)? + } + + struct Streaming { + /// The backpressure strategy. + var backPressureStrategy: _InternalBackPressureStrategy + /// Indicates if the iterator was initialized. + var iteratorInitialized: Bool + /// The onTermination callback. + var onTermination: (@Sendable () -> Void)? + /// The buffer of elements. + var buffer: Deque + /// The optional consumer continuation. + var consumerContinuation: CheckedContinuation? + /// The producer continuations. + var producerContinuations: Deque<(UInt, (Result) -> Void)> + /// The producers that have been cancelled. + var cancelledAsyncProducers: Deque + /// Indicates if we currently have outstanding demand. + var hasOutstandingDemand: Bool + } + + struct SourceFinished { + /// Indicates if the iterator was initialized. + var iteratorInitialized: Bool + /// The buffer of elements. + var buffer: Deque + /// The failure that should be thrown after the last element has been consumed. + var failure: Error? + /// The onTermination callback. + var onTermination: (@Sendable () -> Void)? + } + + case initial(Initial) + /// The state once either any element was yielded or `next()` was called. + case streaming(Streaming) + /// The state once the underlying source signalled that it is finished. + case sourceFinished(SourceFinished) + + /// The state once there can be no outstanding demand. This can happen if: + /// 1. The iterator was deinited + /// 2. The underlying source finished and all buffered elements have been consumed + case finished(iteratorInitialized: Bool) + + /// An intermediate state to avoid CoWs. + case modify + } + + /// The state machine's current state. + var _state: _State + + // The ID used for the next CallbackToken. + var nextCallbackTokenID: UInt = 0 + + var _onTermination: (@Sendable () -> Void)? { + set { + switch self._state { + case .initial(var initial): + initial.onTermination = newValue + self._state = .initial(initial) + + case .streaming(var streaming): + streaming.onTermination = newValue + self._state = .streaming(streaming) + + case .sourceFinished(var sourceFinished): + sourceFinished.onTermination = newValue + self._state = .sourceFinished(sourceFinished) + + case .finished: + break + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + get { + switch self._state { + case .initial(let initial): + return initial.onTermination + + case .streaming(let streaming): + return streaming.onTermination + + case .sourceFinished(let sourceFinished): + return sourceFinished.onTermination + + case .finished: + return nil + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + } + + /// Initializes a new `StateMachine`. + /// + /// We are passing and holding the back-pressure strategy here because + /// it is a customizable extension of the state machine. + /// + /// - Parameter backPressureStrategy: The back-pressure strategy. + init( + backPressureStrategy: _InternalBackPressureStrategy + ) { + self._state = .initial( + .init( + backPressureStrategy: backPressureStrategy, + iteratorInitialized: false, + onTermination: nil + ) + ) + } + + /// Generates the next callback token. + mutating func nextCallbackToken() -> Source.WriteResult.CallbackToken { + let id = self.nextCallbackTokenID + self.nextCallbackTokenID += 1 + return .init(id: id) + } + + /// Actions returned by `sequenceDeinitialized()`. + enum SequenceDeinitializedAction { + /// Indicates that `onTermination` should be called. + case callOnTermination((@Sendable () -> Void)?) + /// Indicates that all producers should be failed and `onTermination` should be called. + case failProducersAndCallOnTermination( + [(Result) -> Void], + (@Sendable () -> Void)? + ) + } + + mutating func sequenceDeinitialized() -> SequenceDeinitializedAction? { + switch self._state { + case .initial(let initial): + if initial.iteratorInitialized { + // An iterator was created and we deinited the sequence. + // This is an expected pattern and we just continue on normal. + return .none + } else { + // No iterator was created so we can transition to finished right away. + self._state = .finished(iteratorInitialized: false) + + return .callOnTermination(initial.onTermination) + } + + case .streaming(let streaming): + if streaming.iteratorInitialized { + // An iterator was created and we deinited the sequence. + // This is an expected pattern and we just continue on normal. + return .none + } else { + // No iterator was created so we can transition to finished right away. + self._state = .finished(iteratorInitialized: false) + + return .failProducersAndCallOnTermination( + Array(streaming.producerContinuations.map { $0.1 }), + streaming.onTermination + ) + } + + case .sourceFinished(let sourceFinished): + if sourceFinished.iteratorInitialized { + // An iterator was created and we deinited the sequence. + // This is an expected pattern and we just continue on normal. + return .none + } else { + // No iterator was created so we can transition to finished right away. + self._state = .finished(iteratorInitialized: false) + + return .callOnTermination(sourceFinished.onTermination) + } + + case .finished: + // We are already finished so there is nothing left to clean up. + // This is just the references dropping afterwards. + return .none + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + mutating func iteratorInitialized() { + switch self._state { + case .initial(var initial): + if initial.iteratorInitialized { + // Our sequence is a unicast sequence and does not support multiple AsyncIterator's + fatalError("Only a single AsyncIterator can be created") + } else { + // The first and only iterator was initialized. + initial.iteratorInitialized = true + self._state = .initial(initial) + } + + case .streaming(var streaming): + if streaming.iteratorInitialized { + // Our sequence is a unicast sequence and does not support multiple AsyncIterator's + fatalError("Only a single AsyncIterator can be created") + } else { + // The first and only iterator was initialized. + streaming.iteratorInitialized = true + self._state = .streaming(streaming) + } + + case .sourceFinished(var sourceFinished): + if sourceFinished.iteratorInitialized { + // Our sequence is a unicast sequence and does not support multiple AsyncIterator's + fatalError("Only a single AsyncIterator can be created") + } else { + // The first and only iterator was initialized. + sourceFinished.iteratorInitialized = true + self._state = .sourceFinished(sourceFinished) + } + + case .finished(iteratorInitialized: true): + // Our sequence is a unicast sequence and does not support multiple AsyncIterator's + fatalError("Only a single AsyncIterator can be created") + + case .finished(iteratorInitialized: false): + // It is strange that an iterator is created after we are finished + // but it can definitely happen, e.g. + // Sequence.init -> source.finish -> sequence.makeAsyncIterator + self._state = .finished(iteratorInitialized: true) + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `iteratorDeinitialized()`. + enum IteratorDeinitializedAction { + /// Indicates that `onTermination` should be called. + case callOnTermination((@Sendable () -> Void)?) + /// Indicates that all producers should be failed and `onTermination` should be called. + case failProducersAndCallOnTermination( + [(Result) -> Void], + (@Sendable () -> Void)? + ) + } + + mutating func iteratorDeinitialized() -> IteratorDeinitializedAction? { + switch self._state { + case .initial(let initial): + if initial.iteratorInitialized { + // An iterator was created and deinited. Since we only support + // a single iterator we can now transition to finish. + self._state = .finished(iteratorInitialized: true) + return .callOnTermination(initial.onTermination) + } else { + // An iterator needs to be initialized before it can be deinitialized. + fatalError("AsyncStream internal inconsistency") + } + + case .streaming(let streaming): + if streaming.iteratorInitialized { + // An iterator was created and deinited. Since we only support + // a single iterator we can now transition to finish. + self._state = .finished(iteratorInitialized: true) + + return .failProducersAndCallOnTermination( + Array(streaming.producerContinuations.map { $0.1 }), + streaming.onTermination + ) + } else { + // An iterator needs to be initialized before it can be deinitialized. + fatalError("AsyncStream internal inconsistency") + } + + case .sourceFinished(let sourceFinished): + if sourceFinished.iteratorInitialized { + // An iterator was created and deinited. Since we only support + // a single iterator we can now transition to finish. + self._state = .finished(iteratorInitialized: true) + return .callOnTermination(sourceFinished.onTermination) + } else { + // An iterator needs to be initialized before it can be deinitialized. + fatalError("AsyncStream internal inconsistency") + } + + case .finished: + // We are already finished so there is nothing left to clean up. + // This is just the references dropping afterwards. + return .none + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `sourceDeinitialized()`. + enum SourceDeinitializedAction { + /// Indicates that `onTermination` should be called. + case callOnTermination((() -> Void)?) + /// Indicates that all producers should be failed and `onTermination` should be called. + case failProducersAndCallOnTermination( + [(Result) -> Void], + (@Sendable () -> Void)? + ) + /// Indicates that all producers should be failed. + case failProducers([(Result) -> Void]) + } + + mutating func sourceDeinitialized() -> SourceDeinitializedAction? { + switch self._state { + case .initial(let initial): + // The source got deinited before anything was written + self._state = .finished(iteratorInitialized: initial.iteratorInitialized) + return .callOnTermination(initial.onTermination) + + case .streaming(let streaming): + if streaming.buffer.isEmpty { + // We can transition to finished right away since the buffer is empty now + self._state = .finished(iteratorInitialized: streaming.iteratorInitialized) + + return .failProducersAndCallOnTermination( + Array(streaming.producerContinuations.map { $0.1 }), + streaming.onTermination + ) + } else { + // The continuation must be `nil` if the buffer has elements + precondition(streaming.consumerContinuation == nil) + + self._state = .sourceFinished( + .init( + iteratorInitialized: streaming.iteratorInitialized, + buffer: streaming.buffer, + failure: nil, + onTermination: streaming.onTermination + ) + ) + + return .failProducers( + Array(streaming.producerContinuations.map { $0.1 }) + ) + } + + case .sourceFinished, .finished: + // This is normal and we just have to tolerate it + return .none + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `write()`. + enum WriteAction { + /// Indicates that the producer should be notified to produce more. + case returnProduceMore + /// Indicates that the producer should be suspended to stop producing. + case returnEnqueue( + callbackToken: Source.WriteResult.CallbackToken + ) + /// Indicates that the consumer should be resumed and the producer should be notified to produce more. + case resumeConsumerAndReturnProduceMore( + continuation: CheckedContinuation, + element: Element + ) + /// Indicates that the consumer should be resumed and the producer should be suspended. + case resumeConsumerAndReturnEnqueue( + continuation: CheckedContinuation, + element: Element, + callbackToken: Source.WriteResult.CallbackToken + ) + /// Indicates that the producer has been finished. + case throwFinishedError + + init( + callbackToken: Source.WriteResult.CallbackToken?, + continuationAndElement: (CheckedContinuation, Element)? = nil + ) { + switch (callbackToken, continuationAndElement) { + case (.none, .none): + self = .returnProduceMore + + case (.some(let callbackToken), .none): + self = .returnEnqueue(callbackToken: callbackToken) + + case (.none, .some((let continuation, let element))): + self = .resumeConsumerAndReturnProduceMore( + continuation: continuation, + element: element + ) + + case (.some(let callbackToken), .some((let continuation, let element))): + self = .resumeConsumerAndReturnEnqueue( + continuation: continuation, + element: element, + callbackToken: callbackToken + ) + } + } + } + + mutating func write(_ sequence: some Sequence) -> WriteAction { + switch self._state { + case .initial(var initial): + var buffer = Deque() + buffer.append(contentsOf: sequence) + + let shouldProduceMore = initial.backPressureStrategy.didYield( + bufferDepth: buffer.count + ) + let callbackToken = shouldProduceMore ? nil : self.nextCallbackToken() + + self._state = .streaming( + .init( + backPressureStrategy: initial.backPressureStrategy, + iteratorInitialized: initial.iteratorInitialized, + onTermination: initial.onTermination, + buffer: buffer, + consumerContinuation: nil, + producerContinuations: .init(), + cancelledAsyncProducers: .init(), + hasOutstandingDemand: shouldProduceMore + ) + ) + + return .init(callbackToken: callbackToken) + + case .streaming(var streaming): + self._state = .modify + + streaming.buffer.append(contentsOf: sequence) + + // We have an element and can resume the continuation + let shouldProduceMore = streaming.backPressureStrategy.didYield( + bufferDepth: streaming.buffer.count + ) + streaming.hasOutstandingDemand = shouldProduceMore + let callbackToken = shouldProduceMore ? nil : self.nextCallbackToken() + + if let consumerContinuation = streaming.consumerContinuation { + guard let element = streaming.buffer.popFirst() else { + // We got a yield of an empty sequence. We just tolerate this. + self._state = .streaming(streaming) + + return .init(callbackToken: callbackToken) + } + + // We got a consumer continuation and an element. We can resume the consumer now + streaming.consumerContinuation = nil + self._state = .streaming(streaming) + return .init( + callbackToken: callbackToken, + continuationAndElement: (consumerContinuation, element) + ) + } else { + // We don't have a suspended consumer so we just buffer the elements + self._state = .streaming(streaming) + return .init( + callbackToken: callbackToken + ) + } + + case .sourceFinished, .finished: + // If the source has finished we are dropping the elements. + return .throwFinishedError + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `enqueueProducer()`. + enum EnqueueProducerAction { + /// Indicates that the producer should be notified to produce more. + case resumeProducer((Result) -> Void) + /// Indicates that the producer should be notified about an error. + case resumeProducerWithError((Result) -> Void, Error) + } + + mutating func enqueueProducer( + callbackToken: Source.WriteResult.CallbackToken, + onProduceMore: @Sendable @escaping (Result) -> Void + ) -> EnqueueProducerAction? { + switch self._state { + case .initial: + // We need to transition to streaming before we can suspend + // This is enforced because the CallbackToken has no internal init so + // one must create it by calling `write` first. + fatalError("AsyncStream internal inconsistency") + + case .streaming(var streaming): + if let index = streaming.cancelledAsyncProducers.firstIndex(of: callbackToken.id) { + // Our producer got marked as cancelled. + self._state = .modify + streaming.cancelledAsyncProducers.remove(at: index) + self._state = .streaming(streaming) + + return .resumeProducerWithError(onProduceMore, CancellationError()) + } else if streaming.hasOutstandingDemand { + // We hit an edge case here where we wrote but the consuming thread got interleaved + return .resumeProducer(onProduceMore) + } else { + self._state = .modify + streaming.producerContinuations.append((callbackToken.id, onProduceMore)) + + self._state = .streaming(streaming) + return .none + } + + case .sourceFinished, .finished: + // Since we are unlocking between yielding and suspending the yield + // It can happen that the source got finished or the consumption fully finishes. + return .resumeProducerWithError(onProduceMore, AlreadyFinishedError()) + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `cancelProducer()`. + enum CancelProducerAction { + /// Indicates that the producer should be notified about cancellation. + case resumeProducerWithCancellationError((Result) -> Void) + } + + mutating func cancelProducer( + callbackToken: Source.WriteResult.CallbackToken + ) -> CancelProducerAction? { + switch self._state { + case .initial: + // We need to transition to streaming before we can suspend + fatalError("AsyncStream internal inconsistency") + + case .streaming(var streaming): + if let index = streaming.producerContinuations.firstIndex(where: { + $0.0 == callbackToken.id + }) { + // We have an enqueued producer that we need to resume now + self._state = .modify + let continuation = streaming.producerContinuations.remove(at: index).1 + self._state = .streaming(streaming) + + return .resumeProducerWithCancellationError(continuation) + } else { + // The task that yields was cancelled before yielding so the cancellation handler + // got invoked right away + self._state = .modify + streaming.cancelledAsyncProducers.append(callbackToken.id) + self._state = .streaming(streaming) + + return .none + } + + case .sourceFinished, .finished: + // Since we are unlocking between yielding and suspending the yield + // It can happen that the source got finished or the consumption fully finishes. + return .none + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `finish()`. + enum FinishAction { + /// Indicates that `onTermination` should be called. + case callOnTermination((() -> Void)?) + /// Indicates that the consumer should be resumed with the failure, the producers + /// should be resumed with an error and `onTermination` should be called. + case resumeConsumerAndCallOnTermination( + consumerContinuation: CheckedContinuation, + failure: Error?, + onTermination: (() -> Void)? + ) + /// Indicates that the producers should be resumed with an error. + case resumeProducers( + producerContinuations: [(Result) -> Void] + ) + } + + @inlinable + mutating func finish(_ failure: Error?) -> FinishAction? { + switch self._state { + case .initial(let initial): + // Nothing was yielded nor did anybody call next + // This means we can transition to sourceFinished and store the failure + self._state = .sourceFinished( + .init( + iteratorInitialized: initial.iteratorInitialized, + buffer: .init(), + failure: failure, + onTermination: initial.onTermination + ) + ) + + return .callOnTermination(initial.onTermination) + + case .streaming(let streaming): + if let consumerContinuation = streaming.consumerContinuation { + // We have a continuation, this means our buffer must be empty + // Furthermore, we can now transition to finished + // and resume the continuation with the failure + precondition(streaming.buffer.isEmpty, "Expected an empty buffer") + precondition( + streaming.producerContinuations.isEmpty, + "Expected no suspended producers" + ) + + self._state = .finished(iteratorInitialized: streaming.iteratorInitialized) + + return .resumeConsumerAndCallOnTermination( + consumerContinuation: consumerContinuation, + failure: failure, + onTermination: streaming.onTermination + ) + } else { + self._state = .sourceFinished( + .init( + iteratorInitialized: streaming.iteratorInitialized, + buffer: streaming.buffer, + failure: failure, + onTermination: streaming.onTermination + ) + ) + + return .resumeProducers( + producerContinuations: Array(streaming.producerContinuations.map { $0.1 }) + ) + } + + case .sourceFinished, .finished: + // If the source has finished, finishing again has no effect. + return .none + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `next()`. + enum NextAction { + /// Indicates that the element should be returned to the caller. + case returnElement(Element) + /// Indicates that the element should be returned to the caller and that all producers should be called. + case returnElementAndResumeProducers(Element, [(Result) -> Void]) + /// Indicates that the `Error` should be returned to the caller and that `onTermination` should be called. + case returnErrorAndCallOnTermination(Error?, (() -> Void)?) + /// Indicates that the `nil` should be returned to the caller. + case returnNil + /// Indicates that the `Task` of the caller should be suspended. + case suspendTask + } + + mutating func next() -> NextAction { + switch self._state { + case .initial(let initial): + // We are not interacting with the back-pressure strategy here because + // we are doing this inside `next(:)` + self._state = .streaming( + .init( + backPressureStrategy: initial.backPressureStrategy, + iteratorInitialized: initial.iteratorInitialized, + onTermination: initial.onTermination, + buffer: Deque(), + consumerContinuation: nil, + producerContinuations: .init(), + cancelledAsyncProducers: .init(), + hasOutstandingDemand: false + ) + ) + + return .suspendTask + case .streaming(var streaming): + guard streaming.consumerContinuation == nil else { + // We have multiple AsyncIterators iterating the sequence + fatalError("AsyncStream internal inconsistency") + } + + self._state = .modify + + if let element = streaming.buffer.popFirst() { + // We have an element to fulfil the demand right away. + let shouldProduceMore = streaming.backPressureStrategy.didConsume( + bufferDepth: streaming.buffer.count + ) + streaming.hasOutstandingDemand = shouldProduceMore + + if shouldProduceMore { + // There is demand and we have to resume our producers + let producers = Array(streaming.producerContinuations.map { $0.1 }) + streaming.producerContinuations.removeAll() + self._state = .streaming(streaming) + return .returnElementAndResumeProducers(element, producers) + } else { + // We don't have any new demand, so we can just return the element. + self._state = .streaming(streaming) + return .returnElement(element) + } + } else { + // There is nothing in the buffer to fulfil the demand so we need to suspend. + // We are not interacting with the back-pressure strategy here because + // we are doing this inside `suspendNext` + self._state = .streaming(streaming) + + return .suspendTask + } + + case .sourceFinished(var sourceFinished): + // Check if we have an element left in the buffer and return it + self._state = .modify + + if let element = sourceFinished.buffer.popFirst() { + self._state = .sourceFinished(sourceFinished) + + return .returnElement(element) + } else { + // We are returning the queued failure now and can transition to finished + self._state = .finished(iteratorInitialized: sourceFinished.iteratorInitialized) + + return .returnErrorAndCallOnTermination( + sourceFinished.failure, + sourceFinished.onTermination + ) + } + + case .finished: + return .returnNil + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `suspendNext()`. + enum SuspendNextAction { + /// Indicates that the consumer should be resumed. + case resumeConsumerWithElement(CheckedContinuation, Element) + /// Indicates that the consumer and all producers should be resumed. + case resumeConsumerWithElementAndProducers( + CheckedContinuation, + Element, + [(Result) -> Void] + ) + /// Indicates that the consumer should be resumed with the failure and that `onTermination` should be called. + case resumeConsumerWithErrorAndCallOnTermination( + CheckedContinuation, + Error?, + (() -> Void)? + ) + /// Indicates that the consumer should be resumed with `nil`. + case resumeConsumerWithNil(CheckedContinuation) + } + + mutating func suspendNext( + continuation: CheckedContinuation + ) -> SuspendNextAction? { + switch self._state { + case .initial: + // We need to transition to streaming before we can suspend + preconditionFailure("AsyncStream internal inconsistency") + + case .streaming(var streaming): + guard streaming.consumerContinuation == nil else { + // We have multiple AsyncIterators iterating the sequence + fatalError( + "This should never happen since we only allow a single Iterator to be created" + ) + } + + self._state = .modify + + // We have to check here again since we might have a producer interleave next and suspendNext + if let element = streaming.buffer.popFirst() { + // We have an element to fulfil the demand right away. + + let shouldProduceMore = streaming.backPressureStrategy.didConsume( + bufferDepth: streaming.buffer.count + ) + streaming.hasOutstandingDemand = shouldProduceMore + + if shouldProduceMore { + // There is demand and we have to resume our producers + let producers = Array(streaming.producerContinuations.map { $0.1 }) + streaming.producerContinuations.removeAll() + self._state = .streaming(streaming) + return .resumeConsumerWithElementAndProducers( + continuation, + element, + producers + ) + } else { + // We don't have any new demand, so we can just return the element. + self._state = .streaming(streaming) + return .resumeConsumerWithElement(continuation, element) + } + } else { + // There is nothing in the buffer to fulfil the demand so we to store the continuation. + streaming.consumerContinuation = continuation + self._state = .streaming(streaming) + + return .none + } + + case .sourceFinished(var sourceFinished): + // Check if we have an element left in the buffer and return it + self._state = .modify + + if let element = sourceFinished.buffer.popFirst() { + self._state = .sourceFinished(sourceFinished) + + return .resumeConsumerWithElement(continuation, element) + } else { + // We are returning the queued failure now and can transition to finished + self._state = .finished(iteratorInitialized: sourceFinished.iteratorInitialized) + + return .resumeConsumerWithErrorAndCallOnTermination( + continuation, + sourceFinished.failure, + sourceFinished.onTermination + ) + } + + case .finished: + return .resumeConsumerWithNil(continuation) + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + + /// Actions returned by `cancelNext()`. + enum CancelNextAction { + /// Indicates that the continuation should be resumed with a cancellation error, the producers should be finished and call onTermination. + case resumeConsumerWithCancellationErrorAndCallOnTermination( + CheckedContinuation, + (() -> Void)? + ) + /// Indicates that the producers should be finished and call onTermination. + case failProducersAndCallOnTermination([(Result) -> Void], (() -> Void)?) + } + + mutating func cancelNext() -> CancelNextAction? { + switch self._state { + case .initial: + // We need to transition to streaming before we can suspend + fatalError("AsyncStream internal inconsistency") + + case .streaming(let streaming): + self._state = .finished(iteratorInitialized: streaming.iteratorInitialized) + + if let consumerContinuation = streaming.consumerContinuation { + precondition( + streaming.producerContinuations.isEmpty, + "Internal inconsistency. Unexpected producer continuations." + ) + return .resumeConsumerWithCancellationErrorAndCallOnTermination( + consumerContinuation, + streaming.onTermination + ) + } else { + return .failProducersAndCallOnTermination( + Array(streaming.producerContinuations.map { $0.1 }), + streaming.onTermination + ) + } + + case .sourceFinished, .finished: + return .none + + case .modify: + fatalError("AsyncStream internal inconsistency") + } + } + } +} diff --git a/Sources/NIOFileSystem/Internal/Cancellation.swift b/Sources/NIOFileSystem/Internal/Cancellation.swift new file mode 100644 index 0000000000..b74b4d6b44 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/Cancellation.swift @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/// Executes the closure and masks cancellation. +@_spi(Testing) +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public func withoutCancellation( + _ execute: @escaping () async throws -> R +) async throws -> R { + // Okay as we immediately wait for the result of the task. + let unsafeExecute = UnsafeTransfer(execute) + let t = Task { + try await unsafeExecute.wrappedValue() + } + return try await t.value +} + +/// Executes `fn` and then `tearDown`, which cannot be cancelled. +@_spi(Testing) +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public func withUncancellableTearDown( + _ fn: () async throws -> R, + tearDown: @escaping (Result) async throws -> Void +) async throws -> R { + let result: Result + do { + result = .success(try await fn()) + } catch { + result = .failure(error) + } + + let tearDownResult: Result = try await withoutCancellation { + do { + return .success(try await tearDown(result)) + } catch { + return .failure(error) + } + } + + try tearDownResult.get() + return try result.get() +} diff --git a/Sources/NIOFileSystem/Internal/Concurrency Primitives/IOExecutor.swift b/Sources/NIOFileSystem/Internal/Concurrency Primitives/IOExecutor.swift new file mode 100644 index 0000000000..45344c4bb5 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/Concurrency Primitives/IOExecutor.swift @@ -0,0 +1,421 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Atomics +import DequeModule +import Dispatch +import NIOConcurrencyHelpers + +@_spi(Testing) +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public final class IOExecutor: Sendable { + /// Used to generate IDs for work items. Don't use directly, + /// use ``generateWorkID()`` instead. + private let workID = ManagedAtomic(UInt64(0)) + /// The workers. + private let workers: [Worker] + + /// Generates a unique ID for a work item. This is used to identify the work + /// for cancellation. + private func generateWorkID() -> UInt64 { + // We only require uniqueness: relaxed is sufficient. + return self.workID.loadThenWrappingIncrement(ordering: .relaxed) + } + + /// Create a running executor with the given number of threads. + /// + /// - Precondition: `numberOfThreads` must be greater than zero. + /// - Returns: a running executor which does work on `numberOfThreads` threads. + @_spi(Testing) + public static func running(numberOfThreads: Int) async -> IOExecutor { + let executor = IOExecutor(numberOfThreads: numberOfThreads) + await executor.start() + return executor + } + + /// Create a running executor with the given number of threads. + /// + /// - Precondition: `numberOfThreads` must be greater than zero. + /// - Returns: a running executor which does work on `numberOfThreads` threads. + @_spi(Testing) + public static func runningAsync(numberOfThreads: Int) -> IOExecutor { + let executor = IOExecutor(numberOfThreads: numberOfThreads) + Task { await executor.start() } + return executor + } + + private init(numberOfThreads: Int) { + precondition(numberOfThreads > 0, "numberOfThreads must be greater than zero") + var workers = [Worker]() + workers.reserveCapacity(numberOfThreads) + for _ in 0..( + _ work: @Sendable @escaping () throws -> R + ) async throws -> R { + let workerIndex = self.pickWorker() + let workID = self.generateWorkID() + + return try await withTaskCancellationHandler { + return try await withUnsafeThrowingContinuation { continuation in + self.workers[workerIndex].enqueue(id: workID) { action in + switch action { + case .run: + continuation.resume(with: Result(catching: work)) + case .cancel: + continuation.resume(throwing: CancellationError()) + case .reject: + let error = FileSystemError( + code: .unavailable, + message: "The executor has been shutdown.", + cause: nil, + location: .here() + ) + continuation.resume(throwing: error) + } + } + } + } onCancel: { + self.workers[workerIndex].cancel(workID: workID) + } + } + + /// Executes work on a thread owned by the executor and notifies a completion + /// handler with the result. + /// + /// - Parameters: + /// - work: A closure to execute. + /// - onCompletion: A closure to notify with the result of the work. + @_spi(Testing) + public func execute( + work: @Sendable @escaping () throws -> R, + onCompletion: @Sendable @escaping (Result) -> Void + ) { + let workerIndex = self.pickWorker() + let workID = self.generateWorkID() + self.workers[workerIndex].enqueue(id: workID) { action in + switch action { + case .run: + onCompletion(Result(catching: work)) + case .cancel: + onCompletion(.failure(CancellationError())) + case .reject: + let error = FileSystemError( + code: .unavailable, + message: "The executor has been shutdown.", + cause: nil, + location: .here() + ) + onCompletion(.failure(error)) + } + } + } + + /// Stop accepting new tasks and wait for all enqueued work to finish. + /// + /// Any work submitted to the executor whilst it is draining or once it + /// has finished draining will not be executed. + @_spi(Testing) + public func drain() async { + await withTaskGroup(of: Void.self) { group in + for worker in self.workers { + group.addTask { + await worker.stop() + } + } + } + } + + /// Returns the index of the work to submit the next item of work on. + /// + /// If there are more than two workers then the "power of two random choices" + /// approach (https://www.eecs.harvard.edu/~michaelm/postscripts/tpds2001.pdf) is + /// used whereby two workers are selected at random and the one with the least + /// amount of work is chosen. + /// + /// If there are one or two workers then the one with the least work is + /// chosen. + private func pickWorker() -> Int { + // 'numberOfThreads' is guaranteed to be > 0. + switch self.workers.count { + case 1: + return 0 + case 2: + return self.indexOfLeastBusyWorkerAtIndices(0, 1) + default: + // For more than two threads use the 'power of two random choices'. + let i = self.workers.indices.randomElement()! + let j = self.workers.indices.randomElement()! + + if i == j { + return i + } else { + return self.indexOfLeastBusyWorkerAtIndices(i, j) + } + } + } + + private func indexOfLeastBusyWorkerAtIndices(_ i: Int, _ j: Int) -> Int { + let workOnI = self.workers[i].numberOfQueuedTasks + let workOnJ = self.workers[j].numberOfQueuedTasks + return workOnI < workOnJ ? i : j + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension IOExecutor { + final class Worker: @unchecked Sendable { + /// An item of work executed by the worker. + typealias WorkItem = @Sendable (WorkAction) -> Void + + /// Whether the work should be executed or cancelled. + enum WorkAction { + case run + case cancel + case reject + } + + /// The state of the worker. + private enum State { + case notStarted + case starting(CheckedContinuation) + case active(Thread) + case draining(CheckedContinuation) + case stopped + + mutating func starting(continuation: CheckedContinuation) { + switch self { + case .notStarted: + self = .starting(continuation) + case .active, .starting, .draining, .stopped: + fatalError("\(#function) while worker was active/starting/draining/stopped") + } + } + + mutating func activate(_ thread: Thread) -> CheckedContinuation { + switch self { + case let .starting(continuation): + self = .active(thread) + return continuation + case .notStarted, .active, .draining, .stopped: + fatalError("\(#function) while worker was notStarted/active/draining/stopped") + } + } + + mutating func drained() -> CheckedContinuation { + switch self { + case let .draining(continuation): + self = .stopped + return continuation + case .notStarted, .starting, .active, .stopped: + fatalError("\(#function) while worker was notStarted/starting/active/stopped") + } + } + + var isStopped: Bool { + switch self { + case .stopped: + return true + case .notStarted, .starting, .active, .draining: + return true + } + } + } + + private let lock = NIOLock() + private var state: State + /// Items of work waiting to be executed and their ID. + private var workQueue: Deque<(UInt64, WorkItem)> + /// IDs of work items which have been marked as cancelled but not yet + /// processed. + private var cancelledTasks: [UInt64] + // TODO: make checking for cancellation cheaper by maintaining a heap of cancelled + // IDs and ensuring work is in order of increasing ID. With that we can check the ID + // of the popped work item against the top of the heap. + + /// Signalled when an item of work is added to the work queue. + private let semaphore: DispatchSemaphore + /// The number of items in the work queue. This is used by the executor to + /// make decisions on where to enqueue work so relaxed memory ordering is fine. + private let queuedWork = ManagedAtomic(0) + + internal init() { + self.state = .notStarted + self.semaphore = DispatchSemaphore(value: 0) + self.workQueue = [] + self.cancelledTasks = [] + } + + deinit { + assert(self.state.isStopped, "The IOExecutor MUST be shutdown before deinit") + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension IOExecutor.Worker { + func start(name: String) async { + self.lock.lock() + await withCheckedContinuation { continuation in + self.state.starting(continuation: continuation) + self.lock.unlock() + + Thread.spawnAndRun(name: name, detachThread: false) { thread in + let continuation = self.lock.withLock { self.state.activate(thread) } + continuation.resume() + self.process() + } + } + } + + var numberOfQueuedTasks: Int { + return self.queuedWork.load(ordering: .relaxed) + } + + func enqueue(id: UInt64, _ item: @escaping WorkItem) { + let didEnqueue = self.lock.withLock { + self._enqueue(id: id, item) + } + + if didEnqueue { + self.queuedWork.wrappingIncrement(ordering: .relaxed) + self.semaphore.signal() + } else { + // Draining or stopped. + item(.reject) + } + } + + private func _enqueue(id: UInt64, _ item: @escaping WorkItem) -> Bool { + switch self.state { + case .notStarted, .starting, .active: + self.workQueue.append((id, item)) + return true + case .draining, .stopped: + // New work is rejected in these states. + return false + } + } + + func cancel(workID: UInt64) { + self.lock.withLock { + // The work will be cancelled when pulled from the work queue. This means + // there's a delay before the work is actually cancelled; we trade that off + // against the cost of scanning the work queue for an item to cancel and then + // removing it which is O(n) (which could be O(n^2) for bulk cancellation). + self.cancelledTasks.append(workID) + } + } + + func stop() async { + self.lock.lock() + switch self.state { + case .notStarted: + self.state = .stopped + while let work = self.workQueue.popFirst() { + work.1(.reject) + } + + case let .active(thread): + await withCheckedContinuation { continuation in + self.state = .draining(continuation) + self.lock.unlock() + + // Signal the semaphore: 'process()' waits on the semaphore for work so + // always expects the queue to be non-empty. This is the exception that + // indicates to 'process()' that it can stop. + self.semaphore.signal() + } + precondition(self.lock.withLock({ self.state.isStopped })) + // This won't block, we just drained the work queue so 'self.process()' will have + // returned + thread.join() + + case .starting, .draining: + fatalError("Worker is already starting/draining") + + case .stopped: + self.lock.unlock() + } + } + + private func process() { + enum Instruction { + case run(WorkItem) + case cancel(WorkItem) + case stopWorking(CheckedContinuation) + } + + while true { + // Wait for work to be signalled via the semaphore. It is signalled for every time an + // item of work is added to the queue. It is signalled an additional time when the worker + // is shutting down: in that case there is no corresponding work item in the queue so + // 'popFirst()' will return 'nil'; that's the signal to stop looping. + self.semaphore.wait() + + let instruction: Instruction = self.lock.withLock { + switch self.state { + case let .draining(continuation): + if self.workQueue.isEmpty { + self.state = .stopped + return .stopWorking(continuation) + } + // There's still work to do: continue as if active. + fallthrough + + case .active: + let (id, work) = self.workQueue.removeFirst() + if let index = self.cancelledTasks.firstIndex(of: id) { + self.cancelledTasks.remove(at: index) + return .cancel(work) + } else { + return .run(work) + } + + case .notStarted, .starting, .stopped: + fatalError("Impossible state") + } + } + + switch instruction { + case let .run(work): + work(.run) + case let .cancel(work): + work(.cancel) + case let .stopWorking(continuation): + continuation.resume() + return + } + } + } +} diff --git a/Sources/NIOFileSystem/Internal/Concurrency Primitives/Thread.swift b/Sources/NIOFileSystem/Internal/Concurrency Primitives/Thread.swift new file mode 100644 index 0000000000..d75841eb1f --- /dev/null +++ b/Sources/NIOFileSystem/Internal/Concurrency Primitives/Thread.swift @@ -0,0 +1,97 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if os(Linux) || os(FreeBSD) || os(Android) +import CNIOLinux +#endif + +protocol ThreadOps { + associatedtype ThreadHandle + associatedtype ThreadSpecificKey + associatedtype ThreadSpecificKeyDestructor + + static func run( + handle: inout ThreadHandle?, + args: Ref, + detachThread: Bool + ) + static func joinThread(_ thread: ThreadHandle) + static func threadName(_ thread: ThreadHandle) -> String? + static func compareThreads(_ lhs: ThreadHandle, _ rhs: ThreadHandle) -> Bool +} + +/// A Thread that executes some runnable block. +/// +/// All methods exposed are thread-safe. +final class Thread { + internal typealias ThreadBoxValue = (body: (Thread) -> Void, name: String?) + internal typealias ThreadBox = Ref + + private let desiredName: String? + + /// The thread handle used by this instance. + private let handle: ThreadOpsSystem.ThreadHandle + + /// Create a new instance + /// + /// - arguments: + /// - handle: The `ThreadOpsSystem.ThreadHandle` that is wrapped and used by the `Thread`. + internal init(handle: ThreadOpsSystem.ThreadHandle, desiredName: String?) { + self.handle = handle + self.desiredName = desiredName + } + + /// Execute the given body with the `pthread_t` that is used by this `Thread` as argument. + /// + /// - warning: Do not escape `pthread_t` from the closure for later use. + /// + /// - parameters: + /// - body: The closure that will accept the `pthread_t`. + /// - returns: The value returned by `body`. + internal func withUnsafeThreadHandle( + _ body: (ThreadOpsSystem.ThreadHandle) throws -> T + ) rethrows -> T { + return try body(self.handle) + } + + /// Get current name of the `Thread` or `nil` if not set. + var currentName: String? { + return ThreadOpsSystem.threadName(self.handle) + } + + func join() { + ThreadOpsSystem.joinThread(self.handle) + } + + /// Spawns and runs some task in a `Thread`. + /// + /// - arguments: + /// - name: The name of the `Thread` or `nil` if no specific name should be set. + /// - body: The function to execute within the spawned `Thread`. + /// - detach: Whether to detach the thread. If the thread is not detached it must be `join`ed. + static func spawnAndRun( + name: String? = nil, + detachThread: Bool = true, + body: @escaping (Thread) -> Void + ) { + var handle: ThreadOpsSystem.ThreadHandle? = nil + + // Store everything we want to pass into the c function in a Box so we + // can hand-over the reference. + let tuple: ThreadBoxValue = (body: body, name: name) + let box = ThreadBox(tuple) + + ThreadOpsSystem.run(handle: &handle, args: box, detachThread: detachThread) + } +} diff --git a/Sources/NIOFileSystem/Internal/Concurrency Primitives/ThreadPosix.swift b/Sources/NIOFileSystem/Internal/Concurrency Primitives/ThreadPosix.swift new file mode 100644 index 0000000000..5eb30a1f73 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/Concurrency Primitives/ThreadPosix.swift @@ -0,0 +1,141 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Glibc) +import CNIOLinux + +private let sys_pthread_getname_np = CNIOLinux_pthread_getname_np +private let sys_pthread_setname_np = CNIOLinux_pthread_setname_np +private typealias ThreadDestructor = @convention(c) (UnsafeMutableRawPointer?) -> + UnsafeMutableRawPointer? +#elseif canImport(Darwin) +import Darwin + +private let sys_pthread_getname_np = pthread_getname_np +// Emulate the same method signature as pthread_setname_np on Linux. +private func sys_pthread_setname_np(_ p: pthread_t, _ pointer: UnsafePointer) -> Int32 { + assert(pthread_equal(pthread_self(), p) != 0) + pthread_setname_np(pointer) + // Will never fail on macOS so just return 0 which will be used on linux to signal it not failed. + return 0 +} +private typealias ThreadDestructor = @convention(c) (UnsafeMutableRawPointer) -> + UnsafeMutableRawPointer? +#endif + +private func sysPthread_create( + handle: UnsafeMutablePointer, + destructor: @escaping ThreadDestructor, + args: UnsafeMutableRawPointer? +) -> CInt { + #if canImport(Darwin) + return pthread_create(handle, nil, destructor, args) + #elseif canImport(Glibc) + var handleLinux = pthread_t() + let result = pthread_create( + &handleLinux, + nil, + destructor, + args + ) + handle.pointee = handleLinux + return result + #endif +} + +typealias ThreadOpsSystem = ThreadOpsPosix + +enum ThreadOpsPosix: ThreadOps { + typealias ThreadHandle = pthread_t + typealias ThreadSpecificKey = pthread_key_t + #if canImport(Darwin) + typealias ThreadSpecificKeyDestructor = @convention(c) (UnsafeMutableRawPointer) -> Void + #elseif canImport(Glibc) + typealias ThreadSpecificKeyDestructor = @convention(c) (UnsafeMutableRawPointer?) -> Void + #endif + + static func threadName(_ thread: ThreadOpsSystem.ThreadHandle) -> String? { + // 64 bytes should be good enough as on Linux the limit is usually 16 + // and it's very unlikely a user will ever set something longer + // anyway. + var chars: [CChar] = Array(repeating: 0, count: 64) + return chars.withUnsafeMutableBufferPointer { ptr in + guard sys_pthread_getname_np(thread, ptr.baseAddress!, ptr.count) == 0 else { + return nil + } + + let buffer: UnsafeRawBufferPointer = + UnsafeRawBufferPointer(UnsafeBufferPointer(rebasing: ptr.prefix { $0 != 0 })) + return String(decoding: buffer, as: Unicode.UTF8.self) + } + } + + static func run( + handle: inout ThreadOpsSystem.ThreadHandle?, + args: Ref, + detachThread: Bool + ) { + let argv0 = Unmanaged.passRetained(args).toOpaque() + let res = sysPthread_create( + handle: &handle, + destructor: { + // Cast to UnsafeMutableRawPointer? and force unwrap to make the + // same code work on macOS and Linux. + let boxed = Unmanaged + .fromOpaque(($0 as UnsafeMutableRawPointer?)!) + .takeRetainedValue() + let (body, name) = (boxed.value.body, boxed.value.name) + let hThread: ThreadOpsSystem.ThreadHandle = pthread_self() + + if let name = name { + let maximumThreadNameLength: Int + #if canImport(Glibc) + maximumThreadNameLength = 15 + #elseif canImport(Darwin) + maximumThreadNameLength = .max + #endif + name.prefix(maximumThreadNameLength).withCString { namePtr in + // this is non-critical so we ignore the result here, we've seen + // EPERM in containers. + _ = sys_pthread_setname_np(hThread, namePtr) + } + } + + body(Thread(handle: hThread, desiredName: name)) + + return nil + }, + args: argv0 + ) + precondition(res == 0, "Unable to create thread: \(res)") + + if detachThread { + let detachError = pthread_detach(handle!) + precondition(detachError == 0, "pthread_detach failed with error \(detachError)") + } + + } + + static func joinThread(_ thread: ThreadOpsSystem.ThreadHandle) { + let err = pthread_join(thread, nil) + assert(err == 0, "pthread_join failed with \(err)") + } + + static func compareThreads( + _ lhs: ThreadOpsSystem.ThreadHandle, + _ rhs: ThreadOpsSystem.ThreadHandle + ) -> Bool { + return pthread_equal(lhs, rhs) != 0 + } +} diff --git a/Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift b/Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift new file mode 100644 index 0000000000..c39dd0b5e7 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@usableFromInline +struct UnsafeTransfer: @unchecked Sendable { + @usableFromInline + var wrappedValue: Value + + @inlinable + init(_ wrappedValue: Value) { + self.wrappedValue = wrappedValue + } +} diff --git a/Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift b/Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift new file mode 100644 index 0000000000..54c20a92af --- /dev/null +++ b/Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +extension String { + @inlinable + init( + backportUnsafeUninitializedCapacity capacity: Int, + initializingUTF8With initializer: (_ buffer: UnsafeMutableBufferPointer) throws -> Int + ) rethrows { + // The buffer will store zero terminated C string + let buffer = UnsafeMutableBufferPointer.allocate(capacity: capacity + 1) + defer { + buffer.deallocate() + } + + let initializedCount = try initializer(buffer) + precondition(initializedCount <= capacity, "Overran buffer in initializer!") + + // add zero termination + buffer[initializedCount] = 0 + + self = String(cString: buffer.baseAddress!) + } +} + +// Frustratingly, Swift 5.3 shipped before the macOS 11 SDK did, so we cannot gate the availability of +// this declaration on having the 5.3 compiler. This has caused a number of build issues. While updating +// to newer Xcodes does work, we can save ourselves some hassle and just wait until 5.4 to get this +// enhancement on Apple platforms. +extension String { + @inlinable + init( + customUnsafeUninitializedCapacity capacity: Int, + initializingUTF8With initializer: ( + _ buffer: UnsafeMutableBufferPointer + ) throws -> Int + ) rethrows { + if #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) { + try self.init( + unsafeUninitializedCapacity: capacity, + initializingUTF8With: initializer + ) + } else { + try self.init( + backportUnsafeUninitializedCapacity: capacity, + initializingUTF8With: initializer + ) + } + } +} diff --git a/Sources/NIOFileSystem/Internal/System Calls/CInterop.swift b/Sources/NIOFileSystem/Internal/System Calls/CInterop.swift new file mode 100644 index 0000000000..e6b3e9a170 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/System Calls/CInterop.swift @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(Darwin) +import Darwin +import CNIODarwin +#elseif canImport(Glibc) +import Glibc +import CNIOLinux +#endif + +/// Aliases for platform-dependent types used for system calls. +extension CInterop { + #if canImport(Darwin) + public typealias Stat = Darwin.stat + #elseif canImport(Glibc) + public typealias Stat = Glibc.stat + #endif + + #if canImport(Darwin) + @_spi(Testing) + public static let maxPathLength = Darwin.PATH_MAX + #elseif canImport(Glibc) + @_spi(Testing) + public static let maxPathLength = Glibc.PATH_MAX + #endif + + #if canImport(Darwin) + typealias DirPointer = UnsafeMutablePointer + #elseif canImport(Glibc) + typealias DirPointer = OpaquePointer + #endif + + #if canImport(Darwin) + typealias DirEnt = Darwin.dirent + #elseif canImport(Glibc) + typealias DirEnt = Glibc.dirent + #endif + + #if canImport(Darwin) + typealias FTS = CNIODarwin.FTS + typealias FTSEnt = CNIODarwin.FTSENT + #elseif canImport(Glibc) + typealias FTS = CNIOLinux.FTS + typealias FTSEnt = CNIOLinux.FTSENT + #endif + + typealias FTSPointer = UnsafeMutablePointer + typealias FTSEntPointer = UnsafeMutablePointer +} diff --git a/Sources/NIOFileSystem/Internal/System Calls/Errno.swift b/Sources/NIOFileSystem/Internal/System Calls/Errno.swift new file mode 100644 index 0000000000..bd32734c33 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/System Calls/Errno.swift @@ -0,0 +1,132 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +extension Errno { + @_spi(Testing) + public static var _current: Errno { + get { + #if canImport(Darwin) + return Errno(rawValue: Darwin.errno) + #elseif canImport(Glibc) + return Errno(rawValue: Glibc.errno) + #endif + } + set { + #if canImport(Darwin) + Darwin.errno = newValue.rawValue + #elseif canImport(Glibc) + Glibc.errno = newValue.rawValue + #endif + } + } + + fileprivate static func clear() { + #if canImport(Darwin) + Darwin.errno = 0 + #elseif canImport(Glibc) + Glibc.errno = 0 + #endif + } +} + +/// Returns a `Result` representing the value returned from the given closure +/// or an `Errno` if that value was -1. +/// +/// If desired this function can call the closure in a loop until it does not +/// result in `Errno` being `.interrupted`. +@_spi(Testing) +public func valueOrErrno( + retryOnInterrupt: Bool = true, + _ fn: () -> I +) -> Result { + while true { + Errno.clear() + let result = fn() + if result == -1 { + let errno = Errno._current + if errno == .interrupted, retryOnInterrupt { + continue + } else { + return .failure(errno) + } + } else { + return .success(result) + } + } +} + +/// As `valueOrErrno` but discards the success value. +@_spi(Testing) +public func nothingOrErrno( + retryOnInterrupt: Bool = true, + _ fn: () -> I +) -> Result { + return valueOrErrno(retryOnInterrupt: retryOnInterrupt, fn).map { _ in } +} + +/// Returns a `Result` representing the value returned from the given closure +/// or an `Errno` if that value was `nil`. +/// +/// If desired this function can call the closure in a loop until it does not +/// result in `Errno` being `.interrupted`. `Errno` is only checked if the +/// closure returns `nil`. +@_spi(Testing) +public func optionalValueOrErrno( + retryOnInterrupt: Bool = true, + _ fn: () -> R? +) -> Result { + while true { + Errno.clear() + if let result = fn() { + return .success(result) + } else { + let errno = Errno._current + if errno == .interrupted, retryOnInterrupt { + continue + } else if errno.rawValue == 0 { + return .success(nil) + } else { + return .failure(errno) + } + } + } +} + +/// As `valueOrErrno` but unconditionally checks the current `Errno`. +@_spi(Testing) +public func valueOrErrno( + retryOnInterrupt: Bool = true, + _ fn: () -> R +) -> Result { + while true { + Errno.clear() + let value = fn() + let errno = Errno._current + if errno.rawValue == 0 { + return .success(value) + } else if errno == .interrupted, retryOnInterrupt { + continue + } else { + return .failure(errno) + } + } +} diff --git a/Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift b/Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift new file mode 100644 index 0000000000..3f0472905d --- /dev/null +++ b/Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift @@ -0,0 +1,320 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +import CNIOLinux +#endif + +extension FileDescriptor { + /// Opens or creates a file for reading or writing. + /// + /// The corresponding C function is `fdopenat`. + /// + /// - Parameters: + /// - path: The location of the file to open. If the path is relative then the file is opened + /// relative to the descriptor. + /// - mode: The read and write access to use. + /// - options: The behavior for opening the file. + /// - permissions: The file permissions to use for created files. + /// - retryOnInterrupt: Whether to retry the operation if it throws `Errno.interrupted`. The + /// default is true. Pass false to try only once and throw an error upon interruption. + /// - Returns: A file descriptor for the open file. + @_spi(Testing) + public func `open`( + atPath path: FilePath, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions, + permissions: FilePermissions?, + retryOnInterrupt: Bool = true + ) -> Result { + let oFlag = mode.rawValue | options.rawValue + let rawValue = valueOrErrno(retryOnInterrupt: retryOnInterrupt) { + path.withPlatformString { + if let permissions = permissions { + return system_fdopenat(self.rawValue, $0, oFlag, permissions.rawValue) + } else { + precondition(!options.contains(.create), "Create must be given permissions") + return system_fdopenat(self.rawValue, $0, oFlag) + } + } + } + + return rawValue.map { FileDescriptor(rawValue: $0) } + } + + /// Returns information about the status of the open file. + /// + /// The corresponding C function is `fstat`. + /// + /// - Returns: Information about the open file. + @_spi(Testing) + public func status() -> Result { + var status = CInterop.Stat() + return nothingOrErrno(retryOnInterrupt: false) { + system_fstat(self.rawValue, &status) + }.map { status } + } + + /// Sets the permission bits of the open file. + /// + /// The corresponding C function is `fchmod`. + /// + /// - Parameters: + /// - mode: The permissions to set on the file. + /// - retryOnInterrupt: Whether to retry the operation if it throws `Errno.interrupted`. The + /// default is true. Pass false to try only once and throw an error upon interruption. + @_spi(Testing) + public func changeMode( + _ mode: FilePermissions, + retryOnInterrupt: Bool = true + ) -> Result { + nothingOrErrno(retryOnInterrupt: retryOnInterrupt) { + system_fchmod(self.rawValue, mode.rawValue) + } + } + + /// List the names of extended attributes. + /// + /// The corresponding C function is `flistxattr`. + /// + /// - Parameter buffer: The buffer into which names are written. Names are written are NULL + /// terminated UTF-8 strings and are returned in an arbitrary order. There is no padding + /// between strings. If `buffer` is `nil` then the return value is the size of the buffer + /// required to list all extended attributes. If there is not enough space in the `buffer` + /// then `Errno.outOfRange` is returned. + /// - Returns: The size of the extended attribute list. + @_spi(Testing) + public func listExtendedAttributes( + _ buffer: UnsafeMutableBufferPointer? + ) -> Result { + return valueOrErrno(retryOnInterrupt: false) { + system_flistxattr(self.rawValue, buffer?.baseAddress, buffer?.count ?? 0) + } + } + + /// Get the value of the named extended attribute. + /// + /// The corresponding C function is `fgetxattr`. + /// + /// - Parameters: + /// - name: The name of the extended attribute. + /// - buffer: The buffer into which the value is written. If `buffer` is `nil` then the return + /// value is the size of the buffer required to read the value. If there is not enough + /// space in the `buffer` then `Errno.outOfRange` is returned. + /// - Returns: The size of the extended attribute value. + @_spi(Testing) + public func getExtendedAttribute( + named name: String, + buffer: UnsafeMutableRawBufferPointer? + ) -> Result { + return valueOrErrno(retryOnInterrupt: false) { + name.withPlatformString { + system_fgetxattr(self.rawValue, $0, buffer?.baseAddress, buffer?.count ?? 0) + } + } + } + + /// Set the value of the named extended attribute. + /// + /// The corresponding C function is `fsetxattr`. + /// + /// - Parameters: + /// - name: The name of the extended attribute. + /// - value: The data to set for the attribute. + @_spi(Testing) + public func setExtendedAttribute( + named name: String, + to value: UnsafeRawBufferPointer? + ) -> Result { + nothingOrErrno(retryOnInterrupt: false) { + name.withPlatformString { namePointer in + system_fsetxattr(self.rawValue, namePointer, value?.baseAddress, value?.count ?? 0) + } + } + } + + /// Remove the value for the named extended attribute. + /// + /// The corresponding C function is `fremovexattr`. + /// + /// - Parameters: + /// - name: The name of the extended attribute. + @_spi(Testing) + public func removeExtendedAttribute(_ name: String) -> Result { + nothingOrErrno(retryOnInterrupt: false) { + name.withPlatformString { + system_fremovexattr(self.rawValue, $0) + } + } + } + + /// Synchronize modified data and metadata to a permanent storage device. + /// + /// The corresponding C functions is `fsync`. + /// + /// - Parameter retryOnInterrupt: Whether the call should be retried on `Errno.interrupted`. + @_spi(Testing) + public func synchronize(retryOnInterrupt: Bool = true) -> Result { + nothingOrErrno(retryOnInterrupt: retryOnInterrupt) { + system_fsync(self.rawValue) + } + } + + /// Returns a pointer to a directory structure. + /// + /// The corresponding C function is `fdopendir` + /// + /// - Important: Calling this function cedes ownership of the file descriptor to the system. The + /// caller should not modify the descriptor or close the descriptor via `close()`. Once + /// directory iteration has been completed then `Libc.closdir(_:)` must be called. + internal func opendir() -> Result { + valueOrErrno(retryOnInterrupt: false) { + libc_fdopendir(self.rawValue) + } + } +} + +extension FileDescriptor { + func listExtendedAttributes() -> Result<[String], Errno> { + // Required capacity is returned if a no buffer is passed to flistxattr. + return self.listExtendedAttributes(nil).flatMap { capacity in + guard capacity > 0 else { + // Required capacity is zero: no attributes to read. + return .success([]) + } + + // Read and decode. + var buffer = [CChar](repeating: 0, count: capacity) + return buffer.withUnsafeMutableBufferPointer { pointer in + self.listExtendedAttributes(pointer) + }.map { size in + // The buffer contains null terminated C-strings. + var attributes = [String]() + var slice = buffer.prefix(size) + while let index = slice.firstIndex(of: 0) { + // TODO: can we do this more cheaply? + let prefix = slice[...index] + attributes.append(String(cString: Array(prefix))) + slice = slice.dropFirst(prefix.count) + } + + return attributes + } + } + } + + func readExtendedAttribute(named name: String) -> Result<[UInt8], Errno> { + // Required capacity is returned if a no buffer is passed to fgetxattr. + return self.getExtendedAttribute(named: name, buffer: nil).flatMap { capacity in + guard capacity > 0 else { + // Required capacity is zero: no values to read. + return .success([]) + } + + // Read and decode. + var buffer = [UInt8](repeating: 0, count: capacity) + return buffer.withUnsafeMutableBytes { bytes in + self.getExtendedAttribute(named: name, buffer: bytes) + }.map { size in + // Remove any trailing zeros. + buffer.removeLast(buffer.count - size) + return buffer + } + } + } +} + +extension FileDescriptor { + func readChunk(fromAbsoluteOffset offset: Int64, length: Int64) -> Result { + self._readChunk(fromAbsoluteOffset: offset, length: length) + } + + func readChunk(length: Int64) -> Result { + self._readChunk(fromAbsoluteOffset: nil, length: length) + } + + private func _readChunk( + fromAbsoluteOffset offset: Int64?, + length: Int64 + ) -> Result { + // This is used by the `FileChunks` and means we allocate for every chunk that we read for + // the file. That's fine for now because the syscall cost is likely to be the dominant + // factor here. However we should investigate whether it's possible to have a pool of + // buffers which we can reuse. This would need to be at least as large as the high watermark + // of the chunked file for it to be useful. + return Result { + var buffer = ByteBuffer() + try buffer.writeWithUnsafeMutableBytes(minimumWritableBytes: Int(length)) { buffer in + let bufferPointer: UnsafeMutableRawBufferPointer + + // Don't vend a buffer which is larger than `length`; we can read less but we must + // not read more. + if length < buffer.count { + bufferPointer = UnsafeMutableRawBufferPointer( + start: buffer.baseAddress, + count: Int(length) + ) + } else { + bufferPointer = buffer + } + + if let offset { + return try self.read(fromAbsoluteOffset: offset, into: bufferPointer) + } else { + return try self.read(into: bufferPointer) + } + } + return buffer + } + } +} + +extension FileDescriptor { + func write( + contentsOf bytes: some Sequence, + toAbsoluteOffset offset: Int64 + ) -> Result { + return Result { + Int64(try self.writeAll(toAbsoluteOffset: offset, bytes)) + } + } + + func write( + contentsOf bytes: some Sequence + ) -> Result { + return Result { + Int64(try self.writeAll(bytes)) + } + } +} + +#if canImport(Glibc) +extension FileDescriptor.OpenOptions { + static var temporaryFile: Self { + Self(rawValue: CNIOLinux_O_TMPFILE) + } +} + +extension FileDescriptor { + static var currentWorkingDirectory: Self { + Self(rawValue: AT_FDCWD) + } +} +#endif diff --git a/Sources/NIOFileSystem/Internal/System Calls/Mocking.swift b/Sources/NIOFileSystem/Internal/System Calls/Mocking.swift new file mode 100644 index 0000000000..ff8ee77d6d --- /dev/null +++ b/Sources/NIOFileSystem/Internal/System Calls/Mocking.swift @@ -0,0 +1,359 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/* + This source file is part of the Swift System open source project + + Copyright (c) 2020 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + */ + +import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +import CNIOLinux +#endif + +// Syscall mocking support. +// +// NOTE: This is currently the bare minimum needed for System's testing purposes, though we do +// eventually want to expose some solution to users. +// +// Mocking is contextual, accessible through MockingDriver.withMockingEnabled. Mocking +// state, including whether it is enabled, is stored in thread-local storage. Mocking is only +// enabled in testing builds of System currently, to minimize runtime overhead of release builds. +// + +#if ENABLE_MOCKING +@_spi(Testing) +public struct Trace { + @_spi(Testing) + public struct Entry { + @_spi(Testing) + public var name: String + @_spi(Testing) + public var arguments: [AnyHashable] + + @_spi(Testing) + public init(name: String, _ arguments: [AnyHashable]) { + self.name = name + self.arguments = arguments + } + } + + private var entries: [Entry] = [] + private var firstEntry: Int = 0 + + @_spi(Testing) + public var isEmpty: Bool { firstEntry >= entries.count } + + @_spi(Testing) + public mutating func dequeue() -> Entry? { + guard !self.isEmpty else { return nil } + defer { firstEntry += 1 } + return entries[firstEntry] + } + + fileprivate mutating func add(_ e: Entry) { + entries.append(e) + } +} + +@_spi(Testing) +public enum ForceErrno: Equatable { + case none + case always(errno: CInt) + + case counted(errno: CInt, count: Int) +} + +// Provide access to the driver, context, and trace stack of mocking +@_spi(Testing) +public final class MockingDriver { + // Record syscalls and their arguments + @_spi(Testing) + public var trace = Trace() + + // Mock errors inside syscalls + @_spi(Testing) + public var forceErrno = ForceErrno.none + + // Whether we should pretend to be Windows for syntactic operations + // inside FilePath + fileprivate var forceWindowsSyntaxForPaths = false +} + +private let driverKey: _PlatformTLSKey = { makeTLSKey() }() + +internal var currentMockingDriver: MockingDriver? { + #if !ENABLE_MOCKING + fatalError("Contextual mocking in non-mocking build") + #endif + guard let rawPtr = getTLS(driverKey) else { return nil } + + return Unmanaged.fromOpaque(rawPtr).takeUnretainedValue() +} + +extension MockingDriver { + /// Enables mocking for the duration of `f` with a clean trace queue + /// Restores prior mocking status and trace queue after execution + @_spi(Testing) + public static func withMockingEnabled( + _ f: (MockingDriver) throws -> Void + ) rethrows { + let priorMocking = currentMockingDriver + let driver = MockingDriver() + + defer { + if let object = priorMocking { + setTLS(driverKey, Unmanaged.passUnretained(object).toOpaque()) + } else { + setTLS(driverKey, nil) + } + _fixLifetime(driver) + } + + setTLS(driverKey, Unmanaged.passUnretained(driver).toOpaque()) + return try f(driver) + } +} + +// Check TLS for mocking +@inline(never) +private var contextualMockingEnabled: Bool { + return currentMockingDriver != nil +} + +extension MockingDriver { + @_spi(Testing) + public static var enabled: Bool { mockingEnabled } + + @_spi(Testing) + public static var forceWindowsPaths: Bool { + currentMockingDriver?.forceWindowsSyntaxForPaths ?? false + } +} + +#endif // ENABLE_MOCKING + +@inline(__always) +internal var mockingEnabled: Bool { + // Fast constant-foldable check for release builds + #if ENABLE_MOCKING + return contextualMockingEnabled + #else + return false + #endif +} + +@inline(__always) +internal var forceWindowsPaths: Bool { + #if !ENABLE_MOCKING + return false + #else + return MockingDriver.forceWindowsPaths + #endif +} + +#if ENABLE_MOCKING +// Strip the mock_system prefix and the arg list suffix +private func originalSyscallName(_ function: String) -> String { + // `function` must be of format `system_()` + for `prefix` in ["system_", "libc_"] { + if function.starts(with: `prefix`) { + return String(function.dropFirst(`prefix`.count).prefix { $0 != "(" }) + } + } + preconditionFailure("\(function) must start with 'system_' or 'libc_'") +} + +private func mockImpl(syscall name: String, args: [AnyHashable]) -> CInt { + precondition(mockingEnabled) + let origName = originalSyscallName(name) + guard let driver = currentMockingDriver else { + fatalError("Mocking requested from non-mocking context") + } + + driver.trace.add(Trace.Entry(name: origName, args)) + + switch driver.forceErrno { + case .none: break + case .always(let e): + system_errno = e + return -1 + case .counted(let e, let count): + assert(count >= 1) + system_errno = e + driver.forceErrno = count > 1 ? .counted(errno: e, count: count - 1) : .none + return -1 + } + + return 0 +} + +private func reinterpret(_ args: [AnyHashable?]) -> [AnyHashable] { + return args.map { arg in + switch arg { + case let charPointer as UnsafePointer: + return String(_errorCorrectingPlatformString: charPointer) + case is UnsafeMutablePointer: + return "" + case is UnsafeMutableRawPointer: + return "" + case is UnsafeRawPointer: + return "" + case .none: + return "nil" + case let .some(arg): + return arg + } + } +} + +func mock( + syscall name: String = #function, + _ args: AnyHashable?... +) -> CInt { + return mockImpl(syscall: name, args: reinterpret(args)) +} + +func mockInt( + syscall name: String = #function, + _ args: AnyHashable?... +) -> Int { + return Int(mockImpl(syscall: name, args: reinterpret(args))) +} + +#endif // ENABLE_MOCKING + +// Force paths to be treated as Windows syntactically if `enabled` is +// true. +@_spi(Testing) +public func _withWindowsPaths(enabled: Bool, _ body: () -> Void) { + #if ENABLE_MOCKING + guard enabled else { + body() + return + } + MockingDriver.withMockingEnabled { driver in + driver.forceWindowsSyntaxForPaths = true + body() + } + #else + body() + #endif +} + +// Internal wrappers and typedefs which help reduce #if littering in System's +// code base. + +// TODO: Should CSystem just include all the header files we need? + +internal typealias _COffT = off_t + +// MARK: syscalls and variables + +#if canImport(Darwin) +internal var system_errno: CInt { + get { Darwin.errno } + set { Darwin.errno = newValue } +} +#elseif canImport(Glibc) +internal var system_errno: CInt { + get { Glibc.errno } + set { Glibc.errno = newValue } +} +#endif + +// MARK: C stdlib decls + +// Convention: `system_foo` is system's wrapper for `foo`. + +internal func system_strerror(_ __errnum: Int32) -> UnsafeMutablePointer! { + strerror(__errnum) +} + +internal func system_strlen(_ s: UnsafePointer) -> Int { + strlen(s) +} +internal func system_strlen(_ s: UnsafeMutablePointer) -> Int { + strlen(s) +} + +// Convention: `system_platform_foo` is a +// platform-representation-abstracted wrapper around `foo`-like functionality. +// Type and layout differences such as the `char` vs `wchar` are abstracted. +// + +// strlen for the platform string +internal func system_platform_strlen(_ s: UnsafePointer) -> Int { + return strlen(s) +} + +// memset for raw buffers +// FIXME: Do we really not have something like this in the stdlib already? +internal func system_memset( + _ buffer: UnsafeMutableRawBufferPointer, + to byte: UInt8 +) { + guard buffer.count > 0 else { return } + memset(buffer.baseAddress!, CInt(byte), buffer.count) +} + +// Interop between String and platfrom string +extension String { + internal func _withPlatformString( + _ body: (UnsafePointer) throws -> Result + ) rethrows -> Result { + // Need to #if because CChar may be signed + return try withCString(body) + } + + internal init?(_platformString platformString: UnsafePointer) { + // Need to #if because CChar may be signed + self.init(validatingUTF8: platformString) + } + + internal init( + _errorCorrectingPlatformString platformString: UnsafePointer + ) { + // Need to #if because CChar may be signed + self.init(cString: platformString) + } +} + +internal typealias _PlatformTLSKey = pthread_key_t + +internal func makeTLSKey() -> _PlatformTLSKey { + var raw = pthread_key_t() + guard 0 == pthread_key_create(&raw, nil) else { + fatalError("Unable to create key") + } + return raw +} + +internal func setTLS(_ key: _PlatformTLSKey, _ p: UnsafeMutableRawPointer?) { + guard 0 == pthread_setspecific(key, p) else { + fatalError("Unable to set TLS") + } +} + +internal func getTLS(_ key: _PlatformTLSKey) -> UnsafeMutableRawPointer? { + return pthread_getspecific(key) +} diff --git a/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift b/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift new file mode 100644 index 0000000000..e6991fdfff --- /dev/null +++ b/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift @@ -0,0 +1,378 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(Darwin) +import Darwin +import CNIODarwin +#elseif canImport(Glibc) +import Glibc +import CNIOLinux +#endif + +@_spi(Testing) +public enum Syscall { + @_spi(Testing) + public static func stat(path: FilePath) -> Result { + return path.withPlatformString { platformPath in + var status = CInterop.Stat() + return valueOrErrno(retryOnInterrupt: false) { + system_stat(platformPath, &status) + }.map { _ in + status + } + } + } + + @_spi(Testing) + public static func lstat(path: FilePath) -> Result { + return path.withPlatformString { platformPath in + var status = CInterop.Stat() + return valueOrErrno(retryOnInterrupt: false) { + system_lstat(platformPath, &status) + }.map { _ in + status + } + } + } + + @_spi(Testing) + public static func mkdir(at path: FilePath, permissions: FilePermissions) -> Result { + return nothingOrErrno(retryOnInterrupt: false) { + path.withPlatformString { p in + system_mkdir(p, permissions.rawValue) + } + } + } + + @_spi(Testing) + public static func rename(from old: FilePath, to new: FilePath) -> Result { + return nothingOrErrno(retryOnInterrupt: false) { + old.withPlatformString { oldPath in + new.withPlatformString { newPath in + system_rename(oldPath, newPath) + } + } + } + } + + #if canImport(Darwin) + @_spi(Testing) + public static func rename( + from old: FilePath, + to new: FilePath, + options: RenameOptions + ) -> Result { + return nothingOrErrno(retryOnInterrupt: false) { + old.withPlatformString { oldPath in + new.withPlatformString { newPath in + system_renamex_np(oldPath, newPath, options.rawValue) + } + } + } + } + + @_spi(Testing) + public struct RenameOptions: OptionSet { + public var rawValue: CUnsignedInt + + public init(rawValue: CUnsignedInt) { + self.rawValue = rawValue + } + + public static var exclusive: Self { + return Self(rawValue: UInt32(bitPattern: RENAME_EXCL)) + } + + public static var swap: Self { + return Self(rawValue: UInt32(bitPattern: RENAME_SWAP)) + } + } + #endif + + #if canImport(Glibc) + @_spi(Testing) + public static func rename( + from old: FilePath, + relativeTo oldFD: FileDescriptor, + to new: FilePath, + relativeTo newFD: FileDescriptor, + flags: RenameAtFlags + ) -> Result { + return nothingOrErrno(retryOnInterrupt: false) { + old.withPlatformString { oldPath in + new.withPlatformString { newPath in + system_renameat2( + oldFD.rawValue, + oldPath, + newFD.rawValue, + newPath, + flags.rawValue + ) + } + } + } + } + + @_spi(Testing) + public struct RenameAtFlags: OptionSet { + public var rawValue: CUnsignedInt + + public init(rawValue: CUnsignedInt) { + self.rawValue = rawValue + } + + public static var exclusive: Self { + return Self(rawValue: CNIOLinux_RENAME_NOREPLACE) + } + + public static var swap: Self { + return Self(rawValue: CNIOLinux_RENAME_EXCHANGE) + } + } + #endif + + #if canImport(Glibc) + @_spi(Testing) + public struct LinkAtFlags: OptionSet { + @_spi(Testing) + public var rawValue: CInt + + @_spi(Testing) + public init(rawValue: CInt) { + self.rawValue = rawValue + } + + @_spi(Testing) + public static var emptyPath: Self { + Self(rawValue: CNIOLinux_AT_EMPTY_PATH) + } + + @_spi(Testing) + public static var followSymbolicLinks: Self { + Self(rawValue: AT_SYMLINK_FOLLOW) + } + } + + @_spi(Testing) + public static func linkAt( + from source: FilePath, + relativeTo sourceFD: FileDescriptor, + to destination: FilePath, + relativeTo destinationFD: FileDescriptor, + flags: LinkAtFlags + ) -> Result { + return nothingOrErrno(retryOnInterrupt: false) { + source.withPlatformString { src in + destination.withPlatformString { dst in + system_linkat( + sourceFD.rawValue, + src, + destinationFD.rawValue, + dst, + flags.rawValue + ) + } + } + } + } + #endif + + @_spi(Testing) + public static func symlink( + to destination: FilePath, + from source: FilePath + ) -> Result { + return nothingOrErrno(retryOnInterrupt: false) { + source.withPlatformString { src in + destination.withPlatformString { dst in + system_symlink(dst, src) + } + } + } + } + + @_spi(Testing) + public static func readlink(at path: FilePath) -> Result { + do { + let resolved = try path.withPlatformString { p in + try String(customUnsafeUninitializedCapacity: Int(CInterop.maxPathLength)) { pointer in + let result = pointer.withMemoryRebound(to: CInterop.PlatformChar.self) { ptr in + valueOrErrno(retryOnInterrupt: false) { + system_readlink(p, ptr.baseAddress!, ptr.count) + } + } + return try result.get() + } + } + return .success(FilePath(resolved)) + } catch let error as Errno { + return .failure(error) + } catch { + // Shouldn't happen: we deal in Result types and only ever with Errno. + fatalError("Unexpected error '\(error)' caught") + } + } + + #if canImport(Glibc) + @_spi(Testing) + public static func sendfile( + to output: FileDescriptor, + from input: FileDescriptor, + offset: Int, + size: Int + ) -> Result { + valueOrErrno(retryOnInterrupt: false) { + system_sendfile(output.rawValue, input.rawValue, offset, size) + } + } + #endif +} + +@_spi(Testing) +public enum Libc { + static func readdir( + _ dir: CInterop.DirPointer + ) -> Result?, Errno> { + optionalValueOrErrno(retryOnInterrupt: false) { + libc_readdir(dir) + } + } + + static func closedir( + _ dir: CInterop.DirPointer + ) -> Result { + nothingOrErrno(retryOnInterrupt: false) { + libc_closedir(dir) + } + } + + #if canImport(Darwin) + @_spi(Testing) + public static func fcopyfile( + from source: FileDescriptor, + to destination: FileDescriptor, + state: copyfile_state_t?, + flags: copyfile_flags_t + ) -> Result { + nothingOrErrno(retryOnInterrupt: false) { + libc_fcopyfile(source.rawValue, destination.rawValue, state, flags) + } + } + #endif + + @_spi(Testing) + public static func remove( + _ path: FilePath + ) -> Result { + nothingOrErrno(retryOnInterrupt: false) { + path.withPlatformString { + libc_remove($0) + } + } + } + + static func getcwd() -> Result { + var buffer = [CInterop.PlatformChar]( + repeating: 0, + count: Int(CInterop.maxPathLength) + ) + + return optionalValueOrErrno(retryOnInterrupt: false) { + buffer.withUnsafeMutableBufferPointer { pointer in + libc_getcwd(pointer.baseAddress!, pointer.count) + } + }.map { ptr in + // 'ptr' is just the input pointer, we should ignore it and just rely on the bytes + // in buffer. + // + // At this point 'ptr' must be non-nil, because if it were 'nil' we should be on the + // error path. + precondition(ptr != nil) + return FilePath(platformString: buffer) + } + } + + static func constr(_ name: CInt) -> Result { + var buffer = [CInterop.PlatformChar](repeating: 0, count: 128) + + repeat { + let result = valueOrErrno(retryOnInterrupt: false) { + buffer.withUnsafeMutableBufferPointer { pointer in + libc_confstr(name, pointer.baseAddress!, pointer.count) + } + } + + switch result { + case let .success(length): + if length <= buffer.count { + return .success(String(cString: buffer)) + } else { + // The buffer wasn't long enough. Double and try again. + buffer.append(contentsOf: repeatElement(0, count: buffer.capacity)) + } + case let .failure(errno): + return .failure(errno) + } + } while true + } + + static func ftsOpen(_ path: FilePath, options: FTSOpenOptions) -> Result { + // 'fts_open' needs an unsafe mutable pointer to the C-string, `FilePath` doesn't offer this + // so copy out its bytes. + var pathBytes = path.withPlatformString { pointer in + // Length excludes the null terminator, so add it back. + let bufferPointer = UnsafeBufferPointer(start: pointer, count: path.length + 1) + return Array(bufferPointer) + } + + return valueOrErrno { + pathBytes.withUnsafeMutableBufferPointer { pointer in + // The array must be terminated with a nil. + libc_fts_open([pointer.baseAddress, nil], options.rawValue) + } + } + } + + /// Options passed to 'fts_open'. + struct FTSOpenOptions: OptionSet { + var rawValue: CInt + + /// Don't change directory while walking the filesystem hierarchy. + static var noChangeDir: Self { Self(rawValue: FTS_NOCHDIR) } + + /// Return FTS entries for symbolic links rather than their targets. + static var physical: Self { Self(rawValue: FTS_PHYSICAL) } + + /// Return FTS entries for the targets of symbolic links. + static var logical: Self { Self(rawValue: FTS_LOGICAL) } + } + + static func ftsRead( + _ pointer: CInterop.FTSPointer + ) -> Result?, Errno> { + optionalValueOrErrno { + libc_fts_read(pointer) + } + } + + static func ftsClose( + _ pointer: CInterop.FTSPointer + ) -> Result { + nothingOrErrno { + libc_fts_close(pointer) + } + } +} diff --git a/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift b/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift new file mode 100644 index 0000000000..826af57a58 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift @@ -0,0 +1,412 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(Darwin) +import Darwin +import CNIODarwin +#elseif canImport(Glibc) +import Glibc +import CNIOLinux +#endif + +// MARK: - system + +/// openat(2): Open or create a file for reading or writing +func system_fdopenat( + _ fd: FileDescriptor.RawValue, + _ path: UnsafePointer, + _ oflag: Int32 +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(fd, path, oflag) + } + #endif + return openat(fd, path, oflag) +} + +/// openat(2): Open or create a file for reading or writing +func system_fdopenat( + _ fd: FileDescriptor.RawValue, + _ path: UnsafePointer, + _ oflag: Int32, + _ mode: CInterop.Mode +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(fd, path, oflag, mode) + } + #endif + return openat(fd, path, oflag, mode) +} + +/// stat(2): Get file status +func system_stat( + _ path: UnsafePointer, + _ info: inout CInterop.Stat +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(path) + } + #endif + return stat(path, &info) +} + +/// lstat(2): Get file status +internal func system_lstat( + _ path: UnsafePointer, + _ info: inout CInterop.Stat +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(path) + } + #endif + return lstat(path, &info) +} + +/// fstat(2): Get file status +internal func system_fstat( + _ fd: FileDescriptor.RawValue, + _ info: inout CInterop.Stat +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(fd) + } + #endif + return fstat(fd, &info) +} + +/// fchmod(2): Change mode of file +internal func system_fchmod( + _ fd: FileDescriptor.RawValue, + _ mode: CInterop.Mode +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(fd, mode) + } + #endif + return fchmod(fd, mode) +} + +/// fsync(2): Synchronize modifications to a file to permanent storage +internal func system_fsync( + _ fd: FileDescriptor.RawValue +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(fd) + } + #endif + return fsync(fd) +} + +/// mkdir(2): Make a directory file +internal func system_mkdir( + _ path: UnsafePointer, + _ mode: CInterop.Mode +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(path, mode) + } + #endif + return mkdir(path, mode) +} + +/// symlink(2): Make symolic link to a file +internal func system_symlink( + _ destination: UnsafePointer, + _ source: UnsafePointer +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(destination, source) + } + #endif + return symlink(destination, source) +} + +/// readlink(2): Read value of a symolic link +internal func system_readlink( + _ path: UnsafePointer, + _ buffer: UnsafeMutablePointer, + _ bufferSize: Int +) -> Int { + #if ENABLE_MOCKING + if mockingEnabled { + return mockInt(path, buffer, bufferSize) + } + #endif + return readlink(path, buffer, bufferSize) +} + +/// flistxattr(2): List extended attribute names +internal func system_flistxattr( + _ fd: FileDescriptor.RawValue, + _ namebuf: UnsafeMutablePointer?, + _ size: Int +) -> Int { + #if ENABLE_MOCKING + if mockingEnabled { + return mockInt(fd, namebuf, size) + } + #endif + #if canImport(Darwin) + // The final parameter is 'options'; there is no equivalent on Linux. + return flistxattr(fd, namebuf, size, 0) + #elseif canImport(Glibc) + return flistxattr(fd, namebuf, size) + #endif +} + +/// fgetxattr(2): Get an extended attribute value +internal func system_fgetxattr( + _ fd: FileDescriptor.RawValue, + _ name: UnsafePointer, + _ value: UnsafeMutableRawPointer?, + _ size: Int +) -> Int { + #if ENABLE_MOCKING + if mockingEnabled { + return mockInt(fd, name, value, size) + } + #endif + + #if canImport(Darwin) + // Penultimate parameter is position which is reserved and should be zero. + // The final parameter is 'options'; there is no equivalent on Linux. + return fgetxattr(fd, name, value, size, 0, 0) + #elseif canImport(Glibc) + return fgetxattr(fd, name, value, size) + #endif +} + +/// fsetxattr(2): Set an extended attribute value +internal func system_fsetxattr( + _ fd: FileDescriptor.RawValue, + _ name: UnsafePointer, + _ value: UnsafeRawPointer?, + _ size: Int +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(fd, name, value, size) + } + #endif + + // The final parameter is 'options'/'flags' on Darwin/Linux respectively. + #if canImport(Darwin) + // Penultimate parameter is position which is reserved and should be zero. + return fsetxattr(fd, name, value, size, 0, 0) + #elseif canImport(Glibc) + return fsetxattr(fd, name, value, size, 0) + #endif +} + +/// fremovexattr(2): Remove an extended attribute value +internal func system_fremovexattr( + _ fd: FileDescriptor.RawValue, + _ name: UnsafePointer +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(fd, name) + } + #endif + + #if canImport(Darwin) + // The final parameter is 'options'; there is no equivalent on Linux. + return fremovexattr(fd, name, 0) + #elseif canImport(Glibc) + return fremovexattr(fd, name) + #endif +} + +/// rename(2): Change the name of a file +internal func system_rename( + _ old: UnsafePointer, + _ new: UnsafePointer +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(old, new) + } + #endif + return rename(old, new) +} + +#if canImport(Darwin) +internal func system_renamex_np( + _ old: UnsafePointer, + _ new: UnsafePointer, + _ flags: CUnsignedInt +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(old, new, flags) + } + #endif + return renamex_np(old, new, flags) +} +#endif + +#if canImport(Glibc) +internal func system_renameat2( + _ oldFD: FileDescriptor.RawValue, + _ old: UnsafePointer, + _ newFD: FileDescriptor.RawValue, + _ new: UnsafePointer, + _ flags: CUnsignedInt +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(oldFD, old, newFD, new, flags) + } + #endif + return CNIOLinux_renameat2(oldFD, old, newFD, new, flags) +} +#endif + +/// link(2): Creates a new link for a file. +#if canImport(Glibc) +internal func system_linkat( + _ oldFD: FileDescriptor.RawValue, + _ old: UnsafePointer, + _ newFD: FileDescriptor.RawValue, + _ new: UnsafePointer, + _ flags: CInt +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(oldFD, old, newFD, new, flags) + } + #endif + return linkat(oldFD, old, newFD, new, flags) +} +#endif + +#if canImport(Glibc) +/// sendfile(2): Transfer data between descriptors +internal func system_sendfile( + _ outFD: CInt, + _ inFD: CInt, + _ offset: off_t, + _ count: Int +) -> Int { + #if ENABLE_MOCKING + if mockingEnabled { + return mockInt(outFD, inFD, offset, count) + } + #endif + var offset = offset + return sendfile(outFD, inFD, &offset, count) +} +#endif + +// MARK: - libc + +/// fdopendir(3): Opens a directory stream for the file descriptor +internal func libc_fdopendir( + _ fd: FileDescriptor.RawValue +) -> CInterop.DirPointer { + return fdopendir(fd) +} + +/// readdir(3): Returns a pointer to the next directory entry +internal func libc_readdir( + _ dir: CInterop.DirPointer +) -> UnsafeMutablePointer? { + return readdir(dir) +} + +/// readdir(3): Closes the directory stream and frees associated structures +internal func libc_closedir( + _ dir: CInterop.DirPointer +) -> CInt { + return closedir(dir) +} + +/// remove(3): Remove directory entry +internal func libc_remove( + _ path: UnsafePointer +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(path) + } + #endif + return remove(path) +} + +#if canImport(Darwin) +/// copyfile(3): Copy a file from one file to another. +internal func libc_fcopyfile( + _ from: CInt, + _ to: CInt, + _ state: copyfile_state_t?, + _ flags: copyfile_flags_t +) -> CInt { + #if ENABLE_MOCKING + if mockingEnabled { + return mock(from, to, state, flags) + } + #endif + return fcopyfile(from, to, state, flags) +} +#endif + +/// getcwd(3): Get working directory pathname +internal func libc_getcwd( + _ buffer: UnsafeMutablePointer, + _ size: Int +) -> UnsafeMutablePointer? { + return getcwd(buffer, size) +} + +/// confstr(3) +internal func libc_confstr( + _ name: CInt, + _ buffer: UnsafeMutablePointer, + _ size: Int +) -> Int { + return confstr(name, buffer, size) +} + +/// fts(3) +internal func libc_fts_open( + _ path: [UnsafeMutablePointer?], + _ options: CInt +) -> UnsafeMutablePointer { + return fts_open(path, options, nil) +} + +/// fts(3) +internal func libc_fts_read( + _ fts: UnsafeMutablePointer +) -> UnsafeMutablePointer? { + return fts_read(fts) +} + +/// fts(3) +internal func libc_fts_close( + _ fts: UnsafeMutablePointer +) -> CInt { + return fts_close(fts) +} diff --git a/Sources/NIOFileSystem/Internal/SystemFileHandle.swift b/Sources/NIOFileSystem/Internal/SystemFileHandle.swift new file mode 100644 index 0000000000..2b367ba26d --- /dev/null +++ b/Sources/NIOFileSystem/Internal/SystemFileHandle.swift @@ -0,0 +1,1303 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOConcurrencyHelpers +import NIOCore +@preconcurrency import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +/// An implementation of ``FileHandleProtocol`` which is backed by system calls and a file +/// descriptor. +@_spi(Testing) +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +public final class SystemFileHandle { + /// The executor on which to execute system calls. + internal var executor: IOExecutor { self.sendableView.executor } + + /// The path used to open this handle. + internal var path: FilePath { self.sendableView.path } + + @_spi(Testing) + public struct Materialization: Sendable { + /// The path of the file which was created. + var created: FilePath + /// The desired path of the file. + var desired: FilePath + /// Whether the ``desired`` file must be created exclusively. If `true` then if a file + /// already exists at the ``desired`` path then an error is thrown, otherwise any existing + /// file will be replaced.` + var exclusive: Bool + /// The mode used to materialize the file. + var mode: Mode + + enum Mode { + /// Rename the created file to become the desired file. + case rename + #if canImport(Glibc) + /// Link the unnamed file to the desired file using 'linkat(2)'. + case link + #endif + } + } + + fileprivate enum Lifecycle { + case open(FileDescriptor) + case detached + case closed + } + + @_spi(Testing) + public let sendableView: SendableView + + /// The file handle may be for a non-seekable file, so it shouldn't be 'Sendable', however, most + /// of the work performed on behalf of the handle is executed in a thread pool which means that + /// its state must be 'Sendable'. + @_spi(Testing) + public struct SendableView: Sendable { + /// The lifecycle of the file handle. + fileprivate let lifecycle: NIOLockedValueBox + + /// The executor on which to execute system calls. + internal let executor: IOExecutor + + /// The path used to open this handle. + internal let path: FilePath + + /// An action to take when closing the file handle. + fileprivate let materialization: Materialization? + + fileprivate init( + lifecycle: Lifecycle, + executor: IOExecutor, + path: FilePath, + materialization: Materialization? + ) { + self.lifecycle = NIOLockedValueBox(lifecycle) + self.executor = executor + self.path = path + self.materialization = materialization + } + } + + /// Creates a handle which takes ownership of the provided descriptor. + /// + /// - Precondition: The descriptor must be open. + /// - Parameters: + /// - descriptor: The open file descriptor. + /// - path: The path to the file used to open the descriptor. + /// - executor: The executor which system calls will be performed on. + @_spi(Testing) + public init( + takingOwnershipOf descriptor: FileDescriptor, + path: FilePath, + materialization: Materialization? = nil, + executor: IOExecutor + ) { + self.sendableView = SendableView( + lifecycle: .open(descriptor), + executor: executor, + path: path, + materialization: materialization + ) + } + + deinit { + self.sendableView.lifecycle.withLockedValue { lifecycle -> Void in + switch lifecycle { + case .open: + fatalError( + """ + Leaking file descriptor: the handle for '\(self.sendableView.path)' MUST be closed or \ + detached with 'close()' or 'detachUnsafeFileDescriptor()' before the final \ + reference to the handle is dropped. + """ + ) + case .detached, .closed: + () + } + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle.SendableView { + /// Returns the file descriptor if it's available; `nil` otherwise. + internal func descriptorIfAvailable() -> FileDescriptor? { + return self.lifecycle.withLockedValue { + switch $0 { + case let .open(descriptor): + return descriptor + case .detached, .closed: + return nil + } + } + } + + /// Executes a closure with the file descriptor it it's available otherwise throws the result + /// of `onUnavailable`. + internal func _withUnsafeDescriptor( + _ execute: (FileDescriptor) throws -> R, + onUnavailable: () -> FileSystemError + ) throws -> R { + if let descriptor = self.descriptorIfAvailable() { + return try execute(descriptor) + } else { + throw onUnavailable() + } + } + + /// Executes a closure with the file descriptor it it's available otherwise throws the result + /// of `onUnavailable`. + internal func _withUnsafeDescriptorResult( + _ execute: (FileDescriptor) -> Result, + onUnavailable: () -> FileSystemError + ) -> Result { + if let descriptor = self.descriptorIfAvailable() { + return execute(descriptor) + } else { + return .failure(onUnavailable()) + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle: FileHandleProtocol { + // Notes which apply to the following block of functions: + // + // 1. Documentation is inherited from ``FileHandleProtocol`` and is not repeated here. + // 2. The functions should be annotated with @_spi(Testing); this is not possible with the + // conformance to FileHandleProtocol which requires them to be only marked public. However + // this is not an issue: the implementing type is annotated with @_spi(Testing) so the + // functions are not actually public. + // 3. Most of these functions call through to a synchronous version prefixed with an underscore, + // this is to make testing possible with the system call mocking infrastructure we are + // currently using. + + public func info() async throws -> FileInfo { + return try await self.executor.execute { [sendableView] in + try sendableView._info().get() + } + } + + public func replacePermissions(_ permissions: FilePermissions) async throws { + return try await self.executor.execute { [sendableView] in + try sendableView._replacePermissions(permissions) + } + } + + public func addPermissions(_ permissions: FilePermissions) async throws -> FilePermissions { + return try await self.executor.execute { [sendableView] in + try sendableView._addPermissions(permissions) + } + } + + public func removePermissions(_ permissions: FilePermissions) async throws -> FilePermissions { + return try await self.executor.execute { [sendableView] in + try sendableView._removePermissions(permissions) + } + } + + public func attributeNames() async throws -> [String] { + return try await self.executor.execute { [sendableView] in + try sendableView._attributeNames() + } + } + + public func valueForAttribute(_ name: String) async throws -> [UInt8] { + return try await self.executor.execute { [sendableView] in + try sendableView._valueForAttribute(name) + } + } + + public func updateValueForAttribute( + _ bytes: some (Sendable & RandomAccessCollection), + attribute name: String + ) async throws { + return try await self.executor.execute { [sendableView] in + try sendableView._updateValueForAttribute(bytes, attribute: name) + } + } + + public func removeValueForAttribute(_ name: String) async throws { + return try await self.executor.execute { [sendableView] in + try sendableView._removeValueForAttribute(name) + } + } + + public func synchronize() async throws { + return try await self.executor.execute { [sendableView] in + try sendableView._synchronize() + } + } + + public func withUnsafeDescriptor( + _ execute: @Sendable @escaping (FileDescriptor) throws -> R + ) async throws -> R { + try await self.executor.execute { [sendableView] in + try sendableView._withUnsafeDescriptor { + return try execute($0) + } onUnavailable: { + FileSystemError( + code: .closed, + message: "File is closed ('\(sendableView.path)').", + cause: nil, + location: .here() + ) + } + } + } + + public func detachUnsafeFileDescriptor() throws -> FileDescriptor { + return try self.sendableView.lifecycle.withLockedValue { lifecycle in + switch lifecycle { + case let .open(descriptor): + lifecycle = .detached + return descriptor + + case .detached: + throw FileSystemError( + code: .closed, + message: """ + File descriptor has already been detached ('\(self.path)'). Handles may \ + only be detached once. + """, + cause: nil, + location: .here() + ) + + case .closed: + throw FileSystemError( + code: .closed, + message: """ + Cannot detach descriptor for closed file ('\(self.path)'). Handles may \ + only be detached while they are open. + """, + cause: nil, + location: .here() + ) + } + } + } + + public func close() async throws { + try await self.executor.execute { [sendableView] in + try sendableView._close(materialize: true).get() + } + } + + public func close(makeChangesVisible: Bool) async throws { + try await self.executor.execute { [sendableView] in + try sendableView._close(materialize: makeChangesVisible).get() + } + } + + @_spi(Testing) + public enum UpdatePermissionsOperation { case set, add, remove } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle.SendableView { + /// Returns a string in the format: "{message}, the file '{path}' is closed." + private func fileIsClosed(_ message: String) -> String { + return "\(message), the file '\(self.path)' is closed." + } + + /// Returns a string in the format: "{message} for '{path}'." + private func unknown(_ message: String) -> String { + return "\(message) for '\(self.path)'." + } + + @_spi(Testing) + public func _info() -> Result { + self._withUnsafeDescriptorResult { descriptor in + return descriptor.status().map { stat in + FileInfo(platformSpecificStatus: stat) + }.mapError { errno in + .stat("fstat", errno: errno, path: self.path, location: .here()) + } + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed("Unable to get information"), + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _replacePermissions(_ permissions: FilePermissions) throws { + try self._withUnsafeDescriptor { descriptor in + try self.updatePermissions( + permissions, + operation: .set, + operand: permissions, + descriptor: descriptor + ) + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed("Unable to replace permissions"), + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _addPermissions(_ permissions: FilePermissions) throws -> FilePermissions { + try self._withUnsafeDescriptor { descriptor in + switch descriptor.status() { + case let .success(status): + let info = FileInfo(platformSpecificStatus: status) + let merged = info.permissions.union(permissions) + + // Check if we need to make any changes. + if merged == info.permissions { + return merged + } + + // Apply the new permissions. + try self.updatePermissions( + merged, + operation: .add, + operand: permissions, + descriptor: descriptor + ) + + return merged + + case let .failure(errno): + throw FileSystemError( + message: "Unable to add permissions.", + wrapping: .stat("fstat", errno: errno, path: self.path, location: .here()) + ) + } + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed("Unable to add permissions"), + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _removePermissions(_ permissions: FilePermissions) throws -> FilePermissions { + try self._withUnsafeDescriptor { descriptor in + switch descriptor.status() { + case let .success(status): + let info = FileInfo(platformSpecificStatus: status) + let merged = info.permissions.subtracting(permissions) + + // Check if we need to make any changes. + if merged == info.permissions { + return merged + } + + // Apply the new permissions. + try self.updatePermissions( + merged, + operation: .remove, + operand: permissions, + descriptor: descriptor + ) + + return merged + + case let .failure(errno): + throw FileSystemError( + message: "Unable to remove permissions.", + wrapping: .stat("fstat", errno: errno, path: self.path, location: .here()) + ) + + } + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed("Unable to remove permissions"), + cause: nil, + location: .here() + ) + } + } + + private func updatePermissions( + _ permissions: FilePermissions, + operation: SystemFileHandle.UpdatePermissionsOperation, + operand: FilePermissions, + descriptor: FileDescriptor + ) throws { + return try descriptor.changeMode(permissions).mapError { errno in + FileSystemError.fchmod( + operation: operation, + operand: operand, + permissions: permissions, + errno: errno, + path: self.path, + location: .here() + ) + }.get() + } + + @_spi(Testing) + public func _attributeNames() throws -> [String] { + return try self._withUnsafeDescriptor { descriptor in + return try descriptor.listExtendedAttributes().mapError { errno in + FileSystemError.flistxattr(errno: errno, path: self.path, location: .here()) + }.get() + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed("Could not list extended attributes"), + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _valueForAttribute(_ name: String) throws -> [UInt8] { + return try self._withUnsafeDescriptor { descriptor in + return try descriptor.readExtendedAttribute( + named: name + ).flatMapError { errno -> Result<[UInt8], FileSystemError> in + switch errno { + #if canImport(Darwin) + case .attributeNotFound: + // Okay, return empty value. + return .success([]) + #endif + case .noData: + // Okay, return empty value. + return .success([]) + default: + let error = FileSystemError.fgetxattr( + attribute: name, + errno: errno, + path: self.path, + location: .here() + ) + return .failure(error) + } + }.get() + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed( + "Could not get value for extended attribute ('\(name)')" + ), + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _updateValueForAttribute( + _ bytes: some RandomAccessCollection, + attribute name: String + ) throws { + return try self._withUnsafeDescriptor { descriptor in + func withUnsafeBufferPointer(_ body: (UnsafeBufferPointer) throws -> Void) throws { + try bytes.withContiguousStorageIfAvailable(body) + ?? Array(bytes).withUnsafeBufferPointer(body) + } + + try withUnsafeBufferPointer { pointer in + let rawBufferPointer = UnsafeRawBufferPointer(pointer) + return try descriptor.setExtendedAttribute( + named: name, + to: rawBufferPointer + ).mapError { errno in + FileSystemError.fsetxattr( + attribute: name, + errno: errno, + path: self.path, + location: .here() + ) + }.get() + } + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed( + "Could not set value for extended attribute ('\(name)')" + ), + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _removeValueForAttribute(_ name: String) throws { + return try self._withUnsafeDescriptor { descriptor in + try descriptor.removeExtendedAttribute(name).mapError { errno in + FileSystemError.fremovexattr( + attribute: name, + errno: errno, + path: self.path, + location: .here() + ) + }.get() + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed("Could not remove extended attribute ('\(name)')"), + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _synchronize() throws { + try self._withUnsafeDescriptor { descriptor in + try descriptor.synchronize().mapError { errno in + FileSystemError.fsync(errno: errno, path: self.path, location: .here()) + }.get() + } onUnavailable: { + FileSystemError( + code: .closed, + message: self.fileIsClosed("Could not synchronize"), + cause: nil, + location: .here() + ) + } + } + + internal func _duplicate() -> Result { + return self._withUnsafeDescriptorResult { descriptor in + Result { + try descriptor.duplicate() + }.mapError { error in + FileSystemError.dup(error: error, path: self.path, location: .here()) + } + } onUnavailable: { + FileSystemError( + code: .closed, + message: "Unable to duplicate descriptor of closed handle for '\(self.path)'.", + cause: nil, + location: .here() + ) + } + } + + @_spi(Testing) + public func _close(materialize: Bool) -> Result { + let descriptor: FileDescriptor? = self.lifecycle.withLockedValue { lifecycle in + switch lifecycle { + case let .open(descriptor): + lifecycle = .closed + return descriptor + case .detached, .closed: + return nil + } + } + + guard let descriptor = descriptor else { + return .success(()) + } + + // Materialize then close. + let materializeResult = self._materialize(materialize, descriptor: descriptor) + + return Result { + try descriptor.close() + }.mapError { error in + .close(error: error, path: self.path, location: .here()) + }.flatMap { + materializeResult + } + } + + private func _materialize( + _ materialize: Bool, + descriptor: FileDescriptor + ) -> Result { + guard let materialization = self.materialization else { return .success(()) } + + let createdPath = materialization.created + let desiredPath = materialization.desired + + let result: Result + switch materialization.mode { + #if canImport(Glibc) + case .link: + if materialize { + func linkAtEmptyPath() -> Result { + Syscall.linkAt( + from: "", + relativeTo: descriptor, + to: desiredPath, + relativeTo: .currentWorkingDirectory, + flags: [.emptyPath] + ) + } + + func linkAtProcFS() -> Result { + Syscall.linkAt( + from: FilePath("/proc/self/fd/\(descriptor.rawValue)"), + relativeTo: .currentWorkingDirectory, + to: desiredPath, + relativeTo: .currentWorkingDirectory, + flags: [.followSymbolicLinks] + ) + } + + switch linkAtEmptyPath() { + case .success: + result = .success(()) + + case .failure(.fileExists) where !materialization.exclusive: + // File exists and materialization _isn't_ exclusive. Remove the existing + // file and try again. + let removeResult = Libc.remove(desiredPath).mapError { errno in + FileSystemError.remove(errno: errno, path: desiredPath, location: .here()) + } + + let linkAtResult = linkAtEmptyPath().flatMapError { errno in + // ENOENT means we likely didn't have the 'CAP_DAC_READ_SEARCH' capability + // so try again by linking to the descriptor via procfs. + if errno == .noSuchFileOrDirectory { + return linkAtProcFS() + } else { + return .failure(errno) + } + }.mapError { errno in + FileSystemError.link( + errno: errno, + from: createdPath, + to: desiredPath, + location: .here() + ) + } + + result = removeResult.flatMap { linkAtResult } + + case .failure(.noSuchFileOrDirectory): + result = linkAtProcFS().flatMapError { errno in + if errno == .fileExists, !materialization.exclusive { + return Libc.remove(desiredPath).mapError { errno in + FileSystemError.remove( + errno: errno, + path: desiredPath, + location: .here() + ) + }.flatMap { + return linkAtProcFS().mapError { errno in + FileSystemError.link( + errno: errno, + from: createdPath, + to: desiredPath, + location: .here() + ) + } + } + } else { + let error = FileSystemError.link( + errno: errno, + from: createdPath, + to: desiredPath, + location: .here() + ) + return .failure(error) + } + } + + case .failure(let errno): + result = .failure( + .link(errno: errno, from: createdPath, to: desiredPath, location: .here()) + ) + } + } else { + result = .success(()) + } + #endif + + case .rename: + if materialize { + let renameResult: Result + #if canImport(Darwin) + renameResult = Syscall.rename( + from: createdPath, + to: desiredPath, + options: materialization.exclusive ? [.exclusive] : [] + ) + #elseif canImport(Glibc) + // The created and desired paths are absolute, so the relative descriptors are + // ignored. However, they must still be provided to 'rename' in order to pass + // flags. + renameResult = Syscall.rename( + from: createdPath, + relativeTo: .currentWorkingDirectory, + to: desiredPath, + relativeTo: .currentWorkingDirectory, + flags: materialization.exclusive ? [.exclusive] : [] + ) + #endif + + // A file exists at the desired path and the user specified exclusive creation, + // clear up by removing the file we did create. + if materialization.exclusive, case .failure(.fileExists) = renameResult { + _ = Libc.remove(createdPath) + } + + result = renameResult.mapError { errno in + .rename( + errno: errno, + oldName: createdPath, + newName: desiredPath, + location: .here() + ) + } + } else { + // Don't materialize the source, remove it + result = Libc.remove(createdPath).mapError { + .remove(errno: $0, path: createdPath, location: .here()) + } + } + } + + return result + } +} + +// MARK: - Readable File Handle + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle: ReadableFileHandleProtocol { + // Notes which apply to the following block of functions: + // + // 1. Documentation is inherited from ``FileHandleProtocol`` and is not repeated here. + // 2. The functions should be annotated with @_spi(Testing); this is not possible with the + // conformance to FileHandleProtocol which requires them to be only marked public. However + // this is not an issue: the implementing type is annotated with @_spi(Testing) so the + // functions are not actually public. + + public func readChunk( + fromAbsoluteOffset offset: Int64, + length: ByteCount + ) async throws -> ByteBuffer { + return try await self.executor.execute { [sendableView] in + return try sendableView._withUnsafeDescriptor { descriptor in + try descriptor.readChunk( + fromAbsoluteOffset: offset, + length: length.bytes + ).flatMapError { error in + if let errno = error as? Errno, errno == .illegalSeek { + guard offset == 0 else { + return .failure( + FileSystemError( + code: .unsupported, + message: "File is unseekable.", + cause: nil, + location: .here() + ) + ) + } + + return descriptor.readChunk(length: length.bytes).mapError { error in + FileSystemError.read( + usingSyscall: .read, + error: error, + path: sendableView.path, + location: .here() + ) + } + } else { + return .failure( + FileSystemError.read( + usingSyscall: .pread, + error: error, + path: sendableView.path, + location: .here() + ) + ) + } + } + .get() + } onUnavailable: { + FileSystemError( + code: .closed, + message: "Couldn't read chunk, the file '\(sendableView.path)' is closed.", + cause: nil, + location: .here() + ) + } + } + } + + public func readChunks( + in range: Range, + chunkLength size: ByteCount + ) -> FileChunks { + return FileChunks(handle: self, chunkLength: size, range: range) + } +} + +// MARK: - Writable File Handle + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle: WritableFileHandleProtocol { + @discardableResult + public func write( + contentsOf bytes: some (Sequence & Sendable), + toAbsoluteOffset offset: Int64 + ) async throws -> Int64 { + return try await self.executor.execute { [sendableView] in + return try sendableView._withUnsafeDescriptor { descriptor in + try descriptor.write(contentsOf: bytes, toAbsoluteOffset: offset) + .flatMapError { error in + if let errno = error as? Errno, errno == .illegalSeek { + guard offset == 0 else { + return .failure( + FileSystemError( + code: .unsupported, + message: "File is unseekable.", + cause: nil, + location: .here() + ) + ) + } + + return descriptor.write(contentsOf: bytes) + .mapError { error in + FileSystemError.write( + usingSyscall: .write, + error: error, + path: sendableView.path, + location: .here() + ) + } + } else { + return .failure( + FileSystemError.write( + usingSyscall: .pwrite, + error: error, + path: sendableView.path, + location: .here() + ) + ) + } + } + .get() + } onUnavailable: { + FileSystemError( + code: .closed, + message: "Couldn't write bytes, the file '\(sendableView.path)' is closed.", + cause: nil, + location: .here() + ) + } + } + } + + public func resize(to size: ByteCount) async throws { + try await self.executor.execute { [sendableView] in + try sendableView._resize(to: size).get() + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle.SendableView { + func _resize(to size: ByteCount) -> Result<(), FileSystemError> { + return self._withUnsafeDescriptorResult { descriptor in + Result { + try descriptor.resize(to: size.bytes, retryOnInterrupt: true) + }.mapError { error in + FileSystemError.ftruncate(error: error, path: self.path, location: .here()) + } + } onUnavailable: { + FileSystemError( + code: .closed, + message: "Unable to resize file '\(self.path)'.", + cause: nil, + location: .here() + ) + } + } +} + +// MARK: - Directory File Handle + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle: DirectoryFileHandleProtocol { + public typealias ReadFileHandle = SystemFileHandle + public typealias WriteFileHandle = SystemFileHandle + public typealias ReadWriteFileHandle = SystemFileHandle + + public func listContents(recursive: Bool) -> DirectoryEntries { + return DirectoryEntries(handle: self, recursive: recursive) + } + + public func openFile( + forReadingAt path: FilePath, + options: OpenOptions.Read + ) async throws -> SystemFileHandle { + let opts = options.descriptorOptions.union(.nonBlocking) + let handle = try await self.executor.execute { [sendableView] in + let handle = try sendableView._open( + atPath: path, + mode: .readOnly, + options: opts, + transactionalIfPossible: false + ).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } + + public func openFile( + forReadingAndWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> SystemFileHandle { + let perms = options.permissionsForRegularFile + let opts = options.descriptorOptions.union(.nonBlocking) + let handle = try await self.executor.execute { [sendableView] in + let handle = try sendableView._open( + atPath: path, + mode: .readWrite, + options: opts, + permissions: perms, + transactionalIfPossible: options.newFile?.transactionalCreation ?? false + ).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } + + public func openFile( + forWritingAt path: FilePath, + options: OpenOptions.Write + ) async throws -> SystemFileHandle { + let perms = options.permissionsForRegularFile + let opts = options.descriptorOptions.union(.nonBlocking) + let handle = try await self.executor.execute { [sendableView] in + let handle = try sendableView._open( + atPath: path, + mode: .writeOnly, + options: opts, + permissions: perms, + transactionalIfPossible: options.newFile?.transactionalCreation ?? false + ).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } + + public func openDirectory( + atPath path: FilePath, + options: OpenOptions.Directory + ) async throws -> SystemFileHandle { + let opts = options.descriptorOptions.union(.nonBlocking) + let handle = try await self.executor.execute { [sendableView] in + let handle = try sendableView._open( + atPath: path, + mode: .readOnly, + options: opts, + transactionalIfPossible: false + ).get() + // Okay to transfer: we just created it and are now moving back to the callers task. + return UnsafeTransfer(handle) + } + return handle.wrappedValue + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle.SendableView { + func _open( + atPath path: FilePath, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions, + permissions: FilePermissions? = nil, + transactionalIfPossible transactional: Bool + ) -> Result { + if transactional { + if path.isAbsolute { + // The provided path is absolute: just open the handle normally. + return SystemFileHandle.syncOpen( + atPath: path, + mode: mode, + options: options, + permissions: permissions, + transactionalIfPossible: transactional, + executor: self.executor + ) + } else if self.path.isAbsolute { + // The parent path is absolute and the provided path is relative; combine them. + return SystemFileHandle.syncOpen( + atPath: self.path.appending(path.components), + mode: mode, + options: options, + permissions: permissions, + transactionalIfPossible: transactional, + executor: self.executor + ) + } + + // At this point transactional file creation isn't possible. Fallback to + // non-transactional. + } + + // Provided and parent paths are relative. There's no way we can safely delay + // materialization as we don't know if the parent descriptor will be available when + // closing the opened file. + return self._withUnsafeDescriptorResult { descriptor in + descriptor.open( + atPath: path, + mode: mode, + options: options, + permissions: permissions + ).map { newDescriptor in + SystemFileHandle( + takingOwnershipOf: newDescriptor, + path: self.path.appending(path.components).lexicallyNormalized(), + executor: self.executor + ) + }.mapError { errno in + .open("openat", error: errno, path: self.path, location: .here()) + } + } onUnavailable: { + FileSystemError( + code: .closed, + message: """ + Unable to open file at path '\(path)' relative to '\(self.path)', the file \ + is closed. + """, + cause: nil, + location: .here() + ) + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SystemFileHandle { + static func syncOpen( + atPath path: FilePath, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions, + permissions: FilePermissions?, + transactionalIfPossible transactional: Bool, + executor: IOExecutor + ) -> Result { + let isWritable = (mode == .writeOnly || mode == .readWrite) + let exclusiveCreate = options.contains(.exclusiveCreate) + let truncate = options.contains(.truncate) + let delayMaterialization = transactional && isWritable && (exclusiveCreate || truncate) + + if delayMaterialization { + // When opening in this mode we can more "atomically" create the file, that is, by not + // leaving the user with a half written file should e.g. the system crash or throw an + // error while writing. On Linux we do this by opening the directory for the path + // with `O_TMPFILE` and creating a hard link when closing the file. On other platforms + // we generate a dot file with a randomised suffix name and rename it to the + // destination. + return Self.syncOpenWithMaterialization( + atPath: path, + mode: mode, + options: options, + permissions: permissions, + executor: executor + ) + } else { + return Self.syncOpen( + atPath: path, + mode: mode, + options: options, + permissions: permissions, + executor: executor + ) + } + } + + static func syncOpen( + atPath path: FilePath, + mode: FileDescriptor.AccessMode, + options: FileDescriptor.OpenOptions, + permissions: FilePermissions?, + executor: IOExecutor + ) -> Result { + return Result { + try FileDescriptor.open( + path, + mode, + options: options, + permissions: permissions + ) + }.map { descriptor in + SystemFileHandle( + takingOwnershipOf: descriptor, + path: path, + executor: executor + ) + }.mapError { errno in + FileSystemError.open("open", error: errno, path: path, location: .here()) + } + } + + static func syncOpenWithMaterialization( + atPath path: FilePath, + mode: FileDescriptor.AccessMode, + options originalOptions: FileDescriptor.OpenOptions, + permissions: FilePermissions?, + executor: IOExecutor, + useTemporaryFileIfPossible: Bool = true + ) -> Result { + let openedPath: FilePath + let desiredPath: FilePath + + // There are two different approaches to materializing the file. On Linux, and where + // supported, we can open the file with the 'O_TMPFILE' flag which creates a temporary + // unnamed file. If we later decide the make the file visible we use 'linkat(2)' with + // the appropriate flags to link the unnamed temporary file to the desired file path. + // + // On other platforms, and when not supported on Linux, we create a regular file as we + // normally would and when we decide to materialize it we simply rename it to the desired + // name (or remove it if we aren't materializing it). + // + // There are, however, some wrinkles. + // + // Normally when a file is opened the system will open files specified with relative paths + // relative to the current working directory. However, when we delay making a file visible + // the current working directory could change which introduces an awkward race. Consider + // the following sequence of events: + // + // 1. User opens a file with delay materialization using a relative path + // 2. A temporary file is opened relative to the current working directory + // 3. The current working directory is changed + // 4. The file is closed. + // + // Where is the file created? It *should* be relative to the working directory at the time + // the user opened the file. However, if materializing the file relative to the new + // working directory would be very surprising behaviour for the user. + // + // To work around this we will get the current working directory only if the provided path + // is relative. That way all operations can be done on a path relative to a fixed point + // (i.e. the current working directory at this point in time). + if path.isRelative { + let currentWorkingDirectory: FilePath + + switch Libc.getcwd() { + case .success(let path): + currentWorkingDirectory = path + case .failure(let errno): + let error = FileSystemError( + message: """ + Can't open relative '\(path)' as the current working directory couldn't \ + be determined. + """, + wrapping: .getcwd(errno: errno, location: .here()) + ) + return .failure(error) + } + + func makePath() -> FilePath { + #if canImport(Glibc) + if useTemporaryFileIfPossible { + return currentWorkingDirectory.appending(path.components.dropLast()) + } + #endif + return currentWorkingDirectory.appending(path.components.dropLast()) + .appending(".tmp-" + String(randomAlphaNumericOfLength: 6)) + } + + openedPath = makePath() + desiredPath = currentWorkingDirectory.appending(path.components) + } else { + func makePath() -> FilePath { + #if canImport(Glibc) + if useTemporaryFileIfPossible { + return path.removingLastComponent() + } + #endif + return path.removingLastComponent() + .appending(".tmp-" + String(randomAlphaNumericOfLength: 6)) + } + + openedPath = makePath() + desiredPath = path + } + + let materializationMode: Materialization.Mode + let options: FileDescriptor.OpenOptions + + #if canImport(Glibc) + if useTemporaryFileIfPossible { + options = [.temporaryFile] + materializationMode = .link + } else { + options = originalOptions + materializationMode = .rename + } + #else + options = originalOptions + materializationMode = .rename + #endif + + let materialization = Materialization( + created: openedPath, + desired: desiredPath, + exclusive: originalOptions.contains(.exclusiveCreate), + mode: materializationMode + ) + + do { + let descriptor = try FileDescriptor.open( + openedPath, + mode, + options: options, + permissions: permissions + ) + + let handle = SystemFileHandle( + takingOwnershipOf: descriptor, + path: openedPath, + materialization: materialization, + executor: executor + ) + + return .success(handle) + } catch { + #if canImport(Glibc) + // 'O_TMPFILE' isn't supported for the current file system, try again but using + // rename instead. + if useTemporaryFileIfPossible, let errno = error as? Errno, errno == .notSupported { + return Self.syncOpenWithMaterialization( + atPath: path, + mode: mode, + options: originalOptions, + permissions: permissions, + executor: executor, + useTemporaryFileIfPossible: false + ) + } + #endif + return .failure(.open("open", error: error, path: path, location: .here())) + } + } +} diff --git a/Sources/NIOFileSystem/Internal/Utilities.swift b/Sources/NIOFileSystem/Internal/Utilities.swift new file mode 100644 index 0000000000..9d6b944582 --- /dev/null +++ b/Sources/NIOFileSystem/Internal/Utilities.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +@usableFromInline +internal final class Ref { + @usableFromInline + var value: Value + @inlinable + init(_ value: Value) { + self.value = value + } +} + +extension String { + init(randomAlphaNumericOfLength length: Int) { + precondition(length > 0) + + let characters = (0.. Self { + return Write( + existingFile: replaceExisting ? .truncate : .none, + newFile: NewFile(permissions: permissions) + ) + } + + /// Opens a file for modifying. + /// + /// - Parameters: + /// - createIfNecessary: Whether a file should be created if one doesn't exist. If + /// `false` and a file doesn't exist then an error is thrown. + /// - permissions: The permissions to apply to the newly created file. Default permissions + /// (read-write owner permissions and read permissions for everyone else) are applied + /// if `nil`. Ignored if `createIfNonExistent` is `false`. + /// - Returns: Options for modifying an existing file for writing. + public static func modifyFile( + createIfNecessary: Bool, + permissions: FilePermissions? = nil + ) -> Self { + return Write( + existingFile: .open, + newFile: createIfNecessary ? NewFile(permissions: permissions) : nil + ) + } + } +} + +extension OpenOptions { + /// Options for opening an existing file. + public enum ExistingFile: Sendable, Hashable { + /// Indicates that no file exists. If a file does exist then an error is thrown when + /// opening the file. + case none + + /// Any existing file should be opened without modification. + case open + + /// Truncate the existing file. + /// + /// Setting this is equivalent to opening a file with `O_TRUNC`. + case truncate + } + + /// Options for creating a new file. + public struct NewFile: Sendable, Hashable { + /// The permissions to apply to the new file. `nil` implies default permissions + /// should be applied. + public var permissions: FilePermissions? + + /// Whether the file should be created and updated as a single transaction, if + /// applicable. + /// + /// When this option is set and applied the newly created file will only materialize + /// on the file system when the file is closed. When used in conjunction with + /// ``FileSystemProtocol/withFileHandle(forWritingAt:options:execute:)`` and + /// ``FileSystemProtocol/withFileHandle(forReadingAndWritingAt:options:execute:)`` the + /// file will only materialize when the file is closed and no errors have been thrown. + /// + /// - Important: This flag is only applied if ``OpenOptions/Write/existingFile`` is + /// ``OpenOptions/ExistingFile/none``. + public var transactionalCreation: Bool + + public init( + permissions: FilePermissions? = nil, + transactionalCreation: Bool = true + ) { + self.permissions = permissions + self.transactionalCreation = transactionalCreation + } + } +} + +extension OpenOptions.Write { + @_spi(Testing) + public var permissionsForRegularFile: FilePermissions? { + if let newFile = self.newFile { + return newFile.permissions ?? .defaultsForRegularFile + } else { + return nil + } + } + + var descriptorOptions: FileDescriptor.OpenOptions { + var options = FileDescriptor.OpenOptions() + + if !self.followSymbolicLinks { + options.insert(.noFollow) + } + + if self.closeOnExec { + options.insert(.closeOnExec) + } + + if self.newFile != nil { + options.insert(.create) + } + + switch self.existingFile { + case .none: + options.insert(.exclusiveCreate) + case .open: + () + case .truncate: + options.insert(.truncate) + } + + return options + } +} + +extension OpenOptions.Read { + var descriptorOptions: FileDescriptor.OpenOptions { + var options = FileDescriptor.OpenOptions() + + if !self.followSymbolicLinks { + options.insert(.noFollow) + } + + if self.closeOnExec { + options.insert(.closeOnExec) + } + + return options + } +} + +extension OpenOptions.Directory { + var descriptorOptions: FileDescriptor.OpenOptions { + var options = FileDescriptor.OpenOptions([.directory]) + + if !self.followSymbolicLinks { + options.insert(.noFollow) + } + + if self.closeOnExec { + options.insert(.closeOnExec) + } + + return options + } +} + +extension FileDescriptor.OpenOptions { + public init(_ options: OpenOptions.Read) { + self = options.descriptorOptions + } + + public init(_ options: OpenOptions.Write) { + self = options.descriptorOptions + } + + public init(_ options: OpenOptions.Directory) { + self = options.descriptorOptions + } +} + +extension FilePermissions { + /// Default permissions for regular files; owner read-write, group read, other read. + internal static let defaultsForRegularFile: FilePermissions = [ + .ownerReadWrite, + .groupRead, + .otherRead, + ] + + /// Default permissions for directories; owner read-write-execute, group read-execute, other + /// read-execute. + internal static let defaultsForDirectory: FilePermissions = [ + .ownerReadWriteExecute, + .groupReadExecute, + .otherReadExecute, + ] +} diff --git a/Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift b/Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift new file mode 100644 index 0000000000..552401fb6f --- /dev/null +++ b/Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOFileSystem + +import struct Foundation.Date + +extension Date { + public init(timespec: FileInfo.Timespec) { + let timeInterval = Double(timespec.seconds) + Double(timespec.nanoseconds) / 1_000_000_000 + self = Date(timeIntervalSince1970: timeInterval) + } +} + +extension FileInfo.Timespec { + /// The UTC time of the timestamp. + public var date: Date { + return Date(timespec: self) + } +} diff --git a/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift b/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift new file mode 100644 index 0000000000..4dcaba4f0e --- /dev/null +++ b/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOFileSystem +import NIOFileSystemFoundationCompat +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class FileSystemBytesConformanceTests: XCTestCase { + func testTimepecToDate() async throws { + XCTAssertEqual( + FileInfo.Timespec(seconds: 0, nanoseconds: 0).date, + Date(timeIntervalSince1970: 0) + ) + XCTAssertEqual( + FileInfo.Timespec(seconds: 1, nanoseconds: 0).date, + Date(timeIntervalSince1970: 1) + ) + XCTAssertEqual( + FileInfo.Timespec(seconds: 1, nanoseconds: 1).date, + Date(timeIntervalSince1970: 1.000000001) + ) + } +} diff --git a/Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift b/Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift new file mode 100644 index 0000000000..ccf26350ec --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift @@ -0,0 +1,261 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +import NIOFileSystem +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class BufferedReaderTests: XCTestCase { + func testBufferedReaderSizeAndCapacity() async throws { + let fs = FileSystem.shared + try await fs.withFileHandle(forReadingAt: #filePath) { handle in + var reader = handle.bufferedReader(capacity: .bytes(128 * 1024)) + XCTAssertEqual(reader.count, 0) + XCTAssertEqual(reader.capacity, 128 * 1024) + + let byte = try await reader.read(.bytes(1)) + XCTAssertEqual(byte.readableBytes, 1) + + // Buffer should be non-empty; there's more than one byte in this file. + XCTAssertGreaterThan(reader.count, 0) + // It should be no greater than the buffer capacity, however. + XCTAssertLessThanOrEqual(reader.count, reader.capacity) + } + } + + func testBufferedReaderReadFixedSize() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { handle in + var writer = handle.bufferedWriter() + try await writer.write(contentsOf: repeatElement(0, count: 1024 * 1024)) + try await writer.flush() + } + + try await fs.withFileHandle(forReadingAt: path) { handle in + var reader = handle.bufferedReader() + let allTheBytes = try await reader.read(.bytes(1024 * 1024)) + XCTAssertEqual(allTheBytes.readableBytes, 1024 * 1024) + + let noBytes = try await reader.read(.bytes(1024)) + XCTAssertEqual(noBytes.readableBytes, 0) + } + } + + func testBufferedReaderMultipleReads() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { handle in + var writer = handle.bufferedWriter() + try await writer.write(contentsOf: repeatElement(0, count: 1024 * 1024)) + try await writer.flush() + } + + try await fs.withFileHandle(forReadingAt: path) { handle in + var reader = handle.bufferedReader() + var allTheBytes = [UInt8]() + + while true { + let byte = try await reader.read(.bytes(100)) + if byte.readableBytes == 0 { break } + allTheBytes.append(contentsOf: byte.readableBytesView) + } + + XCTAssertEqual(allTheBytes.count, 1024 * 1024) + } + } + + func testBufferedReaderReadWhile() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { handle in + var writer = handle.bufferedWriter() + for byte in UInt8.min...UInt8.max { + try await writer.write(contentsOf: repeatElement(byte, count: 1024)) + } + try await writer.flush() + } + + try await fs.withFileHandle(forReadingAt: path) { handle in + var reader = handle.bufferedReader() + let zeros = try await reader.read { $0 == 0 } + XCTAssertEqual(zeros, ByteBuffer(bytes: Array(repeating: 0, count: 1024))) + + let onesAndTwos = try await reader.read { $0 < 3 } + var expectedOnesAndTwos = ByteBuffer() + expectedOnesAndTwos.writeRepeatingByte(1, count: 1024) + expectedOnesAndTwos.writeRepeatingByte(2, count: 1024) + + XCTAssertEqual(onesAndTwos, expectedOnesAndTwos) + + let threesThroughNines = try await reader.read { $0 < 10 } + var expectedThreesThroughNines = ByteBuffer() + for byte in UInt8(3)...9 { + expectedThreesThroughNines.writeRepeatingByte(byte, count: 1024) + } + XCTAssertEqual(threesThroughNines, expectedThreesThroughNines) + + let theRest = try await reader.read { _ in true } + XCTAssertEqual(theRest.readableBytes, 246 * 1024) + } + } + + func testBufferedReaderDropFixedSize() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { handle in + var writer = handle.bufferedWriter() + for byte in UInt8.min...UInt8.max { + try await writer.write(contentsOf: repeatElement(byte, count: 1000)) + } + try await writer.flush() + } + + try await fs.withFileHandle(forReadingAt: path) { handle in + var reader = handle.bufferedReader() + try await reader.drop(1000) + let ones = try await reader.read(.bytes(1000)) + XCTAssertEqual(ones, ByteBuffer(repeating: 1, count: 1000)) + + try await reader.drop(1500) + let threes = try await reader.read(.bytes(500)) + XCTAssertEqual(threes, ByteBuffer(repeating: 3, count: 500)) + + // More than remains in the file. + try await reader.drop(1_000_000) + let empty = try await reader.read(.bytes(1000)) + XCTAssertEqual(empty.readableBytes, 0) + } + } + + func testBufferedReaderDropWhile() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { handle in + var writer = handle.bufferedWriter() + for byte in UInt8.min...UInt8.max { + try await writer.write(contentsOf: repeatElement(byte, count: 1024)) + } + try await writer.flush() + } + + try await fs.withFileHandle(forReadingAt: path) { handle in + var reader = handle.bufferedReader() + try await reader.drop(while: { $0 < 255 }) + let bytes = try await reader.read(.bytes(1024)) + XCTAssertEqual(bytes, ByteBuffer(repeating: 255, count: 1024)) + + let empty = try await reader.read(.bytes(1)) + XCTAssertEqual(empty.readableBytes, 0) + } + } + + func testBufferedReaderReadingText() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { handle in + let text = """ + Here's to the crazy ones, the misfits, the rebels, the troublemakers, \ + the round pegs in the square holes, the ones who see things differently. + """ + + var writer = handle.bufferedWriter() + try await writer.write(contentsOf: text.utf8) + try await writer.flush() + } + + try await fs.withFileHandle(forReadingAt: path) { file in + var reader = file.bufferedReader() + var words = [String]() + + func isWordIsh(_ byte: UInt8) -> Bool { + switch byte { + case UInt8(ascii: "a")...UInt8(ascii: "z"), + UInt8(ascii: "A")...UInt8(ascii: "Z"), + UInt8(ascii: "'"): + return true + default: + return false + } + } + + repeat { + // Gobble up whitespace etc.. + try await reader.drop(while: { !isWordIsh($0) }) + // Read the next word. + var characters = try await reader.read(while: isWordIsh(_:)) + + if characters.readableBytes == 0 { + break // Done. + } else { + words.append(characters.readString(length: characters.readableBytes)!) + } + } while true + + let expected: [String] = [ + "Here's", + "to", + "the", + "crazy", + "ones", + "the", + "misfits", + "the", + "rebels", + "the", + "troublemakers", + "the", + "round", + "pegs", + "in", + "the", + "square", + "holes", + "the", + "ones", + "who", + "see", + "things", + "differently", + ] + + XCTAssertEqual(words, expected) + } + } +} diff --git a/Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift b/Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift new file mode 100644 index 0000000000..68dad5290c --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift @@ -0,0 +1,143 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +@_spi(Testing) import NIOFileSystem +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class BufferedWriterTests: XCTestCase { + func testBufferedWriter() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forReadingAndWritingAt: path, + options: .newFile(replaceExisting: false) + ) { file in + let bufferSize = 8192 + var writer = file.bufferedWriter(capacity: .bytes(Int64(bufferSize))) + XCTAssertEqual(writer.bufferedBytes, 0) + + // Write a full buffers worth of bytes, should be flushed immediately. + try await writer.write(contentsOf: repeatElement(0, count: bufferSize)) + XCTAssertEqual(writer.bufferedBytes, 0) + + // Write just under a buffer. + try await writer.write(contentsOf: repeatElement(1, count: bufferSize - 1)) + XCTAssertEqual(writer.bufferedBytes, bufferSize - 1) + + // Try to read the as-yet-unwritten bytes. + let emptyChunk = try await file.readChunk( + fromAbsoluteOffset: Int64(bufferSize), + length: .bytes(Int64(bufferSize - 1)) + ) + XCTAssertEqual(emptyChunk.readableBytes, 0) + + // Write one more byte to flush out the buffer. + try await writer.write(contentsOf: repeatElement(1, count: 1)) + XCTAssertEqual(writer.bufferedBytes, 0) + + // Try to read now that the bytes have been finished. + let chunk = try await file.readChunk( + fromAbsoluteOffset: Int64(bufferSize), + length: .bytes(Int64(bufferSize)) + ) + XCTAssertEqual(chunk, ByteBuffer(repeating: 1, count: bufferSize)) + } + } + + func testBufferedWriterAsyncSequenceOfBytes() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forReadingAndWritingAt: path, + options: .newFile(replaceExisting: false) + ) { file in + let bufferSize = 8192 + var writer = file.bufferedWriter(capacity: .bytes(Int64(bufferSize))) + XCTAssertEqual(writer.bufferedBytes, 0) + + let streamOfBytes = AsyncStream(UInt8.self) { continuation in + for _ in 0..<16384 { + continuation.yield(0) + } + continuation.finish() + } + + var written = try await writer.write(contentsOf: streamOfBytes) + XCTAssertEqual(written, 16384) + XCTAssertEqual(writer.bufferedBytes, 0) + + let streamOfChunks = AsyncStream([UInt8].self) { continuation in + for _ in stride(from: 0, to: 16384, by: 1024) { + continuation.yield(Array(repeating: 0, count: 1024)) + } + continuation.finish() + } + + written = try await writer.write(contentsOf: streamOfChunks) + XCTAssertEqual(written, 16384) + XCTAssertEqual(writer.bufferedBytes, 0) + + let bytes = try await file.readToEnd(maximumSizeAllowed: .bytes(1024 * 1024)) + XCTAssertEqual(bytes.readableBytes, 16384 * 2) + XCTAssertTrue(bytes.readableBytesView.allSatisfy { $0 == 0 }) + } + } + + func testBufferedWriterManualFlushing() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forReadingAndWritingAt: path, + options: .newFile(replaceExisting: false) + ) { file in + var writer = file.bufferedWriter(capacity: .bytes(1024)) + try await writer.write(contentsOf: Array(repeating: 0, count: 128)) + XCTAssertEqual(writer.bufferedBytes, 128) + + try await writer.flush() + XCTAssertEqual(writer.bufferedBytes, 0) + } + } + + func testBufferedWriterReclaimsStorageAfterLargeWrite() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle( + forReadingAndWritingAt: path, + options: .newFile(replaceExisting: false) + ) { file in + let bufferSize = 128 + var writer = file.bufferedWriter(capacity: .bytes(Int64(bufferSize))) + XCTAssertEqual(writer.bufferCapacity, 0) + + // Fill up the buffer. The capacity should be >= the buffer size. + try await writer.write(contentsOf: Array(repeating: 0, count: bufferSize)) + XCTAssertEqual(writer.bufferedBytes, 0) + XCTAssertGreaterThanOrEqual(writer.bufferCapacity, bufferSize) + + // Writes which take its internal buffer capacity over double its configured capacity + // will result in memory being reclaimed. + let doubleSize = bufferSize * 2 + try await writer.write(contentsOf: Array(repeating: 1, count: doubleSize + 1)) + XCTAssertEqual(writer.bufferedBytes, 0) + XCTAssertEqual(writer.bufferCapacity, 0) + } + } +} diff --git a/Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift b/Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift new file mode 100644 index 0000000000..a1e533b89a --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift @@ -0,0 +1,73 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +import NIOFileSystem +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class ConvenienceTests: XCTestCase { + static let fs = FileSystem.shared + + func testWriteStringToFile() async throws { + let path = try await Self.fs.temporaryFilePath() + let bytesWritten = try await "some text".write(toFileAt: path) + XCTAssertEqual(bytesWritten, 9) + + let bytes = try await ByteBuffer(contentsOf: path, maximumSizeAllowed: .bytes(1024)) + XCTAssertEqual(bytes, ByteBuffer(string: "some text")) + } + + func testWriteSequenceToFile() async throws { + let path = try await Self.fs.temporaryFilePath() + let byteSequence = stride(from: UInt8(0), to: UInt8(64), by: 1) + let bytesWritten = try await byteSequence.write(toFileAt: path) + XCTAssertEqual(bytesWritten, 64) + + let bytes = try await ByteBuffer(contentsOf: path, maximumSizeAllowed: .bytes(1024)) + XCTAssertEqual(bytes, ByteBuffer(bytes: byteSequence)) + } + + func testWriteAsyncSequenceOfBytesToFile() async throws { + let path = try await Self.fs.temporaryFilePath() + let stream = AsyncStream(UInt8.self) { continuation in + for byte in UInt8(0)..<64 { + continuation.yield(byte) + } + continuation.finish() + } + + let bytesWritten = try await stream.write(toFileAt: path) + XCTAssertEqual(bytesWritten, 64) + + let bytes = try await ByteBuffer(contentsOf: path, maximumSizeAllowed: .bytes(1024)) + XCTAssertEqual(bytes, ByteBuffer(bytes: Array(0..<64))) + } + + func testWriteAsyncSequenceOfChunksToFile() async throws { + let path = try await Self.fs.temporaryFilePath() + let stream = AsyncStream([UInt8].self) { continuation in + for lowerByte in stride(from: UInt8(0), to: 64, by: 8) { + continuation.yield(Array(lowerByte.. FilePath { + return FilePath("swift-filesystem-tests-\(UInt64.random(in: .min ... .max))") + } + + func withTemporaryFile( + autoClose: Bool = true, + _ execute: @Sendable (SystemFileHandle) async throws -> Void + ) async throws { + let path = FilePath("/tmp/\(Self.temporaryFileName())") + defer { + // Remove the file when we're done. + XCTAssertNoThrow(try Libc.remove(path).get()) + } + + try await withHandle( + forFileAtPath: path, + accessMode: .readWrite, + options: [.create, .exclusiveCreate], + permissions: .ownerReadWrite, + autoClose: autoClose + ) { handle in + try await execute(handle) + } + } + + func withTestDataDirectory( + autoClose: Bool = true, + _ execute: @Sendable (SystemFileHandle) async throws -> Void + ) async throws { + try await self.withHandle( + forFileAtPath: Self.testData, + accessMode: .readOnly, + options: [.directory, .nonBlocking], + autoClose: autoClose + ) { + try await execute($0) + } + } + + private static func removeFile(atPath path: FilePath) { + XCTAssertNoThrow(try Libc.remove(path).get()) + } + + func withHandle( + forFileAtPath path: FilePath, + accessMode: FileDescriptor.AccessMode = .readOnly, + options: FileDescriptor.OpenOptions = [], + permissions: FilePermissions? = nil, + autoClose: Bool = true, + _ execute: @Sendable (SystemFileHandle) async throws -> Void + ) async throws { + let descriptor = try FileDescriptor.open( + path, + accessMode, + options: options, + permissions: permissions + ) + let executor = await IOExecutor.running(numberOfThreads: 1) + let handle = SystemFileHandle(takingOwnershipOf: descriptor, path: path, executor: executor) + + do { + try await execute(handle) + if autoClose { + try? await handle.close() + } + await executor.drain() + } catch let skip as XCTSkip { + try? await handle.close() + await executor.drain() + throw skip + } catch { + XCTFail("Test threw error: '\(error)'") + // Always close on error. + try await handle.close() + await executor.drain() + } + } + + func testInfo() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + let info = try await handle.info() + // It's hard to make more assertions than this... + XCTAssertEqual(info.type, .regular) + XCTAssertGreaterThan(info.size, 1024) + } + + try await self.withTemporaryFile { handle in + let info = try await handle.info() + // It's hard to make more assertions than this... + XCTAssertEqual(info.type, .regular) + XCTAssertEqual(info.size, 0) + } + } + + func testExtendedAttributes() async throws { + let attribute = "attribute-name" + try await self.withTemporaryFile { handle in + do { + // We just created this but we can't assert that there won't be any attributes + // (who knows what the filesystem will do?) so we'll use this number as a baseline. + var originalAttributes = try await handle.attributeNames() + originalAttributes.sort() + + // There should be no value for this attribute, yet. + let value = try await handle.valueForAttribute(attribute) + XCTAssertEqual(value, []) + + // Set a value. + let someBytes = Array("hello, world".utf8) + try await handle.updateValueForAttribute(someBytes, attribute: attribute) + + // Retrieve it again. + let retrieved = try await handle.valueForAttribute(attribute) + XCTAssertEqual(retrieved, someBytes) + + // There should be an attribute now. + let attributes = try await handle.attributeNames() + XCTAssert(Set(attributes).isSuperset(of: originalAttributes)) + + // Remove it. + try await handle.removeValueForAttribute(attribute) + + // Should be back to the original values. + var maybeOriginalAttributes = try await handle.attributeNames() + maybeOriginalAttributes.sort() + XCTAssertEqual(originalAttributes, maybeOriginalAttributes) + } catch let error as FileSystemError where error.code == .unsupported { + throw XCTSkip("Extended attributes are not supported on this platform.") + } + } + } + + func testListExtendedAttributes() async throws { + try await self.withTemporaryFile { handle in + do { + // Set some attributes. + let attributeNames = Set((0..<5).map { "attr-\($0)" }) + for attribute in attributeNames { + try await handle.updateValueForAttribute([0, 1, 2], attribute: attribute) + } + + // List the attributes. + let attributes = try await handle.attributeNames() + XCTAssert(Set(attributes).isSuperset(of: attributeNames)) + } catch let error as FileSystemError where error.code == .unsupported { + throw XCTSkip("Extended attributes are not supported on this platform.") + } + } + } + + func testUpdatePermissions() async throws { + try await self.withTemporaryFile { handle in + let info = try await handle.info() + // Default permissions we use for temporary files. + XCTAssertEqual(info.permissions, .ownerReadWrite) + + try await handle.replacePermissions(.ownerReadWriteExecute) + let actual = try await handle.info().permissions + XCTAssertEqual(actual, .ownerReadWriteExecute) + } + } + + func testAddPermissions() async throws { + try await self.withTemporaryFile { handle in + let info = try await handle.info() + // Default permissions we use for temporary files. + XCTAssertEqual(info.permissions, .ownerReadWrite) + + let computed = try await handle.addPermissions(.ownerExecute) + let actual = try await handle.info().permissions + XCTAssertEqual(computed, actual) + XCTAssertEqual(computed, .ownerReadWriteExecute) + } + } + + func testRemovePermissions() async throws { + try await self.withTemporaryFile { handle in + let info = try await handle.info() + // Default permissions we use for temporary files. + XCTAssertEqual(info.permissions, .ownerReadWrite) + + // Set execute so we can remove it. + try await handle.replacePermissions(.ownerReadWriteExecute) + + // Remove owner execute. + let computed = try await handle.removePermissions(.ownerExecute) + let actual = try await handle.info().permissions + XCTAssertEqual(computed, actual) + XCTAssertEqual(computed, .ownerReadWrite) + } + } + + func testWithUnsafeDescriptor() async throws { + try await self.withTemporaryFile { handle in + // Check we can successfully return a value. + let value = try await handle.withUnsafeDescriptor { descriptor in + return 42 + } + XCTAssertEqual(value, 42) + } + } + + func testDetach() async throws { + try await self.withTemporaryFile(autoClose: false) { handle in + let descriptor = try handle.detachUnsafeFileDescriptor() + // We don't need this: just close it. + XCTAssertNoThrow(try descriptor.close()) + // Closing a detached handle is a no-op. + try await handle.close() + // All other methods should throw. + try await Self.testAllMethodsThrowClosed(handle) + } + } + + func testClose() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile, autoClose: false) { handle in + // Close. + try await handle.close() + // Closing is idempotent: this is fine. + try await handle.close() + // All other methods should throw. + try await Self.testAllMethodsThrowClosed(handle) + } + } + + func testReadChunk() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + do { + // Zero offset. + let bytes = try await handle.readChunk(fromAbsoluteOffset: 0, length: .bytes(80)) + let line = String(buffer: bytes) + XCTAssertEqual(line, "//===----------------------------------------------------------------------===//") + } + + do { + // Non-zero offset. + let bytes = try await handle.readChunk(fromAbsoluteOffset: 5, length: .bytes(10)) + let line = String(buffer: bytes) + XCTAssertEqual(line, "----------") + } + + do { + // Length longer than file. + let info = try await handle.info() + let bytes = try await handle.readChunk( + fromAbsoluteOffset: 0, + length: .bytes(info.size + 10) + ) + // Bytes should not be larger than the file. + XCTAssertEqual(bytes.readableBytes, Int(info.size)) + } + } + } + + func testReadWholeFile() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + // Check errors are thrown if we don't allow enough bytes. + await XCTAssertThrowsFileSystemErrorAsync { + try await handle.readToEnd(maximumSizeAllowed: .bytes(0)) + } onError: { error in + XCTAssertEqual(error.code, .resourceExhausted) + } + + // Validate that we can read the whole file when at the limit. + let info = try await handle.info() + let contents = try await handle.readToEnd(maximumSizeAllowed: .bytes(info.size)) + // Compare against the data as read by Foundation. + let readByFoundation = try Data(contentsOf: URL(fileURLWithPath: Self.thisFile.string)) + XCTAssertEqual( + contents, + ByteBuffer(data: readByFoundation), + "Contents of \(Self.thisFile) differ to that read by Foundation" + ) + } + } + + func testWriteAndReadUnseekableFile() async throws { + let privateTempDirPath = try await FileSystem.shared.createTemporaryDirectory(template: "test-XXX") + guard mkfifo(privateTempDirPath.appending("fifo").string, 0o644) == 0 else { + XCTFail("Error calling mkfifo.") + return + } + + try await self.withHandle(forFileAtPath: privateTempDirPath.appending("fifo"), accessMode: .readWrite) { + handle in + let someBytes = ByteBuffer(repeating: 42, count: 1546) + try await handle.write(contentsOf: someBytes.readableBytesView, toAbsoluteOffset: 0) + + let readSomeBytes = try await handle.readToEnd(maximumSizeAllowed: .bytes(1546)) + XCTAssertEqual(readSomeBytes, someBytes) + } + } + + func testWriteAndReadUnseekableFileOverMaximumSizeAllowedThrowsError() async throws { + let privateTempDirPath = try await FileSystem.shared.createTemporaryDirectory(template: "test-XXX") + guard mkfifo(privateTempDirPath.appending("fifo").string, 0o644) == 0 else { + XCTFail("Error calling mkfifo.") + return + } + + try await self.withHandle(forFileAtPath: privateTempDirPath.appending("fifo"), accessMode: .readWrite) { + handle in + let someBytes = [UInt8](repeating: 42, count: 10) + try await handle.write(contentsOf: someBytes, toAbsoluteOffset: 0) + + await XCTAssertThrowsFileSystemErrorAsync { + try await handle.readToEnd(maximumSizeAllowed: .bytes(9)) + } onError: { error in + XCTAssertEqual(error.code, .resourceExhausted) + } + } + } + + func testWriteAndReadUnseekableFileWithOffsetsThrows() async throws { + let privateTempDirPath = try await FileSystem.shared.createTemporaryDirectory(template: "test-XXX") + guard mkfifo(privateTempDirPath.appending("fifo").string, 0o644) == 0 else { + XCTFail("Error calling mkfifo.") + return + } + + try await self.withHandle(forFileAtPath: privateTempDirPath.appending("fifo"), accessMode: .readWrite) { + handle in + let someBytes = [UInt8](repeating: 42, count: 1546) + + await XCTAssertThrowsErrorAsync { + try await handle.write(contentsOf: someBytes, toAbsoluteOffset: 42) + XCTFail("Should have thrown") + } onError: { error in + let fileSystemError = error as! FileSystemError + XCTAssertEqual(fileSystemError.code, .unsupported) + XCTAssertEqual(fileSystemError.message, "File is unseekable.") + } + + await XCTAssertThrowsErrorAsync { + _ = try await handle.readToEnd(fromAbsoluteOffset: 42, maximumSizeAllowed: .bytes(1)) + XCTFail("Should have thrown") + } onError: { error in + let fileSystemError = error as! FileSystemError + XCTAssertEqual(fileSystemError.code, .unsupported) + XCTAssertEqual(fileSystemError.message, "File is unseekable.") + } + } + } + + func testReadWholeFileWithOffsets() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + let info = try await handle.info() + + // We should be able to do a zero-length read at the end of the file with a max size + // allowed of zero. + let empty = try await handle.readToEnd( + fromAbsoluteOffset: info.size, + maximumSizeAllowed: .bytes(0) + ) + XCTAssertEqual(empty.readableBytes, 0) + + // Read the last 100 bytes. + let bytes = try await handle.readToEnd( + fromAbsoluteOffset: info.size - 100, + maximumSizeAllowed: .bytes(100) + ) + + // Compare against the data as read by Foundation. + let readByFoundation = try Data(contentsOf: URL(fileURLWithPath: Self.thisFile.string)) + let tail = readByFoundation.dropFirst(readByFoundation.count - 100) + XCTAssertEqual( + bytes, + ByteBuffer(data: tail), + "Contents of \(Self.thisFile) differ to that read by Foundation" + ) + } + } + + func testReadFileAsChunks() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + var bytes = ByteBuffer() + + for try await chunk in handle.readChunks(in: ..., chunkLength: .bytes(128)) { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + + var contents = try await handle.readToEnd(maximumSizeAllowed: .bytes(1024 * 1024)) + XCTAssertEqual( + bytes, + contents, + """ + Read \(bytes.readableBytes) which were different to the \(contents.readableBytes) expected bytes. + """ + ) + + // Read from an offset. + bytes.clear() + for try await chunk in handle.readChunks(in: 100..., chunkLength: .bytes(128)) { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + + contents.moveReaderIndex(forwardBy: 100) + XCTAssertEqual( + bytes, + contents, + """ + Read \(bytes.readableBytes) which were different to the \(contents.readableBytes) \ + expected bytes. + """ + ) + } + } + + func testReadEmptyRange() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + // No bytes should be read. + for try await _ in handle.readChunks(in: 0..<0, chunkLength: .bytes(128)) { + XCTFail("We shouldn't read any chunks.") + } + + // No bytes should be read. + for try await _ in handle.readChunks(in: 100..<100, chunkLength: .bytes(128)) { + XCTFail("We shouldn't read any chunks.") + } + } + } + + enum RangeType { + case closed(ClosedRange) + case partialThrough(PartialRangeThrough) + case partialUpTo(PartialRangeUpTo) + } + + static func testReadEndOffsetExceedsEOF(range: RangeType, handle: SystemFileHandle) async throws { + var bytes = ByteBuffer() + let fileChunks: FileChunks + + switch range { + case .closed(let offsets): + fileChunks = handle.readChunks(in: offsets, chunkLength: .bytes(128)) + case .partialUpTo(let offsets): + fileChunks = handle.readChunks(in: offsets, chunkLength: .bytes(128)) + case .partialThrough(let offsets): + fileChunks = handle.readChunks(in: offsets, chunkLength: .bytes(128)) + } + + for try await chunk in fileChunks { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + + // We should read bytes only before the EOF. + let contents = try await handle.readToEnd(maximumSizeAllowed: .bytes(1024 * 1024)) + XCTAssertEqual(bytes, contents) + } + + func testReadEndOffsetExceedsEOFClosedrange() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + let info = try await handle.info() + try await Self.testReadEndOffsetExceedsEOF( + range: .closed(0...(info.size + 3)), + handle: handle + ) + } + } + + func testReadEndOffsetExceedsEOFPartialThrough() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + let info = try await handle.info() + try await Self.testReadEndOffsetExceedsEOF( + range: .partialThrough(...(info.size + 3)), + handle: handle + ) + } + } + + func testReadEndOffsetExceedsEOFPartialUpTo() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + let info = try await handle.info() + try await Self.testReadEndOffsetExceedsEOF( + range: .partialUpTo(..<(info.size + 3)), + handle: handle + ) + } + } + + func testReadRangeShorterThanChunklength() async throws { + // Reading chunks of bytes from within a range that is shorter than the chunklength + // and the length of the file. + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + var bytes = ByteBuffer() + for try await chunk in handle.readChunks(in: 0...120, chunkLength: .bytes(128)) { + XCTAssertEqual(chunk.readableBytes, 121) + bytes.writeImmutableBuffer(chunk) + } + + // We should only read bytes from within the range. + XCTAssertEqual( + bytes.readableBytes, + 121, + """ + Read \(bytes.readableBytes) which were different to the 121 \ + expected bytes. + """ + ) + } + } + + func testReadRangeLongerThanChunkAndNotMultipleOfChunkLength() async throws { + // Reading chunks of bytes from within a range longer than the chunklength + // and with size not a multiple of the chunklegth. + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + var bytes = ByteBuffer() + for try await chunk in handle.readChunks(in: 0...200, chunkLength: .bytes(128)) { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + + // We should only read bytes from within the range. + XCTAssertEqual( + bytes.readableBytes, + 201, + """ + Read \(bytes.readableBytes) which were different to the 201 \ + expected bytes. + """ + ) + } + } + + func testReadPartialFromRange() async throws { + // Reading chunks of bytes from a PartialRangeFrom. + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + var bytes = ByteBuffer() + for try await chunk in handle.readChunks(in: 0..., chunkLength: .bytes(128)) { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + let contents = try await handle.readToEnd(maximumSizeAllowed: .bytes(1024 * 1024)) + + // We should read bytes until EOF. + XCTAssertEqual( + bytes.readableBytes, + contents.readableBytes, + """ + Read \(bytes.readableBytes) which were different to the \(contents.readableBytes) \ + expected bytes. + """ + ) + } + } + + func testUnboundedRange() async throws { + // Reading chunks of bytes from an UnboundedRange. + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + var bytes = ByteBuffer() + for try await chunk in handle.readChunks(in: ..., chunkLength: .bytes(128)) { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + let contents = try await handle.readToEnd(maximumSizeAllowed: .bytes(1024 * 1024)) + + // We should read bytes until EOF. + XCTAssertEqual( + bytes.readableBytes, + contents.readableBytes, + """ + Read \(bytes.readableBytes) which were different to the \(contents.readableBytes) \ + expected bytes. + """ + ) + } + } + + func testReadPartialRange() async throws { + // Reading chunks of bytes from a PartialRangeThrough with the upper bound inside the file. + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + var bytes = ByteBuffer() + let contents = try await handle.readToEnd(maximumSizeAllowed: .bytes(1024 * 1024)) + + for try await chunk in handle.readChunks(in: ...200, chunkLength: .bytes(128)) { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + + // We should read the first bytes from the beginning of the file, until reaching the + // upper bound of the range (inclusive). + XCTAssertEqual(bytes, contents.getSlice(at: contents.readerIndex, length: 201)) + + bytes.clear() + + for try await chunk in handle.readChunks(in: ..<200, chunkLength: .bytes(128)) { + XCTAssertLessThanOrEqual(chunk.readableBytes, 128) + bytes.writeImmutableBuffer(chunk) + } + + // We should read the first bytes from the beginning of the file, until reaching the + // upper bound of the range (inclusive). + XCTAssertEqual(bytes, contents.getSlice(at: contents.readerIndex, length: 200)) + } + } + + func testReadChunksOverloadAmbiguity() async throws { + try await self.withHandle(forFileAtPath: Self.thisFile) { handle in + // Seven possibilities for range: + // 1. ... + // 2. x...y + // 3. x..( + line: UInt = #line, + _ expression: () async throws -> R +) async throws { + await XCTAssertThrowsFileSystemErrorAsync { + try await expression() + } onError: { error in + XCTAssertEqual(error.code, .closed) + } +} diff --git a/Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift b/Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift new file mode 100644 index 0000000000..0b6b5d74af --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) import NIOFileSystem +import SystemPackage +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension FileSystemTests { + func testRemoveOneItemIgnoresNonExistentFile() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + let removed = try await fs.removeOneItem(at: path) + XCTAssertEqual(removed, 0) + } +} diff --git a/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift new file mode 100644 index 0000000000..1074ca0907 --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift @@ -0,0 +1,1430 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +@_spi(Testing) import NIOFileSystem +@preconcurrency import SystemPackage +import XCTest + +extension FilePath { + static let testData = FilePath(#filePath) + .removingLastComponent() // FileHandleTests.swift + .appending("Test Data") + .lexicallyNormalized() + + static let testDataReadme = Self.testData.appending("README.md") + static let testDataReadmeSymlink = Self.testData.appending("README.md.symlink") +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension FileSystem { + func temporaryFilePath( + _ function: String = #function, + inTemporaryDirectory: Bool = true + ) async throws -> FilePath { + if inTemporaryDirectory { + let directory = try await self.temporaryDirectory + return self.temporaryFilePath(function, inDirectory: directory) + } else { + return self.temporaryFilePath(function, inDirectory: nil) + } + } + + func temporaryFilePath( + _ function: String = #function, + inDirectory directory: FilePath? + ) -> FilePath { + let index = function.firstIndex(of: "(")! + let functionName = function.prefix(upTo: index) + let random = UInt32.random(in: .min ... .max) + let fileName = "\(functionName)-\(random)" + + if let directory = directory { + return directory.appending(fileName) + } else { + return FilePath(fileName) + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class FileSystemTests: XCTestCase { + var fs: FileSystem { return .shared } + + func testOpenFileForReading() async throws { + try await self.fs.withFileHandle(forReadingAt: .testDataReadme) { file in + let info = try await file.info() + XCTAssertEqual(info.type, .regular) + XCTAssertGreaterThan(info.size, 0) + } + } + + func testOpenFileForReadingFollowsSymlink() async throws { + try await self.fs.withFileHandle(forReadingAt: .testDataReadmeSymlink) { file in + let info = try await file.info() + XCTAssertEqual(info.type, .regular) + } + } + + func testOpenSymlinkForReadingWithoutFollow() async throws { + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withFileHandle( + forReadingAt: .testDataReadmeSymlink, + options: OpenOptions.Read(followSymbolicLinks: false) + ) { _ in + } + } onError: { error in + XCTAssertEqual(error.code, .invalidArgument) + } + } + + func testOpenNonExistentFileForReading() async throws { + let path = try await self.fs.temporaryFilePath() + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withFileHandle(forReadingAt: path) { _ in } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testOpenFileWhereIntermediateIsNotADirectory() async throws { + let path = FilePath(#filePath).appending("foobar") + + // For reading: + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withFileHandle(forReadingAt: path) { _ in + XCTFail("File unexpectedly opened") + } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + + // For writing: + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withFileHandle(forWritingAt: path) { _ in + XCTFail("File unexpectedly opened") + } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + + // As a directory: + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withDirectoryHandle(atPath: path) { _ in + XCTFail("File unexpectedly opened") + } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testOpenFileForWriting() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { file in + let info = try await file.info() + XCTAssertEqual(info.type, .regular) + XCTAssertEqual(info.size, 0) + } + } + + func testOpenNonExistentFileForWritingWithoutCreating() async throws { + let path = try await self.fs.temporaryFilePath() + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withFileHandle( + forWritingAt: path, + options: .modifyFile(createIfNecessary: false) + ) { _ in } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testOpenForWritingFollowingSymlink() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { _ in } + + let link = try await self.fs.temporaryFilePath() + try await self.fs.createSymbolicLink(at: link, withDestination: path) + + // Open via the link and write. + try await self.fs.withFileHandle( + forWritingAt: link, + options: .modifyFile(createIfNecessary: false) + ) { file in + let info = try await file.info() + XCTAssertEqual(info.type, .regular) + + try await file.write(contentsOf: [0, 1, 2], toAbsoluteOffset: 0) + } + + let contents = try await ByteBuffer( + contentsOf: path, + maximumSizeAllowed: .bytes(1024), + fileSystem: self.fs + ) + XCTAssertEqual(contents, ByteBuffer(bytes: [0, 1, 2])) + } + + func testOpenNonExistentFileForWritingWithMaterialization() async throws { + for isAbsolute in [true, false] { + let path = try await self.fs.temporaryFilePath(inTemporaryDirectory: isAbsolute) + XCTAssertEqual(path.isAbsolute, isAbsolute) + + await XCTAssertThrowsErrorAsync { + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { file in + // The file hasn't materialized yet, so no file at the expected path + // should exist. + let info = try await self.fs.info(forFileAt: path) + XCTAssertNil(info) + + try await file.write( + contentsOf: repeatElement(0, count: 1024), + toAbsoluteOffset: 0 + ) + throw CancellationError() + } + } onError: { error in + XCTAssert(error is CancellationError) + } + + // Threw in the 'with' block; the file shouldn't exist anymore. + let info = try await self.fs.info(forFileAt: path) + XCTAssertNil(info) + } + } + + func testOpenExistingFileForWritingWithMaterialization() async throws { + for isAbsolute in [true, false] { + let path = try await self.fs.temporaryFilePath(inTemporaryDirectory: isAbsolute) + XCTAssertEqual(path.isAbsolute, isAbsolute) + + // Avoid dirtying the current working directory. + if path.isRelative { + self.addTeardownBlock { [fileSystem = self.fs] in + try await fileSystem.removeItem(at: path) + } + } + + // Create a file and write some data to it. + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { file in + _ = try await file.write(contentsOf: [0, 1, 2], toAbsoluteOffset: 0) + } + + // File must exist now. + let info = try await self.fs.info(forFileAt: path) + XCTAssertNotNil(info) + + // Open the existing file and truncate it. Write different bytes to it but then throw an + // error. The changes shouldn't persist because of the error. + await XCTAssertThrowsErrorAsync { + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: true) + ) { file in + try await file.write(contentsOf: [3, 4, 5], toAbsoluteOffset: 0) + throw CancellationError() + } + } onError: { error in + XCTAssert(error is CancellationError) + } + + // Read the file again, it should contain the original bytes. + let bytes = try await ByteBuffer(contentsOf: path, maximumSizeAllowed: .megabytes(1)) + XCTAssertEqual(bytes, ByteBuffer(bytes: [0, 1, 2])) + } + } + + func testOpenFileForReadingAndWriting() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.withFileHandle( + forReadingAndWritingAt: path, + options: .newFile(replaceExisting: false) + ) { + file in + let info = try await file.info() + XCTAssertEqual(info.type, .regular) + XCTAssertEqual(info.size, 0) + } + } + + func testOpenNonExistentFileForReadingAndWritingWithoutCreating() async throws { + let path = try await self.fs.temporaryFilePath() + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withFileHandle( + forReadingAndWritingAt: path, + options: .modifyFile(createIfNecessary: false) + ) { + _ in + } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testOpenDirectory() async throws { + try await self.fs.withDirectoryHandle(atPath: .testData) { dir in + let info = try await dir.info() + XCTAssertEqual(info.type, .directory) + XCTAssertGreaterThan(info.size, 0) + } + } + + func testOpenNonExistentDirectory() async throws { + let path = try await self.fs.temporaryFilePath() + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.withDirectoryHandle(atPath: path) { _ in } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testOpenDirectoryFollowingSymlink() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.createDirectory(at: path, withIntermediateDirectories: true) + + let link = try await self.fs.temporaryFilePath() + try await self.fs.createSymbolicLink(at: link, withDestination: path) + + try await self.fs.withDirectoryHandle(atPath: link) { dir in + let info = try await dir.info() + XCTAssertEqual(info.type, .directory) + } + } + + func testOpenNonExistentFileForWritingRelativeToDirectoryWithMaterialization() async throws { + // (false, false) isn't supported. + let isPathAbsolute: [(Bool, Bool)] = [(true, true), (true, false), (false, true)] + + for (isDirectoryAbsolute, isFileAbsolute) in isPathAbsolute { + let directoryPath = try await self.fs.temporaryFilePath( + inTemporaryDirectory: isDirectoryAbsolute + ) + XCTAssertEqual(directoryPath.isAbsolute, isDirectoryAbsolute) + + // Avoid dirtying the current working directory. + if directoryPath.isRelative { + self.addTeardownBlock { [fileSystem = self.fs] in + try await fileSystem.removeItem(at: directoryPath) + } + } + + // Create the directory and open it + try await self.fs.createDirectory(at: directoryPath, withIntermediateDirectories: true) + try await self.fs.withDirectoryHandle(atPath: directoryPath) { directory in + let filePath = try await self.fs.temporaryFilePath( + inTemporaryDirectory: isFileAbsolute + ) + + XCTAssertEqual(filePath.isAbsolute, isFileAbsolute) + + // Create the file and throw. + await XCTAssertThrowsErrorAsync { + try await directory.withFileHandle( + forWritingAt: filePath, + options: .newFile(replaceExisting: false) + ) { handle in + throw CancellationError() + } + } onError: { error in + XCTAssert(error is CancellationError) + } + + // The file shouldn't exist. + await XCTAssertThrowsFileSystemErrorAsync { + try await directory.withFileHandle(forReadingAt: filePath) { _ in } + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + } + } + + func testOpenExistingFileForWritingRelativeToDirectoryWithMaterialization() async throws { + // (false, false) isn't supported. + let isPathAbsolute: [(Bool, Bool)] = [(true, true), (true, false), (false, true)] + + for (isDirectoryAbsolute, isFileAbsolute) in isPathAbsolute { + let directoryPath = try await self.fs.temporaryFilePath( + inTemporaryDirectory: isDirectoryAbsolute + ) + + XCTAssertEqual(directoryPath.isAbsolute, isDirectoryAbsolute) + + if directoryPath.isRelative { + self.addTeardownBlock { [fileSystem = self.fs] in + try await fileSystem.removeItem(at: directoryPath, recursively: true) + } + } + + // Create the directory and open it + try await self.fs.createDirectory(at: directoryPath, withIntermediateDirectories: true) + try await self.fs.withDirectoryHandle(atPath: directoryPath) { directory in + let filePath = try await self.fs.temporaryFilePath( + inTemporaryDirectory: isFileAbsolute + ) + + XCTAssertEqual(filePath.isAbsolute, isFileAbsolute) + + // Create the file and write some bytes. + try await directory.withFileHandle( + forWritingAt: filePath, + options: .newFile(replaceExisting: false) + ) { file in + _ = try await file.write( + contentsOf: repeatElement(0, count: 1024), + toAbsoluteOffset: 0 + ) + } + + // Create the file and throw. + await XCTAssertThrowsErrorAsync { + try await directory.withFileHandle( + forWritingAt: filePath, + options: .newFile(replaceExisting: true) + ) { handle in + throw CancellationError() + } + } onError: { error in + XCTAssert(error is CancellationError) + } + + // The file should contain the original bytes. + try await directory.withFileHandle(forReadingAt: filePath) { file in + let bytes = try await file.readToEnd(maximumSizeAllowed: .megabytes(1)) + XCTAssertEqual(bytes, ByteBuffer(repeating: 0, count: 1024)) + } + } + } + } + + func testCreateDirectory() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.createDirectory( + at: path, + withIntermediateDirectories: false, + permissions: nil + ) + + try await self.fs.withDirectoryHandle(atPath: path) { dir in + let info = try await dir.info() + XCTAssertEqual(info.type, .directory) + XCTAssertGreaterThan(info.size, 0) + } + } + + func testCreateDirectoryWithIntermediatePaths() async throws { + var path = try await self.fs.temporaryFilePath() + for i in 0..<100 { + path.append("\(i)") + } + + try await self.fs.createDirectory( + at: path, + withIntermediateDirectories: true, + permissions: nil + ) + + try await self.fs.withDirectoryHandle(atPath: path) { dir in + let info = try await dir.info() + XCTAssertEqual(info.type, .directory) + XCTAssertGreaterThan(info.size, 0) + } + } + + func testCreateDirectoryAtPathWhichExists() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { _ in } + + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.createDirectory(at: path, withIntermediateDirectories: true) + } onError: { error in + XCTAssertEqual(error.code, .fileAlreadyExists) + } + } + + func testCreateDirectoryAtPathWhereParentDoesNotExist() async throws { + let parent = try await self.fs.temporaryFilePath() + let path = parent.appending("path") + + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.createDirectory(at: path, withIntermediateDirectories: false) + } onError: { error in + XCTAssertEqual(error.code, .invalidArgument) + } + } + + func testCurrentWorkingDirectory() async throws { + let directory = try await self.fs.currentWorkingDirectory + XCTAssert(!directory.isEmpty) + XCTAssert(directory.isAbsolute) + } + + func testTemporaryDirectory() async throws { + let directory = try await self.fs.temporaryDirectory + XCTAssert(!directory.isEmpty) + XCTAssert(directory.isAbsolute) + } + + func testInfo() async throws { + let info = try await self.fs.info(forFileAt: .testDataReadme, infoAboutSymbolicLink: false) + XCTAssertEqual(info?.type, .regular) + XCTAssertGreaterThan(info?.size ?? -1, Int64(0)) + } + + func testInfoResolvingSymbolicLinks() async throws { + let info = try await self.fs.info( + forFileAt: .testDataReadmeSymlink, + infoAboutSymbolicLink: false + ) + XCTAssertEqual(info?.type, .regular) + XCTAssertGreaterThan(info?.size ?? -1, Int64(0)) + } + + func testInfoWithoutResolvingSymbolicLinks() async throws { + let info = try await self.fs.info( + forFileAt: .testDataReadmeSymlink, + infoAboutSymbolicLink: true + ) + XCTAssertEqual(info?.type, .symlink) + XCTAssertGreaterThan(info?.size ?? -1, Int64(0)) + } + + func testCreateSymbolicLink() async throws { + let path = try await self.fs.temporaryFilePath() + let destination = FilePath.testDataReadme + + try await self.fs.createSymbolicLink(at: path, withDestination: destination) + let info = try await self.fs.info(forFileAt: destination, infoAboutSymbolicLink: true) + let infoViaLink = try await self.fs.info(forFileAt: path, infoAboutSymbolicLink: false) + XCTAssertEqual(info, infoViaLink) + } + + func testDestinationOfSymbolicLink() async throws { + do { + // Relative symbolic link. + let destination = try await self.fs.destinationOfSymbolicLink( + at: .testDataReadmeSymlink + ) + XCTAssertEqual(destination, "README.md") + } + + do { + // Absolute symbolic link. + let path = try await self.fs.temporaryFilePath() + try await self.fs.createSymbolicLink(at: path, withDestination: .testDataReadme) + let destination = try await self.fs.destinationOfSymbolicLink(at: path) + XCTAssertEqual(destination, .testDataReadme) + } + } + + func testCopySingleFile() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.copyItem(at: .testDataReadme, to: path) + + try await self.fs.withFileHandle(forReadingAt: path) { copy in + try await self.fs.withFileHandle(forReadingAt: .testDataReadme) { original in + let originalContents = try await original.readToEnd( + maximumSizeAllowed: .bytes(1024 * 1024) + ) + let copyContents = try await copy.readToEnd(maximumSizeAllowed: .bytes(1024 * 1024)) + XCTAssertEqual(originalContents, copyContents) + } + } + } + + func testCopyLargeFile() async throws { + let sourcePath = try await self.fs.temporaryFilePath() + let destPath = try await self.fs.temporaryFilePath() + let sourceInfo = try await self.fs.withFileHandle( + forWritingAt: sourcePath, + options: .newFile(replaceExisting: false) + ) { file in + // On Linux we use sendfile to copy which has a limit of 2GB; write at least that much + // to much sure we handle files above that size correctly. + var bytesToWrite = 3 * 1024 * 1024 * 1024 + var offset: Int64 = 0 + // Write a blob a handful of times to avoid consuming too much memory in one go. + let blob = [UInt8](repeating: 0, count: 1024 * 1024 * 32) // 32MB + while bytesToWrite > 0 { + try await file.write(contentsOf: blob, toAbsoluteOffset: offset) + offset += Int64(blob.count) + bytesToWrite -= blob.count + } + + return try await file.info() + } + + try await self.fs.copyItem(at: sourcePath, to: destPath) + let destInfo = try await self.fs.info(forFileAt: destPath) + XCTAssertEqual(destInfo?.size, sourceInfo.size) + } + + func testCopySingleFileCopiesAttributesAndPermissions() async throws { + let original = try await self.fs.temporaryFilePath() + let copy = try await self.fs.temporaryFilePath() + + try await self.fs.withFileHandle( + forWritingAt: original, + options: .newFile(replaceExisting: false, permissions: .ownerReadWrite) + ) { file1 in + do { + try await file1.updateValueForAttribute([0, 1, 2, 3], attribute: "foo") + } catch let error as FileSystemError where error.code == .unsupported { + // Extended attributes are not always supported; swallow the error if we hit it. + } + } + + try await self.fs.copyItem(at: original, to: copy) + + try await self.fs.withFileHandle(forReadingAt: copy) { file2 in + let info = try await file2.info() + XCTAssertEqual(info.permissions, [.ownerReadWrite]) + + do { + let value = try await file2.valueForAttribute("foo") + XCTAssertEqual(value, [0, 1, 2, 3]) + } catch let error as FileSystemError where error.code == .unsupported { + // Extended attributes are not always supported; swallow the error if we hit it. + } + } + } + + func testCopySymlink() async throws { + let copy = try await self.fs.temporaryFilePath() + try await self.fs.copyItem(at: .testDataReadmeSymlink, to: copy) + + let info = try await self.fs.info(forFileAt: copy, infoAboutSymbolicLink: true) + XCTAssertEqual(info?.type, .symlink) + + let destination = try await self.fs.destinationOfSymbolicLink(at: copy) + XCTAssertEqual(destination, "README.md") + } + + func testCopyEmptyDirectory() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.createDirectory(at: path, withIntermediateDirectories: false) + + let copy = try await self.fs.temporaryFilePath() + try await self.fs.copyItem(at: path, to: copy) + + try await self.checkDirectoriesMatch(path, copy) + } + + func testCopyGeneratedTreeStructure() async throws { + let path = try await self.fs.temporaryFilePath() + let items = try await self.generateDirectoryStructure( + root: path, + maxDepth: 4, + maxFilesPerDirectory: 10 + ) + + let copy = try await self.fs.temporaryFilePath() + do { + try await self.fs.copyItem(at: path, to: copy) + } catch { + // Leave breadcrumbs to make debugging easier. + XCTFail("Failed to copy \(items) from '\(path)' to '\(copy)'") + throw error + } + + do { + try await self.checkDirectoriesMatch(path, copy) + } catch { + // Leave breadcrumbs to make debugging easier. + XCTFail("Failed to validate \(items) copied from '\(path)' to '\(copy)'") + throw error + } + } + + func testCopySelectively() async throws { + let path = try await self.fs.temporaryFilePath() + + // Only generate regular files. They'll be in the format 'file-N-regular'. + let _ = try await self.generateDirectoryStructure( + root: path, + maxDepth: 1, + maxFilesPerDirectory: 10, + directoryProbability: 0.0, + symbolicLinkProbability: 0.0 + ) + + let copyPath = try await self.fs.temporaryFilePath() + try await self.fs.copyItem(at: path, to: copyPath) { _, error in + throw error + } shouldCopyFile: { source, destination in + // Copy the directory and 'file-1-regular' + return (source == path) || (source.lastComponent!.string == "file-0-regular") + } + + let paths = try await self.fs.withDirectoryHandle(atPath: copyPath) { dir in + try await dir.listContents().reduce(into: []) { $0.append($1) } + } + + XCTAssertEqual(paths.count, 1) + XCTAssertEqual(paths.first?.name, "file-0-regular") + } + + func testCopyNonExistentFile() async throws { + let source = try await self.fs.temporaryFilePath() + let destination = try await self.fs.temporaryFilePath() + + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.copyItem(at: source, to: destination) + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testCopyToExistingDestination() async throws { + let source = try await self.fs.temporaryFilePath() + let destination = try await self.fs.temporaryFilePath() + + // Touch both files. + for path in [source, destination] { + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { _ in } + } + + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.copyItem(at: source, to: destination) + } onError: { error in + XCTAssertEqual(error.code, .fileAlreadyExists) + } + } + + func testRemoveSingleFile() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { _ in } + + let infoAfterCreation = try await self.fs.info(forFileAt: path) + XCTAssertNotNil(infoAfterCreation) + + let removed = try await self.fs.removeItem(at: path) + XCTAssertEqual(removed, 1) + + let infoAfterRemoval = try await self.fs.info(forFileAt: path) + XCTAssertNil(infoAfterRemoval) + } + + func testRemoveNonExistentFile() async throws { + let path = try await self.fs.temporaryFilePath() + let info = try await self.fs.info(forFileAt: path) + XCTAssertNil(info) + let removed = try await self.fs.removeItem(at: path) + XCTAssertEqual(removed, 0) + } + + func testRemoveDirectory() async throws { + let path = try await self.fs.temporaryFilePath() + let created = try await self.generateDirectoryStructure( + root: path, + maxDepth: 3, + maxFilesPerDirectory: 10 + ) + + let infoAfterCreation = try await self.fs.info(forFileAt: path) + XCTAssertNotNil(infoAfterCreation) + + // Removing a non-empty directory recursively should throw 'notEmpty' + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.removeItem(at: path, recursively: false) + } onError: { error in + XCTAssertEqual(error.code, .notEmpty) + } + + let removed = try await self.fs.removeItem(at: path) + XCTAssertEqual(created, removed) + + let infoAfterRemoval = try await self.fs.info(forFileAt: path) + XCTAssertNil(infoAfterRemoval) + } + + func testMoveRegularFile() async throws { + let source = try await self.fs.temporaryFilePath() + try await self.fs.withFileHandle( + forWritingAt: source, + options: .newFile(replaceExisting: false) + ) { _ in } + let destination = try await self.fs.temporaryFilePath() + + do { + let sourceInfo = try await self.fs.info(forFileAt: source) + XCTAssertNotNil(sourceInfo) + let destinationInfo = try await self.fs.info(forFileAt: destination) + XCTAssertNil(destinationInfo) + } + + try await self.fs.moveItem(at: source, to: destination) + + do { + let sourceInfo = try await self.fs.info(forFileAt: source) + XCTAssertNil(sourceInfo) + let destinationInfo = try await self.fs.info(forFileAt: destination) + XCTAssertNotNil(destinationInfo) + } + } + + func testMoveSymbolicLink() async throws { + let source = try await self.fs.temporaryFilePath() + let destination = try await self.fs.temporaryFilePath() + + try await self.fs.createSymbolicLink(at: source, withDestination: .testDataReadme) + + do { + let sourceInfo = try await self.fs.info(forFileAt: source, infoAboutSymbolicLink: true) + XCTAssertNotNil(sourceInfo) + XCTAssertEqual(sourceInfo?.type, .symlink) + let destinationInfo = try await self.fs.info(forFileAt: destination) + XCTAssertNil(destinationInfo) + } + + try await self.fs.moveItem(at: source, to: destination) + + do { + let sourceInfo = try await self.fs.info(forFileAt: source) + XCTAssertNil(sourceInfo) + let destinationInfo = try await self.fs.info( + forFileAt: destination, + infoAboutSymbolicLink: true + ) + XCTAssertNotNil(destinationInfo) + XCTAssertEqual(destinationInfo?.type, .symlink) + } + + let linkDestination = try await self.fs.destinationOfSymbolicLink(at: destination) + XCTAssertEqual(linkDestination, .testDataReadme) + } + + func testMoveDirectory() async throws { + let source = try await self.fs.temporaryFilePath() + let destination = try await self.fs.temporaryFilePath() + + try await self.fs.createDirectory(at: source, withIntermediateDirectories: true) + try await self.fs.withFileHandle( + forWritingAt: source.appending("foo"), + options: .newFile(replaceExisting: false) + ) { _ in } + + do { + let sourceInfo = try await self.fs.info(forFileAt: source, infoAboutSymbolicLink: false) + XCTAssertNotNil(sourceInfo) + XCTAssertEqual(sourceInfo?.type, .directory) + let destinationInfo = try await self.fs.info(forFileAt: destination) + XCTAssertNil(destinationInfo) + } + + try await self.fs.moveItem(at: source, to: destination) + + let sourceInfo = try await self.fs.info(forFileAt: source) + XCTAssertNil(sourceInfo) + + let items = try await self.fs.withDirectoryHandle(atPath: destination) { directory in + try await directory.listContents().reduce(into: []) { $0.append($1) } + } + + XCTAssertEqual(items.count, 1) + XCTAssertEqual(items.first?.name, "foo") + } + + func testMoveWhenSourceDoesNotExist() async throws { + let source = try await self.fs.temporaryFilePath() + let destination = try await self.fs.temporaryFilePath() + + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.moveItem(at: source, to: destination) + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testMoveWhenDestinationAlreadyExists() async throws { + let source = try await self.fs.temporaryFilePath() + let destination = try await self.fs.temporaryFilePath() + for path in [source, destination] { + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { _ in } + } + + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.moveItem(at: source, to: destination) + } onError: { error in + XCTAssertEqual(error.code, .fileAlreadyExists) + } + } + + func testReplaceFile(_ existingType: FileType?, with replacementType: FileType) async throws { + func makeRegularFile(at path: FilePath) async throws { + try await self.fs.withFileHandle( + forWritingAt: path, + options: .newFile(replaceExisting: false) + ) { _ in } + } + + func makeSymbolicLink(at path: FilePath) async throws { + try await self.fs.createSymbolicLink(at: path, withDestination: "/whatever") + } + + func makeDirectory(at path: FilePath) async throws { + try await self.fs.createDirectory(at: path, withIntermediateDirectories: true) + } + + func makeFile(ofType type: FileType, at path: FilePath) async throws { + switch type { + case .regular: + try await makeRegularFile(at: path) + case .symlink: + try await makeSymbolicLink(at: path) + case .directory: + try await makeDirectory(at: path) + default: + XCTFail("Unexpected file type '\(type)'") + } + } + + let existingPath = try await self.fs.temporaryFilePath() + let replacementPath = try await self.fs.temporaryFilePath() + if let existingType = existingType { + try await makeFile(ofType: existingType, at: existingPath) + } + try await makeFile(ofType: replacementType, at: replacementPath) + + try await self.fs.replaceItem(at: existingPath, withItemAt: replacementPath) + + let sourceInfo = try await self.fs.info( + forFileAt: existingPath, + infoAboutSymbolicLink: true + ) + XCTAssertNotNil(sourceInfo) + XCTAssertEqual(sourceInfo?.type, replacementType) + + let destinationInfo = try await self.fs.info( + forFileAt: replacementPath, + infoAboutSymbolicLink: true + ) + XCTAssertNil(destinationInfo) + } + + func testReplaceRegularFileWithRegularFile() async throws { + try await self.testReplaceFile(.regular, with: .regular) + } + + func testReplaceRegularFileWithSymbolicLink() async throws { + try await self.testReplaceFile(.regular, with: .symlink) + } + + func testReplaceRegularFileWithDirectory() async throws { + try await self.testReplaceFile(.regular, with: .directory) + } + + func testReplaceSymbolicLinkWithRegularFile() async throws { + try await self.testReplaceFile(.symlink, with: .regular) + } + + func testReplaceSymbolicLinkWithSymbolicLink() async throws { + try await self.testReplaceFile(.symlink, with: .symlink) + } + + func testReplaceSymbolicLinkWithDirectory() async throws { + try await self.testReplaceFile(.symlink, with: .directory) + } + + func testReplaceDirectoryWithRegularFile() async throws { + try await self.testReplaceFile(.directory, with: .regular) + } + + func testReplaceDirectoryWithSymbolicLink() async throws { + try await self.testReplaceFile(.directory, with: .symlink) + } + + func testReplaceDirectoryWithDirectory() async throws { + try await self.testReplaceFile(.directory, with: .directory) + } + + func testReplaceNothingWithRegularFile() async throws { + try await self.testReplaceFile(.none, with: .regular) + } + + func testReplaceNothingWithSymbolicLink() async throws { + try await self.testReplaceFile(.none, with: .symlink) + } + + func testReplaceNothingWithDirectory() async throws { + try await self.testReplaceFile(.none, with: .directory) + } + + func testReplaceWhenExistingFileDoesNotExist() async throws { + let existing = try await self.fs.temporaryFilePath() + let replacement = try await self.fs.temporaryFilePath() + + await XCTAssertThrowsFileSystemErrorAsync { + try await self.fs.replaceItem(at: replacement, withItemAt: existing) + } onError: { error in + XCTAssertEqual(error.code, .notFound) + } + } + + func testWithFileSystem() async throws { + try await withFileSystem(numberOfThreads: 1) { fs in + let info = try await fs.info(forFileAt: .testDataReadme) + XCTAssertEqual(info?.type, .regular) + } + } + + func testListContentsOfLargeDirectory() async throws { + let path = try await self.fs.temporaryFilePath() + try await self.fs.createDirectory(at: path, withIntermediateDirectories: true) + + try await self.fs.withDirectoryHandle(atPath: path) { handle in + for i in 0..<1024 { + try await self.fs.withFileHandle( + forWritingAt: path.appending("\(i)"), + options: .newFile(replaceExisting: false) + ) { _ in } + } + + let names = try await handle.listContents().reduce(into: []) { + $0.append($1.name.string) + } + + let expected = (0..<1024).map { + String($0) + } + + XCTAssertEqual(names.sorted(), expected.sorted()) + } + } +} + +extension FileSystemTests { + private func checkDirectoriesMatch(_ root1: FilePath, _ root2: FilePath) async throws { + func namesAndTypes(_ root: FilePath) async throws -> [(FilePath.Component, FileType)] { + try await self.fs.withDirectoryHandle(atPath: root) { dir in + try await dir.listContents() + .reduce(into: []) { $0.append($1) } + .map { ($0.name, $0.type) } + .sorted(by: { lhs, rhs in lhs.0.string < rhs.0.string }) + } + } + + // Check if all named entries and types match. + let root1Entries = try await namesAndTypes(root1) + let root2Entries = try await namesAndTypes(root2) + XCTAssertEqual(root1Entries.map { $0.0 }, root2Entries.map { $0.0 }) + XCTAssertEqual(root1Entries.map { $0.1 }, root2Entries.map { $0.1 }) + + // Now look at regular files: are they all the same? + for (path, type) in root1Entries where type == .regular { + try await self.checkRegularFilesMatch(root1.appending(path), root2.appending(path)) + } + + // Are symbolic links all the same? + for (path, type) in root1Entries where type == .symlink { + try await self.checkSymbolicLinksMatch(root1.appending(path), root2.appending(path)) + } + + // Finally, check directories. + for (path, type) in root1Entries where type == .directory { + try await self.checkDirectoriesMatch(root1.appending(path), root2.appending(path)) + } + } + + private func checkRegularFilesMatch(_ path1: FilePath, _ path2: FilePath) async throws { + try await self.fs.withFileHandle(forReadingAt: path1) { file1 in + try await self.fs.withFileHandle(forReadingAt: path2) { file2 in + let info1 = try await file1.info() + let info2 = try await file2.info() + XCTAssertEqual(info1.type, info2.type) + XCTAssertEqual(info1.size, info2.size) + XCTAssertEqual(info1.permissions, info2.permissions) + + let file1Contents = try await file1.readToEnd( + maximumSizeAllowed: .bytes(1024 * 1024) + ) + let file2Contents = try await file2.readToEnd( + maximumSizeAllowed: .bytes(1024 * 1024) + ) + XCTAssertEqual(file1Contents, file2Contents) + + do { + let file1Attributes = try await file1.attributeNames().sorted() + let file2Attributes = try await file2.attributeNames().sorted() + XCTAssertEqual(file1Attributes, file2Attributes) + + for attribute in file1Attributes { + let value1 = try await file1.valueForAttribute(attribute) + let value2 = try await file2.valueForAttribute(attribute) + XCTAssertEqual(value1, value2) + } + } catch let error as FileSystemError where error.code == .unsupported { + // Extended attributes aren't supported on all platforms, so swallow any + // unavailable errors. + } + } + } + } + + private func checkSymbolicLinksMatch(_ path1: FilePath, _ path2: FilePath) async throws { + let destination1 = try await self.fs.destinationOfSymbolicLink(at: path1) + let destination2 = try await self.fs.destinationOfSymbolicLink(at: path2) + XCTAssertEqual(destination1, destination2) + } + + private func generateDirectoryStructure( + root: FilePath, + maxDepth: Int, + maxFilesPerDirectory: Int, + directoryProbability: Double = 0.3, + symbolicLinkProbability: Double = 0.2 + ) async throws -> Int { + guard maxDepth > 0 else { return 0 } + + func makeDirectory() -> Bool { + Double.random(in: 0..<1.0) <= directoryProbability + } + + func makeSymbolicLink() -> Bool { + Double.random(in: 0..<1.0) <= symbolicLinkProbability + } + + try await self.fs.createDirectory( + at: root, + withIntermediateDirectories: false, + permissions: nil + ) + + let itemsInThisDir = Int.random(in: 1...maxFilesPerDirectory) + var itemsCreated = 1 + + guard itemsInThisDir > 0 else { + return itemsCreated + } + + let dirsToMake = try await self.fs.withDirectoryHandle(atPath: root) { dir in + var directoriesToMake = [FilePath]() + + for i in 0.., Int)] = [ + (0..<0, 0), + (0..<1, 1), + (0.., Int)] = [ + (0...0, 1), + (0...1, 2), + // Clamped to file size. + (0...endIndex, Int(info.size)), + (0...(endIndex - 1), Int(info.size)), + // Short one byte. + (1...(endIndex - 1), Int(info.size - 1)), + (1...endIndex, Int(info.size - 1)), + (0...(Int64.max - 1), Int(size)), + ] + + for (range, expected) in ranges { + let byteCount = try await handle.readChunks(in: range).reduce(into: 0) { + $0 += $1.readableBytes + } + XCTAssertEqual(byteCount, expected) + } + } + } + + func testReadChunksPartialRangeUpTo() async throws { + try await self.fs.withFileHandle(forReadingAt: FilePath(#filePath)) { handle in + let info = try await handle.info() + let size = info.size + let endIndex = size + 1 + + let ranges: [(PartialRangeUpTo, Int)] = [ + (..<0, 0), + (..<1, 1), + // Clamped to file size. + (.., Int)] = [ + (...0, 1), + (...1, 2), + // Clamped to size. + (...endIndex, Int(info.size)), + (...(endIndex - 1), Int(info.size)), + // Exact size. + (...(endIndex - 2), Int(info.size)), + // One byte short + (...(endIndex - 3), Int(info.size - 1)), + (...(Int64.max - 1), Int(size)), + ] + + for (range, expected) in ranges { + let byteCount = try await handle.readChunks(in: range).reduce(into: 0) { + $0 += $1.readableBytes + } + + XCTAssertEqual(byteCount, expected) + } + } + } + + func testReadChunksPartialRangeFrom() async throws { + try await self.fs.withFileHandle(forReadingAt: FilePath(#filePath)) { handle in + let info = try await handle.info() + let size = info.size + let endIndex = size + 1 + + let ranges: [(PartialRangeFrom, Int)] = [ + (0..., Int(size)), + (1..., Int(size - 1)), + (endIndex..., 0), + ((endIndex - 1)..., 0), + ((endIndex - 2)..., 1), + ] + + for (range, expected) in ranges { + let byteCount = try await handle.readChunks(in: range).reduce(into: 0) { + $0 += $1.readableBytes + } + + XCTAssertEqual(byteCount, expected, "\(range)") + } + } + } + + func testReadChunksUnboundedRange() async throws { + try await self.fs.withFileHandle(forReadingAt: FilePath(#filePath)) { handle in + let info = try await handle.info() + let size = info.size + + let byteCount = try await handle.readChunks(in: ...).reduce(into: 0) { + $0 += $1.readableBytes + } + + XCTAssertEqual(byteCount, Int(size)) + } + } +} diff --git a/Tests/NIOFileSystemIntegrationTests/Test Data/Foo.symlink b/Tests/NIOFileSystemIntegrationTests/Test Data/Foo.symlink new file mode 120000 index 0000000000..9f26b637f0 --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/Test Data/Foo.symlink @@ -0,0 +1 @@ +Foo \ No newline at end of file diff --git a/Tests/NIOFileSystemIntegrationTests/Test Data/Foo/README.txt b/Tests/NIOFileSystemIntegrationTests/Test Data/Foo/README.txt new file mode 100644 index 0000000000..1b1880b8f0 --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/Test Data/Foo/README.txt @@ -0,0 +1 @@ +This file exists so that the containing directory is not empty. diff --git a/Tests/NIOFileSystemIntegrationTests/Test Data/README.md b/Tests/NIOFileSystemIntegrationTests/Test Data/README.md new file mode 100644 index 0000000000..181f96f34b --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/Test Data/README.md @@ -0,0 +1,5 @@ +# Test Data + +This directory contains known files and directories for integration testing. This avoids having to +rely on APIs for creating files and directories in order to test APIs which read files and +directories. diff --git a/Tests/NIOFileSystemIntegrationTests/Test Data/README.md.symlink b/Tests/NIOFileSystemIntegrationTests/Test Data/README.md.symlink new file mode 120000 index 0000000000..42061c01a1 --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/Test Data/README.md.symlink @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift b/Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift new file mode 100644 index 0000000000..27e754604f --- /dev/null +++ b/Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift @@ -0,0 +1,68 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOFileSystem +import XCTest + +func XCTAssertThrowsErrorAsync( + file: StaticString = #file, + line: UInt = #line, + expression: () async throws -> R, + onError: (Error) -> Void = { _ in } +) async { + do { + _ = try await expression() + XCTFail("expression did not throw", file: file, line: line) + } catch { + onError(error) + } +} + +func XCTAssertThrowsFileSystemError( + _ expression: @autoclosure () throws -> R, + file: StaticString = #file, + line: UInt = #line, + _ onError: (FileSystemError) -> Void = { _ in } +) { + XCTAssertThrowsError(try expression(), file: file, line: line) { error in + if let fsError = error as? FileSystemError { + onError(fsError) + } else { + XCTFail( + "Expected 'FileSystemError' but found '\(type(of: error))'", + file: file, + line: line + ) + } + } +} + +func XCTAssertThrowsFileSystemErrorAsync( + file: StaticString = #file, + line: UInt = #line, + _ expression: () async throws -> R, + onError: (FileSystemError) -> Void = { _ in } +) async { + await XCTAssertThrowsErrorAsync(file: file, line: line, expression: expression) { error in + if let fsError = error as? FileSystemError { + onError(fsError) + } else { + XCTFail( + "Expected 'FileSystemError' but found '\(type(of: error))'", + file: file, + line: line + ) + } + } +} diff --git a/Tests/NIOFileSystemTests/ByteCountTests.swift b/Tests/NIOFileSystemTests/ByteCountTests.swift new file mode 100644 index 0000000000..bb4146483f --- /dev/null +++ b/Tests/NIOFileSystemTests/ByteCountTests.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOFileSystem +import XCTest + +class ByteCountTests: XCTestCase { + func testByteCountBytes() { + let byteCount = ByteCount.bytes(10) + XCTAssertEqual(byteCount.bytes, 10) + } + + func testByteCountKilobytes() { + let byteCount = ByteCount.kilobytes(10) + XCTAssertEqual(byteCount.bytes, 10_000) + } + + func testByteCountMegabytes() { + let byteCount = ByteCount.megabytes(10) + XCTAssertEqual(byteCount.bytes, 10_000_000) + } + + func testByteCountGigabytes() { + let byteCount = ByteCount.gigabytes(10) + XCTAssertEqual(byteCount.bytes, 10_000_000_000) + } + + func testByteCountKibibytes() { + let byteCount = ByteCount.kibibytes(10) + XCTAssertEqual(byteCount.bytes, 10_240) + } + + func testByteCountMebibytes() { + let byteCount = ByteCount.mebibytes(10) + XCTAssertEqual(byteCount.bytes, 10_485_760) + } + + func testByteCountGibibytes() { + let byteCount = ByteCount.gibibytes(10) + XCTAssertEqual(byteCount.bytes, 10_737_418_240) + } + + func testByteCountEquality() { + let byteCount1 = ByteCount.bytes(10) + let byteCount2 = ByteCount.bytes(20) + XCTAssertEqual(byteCount1, byteCount1) + XCTAssertNotEqual(byteCount1, byteCount2) + } +} diff --git a/Tests/NIOFileSystemTests/DirectoryEntriesTests.swift b/Tests/NIOFileSystemTests/DirectoryEntriesTests.swift new file mode 100644 index 0000000000..8f15321620 --- /dev/null +++ b/Tests/NIOFileSystemTests/DirectoryEntriesTests.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOFileSystem +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class DirectoryEntriesTests: XCTestCase { + func testDirectoryEntriesWrapsAsyncStream() async throws { + let stream = AsyncThrowingStream<[DirectoryEntry], Error> { + $0.yield([DirectoryEntry(path: "foo", type: .regular)!]) + $0.yield( + [ + DirectoryEntry(path: "bar", type: .block)!, + DirectoryEntry(path: "baz", type: .character)!, + ] + ) + $0.finish() + } + + let directoryEntries = DirectoryEntries(wrapping: stream) + var iterator = directoryEntries.makeAsyncIterator() + let entry0 = try await iterator.next() + XCTAssertEqual(entry0, DirectoryEntry(path: "foo", type: .regular)) + let entry1 = try await iterator.next() + XCTAssertEqual(entry1, DirectoryEntry(path: "bar", type: .block)) + let entry2 = try await iterator.next() + XCTAssertEqual(entry2, DirectoryEntry(path: "baz", type: .character)) + let end = try await iterator.next() + XCTAssertNil(end) + } + + func testDirectoryEntriesBatchesWrapsAsyncStream() async throws { + let stream = AsyncThrowingStream<[DirectoryEntry], Error> { + $0.yield([DirectoryEntry(path: "foo", type: .regular)!]) + $0.yield( + [ + DirectoryEntry(path: "bar", type: .block)!, + DirectoryEntry(path: "baz", type: .character)!, + ] + ) + $0.finish() + } + + let directoryEntries = DirectoryEntries(wrapping: stream) + var iterator = directoryEntries.batched().makeAsyncIterator() + let entry0 = try await iterator.next() + XCTAssertEqual(entry0, [DirectoryEntry(path: "foo", type: .regular)!]) + + let entry1 = try await iterator.next() + XCTAssertEqual( + entry1, + [ + DirectoryEntry(path: "bar", type: .block)!, + DirectoryEntry(path: "baz", type: .character)!, + ] + ) + let end = try await iterator.next() + XCTAssertNil(end) + } +} diff --git a/Tests/NIOFileSystemTests/FileChunksTests.swift b/Tests/NIOFileSystemTests/FileChunksTests.swift new file mode 100644 index 0000000000..ba6f4b9315 --- /dev/null +++ b/Tests/NIOFileSystemTests/FileChunksTests.swift @@ -0,0 +1,40 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +import NIOFileSystem +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class FileChunksTests: XCTestCase { + func testFileChunksWrapsAsyncStream() async throws { + let stream = AsyncThrowingStream { + $0.yield(ByteBuffer(bytes: [0, 1, 2])) + $0.yield(ByteBuffer(bytes: [3, 4, 5])) + $0.yield(ByteBuffer(bytes: [6, 7, 8])) + $0.finish() + } + + let fileChunks = FileChunks(wrapping: stream) + var iterator = fileChunks.makeAsyncIterator() + let chunk0 = try await iterator.next() + XCTAssertEqual(chunk0, ByteBuffer(bytes: [0, 1, 2])) + let chunk1 = try await iterator.next() + XCTAssertEqual(chunk1, ByteBuffer(bytes: [3, 4, 5])) + let chunk2 = try await iterator.next() + XCTAssertEqual(chunk2, ByteBuffer(bytes: [6, 7, 8])) + let end = try await iterator.next() + XCTAssertNil(end) + } +} diff --git a/Tests/NIOFileSystemTests/FileHandleTests.swift b/Tests/NIOFileSystemTests/FileHandleTests.swift new file mode 100644 index 0000000000..f9fb80e1dc --- /dev/null +++ b/Tests/NIOFileSystemTests/FileHandleTests.swift @@ -0,0 +1,269 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) import NIOFileSystem +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +internal final class FileHandleTests: XCTestCase { + private func withHandleForMocking( + path: FilePath = "/probably/does/not/exist", + _ execute: (SystemFileHandle, MockingDriver) throws -> Void + ) async throws { + #if ENABLE_MOCKING + // The executor is required to create the handle, we won't do any async work if we're + // mocking as the driver requires synchronous code (as it uses thread local storage). + let executor = await IOExecutor.running(numberOfThreads: 1) + await executor.drain() + + // Not a real descriptor. + let descriptor = FileDescriptor(rawValue: -1) + let handle = SystemFileHandle(takingOwnershipOf: descriptor, path: path, executor: executor) + defer { + // Not a 'real' descriptor so just detach to avoid "leaking" the descriptor and + // trapping in the deinit of handle. + XCTAssertNoThrow(try handle.detachUnsafeFileDescriptor()) + } + + try MockingDriver.withMockingEnabled { driver in + try execute(handle, driver) + } + #else + throw XCTSkip("Mocking is not enabled (via 'ENABLE_MOCKING')") + #endif + } + + /// A test case which mocks system calls made by the a file handle. + private struct MockHandleTest { + /// The function to run which makes a system call. + var fn: (SystemFileHandle) throws -> Void + + /// The name of the system call being made. This is used to verify information provided + /// in expected errors. + var systemCall: String + + /// Errnos which when emitted by the system are not expected to result in errors being + /// thrown. + var nonThrowingErrnos: [Errno] + + /// Errnos which map to known file system error codes. + var knownErrnos: [Errno: FileSystemError.Code] + + /// Errnos with no known mapping to file system error codes. + var unknownErrnos: [Errno] + + init( + expectedSystemCall: String, + nonThrowingErrnos: [Errno] = [], + knownErrnos: [Errno: FileSystemError.Code] = [:], + unknownErrnos: [Errno] = [], + expression: @escaping (SystemFileHandle) throws -> R + ) { + self.systemCall = expectedSystemCall + self.nonThrowingErrnos = nonThrowingErrnos + self.knownErrnos = knownErrnos + self.unknownErrnos = unknownErrnos + self.fn = { _ = try expression($0) } + + // Verify that EBADF results in the '.closed' status. + let existing = self.knownErrnos.updateValue(.closed, forKey: .badFileDescriptor) + assert(existing == nil) + } + } + + private func run(_ test: MockHandleTest) async throws { + try await self.withHandleForMocking { handle, driver in + // No errno should not throw an error. + try driver.testNoErrnoDoesNotThrow(try test.fn(handle)) + + // Some errnos are safe to ignore and should not throw. + for errno in test.nonThrowingErrnos { + try driver.testNoErrnoDoesNotThrow(errno) + } + + // Each of these should result in an known status code. + for (errno, code) in test.knownErrnos { + try driver.testErrnoThrowsError(errno, try test.fn(handle)) { error in + XCTAssertEqual( + error.code, + code, + """ + \(test.systemCall) throwing \(errno) resulted in '\(error.code)',\ + expected '\(code)'. + """ + ) + XCTAssertSystemCallError(error.cause, name: test.systemCall, errno: errno) + } + } + + // Each of these should result in an unknown status code. + for errno in test.unknownErrnos { + try driver.testErrnoThrowsError(errno, try test.fn(handle)) { error in + XCTAssertEqual(error.code, .unknown) + XCTAssertEqual( + error.code, + .unknown, + """ + \(test.systemCall) throwing \(errno) resulted in '\(error.code)',\ + expected 'unknown'. + """ + ) + XCTAssertSystemCallError(error.cause, name: test.systemCall, errno: errno) + } + } + } + } + + func testInfo() async throws { + let testCase = MockHandleTest( + expectedSystemCall: "fstat", + unknownErrnos: [.deadlock, .ioError] + ) { handle in + try handle.sendableView._info().get() + } + + try await self.run(testCase) + } + + func testReplacePermissions() async throws { + let testCase = MockHandleTest( + expectedSystemCall: "fchmod", + knownErrnos: [ + .invalidArgument: .invalidArgument, + .notPermitted: .permissionDenied, + ], + unknownErrnos: [.deadlock, .ioError] + ) { handle in + try handle.sendableView._replacePermissions(.groupRead) + } + + try await self.run(testCase) + } + + func testListAttributeNames() async throws { + let testCase = MockHandleTest( + expectedSystemCall: "flistxattr", + knownErrnos: [ + .notSupported: .unsupported, + .notPermitted: .unsupported, + .permissionDenied: .permissionDenied, + ], + unknownErrnos: [.deadlock, .ioError] + ) { handle in + try handle.sendableView._attributeNames() + } + + try await self.run(testCase) + } + + func testValueForAttribute() async throws { + var nonThrowingErrnos: [Errno] = [.noData] + var knownErrnos: [Errno: FileSystemError.Code] = [.notSupported: .unsupported] + #if canImport(Darwin) + nonThrowingErrnos.append(.attributeNotFound) + knownErrnos[.fileNameTooLong] = .invalidArgument + #endif + + let testCase = MockHandleTest( + expectedSystemCall: "fgetxattr", + nonThrowingErrnos: nonThrowingErrnos, + knownErrnos: knownErrnos, + unknownErrnos: [.deadlock, .ioError] + ) { handle in + try handle.sendableView._valueForAttribute("foobar") + } + + try await self.run(testCase) + } + + func testSynchronize() async throws { + let testCase = MockHandleTest( + expectedSystemCall: "fsync", + knownErrnos: [.ioError: .io], + unknownErrnos: [.deadlock, .addressInUse] + ) { handle in + try handle.sendableView._synchronize() + } + + try await self.run(testCase) + } + + func testUpdateValueForAttribute() async throws { + var knownErrnos: [Errno: FileSystemError.Code] = [ + .notSupported: .unsupported, + .invalidArgument: .invalidArgument, + ] + #if canImport(Darwin) + knownErrnos[.fileNameTooLong] = .invalidArgument + #endif + + let testCase = MockHandleTest( + expectedSystemCall: "fsetxattr", + knownErrnos: knownErrnos, + unknownErrnos: [.deadlock, .ioError] + ) { handle in + try handle.sendableView._updateValueForAttribute([0, 1, 2, 4], attribute: "foobar") + } + + try await self.run(testCase) + } + + func testRemoveValueForAttribute() async throws { + var knownErrnos: [Errno: FileSystemError.Code] = [.notSupported: .unsupported] + #if canImport(Darwin) + knownErrnos[.fileNameTooLong] = .invalidArgument + #endif + + let testCase = MockHandleTest( + expectedSystemCall: "fremovexattr", + knownErrnos: knownErrnos, + unknownErrnos: [.deadlock, .ioError] + ) { handle in + try handle.sendableView._removeValueForAttribute("foobar") + } + + try await self.run(testCase) + } +} + +extension MockingDriver { + fileprivate func testNoErrnoDoesNotThrow( + _ expression: @autoclosure () throws -> R, + line: UInt = #line + ) throws { + self.forceErrno = .none + XCTAssertNoThrow(try expression(), line: line) + } + + fileprivate func testErrnoDoesNotThrowError( + _ errno: Errno, + _ expression: @autoclosure () throws -> R, + line: UInt = #line + ) throws { + self.forceErrno = .always(errno: errno.rawValue) + XCTAssertNoThrow(try expression(), line: line) + } + + fileprivate func testErrnoThrowsError( + _ errno: Errno, + _ expression: @autoclosure () throws -> R, + line: UInt = #line, + onError: (FileSystemError) -> Void + ) throws { + self.forceErrno = .always(errno: errno.rawValue) + XCTAssertThrowsFileSystemError(try expression(), line: line) { error in + onError(error) + } + } +} diff --git a/Tests/NIOFileSystemTests/FileInfoTests.swift b/Tests/NIOFileSystemTests/FileInfoTests.swift new file mode 100644 index 0000000000..b26dcaf546 --- /dev/null +++ b/Tests/NIOFileSystemTests/FileInfoTests.swift @@ -0,0 +1,158 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOFileSystem +import XCTest + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +final class FileInfoTests: XCTestCase { + private var status: CInterop.Stat { + var status = CInterop.Stat() + status.st_dev = 1 + status.st_mode = S_IFREG | 0o777 + status.st_nlink = 3 + status.st_ino = 4 + status.st_uid = 5 + status.st_gid = 6 + status.st_rdev = 7 + status.st_size = 8 + status.st_blocks = 9 + status.st_blksize = 10 + + #if canImport(Darwin) + status.st_atimespec = timespec(tv_sec: 0, tv_nsec: 0) + status.st_mtimespec = timespec(tv_sec: 1, tv_nsec: 0) + status.st_ctimespec = timespec(tv_sec: 2, tv_nsec: 0) + status.st_birthtimespec = timespec(tv_sec: 3, tv_nsec: 0) + status.st_flags = 11 + status.st_gen = 12 + #elseif canImport(Glibc) + status.st_atim = timespec(tv_sec: 0, tv_nsec: 0) + status.st_mtim = timespec(tv_sec: 1, tv_nsec: 0) + status.st_ctim = timespec(tv_sec: 2, tv_nsec: 0) + #endif + + return status + } + + func testConversionFromStat() { + let info = FileInfo(platformSpecificStatus: self.status) + XCTAssertEqual(info.type, .regular) + XCTAssertEqual( + info.permissions, + [.groupReadWriteExecute, .ownerReadWriteExecute, .otherReadWriteExecute] + ) + XCTAssertEqual(info.userID, FileInfo.UserID(rawValue: 5)) + XCTAssertEqual(info.groupID, FileInfo.GroupID(rawValue: 6)) + XCTAssertEqual(info.size, 8) + + XCTAssertEqual(info.lastAccessTime, FileInfo.Timespec(seconds: 0, nanoseconds: 0)) + XCTAssertEqual(info.lastDataModificationTime, FileInfo.Timespec(seconds: 1, nanoseconds: 0)) + XCTAssertEqual(info.lastStatusChangeTime, FileInfo.Timespec(seconds: 2, nanoseconds: 0)) + } + + func testEquatableConformance() { + let info = FileInfo(platformSpecificStatus: self.status) + let other = FileInfo(platformSpecificStatus: self.status) + XCTAssertEqual(info, other) + + func assertNotEqualAfterMutation(line: UInt = #line, _ mutate: (inout FileInfo) -> Void) { + var modified = info + mutate(&modified) + XCTAssertNotEqual(info, modified, line: line) + } + + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_dev += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_mode += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_nlink += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_ino += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_uid += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_gid += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_rdev += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_size += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_blocks += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_blksize += 1 } + + #if canImport(Darwin) + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_atimespec.tv_sec += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_mtimespec.tv_sec += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_ctimespec.tv_sec += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_birthtimespec.tv_sec += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_flags += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_gen += 1 } + #elseif canImport(Glibc) + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_atim.tv_sec += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_mtim.tv_sec += 1 } + assertNotEqualAfterMutation { $0.platformSpecificStatus!.st_ctim.tv_sec += 1 } + #endif + } + + func testHashableConformance() { + let info = FileInfo(platformSpecificStatus: self.status) + let other = FileInfo(platformSpecificStatus: self.status) + XCTAssertEqual(Set([info, other]), [info]) + + func assertDifferentHashValueAfterMutation( + line: UInt = #line, + _ mutate: (inout FileInfo) -> Void + ) { + // Hash values _should_ be unique but aren't guaranteed to be. Generate 100 different + // versions of the value and expect that at least 80 different hash values. + var modified = info + var hashValues: Set = [info.hashValue] + for _ in 1..<100 { + mutate(&modified) + hashValues.insert(modified.hashValue) + } + XCTAssertGreaterThanOrEqual(hashValues.count, 80, line: line) + } + + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_dev += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_mode += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_nlink += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_ino += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_uid += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_gid += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_rdev += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_size += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_blocks += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_blksize += 1 } + + #if canImport(Darwin) + assertDifferentHashValueAfterMutation { + $0.platformSpecificStatus!.st_atimespec.tv_sec += 1 + } + assertDifferentHashValueAfterMutation { + $0.platformSpecificStatus!.st_mtimespec.tv_sec += 1 + } + assertDifferentHashValueAfterMutation { + $0.platformSpecificStatus!.st_ctimespec.tv_sec += 1 + } + assertDifferentHashValueAfterMutation { + $0.platformSpecificStatus!.st_birthtimespec.tv_sec += 1 + } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_flags += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_gen += 1 } + #elseif canImport(Glibc) + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_atim.tv_sec += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_mtim.tv_sec += 1 } + assertDifferentHashValueAfterMutation { $0.platformSpecificStatus!.st_ctim.tv_sec += 1 } + #endif + } +} diff --git a/Tests/NIOFileSystemTests/FileOpenOptionsTests.swift b/Tests/NIOFileSystemTests/FileOpenOptionsTests.swift new file mode 100644 index 0000000000..9f82e85c3a --- /dev/null +++ b/Tests/NIOFileSystemTests/FileOpenOptionsTests.swift @@ -0,0 +1,100 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) import NIOFileSystem +import XCTest + +final class FileOpenOptionsTests: XCTestCase { + private let expectedDefaults: FilePermissions = [ + .ownerReadWrite, + .groupRead, + .otherRead, + ] + + func testReadOptions() { + var options = OpenOptions.Read() + XCTAssertEqual(FileDescriptor.OpenOptions(options), []) + + options.followSymbolicLinks = false + options.closeOnExec = true + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.noFollow, .closeOnExec]) + } + + func testDirectoryOptions() { + var options = OpenOptions.Directory() + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.directory]) + + options.followSymbolicLinks = false + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.noFollow, .directory]) + + options.followSymbolicLinks = true + options.closeOnExec = true + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.directory, .closeOnExec]) + } + + func testWriteOpenOrCreate() { + var options = OpenOptions.Write.modifyFile(createIfNecessary: true) + XCTAssertEqual(options.existingFile, .open) + XCTAssertEqual(options.permissionsForRegularFile, self.expectedDefaults) + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create]) + + options = OpenOptions.Write.modifyFile(createIfNecessary: true, permissions: .groupExecute) + XCTAssertEqual(options.existingFile, .open) + XCTAssertEqual(options.permissionsForRegularFile, .groupExecute) + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create]) + + options.followSymbolicLinks = false + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .noFollow]) + } + + func testWriteTruncateOrCreate() { + var options = OpenOptions.Write.newFile(replaceExisting: true) + XCTAssertEqual(options.existingFile, .truncate) + XCTAssertEqual(options.permissionsForRegularFile, self.expectedDefaults) + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .truncate]) + + options = OpenOptions.Write.newFile(replaceExisting: true, permissions: .groupExecute) + XCTAssertEqual(options.existingFile, .truncate) + XCTAssertEqual(options.permissionsForRegularFile, .groupExecute) + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .truncate]) + + options.followSymbolicLinks = false + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .truncate, .noFollow]) + } + + func testWriteExclusiveOpen() { + var options = OpenOptions.Write(existingFile: .open, newFile: nil) + XCTAssertEqual(options.existingFile, .open) + XCTAssertNil(options.newFile) + XCTAssertEqual(FileDescriptor.OpenOptions(options), []) + + options.followSymbolicLinks = false + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.noFollow]) + } + + func testWriteExclusiveCreate() { + var options = OpenOptions.Write.newFile(replaceExisting: false) + XCTAssertEqual(options.existingFile, .none) + XCTAssertEqual(options.permissionsForRegularFile, self.expectedDefaults) + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .exclusiveCreate]) + + options = OpenOptions.Write.newFile(replaceExisting: false, permissions: .groupExecute) + XCTAssertEqual(options.existingFile, .none) + XCTAssertEqual(options.permissionsForRegularFile, .groupExecute) + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .exclusiveCreate]) + + options.followSymbolicLinks = false + XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .exclusiveCreate, .noFollow]) + } +} diff --git a/Tests/NIOFileSystemTests/FileSystemErrorTests.swift b/Tests/NIOFileSystemTests/FileSystemErrorTests.swift new file mode 100644 index 0000000000..6742afbaf6 --- /dev/null +++ b/Tests/NIOFileSystemTests/FileSystemErrorTests.swift @@ -0,0 +1,607 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) import NIOFileSystem +import XCTest + +final class FileSystemErrorTests: XCTestCase { + func testFileSystemErrorCustomStringConvertible() throws { + var error = FileSystemError( + code: .unsupported, + message: "An error message.", + cause: nil, + location: .init(function: "fn(_:)", file: "file.swift", line: 42) + ) + + XCTAssertEqual(String(describing: error), "Unsupported: An error message.") + + struct SomeCausalInfo: Error {} + error.cause = SomeCausalInfo() + + XCTAssertEqual( + String(describing: error), + "Unsupported: An error message. (SomeCausalInfo())" + ) + } + + func testFileSystemErrorCustomDebugStringConvertible() throws { + var error = FileSystemError( + code: .permissionDenied, + message: "An error message.", + cause: nil, + location: .init(function: "fn(_:)", file: "file.swift", line: 42) + ) + + XCTAssertEqual( + String(reflecting: error), + """ + Permission denied: "An error message." + """ + ) + + struct SomeCausalInfo: Error, CustomStringConvertible, CustomDebugStringConvertible { + var description: String { "SomeCausalInfo()" } + var debugDescription: String { + String(reflecting: self.description) + } + } + error.cause = SomeCausalInfo() + + XCTAssertEqual( + String(reflecting: error), + """ + Permission denied: "An error message." ("SomeCausalInfo()") + """ + ) + } + + func testFileSystemErrorDetailedDescription() throws { + var error = FileSystemError( + code: .permissionDenied, + message: "An error message.", + cause: nil, + location: .init(function: "fn(_:)", file: "file.swift", line: 42) + ) + + XCTAssertEqual( + error.detailedDescription(), + """ + FileSystemError: Permission denied + ├─ Reason: An error message. + └─ Source location: fn(_:) (file.swift:42) + """ + ) + + struct SomeCausalInfo: Error, CustomStringConvertible { + var description: String { "SomeCausalInfo()" } + } + error.cause = SomeCausalInfo() + + XCTAssertEqual( + error.detailedDescription(), + """ + FileSystemError: Permission denied + ├─ Reason: An error message. + ├─ Cause: SomeCausalInfo() + └─ Source location: fn(_:) (file.swift:42) + """ + ) + } + + func testFileSystemErrorCustomDebugStringConvertibleWithNestedCause() throws { + let location = FileSystemError.SourceLocation( + function: "fn(_:)", + file: "file.swift", + line: 42 + ) + + let subCause = FileSystemError( + code: .notFound, + message: "Where did I put that?", + cause: FileSystemError.SystemCallError(systemCall: "close", errno: .badFileDescriptor), + location: location + ) + + let cause = FileSystemError( + code: .invalidArgument, + message: "Can't close a file which is already closed.", + cause: subCause, + location: location + ) + + let error = FileSystemError( + code: .permissionDenied, + message: "I'm afraid I can't let you do that Dave.", + cause: cause, + location: location + ) + + XCTAssertEqual( + error.detailedDescription(), + """ + FileSystemError: Permission denied + ├─ Reason: I'm afraid I can't let you do that Dave. + ├─ Cause: + │ └─ FileSystemError: Invalid argument + │ ├─ Reason: Can't close a file which is already closed. + │ ├─ Cause: + │ │ └─ FileSystemError: Not found + │ │ ├─ Reason: Where did I put that? + │ │ ├─ Cause: 'close' system call failed with '(9) Bad file descriptor'. + │ │ └─ Source location: fn(_:) (file.swift:42) + │ └─ Source location: fn(_:) (file.swift:42) + └─ Source location: fn(_:) (file.swift:42) + """ + ) + } + + func testCopyOnWrite() throws { + let error1 = FileSystemError( + code: .io, + message: "a message", + cause: nil, + location: .init(function: "fn(_:)", file: "file.swift", line: 42) + ) + + var error2 = error1 + error2.code = .invalidArgument + XCTAssertEqual(error1.code, .io) + XCTAssertEqual(error2.code, .invalidArgument) + + var error3 = error1 + error3.message = "a different message" + XCTAssertEqual(error1.message, "a message") + XCTAssertEqual(error3.message, "a different message") + + var error4 = error1 + error4.cause = CancellationError() + XCTAssertNil(error1.cause) + XCTAssert(error4.cause is CancellationError) + + var error5 = error1 + error5.location.file = "different-file.swift" + XCTAssertEqual(error1.location.function, "fn(_:)") + XCTAssertEqual(error1.location.file, "file.swift") + XCTAssertEqual(error1.location.line, 42) + + XCTAssertEqual(error5.location.function, "fn(_:)") + XCTAssertEqual(error5.location.file, "different-file.swift") + XCTAssertEqual(error5.location.line, 42) + } + + func testErrorsMapToCorrectSyscallCause() throws { + let here = FileSystemError.SourceLocation(function: "fn", file: "file", line: 42) + let path = FilePath("/foo") + + for statName in ["stat", "lstat", "fstat"] { + assertCauseIsSyscall(statName, here) { + .stat(statName, errno: .badFileDescriptor, path: path, location: here) + } + } + + assertCauseIsSyscall("fchmod", here) { + .fchmod( + operation: .add, + operand: [], + permissions: [], + errno: .badFileDescriptor, + path: path, + location: here + ) + } + + assertCauseIsSyscall("flistxattr", here) { + .flistxattr(errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("fgetxattr", here) { + .fgetxattr(attribute: "attr", errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("fsetxattr", here) { + .fsetxattr(attribute: "attr", errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("fremovexattr", here) { + .fremovexattr(attribute: "attr", errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("fsync", here) { + .fsync(errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("dup", here) { + .dup(error: Errno.badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("close", here) { + .close(error: Errno.badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("read", here) { + .read(usingSyscall: .read, error: Errno.badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("pread", here) { + .read(usingSyscall: .pread, error: Errno.badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("write", here) { + .write(usingSyscall: .write, error: Errno.badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("pwrite", here) { + .write(usingSyscall: .pwrite, error: Errno.badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("fdopendir", here) { + .fdopendir(errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("readdir", here) { + .readdir(errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("openat", here) { + .open("openat", error: Errno.badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("mkdir", here) { + .mkdir(errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("rename", here) { + .rename(errno: .badFileDescriptor, oldName: "old", newName: "new", location: here) + } + + assertCauseIsSyscall("remove", here) { + .remove(errno: .badFileDescriptor, path: path, location: here) + } + + assertCauseIsSyscall("symlink", here) { + .symlink(errno: .badFileDescriptor, link: "link", target: "target", location: here) + } + + assertCauseIsSyscall("readlink", here) { + .readlink(errno: .badFileDescriptor, path: "link", location: here) + } + + assertCauseIsSyscall("getcwd", here) { + .getcwd(errno: .badFileDescriptor, location: here) + } + + assertCauseIsSyscall("confstr", here) { + .confstr(name: "foo", errno: .badFileDescriptor, location: here) + } + + assertCauseIsSyscall("fcopyfile", here) { + .fcopyfile(errno: .badFileDescriptor, from: "src", to: "dst", location: here) + } + + assertCauseIsSyscall("sendfile", here) { + .sendfile(errno: .badFileDescriptor, from: "src", to: "dst", location: here) + } + + assertCauseIsSyscall("ftruncate", here) { + .ftruncate(error: Errno.badFileDescriptor, path: path, location: here) + } + } + + func testErrnoMapping_stat() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed + ] + ) { errno in + .stat("stat", errno: errno, path: "path", location: .fixed) + } + } + + func testErrnoMapping_fchmod() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .invalidArgument: .invalidArgument, + .notPermitted: .permissionDenied, + ] + ) { errno in + .fchmod( + operation: .add, + operand: [], + permissions: [], + errno: errno, + path: "", + location: .fixed + ) + } + } + + func testErrnoMapping_flistxattr() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .notSupported: .unsupported, + .notPermitted: .unsupported, + .permissionDenied: .permissionDenied, + ] + ) { errno in + .flistxattr(errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_fgetxattr() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .notSupported: .unsupported, + ] + ) { errno in + .fgetxattr(attribute: "attr", errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_fsetxattr() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .notSupported: .unsupported, + .invalidArgument: .invalidArgument, + ] + ) { errno in + .fsetxattr(attribute: "attr", errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_fremovexattr() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .notSupported: .unsupported, + ] + ) { errno in + .fremovexattr(attribute: "attr", errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_fsync() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .ioError: .io, + ] + ) { errno in + .fsync(errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_dup() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed + ] + ) { errno in + .dup(error: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_close() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .ioError: .io, + ] + ) { errno in + .close(error: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_read() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .ioError: .io, + ] + ) { errno in + .read(usingSyscall: .read, error: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_pread() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .ioError: .io, + .illegalSeek: .unsupported, + ] + ) { errno in + .read(usingSyscall: .pread, error: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_write() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .ioError: .io, + ] + ) { errno in + .write(usingSyscall: .write, error: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_pwrite() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .ioError: .io, + .illegalSeek: .unsupported, + ] + ) { errno in + .write(usingSyscall: .pwrite, error: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_open() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .permissionDenied: .permissionDenied, + .fileExists: .fileAlreadyExists, + .ioError: .io, + .tooManyOpenFiles: .unavailable, + .noSuchFileOrDirectory: .notFound, + .notDirectory: .notFound, + ] + ) { errno in + .open("open", error: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_mkdir() { + self.testErrnoToErrorCode( + expected: [ + .permissionDenied: .permissionDenied, + .isDirectory: .invalidArgument, + .notDirectory: .invalidArgument, + .noSuchFileOrDirectory: .invalidArgument, + .ioError: .io, + .fileExists: .fileAlreadyExists, + ] + ) { errno in + .mkdir(errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_rename() { + self.testErrnoToErrorCode( + expected: [ + .permissionDenied: .permissionDenied, + .invalidArgument: .invalidArgument, + .noSuchFileOrDirectory: .notFound, + .ioError: .io, + ] + ) { errno in + .rename(errno: errno, oldName: "old", newName: "new", location: .fixed) + } + } + + func testErrnoMapping_remove() { + self.testErrnoToErrorCode( + expected: [ + .permissionDenied: .permissionDenied, + .notPermitted: .permissionDenied, + .resourceBusy: .unavailable, + .ioError: .io, + ] + ) { errno in + .remove(errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_symlink() { + self.testErrnoToErrorCode( + expected: [ + .permissionDenied: .permissionDenied, + .notPermitted: .permissionDenied, + .fileExists: .fileAlreadyExists, + .noSuchFileOrDirectory: .invalidArgument, + .notDirectory: .invalidArgument, + .ioError: .io, + ] + ) { errno in + .symlink(errno: errno, link: "link", target: "target", location: .fixed) + } + } + + func testErrnoMapping_readlink() { + self.testErrnoToErrorCode( + expected: [ + .permissionDenied: .permissionDenied, + .invalidArgument: .invalidArgument, + .noSuchFileOrDirectory: .notFound, + .ioError: .io, + ] + ) { errno in + .readlink(errno: errno, path: "", location: .fixed) + } + } + + func testErrnoMapping_fcopyfile() { + self.testErrnoToErrorCode( + expected: [ + .notSupported: .invalidArgument, + .invalidArgument: .invalidArgument, + .permissionDenied: .permissionDenied, + ] + ) { errno in + .fcopyfile(errno: errno, from: "src", to: "dst", location: .fixed) + } + } + + func testErrnoMapping_sendfile() { + self.testErrnoToErrorCode( + expected: [ + .ioError: .io, + .noMemory: .io, + ] + ) { errno in + .sendfile(errno: errno, from: "src", to: "dst", location: .fixed) + } + } + + func testErrnoMapping_resize() { + self.testErrnoToErrorCode( + expected: [ + .badFileDescriptor: .closed, + .fileTooLarge: .invalidArgument, + .invalidArgument: .invalidArgument, + ] + ) { errno in + .ftruncate(error: errno, path: "", location: .fixed) + } + } + + private func testErrnoToErrorCode( + expected mapping: [Errno: FileSystemError.Code], + _ makeError: (Errno) -> FileSystemError + ) { + for (errno, code) in mapping { + let error = makeError(errno) + XCTAssertEqual(error.code, code, "\(error)") + } + + let errno = Errno(rawValue: -1) + let error = makeError(errno) + XCTAssertEqual(error.code, .unknown, "\(error)") + } +} + +private func assertCauseIsSyscall( + _ name: String, + _ location: FileSystemError.SourceLocation, + _ buildError: () -> FileSystemError +) { + let error = buildError() + + XCTAssertEqual(error.location, location) + if let cause = error.cause as? FileSystemError.SystemCallError { + XCTAssertEqual(cause.systemCall, name) + } else { + XCTFail("Unexpected error: \(String(describing: error.cause))") + } +} + +extension FileSystemError.SourceLocation { + fileprivate static let fixed = Self(function: "fn", file: "file", line: 1) +} diff --git a/Tests/NIOFileSystemTests/FileTypeTests.swift b/Tests/NIOFileSystemTests/FileTypeTests.swift new file mode 100644 index 0000000000..179079d8f5 --- /dev/null +++ b/Tests/NIOFileSystemTests/FileTypeTests.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) import NIOFileSystem +import XCTest + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +final class FileTypeTests: XCTestCase { + func testFileTypeEquatable() { + let types = FileType.allCases + // Use indices to avoid using the `Equatable` conformance to write the tests. + for index in types.indices { + XCTAssertEqual(types[index], types[index]) + for otherIndex in types.indices where otherIndex != index { + XCTAssertNotEqual(types[index], types[otherIndex]) + } + } + } + + func testFileTypeHashable() { + // Tests there is `Hashable` conformance is sufficient as we rely on the compiler + // synthesising the implementation. + let types = FileType.allCases + let unique = Set(types) + XCTAssertEqual(types.count, unique.count) + } + + func testCustomStringConvertible() { + XCTAssertEqual(String(describing: FileType.block), "block") + XCTAssertEqual(String(describing: FileType.character), "character") + XCTAssertEqual(String(describing: FileType.directory), "directory") + XCTAssertEqual(String(describing: FileType.fifo), "fifo") + XCTAssertEqual(String(describing: FileType.regular), "regular") + XCTAssertEqual(String(describing: FileType.socket), "socket") + XCTAssertEqual(String(describing: FileType.symlink), "symlink") + XCTAssertEqual(String(describing: FileType.unknown), "unknown") + XCTAssertEqual(String(describing: FileType.whiteout), "whiteout") + } + + func testConversionFromModeT() { + XCTAssertEqual(FileType(platformSpecificMode: S_IFIFO), .fifo) + XCTAssertEqual(FileType(platformSpecificMode: S_IFCHR), .character) + XCTAssertEqual(FileType(platformSpecificMode: S_IFDIR), .directory) + XCTAssertEqual(FileType(platformSpecificMode: S_IFBLK), .block) + XCTAssertEqual(FileType(platformSpecificMode: S_IFREG), .regular) + XCTAssertEqual(FileType(platformSpecificMode: S_IFLNK), .symlink) + XCTAssertEqual(FileType(platformSpecificMode: S_IFSOCK), .socket) + #if canImport(Darwin) + XCTAssertEqual(FileType(platformSpecificMode: S_IFWHT), .whiteout) + #endif + } + + func testConversionFromDirentType() { + XCTAssertEqual(FileType(direntType: numericCast(DT_FIFO)), .fifo) + XCTAssertEqual(FileType(direntType: numericCast(DT_CHR)), .character) + XCTAssertEqual(FileType(direntType: numericCast(DT_DIR)), .directory) + XCTAssertEqual(FileType(direntType: numericCast(DT_BLK)), .block) + XCTAssertEqual(FileType(direntType: numericCast(DT_REG)), .regular) + XCTAssertEqual(FileType(direntType: numericCast(DT_LNK)), .symlink) + XCTAssertEqual(FileType(direntType: numericCast(DT_SOCK)), .socket) + #if canImport(Darwin) + XCTAssertEqual(FileType(direntType: numericCast(DT_WHT)), .whiteout) + #endif + } +} diff --git a/Tests/NIOFileSystemTests/Internal/CancellationTests.swift b/Tests/NIOFileSystemTests/Internal/CancellationTests.swift new file mode 100644 index 0000000000..d89c81e802 --- /dev/null +++ b/Tests/NIOFileSystemTests/Internal/CancellationTests.swift @@ -0,0 +1,89 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Atomics +@_spi(Testing) import NIOFileSystem +import XCTest + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +final class CancellationTests: XCTestCase { + func testWithoutCancellation() async throws { + try await withThrowingTaskGroup(of: Void.self) { group in + group.cancelAll() + + group.addTask { + XCTAssertTrue(Task.isCancelled) + try await withoutCancellation { + XCTAssertFalse(Task.isCancelled) + } + } + + group.addTask { + try await withoutCancellation { + try Task.checkCancellation() + } + } + + try await group.waitForAll() + } + } + + func testUncancellableTearDownWhenNotThrowing() async throws { + try await withThrowingTaskGroup(of: Void.self) { group in + group.cancelAll() + + group.addTask { + let ranTearDown = ManagedAtomic(false) + + let isCancelled = try await withUncancellableTearDown { + return Task.isCancelled + } tearDown: { _ in + ranTearDown.store(true, ordering: .releasing) + } + + XCTAssertTrue(isCancelled) + + let didRunTearDown = ranTearDown.load(ordering: .acquiring) + XCTAssertTrue(didRunTearDown) + } + + try await group.waitForAll() + } + } + + func testUncancellableTearDownWhenThrowing() async throws { + try await withThrowingTaskGroup(of: Void.self) { group in + group.cancelAll() + + group.addTask { + let ranTearDown = ManagedAtomic(false) + + await XCTAssertThrowsErrorAsync { + try await withUncancellableTearDown { + try Task.checkCancellation() + } tearDown: { _ in + ranTearDown.store(true, ordering: .releasing) + } + } onError: { error in + XCTAssert(error is CancellationError) + } + + let didRunTearDown = ranTearDown.load(ordering: .acquiring) + XCTAssertTrue(didRunTearDown) + } + + try await group.waitForAll() + } + } +} diff --git a/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift b/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift new file mode 100644 index 0000000000..6b82266278 --- /dev/null +++ b/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift @@ -0,0 +1,1136 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import XCTest + +@testable import NIOFileSystem + +final class BufferedStreamTests: XCTestCase { + // MARK: - sequenceDeinitialized + + func testSequenceDeinitialized_whenNoIterator() async throws { + var (stream, source): (BufferedStream?, BufferedStream.Source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source.onTermination = { + onTerminationContinuation.finish() + } + + await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + withExtendedLifetime(stream) {} + stream = nil + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + do { + _ = try { try source.write(2) }() + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is AlreadyFinishedError) + } + + group.cancelAll() + } + } + + func testSequenceDeinitialized_whenIterator() async throws { + var (stream, source): (BufferedStream?, BufferedStream.Source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + var iterator = stream?.makeAsyncIterator() + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source.onTermination = { + onTerminationContinuation.finish() + } + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + try withExtendedLifetime(stream) { + let writeResult = try source.write(1) + writeResult.assertIsProducerMore() + } + + stream = nil + + do { + let writeResult = try { try source.write(2) }() + writeResult.assertIsProducerMore() + } catch { + XCTFail("Expected no error to be thrown") + } + + let element1 = try await iterator?.next() + XCTAssertEqual(element1, 1) + let element2 = try await iterator?.next() + XCTAssertEqual(element2, 2) + + group.cancelAll() + } + } + + func testSequenceDeinitialized_whenFinished() async throws { + var (stream, source): (BufferedStream?, BufferedStream.Source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source.onTermination = { + onTerminationContinuation.finish() + } + + await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + withExtendedLifetime(stream) { + source.finish(throwing: nil) + } + + stream = nil + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + do { + _ = try { try source.write(1) }() + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is AlreadyFinishedError) + } + + group.cancelAll() + } + } + + func testSequenceDeinitialized_whenStreaming_andSuspendedProducer() async throws { + var (stream, source): (BufferedStream?, BufferedStream.Source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 2) + ) + + _ = try { try source.write(1) }() + + do { + try await withCheckedThrowingContinuation { continuation in + source.write(1) { result in + continuation.resume(with: result) + } + + stream = nil + _ = stream?.makeAsyncIterator() + } + } catch { + XCTAssertTrue(error is AlreadyFinishedError) + } + } + + // MARK: - iteratorInitialized + + func testIteratorInitialized_whenInitial() async throws { + let (stream, _) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + _ = stream.makeAsyncIterator() + } + + func testIteratorInitialized_whenStreaming() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + try await source.write(1) + + var iterator = stream.makeAsyncIterator() + let element = try await iterator.next() + XCTAssertEqual(element, 1) + } + + func testIteratorInitialized_whenSourceFinished() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + try await source.write(1) + source.finish(throwing: nil) + + var iterator = stream.makeAsyncIterator() + let element1 = try await iterator.next() + XCTAssertEqual(element1, 1) + let element2 = try await iterator.next() + XCTAssertNil(element2) + } + + func testIteratorInitialized_whenFinished() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + source.finish(throwing: nil) + + var iterator = stream.makeAsyncIterator() + let element = try await iterator.next() + XCTAssertNil(element) + } + + // MARK: - iteratorDeinitialized + + func testIteratorDeinitialized_whenInitial() async throws { + var (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source.onTermination = { + onTerminationContinuation.finish() + } + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + var iterator: BufferedStream.AsyncIterator? = stream.makeAsyncIterator() + iterator = nil + _ = try await iterator?.next() + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testIteratorDeinitialized_whenStreaming() async throws { + var (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source.onTermination = { + onTerminationContinuation.finish() + } + + try await source.write(1) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + var iterator: BufferedStream.AsyncIterator? = stream.makeAsyncIterator() + iterator = nil + _ = try await iterator?.next() + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testIteratorDeinitialized_whenSourceFinished() async throws { + var (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source.onTermination = { + onTerminationContinuation.finish() + } + + try await source.write(1) + source.finish(throwing: nil) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + var iterator: BufferedStream.AsyncIterator? = stream.makeAsyncIterator() + iterator = nil + _ = try await iterator?.next() + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testIteratorDeinitialized_whenFinished() async throws { + var (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source.onTermination = { + onTerminationContinuation.finish() + } + + source.finish(throwing: nil) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + var iterator: BufferedStream.AsyncIterator? = stream.makeAsyncIterator() + iterator = nil + _ = try await iterator?.next() + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testIteratorDeinitialized_whenStreaming_andSuspendedProducer() async throws { + var (stream, source): (BufferedStream?, BufferedStream.Source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 2) + ) + + var iterator: BufferedStream.AsyncIterator? = stream?.makeAsyncIterator() + stream = nil + + _ = try { try source.write(1) }() + + do { + try await withCheckedThrowingContinuation { continuation in + source.write(1) { result in + continuation.resume(with: result) + } + + iterator = nil + } + } catch { + XCTAssertTrue(error is AlreadyFinishedError) + } + + _ = try await iterator?.next() + } + + // MARK: - sourceDeinitialized + + func testSourceDeinitialized_whenInitial() async throws { + var (stream, source): (BufferedStream, BufferedStream.Source?) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source?.onTermination = { + onTerminationContinuation.finish() + } + + await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + source = nil + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + + withExtendedLifetime(stream) {} + } + + func testSourceDeinitialized_whenStreaming_andEmptyBuffer() async throws { + var (stream, source): (BufferedStream, BufferedStream.Source?) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source?.onTermination = { + onTerminationContinuation.finish() + } + + try await source?.write(1) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + var iterator: BufferedStream.AsyncIterator? = stream.makeAsyncIterator() + _ = try await iterator?.next() + + source = nil + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testSourceDeinitialized_whenStreaming_andNotEmptyBuffer() async throws { + var (stream, source): (BufferedStream, BufferedStream.Source?) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source?.onTermination = { + onTerminationContinuation.finish() + } + + try await source?.write(1) + try await source?.write(2) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + var iterator: BufferedStream.AsyncIterator? = stream.makeAsyncIterator() + _ = try await iterator?.next() + + source = nil + + _ = await onTerminationIterator.next() + + _ = try await iterator?.next() + _ = try await iterator?.next() + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testSourceDeinitialized_whenSourceFinished() async throws { + var (stream, source): (BufferedStream, BufferedStream.Source?) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source?.onTermination = { + onTerminationContinuation.finish() + } + + try await source?.write(1) + try await source?.write(2) + source?.finish(throwing: nil) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + var iterator: BufferedStream.AsyncIterator? = stream.makeAsyncIterator() + _ = try await iterator?.next() + + source = nil + + _ = await onTerminationIterator.next() + + _ = try await iterator?.next() + _ = try await iterator?.next() + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testSourceDeinitialized_whenFinished() async throws { + var (stream, source): (BufferedStream, BufferedStream.Source?) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 5, high: 10) + ) + + let (onTerminationStream, onTerminationContinuation) = AsyncStream.makeStream() + source?.onTermination = { + onTerminationContinuation.finish() + } + + source?.finish(throwing: nil) + + await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while !Task.isCancelled { + onTerminationContinuation.yield() + try await Task.sleep(nanoseconds: 200_000_000) + } + } + + var onTerminationIterator = onTerminationStream.makeAsyncIterator() + _ = await onTerminationIterator.next() + + _ = stream.makeAsyncIterator() + + source = nil + + _ = await onTerminationIterator.next() + + let terminationResult: Void? = await onTerminationIterator.next() + XCTAssertNil(terminationResult) + + group.cancelAll() + } + } + + func testSourceDeinitialized_whenStreaming_andSuspendedProducer() async throws { + var (stream, source): (BufferedStream, BufferedStream.Source?) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 0, high: 0) + ) + let (producerStream, producerContinuation) = AsyncThrowingStream.makeStream() + var iterator = stream.makeAsyncIterator() + + source?.write(1) { + producerContinuation.yield(with: $0) + } + + _ = try await iterator.next() + source = nil + + do { + try await producerStream.first { _ in true } + XCTFail("We expected to throw here") + } catch { + XCTAssertTrue(error is AlreadyFinishedError) + } + } + + // MARK: - write + + func testWrite_whenInitial() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 5) + ) + + try await source.write(1) + + var iterator = stream.makeAsyncIterator() + let element = try await iterator.next() + XCTAssertEqual(element, 1) + } + + func testWrite_whenStreaming_andNoConsumer() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 5) + ) + + try await source.write(1) + try await source.write(2) + + var iterator = stream.makeAsyncIterator() + let element1 = try await iterator.next() + XCTAssertEqual(element1, 1) + let element2 = try await iterator.next() + XCTAssertEqual(element2, 2) + } + + func testWrite_whenStreaming_andSuspendedConsumer() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 5) + ) + + try await withThrowingTaskGroup(of: Int?.self) { group in + group.addTask { + return try await stream.first { _ in true } + } + + // This is always going to be a bit racy since we need the call to next() suspend + try await Task.sleep(nanoseconds: 500_000_000) + + try await source.write(1) + let element = try await group.next() + XCTAssertEqual(element, 1) + } + } + + func testWrite_whenStreaming_andSuspendedConsumer_andEmptySequence() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 5) + ) + + try await withThrowingTaskGroup(of: Int?.self) { group in + group.addTask { + return try await stream.first { _ in true } + } + + // This is always going to be a bit racy since we need the call to next() suspend + try await Task.sleep(nanoseconds: 500_000_000) + + try await source.write(contentsOf: []) + try await source.write(contentsOf: [1]) + let element = try await group.next() + XCTAssertEqual(element, 1) + } + } + + // MARK: - enqueueProducer + + func testEnqueueProducer_whenStreaming_andAndCancelled() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 2) + ) + + let (producerStream, producerSource) = AsyncThrowingStream.makeStream() + + try await source.write(1) + + let writeResult = try { try source.write(2) }() + + switch writeResult { + case .produceMore: + preconditionFailure() + case .enqueueCallback(let callbackToken): + source.cancelCallback(callbackToken: callbackToken) + + source.enqueueCallback(callbackToken: callbackToken) { result in + producerSource.yield(with: result) + } + } + + do { + _ = try await producerStream.first { _ in true } + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is CancellationError) + } + + let element = try await stream.first { _ in true } + XCTAssertEqual(element, 1) + } + + func testEnqueueProducer_whenStreaming_andAndCancelled_andAsync() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 2) + ) + + try await source.write(1) + + await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await source.write(2) + } + + group.cancelAll() + do { + try await group.next() + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is CancellationError) + } + } + + let element = try await stream.first { _ in true } + XCTAssertEqual(element, 1) + } + + func testEnqueueProducer_whenStreaming_andInterleaving() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 1) + ) + var iterator = stream.makeAsyncIterator() + + let (producerStream, producerSource) = AsyncThrowingStream.makeStream() + + let writeResult = try { try source.write(1) }() + + switch writeResult { + case .produceMore: + preconditionFailure() + case .enqueueCallback(let callbackToken): + let element = try await iterator.next() + XCTAssertEqual(element, 1) + + source.enqueueCallback(callbackToken: callbackToken) { result in + producerSource.yield(with: result) + } + } + + do { + _ = try await producerStream.first { _ in true } + } catch { + XCTFail("Expected no error to be thrown") + } + } + + func testEnqueueProducer_whenStreaming_andSuspending() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 1) + ) + var iterator = stream.makeAsyncIterator() + + let (producerStream, producerSource) = AsyncThrowingStream.makeStream() + + let writeResult = try { try source.write(1) }() + + switch writeResult { + case .produceMore: + preconditionFailure() + case .enqueueCallback(let callbackToken): + source.enqueueCallback(callbackToken: callbackToken) { result in + producerSource.yield(with: result) + } + } + + let element = try await iterator.next() + XCTAssertEqual(element, 1) + + do { + _ = try await producerStream.first { _ in true } + } catch { + XCTFail("Expected no error to be thrown") + } + } + + func testEnqueueProducer_whenFinished() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 1) + ) + var iterator = stream.makeAsyncIterator() + + let (producerStream, producerSource) = AsyncThrowingStream.makeStream() + + let writeResult = try { try source.write(1) }() + + switch writeResult { + case .produceMore: + preconditionFailure() + case .enqueueCallback(let callbackToken): + source.finish(throwing: nil) + + source.enqueueCallback(callbackToken: callbackToken) { result in + producerSource.yield(with: result) + } + } + + let element = try await iterator.next() + XCTAssertEqual(element, 1) + + do { + _ = try await producerStream.first { _ in true } + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is AlreadyFinishedError) + } + } + + // MARK: - cancelProducer + + func testCancelProducer_whenStreaming() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 2) + ) + + let (producerStream, producerSource) = AsyncThrowingStream.makeStream() + + try await source.write(1) + + let writeResult = try { try source.write(2) }() + + switch writeResult { + case .produceMore: + preconditionFailure() + case .enqueueCallback(let callbackToken): + source.enqueueCallback(callbackToken: callbackToken) { result in + producerSource.yield(with: result) + } + + source.cancelCallback(callbackToken: callbackToken) + } + + do { + _ = try await producerStream.first { _ in true } + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is CancellationError) + } + + let element = try await stream.first { _ in true } + XCTAssertEqual(element, 1) + } + + func testCancelProducer_whenSourceFinished() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 2) + ) + + let (producerStream, producerSource) = AsyncThrowingStream.makeStream() + + try await source.write(1) + + let writeResult = try { try source.write(2) }() + + switch writeResult { + case .produceMore: + preconditionFailure() + case .enqueueCallback(let callbackToken): + source.enqueueCallback(callbackToken: callbackToken) { result in + producerSource.yield(with: result) + } + + source.finish(throwing: nil) + + source.cancelCallback(callbackToken: callbackToken) + } + + do { + _ = try await producerStream.first { _ in true } + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is AlreadyFinishedError) + } + + let element = try await stream.first { _ in true } + XCTAssertEqual(element, 1) + } + + // MARK: - finish + + func testFinish_whenStreaming_andConsumerSuspended() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 1) + ) + + try await withThrowingTaskGroup(of: Int?.self) { group in + group.addTask { + return try await stream.first { $0 == 2 } + } + + // This is always going to be a bit racy since we need the call to next() suspend + try await Task.sleep(nanoseconds: 500_000_000) + + source.finish(throwing: nil) + let element = try await group.next() + XCTAssertEqual(element, .some(nil)) + } + } + + func testFinish_whenInitial() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 1, high: 1) + ) + + source.finish(throwing: CancellationError()) + + do { + for try await _ in stream {} + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is CancellationError) + } + + } + + // MARK: - Backpressure + + func testBackPressure() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 4) + ) + + let (backPressureEventStream, backPressureEventContinuation) = AsyncStream.makeStream( + of: Void.self + ) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + while true { + backPressureEventContinuation.yield(()) + try await source.write(contentsOf: [1]) + } + } + + var backPressureEventIterator = backPressureEventStream.makeAsyncIterator() + var iterator = stream.makeAsyncIterator() + + await backPressureEventIterator.next() + await backPressureEventIterator.next() + await backPressureEventIterator.next() + await backPressureEventIterator.next() + + _ = try await iterator.next() + _ = try await iterator.next() + _ = try await iterator.next() + + await backPressureEventIterator.next() + await backPressureEventIterator.next() + await backPressureEventIterator.next() + + group.cancelAll() + } + } + + func testBackPressureSync() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 4) + ) + + let (backPressureEventStream, backPressureEventContinuation) = AsyncStream.makeStream( + of: Void.self + ) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + @Sendable func yield() { + backPressureEventContinuation.yield(()) + source.write(contentsOf: [1]) { result in + switch result { + case .success: + yield() + + case .failure: + break + } + } + } + + yield() + } + + var backPressureEventIterator = backPressureEventStream.makeAsyncIterator() + var iterator = stream.makeAsyncIterator() + + await backPressureEventIterator.next() + await backPressureEventIterator.next() + await backPressureEventIterator.next() + await backPressureEventIterator.next() + + _ = try await iterator.next() + _ = try await iterator.next() + _ = try await iterator.next() + + await backPressureEventIterator.next() + await backPressureEventIterator.next() + await backPressureEventIterator.next() + + group.cancelAll() + } + } + + func testThrowsError() async throws { + let (stream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 4) + ) + + try await source.write(1) + try await source.write(2) + source.finish(throwing: CancellationError()) + + var elements = [Int]() + var iterator = stream.makeAsyncIterator() + + do { + while let element = try await iterator.next() { + elements.append(element) + } + XCTFail("Expected an error to be thrown") + } catch { + XCTAssertTrue(error is CancellationError) + XCTAssertEqual(elements, [1, 2]) + } + + let element = try await iterator.next() + XCTAssertNil(element) + } + + func testAsyncSequenceWrite() async throws { + let (stream, continuation) = AsyncStream.makeStream() + let (backpressuredStream, source) = BufferedStream.makeStream( + of: Int.self, + backPressureStrategy: .watermark(low: 2, high: 4) + ) + + continuation.yield(1) + continuation.yield(2) + continuation.finish() + + try await source.write(contentsOf: stream) + source.finish(throwing: nil) + + let elements = try await backpressuredStream.collect() + XCTAssertEqual(elements, [1, 2]) + } +} + +extension AsyncSequence { + /// Collect all elements in the sequence into an array. + fileprivate func collect() async rethrows -> [Element] { + try await self.reduce(into: []) { accumulated, next in + accumulated.append(next) + } + } +} + +extension BufferedStream.Source.WriteResult { + func assertIsProducerMore() { + switch self { + case .produceMore: + return + + case .enqueueCallback: + XCTFail("Expected produceMore") + } + } + + func assertIsEnqueueCallback() { + switch self { + case .produceMore: + XCTFail("Expected enqueueCallback") + + case .enqueueCallback: + return + } + } +} + +extension AsyncStream { + static func makeStream( + of elementType: Element.Type = Element.self, + bufferingPolicy limit: AsyncStream.Continuation.BufferingPolicy = .unbounded + ) -> (stream: AsyncStream, continuation: AsyncStream.Continuation) { + var continuation: AsyncStream.Continuation! + let stream = AsyncStream(bufferingPolicy: limit) { continuation = $0 } + return (stream, continuation!) + } +} + +extension AsyncThrowingStream { + static func makeStream( + of elementType: Element.Type = Element.self, + throwing failureType: Failure.Type = Failure.self, + bufferingPolicy limit: AsyncThrowingStream.Continuation.BufferingPolicy = + .unbounded + ) -> ( + stream: AsyncThrowingStream, + continuation: AsyncThrowingStream.Continuation + ) where Failure == Error { + var continuation: AsyncThrowingStream.Continuation! + let stream = AsyncThrowingStream(bufferingPolicy: limit) { continuation = $0 } + return (stream, continuation!) + } +} diff --git a/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/IOExecutorTests.swift b/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/IOExecutorTests.swift new file mode 100644 index 0000000000..6dd3469684 --- /dev/null +++ b/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/IOExecutorTests.swift @@ -0,0 +1,172 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) import NIOFileSystem +import XCTest + +internal final class IOExecutorTests: XCTestCase { + func testExecuteAsync() async throws { + try await withExecutor { executor in + let n = try await executor.execute { + return fibonacci(13) + } + XCTAssertEqual(n, 233) + } + } + + func testExecuteWithCallback() async throws { + try await withExecutor { executor in + await withCheckedContinuation { continuation in + executor.execute { + fibonacci(13) + } onCompletion: { result in + switch result { + case let .success(n): + XCTAssertEqual(n, 233) + case let .failure(error): + XCTFail("Unexpected error: \(error)") + } + continuation.resume() + } + } + } + } + + func testExecuteManyWorkItemsOn2Threads() async throws { + // There are (slightly) different code paths depending on the number + // of threads which determine which executor to pick so run this test + // on 2 and 4 threads to exercise those paths. + try await self.testExecuteManyWorkItems(threadCount: 2) + } + + func testExecuteManyWorkItemsOn4Threads() async throws { + // There are (slightly) different code paths depending on the number + // of threads which determine which executor to pick so run this test + // on 2 and 4 threads to exercise those paths. + try await self.testExecuteManyWorkItems(threadCount: 4) + } + + func testExecuteManyWorkItems(threadCount threads: Int) async throws { + try await withExecutor(numberOfThreads: threads) { executor in + try await withThrowingTaskGroup(of: Int.self) { group in + let values = Array(0..<10_000) + + for value in values { + group.addTask { + try await executor.execute { return value } + } + } + + // Wait for the values. + let processed = try await group.reduce(into: [], { $0.append($1) }) + + // They should be the same but may be in a different order. + XCTAssertEqual(processed.sorted(), values) + } + } + } + + func testExecuteCancellation() async throws { + try await withExecutor { executor in + await withThrowingTaskGroup(of: Void.self) { group in + group.cancelAll() + group.addTask { + try await executor.execute { + XCTFail("Should be cancelled before executed") + } + } + + await XCTAssertThrowsErrorAsync { + try await group.waitForAll() + } onError: { error in + XCTAssert(error is CancellationError) + } + } + } + } + + func testDrainWhenEmpty() async throws { + let executor = await IOExecutor.running(numberOfThreads: 1) + await executor.drain() + } + + func testExecuteAfterDraining() async throws { + let executor = await IOExecutor.running(numberOfThreads: 1) + await executor.drain() + + // Callback should fire with an error. + executor.execute { + XCTFail("Work unexpectedly ran in drained pool.") + } onCompletion: { result in + switch result { + case .success: + XCTFail("Work unexpectedly ran in drained pool.") + case let .failure(error): + if let fsError = error as? FileSystemError { + XCTAssertEqual(fsError.code, .unavailable) + } else { + XCTFail("Unexpected error: \(error)") + } + } + } + + // Should throw immediately. + await XCTAssertThrowsErrorAsync { + try await executor.execute { + XCTFail("Work unexpectedly ran in drained pool.") + } + } onError: { error in + if let fsError = error as? FileSystemError { + XCTAssertEqual(fsError.code, .unavailable) + } else { + XCTFail("Unexpected error: \(error)") + } + } + } + + func testWorkWhenStartingAsync() async throws { + try await withThrowingTaskGroup(of: Int.self) { group in + let executor = IOExecutor.runningAsync(numberOfThreads: 4) + for _ in 0..<1000 { + group.addTask { + try await executor.execute { fibonacci(8) } + } + } + + try await group.waitForAll() + await executor.drain() + } + } +} + +private func fibonacci(_ n: Int) -> Int { + if n < 2 { + return n + } else { + return fibonacci(n - 1) + fibonacci(n - 2) + } +} + +private func withExecutor( + numberOfThreads: Int = 1, + execute: (IOExecutor) async throws -> Void +) async throws { + let executor = await IOExecutor.running(numberOfThreads: numberOfThreads) + do { + try await execute(executor) + await executor.drain() + } catch { + await executor.drain() + } +} diff --git a/Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift b/Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift new file mode 100644 index 0000000000..51091c0d27 --- /dev/null +++ b/Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift @@ -0,0 +1,254 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/* + This source file is part of the Swift System open source project + + Copyright (c) 2020 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + */ + +@_spi(Testing) import NIOFileSystem +import SystemPackage +import XCTest + +internal struct Wildcard: Hashable {} + +extension Trace.Entry { + /// This implements `==` with wildcard matching. + /// (`Entry` cannot conform to `Equatable`/`Hashable` this way because + /// the wildcard matching `==` relation isn't transitive.) + internal func matches(_ other: Self) -> Bool { + guard self.name == other.name else { return false } + guard self.arguments.count == other.arguments.count else { return false } + for i in self.arguments.indices { + if self.arguments[i] is Wildcard || other.arguments[i] is Wildcard { + continue + } + guard self.arguments[i] == other.arguments[i] else { return false } + } + return true + } +} + +internal protocol TestCase { + // TODO: want a source location stack, more fidelity, kinds of stack entries, etc + var file: StaticString { get } + var line: UInt { get } + + // TODO: Instead have an attribute to register a test in a allTests var, similar to the argument parser. + func runAllTests() + + // Customization hook: add adornment to reported failure reason + // Defaut: reason or empty + func failureMessage(_ reason: String?) -> String +} + +extension TestCase { + // Default implementation + func failureMessage(_ reason: String?) -> String { reason ?? "" } + + func expectEqualSequence( + _ expected: S1, + _ actual: S2, + _ message: String? = nil + ) where S1.Element: Equatable, S1.Element == S2.Element { + if !expected.elementsEqual(actual) { + defer { print("expected: \(expected)\n actual: \(actual)") } + fail(message) + } + } + func expectEqual( + _ expected: E, + _ actual: E, + _ message: String? = nil + ) { + if actual != expected { + defer { print("expected: \(expected)\n actual: \(actual)") } + fail(message) + } + } + func expectNotEqual( + _ expected: E, + _ actual: E, + _ message: String? = nil + ) { + if actual == expected { + defer { print("expected not equal: \(expected) and \(actual)") } + fail(message) + } + } + func expectMatch( + _ expected: Trace.Entry?, + _ actual: Trace.Entry?, + _ message: String? = nil + ) { + func check() -> Bool { + switch (expected, actual) { + case let (expected?, actual?): + return expected.matches(actual) + case (nil, nil): + return true + default: + return false + } + } + if !check() { + let e = expected.map { "\($0)" } ?? "nil" + let a = actual.map { "\($0)" } ?? "nil" + defer { print("expected: \(e)\n actual: \(a)") } + fail(message) + } + } + func expectNil( + _ actual: T?, + _ message: String? = nil + ) { + if actual != nil { + defer { print("expected nil: \(actual!)") } + fail(message) + } + } + func expectNotNil( + _ actual: T?, + _ message: String? = nil + ) { + if actual == nil { + defer { print("expected non-nil") } + fail(message) + } + } + func expectTrue( + _ actual: Bool, + _ message: String? = nil + ) { + if !actual { fail(message) } + } + func expectFalse( + _ actual: Bool, + _ message: String? = nil + ) { + if actual { fail(message) } + } + + func fail(_ reason: String? = nil) { + XCTAssert(false, failureMessage(reason), file: file, line: line) + } +} + +internal struct MockTestCase: TestCase { + var file: StaticString + var line: UInt + + var expected: Trace.Entry + var interruptBehavior: InterruptBehavior + + var interruptable: Bool { return interruptBehavior == .interruptable } + + internal enum InterruptBehavior { + // Retry the syscall on EINTR + case interruptable + + // Cannot return EINTR + case noInterrupt + + // Cannot error at all + case noError + } + + var body: (_ retryOnInterrupt: Bool) throws -> Void + + init( + _ file: StaticString = #file, + _ line: UInt = #line, + name: String, + _ interruptable: InterruptBehavior, + _ args: AnyHashable..., + body: @escaping (_ retryOnInterrupt: Bool) throws -> Void + ) { + self.file = file + self.line = line + self.expected = Trace.Entry(name: name, args) + self.interruptBehavior = interruptable + self.body = body + } + + func runAllTests() { + XCTAssertFalse(MockingDriver.enabled) + MockingDriver.withMockingEnabled { mocking in + // Make sure we completely match the trace queue + self.expectTrue(mocking.trace.isEmpty) + defer { self.expectTrue(mocking.trace.isEmpty) } + + // Test our API mappings to the lower-level syscall invocation + do { + try body(true) + self.expectMatch(self.expected, mocking.trace.dequeue()) + } catch { + self.fail() + } + + // Non-error-ing syscalls shouldn't ever throw + guard interruptBehavior != .noError else { + do { + try body(interruptable) + self.expectMatch(self.expected, mocking.trace.dequeue()) + try body(!interruptable) + self.expectMatch(self.expected, mocking.trace.dequeue()) + } catch { + self.fail() + } + return + } + + // Test interupt behavior. Interruptable calls will be told not to + // retry to catch the EINTR. Non-interruptable calls will be told to + // retry, to make sure they don't spin (e.g. if API changes to include + // interruptable) + do { + mocking.forceErrno = .always(errno: EINTR) + try body(!interruptable) + self.fail() + } catch Errno.interrupted { + // Success! + self.expectMatch(self.expected, mocking.trace.dequeue()) + } catch { + self.fail() + } + + // Force a limited number of EINTRs, and make sure interruptable functions + // retry that number of times. Non-interruptable functions should throw it. + do { + mocking.forceErrno = .counted(errno: EINTR, count: 3) + + try body(interruptable) + self.expectMatch(self.expected, mocking.trace.dequeue()) // EINTR + self.expectMatch(self.expected, mocking.trace.dequeue()) // EINTR + self.expectMatch(self.expected, mocking.trace.dequeue()) // EINTR + self.expectMatch(self.expected, mocking.trace.dequeue()) // Success + } catch Errno.interrupted { + self.expectFalse(interruptable) + self.expectMatch(self.expected, mocking.trace.dequeue()) // EINTR + } catch { + self.fail() + } + } + } +} + +internal func withWindowsPaths(enabled: Bool, _ body: () -> Void) { + _withWindowsPaths(enabled: enabled, body) +} diff --git a/Tests/NIOFileSystemTests/Internal/SyscallTests.swift b/Tests/NIOFileSystemTests/Internal/SyscallTests.swift new file mode 100644 index 0000000000..de7ed73b85 --- /dev/null +++ b/Tests/NIOFileSystemTests/Internal/SyscallTests.swift @@ -0,0 +1,470 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) import NIOFileSystem +import SystemPackage +import XCTest + +final class SyscallTests: XCTestCase { + func test_openat() throws { + let fd = FileDescriptor(rawValue: 42) + let testCases = [ + MockTestCase( + name: "fdopenat", + .interruptable, + fd.rawValue, + "a path", + O_RDONLY | O_NONBLOCK + ) { retryOnInterrupt in + _ = try fd.open( + atPath: "a path", + mode: .readOnly, + options: [.nonBlocking], + permissions: nil, + retryOnInterrupt: retryOnInterrupt + ).get() + }, + + MockTestCase( + name: "fdopenat", + .interruptable, + fd.rawValue, + "a path", + O_WRONLY | O_CREAT, + 0o777 + ) { retryOnInterrupt in + _ = try fd.open( + atPath: "a path", + mode: .writeOnly, + options: [.create], + permissions: [ + .groupReadWriteExecute, .ownerReadWriteExecute, .otherReadWriteExecute, + ], + retryOnInterrupt: retryOnInterrupt + ).get() + }, + + ] + + testCases.run() + } + + func test_stat() throws { + let testCases = [ + MockTestCase(name: "stat", .noInterrupt, "a path") { _ in + _ = try Syscall.stat(path: "a path").get() + }, + + MockTestCase(name: "lstat", .noInterrupt, "a path") { _ in + _ = try Syscall.lstat(path: "a path").get() + }, + + MockTestCase(name: "fstat", .noInterrupt, 42) { _ in + _ = try FileDescriptor(rawValue: 42).status().get() + }, + ] + + testCases.run() + } + + func test_fchmod() throws { + let fd = FileDescriptor(rawValue: 42) + let permissions: FilePermissions = [ + .groupReadWriteExecute, + .otherReadWriteExecute, + .ownerReadWriteExecute, + ] + + let testCases = [ + MockTestCase(name: "fchmod", .interruptable, 42, 0) { retryOnInterrupt in + try fd.changeMode([], retryOnInterrupt: retryOnInterrupt).get() + }, + + MockTestCase(name: "fchmod", .interruptable, 42, 0o777) { retryOnInterrupt in + try fd.changeMode(permissions, retryOnInterrupt: retryOnInterrupt).get() + }, + ] + + testCases.run() + } + + func test_fsync() throws { + let fd = FileDescriptor(rawValue: 42) + let testCases = [ + MockTestCase(name: "fsync", .interruptable, 42) { retryOnInterrupt in + try fd.synchronize(retryOnInterrupt: retryOnInterrupt).get() + } + ] + + testCases.run() + } + + func test_mkdir() throws { + let testCases = [ + MockTestCase(name: "mkdir", .noInterrupt, "a path", 0) { _ in + try Syscall.mkdir(at: "a path", permissions: []).get() + }, + + MockTestCase(name: "mkdir", .noInterrupt, "a path", 0o777) { _ in + try Syscall.mkdir( + at: "a path", + permissions: [ + .groupReadWriteExecute, .otherReadWriteExecute, .ownerReadWriteExecute, + ] + ).get() + }, + ] + + testCases.run() + } + + func test_linkat() throws { + #if canImport(Glibc) + let fd1 = FileDescriptor(rawValue: 13) + let fd2 = FileDescriptor(rawValue: 42) + + let testCases = [ + MockTestCase(name: "linkat", .noInterrupt, 13, "src", 42, "dst", 0) { _ in + try Syscall.linkAt( + from: "src", + relativeTo: fd1, + to: "dst", + relativeTo: fd2, + flags: [] + ).get() + }, + MockTestCase(name: "linkat", .noInterrupt, 13, "src", 42, "dst", 4096) { _ in + try Syscall.linkAt( + from: "src", + relativeTo: fd1, + to: "dst", + relativeTo: fd2, + flags: [.emptyPath] + ).get() + }, + ] + testCases.run() + #else + throw XCTSkip("'linkat' is only supported on Linux") + #endif + } + + func test_symlink() throws { + let testCases = [ + MockTestCase(name: "symlink", .noInterrupt, "one", "two") { _ in + try Syscall.symlink(to: "one", from: "two").get() + } + ] + + testCases.run() + } + + func test_readlink() throws { + let testCases = [ + MockTestCase( + name: "readlink", + .noInterrupt, + "a path", + "", + CInterop.maxPathLength + ) { _ in + _ = try Syscall.readlink(at: "a path").get() + } + ] + + testCases.run() + } + + func test_flistxattr() throws { + let fd = FileDescriptor(rawValue: 42) + let buffer = UnsafeMutableBufferPointer.allocate(capacity: 1024) + defer { buffer.deallocate() } + + let testCases: [MockTestCase] = [ + MockTestCase(name: "flistxattr", .noInterrupt, 42, "nil", 0) { _ in + _ = try fd.listExtendedAttributes(nil).get() + }, + + MockTestCase(name: "flistxattr", .noInterrupt, 42, "", 1024) { _ in + _ = try fd.listExtendedAttributes(buffer).get() + }, + ] + + testCases.run() + } + + func test_fgetxattr() throws { + let fd = FileDescriptor(rawValue: 42) + let buffer = UnsafeMutableRawBufferPointer.allocate(byteCount: 1024, alignment: 1) + defer { buffer.deallocate() } + + let testCases: [MockTestCase] = [ + MockTestCase(name: "fgetxattr", .noInterrupt, 42, "an attribute", "nil", 0) { _ in + _ = try fd.getExtendedAttribute(named: "an attribute", buffer: nil).get() + }, + + MockTestCase(name: "fgetxattr", .noInterrupt, 42, "an attribute", "", 1024) { + _ in + _ = try fd.getExtendedAttribute(named: "an attribute", buffer: buffer).get() + }, + ] + + testCases.run() + } + + func test_fsetxattr() throws { + let fd = FileDescriptor(rawValue: 42) + let buffer = UnsafeMutableRawBufferPointer.allocate(byteCount: 1024, alignment: 1) + defer { buffer.deallocate() } + + let testCases: [MockTestCase] = [ + MockTestCase(name: "fsetxattr", .noInterrupt, 42, "attr name", "nil", 0) { _ in + _ = try fd.setExtendedAttribute(named: "attr name", to: nil).get() + }, + + MockTestCase(name: "fsetxattr", .noInterrupt, 42, "attr name", "", 1024) { _ in + _ = try fd.setExtendedAttribute(named: "attr name", to: .init(buffer)).get() + }, + ] + + testCases.run() + } + + func test_fremovexattr() throws { + let fd = FileDescriptor(rawValue: 42) + let testCases: [MockTestCase] = [ + MockTestCase(name: "fremovexattr", .noInterrupt, 42, "attr name") { _ in + _ = try fd.removeExtendedAttribute("attr name").get() + } + ] + testCases.run() + } + + func test_rename() throws { + let testCases: [MockTestCase] = [ + MockTestCase(name: "rename", .noInterrupt, "old", "new") { _ in + try Syscall.rename(from: "old", to: "new").get() + } + ] + testCases.run() + } + + func test_renamex_np() throws { + #if canImport(Darwin) + let testCases: [MockTestCase] = [ + MockTestCase(name: "renamex_np", .noInterrupt, "foo", "bar", 0) { _ in + _ = try Syscall.rename(from: "foo", to: "bar", options: []).get() + }, + MockTestCase(name: "renamex_np", .noInterrupt, "bar", "baz", 2) { _ in + _ = try Syscall.rename(from: "bar", to: "baz", options: [.swap]).get() + }, + MockTestCase(name: "renamex_np", .noInterrupt, "bar", "baz", 4) { _ in + _ = try Syscall.rename(from: "bar", to: "baz", options: [.exclusive]).get() + }, + MockTestCase(name: "renamex_np", .noInterrupt, "bar", "baz", 2) { _ in + _ = try Syscall.rename(from: "bar", to: "baz", options: [.swap]).get() + }, + MockTestCase(name: "renamex_np", .noInterrupt, "bar", "baz", 6) { _ in + _ = try Syscall.rename(from: "bar", to: "baz", options: [.exclusive, .swap]).get() + }, + ] + testCases.run() + #else + throw XCTSkip("'renamex_np' is only supported on Darwin") + #endif + } + + func test_renameat2() throws { + #if canImport(Glibc) + let fd1 = FileDescriptor(rawValue: 13) + let fd2 = FileDescriptor(rawValue: 42) + + let testCases: [MockTestCase] = [ + MockTestCase(name: "renameat2", .noInterrupt, 13, "foo", 42, "bar", 0) { _ in + _ = try Syscall.rename( + from: "foo", + relativeTo: fd1, + to: "bar", + relativeTo: fd2, + flags: [] + ).get() + }, + MockTestCase(name: "renameat2", .noInterrupt, 13, "foo", 42, "bar", 1) { _ in + _ = try Syscall.rename( + from: "foo", + relativeTo: fd1, + to: "bar", + relativeTo: fd2, + flags: [.exclusive] + ).get() + }, + MockTestCase(name: "renameat2", .noInterrupt, 13, "foo", 42, "bar", 2) { _ in + _ = try Syscall.rename( + from: "foo", + relativeTo: fd1, + to: "bar", + relativeTo: fd2, + flags: [.swap] + ).get() + }, + MockTestCase(name: "renameat2", .noInterrupt, 13, "foo", 42, "bar", 3) { _ in + _ = try Syscall.rename( + from: "foo", + relativeTo: fd1, + to: "bar", + relativeTo: fd2, + flags: [.swap, .exclusive] + ).get() + }, + ] + testCases.run() + #else + throw XCTSkip("'renameat2' is only supported on Linux") + #endif + } + + func test_sendfile() throws { + #if canImport(Glibc) + let input = FileDescriptor(rawValue: 42) + let output = FileDescriptor(rawValue: 1) + + let testCases: [MockTestCase] = [ + MockTestCase(name: "sendfile", .noInterrupt, 1, 42, 0, 1024) { _ in + _ = try Syscall.sendfile(to: output, from: input, offset: 0, size: 1024).get() + }, + MockTestCase(name: "sendfile", .noInterrupt, 1, 42, 100, 512) { _ in + _ = try Syscall.sendfile(to: output, from: input, offset: 100, size: 512).get() + }, + ] + testCases.run() + #else + throw XCTSkip("'sendfile' is only supported on Linux") + #endif + } + + func test_fcopyfile() throws { + #if canImport(Darwin) + let input = FileDescriptor(rawValue: 42) + let output = FileDescriptor(rawValue: 1) + + let testCases: [MockTestCase] = [ + MockTestCase(name: "fcopyfile", .noInterrupt, 42, 1, "nil", 0) { _ in + try Libc.fcopyfile(from: input, to: output, state: nil, flags: 0).get() + } + ] + testCases.run() + #else + throw XCTSkip("'fcopyfile' is only supported on Darwin") + #endif + } + + func test_remove() throws { + let testCases: [MockTestCase] = [ + MockTestCase(name: "remove", .noInterrupt, "somepath") { _ in + try Libc.remove("somepath").get() + } + ] + testCases.run() + } + + func testValueOrErrno() throws { + let r1: Result = valueOrErrno(retryOnInterrupt: false) { + Errno._current = .addressInUse + return -1 + } + XCTAssertEqual(r1, .failure(.addressInUse)) + + var shouldInterrupt = true + let r2: Result = valueOrErrno(retryOnInterrupt: true) { + if shouldInterrupt { + shouldInterrupt = false + Errno._current = .interrupted + return -1 + } else { + Errno._current = .permissionDenied + return -1 + } + } + XCTAssertFalse(shouldInterrupt) + XCTAssertEqual(r2, .failure(.permissionDenied)) + + let r3: Result = valueOrErrno(retryOnInterrupt: false) { 0 } + XCTAssertEqual(r3, .success(0)) + } + + func testNothingOrErrno() throws { + let r1: Result = nothingOrErrno(retryOnInterrupt: false) { + Errno._current = .addressInUse + return -1 + } + + XCTAssertThrowsError(try r1.get()) { error in + XCTAssertEqual(error as? Errno, .addressInUse) + } + + var shouldInterrupt = true + let r2: Result = nothingOrErrno(retryOnInterrupt: true) { + if shouldInterrupt { + shouldInterrupt = false + Errno._current = .interrupted + return -1 + } else { + Errno._current = .permissionDenied + return -1 + } + } + XCTAssertFalse(shouldInterrupt) + XCTAssertThrowsError(try r2.get()) { error in + XCTAssertEqual(error as? Errno, .permissionDenied) + } + + let r3: Result = nothingOrErrno(retryOnInterrupt: false) { 0 } + XCTAssertNoThrow(try r3.get()) + } + + func testOptionalValueOrErrno() throws { + let r1: Result = optionalValueOrErrno(retryOnInterrupt: false) { + Errno._current = .addressInUse + return nil + } + XCTAssertEqual(r1, .failure(.addressInUse)) + + var shouldInterrupt = true + let r2: Result = optionalValueOrErrno(retryOnInterrupt: true) { + if shouldInterrupt { + shouldInterrupt = false + Errno._current = .interrupted + return nil + } else { + return "foo" + } + } + XCTAssertFalse(shouldInterrupt) + XCTAssertEqual(r2, .success("foo")) + + let r3: Result = optionalValueOrErrno(retryOnInterrupt: false) { "bar" } + XCTAssertEqual(r3, .success("bar")) + + let r4: Result = optionalValueOrErrno(retryOnInterrupt: false) { nil } + XCTAssertEqual(r4, .success(nil)) + } +} + +extension Array where Element == MockTestCase { + fileprivate func run() { + for testCase in self { + testCase.runAllTests() + } + } +} diff --git a/Tests/NIOFileSystemTests/XCTestExtensions.swift b/Tests/NIOFileSystemTests/XCTestExtensions.swift new file mode 100644 index 0000000000..3419d1bd21 --- /dev/null +++ b/Tests/NIOFileSystemTests/XCTestExtensions.swift @@ -0,0 +1,68 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOFileSystem +import XCTest + +func XCTAssertThrowsErrorAsync( + file: StaticString = #file, + line: UInt = #line, + expression: () async throws -> R, + onError: (Error) -> Void = { _ in } +) async { + do { + _ = try await expression() + XCTFail("expression did not throw", file: file, line: line) + } catch { + onError(error) + } +} + +func XCTAssertThrowsFileSystemError( + _ expression: @autoclosure () throws -> R, + file: StaticString = #file, + line: UInt = #line, + _ onError: (FileSystemError) -> Void = { _ in } +) { + XCTAssertThrowsError(try expression(), file: file, line: line) { error in + if let fsError = error as? FileSystemError { + onError(fsError) + } else { + XCTFail( + "Expected 'FileSystemError' but found '\(type(of: error))'", + file: file, + line: line + ) + } + } +} + +func XCTAssertSystemCallError( + _ error: (any Error)?, + name: String, + errno: Errno, + file: StaticString = #file, + line: UInt = #line +) { + guard let systemCallError = error as? FileSystemError.SystemCallError else { + return XCTFail( + "Expected FileSystemError.SystemCallError but found '\(type(of: error))'", + file: file, + line: line + ) + } + + XCTAssertEqual(systemCallError.systemCall, name, file: file, line: line) + XCTAssertEqual(systemCallError.errno, errno, file: file, line: line) +}