Add NIOFilesystem (#2615)

Motivation:

I/O is typically blocking, but options for interacting with the
filesystem using `async` APIs are limited. This change adds a new module
to NIO, `NIOFileSystem`. At the moment this is exposed as the
`_NIOFileSystem` product while users get a chance to try it out and
provide feedback.

During this time the API of `NIOFileSystem` is subject to change.
Breaking changes will come in _minor_ NIO releases and additive changes
may come in _patch_ releases of NIO.

Modifications:

- Add `NIOFileSystem`, `NIOFileSystemFoundationCompat`,
  `NIOFileSystemTests`, `NIOFileSystemIntegrationTests`, and
  `NIOFileSystemFoundationCompatTests`.
- Update docs
- Update CNIOLinux and CNIODarwin shims

Result:

File system can be accessed via `async` APIs

.spi.yml

@@ -1,4 +1,4 @@
 version: 1
 builder:
   configs:
-    - documentation_targets: [NIO, NIOConcurrencyHelpers, NIOCore, NIOEmbedded, NIOFoundationCompat, NIOHTTP1, NIOPosix, NIOTLS, NIOWebSocket, NIOTestUtils]
+    - documentation_targets: [NIO, NIOConcurrencyHelpers, NIOCore, NIOEmbedded, NIOFoundationCompat, NIOHTTP1, NIOPosix, NIOTLS, NIOWebSocket, NIOTestUtils, NIOFileSystem]

NOTICE.txt

@@ -96,3 +96,12 @@ This product contains a derivation of "_TinyArray.swift" from SwiftCertificates.
     * https://www.apache.org/licenses/LICENSE-2.0
   * HOMEPAGE:
     * https://github.com/apple/swift-certificates
+
+---
+
+This product contains a derivation of the mocking infrastructure from Swift System.
+
+  * LICENSE (Apache License 2.0):
+    * https://www.apache.org/licenses/LICENSE-2.0
+  * HOMEPAGE:
+    * https://github.com/apple/swift-system

Package.swift

@@ -17,6 +17,7 @@ import PackageDescription
 
 let swiftAtomics: PackageDescription.Target.Dependency = .product(name: "Atomics", package: "swift-atomics")
 let swiftCollections: PackageDescription.Target.Dependency = .product(name: "DequeModule", package: "swift-collections")
+let swiftSystem: PackageDescription.Target.Dependency = .product(name: "SystemPackage", package: "swift-system")
 
 
 let package = Package(
@@ -33,10 +34,13 @@ let package = Package(
         .library(name: "NIOFoundationCompat", targets: ["NIOFoundationCompat"]),
         .library(name: "NIOWebSocket", targets: ["NIOWebSocket"]),
         .library(name: "NIOTestUtils", targets: ["NIOTestUtils"]),
+        .library(name: "_NIOFileSystem", targets: ["NIOFileSystem"]),
+        .library(name: "_NIOFileSystemFoundationCompat", targets: ["NIOFileSystemFoundationCompat"]),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-atomics.git", from: "1.1.0"),
         .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.2"),
+        .package(url: "https://github.com/apple/swift-system.git", from: "1.2.0"),
         .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"),
     ],
     targets: [
@@ -185,6 +189,26 @@ let package = Package(
                 swiftAtomics,
             ]
         ),
+        .target(
+            name: "NIOFileSystem",
+            dependencies: [
+                "NIOCore",
+                "CNIOLinux",
+                "CNIODarwin",
+                swiftAtomics,
+                swiftCollections,
+                swiftSystem,
+            ],
+            swiftSettings: [
+                .define("ENABLE_MOCKING", .when(configuration: .debug))
+            ]
+        ),
+        .target(
+            name: "NIOFileSystemFoundationCompat",
+            dependencies: [
+                "NIOFileSystem",
+            ]
+        ),
 
         // MARK: - Examples
 
@@ -434,5 +458,39 @@ let package = Package(
             name: "NIOSingletonsTests",
             dependencies: ["NIOCore", "NIOPosix"]
         ),
+        .testTarget(
+            name: "NIOFileSystemTests",
+            dependencies: [
+                "NIOCore",
+                "NIOFileSystem",
+                swiftAtomics,
+                swiftCollections,
+                swiftSystem,
+            ],
+            swiftSettings: [
+                .define("ENABLE_MOCKING", .when(configuration: .debug))
+            ]
+        ),
+        .testTarget(
+            name: "NIOFileSystemIntegrationTests",
+            dependencies: [
+                "NIOCore",
+                "NIOFileSystem",
+                "NIOFoundationCompat",
+            ],
+            exclude: [
+                // Contains known files and directory structures used
+                // for the integration tests. Exclude the whole tree from
+                // the build.
+                "Test Data",
+            ]
+        ),
+        .testTarget(
+            name: "NIOFileSystemFoundationCompatTests",
+            dependencies: [
+                "NIOFileSystem",
+                "NIOFileSystemFoundationCompat",
+            ]
+        )
     ]
 )

README.md

@@ -34,6 +34,7 @@ Within this repository we have a number of products that provide different funct
 - `NIOHTTP1`. This provides a low-level HTTP/1.1 protocol implementation.
 - `NIOWebSocket`. This provides a low-level WebSocket protocol implementation.
 - `NIOTestUtils`. This provides a number of helpers for testing projects that use SwiftNIO.
+- `NIOFileSystem`. This provides `async` APIs for interacting with the file system.
 
 ### Protocol Implementations
 
@@ -354,16 +355,16 @@ dnf install swift-lang /usr/bin/nc /usr/bin/lsof /usr/bin/shasum
 
 ### Benchmarks
 
-Benchmarks for `swift-nio` are in a separate Swift Package in the `Benchmarks` subfolder of this repository. 
+Benchmarks for `swift-nio` are in a separate Swift Package in the `Benchmarks` subfolder of this repository.
 They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin.
 Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics.
-An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. 
+An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`.
 Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking:
 ```
 swift package benchmark
 ```
 
-For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). 
+For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark).
 
 [ch]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channelhandler
 [c]: https://swiftpackageindex.com/apple/swift-nio/main/documentation/niocore/channel

Sources/CNIODarwin/include/CNIODarwin.h

@@ -19,6 +19,10 @@
 #include <sys/vsock.h>
 #include <sys/ioctl.h>
 #include <time.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fts.h>
 
 // Darwin platforms do not have a sendmmsg implementation available to them. This C module
 // provides a shim that implements sendmmsg on top of sendmsg. It also provides a shim for
@@ -54,5 +58,7 @@ size_t CNIODarwin_CMSG_SPACE(size_t);
 
 extern const unsigned long CNIODarwin_IOCTL_VM_SOCKETS_GET_LOCAL_CID;
 
+const char* CNIODarwin_dirent_dname(struct dirent* ent);
+
 #endif  // __APPLE__
 #endif  // C_NIO_DARWIN_H

Sources/CNIODarwin/shim.c

@@ -21,6 +21,7 @@
 #include <assert.h>
 #include <netinet/ip.h>
 #include <netinet/in.h>
+#include "dirent.h"
 
 int CNIODarwin_sendmmsg(int sockfd, CNIODarwin_mmsghdr *msgvec, unsigned int vlen, int flags) {
     // Some quick error checking. If vlen can't fit into int, we bail.
@@ -92,4 +93,8 @@ const int CNIODarwin_IPV6_PKTINFO = IPV6_PKTINFO;
 
 const unsigned long CNIODarwin_IOCTL_VM_SOCKETS_GET_LOCAL_CID = IOCTL_VM_SOCKETS_GET_LOCAL_CID;
 
+const char* CNIODarwin_dirent_dname(struct dirent* ent) {
+    return ent->d_name;
+}
+
 #endif  // __APPLE__

Sources/CNIOLinux/include/CNIOLinux.h

@@ -22,6 +22,9 @@
 #include <sys/sysinfo.h>
 #include <sys/socket.h>
 #include <sys/utsname.h>
+#include <sys/xattr.h>
+#include <sys/types.h>
+#include <sys/stat.h>
 #include <sched.h>
 #include <stdbool.h>
 #include <errno.h>
@@ -30,6 +33,10 @@
 #include <netinet/udp.h>
 #include "liburing_nio.h"
 #include <linux/vm_sockets.h>
+#include <fcntl.h>
+#include <fts.h>
+#include <stdio.h>
+#include <dirent.h>
 
 #if __has_include(<linux/mptcp.h>)
 #include <linux/mptcp.h>
@@ -117,5 +124,14 @@ int CNIOLinux_system_info(struct utsname* uname_data);
 
 extern const unsigned long CNIOLinux_IOCTL_VM_SOCKETS_GET_LOCAL_CID;
 
+const char* CNIOLinux_dirent_dname(struct dirent* ent);
+
+int CNIOLinux_renameat2(int oldfd, const char* old, int newfd, const char* newName, unsigned int flags);
+
+extern const int CNIOLinux_O_TMPFILE;
+extern const int CNIOLinux_AT_EMPTY_PATH;
+extern const unsigned int CNIOLinux_RENAME_NOREPLACE;
+extern const unsigned int CNIOLinux_RENAME_EXCHANGE;
+
 #endif
 #endif

Sources/CNIOLinux/shim.c

@@ -188,4 +188,17 @@ int CNIOLinux_system_info(struct utsname* uname_data) {
 
 const unsigned long CNIOLinux_IOCTL_VM_SOCKETS_GET_LOCAL_CID = IOCTL_VM_SOCKETS_GET_LOCAL_CID;
 
+const char* CNIOLinux_dirent_dname(struct dirent* ent) {
+  return ent->d_name;
+}
+
+int CNIOLinux_renameat2(int oldfd, const char* old, int newfd, const char* newName, unsigned int flags) {
+  return renameat2(oldfd, old, newfd, newName, flags);
+}
+
+const int CNIOLinux_O_TMPFILE = O_TMPFILE;
+const unsigned int CNIOLinux_RENAME_NOREPLACE = RENAME_NOREPLACE;
+const unsigned int CNIOLinux_RENAME_EXCHANGE = RENAME_EXCHANGE;
+const int CNIOLinux_AT_EMPTY_PATH = AT_EMPTY_PATH;
+
 #endif

Sources/NIO/Docs.docc/index.md

@@ -35,6 +35,7 @@ SwiftNIO has a number of products that provide different functionality. This pac
 - [NIOHTTP1][module-http1]. This provides a low-level HTTP/1.1 protocol implementation.
 - [NIOWebSocket][module-websocket]. This provides a low-level WebSocket protocol implementation.
 - [NIOTestUtils][module-test-utilities]. This provides a number of helpers for testing projects that use SwiftNIO.
+- [NIOFileSystem][module-filesystem]. This provides `async` APIs for interacting with the file system.
 
 ### Conceptual Overview
 
@@ -160,6 +161,7 @@ The core SwiftNIO repository will contain a few extremely important protocol imp
 [module-tls]: ./NIOTLS
 [module-websocket]: ./NIOWebSocket
 [module-test-utilities]: ./NIOTestUtils
+[module-filesystem]: ./NIOFileSystem
 
 [ch]: ./NIOCore/ChannelHandler
 [c]: ./NIOCore/Channel