Merge pull request #2577 from apple/compiled-manifests-5.1

[5.1] Compile manifests instead of interpreting them

Sources/PackageLoading/ManifestLoader.swift

@@ -401,31 +401,21 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 
             // Compute the path to runtime we need to load.
             let runtimePath = self.runtimePath(for: manifestVersion).pathString
-            let interpreterFlags = self.interpreterFlags(for: manifestVersion)
+            let compilerFlags = self.interpreterFlags(for: manifestVersion)
 
             // FIXME: Workaround for the module cache bug that's been haunting Swift CI
             // <rdar://problem/48443680>
             let moduleCachePath = Process.env["SWIFTPM_MODULECACHE_OVERRIDE"] ?? Process.env["SWIFTPM_TESTS_MODULECACHE"]
 
+            let bootstrapArgs = self.bootstrapArgs()
+
             var cmd = [String]()
-          #if os(macOS)
-            // If enabled, use sandbox-exec on macOS. This provides some safety against
-            // arbitrary code execution when parsing manifest files. We only allow
-            // the permissions which are absolutely necessary for manifest parsing.
-            if isManifestSandboxEnabled {
-                let cacheDirs = [
-                    cacheDir,
-                    moduleCachePath.map{ AbsolutePath($0) }
-                ].compactMap{$0}
-                cmd += ["sandbox-exec", "-p", sandboxProfile(cacheDirs)]
-            }
-          #endif
             cmd += [resources.swiftCompiler.pathString]
-            cmd += ["--driver-mode=swift"]
-            cmd += bootstrapArgs()
+            cmd += bootstrapArgs.compileFlags
             cmd += verbosity.ccArgs
             cmd += ["-L", runtimePath, "-lPackageDescription"]
-            cmd += interpreterFlags
+            cmd += ["-Xlinker", "-rpath", "-Xlinker", runtimePath]
+            cmd += compilerFlags
             if let moduleCachePath = moduleCachePath {
                 cmd += ["-module-cache-path", moduleCachePath]
             }
@@ -441,25 +431,55 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 
             cmd += [manifestPath.pathString]
 
-            // Create and open a temporary file to write json to.
-            let file = try TemporaryFile()
+            let tmpDir = try TemporaryDirectory(removeTreeOnDeinit: true)
+            let compiledManifest = tmpDir.path.appending(component: "manifest")
+            // Set path to compiled manifest executable.
+            cmd += ["-o", compiledManifest.pathString]
+
+            // Compile the manifest.
+            let compilerResult = try Process.popen(arguments: cmd)
+            let compilerOutput = try (compilerResult.utf8Output() + compilerResult.utf8stderrOutput()).spm_chuzzle()
+            manifestParseResult.compilerOutput = compilerOutput
+
+            // Return now if there was an error.
+            if compilerResult.exitStatus != .terminated(code: 0) {
+                return
+            }
+
             // Pass the fd in arguments.
-            cmd += ["-fileno", "\(file.fileHandle.fileDescriptor)"]
+            cmd = [compiledManifest.pathString, "-fileno", "1"]
+
+          #if os(macOS)
+            // If enabled, use sandbox-exec on macOS. This provides some safety against
+            // arbitrary code execution when parsing manifest files. We only allow
+            // the permissions which are absolutely necessary for manifest parsing.
+            if isManifestSandboxEnabled {
+                let cacheDirectories = [
+                    cacheDir,
+                    moduleCachePath.map({ AbsolutePath($0) })
+                ].compactMap({ $0 })
+                let profile = sandboxProfile(cacheDirectories)
+                cmd += ["sandbox-exec", "-p", profile]
+            }
+          #endif
+
+            // Setup the runtime environment for bootstrapping.
+            var env = Process.env
+            if !bootstrapArgs.runtimeFlags.isEmpty {
+                env["LD_LIBRARY_PATH"] = bootstrapArgs.runtimeFlags.joined(separator: ":")
+            }
 
             // Run the command.
-            let result = try Process.popen(arguments: cmd)
-            let output = try (result.utf8Output() + result.utf8stderrOutput()).spm_chuzzle()
-            manifestParseResult.compilerOutput = output
+            let runResult = try Process.popen(arguments: cmd, environment: env)
+            let runOutput = try (runResult.utf8Output() + runResult.utf8stderrOutput()).spm_chuzzle()
 
             // Return now if there was an error.
-            if result.exitStatus != .terminated(code: 0) {
+            if runResult.exitStatus != .terminated(code: 0) {
+                manifestParseResult.errorOutput = runOutput
                 return
             }
 
-            guard let json = try localFileSystem.readFileContents(file.path).validDescription else {
-                throw StringError("the manifest has invalid encoding")
-            }
-            manifestParseResult.parsedManifest = json
+            manifestParseResult.parsedManifest = runOutput
         }
 
         var manifestParseResult = ManifestParseResult()
@@ -478,32 +498,34 @@ public final class ManifestLoader: ManifestLoaderProtocol {
     }
 
     /// Returns the extra manifest args required during SwiftPM's own bootstrap.
-    private func bootstrapArgs() -> [String] {
+    private func bootstrapArgs() -> (compileFlags: [String], runtimeFlags: [String]) {
       #if !os(Linux)
-        return []
+        return ([], [])
       #else
         // The Linux bots require extra arguments in order to locate the corelibs.
         // We can potentially drop this by installing some stable linux toolchain
         // after Swift gets ABI and module stability.
         //
         // Compute if SwiftPM is bootstrapping.
         let env = ProcessInfo.processInfo.environment
-        guard env.keys.contains("SWIFTPM_BOOTSTRAP") else { return [] }
+        guard env.keys.contains("SWIFTPM_BOOTSTRAP") else { return ([], []) }
         guard let buildPathStr = env["SWIFTPM_BUILD_DIR"], let buildPath = try? AbsolutePath(validating: buildPathStr) else {
-            return []
+            return ([], [])
         }
 
         // Construct the required search paths relative to the build directory.
         let libdir = buildPath.appending(RelativePath(".bootstrap/lib/swift/linux"))
         let incdir = libdir.appending(component: "x86_64")
         let dispatchIncdir = incdir.appending(component: "dispatch")
 
-        return [
+        let compileFlags = [
             "-I\(incdir)",
             "-I\(dispatchIncdir)",
             "-L\(libdir)",
             "-Xcc", "-F\(incdir)",
         ]
+
+        return (compileFlags, [libdir.pathString])
       #endif
     }
 

Utilities/bootstrap

@@ -908,7 +908,10 @@ def build_llbuild(args):
     cmake_cache_path = os.path.join(llbuild_build_dir, "CMakeCache.txt")
     if not os.path.isfile(cmake_cache_path) or not args.swiftc_path in open(cmake_cache_path).read():
         mkdir_p(llbuild_build_dir)
-        cmd = ["cmake", "-G", "Ninja", "-DCMAKE_BUILD_TYPE:=Debug", "-DCMAKE_C_COMPILER:=clang", "-DCMAKE_CXX_COMPILER:=clang++", "-DLLBUILD_SUPPORT_BINDINGS:=Swift", "-DSWIFTC_EXECUTABLE:=%s" % (args.swiftc_path), llbuild_source_dir]
+        swift_flags = ""
+        if args.sysroot:
+            swift_flags = "-sdk %s" % args.sysroot
+        cmd = ["cmake", "-G", "Ninja", "-DCMAKE_BUILD_TYPE:=Debug", "-DCMAKE_C_COMPILER:=clang", "-DCMAKE_CXX_COMPILER:=clang++", "-DLLBUILD_SUPPORT_BINDINGS:=Swift", "-DCMAKE_Swift_FLAGS:=%s" % (swift_flags), "-DSWIFTC_EXECUTABLE:=%s" % (args.swiftc_path), llbuild_source_dir]
         subprocess.check_call(cmd, cwd=llbuild_build_dir)
 
     # Build.