Skip to content

Propagate build jobs through plugin compilation #9541

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kateinoigakukun wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Propagate build jobs through plugin compilation #9541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Sources/Build/BuildOperation.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,6 +624,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
sourceFiles: plugin.sources.paths,
pluginName: plugin.moduleName,
toolsVersion: plugin.toolsVersion,
workers: config.toolsBuildParameters.workers,
observabilityScope: self.observabilityScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate
Expand Down
1 change: 1 addition & 0 deletions Sources/Build/BuildPlan/BuildPlan.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -817,6 +817,7 @@ extension BuildPlan {
pluginGeneratedResources: pluginDerivedResources.map(\.path)
),
buildEnvironment: buildParameters.buildEnvironment,
workers: buildParameters.workers,
scriptRunner: configuration.scriptRunner,
workingDirectory: package.path,
outputDirectory: pluginOutputDir,
Expand Down
1 change: 1 addition & 0 deletions Sources/Commands/PackageCommands/PluginCommand.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,7 @@ struct PluginCommand: AsyncSwiftCommand {
let _ = try await pluginTarget.invoke(
action: .performCommand(package: package, arguments: arguments),
buildEnvironment: buildEnvironment,
workers: buildParameters.workers,
scriptRunner: pluginScriptRunner,
workingDirectory: swiftCommandState.originalWorkingDirectory,
outputDirectory: outputDir,
Expand Down
2 changes: 1 addition & 1 deletion Sources/CoreCommands/Options.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,7 @@ public struct BuildOptions: ParsableArguments {

/// The number of jobs for llbuild to start (aka the number of schedulerLanes)
@Option(name: .shortAndLong, help: "The number of jobs to spawn in parallel during the build process.")
public var jobs: UInt32?
public var jobs: UInt32 = UInt32(ProcessInfo.processInfo.activeProcessorCount)

/// Whether to use the integrated Swift driver rather than shelling out
/// to a separate process.
Expand Down
2 changes: 1 addition & 1 deletion Sources/CoreCommands/SwiftCommandState.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -936,7 +936,7 @@ public final class SwiftCommandState {
buildSystemKind: options.build.buildSystem,
pkgConfigDirectories: options.locations.pkgConfigDirectories,
architectures: options.build.architectures,
workers: options.build.jobs ?? UInt32(ProcessInfo.processInfo.activeProcessorCount),
workers: options.build.jobs,
shouldCreateDylibForDynamicProducts: !self.options.build.shouldBuildDylibsAsFrameworks,
sanitizers: options.build.enabledSanitizers,
indexStoreMode: options.build.indexStoreMode.buildParameter,
Expand Down
6 changes: 6 additions & 0 deletions Sources/SPMBuildCore/Plugins/DefaultPluginScriptRunner.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner, Cancellable {
writableDirectories: [Basics.AbsolutePath],
readOnlyDirectories: [Basics.AbsolutePath],
allowNetworkConnections: [SandboxNetworkPermission],
workers: UInt32,
fileSystem: FileSystem,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
Expand All @@ -77,6 +78,7 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner, Cancellable {
sourceFiles: sourceFiles,
pluginName: pluginName,
toolsVersion: toolsVersion,
workers: workers,
observabilityScope: observabilityScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate,
Expand Down Expand Up @@ -119,6 +121,7 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner, Cancellable {
sourceFiles: [Basics.AbsolutePath],
pluginName: String,
toolsVersion: ToolsVersion,
workers: UInt32,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
delegate: PluginScriptCompilerDelegate,
Expand Down Expand Up @@ -221,6 +224,9 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner, Cancellable {
// Parse the plugin as a library so that `@main` is supported even though there might be only a single source file.
commandLine += ["-parse-as-library"]

// Enable concurrent compilation.
commandLine += ["-j\(workers)"]

// Ask the compiler to create a diagnostics file (we'll put it next to the executable).
commandLine += ["-Xfrontend", "-serialize-diagnostics-path", "-Xfrontend", diagFilePath.pathString]

Expand Down
8 changes: 8 additions & 0 deletions Sources/SPMBuildCore/Plugins/PluginInvocation.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ extension PluginModule {
public func invoke(
action: PluginAction,
buildEnvironment: BuildEnvironment,
workers: UInt32,
scriptRunner: PluginScriptRunner,
workingDirectory: AbsolutePath,
outputDirectory: AbsolutePath,
Expand All @@ -83,6 +84,7 @@ extension PluginModule {
self.invoke(
action: action,
buildEnvironment: buildEnvironment,
workers: workers,
scriptRunner: scriptRunner,
workingDirectory: workingDirectory,
outputDirectory: outputDirectory,
Expand Down Expand Up @@ -132,6 +134,7 @@ extension PluginModule {
public func invoke(
action: PluginAction,
buildEnvironment: BuildEnvironment,
workers: UInt32,
scriptRunner: PluginScriptRunner,
workingDirectory: AbsolutePath,
outputDirectory: AbsolutePath,
Expand Down Expand Up @@ -419,6 +422,7 @@ extension PluginModule {
writableDirectories: writableDirectories,
readOnlyDirectories: readOnlyDirectories,
allowNetworkConnections: allowNetworkConnections,
workers: workers,
fileSystem: fileSystem,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue,
Expand All @@ -442,6 +446,7 @@ extension PluginModule {
module: ResolvedModule,
action: PluginAction,
buildEnvironment: BuildEnvironment,
workers: UInt32,
scriptRunner: PluginScriptRunner,
workingDirectory: AbsolutePath,
outputDirectory: AbsolutePath,
Expand All @@ -461,6 +466,7 @@ extension PluginModule {
module: module,
action: action,
buildEnvironment: buildEnvironment,
workers: workers,
scriptRunner: scriptRunner,
workingDirectory: workingDirectory,
outputDirectory: outputDirectory,
Expand Down Expand Up @@ -488,6 +494,7 @@ extension PluginModule {
module: ResolvedModule,
action: PluginAction,
buildEnvironment: BuildEnvironment,
workers: UInt32,
scriptRunner: PluginScriptRunner,
workingDirectory: AbsolutePath,
outputDirectory: AbsolutePath,
Expand Down Expand Up @@ -531,6 +538,7 @@ extension PluginModule {
self.invoke(
action: action,
buildEnvironment: buildEnvironment,
workers: workers,
scriptRunner: scriptRunner,
workingDirectory: workingDirectory,
outputDirectory: outputDirectory,
Expand Down
4 changes: 4 additions & 0 deletions Sources/SPMBuildCore/Plugins/PluginScriptRunner.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public protocol PluginScriptRunner {
sourceFiles: [Basics.AbsolutePath],
pluginName: String,
toolsVersion: ToolsVersion,
workers: UInt32,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
delegate: PluginScriptCompilerDelegate,
Expand All @@ -53,6 +54,7 @@ public protocol PluginScriptRunner {
writableDirectories: [Basics.AbsolutePath],
readOnlyDirectories: [Basics.AbsolutePath],
allowNetworkConnections: [SandboxNetworkPermission],
workers: UInt32,
fileSystem: FileSystem,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
Expand All @@ -70,6 +72,7 @@ public extension PluginScriptRunner {
sourceFiles: [Basics.AbsolutePath],
pluginName: String,
toolsVersion: ToolsVersion,
workers: UInt32,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
delegate: PluginScriptCompilerDelegate
Expand All @@ -79,6 +82,7 @@ public extension PluginScriptRunner {
sourceFiles: sourceFiles,
pluginName: pluginName,
toolsVersion: toolsVersion,
workers: workers,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue,
delegate: delegate,
Expand Down
1 change: 1 addition & 0 deletions Sources/SwiftBuildSupport/PIFBuilder.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,7 @@ public final class PIFBuilder {
pluginGeneratedResources: pluginDerivedResources.map(\.path)
),
buildEnvironment: buildParameters.buildEnvironment,
workers: buildParameters.workers,
scriptRunner: pluginScriptRunner,
workingDirectory: package.path,
outputDirectory: pluginOutputDir,
Expand Down
1 change: 1 addition & 0 deletions Sources/SwiftBuildSupport/SwiftBuildSystem.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
sourceFiles: plugin.sources.paths,
pluginName: plugin.moduleName,
toolsVersion: plugin.toolsVersion,
workers: self.buildParameters.workers,
observabilityScope: observabilityScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate
Expand Down
8 changes: 8 additions & 0 deletions Tests/BuildTests/PluginInvocationTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ final class PluginInvocationTests: XCTestCase {
sourceFiles: [AbsolutePath],
pluginName: String,
toolsVersion: ToolsVersion,
workers: UInt32,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
delegate: PluginScriptCompilerDelegate,
Expand All @@ -159,6 +160,7 @@ final class PluginInvocationTests: XCTestCase {
writableDirectories: [AbsolutePath],
readOnlyDirectories: [AbsolutePath],
allowNetworkConnections: [SandboxNetworkPermission],
workers: UInt32,
fileSystem: FileSystem,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
Expand Down Expand Up @@ -384,6 +386,7 @@ final class PluginInvocationTests: XCTestCase {
sourceFiles: buildToolPlugin.sources.paths,
pluginName: buildToolPlugin.name,
toolsVersion: buildToolPlugin.apiVersion,
workers: 1,
observabilityScope: observability.topScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate
Expand All @@ -393,6 +396,7 @@ final class PluginInvocationTests: XCTestCase {
XCTAssert(result.succeeded == false)
XCTAssert(result.cached == false)
XCTAssert(result.commandLine.contains(result.executableFile.pathString), "\(result.commandLine)")
XCTAssert(result.commandLine.contains("-j1"), "expected workers flag in \(result.commandLine)")
XCTAssert(result.executableFile.components.contains("plugin-cache"), "\(result.executableFile.pathString)")
XCTAssert(result.compilerOutput.contains("error: missing return"), "\(result.compilerOutput)")
XCTAssert(result.diagnosticsFile.suffix == ".dia", "\(result.diagnosticsFile.pathString)")
Expand Down Expand Up @@ -436,6 +440,7 @@ final class PluginInvocationTests: XCTestCase {
sourceFiles: buildToolPlugin.sources.paths,
pluginName: buildToolPlugin.name,
toolsVersion: buildToolPlugin.apiVersion,
workers: 1,
observabilityScope: observability.topScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate
Expand Down Expand Up @@ -486,6 +491,7 @@ final class PluginInvocationTests: XCTestCase {
sourceFiles: buildToolPlugin.sources.paths,
pluginName: buildToolPlugin.name,
toolsVersion: buildToolPlugin.apiVersion,
workers: 1,
observabilityScope: observability.topScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate
Expand Down Expand Up @@ -548,6 +554,7 @@ final class PluginInvocationTests: XCTestCase {
sourceFiles: buildToolPlugin.sources.paths,
pluginName: buildToolPlugin.name,
toolsVersion: buildToolPlugin.apiVersion,
workers: 1,
observabilityScope: observability.topScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate
Expand Down Expand Up @@ -601,6 +608,7 @@ final class PluginInvocationTests: XCTestCase {
sourceFiles: buildToolPlugin.sources.paths,
pluginName: buildToolPlugin.name,
toolsVersion: buildToolPlugin.apiVersion,
workers: 1,
observabilityScope: observability.topScope,
callbackQueue: DispatchQueue.sharedConcurrent,
delegate: delegate
Expand Down
2 changes: 2 additions & 0 deletions Tests/FunctionalTests/PluginTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,7 @@ struct PluginTests {
plugin.invoke(
action: .performCommand(package: package, arguments: arguments),
buildEnvironment: BuildEnvironment(platform: .macOS, configuration: .debug),
workers: 1,
scriptRunner: scriptRunner,
workingDirectory: package.path,
outputDirectory: pluginDir.appending("output"),
Expand Down Expand Up @@ -1112,6 +1113,7 @@ struct PluginTests {
_ = try await plugin.invoke(
action: .performCommand(package: package, arguments: []),
buildEnvironment: BuildEnvironment(platform: .macOS, configuration: .debug),
workers: 1,
scriptRunner: scriptRunner,
workingDirectory: package.path,
outputDirectory: pluginDir.appending("output"),
Expand Down
Loading