diff --git a/Cargo.toml b/Cargo.toml index 36dbed8218a..087da74e3ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -251,6 +251,11 @@ name = "cargo" test = false doc = false +[[bin]] +name = "cargo-batch" +test = false +doc = false + [features] vendored-openssl = ["openssl/vendored"] vendored-libgit2 = ["libgit2-sys/vendored"] diff --git a/src/bin/cargo-batch.rs b/src/bin/cargo-batch.rs new file mode 100644 index 00000000000..92a662c2cb3 --- /dev/null +++ b/src/bin/cargo-batch.rs @@ -0,0 +1,300 @@ +#![warn(rust_2018_idioms)] // while we're getting used to 2018 +#![allow(clippy::all)] +#![warn(clippy::needless_borrow)] +#![warn(clippy::redundant_clone)] + +use cargo::core::compiler::{ + unit_graph, BuildContext, BuildRunner, DefaultExecutor, Executor, UnitInterner, +}; +use cargo::core::shell::Shell; +use cargo::core::Workspace; +use cargo::ops::CompileOptions; +use cargo::util::network::http::{http_handle, needs_custom_http_transport}; +use cargo::util::{command_prelude, CliResult, GlobalContext}; +use std::env; +use std::sync::Arc; + +use crate::command_prelude::*; + +fn main() { + setup_logger(); + + let mut config = match GlobalContext::default() { + Ok(cfg) => cfg, + Err(e) => { + let mut shell = Shell::new(); + cargo::exit_with_error(e.into(), &mut shell) + } + }; + + let result = main2(&mut config); + + match result { + Err(e) => cargo::exit_with_error(e, &mut *config.shell()), + Ok(()) => {} + } +} + +fn main2(config: &mut GlobalContext) -> CliResult { + let args: Vec<_> = env::args().collect(); + let mut subargs = args.split(|x| *x == "---"); + + let mut global_args = subargs.next().unwrap().to_vec(); + if global_args.len() >= 2 && global_args[1] == "batch" { + global_args.remove(1); + } + let global_args = Command::new("cargo-batch") + .arg_unit_graph() + .arg_target_dir() + .arg( + opt( + "verbose", + "Use verbose output (-vv very verbose/build.rs output)", + ) + .short('v') + .action(ArgAction::Count) + .global(true), + ) + .arg_silent_suggestion() + .arg( + opt("color", "Coloring: auto, always, never") + .value_name("WHEN") + .global(true), + ) + .arg( + flag("frozen", "Require Cargo.lock and cache are up to date") + .help_heading(heading::MANIFEST_OPTIONS) + .global(true), + ) + .arg( + flag("locked", "Require Cargo.lock is up to date") + .help_heading(heading::MANIFEST_OPTIONS) + .global(true), + ) + .arg( + flag("offline", "Run without accessing the network") + .help_heading(heading::MANIFEST_OPTIONS) + .global(true), + ) + .arg(multi_opt("config", "KEY=VALUE", "Override a configuration value").global(true)) + .arg( + Arg::new("unstable-features") + .help("Unstable (nightly-only) flags to Cargo, see 'cargo -Z help' for details") + .short('Z') + .value_name("FLAG") + .action(ArgAction::Append) + .global(true), + ) + .try_get_matches_from(global_args)?; + + config_configure(config, &global_args)?; + init_git_transports(config); + + let unit_graph = global_args.flag("unit-graph"); + + struct CommandState<'a> { + ws: Workspace<'a>, + compile_opts: CompileOptions, + } + + let mut cmds = Vec::new(); + for args in subargs { + let cli = build_cli(); + let args = cli.try_get_matches_from(args)?; + //println!("args opts: {:#?}", args); + + let ws = args.workspace(config)?; + + let mut compile_opts = args.compile_options( + config, + CompileMode::Build, + Some(&ws), + ProfileChecking::Custom, + )?; + if let Some(out_dir) = args.value_of_path("out-dir", config) { + compile_opts.build_config.export_dir = Some(out_dir); + } else if let Some(out_dir) = config.build_config()?.out_dir.as_ref() { + let out_dir = out_dir.resolve_path(config); + compile_opts.build_config.export_dir = Some(out_dir); + } + //if compile_opts.build_config.export_dir.is_some() { + // config + // .cli_unstable() + // .fail_if_stable_opt("--out-dir", 6790)?; + //} + + //println!("compile opts: {:#?}", compile_opts); + cmds.push(CommandState { ws, compile_opts }); + } + + let interner = UnitInterner::new(); + let mut merged_bcx: Option> = None; + + for cmd in &cmds { + let mut bcx = cargo::ops::create_bcx(&cmd.ws, &cmd.compile_opts, &interner).unwrap(); + if let Some(export_dir) = &cmd.compile_opts.build_config.export_dir { + for root in &bcx.roots { + bcx.unit_export_dirs + .insert(root.clone(), export_dir.clone()); + } + } + + if let Some(merged_bcx) = &mut merged_bcx { + // merge!!! + merged_bcx.unit_graph.extend(bcx.unit_graph); + merged_bcx.roots.extend(bcx.roots); + merged_bcx.unit_export_dirs.extend(bcx.unit_export_dirs); + merged_bcx.all_kinds.extend(bcx.all_kinds); + merged_bcx + .target_data + .target_config + .extend(bcx.target_data.target_config); + merged_bcx + .target_data + .target_info + .extend(bcx.target_data.target_info); + merged_bcx.packages.packages.extend(bcx.packages.packages); + merged_bcx + .packages + .sources + .borrow_mut() + .add_source_map(bcx.packages.sources.into_inner()); + } else { + merged_bcx = Some(bcx) + } + } + + let bcx = merged_bcx.unwrap(); + + if unit_graph { + unit_graph::emit_serialized_unit_graph(&bcx.roots, &bcx.unit_graph, bcx.ws.gctx())?; + return Ok(()); + } + + // util::profile disappeared between cargo 1.76 and cargo 1.78 + // let _p = cargo::util::profile::start("compiling"); + let cx = BuildRunner::new(&bcx)?; + let exec: Arc = Arc::new(DefaultExecutor); + cx.compile(&exec)?; + + Ok(()) +} + +fn config_configure(config: &mut GlobalContext, args: &ArgMatches) -> CliResult { + let arg_target_dir = &args.value_of_path("target-dir", config); + let verbose = args.verbose(); + // quiet is unusual because it is redefined in some subcommands in order + // to provide custom help text. + let quiet = args.flag("quiet"); + let color = args.get_one::("color").map(String::as_str); + let frozen = args.flag("frozen"); + let locked = args.flag("locked"); + let offline = args.flag("offline"); + let mut unstable_flags = vec![]; + if let Some(values) = args.get_many::("unstable-features") { + unstable_flags.extend(values.cloned()); + } + let mut config_args = vec![]; + if let Some(values) = args.get_many::("config") { + config_args.extend(values.cloned()); + } + config.configure( + verbose, + quiet, + color, + frozen, + locked, + offline, + arg_target_dir, + &unstable_flags, + &config_args, + )?; + Ok(()) +} + +pub fn build_cli() -> Command { + subcommand("build") + .about("Compile a local package and all of its dependencies") + .arg_ignore_rust_version() + .arg_future_incompat_report() + .arg_message_format() + .arg_silent_suggestion() + .arg_package_spec( + "Package to build (see `cargo help pkgid`)", + "Build all packages in the workspace", + "Exclude packages from the build", + ) + .arg_targets_all( + "Build only this package's library", + "Build only the specified binary", + "Build all binaries", + "Build only the specified example", + "Build all examples", + "Build only the specified test target", + "Build all tests", + "Build only the specified bench target", + "Build all benches", + "Build all targets", + ) + .arg_features() + .arg_release("Build artifacts in release mode, with optimizations") + .arg_profile("Build artifacts with the specified profile") + .arg_parallel() + .arg_target_triple("Build for the target triple") + .arg( + opt( + "out-dir", + "Copy final artifacts to this directory (unstable)", + ) + .value_name("PATH") + .help_heading(heading::COMPILATION_OPTIONS), + ) + .arg_build_plan() + .arg_unit_graph() + .arg_timings() + .arg_manifest_path() + .after_help("Run `cargo help build` for more detailed information.\n") +} + +fn setup_logger() { + let env = tracing_subscriber::EnvFilter::from_env("CARGO_LOG"); + + tracing_subscriber::fmt() + .with_timer(tracing_subscriber::fmt::time::Uptime::default()) + .with_ansi(std::io::IsTerminal::is_terminal(&std::io::stderr())) + .with_writer(std::io::stderr) + .with_env_filter(env) + .init(); + tracing::trace!(start = humantime::format_rfc3339(std::time::SystemTime::now()).to_string()); +} + +/// Configure libgit2 to use libcurl if necessary. +/// +/// If the user has a non-default network configuration, then libgit2 will be +/// configured to use libcurl instead of the built-in networking support so +/// that those configuration settings can be used. +fn init_git_transports(config: &GlobalContext) { + match needs_custom_http_transport(config) { + Ok(true) => {} + _ => return, + } + + let handle = match http_handle(config) { + Ok(handle) => handle, + Err(..) => return, + }; + + // The unsafety of the registration function derives from two aspects: + // + // 1. This call must be synchronized with all other registration calls as + // well as construction of new transports. + // 2. The argument is leaked. + // + // We're clear on point (1) because this is only called at the start of this + // binary (we know what the state of the world looks like) and we're mostly + // clear on point (2) because we'd only free it after everything is done + // anyway + unsafe { + git2_curl::register(handle); + } +} diff --git a/src/cargo/core/compiler/build_context/mod.rs b/src/cargo/core/compiler/build_context/mod.rs index 78c1e90ae5a..48556316f0a 100644 --- a/src/cargo/core/compiler/build_context/mod.rs +++ b/src/cargo/core/compiler/build_context/mod.rs @@ -10,6 +10,7 @@ use crate::util::errors::CargoResult; use crate::util::interning::InternedString; use crate::util::Rustc; use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; mod target_info; pub use self::target_info::{ @@ -78,6 +79,8 @@ pub struct BuildContext<'a, 'gctx> { /// The list of all kinds that are involved in this build pub all_kinds: HashSet, + + pub unit_export_dirs: HashMap, } impl<'a, 'gctx> BuildContext<'a, 'gctx> { @@ -111,6 +114,7 @@ impl<'a, 'gctx> BuildContext<'a, 'gctx> { unit_graph, scrape_units, all_kinds, + unit_export_dirs: HashMap::new(), }) } diff --git a/src/cargo/core/compiler/build_context/target_info.rs b/src/cargo/core/compiler/build_context/target_info.rs index ed4b04dc228..849397ca916 100644 --- a/src/cargo/core/compiler/build_context/target_info.rs +++ b/src/cargo/core/compiler/build_context/target_info.rs @@ -873,9 +873,9 @@ pub struct RustcTargetData<'gctx> { host_info: TargetInfo, /// Build information for targets that we're building for. - target_config: HashMap, + pub target_config: HashMap, /// Information about the target platform that we're building for. - target_info: HashMap, + pub target_info: HashMap, } impl<'gctx> RustcTargetData<'gctx> { diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/cargo/core/compiler/build_runner/compilation_files.rs index 27c555a2694..24f13bf4eaa 100644 --- a/src/cargo/core/compiler/build_runner/compilation_files.rs +++ b/src/cargo/core/compiler/build_runner/compilation_files.rs @@ -102,7 +102,7 @@ pub struct CompilationFiles<'a, 'gctx> { /// The target directory layout for the target (if different from then host). pub(super) target: HashMap, /// Additional directory to include a copy of the outputs. - export_dir: Option, + export_dir: HashMap, /// The root targets requested by the user on the command line (does not /// include dependencies). roots: Vec, @@ -156,7 +156,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> { ws: build_runner.bcx.ws, host, target, - export_dir: build_runner.bcx.build_config.export_dir.clone(), + export_dir: build_runner.bcx.unit_export_dirs.clone(), roots: build_runner.bcx.roots.clone(), metas, outputs, @@ -213,10 +213,12 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> { } } + /* /// Additional export directory from `--out-dir`. pub fn export_dir(&self) -> Option { self.export_dir.clone() } + */ /// Directory name to use for a package in the form `NAME-HASH`. /// @@ -409,21 +411,11 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> { } let filename = file_type.uplift_filename(&unit.target); - let uplift_path = if unit.target.is_example() { - // Examples live in their own little world. - self.layout(unit.kind).examples().join(filename) - } else if unit.target.is_custom_build() { - self.build_script_dir(unit).join(filename) + if unit.target.is_custom_build() { + Some(self.build_script_dir(unit).join(filename)) } else { - self.layout(unit.kind).dest().join(filename) - }; - if from_path == uplift_path { - // This can happen with things like examples that reside in the - // same directory, do not have a metadata hash (like on Windows), - // and do not have hyphens. - return None; + None } - Some(uplift_path) } /// Calculates the filenames that the given unit will generate. @@ -530,15 +522,10 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> { // If, the `different_binary_name` feature is enabled, the name of the hardlink will // be the name of the binary provided by the user in `Cargo.toml`. let hardlink = self.uplift_to(unit, &file_type, &path); - let export_path = if unit.target.is_custom_build() { - None - } else { - self.export_dir.as_ref().and_then(|export_dir| { - hardlink - .as_ref() - .map(|hardlink| export_dir.join(hardlink.file_name().unwrap())) - }) - }; + let export_path = self + .export_dir + .get(unit) + .map(|export_dir| export_dir.join(file_type.uplift_filename(&unit.target))); outputs.push(OutputFile { path, hardlink, diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 706ff530a19..adfd2d52187 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -289,6 +289,8 @@ fn rustc( let mut output_options = OutputOptions::new(build_runner, unit); let package_id = unit.pkg.package_id(); let target = Target::clone(&unit.target); + let features = unit.features.clone(); + let kind = unit.kind; let mode = unit.mode; exec.init(build_runner, unit); @@ -422,7 +424,7 @@ fn rustc( count => format!(" due to {} previous errors", count), }; let name = descriptive_pkg_name(&name, &target, &mode); - format!("could not compile {name}{errors}{warnings}") + format!("could not compile {name}{errors}{warnings}\n target: {kind:?}\n features: {features:?}") }); if let Err(e) = result { @@ -528,7 +530,6 @@ fn link_targets( ) -> CargoResult { let bcx = build_runner.bcx; let outputs = build_runner.outputs(unit)?; - let export_dir = build_runner.files().export_dir(); let package_id = unit.pkg.package_id(); let manifest_path = PathBuf::from(unit.pkg.manifest_path()); let profile = unit.profile.clone(); @@ -559,17 +560,15 @@ fn link_targets( if !src.exists() { continue; } - let Some(dst) = output.hardlink.as_ref() else { - destinations.push(src.clone()); - continue; - }; - destinations.push(dst.clone()); - paths::link_or_copy(src, dst)?; - if let Some(ref path) = output.export_path { - let export_dir = export_dir.as_ref().unwrap(); - paths::create_dir_all(export_dir)?; - + if let Some(path) = &output.hardlink { + paths::create_dir_all(path.parent().unwrap())?; + paths::link_or_copy(src, path)?; + destinations.push(path.clone()); + } + if let Some(path) = &output.export_path { + paths::create_dir_all(path.parent().unwrap())?; paths::link_or_copy(src, path)?; + destinations.push(path.clone()); } } diff --git a/src/cargo/core/package.rs b/src/cargo/core/package.rs index 5e054794a78..61ba23814bc 100644 --- a/src/cargo/core/package.rs +++ b/src/cargo/core/package.rs @@ -280,8 +280,8 @@ impl hash::Hash for Package { /// This is primarily used to convert a set of `PackageId`s to `Package`s. It /// will download as needed, or used the cached download if available. pub struct PackageSet<'gctx> { - packages: HashMap>, - sources: RefCell>, + pub packages: HashMap>, + pub sources: RefCell>, gctx: &'gctx GlobalContext, multi: Multi, /// Used to prevent reusing the PackageSet to download twice.