From cc239ed1d1ca387c443636a4a8185956c37b3100 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 3 Mar 2023 19:54:11 +0100 Subject: [PATCH 01/11] start implementing parallel job execution --- src/alias.rs | 5 +- src/analyzer.rs | 9 +-- src/dependency.rs | 3 +- src/justfile.rs | 120 +++++++++++++++++++++++++-------------- src/keyed.rs | 3 +- src/lib.rs | 1 - src/recipe_resolver.rs | 17 +++--- src/unresolved_recipe.rs | 3 +- 8 files changed, 101 insertions(+), 60 deletions(-) diff --git a/src/alias.rs b/src/alias.rs index 0c19fe5a5e..cc8f3a5f61 100644 --- a/src/alias.rs +++ b/src/alias.rs @@ -1,8 +1,9 @@ use super::*; +use std::sync::Arc; /// An alias, e.g. `name := target` #[derive(Debug, PartialEq, Clone, Serialize)] -pub(crate) struct Alias<'src, T = Rc>> { +pub(crate) struct Alias<'src, T = Arc>> { pub(crate) attributes: BTreeSet, pub(crate) name: Name<'src>, #[serde( @@ -17,7 +18,7 @@ impl<'src> Alias<'src, Name<'src>> { self.name.line } - pub(crate) fn resolve(self, target: Rc>) -> Alias<'src> { + pub(crate) fn resolve(self, target: Arc>) -> Alias<'src> { assert_eq!(self.target.lexeme(), target.name.lexeme()); Alias { diff --git a/src/analyzer.rs b/src/analyzer.rs index 6899796311..986f3a2531 100644 --- a/src/analyzer.rs +++ b/src/analyzer.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use {super::*, CompileErrorKind::*}; const VALID_ALIAS_ATTRIBUTES: [Attribute; 1] = [Attribute::Private]; @@ -71,11 +72,11 @@ impl<'src> Analyzer<'src> { first: recipes .values() .fold(None, |accumulator, next| match accumulator { - None => Some(Rc::clone(next)), + None => Some(Arc::clone(next)), Some(previous) => Some(if previous.line_number() < next.line_number() { previous } else { - Rc::clone(next) + Arc::clone(next) }), }), aliases, @@ -173,7 +174,7 @@ impl<'src> Analyzer<'src> { } fn resolve_alias( - recipes: &Table<'src, Rc>>, + recipes: &Table<'src, Arc>>, alias: Alias<'src, Name<'src>>, ) -> CompileResult<'src, Alias<'src>> { let token = alias.name.token(); @@ -187,7 +188,7 @@ impl<'src> Analyzer<'src> { // Make sure the target recipe exists match recipes.get(alias.target.lexeme()) { - Some(target) => Ok(alias.resolve(Rc::clone(target))), + Some(target) => Ok(alias.resolve(Arc::clone(target))), None => Err(token.error(UnknownAliasTarget { alias: alias.name.lexeme(), target: alias.target.lexeme(), diff --git a/src/dependency.rs b/src/dependency.rs index 2d1da1173d..f9de949972 100644 --- a/src/dependency.rs +++ b/src/dependency.rs @@ -1,10 +1,11 @@ use super::*; +use std::sync::Arc; #[derive(PartialEq, Debug, Serialize)] pub(crate) struct Dependency<'src> { pub(crate) arguments: Vec>, #[serde(serialize_with = "keyed::serialize")] - pub(crate) recipe: Rc>, + pub(crate) recipe: Arc>, } impl<'src> Display for Dependency<'src> { diff --git a/src/justfile.rs b/src/justfile.rs index 1919999548..3d13008526 100644 --- a/src/justfile.rs +++ b/src/justfile.rs @@ -1,3 +1,4 @@ +use std::sync::Arc; use {super::*, serde::Serialize}; #[derive(Debug, PartialEq, Serialize)] @@ -5,8 +6,8 @@ pub(crate) struct Justfile<'src> { pub(crate) aliases: Table<'src, Alias<'src>>, pub(crate) assignments: Table<'src, Assignment<'src>>, #[serde(serialize_with = "keyed::serialize_option")] - pub(crate) first: Option>>, - pub(crate) recipes: Table<'src, Rc>>, + pub(crate) first: Option>>, + pub(crate) recipes: Table<'src, Arc>>, pub(crate) settings: Settings<'src>, pub(crate) warnings: Vec, } @@ -253,10 +254,21 @@ impl<'src> Justfile<'src> { search, }; - let mut ran = BTreeSet::new(); - for (recipe, arguments) in grouped { - Self::run_recipe(&context, recipe, arguments, &dotenv, search, &mut ran)?; - } + // let mut ran = BTreeSet::new(); + std::thread::scope(|scope| -> RunResult<'src, ()> { + let mut threads = Vec::new(); + for (recipe, arguments) in grouped { + threads.push(scope.spawn(|| { + Self::run_recipe( + &context, recipe, arguments, &dotenv, search, /*&mut ran*/ + ) + })); + } + for thread in threads { + thread.join().unwrap()?; + } + Ok(()) + })?; Ok(()) } @@ -269,7 +281,7 @@ impl<'src> Justfile<'src> { self .recipes .get(name) - .map(Rc::as_ref) + .map(Arc::as_ref) .or_else(|| self.aliases.get(name).map(|alias| alias.target.as_ref())) } @@ -279,16 +291,16 @@ impl<'src> Justfile<'src> { arguments: &[&str], dotenv: &BTreeMap, search: &Search, - ran: &mut BTreeSet>, + // ran: &mut BTreeSet>, ) -> RunResult<'src, ()> { let mut invocation = vec![recipe.name().to_owned()]; for argument in arguments { invocation.push((*argument).to_string()); } - if ran.contains(&invocation) { - return Ok(()); - } + // if ran.contains(&invocation) { + // return Ok(()); + // } let (outer, positional) = Evaluator::evaluate_parameters( context.config, @@ -305,46 +317,70 @@ impl<'src> Justfile<'src> { let mut evaluator = Evaluator::recipe_evaluator(context.config, dotenv, &scope, context.settings, search); - for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) { - let arguments = arguments - .iter() - .map(|argument| evaluator.evaluate_expression(argument)) - .collect::>>()?; - - Self::run_recipe( - context, - recipe, - &arguments.iter().map(String::as_ref).collect::>(), - dotenv, - search, - ran, - )?; - } + std::thread::scope(|scope| -> RunResult<'src, ()> { + let mut threads = Vec::new(); + for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) { + let arguments = arguments + .iter() + .map(|argument| evaluator.evaluate_expression(argument)) + .collect::>>()?; + + threads.push(scope.spawn(move || { + Self::run_recipe( + context, + recipe, + &arguments.iter().map(String::as_ref).collect::>(), + dotenv, + search, + // ran, + ) + })); + } + + for thread in threads { + thread.join().unwrap()?; + } + Ok(()) + })?; recipe.run(context, dotenv, scope.child(), search, &positional)?; { - let mut ran = BTreeSet::new(); - - for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { - let mut evaluated = Vec::new(); + // let mut ran = BTreeSet::new(); + + std::thread::scope(|scope| -> RunResult<'src, ()> { + let mut threads = Vec::new(); + for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { + let mut evaluated = Vec::new(); + + for argument in arguments { + evaluated.push( + evaluator + .evaluate_expression(argument) + .expect("error evaluating expression"), + ); + } - for argument in arguments { - evaluated.push(evaluator.evaluate_expression(argument)?); + threads.push(scope.spawn(move || { + Self::run_recipe( + context, + recipe, + &evaluated.iter().map(String::as_ref).collect::>(), + dotenv, + search, + // &mut ran, + ) + })); } - Self::run_recipe( - context, - recipe, - &evaluated.iter().map(String::as_ref).collect::>(), - dotenv, - search, - &mut ran, - )?; - } + for thread in threads { + thread.join().unwrap()?; + } + Ok(()) + })?; } - ran.insert(invocation); + // ran.insert(invocation); Ok(()) } diff --git a/src/keyed.rs b/src/keyed.rs index e171451eb8..42bb34f8c4 100644 --- a/src/keyed.rs +++ b/src/keyed.rs @@ -1,10 +1,11 @@ use super::*; +use std::sync::Arc; pub(crate) trait Keyed<'key> { fn key(&self) -> &'key str; } -impl<'key, T: Keyed<'key>> Keyed<'key> for Rc { +impl<'key, T: Keyed<'key>> Keyed<'key> for Arc { fn key(&self) -> &'key str { self.as_ref().key() } diff --git a/src/lib.rs b/src/lib.rs index c69bfb6bc2..5d1cdb359a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,6 @@ pub(crate) use { ops::{Index, Range, RangeInclusive}, path::{self, Path, PathBuf}, process::{self, Command, ExitStatus, Stdio}, - rc::Rc, str::{self, Chars}, sync::{Mutex, MutexGuard}, usize, vec, diff --git a/src/recipe_resolver.rs b/src/recipe_resolver.rs index 2715acc240..2873fa34b0 100644 --- a/src/recipe_resolver.rs +++ b/src/recipe_resolver.rs @@ -1,8 +1,9 @@ +use std::sync::Arc; use {super::*, CompileErrorKind::*}; pub(crate) struct RecipeResolver<'src: 'run, 'run> { unresolved_recipes: Table<'src, UnresolvedRecipe<'src>>, - resolved_recipes: Table<'src, Rc>>, + resolved_recipes: Table<'src, Arc>>, assignments: &'run Table<'src, Assignment<'src>>, } @@ -10,7 +11,7 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> { pub(crate) fn resolve_recipes( unresolved_recipes: Table<'src, UnresolvedRecipe<'src>>, assignments: &Table<'src, Assignment<'src>>, - ) -> CompileResult<'src, Table<'src, Rc>>> { + ) -> CompileResult<'src, Table<'src, Arc>>> { let mut resolver = RecipeResolver { resolved_recipes: Table::new(), unresolved_recipes, @@ -72,20 +73,20 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> { &mut self, stack: &mut Vec<&'src str>, recipe: UnresolvedRecipe<'src>, - ) -> CompileResult<'src, Rc>> { + ) -> CompileResult<'src, Arc>> { if let Some(resolved) = self.resolved_recipes.get(recipe.name()) { - return Ok(Rc::clone(resolved)); + return Ok(Arc::clone(resolved)); } stack.push(recipe.name()); - let mut dependencies: Vec> = Vec::new(); + let mut dependencies: Vec> = Vec::new(); for dependency in &recipe.dependencies { let name = dependency.recipe.lexeme(); if let Some(resolved) = self.resolved_recipes.get(name) { // dependency already resolved - dependencies.push(Rc::clone(resolved)); + dependencies.push(Arc::clone(resolved)); } else if stack.contains(&name) { let first = stack[0]; stack.push(first); @@ -113,8 +114,8 @@ impl<'src: 'run, 'run> RecipeResolver<'src, 'run> { stack.pop(); - let resolved = Rc::new(recipe.resolve(dependencies)?); - self.resolved_recipes.insert(Rc::clone(&resolved)); + let resolved = Arc::new(recipe.resolve(dependencies)?); + self.resolved_recipes.insert(Arc::clone(&resolved)); Ok(resolved) } } diff --git a/src/unresolved_recipe.rs b/src/unresolved_recipe.rs index 0a49443de8..0af5c666c8 100644 --- a/src/unresolved_recipe.rs +++ b/src/unresolved_recipe.rs @@ -1,11 +1,12 @@ use super::*; +use std::sync::Arc; pub(crate) type UnresolvedRecipe<'src> = Recipe<'src, UnresolvedDependency<'src>>; impl<'src> UnresolvedRecipe<'src> { pub(crate) fn resolve( self, - resolved: Vec>>, + resolved: Vec>>, ) -> CompileResult<'src, Recipe<'src>> { assert_eq!( self.dependencies.len(), From d6fb65d06004ca70910303472df933c186ca1210 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 5 Mar 2023 09:40:31 +0100 Subject: [PATCH 02/11] switch to crossbeam::scope, create task-runner scope --- Cargo.lock | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/justfile.rs | 32 +++++------------- src/lib.rs | 1 + src/parallel.rs | 46 +++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 23 deletions(-) create mode 100644 src/parallel.rs diff --git a/Cargo.lock b/Cargo.lock index 52735efeb5..1bc9bf56cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "bitflags" version = "1.3.2" @@ -109,6 +115,73 @@ dependencies = [ "rustversion", ] +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -362,6 +435,7 @@ dependencies = [ "camino", "clap", "cradle", + "crossbeam", "ctrlc", "derivative", "dotenvy", @@ -435,6 +509,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "nix" version = "0.26.2" @@ -613,6 +696,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.152" diff --git a/Cargo.toml b/Cargo.toml index 04a508fba1..f4c694ab6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ atty = "0.2.0" camino = "1.0.4" clap = { version = "2.33.0", features = ["wrap_help"] } ctrlc = { version = "3.1.1", features = ["termination"] } +crossbeam = "0.8.2" derivative = "2.0.0" dotenvy = "0.15" edit-distance = "2.0.0" diff --git a/src/justfile.rs b/src/justfile.rs index 3d13008526..c698f4704d 100644 --- a/src/justfile.rs +++ b/src/justfile.rs @@ -255,17 +255,13 @@ impl<'src> Justfile<'src> { }; // let mut ran = BTreeSet::new(); - std::thread::scope(|scope| -> RunResult<'src, ()> { - let mut threads = Vec::new(); + parallel::task_scope(|scope| { for (recipe, arguments) in grouped { - threads.push(scope.spawn(|| { + scope.spawn(|| { Self::run_recipe( &context, recipe, arguments, &dotenv, search, /*&mut ran*/ ) - })); - } - for thread in threads { - thread.join().unwrap()?; + }); } Ok(()) })?; @@ -317,15 +313,14 @@ impl<'src> Justfile<'src> { let mut evaluator = Evaluator::recipe_evaluator(context.config, dotenv, &scope, context.settings, search); - std::thread::scope(|scope| -> RunResult<'src, ()> { - let mut threads = Vec::new(); + parallel::task_scope(|scope| { for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) { let arguments = arguments .iter() .map(|argument| evaluator.evaluate_expression(argument)) .collect::>>()?; - threads.push(scope.spawn(move || { + scope.spawn(move || { Self::run_recipe( context, recipe, @@ -334,11 +329,7 @@ impl<'src> Justfile<'src> { search, // ran, ) - })); - } - - for thread in threads { - thread.join().unwrap()?; + }); } Ok(()) })?; @@ -348,8 +339,7 @@ impl<'src> Justfile<'src> { { // let mut ran = BTreeSet::new(); - std::thread::scope(|scope| -> RunResult<'src, ()> { - let mut threads = Vec::new(); + parallel::task_scope(|scope| { for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { let mut evaluated = Vec::new(); @@ -361,7 +351,7 @@ impl<'src> Justfile<'src> { ); } - threads.push(scope.spawn(move || { + scope.spawn(move || { Self::run_recipe( context, recipe, @@ -370,11 +360,7 @@ impl<'src> Justfile<'src> { search, // &mut ran, ) - })); - } - - for thread in threads { - thread.join().unwrap()?; + }); } Ok(()) })?; diff --git a/src/lib.rs b/src/lib.rs index 5d1cdb359a..2ee07fa72b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -150,6 +150,7 @@ mod name; mod ordinal; mod output; mod output_error; +mod parallel; mod parameter; mod parameter_kind; mod parser; diff --git a/src/parallel.rs b/src/parallel.rs new file mode 100644 index 0000000000..fffd612849 --- /dev/null +++ b/src/parallel.rs @@ -0,0 +1,46 @@ +use crate::RunResult; +use crossbeam::thread; + +type ScopeResult<'src> = RunResult<'src, ()>; + +pub(crate) struct TaskScope<'env, 'src, 'inner_scope> { + inner: &'inner_scope thread::Scope<'env>, + join_handles: Vec>>, +} + +impl<'env, 'src, 'inner_scope> TaskScope<'env, 'src, 'inner_scope> { + pub(crate) fn spawn<'scope, F>(&'scope mut self, f: F) + where + 'src: 'env, + F: FnOnce() -> ScopeResult<'src>, + F: Send + 'env, + { + self.join_handles.push(self.inner.spawn(|_scope| f())); + } +} + +/// task runner scope, based on `crossbeam::thread::scope`. +/// +/// The `scope` object can be used to `.spawn` new tasks to be +/// run. The first error will be returned as result of this `task_scope`. +/// +/// Only works for tasks with an `RunResult<'src, ()>` result type. +pub(crate) fn task_scope<'env, 'src, F>(f: F) -> ScopeResult<'src> +where + F: for<'inner_scope> FnOnce(&mut TaskScope<'env, 'src, 'inner_scope>) -> ScopeResult<'src>, +{ + thread::scope(|scope| { + let mut task_scope = TaskScope { + inner: scope, + join_handles: Vec::new(), + }; + + f(&mut task_scope)?; + + for handle in task_scope.join_handles { + handle.join().expect("could not join thread")?; + } + Ok(()) + }) + .expect("could not join thread") +} From c21d46a60a269ffbea029b7ffe72a3c29ccf47d1 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 5 Mar 2023 17:23:10 +0100 Subject: [PATCH 03/11] add --parallel command line argument --- src/config.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/config.rs b/src/config.rs index 069a9a8d20..f5a581ec47 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,6 +23,7 @@ pub(crate) struct Config { pub(crate) list_heading: String, pub(crate) list_prefix: String, pub(crate) load_dotenv: bool, + pub(crate) parallel: bool, pub(crate) search_config: SearchConfig, pub(crate) shell: Option, pub(crate) shell_args: Option>, @@ -94,6 +95,7 @@ mod arg { pub(crate) const LIST_PREFIX: &str = "LIST-PREFIX"; pub(crate) const NO_DOTENV: &str = "NO-DOTENV"; pub(crate) const NO_HIGHLIGHT: &str = "NO-HIGHLIGHT"; + pub(crate) const PARALLEL: &str = "PARALLEL"; pub(crate) const QUIET: &str = "QUIET"; pub(crate) const SET: &str = "SET"; pub(crate) const SHELL: &str = "SHELL"; @@ -195,6 +197,12 @@ impl Config { .takes_value(true) .help("Use as justfile"), ) + .arg( + Arg::with_name(arg::PARALLEL) + .short("p") + .long("parallel") + .help("run task dependencies & given tasks in parallel") + ) .arg( Arg::with_name(arg::QUIET) .short("q") @@ -571,6 +579,7 @@ impl Config { Ok(Self { check: matches.is_present(arg::CHECK), dry_run: matches.is_present(arg::DRY_RUN), + parallel: matches.is_present(arg::PARALLEL), dump_format: Self::dump_format_from_matches(matches)?, highlight: !matches.is_present(arg::NO_HIGHLIGHT), shell: matches.value_of(arg::SHELL).map(str::to_owned), @@ -628,6 +637,7 @@ mod tests { args: [$($arg:expr),*], $(color: $color:expr,)? $(dry_run: $dry_run:expr,)? + $(parallel: $parallel:expr,)? $(dump_format: $dump_format:expr,)? $(highlight: $highlight:expr,)? $(search_config: $search_config:expr,)? @@ -647,6 +657,7 @@ mod tests { let want = Config { $(color: $color,)? $(dry_run: $dry_run,)? + $(parallel: $parallel,)? $(dump_format: $dump_format,)? $(highlight: $highlight,)? $(search_config: $search_config,)? @@ -783,6 +794,18 @@ mod tests { dry_run: true, } + test! { + name: parallel_long, + args: ["--parallel"], + parallel: true, + } + + test! { + name: parallel_short, + args: ["-p"], + parallel: true, + } + error! { name: dry_run_quiet, args: ["--dry-run", "--quiet"], From 6439e899ba438e9b16ee99d754b02ef720bdae96 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 5 Mar 2023 17:55:12 +0100 Subject: [PATCH 04/11] make parallelisation optional --- src/justfile.rs | 18 +++++++++--------- src/parallel.rs | 13 ++++++++++--- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/justfile.rs b/src/justfile.rs index c698f4704d..501134821e 100644 --- a/src/justfile.rs +++ b/src/justfile.rs @@ -255,13 +255,13 @@ impl<'src> Justfile<'src> { }; // let mut ran = BTreeSet::new(); - parallel::task_scope(|scope| { + parallel::task_scope(config.parallel, |scope| { for (recipe, arguments) in grouped { - scope.spawn(|| { + scope.run(|| { Self::run_recipe( &context, recipe, arguments, &dotenv, search, /*&mut ran*/ ) - }); + })?; } Ok(()) })?; @@ -313,14 +313,14 @@ impl<'src> Justfile<'src> { let mut evaluator = Evaluator::recipe_evaluator(context.config, dotenv, &scope, context.settings, search); - parallel::task_scope(|scope| { + parallel::task_scope(context.config.parallel, |scope| { for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) { let arguments = arguments .iter() .map(|argument| evaluator.evaluate_expression(argument)) .collect::>>()?; - scope.spawn(move || { + scope.run(move || { Self::run_recipe( context, recipe, @@ -329,7 +329,7 @@ impl<'src> Justfile<'src> { search, // ran, ) - }); + })?; } Ok(()) })?; @@ -339,7 +339,7 @@ impl<'src> Justfile<'src> { { // let mut ran = BTreeSet::new(); - parallel::task_scope(|scope| { + parallel::task_scope(context.config.parallel, |scope| { for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { let mut evaluated = Vec::new(); @@ -351,7 +351,7 @@ impl<'src> Justfile<'src> { ); } - scope.spawn(move || { + scope.run(move || { Self::run_recipe( context, recipe, @@ -360,7 +360,7 @@ impl<'src> Justfile<'src> { search, // &mut ran, ) - }); + })?; } Ok(()) })?; diff --git a/src/parallel.rs b/src/parallel.rs index fffd612849..d38990e8c7 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -6,16 +6,22 @@ type ScopeResult<'src> = RunResult<'src, ()>; pub(crate) struct TaskScope<'env, 'src, 'inner_scope> { inner: &'inner_scope thread::Scope<'env>, join_handles: Vec>>, + parallel: bool, } impl<'env, 'src, 'inner_scope> TaskScope<'env, 'src, 'inner_scope> { - pub(crate) fn spawn<'scope, F>(&'scope mut self, f: F) + pub(crate) fn run<'scope, F>(&'scope mut self, f: F) -> ScopeResult<'src> where 'src: 'env, F: FnOnce() -> ScopeResult<'src>, F: Send + 'env, { - self.join_handles.push(self.inner.spawn(|_scope| f())); + if self.parallel { + self.join_handles.push(self.inner.spawn(|_scope| f())); + Ok(()) + } else { + f() + } } } @@ -25,12 +31,13 @@ impl<'env, 'src, 'inner_scope> TaskScope<'env, 'src, 'inner_scope> { /// run. The first error will be returned as result of this `task_scope`. /// /// Only works for tasks with an `RunResult<'src, ()>` result type. -pub(crate) fn task_scope<'env, 'src, F>(f: F) -> ScopeResult<'src> +pub(crate) fn task_scope<'env, 'src, F>(parallel: bool, f: F) -> ScopeResult<'src> where F: for<'inner_scope> FnOnce(&mut TaskScope<'env, 'src, 'inner_scope>) -> ScopeResult<'src>, { thread::scope(|scope| { let mut task_scope = TaskScope { + parallel, inner: scope, join_handles: Vec::new(), }; From d49b0f2792f69043f856284ee7458640d5d57f78 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 5 Mar 2023 18:54:01 +0100 Subject: [PATCH 05/11] re-add tracking of executed tasks --- src/justfile.rs | 44 ++++++++++++++++++++++---------------------- src/parallel.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/justfile.rs b/src/justfile.rs index 501134821e..ca112b77aa 100644 --- a/src/justfile.rs +++ b/src/justfile.rs @@ -1,3 +1,4 @@ +use parallel::Ran; use std::sync::Arc; use {super::*, serde::Serialize}; @@ -254,14 +255,10 @@ impl<'src> Justfile<'src> { search, }; - // let mut ran = BTreeSet::new(); + let ran = Ran::new(); parallel::task_scope(config.parallel, |scope| { for (recipe, arguments) in grouped { - scope.run(|| { - Self::run_recipe( - &context, recipe, arguments, &dotenv, search, /*&mut ran*/ - ) - })?; + scope.run(|| Self::run_recipe(&context, recipe, arguments, &dotenv, search, &ran))?; } Ok(()) })?; @@ -287,16 +284,16 @@ impl<'src> Justfile<'src> { arguments: &[&str], dotenv: &BTreeMap, search: &Search, - // ran: &mut BTreeSet>, + ran: &Ran, ) -> RunResult<'src, ()> { let mut invocation = vec![recipe.name().to_owned()]; for argument in arguments { invocation.push((*argument).to_string()); } - // if ran.contains(&invocation) { - // return Ok(()); - // } + if ran.contains(&invocation) { + return Ok(()); + } let (outer, positional) = Evaluator::evaluate_parameters( context.config, @@ -327,7 +324,7 @@ impl<'src> Justfile<'src> { &arguments.iter().map(String::as_ref).collect::>(), dotenv, search, - // ran, + ran, ) })?; } @@ -337,7 +334,7 @@ impl<'src> Justfile<'src> { recipe.run(context, dotenv, scope.child(), search, &positional)?; { - // let mut ran = BTreeSet::new(); + let ran = Ran::new(); parallel::task_scope(context.config.parallel, |scope| { for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { @@ -351,22 +348,25 @@ impl<'src> Justfile<'src> { ); } - scope.run(move || { - Self::run_recipe( - context, - recipe, - &evaluated.iter().map(String::as_ref).collect::>(), - dotenv, - search, - // &mut ran, - ) + scope.run({ + let ran = ran.clone(); + move || { + Self::run_recipe( + context, + recipe, + &evaluated.iter().map(String::as_ref).collect::>(), + dotenv, + search, + &ran, + ) + } })?; } Ok(()) })?; } - // ran.insert(invocation); + ran.insert(invocation); Ok(()) } diff --git a/src/parallel.rs b/src/parallel.rs index d38990e8c7..a30e91d518 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -1,5 +1,7 @@ use crate::RunResult; use crossbeam::thread; +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; type ScopeResult<'src> = RunResult<'src, ()>; @@ -10,6 +12,7 @@ pub(crate) struct TaskScope<'env, 'src, 'inner_scope> { } impl<'env, 'src, 'inner_scope> TaskScope<'env, 'src, 'inner_scope> { + /// run the given task, either directly synchronously or spawned in a background thread. pub(crate) fn run<'scope, F>(&'scope mut self, f: F) -> ScopeResult<'src> where 'src: 'env, @@ -31,6 +34,9 @@ impl<'env, 'src, 'inner_scope> TaskScope<'env, 'src, 'inner_scope> { /// run. The first error will be returned as result of this `task_scope`. /// /// Only works for tasks with an `RunResult<'src, ()>` result type. +/// +/// When `parallel` is set to `false`, the tasks are directly executed +/// when calling `run`. pub(crate) fn task_scope<'env, 'src, F>(parallel: bool, f: F) -> ScopeResult<'src> where F: for<'inner_scope> FnOnce(&mut TaskScope<'env, 'src, 'inner_scope>) -> ScopeResult<'src>, @@ -51,3 +57,41 @@ where }) .expect("could not join thread") } + +/// track which tasks were already run, across all running threads. +#[derive(Clone)] +pub(crate) struct Ran(Arc>>>); + +impl Ran { + pub(crate) fn new() -> Self { + Self(Arc::new(Mutex::new(HashSet::new()))) + } + + pub(crate) fn insert(&self, args: Vec) { + let mut ran = self.0.lock().unwrap(); + ran.insert(args); + } + + pub(crate) fn contains(&self, args: &Vec) -> bool { + let ran = self.0.lock().unwrap(); + ran.contains(args) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ran_empty() { + let r = Ran::new(); + assert!(!r.contains(&vec![])); + } + + #[test] + fn test_ran_insert_contains() { + let r = Ran::new(); + r.insert(vec!["1".into(), "2".into(), "3".into()]); + assert!(r.contains(&vec!["1".into(), "2".into(), "3".into()])); + } +} From 6386702d05e97c4386392a0b84b126df4dad7f26 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Mon, 6 Mar 2023 06:09:46 +0100 Subject: [PATCH 06/11] use [parallel] attribute on tasks to enable parallel dependency execution --- src/attribute.rs | 1 + src/config.rs | 2 +- src/justfile.rs | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/attribute.rs b/src/attribute.rs index ffd532f480..38c6d66b75 100644 --- a/src/attribute.rs +++ b/src/attribute.rs @@ -11,6 +11,7 @@ pub(crate) enum Attribute { NoCd, NoExitMessage, Private, + Parallel, Unix, Windows, } diff --git a/src/config.rs b/src/config.rs index f5a581ec47..ad154f498f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -201,7 +201,7 @@ impl Config { Arg::with_name(arg::PARALLEL) .short("p") .long("parallel") - .help("run task dependencies & given tasks in parallel") + .help("run given tasks in parallel") ) .arg( Arg::with_name(arg::QUIET) diff --git a/src/justfile.rs b/src/justfile.rs index ca112b77aa..8585322927 100644 --- a/src/justfile.rs +++ b/src/justfile.rs @@ -310,7 +310,8 @@ impl<'src> Justfile<'src> { let mut evaluator = Evaluator::recipe_evaluator(context.config, dotenv, &scope, context.settings, search); - parallel::task_scope(context.config.parallel, |scope| { + let run_dependencies_in_parallel = recipe.attributes.contains(&Attribute::Parallel); + parallel::task_scope(run_dependencies_in_parallel, |scope| { for Dependency { recipe, arguments } in recipe.dependencies.iter().take(recipe.priors) { let arguments = arguments .iter() @@ -336,7 +337,7 @@ impl<'src> Justfile<'src> { { let ran = Ran::new(); - parallel::task_scope(context.config.parallel, |scope| { + parallel::task_scope(run_dependencies_in_parallel, |scope| { for Dependency { recipe, arguments } in recipe.dependencies.iter().skip(recipe.priors) { let mut evaluated = Vec::new(); From 0b3388836937b7d0e97676c1658ae98470c11354 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 7 Mar 2023 16:37:35 +0100 Subject: [PATCH 07/11] generate completions --- completions/just.bash | 2 +- completions/just.elvish | 2 ++ completions/just.fish | 1 + completions/just.powershell | 2 ++ completions/just.zsh | 2 ++ 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/completions/just.bash b/completions/just.bash index eb84e3e87a..f2172bc35a 100644 --- a/completions/just.bash +++ b/completions/just.bash @@ -20,7 +20,7 @@ _just() { case "${cmd}" in just) - opts=" -n -q -u -v -e -l -h -V -f -d -c -s --check --dry-run --highlight --no-dotenv --no-highlight --quiet --shell-command --clear-shell-args --unsorted --unstable --verbose --changelog --choose --dump --edit --evaluate --fmt --init --list --summary --variables --help --version --chooser --color --dump-format --list-heading --list-prefix --justfile --set --shell --shell-arg --working-directory --command --completions --show --dotenv-filename --dotenv-path ... " + opts=" -n -p -q -u -v -e -l -h -V -f -d -c -s --check --dry-run --highlight --no-dotenv --no-highlight --parallel --quiet --shell-command --clear-shell-args --unsorted --unstable --verbose --changelog --choose --dump --edit --evaluate --fmt --init --list --summary --variables --help --version --chooser --color --dump-format --list-heading --list-prefix --justfile --set --shell --shell-arg --working-directory --command --completions --show --dotenv-filename --dotenv-path ... " if [[ ${cur} == -* ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 diff --git a/completions/just.elvish b/completions/just.elvish index 1a208afccd..5d240adb57 100644 --- a/completions/just.elvish +++ b/completions/just.elvish @@ -39,6 +39,8 @@ edit:completion:arg-completer[just] = [@words]{ cand --highlight 'Highlight echoed recipe lines in bold' cand --no-dotenv 'Don''t load `.env` file' cand --no-highlight 'Don''t highlight echoed recipe lines in bold' + cand -p 'run given tasks in parallel' + cand --parallel 'run given tasks in parallel' cand -q 'Suppress all output' cand --quiet 'Suppress all output' cand --shell-command 'Invoke with the shell used to run recipe lines and backticks' diff --git a/completions/just.fish b/completions/just.fish index ebf8eeb748..f753488167 100644 --- a/completions/just.fish +++ b/completions/just.fish @@ -29,6 +29,7 @@ complete -c just -n "__fish_use_subcommand" -s n -l dry-run -d 'Print what just complete -c just -n "__fish_use_subcommand" -l highlight -d 'Highlight echoed recipe lines in bold' complete -c just -n "__fish_use_subcommand" -l no-dotenv -d 'Don\'t load `.env` file' complete -c just -n "__fish_use_subcommand" -l no-highlight -d 'Don\'t highlight echoed recipe lines in bold' +complete -c just -n "__fish_use_subcommand" -s p -l parallel -d 'run given tasks in parallel' complete -c just -n "__fish_use_subcommand" -s q -l quiet -d 'Suppress all output' complete -c just -n "__fish_use_subcommand" -l shell-command -d 'Invoke with the shell used to run recipe lines and backticks' complete -c just -n "__fish_use_subcommand" -l clear-shell-args -d 'Clear shell arguments' diff --git a/completions/just.powershell b/completions/just.powershell index 1ed04d306a..18ece0fa69 100644 --- a/completions/just.powershell +++ b/completions/just.powershell @@ -44,6 +44,8 @@ Register-ArgumentCompleter -Native -CommandName 'just' -ScriptBlock { [CompletionResult]::new('--highlight', 'highlight', [CompletionResultType]::ParameterName, 'Highlight echoed recipe lines in bold') [CompletionResult]::new('--no-dotenv', 'no-dotenv', [CompletionResultType]::ParameterName, 'Don''t load `.env` file') [CompletionResult]::new('--no-highlight', 'no-highlight', [CompletionResultType]::ParameterName, 'Don''t highlight echoed recipe lines in bold') + [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'run given tasks in parallel') + [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'run given tasks in parallel') [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Suppress all output') [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Suppress all output') [CompletionResult]::new('--shell-command', 'shell-command', [CompletionResultType]::ParameterName, 'Invoke with the shell used to run recipe lines and backticks') diff --git a/completions/just.zsh b/completions/just.zsh index a3aff350a2..0d1e894dda 100644 --- a/completions/just.zsh +++ b/completions/just.zsh @@ -40,6 +40,8 @@ _just() { '--highlight[Highlight echoed recipe lines in bold]' \ '--no-dotenv[Don'\''t load `.env` file]' \ '--no-highlight[Don'\''t highlight echoed recipe lines in bold]' \ +'-p[run given tasks in parallel]' \ +'--parallel[run given tasks in parallel]' \ '(-n --dry-run)-q[Suppress all output]' \ '(-n --dry-run)--quiet[Suppress all output]' \ '--shell-command[Invoke with the shell used to run recipe lines and backticks]' \ From 8138ee5584dc1eaf34d9d243f5417c50ae6ad928 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 7 Mar 2023 16:42:29 +0100 Subject: [PATCH 08/11] remove wrong vim filetype from `justfile` --- justfile | 1 - 1 file changed, 1 deletion(-) diff --git a/justfile b/justfile index 52b4ee1fa0..04cc0d701d 100755 --- a/justfile +++ b/justfile @@ -228,4 +228,3 @@ pwd: # Local Variables: # mode: makefile # End: -# vim: set ft=make : From cbfb0fda0328d57ab662baecab31e2b4b2972900 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 7 Mar 2023 16:45:37 +0100 Subject: [PATCH 09/11] add new parallel attribute to readme / book --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c2760a9ace..fee121a99d 100644 --- a/README.md +++ b/README.md @@ -1233,6 +1233,7 @@ Recipes may be annotated with attributes that change their behavior. | `[unix]` | Enable recipe on Unixes. | | `[windows]` | Enable recipe on Windows. | | `[private]` | See [Private Recipes](#private-recipes). | +| `[parallel]` | execute dependencies in parallel. | A recipe can have multiple attributes, either on multiple lines: From 8521c9261646b997fe9e394ba7f5184e297a8547 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 7 Mar 2023 16:57:53 +0100 Subject: [PATCH 10/11] add documentation for argument & attribute --- README.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fee121a99d..c5ddecb765 100644 --- a/README.md +++ b/README.md @@ -1233,7 +1233,7 @@ Recipes may be annotated with attributes that change their behavior. | `[unix]` | Enable recipe on Unixes. | | `[windows]` | Enable recipe on Windows. | | `[private]` | See [Private Recipes](#private-recipes). | -| `[parallel]` | execute dependencies in parallel. | +| `[parallel]` | See [Parallel execution](#parallel-execution). | A recipe can have multiple attributes, either on multiple lines: @@ -2067,6 +2067,32 @@ Available recipes: This is useful for helper recipes which are only meant to be used as dependencies of other recipes. +### Parallel execution + +`just` has two separate ways to enable parallel execution of tasks. + +Run the given recipes on the command line in parallel: + +```sh +$ just --parallel recipe_1 recipe_2 recipe_3 +[...] +``` + +And using the `[parallel]` attribute, task dependencies are allowed to run in +parallel: + +```just +recipe_1: + sleep 1 + +recipe_2: + sleep 2 + +[parallel] +foo: recipe_1 recipe_2 + echo hello +``` + ### Quiet Recipes A recipe name may be prefixed with `@` to invert the meaning of `@` before each line: From 2dbc4e4154c1a901b899a8fc88d6f3342d3278ef Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 7 Mar 2023 17:05:56 +0100 Subject: [PATCH 11/11] update docstring --- src/parallel.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/parallel.rs b/src/parallel.rs index a30e91d518..5fe3f46d22 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -30,13 +30,13 @@ impl<'env, 'src, 'inner_scope> TaskScope<'env, 'src, 'inner_scope> { /// task runner scope, based on `crossbeam::thread::scope`. /// -/// The `scope` object can be used to `.spawn` new tasks to be -/// run. The first error will be returned as result of this `task_scope`. +/// The `scope` object can be used to `.run` new tasks to be +/// executed. Depending on the `parallel` parameter, these are +/// directly run, or spawned in a background thread. /// -/// Only works for tasks with an `RunResult<'src, ()>` result type. +/// The first error will be returned as result of this `task_scope`. /// -/// When `parallel` is set to `false`, the tasks are directly executed -/// when calling `run`. +/// Only works for tasks with an `RunResult<'src, ()>` result type. pub(crate) fn task_scope<'env, 'src, F>(parallel: bool, f: F) -> ScopeResult<'src> where F: for<'inner_scope> FnOnce(&mut TaskScope<'env, 'src, 'inner_scope>) -> ScopeResult<'src>,