Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cached recipes #1906

Draft
wants to merge 36 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
a649e1d
Add ExpressionWalker and refactor Variables
nmay231 Feb 13, 2024
47976a7
Cache recipe by evaluated body, initial
nmay231 Feb 15, 2024
7d24508
Tests
nmay231 Feb 16, 2024
626dd08
Include working directory/project dir in serialized cache
nmay231 Feb 16, 2024
ed9f0aa
Rename RecipeCache.{hash => body_hash}
nmay231 Feb 16, 2024
a1f740c
Shorten cache filename
nmay231 Feb 16, 2024
d885519
Rename JustfileCache.{recipe_caches => recipes}
nmay231 Feb 16, 2024
5a959d1
Use working directory AND justfile path for cache filename
nmay231 Feb 16, 2024
d82e9e1
Remove InvalidCachedRecipe error
nmay231 Mar 7, 2024
c18a0cc
Print 'skipping cached recipe' message by default
nmay231 Mar 7, 2024
6307eb0
Remove `CacheFilename` for now
nmay231 Mar 7, 2024
ed8c05d
Use .as_encoded_bytes() on os strings
nmay231 Mar 7, 2024
bbf7f84
Fix tests
nmay231 Mar 7, 2024
cf3c95a
Make cached_recipes require unstable
nmay231 Mar 7, 2024
36482fd
Put cache files in working directory by default
nmay231 Mar 7, 2024
d5edf65
Added versioning to cache files
nmay231 Mar 7, 2024
f32cc86
Make errors look the same
nmay231 Mar 7, 2024
219c646
Revert "Add ExpressionWalker and refactor Variables"
nmay231 Mar 7, 2024
3a341ea
Clarify error messages
nmay231 Mar 7, 2024
9acd827
Incompatible old versions of cache should also silently be ignored
nmay231 Mar 7, 2024
29d3459
Merge branch 'master' into cached-recipes
nmay231 Mar 12, 2024
0812866
Doc comments
nmay231 Mar 14, 2024
707844c
Move cache_file into Search struct
nmay231 Mar 14, 2024
70fe6ac
Starting to implement cached dependencies
nmay231 Mar 21, 2024
e014285
Return instead of mutate
nmay231 Mar 21, 2024
e9686df
Cached dependencies mostly work
nmay231 Mar 24, 2024
e859bd6
Only impl Recipe::run() for recipes with resolved dependencies
nmay231 Mar 26, 2024
fbc87dd
Finalize cached recipes with deps
nmay231 Mar 26, 2024
8e8e046
Merge branch 'master' into cached-recipes
nmay231 Mar 26, 2024
4961730
Rename src/{cache => justfile_cache}.rs
nmay231 Mar 26, 2024
a17b8dd
Simplify JustfileCache
nmay231 Mar 26, 2024
d12dbc3
Docs
nmay231 Mar 26, 2024
1ca5122
note to add to gitignore
nmay231 Mar 26, 2024
db89a9a
Just make the cache mutable directly, don't use refcell
nmay231 Apr 15, 2024
aa8c2db
Run just check
nmay231 Apr 15, 2024
e666bf5
Merge branch 'master' into cached-recipes
nmay231 Apr 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/attribute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use super::*;
#[serde(rename_all = "kebab-case")]
pub(crate) enum Attribute<'src> {
Confirm(Option<StringLiteral<'src>>),
Cached,
Linux,
Macos,
NoCd,
Expand Down
25 changes: 25 additions & 0 deletions src/cache.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
use std::collections::HashMap;
use std::path::PathBuf;

use serde::{Deserialize, Serialize};

#[derive(Debug, Serialize, Deserialize)]
pub(crate) struct JustfileCache {
/// Only serialized for user debugging
pub(crate) working_directory: PathBuf,
nmay231 marked this conversation as resolved.
Show resolved Hide resolved
pub(crate) recipe_caches: HashMap<String, RecipeCache>,
nmay231 marked this conversation as resolved.
Show resolved Hide resolved
}

impl JustfileCache {
pub(crate) fn new(working_directory: PathBuf) -> Self {
Self {
working_directory,
recipe_caches: HashMap::new(),
}
}
}

#[derive(Debug, Serialize, Deserialize)]
pub(crate) struct RecipeCache {
pub(crate) body_hash: String,
}
20 changes: 20 additions & 0 deletions src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@ pub(crate) enum Error<'src> {
token: Token<'src>,
output_error: OutputError,
},
CacheFileRead {
cache_filename: Option<PathBuf>,
},
CacheFileWrite {
cache_filename: PathBuf,
io_error: io::Error,
},
ChooserInvoke {
shell_binary: String,
shell_arguments: String,
Expand Down Expand Up @@ -96,6 +103,10 @@ pub(crate) enum Error<'src> {
io_error: io::Error,
},
Homedir,
InvalidCachedRecipe {
recipe: &'src str,
describe_invalid: String,
},
InitExists {
justfile: PathBuf,
},
Expand Down Expand Up @@ -267,6 +278,12 @@ impl<'src> ColorDisplay for Error<'src> {
}?,
OutputError::Utf8(utf8_error) => write!(f, "Backtick succeeded but stdout was not utf8: {utf8_error}")?,
}
CacheFileRead {cache_filename} => match cache_filename {
Some(cache_filename) =>
write!(f, "Failed to read cache file: {}", cache_filename.display())?,
None => write!(f, "Failed to get default cache file")?,
}
CacheFileWrite{cache_filename, io_error} => write!(f, "Failed to write cache file ({}): {io_error}", cache_filename.display())?,
ChooserInvoke { shell_binary, shell_arguments, chooser, io_error} => {
let chooser = chooser.to_string_lossy();
write!(f, "Chooser `{shell_binary} {shell_arguments} {chooser}` invocation failed: {io_error}")?;
Expand Down Expand Up @@ -355,6 +372,9 @@ impl<'src> ColorDisplay for Error<'src> {
Homedir => {
write!(f, "Failed to get homedir")?;
}
InvalidCachedRecipe { recipe, describe_invalid } => {
write!(f, "Cached recipe `{recipe}` contains {describe_invalid}, which could run multiple times.\nYou must inline it if possible or set it to a variable outside of the recipe block: my_var := ...")?;
}
InitExists { justfile } => {
write!(f, "Justfile `{}` already exists", justfile.display())?;
}
Expand Down
4 changes: 4 additions & 0 deletions src/expression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ impl<'src> Expression<'src> {
pub(crate) fn variables<'expression>(&'expression self) -> Variables<'expression, 'src> {
Variables::new(self)
}

pub(crate) fn walk<'expression>(&'expression self) -> ExpressionWalker<'expression, 'src> {
ExpressionWalker::new(self)
}
}

impl<'src> Display for Expression<'src> {
Expand Down
83 changes: 83 additions & 0 deletions src/expression_walker.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
use super::*;

pub(crate) struct ExpressionWalker<'expression, 'src> {
stack: Vec<&'expression Expression<'src>>,
}

impl<'expression, 'src> ExpressionWalker<'expression, 'src> {
pub(crate) fn new(root: &'expression Expression<'src>) -> ExpressionWalker<'expression, 'src> {
ExpressionWalker { stack: vec![root] }
}
}

impl<'expression, 'src> Iterator for ExpressionWalker<'expression, 'src> {
type Item = &'expression Expression<'src>;

fn next(&mut self) -> Option<Self::Item> {
let top = self.stack.pop()?;

match top {
Expression::StringLiteral { .. }
| Expression::Variable { .. }
| Expression::Backtick { .. } => {}
Expression::Call { thunk } => match thunk {
Thunk::Nullary { .. } => {}
Thunk::Unary { arg, .. } => self.stack.push(arg),
Thunk::UnaryOpt {
args: (a, opt_b), ..
} => {
self.stack.push(a);
if let Some(b) = opt_b.as_ref() {
self.stack.push(b);
}
}
Thunk::Binary { args, .. } => {
for arg in args.iter().rev() {
self.stack.push(arg);
}
}
Thunk::BinaryPlus {
args: ([a, b], rest),
..
} => {
let first: &[&Expression] = &[a, b];
for arg in first.iter().copied().chain(rest).rev() {
self.stack.push(arg);
}
}
Thunk::Ternary { args, .. } => {
for arg in args.iter().rev() {
self.stack.push(arg);
}
}
},
Expression::Conditional {
lhs,
rhs,
then,
otherwise,
..
} => {
self.stack.push(otherwise);
self.stack.push(then);
self.stack.push(rhs);
self.stack.push(lhs);
}
Expression::Concatenation { lhs, rhs } => {
self.stack.push(rhs);
self.stack.push(lhs);
}
Expression::Join { lhs, rhs } => {
self.stack.push(rhs);
if let Some(lhs) = lhs {
self.stack.push(lhs);
}
}
Expression::Group { contents } => {
self.stack.push(contents);
}
}

Some(top)
}
}
1 change: 1 addition & 0 deletions src/keyword.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use super::*;
pub(crate) enum Keyword {
Alias,
AllowDuplicateRecipes,
CacheFilename,
DotenvFilename,
DotenvLoad,
DotenvPath,
Expand Down
20 changes: 12 additions & 8 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,19 @@ pub(crate) use {
crate::{
alias::Alias, analyzer::Analyzer, assignment::Assignment,
assignment_resolver::AssignmentResolver, ast::Ast, attribute::Attribute, binding::Binding,
color::Color, color_display::ColorDisplay, command_ext::CommandExt, compilation::Compilation,
compile_error::CompileError, compile_error_kind::CompileErrorKind, compiler::Compiler,
cache::JustfileCache, cache::RecipeCache, color::Color, color_display::ColorDisplay,
command_ext::CommandExt, compilation::Compilation, compile_error::CompileError,
compile_error_kind::CompileErrorKind, compiler::Compiler,
conditional_operator::ConditionalOperator, config::Config, config_error::ConfigError,
count::Count, delimiter::Delimiter, dependency::Dependency, dump_format::DumpFormat,
enclosure::Enclosure, error::Error, evaluator::Evaluator, expression::Expression,
fragment::Fragment, function::Function, function_context::FunctionContext,
interrupt_guard::InterruptGuard, interrupt_handler::InterruptHandler, item::Item,
justfile::Justfile, keyed::Keyed, keyword::Keyword, lexer::Lexer, line::Line, list::List,
load_dotenv::load_dotenv, loader::Loader, name::Name, namepath::Namepath, ordinal::Ordinal,
output::output, output_error::OutputError, parameter::Parameter, parameter_kind::ParameterKind,
parser::Parser, platform::Platform, platform_interface::PlatformInterface, position::Position,
expression_walker::ExpressionWalker, fragment::Fragment, function::Function,
function_context::FunctionContext, interrupt_guard::InterruptGuard,
interrupt_handler::InterruptHandler, item::Item, justfile::Justfile, keyed::Keyed,
keyword::Keyword, lexer::Lexer, line::Line, list::List, load_dotenv::load_dotenv,
loader::Loader, name::Name, namepath::Namepath, ordinal::Ordinal, output::output,
output_error::OutputError, parameter::Parameter, parameter_kind::ParameterKind, parser::Parser,
platform::Platform, platform_interface::PlatformInterface, position::Position,
positional::Positional, ran::Ran, range_ext::RangeExt, recipe::Recipe,
recipe_context::RecipeContext, recipe_resolver::RecipeResolver, scope::Scope, search::Search,
search_config::SearchConfig, search_error::SearchError, set::Set, setting::Setting,
Expand Down Expand Up @@ -116,6 +118,7 @@ mod assignment_resolver;
mod ast;
mod attribute;
mod binding;
mod cache;
mod color;
mod color_display;
mod command_ext;
Expand All @@ -135,6 +138,7 @@ mod enclosure;
mod error;
mod evaluator;
mod expression;
mod expression_walker;
mod fragment;
mod function;
mod function_context;
Expand Down
5 changes: 4 additions & 1 deletion src/node.rs
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,10 @@ impl<'src> Node<'src> for Set<'src> {
set.push_mut(Tree::string(&argument.cooked));
}
}
Setting::DotenvFilename(value) | Setting::DotenvPath(value) | Setting::Tempdir(value) => {
Setting::CacheFilename(value)
| Setting::DotenvFilename(value)
| Setting::DotenvPath(value)
| Setting::Tempdir(value) => {
set.push_mut(Tree::string(value));
}
}
Expand Down
1 change: 1 addition & 0 deletions src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,7 @@ impl<'run, 'src> Parser<'run, 'src> {
self.expect(ColonEquals)?;

let set_value = match keyword {
Keyword::CacheFilename => Some(Setting::CacheFilename(self.parse_string_literal()?.cooked)),
Keyword::DotenvFilename => Some(Setting::DotenvFilename(self.parse_string_literal()?.cooked)),
Keyword::DotenvPath => Some(Setting::DotenvPath(self.parse_string_literal()?.cooked)),
Keyword::Shell => Some(Setting::Shell(self.parse_shell()?)),
Expand Down
Loading