From 534fb3ee4d101a56c581482c43b0bd7f67bf69bd Mon Sep 17 00:00:00 2001 From: nichmor Date: Thu, 9 Oct 2025 16:14:30 +0300 Subject: [PATCH 1/5] feat: store intermediate_recipe.yaml in debugdir --- crates/pixi-build-backend/src/generated_recipe.rs | 2 +- .../pixi-build-backend/src/intermediate_backend.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/pixi-build-backend/src/generated_recipe.rs b/crates/pixi-build-backend/src/generated_recipe.rs index 576dfd3f..5d3788cf 100644 --- a/crates/pixi-build-backend/src/generated_recipe.rs +++ b/crates/pixi-build-backend/src/generated_recipe.rs @@ -89,7 +89,7 @@ pub trait GenerateRecipe { } pub trait BackendConfig: DeserializeOwned + Clone { - /// At least debug dir should be provided by the backend config + /// Debug dir provided by the backend config fn debug_dir(&self) -> Option<&Path>; /// Merge this configuration with a target-specific configuration. diff --git a/crates/pixi-build-backend/src/intermediate_backend.rs b/crates/pixi-build-backend/src/intermediate_backend.rs index 8c504e2f..0c3a2f75 100644 --- a/crates/pixi-build-backend/src/intermediate_backend.rs +++ b/crates/pixi-build-backend/src/intermediate_backend.rs @@ -1101,6 +1101,18 @@ where &variants.keys().cloned().collect(), )?; + // Save intermediate recipe in the debug dir + if let Some(debug_dir) = self.debug_dir() { + let debug_path = debug_dir.join("intermediate_recipe.yaml"); + std::fs::create_dir_all(debug_dir).into_diagnostic()?; + std::fs::write( + &debug_path, + recipe.recipe.to_yaml_pretty().into_diagnostic()?, + ) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write intermediate recipe to {debug_path:?}"))?; + } + // Convert the recipe to source code. // TODO(baszalmstra): In the future it would be great if we could just // immediately use the intermediate recipe for some of this rattler-build From 357121fe0487e39e94d7c86478ba4ebd15a3fbfc Mon Sep 17 00:00:00 2001 From: nichmor Date: Thu, 9 Oct 2025 17:45:37 +0300 Subject: [PATCH 2/5] misc: save ir in the working dir --- .../src/intermediate_backend.rs | 105 +++++++++++++++--- 1 file changed, 90 insertions(+), 15 deletions(-) diff --git a/crates/pixi-build-backend/src/intermediate_backend.rs b/crates/pixi-build-backend/src/intermediate_backend.rs index 0c3a2f75..876ac133 100644 --- a/crates/pixi-build-backend/src/intermediate_backend.rs +++ b/crates/pixi-build-backend/src/intermediate_backend.rs @@ -65,6 +65,8 @@ use crate::{ utils::TemporaryRenderedRecipe, }; +use fs_err::tokio as tokio_fs; + #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct IntermediateBackendConfig { @@ -395,6 +397,27 @@ where }, ); + let directories = output_directory( + if number_of_outputs == 1 { + OneOrMultipleOutputs::Single(discovered_output.name.clone()) + } else { + OneOrMultipleOutputs::OneOfMany(discovered_output.name.clone()) + }, + params.work_directory.clone(), + &named_source.path, + ); + // Save intermediate recipe in the debug dir + let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + tokio_fs::create_dir_all(&directories.work_dir) + .await + .into_diagnostic()?; + tokio_fs::write( + &debug_path, + generated_recipe.recipe.to_yaml_pretty().into_diagnostic()?, + ) + .await + .into_diagnostic()?; + let mut output = Output { recipe, build_configuration: BuildConfiguration { @@ -727,6 +750,28 @@ where }, ); + let directories = output_directory( + if number_of_outputs == 1 { + OneOrMultipleOutputs::Single(discovered_output.name.clone()) + } else { + OneOrMultipleOutputs::OneOfMany(discovered_output.name.clone()) + }, + params.work_directory.clone(), + &named_source.path, + ); + + // Save intermediate recipe in the debug dir + let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + tokio_fs::create_dir_all(&directories.work_dir) + .await + .into_diagnostic()?; + tokio_fs::write( + &debug_path, + generated_recipe.recipe.to_yaml_pretty().into_diagnostic()?, + ) + .await + .into_diagnostic()?; + let mut output = Output { recipe, build_configuration: BuildConfiguration { @@ -849,7 +894,7 @@ where let variants = BTreeMap::from_iter(itertools::chain!(recipe_variants, param_variants)); // Construct the intermediate recipe - let recipe = self.generate_recipe.generate_recipe( + let generated_recipe = self.generate_recipe.generate_recipe( &self.project_model, &config, self.source_dir.clone(), @@ -865,7 +910,13 @@ where let recipe_path = self.source_dir.join(&self.manifest_rel_path); let named_source = Source { name: self.manifest_rel_path.display().to_string(), - code: Arc::from(recipe.recipe.to_yaml_pretty().into_diagnostic()?.as_str()), + code: Arc::from( + generated_recipe + .recipe + .to_yaml_pretty() + .into_diagnostic()? + .as_str(), + ), path: recipe_path.clone(), }; @@ -914,6 +965,8 @@ where let mut subpackages = HashMap::new(); let mut outputs = Vec::new(); + + let num_of_outputs = discovered_outputs.len(); for discovered_output in discovered_outputs { let variant = discovered_output.used_vars; let hash = HashInfo::from_variant(&variant, &discovered_output.noarch_type); @@ -945,6 +998,28 @@ where let build_number = recipe.build().number; + let directories = output_directory( + if num_of_outputs == 1 { + OneOrMultipleOutputs::Single(discovered_output.name.clone()) + } else { + OneOrMultipleOutputs::OneOfMany(discovered_output.name.clone()) + }, + params.work_directory.clone(), + &named_source.path, + ); + + // Save intermediate recipe in the debug dir + let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + tokio_fs::create_dir_all(&directories.work_dir) + .await + .into_diagnostic()?; + tokio_fs::write( + &debug_path, + generated_recipe.recipe.to_yaml_pretty().into_diagnostic()?, + ) + .await + .into_diagnostic()?; + subpackages.insert( recipe.package().name().clone(), PackageIdentifier { @@ -1055,7 +1130,7 @@ where Ok(CondaOutputsResult { outputs, - input_globs: recipe.metadata_input_globs, + input_globs: generated_recipe.metadata_input_globs, }) } @@ -1101,18 +1176,6 @@ where &variants.keys().cloned().collect(), )?; - // Save intermediate recipe in the debug dir - if let Some(debug_dir) = self.debug_dir() { - let debug_path = debug_dir.join("intermediate_recipe.yaml"); - std::fs::create_dir_all(debug_dir).into_diagnostic()?; - std::fs::write( - &debug_path, - recipe.recipe.to_yaml_pretty().into_diagnostic()?, - ) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write intermediate recipe to {debug_path:?}"))?; - } - // Convert the recipe to source code. // TODO(baszalmstra): In the future it would be great if we could just // immediately use the intermediate recipe for some of this rattler-build @@ -1158,6 +1221,18 @@ where recipe_path, ); + // Save intermediate recipe in the debug dir + let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + tokio_fs::create_dir_all(&directories.work_dir) + .await + .into_diagnostic()?; + tokio_fs::write( + &debug_path, + recipe.recipe.to_yaml_pretty().into_diagnostic()?, + ) + .await + .into_diagnostic()?; + let tool_config = Configuration::builder() .with_opt_cache_dir(self.cache_dir.clone()) .with_logging_output_handler(self.logging_output_handler.clone()) From 08077338f383604205e52da46e8e7fc499d1f273 Mon Sep 17 00:00:00 2001 From: nichmor Date: Thu, 16 Oct 2025 17:51:55 +0300 Subject: [PATCH 3/5] misc: add a test for intermediate recipe --- .../src/intermediate_backend.rs | 8 +- ...fault_minimal_project_model_for_build.json | 25 +++++ .../tests/integration/protocol.rs | 96 ++++++++++++++++++- ...integration__protocol__conda_build_v1.snap | 10 ++ 4 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 crates/pixi-build-backend/tests/fixtures/default_minimal_project_model_for_build.json create mode 100644 crates/pixi-build-backend/tests/integration/snapshots/integration__protocol__conda_build_v1.snap diff --git a/crates/pixi-build-backend/src/intermediate_backend.rs b/crates/pixi-build-backend/src/intermediate_backend.rs index 876ac133..93873032 100644 --- a/crates/pixi-build-backend/src/intermediate_backend.rs +++ b/crates/pixi-build-backend/src/intermediate_backend.rs @@ -407,7 +407,7 @@ where &named_source.path, ); // Save intermediate recipe in the debug dir - let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + let debug_path = directories.work_dir.join("recipe.yaml"); tokio_fs::create_dir_all(&directories.work_dir) .await .into_diagnostic()?; @@ -761,7 +761,7 @@ where ); // Save intermediate recipe in the debug dir - let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + let debug_path = directories.work_dir.join("recipe.yaml"); tokio_fs::create_dir_all(&directories.work_dir) .await .into_diagnostic()?; @@ -1009,7 +1009,7 @@ where ); // Save intermediate recipe in the debug dir - let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + let debug_path = directories.work_dir.join("recipe.yaml"); tokio_fs::create_dir_all(&directories.work_dir) .await .into_diagnostic()?; @@ -1222,7 +1222,7 @@ where ); // Save intermediate recipe in the debug dir - let debug_path = directories.work_dir.join("intermediate_recipe.yaml"); + let debug_path = directories.work_dir.join("recipe.yaml"); tokio_fs::create_dir_all(&directories.work_dir) .await .into_diagnostic()?; diff --git a/crates/pixi-build-backend/tests/fixtures/default_minimal_project_model_for_build.json b/crates/pixi-build-backend/tests/fixtures/default_minimal_project_model_for_build.json new file mode 100644 index 00000000..d125e55c --- /dev/null +++ b/crates/pixi-build-backend/tests/fixtures/default_minimal_project_model_for_build.json @@ -0,0 +1,25 @@ +{ + "name": "minimal-package", + "version": "1.0.0", + "description": null, + "authors": null, + "license": null, + "license_file": null, + "readme": null, + "homepage": null, + "repository": null, + "documentation": null, + "targets": { + "default_target": { + "host_dependencies": {}, + "build_dependencies": {}, + "run_dependencies": { + "python": { + "binary": { + "version": ">=3.8" + } + } + } + } + } +} diff --git a/crates/pixi-build-backend/tests/integration/protocol.rs b/crates/pixi-build-backend/tests/integration/protocol.rs index e2667abb..2dc8efdb 100644 --- a/crates/pixi-build-backend/tests/integration/protocol.rs +++ b/crates/pixi-build-backend/tests/integration/protocol.rs @@ -1,14 +1,21 @@ use std::sync::Arc; use crate::common::model::{convert_test_model_to_project_model_v1, load_project_model_from_json}; +use dirs::preference_dir; use imp::TestGenerateRecipe; use pixi_build_backend::{intermediate_backend::IntermediateBackend, protocol::Protocol}; use pixi_build_types::{ ChannelConfiguration, PlatformAndVirtualPackages, - procedures::{conda_build_v0::CondaBuildParams, conda_metadata::CondaMetadataParams}, + procedures::{ + conda_build_v0::CondaBuildParams, + conda_build_v1::{CondaBuildV1Output, CondaBuildV1Params}, + conda_metadata::CondaMetadataParams, + }, }; use rattler_build::console_utils::LoggingOutputHandler; -use rattler_conda_types::Platform; +use rattler_conda_types::{ + ChannelUrl, Platform, utils::url_with_trailing_slash::UrlWithTrailingSlash, +}; use serde_json::json; use tempfile::TempDir; use url::Url; @@ -199,3 +206,88 @@ async fn test_conda_build() { ".packages[0].subdir" => "[redacted]", }); } + +#[tokio::test] +async fn test_conda_build_v1() { + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path().to_path_buf(); + + let pixi_manifest = tmp_dir_path.join("pixi.toml"); + let build_dir = tmp_dir_path.join("build"); + + // Load a model from JSON + let original_model = load_project_model_from_json("minimal_project_model_for_build.json"); + + // Serialize it back to JSON + let project_model_v1 = convert_test_model_to_project_model_v1(original_model); + + // save the pixi.toml file to a temporary location + fs_err::write(&pixi_manifest, toml::to_string(&project_model_v1).unwrap()).unwrap(); + + let channel_url = Url::parse("https://prefix.dev/conda-forge").unwrap(); + + let channel_url = ChannelUrl::from(channel_url); + + let build_params = CondaBuildV1Params { + channels: vec![channel_url], + + build_prefix: None, + host_prefix: None, + run_constraints: None, + run_dependencies: None, + run_exports: None, + output: CondaBuildV1Output { + name: "minimal-package".parse().unwrap(), + version: None, + build: None, + subdir: Platform::current(), + variant: Default::default(), + }, + work_directory: build_dir.clone(), + output_directory: None, + editable: None, + }; + + let some_config = json!({ + "debug-dir": "some_debug_dir", + }); + + let target_config = Default::default(); + + let intermediate_backend: IntermediateBackend = IntermediateBackend::new( + pixi_manifest.clone(), + Some(tmp_dir_path.clone()), + project_model_v1, + Arc::default(), + some_config, + target_config, + LoggingOutputHandler::default(), + None, + ) + .unwrap(); + + let conda_build_result = intermediate_backend + .conda_build_v1(build_params) + .await + .unwrap(); + + dbg!(&build_dir); + + // list all dirs inside build_dir + let entries = fs_err::read_dir(&build_dir.join("work")).unwrap(); + for entry in entries { + let entry = entry.unwrap(); + let path = entry.path(); + dbg!(path); + } + + insta::assert_yaml_snapshot!(conda_build_result, { + ".output_file" => "[redacted]", + ".build" => "[redacted]", + ".subdir" => "[redacted]", + }); + + // we also want to assert that we put intermediate_recipe.yaml in the debug dir + + assert!(build_dir.join("work").join("recipe.yaml").exists()); +} diff --git a/crates/pixi-build-backend/tests/integration/snapshots/integration__protocol__conda_build_v1.snap b/crates/pixi-build-backend/tests/integration/snapshots/integration__protocol__conda_build_v1.snap new file mode 100644 index 00000000..0f9eb721 --- /dev/null +++ b/crates/pixi-build-backend/tests/integration/snapshots/integration__protocol__conda_build_v1.snap @@ -0,0 +1,10 @@ +--- +source: crates/pixi-build-backend/tests/integration/protocol.rs +expression: conda_build_result +--- +output_file: "[redacted]" +input_globs: [] +name: minimal-package +version: 1.0.0 +build: "[redacted]" +subdir: "[redacted]" From 3562eb2f79eba0ff11ce93c66ae167e159e10a15 Mon Sep 17 00:00:00 2001 From: nichmor Date: Thu, 16 Oct 2025 17:52:31 +0300 Subject: [PATCH 4/5] misc: add a test for intermediate recipe --- .../tests/integration/protocol.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/crates/pixi-build-backend/tests/integration/protocol.rs b/crates/pixi-build-backend/tests/integration/protocol.rs index 2dc8efdb..28523736 100644 --- a/crates/pixi-build-backend/tests/integration/protocol.rs +++ b/crates/pixi-build-backend/tests/integration/protocol.rs @@ -13,9 +13,7 @@ use pixi_build_types::{ }, }; use rattler_build::console_utils::LoggingOutputHandler; -use rattler_conda_types::{ - ChannelUrl, Platform, utils::url_with_trailing_slash::UrlWithTrailingSlash, -}; +use rattler_conda_types::{ChannelUrl, Platform}; use serde_json::json; use tempfile::TempDir; use url::Url; @@ -271,16 +269,6 @@ async fn test_conda_build_v1() { .await .unwrap(); - dbg!(&build_dir); - - // list all dirs inside build_dir - let entries = fs_err::read_dir(&build_dir.join("work")).unwrap(); - for entry in entries { - let entry = entry.unwrap(); - let path = entry.path(); - dbg!(path); - } - insta::assert_yaml_snapshot!(conda_build_result, { ".output_file" => "[redacted]", ".build" => "[redacted]", From 548b45e3b3a261027e5c17bdc8d6ef248e51dffe Mon Sep 17 00:00:00 2001 From: nichmor Date: Thu, 16 Oct 2025 17:52:48 +0300 Subject: [PATCH 5/5] misc: add a test for intermediate recipe --- crates/pixi-build-backend/tests/integration/protocol.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/pixi-build-backend/tests/integration/protocol.rs b/crates/pixi-build-backend/tests/integration/protocol.rs index 28523736..b90a3eaa 100644 --- a/crates/pixi-build-backend/tests/integration/protocol.rs +++ b/crates/pixi-build-backend/tests/integration/protocol.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use crate::common::model::{convert_test_model_to_project_model_v1, load_project_model_from_json}; -use dirs::preference_dir; use imp::TestGenerateRecipe; use pixi_build_backend::{intermediate_backend::IntermediateBackend, protocol::Protocol}; use pixi_build_types::{