Skip to content
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 114 additions & 19 deletions crates/pixi_core/src/lock_file/outdated.rs
Original file line number Diff line number Diff line change
@@ -1,25 +1,87 @@
use std::collections::{HashMap, HashSet};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};

use super::{verify_environment_satisfiability, verify_platform_satisfiability};
use super::{
CondaPrefixUpdater, resolve::build_dispatch::LazyBuildDispatchDependencies,
satisfiability::VerifySatisfiabilityContext, verify_environment_satisfiability,
verify_platform_satisfiability,
};
use crate::{
Workspace,
lock_file::satisfiability::{EnvironmentUnsat, verify_solve_group_satisfiability},
workspace::{Environment, SolveGroup},
};
use async_once_cell::OnceCell as AsyncOnceCell;
use fancy_display::FancyDisplay;
use itertools::Itertools;
use once_cell::sync::OnceCell;
use pixi_command_dispatcher::CommandDispatcher;
use pixi_consts::consts;
use pixi_manifest::FeaturesExt;
use pixi_manifest::{EnvironmentName, FeaturesExt};
use pixi_uv_context::UvResolutionContext;
use rattler_conda_types::Platform;
use rattler_lock::{LockFile, LockedPackageRef};
use uv_client::RegistryClient;
use uv_configuration::BuildOptions;
use uv_distribution_types::{DependencyMetadata, IndexLocations};
use uv_resolver::FlatIndex;

/// Cache for build-related resources that can be shared between
/// satisfiability checking and PyPI resolution.
pub struct EnvironmentBuildCache {
/// Lazily initialized build dispatch dependencies (interpreter, env, etc.)
pub lazy_build_dispatch_deps: LazyBuildDispatchDependencies,
/// Optional conda prefix updater (created during satisfiability checking)
pub conda_prefix_updater: OnceCell<CondaPrefixUpdater>,
/// Cached registry client (with Online connectivity)
pub registry_client: OnceCell<Arc<RegistryClient>>,
/// Cached index locations
pub index_locations: OnceCell<IndexLocations>,
/// Cached build options
pub build_options: OnceCell<BuildOptions>,
/// Cached flat index (populated lazily when needed, async initialization)
pub flat_index: AsyncOnceCell<FlatIndex>,
/// Cached dependency metadata
pub dependency_metadata: OnceCell<DependencyMetadata>,
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure we need to cache most of these structs, as the most interesting things are chached to disk. I think there are non in-memory caches, only the InFlight which is not here and maybe in the RegistryClient. I'd rather not introduce another struct containing too many OnceCell's. Maybe we can figure out what we need to keep? I think the LazyBuildDispatch is something we should cache?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed all other fields except conda_prefix_updater. I think it's ok to keep it, because it contains an in-memory result ( OnceCell field) of installing a prefix environment.


impl Default for EnvironmentBuildCache {
fn default() -> Self {
Self {
lazy_build_dispatch_deps: LazyBuildDispatchDependencies::default(),
conda_prefix_updater: OnceCell::new(),
registry_client: OnceCell::new(),
index_locations: OnceCell::new(),
build_options: OnceCell::new(),
flat_index: AsyncOnceCell::new(),
dependency_metadata: OnceCell::new(),
}
}
}

/// Key for the build cache, combining environment name and platform.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct BuildCacheKey {
pub environment: EnvironmentName,
pub platform: Platform,
}

impl BuildCacheKey {
pub fn new(environment: EnvironmentName, platform: Platform) -> Self {
Self {
environment,
platform,
}
}
}

/// A struct that contains information about specific outdated environments.
///
/// Use the [`OutdatedEnvironments::from_project_and_lock_file`] to create an
/// instance of this struct by examining the project and lock-file and finding
/// any mismatches.
#[derive(Debug)]
pub struct OutdatedEnvironments<'p> {
/// The conda environments that are considered out of date with the
/// lock-file.
Expand All @@ -33,6 +95,14 @@ pub struct OutdatedEnvironments<'p> {
/// discarded. This is the case for instance when the order of the
/// channels changed.
pub disregard_locked_content: DisregardLockedContent<'p>,

/// Lazily initialized UV context for building dynamic metadata.
/// This is shared between satisfiability checking and pypi resolution.
pub uv_context: OnceCell<UvResolutionContext>,

/// Per-environment-platform build caches for sharing resources between
/// satisfiability checking and PyPI resolution.
pub build_caches: HashMap<BuildCacheKey, Arc<EnvironmentBuildCache>>,
}

/// A struct that stores whether the locked content of certain environments
Expand Down Expand Up @@ -66,11 +136,15 @@ impl<'p> OutdatedEnvironments<'p> {
lock_file: &LockFile,
) -> Self {
// Find all targets that are not satisfied by the lock-file
let UnsatisfiableTargets {
mut outdated_conda,
mut outdated_pypi,
disregard_locked_content,
} = find_unsatisfiable_targets(workspace, command_dispatcher, lock_file).await;
let (
UnsatisfiableTargets {
mut outdated_conda,
mut outdated_pypi,
disregard_locked_content,
},
uv_context,
build_caches,
) = find_unsatisfiable_targets(workspace, command_dispatcher, lock_file).await;

// Extend the outdated targets to include the solve groups
let (mut conda_solve_groups_out_of_date, mut pypi_solve_groups_out_of_date) =
Expand Down Expand Up @@ -118,6 +192,8 @@ impl<'p> OutdatedEnvironments<'p> {
conda: outdated_conda,
pypi: outdated_pypi,
disregard_locked_content,
uv_context,
build_caches,
}
}

Expand All @@ -137,13 +213,30 @@ struct UnsatisfiableTargets<'p> {

/// Find all targets (combination of environment and platform) who's
/// requirements in the `project` are not satisfied by the `lock_file`.
///
/// Returns the unsatisfiable targets, the lazily-initialized UV context
/// (which may have been initialized during satisfiability checking), and
/// build caches for each environment.
async fn find_unsatisfiable_targets<'p>(
project: &'p Workspace,
command_dispatcher: CommandDispatcher,
lock_file: &LockFile,
) -> UnsatisfiableTargets<'p> {
) -> (
UnsatisfiableTargets<'p>,
OnceCell<UvResolutionContext>,
HashMap<BuildCacheKey, Arc<EnvironmentBuildCache>>,
) {
let mut verified_environments = HashMap::new();
let mut unsatisfiable_targets = UnsatisfiableTargets::default();

// Create UV context lazily for building dynamic metadata
let uv_context: OnceCell<UvResolutionContext> = OnceCell::new();

// Create build caches for sharing between satisfiability and resolution
let mut build_caches: HashMap<BuildCacheKey, Arc<EnvironmentBuildCache>> = HashMap::new();

let project_config = project.config();

for environment in project.environments() {
let platforms = environment.platforms();

Expand Down Expand Up @@ -227,15 +320,17 @@ async fn find_unsatisfiable_targets<'p>(

// Verify each individual platform
for platform in platforms {
match verify_platform_satisfiability(
&environment,
command_dispatcher.clone(),
locked_environment,
let mut ctx = VerifySatisfiabilityContext {
environment: &environment,
command_dispatcher: command_dispatcher.clone(),
platform,
project.root(),
)
.await
{
project_root: project.root(),
uv_context: &uv_context,
config: project_config,
project_env_vars: project.env_vars().clone(),
build_caches: &mut build_caches,
};
match verify_platform_satisfiability(&mut ctx, locked_environment).await {
Ok(verified_env) => {
verified_environments.insert((environment.clone(), platform), verified_env);
}
Expand Down Expand Up @@ -317,7 +412,7 @@ async fn find_unsatisfiable_targets<'p>(
.insert(platform);
}

unsatisfiable_targets
(unsatisfiable_targets, uv_context, build_caches)
}

/// Given a mapping of outdated targets, construct a new mapping of all the
Expand Down
Loading
Loading