From e14228148e6995906f1be08defbfb63790f75d32 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 23 Dec 2024 22:00:58 -0500 Subject: [PATCH] Respect PEP 723 script lockfiles in uv run --- crates/uv-configuration/src/dev.rs | 4 +- crates/uv-scripts/src/lib.rs | 20 ++ crates/uv/src/commands/project/add.rs | 1 + crates/uv/src/commands/project/environment.rs | 138 ++++++-- .../uv/src/commands/project/install_target.rs | 136 +++++++- crates/uv/src/commands/project/mod.rs | 4 +- crates/uv/src/commands/project/run.rs | 298 +++++++++++------- crates/uv/src/commands/project/sync.rs | 133 +------- crates/uv/src/commands/tool/run.rs | 2 +- crates/uv/tests/it/run.rs | 187 +++++++++++ 10 files changed, 657 insertions(+), 266 deletions(-) diff --git a/crates/uv-configuration/src/dev.rs b/crates/uv-configuration/src/dev.rs index 80ae4f0640e4..cce967ba1f89 100644 --- a/crates/uv-configuration/src/dev.rs +++ b/crates/uv-configuration/src/dev.rs @@ -316,7 +316,7 @@ impl From for DevGroupsSpecification { /// The manifest of `dependency-groups` to include, taking into account the user-provided /// [`DevGroupsSpecification`] and the project-specific default groups. -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] pub struct DevGroupsManifest { /// The specification for the development dependencies. pub(crate) spec: DevGroupsSpecification, @@ -347,7 +347,7 @@ impl DevGroupsManifest { } /// Returns `true` if the group was enabled by default. - pub fn default(&self, group: &GroupName) -> bool { + pub fn is_default(&self, group: &GroupName) -> bool { if self.spec.contains(group) { // If the group was explicitly requested, then it wasn't enabled by default. false diff --git a/crates/uv-scripts/src/lib.rs b/crates/uv-scripts/src/lib.rs index e5a992e61b8a..ec3c40ec5fd7 100644 --- a/crates/uv-scripts/src/lib.rs +++ b/crates/uv-scripts/src/lib.rs @@ -54,6 +54,14 @@ impl Pep723Item { Self::Remote(_) => None, } } + + /// Return the PEP 723 script, if any. + pub fn as_script(&self) -> Option<&Pep723Script> { + match self { + Self::Script(script) => Some(script), + _ => None, + } + } } /// A PEP 723 script, including its [`Pep723Metadata`]. @@ -193,6 +201,18 @@ impl Pep723Script { Ok(()) } + + /// Return the [`Sources`] defined in the PEP 723 metadata. + pub fn sources(&self) -> &BTreeMap { + static EMPTY: BTreeMap = BTreeMap::new(); + + self.metadata + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.sources.as_ref()) + .unwrap_or(&EMPTY) + } } /// PEP 723 metadata as parsed from a `script` comment block. diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index d06b98f1082c..d56f3dff6314 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -593,6 +593,7 @@ pub(crate) async fn add( Target::Project(project, environment) => (project, environment), // If `--script`, exit early. There's no reason to lock and sync. Target::Script(script, _) => { + // TODO(charlie): Lock the script, if a lockfile already exists. writeln!( printer.stderr(), "Updated `{}`", diff --git a/crates/uv/src/commands/project/environment.rs b/crates/uv/src/commands/project/environment.rs index e4dd28ec2c01..f9e2d6f3408a 100644 --- a/crates/uv/src/commands/project/environment.rs +++ b/crates/uv/src/commands/project/environment.rs @@ -1,6 +1,7 @@ use tracing::debug; use crate::commands::pip::loggers::{InstallLogger, ResolveLogger}; +use crate::commands::project::install_target::InstallTarget; use crate::commands::project::{ resolve_environment, sync_environment, EnvironmentSpecification, ProjectError, }; @@ -9,10 +10,13 @@ use crate::settings::ResolverInstallerSettings; use uv_cache::{Cache, CacheBucket}; use uv_cache_key::{cache_digest, hash_digest}; use uv_client::Connectivity; -use uv_configuration::{Concurrency, PreviewMode, TrustedHost}; +use uv_configuration::{ + Concurrency, DevGroupsManifest, ExtrasSpecification, InstallOptions, PreviewMode, TrustedHost, +}; use uv_dispatch::SharedState; use uv_distribution_types::{Name, Resolution}; use uv_python::{Interpreter, PythonEnvironment}; +use uv_resolver::Installable; /// A [`PythonEnvironment`] stored in the cache. #[derive(Debug)] @@ -25,9 +29,8 @@ impl From for PythonEnvironment { } impl CachedEnvironment { - /// Get or create an [`CachedEnvironment`] based on a given set of requirements and a base - /// interpreter. - pub(crate) async fn get_or_create( + /// Get or create an [`CachedEnvironment`] based on a given set of requirements. + pub(crate) async fn from_spec( spec: EnvironmentSpecification<'_>, interpreter: Interpreter, settings: &ResolverInstallerSettings, @@ -43,21 +46,7 @@ impl CachedEnvironment { printer: Printer, preview: PreviewMode, ) -> Result { - // When caching, always use the base interpreter, rather than that of the virtual - // environment. - let interpreter = if let Some(interpreter) = interpreter.to_base_interpreter(cache)? { - debug!( - "Caching via base interpreter: `{}`", - interpreter.sys_executable().display() - ); - interpreter - } else { - debug!( - "Caching via interpreter: `{}`", - interpreter.sys_executable().display() - ); - interpreter - }; + let interpreter = Self::base_interpreter(interpreter, cache)?; // Resolve the requirements with the interpreter. let resolution = Resolution::from( @@ -78,6 +67,93 @@ impl CachedEnvironment { .await?, ); + Self::from_resolution( + resolution, + interpreter, + settings, + state, + install, + installer_metadata, + connectivity, + concurrency, + native_tls, + allow_insecure_host, + cache, + printer, + preview, + ) + .await + } + + /// Get or create an [`CachedEnvironment`] based on a given [`InstallTarget`]. + pub(crate) async fn from_lock( + target: InstallTarget<'_>, + extras: &ExtrasSpecification, + dev: &DevGroupsManifest, + install_options: InstallOptions, + settings: &ResolverInstallerSettings, + interpreter: Interpreter, + state: &SharedState, + install: Box, + installer_metadata: bool, + connectivity: Connectivity, + concurrency: Concurrency, + native_tls: bool, + allow_insecure_host: &[TrustedHost], + cache: &Cache, + printer: Printer, + preview: PreviewMode, + ) -> Result { + let interpreter = Self::base_interpreter(interpreter, cache)?; + + // Determine the tags, markers, and interpreter to use for resolution. + let tags = interpreter.tags()?; + let marker_env = interpreter.resolver_marker_environment(); + + // Read the lockfile. + let resolution = target.to_resolution( + &marker_env, + tags, + extras, + dev, + &settings.build_options, + &install_options, + )?; + + Self::from_resolution( + resolution, + interpreter, + settings, + state, + install, + installer_metadata, + connectivity, + concurrency, + native_tls, + allow_insecure_host, + cache, + printer, + preview, + ) + .await + } + + /// Get or create an [`CachedEnvironment`] based on a given [`Resolution`]. + pub(crate) async fn from_resolution( + resolution: Resolution, + interpreter: Interpreter, + settings: &ResolverInstallerSettings, + state: &SharedState, + install: Box, + installer_metadata: bool, + connectivity: Connectivity, + concurrency: Concurrency, + native_tls: bool, + allow_insecure_host: &[TrustedHost], + cache: &Cache, + printer: Printer, + preview: PreviewMode, + ) -> Result { // Hash the resolution by hashing the generated lockfile. // TODO(charlie): If the resolution contains any mutable metadata (like a path or URL // dependency), skip this step. @@ -144,4 +220,28 @@ impl CachedEnvironment { pub(crate) fn into_interpreter(self) -> Interpreter { self.0.into_interpreter() } + + /// Return the [`Interpreter`] to use for the cached environment, based on a given + /// [`Interpreter`]. + /// + /// When caching, always use the base interpreter, rather than that of the virtual + /// environment. + fn base_interpreter( + interpreter: Interpreter, + cache: &Cache, + ) -> Result { + if let Some(interpreter) = interpreter.to_base_interpreter(cache)? { + debug!( + "Caching via base interpreter: `{}`", + interpreter.sys_executable().display() + ); + Ok(interpreter) + } else { + debug!( + "Caching via interpreter: `{}`", + interpreter.sys_executable().display() + ); + Ok(interpreter) + } + } } diff --git a/crates/uv/src/commands/project/install_target.rs b/crates/uv/src/commands/project/install_target.rs index 69999d4d77cf..78d88640960f 100644 --- a/crates/uv/src/commands/project/install_target.rs +++ b/crates/uv/src/commands/project/install_target.rs @@ -1,9 +1,14 @@ +use std::borrow::Cow; use std::path::Path; +use std::str::FromStr; use itertools::Either; use uv_normalize::PackageName; +use uv_pypi_types::{LenientRequirement, VerbatimParsedUrl}; use uv_resolver::{Installable, Lock, Package}; +use uv_scripts::Pep723Script; +use uv_workspace::pyproject::{DependencyGroupSpecifier, Source, Sources, ToolUvSources}; use uv_workspace::Workspace; /// A target that can be installed from a lockfile. @@ -25,6 +30,11 @@ pub(crate) enum InstallTarget<'lock> { workspace: &'lock Workspace, lock: &'lock Lock, }, + /// A PEP 723 script. + Script { + script: &'lock Pep723Script, + lock: &'lock Lock, + }, } impl<'lock> Installable<'lock> for InstallTarget<'lock> { @@ -33,6 +43,7 @@ impl<'lock> Installable<'lock> for InstallTarget<'lock> { Self::Project { workspace, .. } => workspace.install_path(), Self::Workspace { workspace, .. } => workspace.install_path(), Self::NonProjectWorkspace { workspace, .. } => workspace.install_path(), + Self::Script { script, .. } => script.path.parent().unwrap(), } } @@ -41,24 +52,28 @@ impl<'lock> Installable<'lock> for InstallTarget<'lock> { Self::Project { lock, .. } => lock, Self::Workspace { lock, .. } => lock, Self::NonProjectWorkspace { lock, .. } => lock, + Self::Script { lock, .. } => lock, } } fn roots(&self) -> impl Iterator { match self { - Self::Project { name, .. } => Either::Right(Either::Left(std::iter::once(*name))), - Self::NonProjectWorkspace { lock, .. } => Either::Left(lock.members().iter()), + Self::Project { name, .. } => Either::Left(Either::Left(std::iter::once(*name))), + Self::NonProjectWorkspace { lock, .. } => { + Either::Left(Either::Right(lock.members().iter())) + } Self::Workspace { lock, .. } => { // Identify the workspace members. // // The members are encoded directly in the lockfile, unless the workspace contains a // single member at the root, in which case, we identify it by its source. if lock.members().is_empty() { - Either::Right(Either::Right(lock.root().into_iter().map(Package::name))) + Either::Right(Either::Left(lock.root().into_iter().map(Package::name))) } else { - Either::Left(lock.members().iter()) + Either::Left(Either::Right(lock.members().iter())) } } + Self::Script { .. } => Either::Right(Either::Right(std::iter::empty())), } } @@ -67,17 +82,120 @@ impl<'lock> Installable<'lock> for InstallTarget<'lock> { Self::Project { name, .. } => Some(name), Self::Workspace { .. } => None, Self::NonProjectWorkspace { .. } => None, + Self::Script { .. } => None, } } } impl<'lock> InstallTarget<'lock> { - /// Return the [`Workspace`] of the target. - pub(crate) fn workspace(&self) -> &'lock Workspace { + /// Return an iterator over all [`Sources`] defined by the target. + pub(crate) fn sources(&self) -> impl Iterator { + match self { + Self::Project { workspace, .. } + | Self::Workspace { workspace, .. } + | Self::NonProjectWorkspace { workspace, .. } => { + Either::Left(workspace.sources().values().flat_map(Sources::iter).chain( + workspace.packages().values().flat_map(|member| { + member + .pyproject_toml() + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.sources.as_ref()) + .map(ToolUvSources::inner) + .into_iter() + .flat_map(|sources| sources.values().flat_map(Sources::iter)) + }), + )) + } + Self::Script { script, .. } => { + Either::Right(script.sources().values().flat_map(Sources::iter)) + } + } + } + + /// Return an iterator over all requirements defined by the target. + pub(crate) fn requirements( + &self, + ) -> impl Iterator>> { match self { - Self::Project { workspace, .. } => workspace, - Self::Workspace { workspace, .. } => workspace, - Self::NonProjectWorkspace { workspace, .. } => workspace, + Self::Project { workspace, .. } + | Self::Workspace { workspace, .. } + | Self::NonProjectWorkspace { workspace, .. } => { + Either::Left( + // Iterate over the non-member requirements in the workspace. + workspace + .requirements() + .into_iter() + .map(Cow::Owned) + .chain(workspace.dependency_groups().ok().into_iter().flat_map( + |dependency_groups| { + dependency_groups.into_values().flatten().map(Cow::Owned) + }, + )) + .chain(workspace.packages().values().flat_map(|member| { + // Iterate over all dependencies in each member. + let dependencies = member + .pyproject_toml() + .project + .as_ref() + .and_then(|project| project.dependencies.as_ref()) + .into_iter() + .flatten(); + let optional_dependencies = member + .pyproject_toml() + .project + .as_ref() + .and_then(|project| project.optional_dependencies.as_ref()) + .into_iter() + .flat_map(|optional| optional.values()) + .flatten(); + let dependency_groups = member + .pyproject_toml() + .dependency_groups + .as_ref() + .into_iter() + .flatten() + .flat_map(|(_, dependencies)| { + dependencies.iter().filter_map(|specifier| { + if let DependencyGroupSpecifier::Requirement(requirement) = + specifier + { + Some(requirement) + } else { + None + } + }) + }); + let dev_dependencies = member + .pyproject_toml() + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.dev_dependencies.as_ref()) + .into_iter() + .flatten(); + dependencies + .chain(optional_dependencies) + .chain(dependency_groups) + .filter_map(|requires_dist| { + LenientRequirement::::from_str(requires_dist) + .map(uv_pep508::Requirement::from) + .map(Cow::Owned) + .ok() + }) + .chain(dev_dependencies.map(Cow::Borrowed)) + })), + ) + } + Self::Script { script, .. } => Either::Right( + script + .metadata + .dependencies + .iter() + .flatten() + .map(Cow::Borrowed), + ), } } } diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index d3506c6164f8..67c78faee22f 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -271,7 +271,7 @@ impl std::fmt::Display for ConflictError { self.conflicts .iter() .map(|conflict| match conflict { - ConflictPackage::Group(ref group) if self.dev.default(group) => + ConflictPackage::Group(ref group) if self.dev.is_default(group) => format!("`{group}` (enabled by default)"), ConflictPackage::Group(ref group) => format!("`{group}`"), ConflictPackage::Extra(..) => unreachable!(), @@ -290,7 +290,7 @@ impl std::fmt::Display for ConflictError { .map(|(i, conflict)| { let conflict = match conflict { ConflictPackage::Extra(ref extra) => format!("extra `{extra}`"), - ConflictPackage::Group(ref group) if self.dev.default(group) => { + ConflictPackage::Group(ref group) if self.dev.is_default(group) => { format!("group `{group}` (enabled by default)") } ConflictPackage::Group(ref group) => format!("group `{group}`"), diff --git a/crates/uv/src/commands/project/run.rs b/crates/uv/src/commands/project/run.rs index 04f6cd9ac4b5..a21b66ea05bd 100644 --- a/crates/uv/src/commands/project/run.rs +++ b/crates/uv/src/commands/project/run.rs @@ -17,8 +17,8 @@ use uv_cache::Cache; use uv_cli::ExternalCommand; use uv_client::{BaseClientBuilder, Connectivity}; use uv_configuration::{ - Concurrency, DevGroupsSpecification, EditableMode, ExtrasSpecification, GroupsSpecification, - InstallOptions, LowerBound, PreviewMode, SourceStrategy, TrustedHost, + Concurrency, DevGroupsManifest, DevGroupsSpecification, EditableMode, ExtrasSpecification, + GroupsSpecification, InstallOptions, LowerBound, PreviewMode, SourceStrategy, TrustedHost, }; use uv_dispatch::SharedState; use uv_distribution::LoweredRequirement; @@ -200,109 +200,57 @@ pub(crate) async fn run( .await? .into_interpreter(); - // Determine the working directory for the script. - let script_dir = match &script { - Pep723Item::Script(script) => std::path::absolute(&script.path)? - .parent() - .expect("script path has no parent") - .to_owned(), - Pep723Item::Stdin(..) | Pep723Item::Remote(..) => std::env::current_dir()?, - }; - let script = script.into_metadata(); - - // Install the script requirements, if necessary. Otherwise, use an isolated environment. - if let Some(dependencies) = script.dependencies { - // Collect any `tool.uv.index` from the script. - let empty = Vec::default(); - let script_indexes = match settings.sources { - SourceStrategy::Enabled => script - .tool - .as_ref() - .and_then(|tool| tool.uv.as_ref()) - .and_then(|uv| uv.top_level.index.as_deref()) - .unwrap_or(&empty), - SourceStrategy::Disabled => &empty, - }; + // If a lockfile already exists, lock the script. + if let Some(target) = script + .as_script() + .map(LockTarget::from) + .filter(|target| target.lock_path().is_file()) + { + debug!("Found existing lockfile for script"); - // Collect any `tool.uv.sources` from the script. - let empty = BTreeMap::default(); - let script_sources = match settings.sources { - SourceStrategy::Enabled => script - .tool - .as_ref() - .and_then(|tool| tool.uv.as_ref()) - .and_then(|uv| uv.sources.as_ref()) - .unwrap_or(&empty), - SourceStrategy::Disabled => &empty, + // Determine the lock mode. + let mode = if frozen { + LockMode::Frozen + } else if locked { + LockMode::Locked(&interpreter) + } else { + LockMode::Write(&interpreter) }; - let requirements = dependencies - .into_iter() - .flat_map(|requirement| { - LoweredRequirement::from_non_workspace_requirement( - requirement, - script_dir.as_ref(), - script_sources, - script_indexes, - &settings.index_locations, - LowerBound::Allow, - ) - .map_ok(LoweredRequirement::into_inner) - }) - .collect::>()?; - let constraints = script - .tool - .as_ref() - .and_then(|tool| tool.uv.as_ref()) - .and_then(|uv| uv.constraint_dependencies.as_ref()) - .into_iter() - .flatten() - .cloned() - .flat_map(|requirement| { - LoweredRequirement::from_non_workspace_requirement( - requirement, - script_dir.as_ref(), - script_sources, - script_indexes, - &settings.index_locations, - LowerBound::Allow, - ) - .map_ok(LoweredRequirement::into_inner) - }) - .collect::, _>>()?; - let overrides = script - .tool - .as_ref() - .and_then(|tool| tool.uv.as_ref()) - .and_then(|uv| uv.override_dependencies.as_ref()) - .into_iter() - .flatten() - .cloned() - .flat_map(|requirement| { - LoweredRequirement::from_non_workspace_requirement( - requirement, - script_dir.as_ref(), - script_sources, - script_indexes, - &settings.index_locations, - LowerBound::Allow, - ) - .map_ok(LoweredRequirement::into_inner) - }) - .collect::, _>>()?; - - let spec = - RequirementsSpecification::from_overrides(requirements, constraints, overrides); - let result = CachedEnvironment::get_or_create( - EnvironmentSpecification::from(spec), - interpreter, - &settings, + // Generate a lockfile. + let lock = project::lock::do_safe_lock( + mode, + target, + settings.as_ref().into(), + LowerBound::Allow, &state, if show_resolution { Box::new(DefaultResolveLogger) } else { Box::new(SummaryResolveLogger) }, + connectivity, + concurrency, + native_tls, + allow_insecure_host, + cache, + printer, + preview, + ) + .await? + .into_lock(); + + let result = CachedEnvironment::from_lock( + InstallTarget::Script { + script: script.as_script().unwrap(), + lock: &lock, + }, + &ExtrasSpecification::default(), + &DevGroupsManifest::default(), + InstallOptions::default(), + &settings, + interpreter, + &state, if show_resolution { Box::new(DefaultInstallLogger) } else { @@ -331,19 +279,151 @@ pub(crate) async fn run( Some(environment.into_interpreter()) } else { - // Create a virtual environment. - temp_dir = cache.venv_dir()?; - let environment = uv_virtualenv::create_venv( - temp_dir.path(), - interpreter, - uv_virtualenv::Prompt::None, - false, - false, - false, - false, - )?; + // Determine the working directory for the script. + let script_dir = match &script { + Pep723Item::Script(script) => std::path::absolute(&script.path)? + .parent() + .expect("script path has no parent") + .to_owned(), + Pep723Item::Stdin(..) | Pep723Item::Remote(..) => std::env::current_dir()?, + }; + let script = script.into_metadata(); + + // Install the script requirements, if necessary. Otherwise, use an isolated environment. + if let Some(dependencies) = script.dependencies { + // Collect any `tool.uv.index` from the script. + let empty = Vec::default(); + let script_indexes = match settings.sources { + SourceStrategy::Enabled => script + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.top_level.index.as_deref()) + .unwrap_or(&empty), + SourceStrategy::Disabled => &empty, + }; - Some(environment.into_interpreter()) + // Collect any `tool.uv.sources` from the script. + let empty = BTreeMap::default(); + let script_sources = match settings.sources { + SourceStrategy::Enabled => script + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.sources.as_ref()) + .unwrap_or(&empty), + SourceStrategy::Disabled => &empty, + }; + + let requirements = dependencies + .into_iter() + .flat_map(|requirement| { + LoweredRequirement::from_non_workspace_requirement( + requirement, + script_dir.as_ref(), + script_sources, + script_indexes, + &settings.index_locations, + LowerBound::Allow, + ) + .map_ok(LoweredRequirement::into_inner) + }) + .collect::>()?; + let constraints = script + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.constraint_dependencies.as_ref()) + .into_iter() + .flatten() + .cloned() + .flat_map(|requirement| { + LoweredRequirement::from_non_workspace_requirement( + requirement, + script_dir.as_ref(), + script_sources, + script_indexes, + &settings.index_locations, + LowerBound::Allow, + ) + .map_ok(LoweredRequirement::into_inner) + }) + .collect::, _>>()?; + let overrides = script + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.override_dependencies.as_ref()) + .into_iter() + .flatten() + .cloned() + .flat_map(|requirement| { + LoweredRequirement::from_non_workspace_requirement( + requirement, + script_dir.as_ref(), + script_sources, + script_indexes, + &settings.index_locations, + LowerBound::Allow, + ) + .map_ok(LoweredRequirement::into_inner) + }) + .collect::, _>>()?; + + let spec = + RequirementsSpecification::from_overrides(requirements, constraints, overrides); + let result = CachedEnvironment::from_spec( + EnvironmentSpecification::from(spec), + interpreter, + &settings, + &state, + if show_resolution { + Box::new(DefaultResolveLogger) + } else { + Box::new(SummaryResolveLogger) + }, + if show_resolution { + Box::new(DefaultInstallLogger) + } else { + Box::new(SummaryInstallLogger) + }, + installer_metadata, + connectivity, + concurrency, + native_tls, + allow_insecure_host, + cache, + printer, + preview, + ) + .await; + + let environment = match result { + Ok(resolution) => resolution, + Err(ProjectError::Operation(err)) => { + return diagnostics::OperationDiagnostic::with_context("script") + .report(err) + .map_or(Ok(ExitStatus::Failure), |err| Err(err.into())) + } + Err(err) => return Err(err.into()), + }; + + Some(environment.into_interpreter()) + } else { + // Create a virtual environment. + temp_dir = cache.venv_dir()?; + let environment = uv_virtualenv::create_venv( + temp_dir.path(), + interpreter, + uv_virtualenv::Prompt::None, + false, + false, + false, + false, + )?; + + Some(environment.into_interpreter()) + } } } else { None @@ -846,7 +926,7 @@ pub(crate) async fn run( Some(spec) => { debug!("Syncing ephemeral requirements"); - let result = CachedEnvironment::get_or_create( + let result = CachedEnvironment::from_spec( EnvironmentSpecification::from(spec).with_lock( lock.as_ref() .map(|(lock, install_path)| (lock, install_path.as_ref())), diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index 78a3595c3ff6..755a4314c3b7 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -1,6 +1,4 @@ -use std::borrow::Cow; use std::path::Path; -use std::str::FromStr; use anyhow::{Context, Result}; use itertools::Itertools; @@ -18,16 +16,14 @@ use uv_distribution_types::{ }; use uv_installer::SitePackages; use uv_normalize::PackageName; -use uv_pep508::{MarkerTree, Requirement, VersionOrUrl}; -use uv_pypi_types::{ - LenientRequirement, ParsedArchiveUrl, ParsedGitUrl, ParsedUrl, VerbatimParsedUrl, -}; +use uv_pep508::{MarkerTree, VersionOrUrl}; +use uv_pypi_types::{ParsedArchiveUrl, ParsedGitUrl, ParsedUrl}; use uv_python::{PythonDownloads, PythonEnvironment, PythonPreference, PythonRequest}; use uv_resolver::{FlatIndex, Installable}; use uv_settings::PythonInstallMirrors; use uv_types::{BuildIsolation, HashStrategy}; use uv_warnings::warn_user; -use uv_workspace::pyproject::{DependencyGroupSpecifier, Source, Sources, ToolUvSources}; +use uv_workspace::pyproject::Source; use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, Workspace}; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger}; @@ -368,7 +364,7 @@ pub(super) async fn do_sync( } // Populate credentials from the workspace. - store_credentials_from_workspace(target.workspace()); + store_credentials_from_workspace(target); // Initialize the registry client. let client = RegistryClientBuilder::new(cache.clone()) @@ -526,9 +522,9 @@ fn apply_editable_mode(resolution: Resolution, editable: EditableMode) -> Resolu /// /// These credentials can come from any of `tool.uv.sources`, `tool.uv.dev-dependencies`, /// `project.dependencies`, and `project.optional-dependencies`. -fn store_credentials_from_workspace(workspace: &Workspace) { - // Iterate over any sources in the workspace root. - for source in workspace.sources().values().flat_map(Sources::iter) { +fn store_credentials_from_workspace(target: InstallTarget<'_>) { + // Iterate over any sources in the target. + for source in target.sources() { match source { Source::Git { git, .. } => { uv_git::store_credentials_from_url(git); @@ -540,29 +536,8 @@ fn store_credentials_from_workspace(workspace: &Workspace) { } } - // Iterate over any dependencies defined in the workspace root. - for requirement in &workspace.requirements() { - let Some(VersionOrUrl::Url(url)) = &requirement.version_or_url else { - continue; - }; - match &url.parsed_url { - ParsedUrl::Git(ParsedGitUrl { url, .. }) => { - uv_git::store_credentials_from_url(url.repository()); - } - ParsedUrl::Archive(ParsedArchiveUrl { url, .. }) => { - uv_auth::store_credentials_from_url(url); - } - _ => {} - } - } - - // Iterate over any dependency groups defined in the workspace root. - for requirement in workspace - .dependency_groups() - .ok() - .iter() - .flat_map(|groups| groups.values().flat_map(|group| group.iter())) - { + // Iterate over any dependencies defined in the target. + for requirement in target.requirements() { let Some(VersionOrUrl::Url(url)) = &requirement.version_or_url else { continue; }; @@ -576,94 +551,4 @@ fn store_credentials_from_workspace(workspace: &Workspace) { _ => {} } } - - // Iterate over each workspace member. - for member in workspace.packages().values() { - // Iterate over the `tool.uv.sources`. - for source in member - .pyproject_toml() - .tool - .as_ref() - .and_then(|tool| tool.uv.as_ref()) - .and_then(|uv| uv.sources.as_ref()) - .map(ToolUvSources::inner) - .iter() - .flat_map(|sources| sources.values().flat_map(Sources::iter)) - { - match source { - Source::Git { git, .. } => { - uv_git::store_credentials_from_url(git); - } - Source::Url { url, .. } => { - uv_auth::store_credentials_from_url(url); - } - _ => {} - } - } - - // Iterate over all dependencies. - let dependencies = member - .pyproject_toml() - .project - .as_ref() - .and_then(|project| project.dependencies.as_ref()) - .into_iter() - .flatten(); - let optional_dependencies = member - .pyproject_toml() - .project - .as_ref() - .and_then(|project| project.optional_dependencies.as_ref()) - .into_iter() - .flat_map(|optional| optional.values()) - .flatten(); - let dependency_groups = member - .pyproject_toml() - .dependency_groups - .as_ref() - .into_iter() - .flatten() - .flat_map(|(_, dependencies)| { - dependencies.iter().filter_map(|specifier| { - if let DependencyGroupSpecifier::Requirement(requirement) = specifier { - Some(requirement) - } else { - None - } - }) - }); - let dev_dependencies = member - .pyproject_toml() - .tool - .as_ref() - .and_then(|tool| tool.uv.as_ref()) - .and_then(|uv| uv.dev_dependencies.as_ref()) - .into_iter() - .flatten(); - - for requirement in dependencies - .chain(optional_dependencies) - .chain(dependency_groups) - .filter_map(|requires_dist| { - LenientRequirement::::from_str(requires_dist) - .map(Requirement::from) - .map(Cow::Owned) - .ok() - }) - .chain(dev_dependencies.map(Cow::Borrowed)) - { - let Some(VersionOrUrl::Url(url)) = &requirement.version_or_url else { - continue; - }; - match &url.parsed_url { - ParsedUrl::Git(ParsedGitUrl { url, .. }) => { - uv_git::store_credentials_from_url(url.repository()); - } - ParsedUrl::Archive(ParsedArchiveUrl { url, .. }) => { - uv_auth::store_credentials_from_url(url); - } - _ => {} - } - } - } } diff --git a/crates/uv/src/commands/tool/run.rs b/crates/uv/src/commands/tool/run.rs index 8d97bc10485d..c45a4ecddd44 100644 --- a/crates/uv/src/commands/tool/run.rs +++ b/crates/uv/src/commands/tool/run.rs @@ -621,7 +621,7 @@ async fn get_or_create_environment( // TODO(zanieb): When implementing project-level tools, discover the project and check if it has the tool. // TODO(zanieb): Determine if we should layer on top of the project environment if it is present. - let environment = CachedEnvironment::get_or_create( + let environment = CachedEnvironment::from_spec( EnvironmentSpecification::from(spec), interpreter, settings, diff --git a/crates/uv/tests/it/run.rs b/crates/uv/tests/it/run.rs index 462d063a3551..c4a230062e5f 100644 --- a/crates/uv/tests/it/run.rs +++ b/crates/uv/tests/it/run.rs @@ -323,6 +323,9 @@ fn run_pep723_script() -> Result<()> { Resolved 1 package in [TIME] "###); + // But neither invocation should create a lockfile. + assert!(!context.temp_dir.child("main.py.lock").exists()); + // Otherwise, the script requirements should _not_ be available, but the project requirements // should. let test_non_script = context.temp_dir.child("main.py"); @@ -773,6 +776,190 @@ fn run_pep723_script_overrides() -> Result<()> { Ok(()) } +/// Run a PEP 723-compatible script with a lockfile. +#[test] +fn run_pep723_script_lock() -> Result<()> { + let context = TestContext::new("3.12"); + + let test_script = context.temp_dir.child("main.py"); + test_script.write_str(indoc! { r#" + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "iniconfig", + # ] + # /// + + import iniconfig + + print("Hello, world!") + "# + })?; + + // Explicitly lock the script. + uv_snapshot!(context.filters(), context.lock().arg("--script").arg("main.py"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + "###); + + let lock = context.read("main.py.lock"); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.11" + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [manifest] + requirements = [{ name = "iniconfig" }] + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, + ] + "### + ); + }); + + uv_snapshot!(context.filters(), context.run().arg("main.py"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + Hello, world! + + ----- stderr ----- + Reading inline script metadata from `main.py` + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + iniconfig==2.0.0 + "###); + + // Modify the metadata. + test_script.write_str(indoc! { r#" + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "anyio", + # ] + # /// + + import anyio + + print("Hello, world!") + "# + })?; + + // Re-running the script with `--locked` should error. + uv_snapshot!(context.filters(), context.run().arg("--locked").arg("main.py"), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Reading inline script metadata from `main.py` + Resolved 3 packages in [TIME] + error: The lockfile at `uv.lock` needs to be updated, but `--locked` was provided. To update the lockfile, run `uv lock`. + "###); + + // Re-running the script with `--frozen` should also error, but at runtime. + uv_snapshot!(context.filters(), context.run().arg("--frozen").arg("main.py"), @r###" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Reading inline script metadata from `main.py` + warning: `--frozen` is a no-op for Python scripts with inline metadata, which always run in isolation + Traceback (most recent call last): + File "[TEMP_DIR]/main.py", line 8, in + import anyio + ModuleNotFoundError: No module named 'anyio' + "###); + + // Re-running the script should update the lockfile. + uv_snapshot!(context.filters(), context.run().arg("main.py"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + Hello, world! + + ----- stderr ----- + Reading inline script metadata from `main.py` + Resolved 3 packages in [TIME] + Prepared 3 packages in [TIME] + Installed 3 packages in [TIME] + + anyio==4.3.0 + + idna==3.6 + + sniffio==1.3.1 + "###); + + let lock = context.read("main.py.lock"); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.11" + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [manifest] + requirements = [{ name = "anyio" }] + + [[package]] + name = "anyio" + version = "4.3.0" + source = { registry = "https://pypi.org/simple" } + dependencies = [ + { name = "idna" }, + { name = "sniffio" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/db/4d/3970183622f0330d3c23d9b8a5f52e365e50381fd484d08e3285104333d3/anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6", size = 159642 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/14/fd/2f20c40b45e4fb4324834aea24bd4afdf1143390242c0b33774da0e2e34f/anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8", size = 85584 }, + ] + + [[package]] + name = "idna" + version = "3.6" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", size = 175426 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", size = 61567 }, + ] + + [[package]] + name = "sniffio" + version = "1.3.1" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, + ] + "### + ); + }); + + Ok(()) +} + /// With `managed = false`, we should avoid installing the project itself. #[test] fn run_managed_false() -> Result<()> {