Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use sys.path for package discovery #9849

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions crates/uv-dispatch/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use uv_distribution_types::{
SourceDist, VersionOrUrlRef,
};
use uv_git::GitResolver;
use uv_installer::{Installer, Plan, Planner, Preparer, SitePackages};
use uv_installer::{InstalledPackages, Installer, Plan, Planner, Preparer};
use uv_pypi_types::{Conflicts, Requirement};
use uv_python::{Interpreter, PythonEnvironment};
use uv_resolver::{
Expand Down Expand Up @@ -229,15 +229,15 @@ impl<'a> BuildContext for BuildDispatch<'a> {
let tags = self.interpreter.tags()?;

// Determine the set of installed packages.
let site_packages = SitePackages::from_environment(venv)?;
let installed_packages = InstalledPackages::from_environment(venv)?;

let Plan {
cached,
remote,
reinstalls,
extraneous: _,
} = Planner::new(resolution).build(
site_packages,
installed_packages,
&Reinstall::default(),
self.build_options,
self.hasher,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use crate::satisfies::RequirementSatisfaction;
///
/// Packages are indexed by both name and (for editable installs) URL.
#[derive(Debug, Clone)]
pub struct SitePackages {
pub struct InstalledPackages {
interpreter: Interpreter,
/// The vector of all installed distributions. The `by_name` and `by_url` indices index into
/// this vector. The vector may contain `None` values, which represent distributions that were
Expand All @@ -39,7 +39,7 @@ pub struct SitePackages {
by_url: FxHashMap<Url, Vec<usize>>,
}

impl SitePackages {
impl InstalledPackages {
/// Build an index of installed packages from the given Python environment.
pub fn from_environment(environment: &PythonEnvironment) -> Result<Self> {
Self::from_interpreter(environment.interpreter())
Expand All @@ -48,15 +48,15 @@ impl SitePackages {
/// Build an index of installed packages from the given Python executable.
pub fn from_interpreter(interpreter: &Interpreter) -> Result<Self> {
let mut distributions: Vec<Option<InstalledDist>> = Vec::new();
let mut by_name = FxHashMap::default();
let mut by_url = FxHashMap::default();
let mut by_name: FxHashMap<PackageName, Vec<usize>> = FxHashMap::default();
let mut by_url: FxHashMap<Url, Vec<usize>> = FxHashMap::default();

for site_packages in interpreter.site_packages() {
for import_path_entry in interpreter.sys_path() {
// Read the site-packages directory.
let site_packages = match fs::read_dir(site_packages) {
Ok(site_packages) => {
let ordered_directory_paths = match fs::read_dir(import_path_entry) {
Ok(import_path_entry) => {
// Collect sorted directory paths; `read_dir` is not stable across platforms
let dist_likes: BTreeSet<_> = site_packages
let dist_likes: BTreeSet<_> = import_path_entry
.filter_map(|read_dir| match read_dir {
Ok(entry) => match entry.file_type() {
Ok(file_type) => (file_type.is_dir()
Expand All @@ -73,18 +73,14 @@ impl SitePackages {
dist_likes
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Ok(Self {
interpreter: interpreter.clone(),
distributions,
by_name,
by_url,
});
// The site-packages directory doesn't exist, skip it.
continue;
}
Err(err) => return Err(err).context("Failed to read site-packages directory"),
};

// Index all installed packages by name.
for path in site_packages {
for path in ordered_directory_paths {
let dist_info = match InstalledDist::try_from_path(&path) {
Ok(Some(dist_info)) => dist_info,
Ok(None) => continue,
Expand Down Expand Up @@ -185,7 +181,7 @@ impl SitePackages {
pub fn diagnostics(
&self,
markers: &ResolverMarkerEnvironment,
) -> Result<Vec<SitePackagesDiagnostic>> {
) -> Result<Vec<InstalledPackagesDiagnostic>> {
let mut diagnostics = Vec::new();

for (package, indexes) in &self.by_name {
Expand All @@ -198,7 +194,7 @@ impl SitePackages {

if let Some(conflict) = distributions.next() {
// There are multiple installed distributions for the same package.
diagnostics.push(SitePackagesDiagnostic::DuplicatePackage {
diagnostics.push(InstalledPackagesDiagnostic::DuplicatePackage {
package: package.clone(),
paths: std::iter::once(distribution.path().to_owned())
.chain(std::iter::once(conflict.path().to_owned()))
Expand All @@ -215,7 +211,7 @@ impl SitePackages {

// Determine the dependencies for the given package.
let Ok(metadata) = distribution.metadata() else {
diagnostics.push(SitePackagesDiagnostic::MetadataUnavailable {
diagnostics.push(InstalledPackagesDiagnostic::MetadataUnavailable {
package: package.clone(),
path: distribution.path().to_owned(),
});
Expand All @@ -225,7 +221,7 @@ impl SitePackages {
// Verify that the package is compatible with the current Python version.
if let Some(requires_python) = metadata.requires_python.as_ref() {
if !requires_python.contains(markers.python_full_version()) {
diagnostics.push(SitePackagesDiagnostic::IncompatiblePythonVersion {
diagnostics.push(InstalledPackagesDiagnostic::IncompatiblePythonVersion {
package: package.clone(),
version: self.interpreter.python_version().clone(),
requires_python: requires_python.clone(),
Expand All @@ -243,7 +239,7 @@ impl SitePackages {
match installed.as_slice() {
[] => {
// No version installed.
diagnostics.push(SitePackagesDiagnostic::MissingDependency {
diagnostics.push(InstalledPackagesDiagnostic::MissingDependency {
package: package.clone(),
requirement: dependency.clone(),
});
Expand All @@ -259,7 +255,7 @@ impl SitePackages {
// The installed version doesn't satisfy the requirement.
if !version_specifier.contains(installed.version()) {
diagnostics.push(
SitePackagesDiagnostic::IncompatibleDependency {
InstalledPackagesDiagnostic::IncompatibleDependency {
package: package.clone(),
version: installed.version().clone(),
requirement: dependency.clone(),
Expand Down Expand Up @@ -394,7 +390,7 @@ pub enum SatisfiesResult {
Unsatisfied(String),
}

impl IntoIterator for SitePackages {
impl IntoIterator for InstalledPackages {
type Item = InstalledDist;
type IntoIter = Flatten<std::vec::IntoIter<Option<InstalledDist>>>;

Expand All @@ -404,7 +400,7 @@ impl IntoIterator for SitePackages {
}

#[derive(Debug)]
pub enum SitePackagesDiagnostic {
pub enum InstalledPackagesDiagnostic {
MetadataUnavailable {
/// The package that is missing metadata.
package: PackageName,
Expand Down Expand Up @@ -441,7 +437,7 @@ pub enum SitePackagesDiagnostic {
},
}

impl Diagnostic for SitePackagesDiagnostic {
impl Diagnostic for InstalledPackagesDiagnostic {
/// Convert the diagnostic into a user-facing message.
fn message(&self) -> String {
match self {
Expand Down Expand Up @@ -495,7 +491,7 @@ impl Diagnostic for SitePackagesDiagnostic {
}
}

impl InstalledPackagesProvider for SitePackages {
impl InstalledPackagesProvider for InstalledPackages {
fn iter(&self) -> impl Iterator<Item = &InstalledDist> {
self.iter()
}
Expand Down
4 changes: 2 additions & 2 deletions crates/uv-installer/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
pub use compile::{compile_tree, CompileError};
pub use installed_packages::{InstalledPackages, InstalledPackagesDiagnostic, SatisfiesResult};
pub use installer::{Installer, Reporter as InstallReporter};
pub use plan::{Plan, Planner};
pub use preparer::{Error as PrepareError, Preparer, Reporter as PrepareReporter};
pub use site_packages::{SatisfiesResult, SitePackages, SitePackagesDiagnostic};
pub use uninstall::{uninstall, UninstallError};

mod compile;
mod preparer;

mod installed_packages;
mod installer;
mod plan;
mod satisfies;
mod site_packages;
mod uninstall;
10 changes: 5 additions & 5 deletions crates/uv-installer/src/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use uv_python::PythonEnvironment;
use uv_types::HashStrategy;

use crate::satisfies::RequirementSatisfaction;
use crate::SitePackages;
use crate::InstalledPackages;

/// A planner to generate an [`Plan`] based on a set of requirements.
#[derive(Debug)]
Expand All @@ -45,7 +45,7 @@ impl<'a> Planner<'a> {
/// return an _installed_ distribution that does not match the required hash.
pub fn build(
self,
mut site_packages: SitePackages,
mut installed_packages: InstalledPackages,
reinstall: &Reinstall,
build_options: &BuildOptions,
hasher: &HashStrategy,
Expand Down Expand Up @@ -77,7 +77,7 @@ impl<'a> Planner<'a> {
let no_build = build_options.no_build_package(dist.name());

// Determine whether the distribution is already installed.
let installed_dists = site_packages.remove_packages(dist.name());
let installed_dists = installed_packages.remove_packages(dist.name());
if reinstall {
reinstalls.extend(installed_dists);
} else {
Expand Down Expand Up @@ -350,11 +350,11 @@ impl<'a> Planner<'a> {
}

// Remove any unnecessary packages.
if site_packages.any() {
if installed_packages.any() {
// Retain seed packages unless: (1) the virtual environment was created by uv and
// (2) the `--seed` argument was not passed to `uv venv`.
let seed_packages = !venv.cfg().is_ok_and(|cfg| cfg.is_uv() && !cfg.is_seed());
for dist_info in site_packages {
for dist_info in installed_packages {
if seed_packages
&& matches!(
dist_info.name().as_ref(),
Expand Down
16 changes: 8 additions & 8 deletions crates/uv-tool/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ pub use receipt::ToolReceipt;
pub use tool::{Tool, ToolEntrypoint};
use uv_cache::Cache;
use uv_fs::{LockedFile, Simplified};
use uv_installer::SitePackages;
use uv_installer::InstalledPackages;
use uv_python::{Interpreter, PythonEnvironment};
use uv_state::{StateBucket, StateStore};
use uv_static::EnvVars;
Expand Down Expand Up @@ -288,9 +288,9 @@ impl InstalledTools {
pub fn version(&self, name: &PackageName, cache: &Cache) -> Result<Version, Error> {
let environment_path = self.tool_dir(name);
let environment = PythonEnvironment::from_root(&environment_path, cache)?;
let site_packages = SitePackages::from_environment(&environment)
let installed_packages = InstalledPackages::from_environment(&environment)
.map_err(|err| Error::EnvironmentRead(environment_path.clone(), err.to_string()))?;
let packages = site_packages.get_packages(name);
let packages = installed_packages.get_packages(name);
let package = packages
.first()
.ok_or_else(|| Error::MissingToolPackage(name.clone()))?;
Expand Down Expand Up @@ -363,11 +363,11 @@ pub fn tool_executable_dir() -> Result<PathBuf, Error> {

/// Find the `.dist-info` directory for a package in an environment.
fn find_dist_info<'a>(
site_packages: &'a SitePackages,
installed_packages: &'a InstalledPackages,
package_name: &PackageName,
package_version: &Version,
) -> Result<&'a Path, Error> {
site_packages
installed_packages
.get_packages(package_name)
.iter()
.find(|package| package.version() == package_version)
Expand All @@ -382,12 +382,12 @@ fn find_dist_info<'a>(
///
/// Returns a list of `(name, path)` tuples.
pub fn entrypoint_paths(
site_packages: &SitePackages,
installed_packages: &InstalledPackages,
package_name: &PackageName,
package_version: &Version,
) -> Result<Vec<(String, PathBuf)>, Error> {
// Find the `.dist-info` directory in the installed environment.
let dist_info_path = find_dist_info(site_packages, package_name, package_version)?;
let dist_info_path = find_dist_info(installed_packages, package_name, package_version)?;
debug!(
"Looking at `.dist-info` at: {}",
dist_info_path.user_display()
Expand All @@ -397,7 +397,7 @@ pub fn entrypoint_paths(
let record = read_record_file(&mut File::open(dist_info_path.join("RECORD"))?)?;

// The RECORD file uses relative paths, so we're looking for the relative path to be a prefix.
let layout = site_packages.interpreter().layout();
let layout = installed_packages.interpreter().layout();
let script_relative = pathdiff::diff_paths(&layout.scheme.scripts, &layout.scheme.purelib)
.ok_or_else(|| {
io::Error::new(
Expand Down
12 changes: 7 additions & 5 deletions crates/uv/src/commands/pip/check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use owo_colors::OwoColorize;

use uv_cache::Cache;
use uv_distribution_types::{Diagnostic, InstalledDist};
use uv_installer::{SitePackages, SitePackagesDiagnostic};
use uv_installer::{InstalledPackages, InstalledPackagesDiagnostic};
use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest};

use crate::commands::pip::operations::report_target_environment;
Expand All @@ -32,8 +32,8 @@ pub(crate) fn pip_check(
report_target_environment(&environment, cache, printer)?;

// Build the installed index.
let site_packages = SitePackages::from_environment(&environment)?;
let packages: Vec<&InstalledDist> = site_packages.iter().collect();
let installed_packages = InstalledPackages::from_environment(&environment)?;
let packages: Vec<&InstalledDist> = installed_packages.iter().collect();

let s = if packages.len() == 1 { "" } else { "s" };
writeln!(
Expand All @@ -51,8 +51,10 @@ pub(crate) fn pip_check(
let markers = environment.interpreter().resolver_marker_environment();

// Run the diagnostics.
let diagnostics: Vec<SitePackagesDiagnostic> =
site_packages.diagnostics(&markers)?.into_iter().collect();
let diagnostics: Vec<InstalledPackagesDiagnostic> = installed_packages
.diagnostics(&markers)?
.into_iter()
.collect();

if diagnostics.is_empty() {
writeln!(
Expand Down
8 changes: 4 additions & 4 deletions crates/uv/src/commands/pip/freeze.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use owo_colors::OwoColorize;

use uv_cache::Cache;
use uv_distribution_types::{Diagnostic, InstalledDist, Name};
use uv_installer::SitePackages;
use uv_installer::InstalledPackages;
use uv_python::{EnvironmentPreference, PythonEnvironment, PythonRequest};

use crate::commands::pip::operations::report_target_environment;
Expand All @@ -32,8 +32,8 @@ pub(crate) fn pip_freeze(
report_target_environment(&environment, cache, printer)?;

// Build the installed index.
let site_packages = SitePackages::from_environment(&environment)?;
for dist in site_packages
let installed_packages = InstalledPackages::from_environment(&environment)?;
for dist in installed_packages
.iter()
.filter(|dist| !(exclude_editable && dist.is_editable()))
.sorted_unstable_by(|a, b| a.name().cmp(b.name()).then(a.version().cmp(b.version())))
Expand Down Expand Up @@ -66,7 +66,7 @@ pub(crate) fn pip_freeze(
// Determine the markers to use for resolution.
let markers = environment.interpreter().resolver_marker_environment();

for diagnostic in site_packages.diagnostics(&markers)? {
for diagnostic in installed_packages.diagnostics(&markers)? {
writeln!(
printer.stderr(),
"{}{} {}",
Expand Down
Loading
Loading