From 484717d42f11d9f7a341228dd62c101b4dc9fe55 Mon Sep 17 00:00:00 2001 From: konsti Date: Tue, 24 Sep 2024 17:16:21 +0200 Subject: [PATCH] Split metadata parsing into a module (#7656) --- .../src/dependency_metadata.rs | 6 +- crates/distribution-types/src/installed.rs | 6 +- crates/pypi-types/src/metadata.rs | 864 ------------------ crates/pypi-types/src/metadata/metadata10.rs | 31 + crates/pypi-types/src/metadata/metadata12.rs | 64 ++ .../src/metadata/metadata_resolver.rs | 229 +++++ crates/pypi-types/src/metadata/mod.rs | 113 +++ .../pypi-types/src/metadata/pyproject_toml.rs | 314 +++++++ .../pypi-types/src/metadata/requires_txt.rs | 170 ++++ crates/uv-cache/src/lib.rs | 4 +- crates/uv-client/src/registry_client.rs | 31 +- crates/uv-distribution/src/download.rs | 6 +- crates/uv-distribution/src/metadata/mod.rs | 8 +- crates/uv-distribution/src/source/mod.rs | 41 +- crates/uv-git/Cargo.toml | 2 +- crates/uv-metadata/src/lib.rs | 12 +- crates/uv-requirements/src/unnamed.rs | 4 +- crates/uv-resolver/src/resolver/mod.rs | 4 +- 18 files changed, 984 insertions(+), 925 deletions(-) delete mode 100644 crates/pypi-types/src/metadata.rs create mode 100644 crates/pypi-types/src/metadata/metadata10.rs create mode 100644 crates/pypi-types/src/metadata/metadata12.rs create mode 100644 crates/pypi-types/src/metadata/metadata_resolver.rs create mode 100644 crates/pypi-types/src/metadata/mod.rs create mode 100644 crates/pypi-types/src/metadata/pyproject_toml.rs create mode 100644 crates/pypi-types/src/metadata/requires_txt.rs diff --git a/crates/distribution-types/src/dependency_metadata.rs b/crates/distribution-types/src/dependency_metadata.rs index 4512f5cb6007..5d2b79b08c02 100644 --- a/crates/distribution-types/src/dependency_metadata.rs +++ b/crates/distribution-types/src/dependency_metadata.rs @@ -1,6 +1,6 @@ use pep440_rs::{Version, VersionSpecifiers}; use pep508_rs::Requirement; -use pypi_types::{Metadata23, VerbatimParsedUrl}; +use pypi_types::{MetadataResolver, VerbatimParsedUrl}; use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; use uv_normalize::{ExtraName, PackageName}; @@ -20,7 +20,7 @@ impl DependencyMetadata { } /// Retrieve a [`StaticMetadata`] entry by [`PackageName`] and [`Version`]. - pub fn get(&self, package: &PackageName, version: &Version) -> Option { + pub fn get(&self, package: &PackageName, version: &Version) -> Option { let versions = self.0.get(package)?; // Search for an exact, then a global match. @@ -29,7 +29,7 @@ impl DependencyMetadata { .find(|v| v.version.as_ref() == Some(version)) .or_else(|| versions.iter().find(|v| v.version.is_none()))?; - Some(Metadata23 { + Some(MetadataResolver { name: metadata.name.clone(), version: version.clone(), requires_dist: metadata.requires_dist.clone(), diff --git a/crates/distribution-types/src/installed.rs b/crates/distribution-types/src/installed.rs index 2cdac4802081..bae7a188a738 100644 --- a/crates/distribution-types/src/installed.rs +++ b/crates/distribution-types/src/installed.rs @@ -285,13 +285,13 @@ impl InstalledDist { } /// Read the `METADATA` file from a `.dist-info` directory. - pub fn metadata(&self) -> Result { + pub fn metadata(&self) -> Result { match self { Self::Registry(_) | Self::Url(_) => { let path = self.path().join("METADATA"); let contents = fs::read(&path)?; // TODO(zanieb): Update this to use thiserror so we can unpack parse errors downstream - pypi_types::Metadata23::parse_metadata(&contents).with_context(|| { + pypi_types::MetadataResolver::parse_metadata(&contents).with_context(|| { format!( "Failed to parse `METADATA` file at: {}", path.user_display() @@ -306,7 +306,7 @@ impl InstalledDist { _ => unreachable!(), }; let contents = fs::read(path.as_ref())?; - pypi_types::Metadata23::parse_metadata(&contents).with_context(|| { + pypi_types::MetadataResolver::parse_metadata(&contents).with_context(|| { format!( "Failed to parse `PKG-INFO` file at: {}", path.user_display() diff --git a/crates/pypi-types/src/metadata.rs b/crates/pypi-types/src/metadata.rs deleted file mode 100644 index f81b4fe1dee6..000000000000 --- a/crates/pypi-types/src/metadata.rs +++ /dev/null @@ -1,864 +0,0 @@ -//! Derived from `pypi_types_crate`. - -use std::io::BufRead; -use std::str::FromStr; - -use indexmap::IndexMap; -use itertools::Itertools; -use mailparse::{MailHeaderMap, MailParseError}; -use serde::de::IntoDeserializer; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tracing::warn; - -use pep440_rs::{Version, VersionParseError, VersionSpecifiers, VersionSpecifiersParseError}; -use pep508_rs::marker::MarkerValueExtra; -use pep508_rs::{ExtraOperator, MarkerExpression, MarkerTree, Pep508Error, Requirement}; -use uv_normalize::{ExtraName, InvalidNameError, PackageName}; - -use crate::lenient_requirement::LenientRequirement; -use crate::{LenientVersionSpecifiers, VerbatimParsedUrl}; - -/// Python Package Metadata 2.3 as specified in -/// . -/// -/// This is a subset of the full metadata specification, and only includes the -/// fields that are relevant to dependency resolution. -/// -/// At present, we support up to version 2.3 of the metadata specification. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "kebab-case")] -pub struct Metadata23 { - // Mandatory fields - pub name: PackageName, - pub version: Version, - // Optional fields - pub requires_dist: Vec>, - pub requires_python: Option, - pub provides_extras: Vec, -} - -/// -/// -/// The error type -#[derive(Error, Debug)] -pub enum MetadataError { - #[error(transparent)] - MailParse(#[from] MailParseError), - #[error("Invalid `pyproject.toml`")] - InvalidPyprojectTomlSyntax(#[source] toml_edit::TomlError), - #[error("`pyproject.toml` is using the `[project]` table, but the required `project.name` field is not set.")] - InvalidPyprojectTomlMissingName(#[source] toml_edit::de::Error), - #[error(transparent)] - InvalidPyprojectTomlSchema(toml_edit::de::Error), - #[error("metadata field {0} not found")] - FieldNotFound(&'static str), - #[error("invalid version: {0}")] - Pep440VersionError(VersionParseError), - #[error(transparent)] - Pep440Error(#[from] VersionSpecifiersParseError), - #[error(transparent)] - Pep508Error(#[from] Box>), - #[error(transparent)] - InvalidName(#[from] InvalidNameError), - #[error("Invalid `Metadata-Version` field: {0}")] - InvalidMetadataVersion(String), - #[error("Reading metadata from `PKG-INFO` requires Metadata 2.2 or later (found: {0})")] - UnsupportedMetadataVersion(String), - #[error("The following field was marked as dynamic: {0}")] - DynamicField(&'static str), - #[error("The project uses Poetry's syntax to declare its dependencies, despite including a `project` table in `pyproject.toml`")] - PoetrySyntax, - #[error("Failed to read `requires.txt` contents")] - RequiresTxtContents(#[from] std::io::Error), -} - -impl From> for MetadataError { - fn from(error: Pep508Error) -> Self { - Self::Pep508Error(Box::new(error)) - } -} - -/// From -impl Metadata23 { - /// Parse the [`Metadata23`] from a `METADATA` file, as included in a built distribution (wheel). - pub fn parse_metadata(content: &[u8]) -> Result { - let headers = Headers::parse(content)?; - - let name = PackageName::new( - headers - .get_first_value("Name") - .ok_or(MetadataError::FieldNotFound("Name"))?, - )?; - let version = Version::from_str( - &headers - .get_first_value("Version") - .ok_or(MetadataError::FieldNotFound("Version"))?, - ) - .map_err(MetadataError::Pep440VersionError)?; - let requires_dist = headers - .get_all_values("Requires-Dist") - .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) - .map_ok(Requirement::from) - .collect::, _>>()?; - let requires_python = headers - .get_first_value("Requires-Python") - .map(|requires_python| LenientVersionSpecifiers::from_str(&requires_python)) - .transpose()? - .map(VersionSpecifiers::from); - let provides_extras = headers - .get_all_values("Provides-Extra") - .filter_map(|provides_extra| match ExtraName::new(provides_extra) { - Ok(extra_name) => Some(extra_name), - Err(err) => { - warn!("Ignoring invalid extra: {err}"); - None - } - }) - .collect::>(); - - Ok(Self { - name, - version, - requires_dist, - requires_python, - provides_extras, - }) - } - - /// Read the [`Metadata23`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2 - /// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and - /// `Provides-Extra`) are marked as dynamic. - pub fn parse_pkg_info(content: &[u8]) -> Result { - let headers = Headers::parse(content)?; - - // To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be - // present and set to a value of at least `2.2`. - let metadata_version = headers - .get_first_value("Metadata-Version") - .ok_or(MetadataError::FieldNotFound("Metadata-Version"))?; - - // Parse the version into (major, minor). - let (major, minor) = parse_version(&metadata_version)?; - if (major, minor) < (2, 2) || (major, minor) >= (3, 0) { - return Err(MetadataError::UnsupportedMetadataVersion(metadata_version)); - } - - // If any of the fields we need are marked as dynamic, we can't use the `PKG-INFO` file. - let dynamic = headers.get_all_values("Dynamic").collect::>(); - for field in dynamic { - match field.as_str() { - "Requires-Python" => return Err(MetadataError::DynamicField("Requires-Python")), - "Requires-Dist" => return Err(MetadataError::DynamicField("Requires-Dist")), - "Provides-Extra" => return Err(MetadataError::DynamicField("Provides-Extra")), - _ => (), - } - } - - // The `Name` and `Version` fields are required, and can't be dynamic. - let name = PackageName::new( - headers - .get_first_value("Name") - .ok_or(MetadataError::FieldNotFound("Name"))?, - )?; - let version = Version::from_str( - &headers - .get_first_value("Version") - .ok_or(MetadataError::FieldNotFound("Version"))?, - ) - .map_err(MetadataError::Pep440VersionError)?; - - // The remaining fields are required to be present. - let requires_dist = headers - .get_all_values("Requires-Dist") - .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) - .map_ok(Requirement::from) - .collect::, _>>()?; - let requires_python = headers - .get_first_value("Requires-Python") - .map(|requires_python| LenientVersionSpecifiers::from_str(&requires_python)) - .transpose()? - .map(VersionSpecifiers::from); - let provides_extras = headers - .get_all_values("Provides-Extra") - .filter_map(|provides_extra| match ExtraName::new(provides_extra) { - Ok(extra_name) => Some(extra_name), - Err(err) => { - warn!("Ignoring invalid extra: {err}"); - None - } - }) - .collect::>(); - - Ok(Self { - name, - version, - requires_dist, - requires_python, - provides_extras, - }) - } - - /// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621. - pub fn parse_pyproject_toml(contents: &str) -> Result { - let pyproject_toml = PyProjectToml::from_toml(contents)?; - - let project = pyproject_toml - .project - .ok_or(MetadataError::FieldNotFound("project"))?; - - // If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file. - let dynamic = project.dynamic.unwrap_or_default(); - for field in dynamic { - match field.as_str() { - "dependencies" => return Err(MetadataError::DynamicField("dependencies")), - "optional-dependencies" => { - return Err(MetadataError::DynamicField("optional-dependencies")) - } - "requires-python" => return Err(MetadataError::DynamicField("requires-python")), - "version" => return Err(MetadataError::DynamicField("version")), - _ => (), - } - } - - // If dependencies are declared with Poetry, and `project.dependencies` is omitted, treat - // the dependencies as dynamic. The inclusion of a `project` table without defining - // `project.dependencies` is almost certainly an error. - if project.dependencies.is_none() - && pyproject_toml.tool.and_then(|tool| tool.poetry).is_some() - { - return Err(MetadataError::PoetrySyntax); - } - - let name = project.name; - let version = project - .version - .ok_or(MetadataError::FieldNotFound("version"))?; - - // Parse the Python version requirements. - let requires_python = project - .requires_python - .map(|requires_python| { - LenientVersionSpecifiers::from_str(&requires_python).map(VersionSpecifiers::from) - }) - .transpose()?; - - // Extract the requirements. - let mut requires_dist = project - .dependencies - .unwrap_or_default() - .into_iter() - .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) - .map_ok(Requirement::from) - .collect::, _>>()?; - - // Extract the optional dependencies. - let mut provides_extras: Vec = Vec::new(); - for (extra, requirements) in project.optional_dependencies.unwrap_or_default() { - requires_dist.extend( - requirements - .into_iter() - .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) - .map_ok(Requirement::from) - .map_ok(|requirement| requirement.with_extra_marker(&extra)) - .collect::, _>>()?, - ); - provides_extras.push(extra); - } - - Ok(Self { - name, - version, - requires_dist, - requires_python, - provides_extras, - }) - } -} - -/// A `pyproject.toml` as specified in PEP 517. -#[derive(Deserialize, Debug)] -#[serde(rename_all = "kebab-case")] -struct PyProjectToml { - project: Option, - tool: Option, -} - -impl PyProjectToml { - fn from_toml(toml: &str) -> Result { - let pyproject_toml: toml_edit::ImDocument<_> = toml_edit::ImDocument::from_str(toml) - .map_err(MetadataError::InvalidPyprojectTomlSyntax)?; - let pyproject_toml: Self = PyProjectToml::deserialize(pyproject_toml.into_deserializer()) - .map_err(|err| { - // TODO(konsti): A typed error would be nicer, this can break on toml upgrades. - if err.message().contains("missing field `name`") { - MetadataError::InvalidPyprojectTomlMissingName(err) - } else { - MetadataError::InvalidPyprojectTomlSchema(err) - } - })?; - Ok(pyproject_toml) - } -} - -/// PEP 621 project metadata. -/// -/// This is a subset of the full metadata specification, and only includes the fields that are -/// relevant for dependency resolution. -/// -/// See . -#[derive(Deserialize, Debug)] -#[serde(rename_all = "kebab-case")] -struct Project { - /// The name of the project - name: PackageName, - /// The version of the project as supported by PEP 440 - version: Option, - /// The Python version requirements of the project - requires_python: Option, - /// Project dependencies - dependencies: Option>, - /// Optional dependencies - optional_dependencies: Option>>, - /// Specifies which fields listed by PEP 621 were intentionally unspecified - /// so another tool can/will provide such metadata dynamically. - dynamic: Option>, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "kebab-case")] -struct Tool { - poetry: Option, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "kebab-case")] -#[allow(clippy::empty_structs_with_brackets)] -struct ToolPoetry {} - -/// Python Package Metadata 1.0 and later as specified in -/// . -/// -/// This is a subset of the full metadata specification, and only includes the -/// fields that have been consistent across all versions of the specification. -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "kebab-case")] -pub struct Metadata10 { - pub name: PackageName, - pub version: String, -} - -impl Metadata10 { - /// Parse the [`Metadata10`] from a `PKG-INFO` file, as included in a source distribution. - pub fn parse_pkg_info(content: &[u8]) -> Result { - let headers = Headers::parse(content)?; - let name = PackageName::new( - headers - .get_first_value("Name") - .ok_or(MetadataError::FieldNotFound("Name"))?, - )?; - let version = headers - .get_first_value("Version") - .ok_or(MetadataError::FieldNotFound("Version"))?; - Ok(Self { name, version }) - } -} - -/// Python Package Metadata 1.2 and later as specified in -/// . -/// -/// This is a subset of the full metadata specification, and only includes the -/// fields that have been consistent across all versions of the specification later than 1.2. -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "kebab-case")] -pub struct Metadata12 { - pub name: PackageName, - pub version: Version, - pub requires_python: Option, -} - -impl Metadata12 { - /// Parse the [`Metadata12`] from a `.dist-info` `METADATA` file, as included in a built - /// distribution. - pub fn parse_metadata(content: &[u8]) -> Result { - let headers = Headers::parse(content)?; - - // To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be - // present and set to a value of at least `2.2`. - let metadata_version = headers - .get_first_value("Metadata-Version") - .ok_or(MetadataError::FieldNotFound("Metadata-Version"))?; - - // Parse the version into (major, minor). - let (major, minor) = parse_version(&metadata_version)?; - - // At time of writing: - // > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”. - if (major, minor) < (1, 0) || (major, minor) >= (3, 0) { - return Err(MetadataError::InvalidMetadataVersion(metadata_version)); - } - - let name = PackageName::new( - headers - .get_first_value("Name") - .ok_or(MetadataError::FieldNotFound("Name"))?, - )?; - let version = Version::from_str( - &headers - .get_first_value("Version") - .ok_or(MetadataError::FieldNotFound("Version"))?, - ) - .map_err(MetadataError::Pep440VersionError)?; - let requires_python = headers - .get_first_value("Requires-Python") - .map(|requires_python| LenientVersionSpecifiers::from_str(&requires_python)) - .transpose()? - .map(VersionSpecifiers::from); - - Ok(Self { - name, - version, - requires_python, - }) - } -} - -/// Parse a `Metadata-Version` field into a (major, minor) tuple. -fn parse_version(metadata_version: &str) -> Result<(u8, u8), MetadataError> { - let (major, minor) = - metadata_version - .split_once('.') - .ok_or(MetadataError::InvalidMetadataVersion( - metadata_version.to_string(), - ))?; - let major = major - .parse::() - .map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?; - let minor = minor - .parse::() - .map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?; - Ok((major, minor)) -} - -/// Python Package Metadata 2.3 as specified in -/// . -/// -/// This is a subset of [`Metadata23`]; specifically, it omits the `version` and `requires-python` -/// fields, which aren't necessary when extracting the requirements of a package without installing -/// the package itself. -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "kebab-case")] -pub struct RequiresDist { - pub name: PackageName, - pub requires_dist: Vec>, - pub provides_extras: Vec, -} - -impl RequiresDist { - /// Extract the [`RequiresDist`] from a `pyproject.toml` file, as specified in PEP 621. - pub fn parse_pyproject_toml(contents: &str) -> Result { - let pyproject_toml = PyProjectToml::from_toml(contents)?; - - let project = pyproject_toml - .project - .ok_or(MetadataError::FieldNotFound("project"))?; - - // If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` - // file. - let dynamic = project.dynamic.unwrap_or_default(); - for field in dynamic { - match field.as_str() { - "dependencies" => return Err(MetadataError::DynamicField("dependencies")), - "optional-dependencies" => { - return Err(MetadataError::DynamicField("optional-dependencies")) - } - _ => (), - } - } - - // If dependencies are declared with Poetry, and `project.dependencies` is omitted, treat - // the dependencies as dynamic. The inclusion of a `project` table without defining - // `project.dependencies` is almost certainly an error. - if project.dependencies.is_none() - && pyproject_toml.tool.and_then(|tool| tool.poetry).is_some() - { - return Err(MetadataError::PoetrySyntax); - } - - let name = project.name; - - // Extract the requirements. - let mut requires_dist = project - .dependencies - .unwrap_or_default() - .into_iter() - .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) - .map_ok(Requirement::from) - .collect::, _>>()?; - - // Extract the optional dependencies. - let mut provides_extras: Vec = Vec::new(); - for (extra, requirements) in project.optional_dependencies.unwrap_or_default() { - requires_dist.extend( - requirements - .into_iter() - .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) - .map_ok(Requirement::from) - .map_ok(|requirement| requirement.with_extra_marker(&extra)) - .collect::, _>>()?, - ); - provides_extras.push(extra); - } - - Ok(Self { - name, - requires_dist, - provides_extras, - }) - } -} - -/// `requires.txt` metadata as defined in . -/// -/// This is a subset of the full metadata specification, and only includes the fields that are -/// included in the legacy `requires.txt` file. -#[derive(Deserialize, Debug, Clone)] -#[serde(rename_all = "kebab-case")] -pub struct RequiresTxt { - pub requires_dist: Vec>, - pub provides_extras: Vec, -} - -impl RequiresTxt { - /// Parse the [`RequiresTxt`] from a `requires.txt` file, as included in an `egg-info`. - /// - /// See: - pub fn parse(content: &[u8]) -> Result { - let mut requires_dist = vec![]; - let mut provides_extras = vec![]; - let mut current_marker = MarkerTree::default(); - - for line in content.lines() { - let line = line.map_err(MetadataError::RequiresTxtContents)?; - - let line = line.trim(); - if line.is_empty() { - continue; - } - - // When encountering a new section, parse the extra and marker from the header, e.g., - // `[:sys_platform == "win32"]` or `[dev]`. - if line.starts_with('[') { - let line = line.trim_start_matches('[').trim_end_matches(']'); - - // Split into extra and marker, both of which can be empty. - let (extra, marker) = { - let (extra, marker) = match line.split_once(':') { - Some((extra, marker)) => (Some(extra), Some(marker)), - None => (Some(line), None), - }; - let extra = extra.filter(|extra| !extra.is_empty()); - let marker = marker.filter(|marker| !marker.is_empty()); - (extra, marker) - }; - - // Parse the extra. - let extra = if let Some(extra) = extra { - if let Ok(extra) = ExtraName::from_str(extra) { - provides_extras.push(extra.clone()); - Some(MarkerValueExtra::Extra(extra)) - } else { - Some(MarkerValueExtra::Arbitrary(extra.to_string())) - } - } else { - None - }; - - // Parse the marker. - let marker = marker.map(MarkerTree::parse_str).transpose()?; - - // Create the marker tree. - match (extra, marker) { - (Some(extra), Some(mut marker)) => { - marker.and(MarkerTree::expression(MarkerExpression::Extra { - operator: ExtraOperator::Equal, - name: extra, - })); - current_marker = marker; - } - (Some(extra), None) => { - current_marker = MarkerTree::expression(MarkerExpression::Extra { - operator: ExtraOperator::Equal, - name: extra, - }); - } - (None, Some(marker)) => { - current_marker = marker; - } - (None, None) => { - current_marker = MarkerTree::default(); - } - } - - continue; - } - - // Parse the requirement. - let requirement = - Requirement::::from(LenientRequirement::from_str(line)?); - - // Add the markers and extra, if necessary. - requires_dist.push(Requirement { - marker: current_marker.clone(), - ..requirement - }); - } - - Ok(Self { - requires_dist, - provides_extras, - }) - } -} - -/// The headers of a distribution metadata file. -#[derive(Debug)] -struct Headers<'a>(Vec>); - -impl<'a> Headers<'a> { - /// Parse the headers from the given metadata file content. - fn parse(content: &'a [u8]) -> Result { - let (headers, _) = mailparse::parse_headers(content)?; - Ok(Self(headers)) - } - - /// Return the first value associated with the header with the given name. - fn get_first_value(&self, name: &str) -> Option { - self.0.get_first_header(name).and_then(|header| { - let value = header.get_value(); - if value == "UNKNOWN" { - None - } else { - Some(value) - } - }) - } - - /// Return all values associated with the header with the given name. - fn get_all_values(&self, name: &str) -> impl Iterator { - self.0 - .get_all_values(name) - .into_iter() - .filter(|value| value != "UNKNOWN") - } -} - -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use pep440_rs::Version; - use uv_normalize::PackageName; - - use crate::{MetadataError, RequiresTxt}; - - use super::Metadata23; - - #[test] - fn test_parse_metadata() { - let s = "Metadata-Version: 1.0"; - let meta = Metadata23::parse_metadata(s.as_bytes()); - assert!(matches!(meta, Err(MetadataError::FieldNotFound("Name")))); - - let s = "Metadata-Version: 1.0\nName: asdf"; - let meta = Metadata23::parse_metadata(s.as_bytes()); - assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version")))); - - let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0"; - let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - - let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0\nAuthor: 中文\n\n一个 Python 包"; - let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - - let s = "Metadata-Version: 1.0\nName: =?utf-8?q?foobar?=\nVersion: 1.0"; - let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap(); - assert_eq!(meta.name, PackageName::from_str("foobar").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - - let s = "Metadata-Version: 1.0\nName: =?utf-8?q?=C3=A4_space?= \nVersion: 1.0"; - let meta = Metadata23::parse_metadata(s.as_bytes()); - assert!(matches!(meta, Err(MetadataError::InvalidName(_)))); - } - - #[test] - fn test_parse_pkg_info() { - let s = "Metadata-Version: 2.1"; - let meta = Metadata23::parse_pkg_info(s.as_bytes()); - assert!(matches!( - meta, - Err(MetadataError::UnsupportedMetadataVersion(_)) - )); - - let s = "Metadata-Version: 2.2\nName: asdf"; - let meta = Metadata23::parse_pkg_info(s.as_bytes()); - assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version")))); - - let s = "Metadata-Version: 2.3\nName: asdf"; - let meta = Metadata23::parse_pkg_info(s.as_bytes()); - assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version")))); - - let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0"; - let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - - let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nDynamic: Requires-Dist"; - let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap_err(); - assert!(matches!(meta, MetadataError::DynamicField("Requires-Dist"))); - - let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo"; - let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); - } - - #[test] - fn test_parse_pyproject_toml() { - let s = r#" - [project] - name = "asdf" - "#; - let meta = Metadata23::parse_pyproject_toml(s); - assert!(matches!(meta, Err(MetadataError::FieldNotFound("version")))); - - let s = r#" - [project] - name = "asdf" - dynamic = ["version"] - "#; - let meta = Metadata23::parse_pyproject_toml(s); - assert!(matches!(meta, Err(MetadataError::DynamicField("version")))); - - let s = r#" - [project] - name = "asdf" - version = "1.0" - "#; - let meta = Metadata23::parse_pyproject_toml(s).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - assert!(meta.requires_python.is_none()); - assert!(meta.requires_dist.is_empty()); - assert!(meta.provides_extras.is_empty()); - - let s = r#" - [project] - name = "asdf" - version = "1.0" - requires-python = ">=3.6" - "#; - let meta = Metadata23::parse_pyproject_toml(s).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); - assert!(meta.requires_dist.is_empty()); - assert!(meta.provides_extras.is_empty()); - - let s = r#" - [project] - name = "asdf" - version = "1.0" - requires-python = ">=3.6" - dependencies = ["foo"] - "#; - let meta = Metadata23::parse_pyproject_toml(s).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); - assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); - assert!(meta.provides_extras.is_empty()); - - let s = r#" - [project] - name = "asdf" - version = "1.0" - requires-python = ">=3.6" - dependencies = ["foo"] - - [project.optional-dependencies] - dotenv = ["bar"] - "#; - let meta = Metadata23::parse_pyproject_toml(s).unwrap(); - assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); - assert_eq!(meta.version, Version::new([1, 0])); - assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); - assert_eq!( - meta.requires_dist, - vec![ - "foo".parse().unwrap(), - "bar; extra == \"dotenv\"".parse().unwrap() - ] - ); - assert_eq!(meta.provides_extras, vec!["dotenv".parse().unwrap()]); - } - - #[test] - fn test_requires_txt() { - let s = r" -Werkzeug>=0.14 -Jinja2>=2.10 - -[dev] -pytest>=3 -sphinx - -[dotenv] -python-dotenv - "; - let meta = RequiresTxt::parse(s.as_bytes()).unwrap(); - assert_eq!( - meta.requires_dist, - vec![ - "Werkzeug>=0.14".parse().unwrap(), - "Jinja2>=2.10".parse().unwrap(), - "pytest>=3; extra == \"dev\"".parse().unwrap(), - "sphinx; extra == \"dev\"".parse().unwrap(), - "python-dotenv; extra == \"dotenv\"".parse().unwrap(), - ] - ); - - let s = r" -Werkzeug>=0.14 - -[dev:] -Jinja2>=2.10 - -[:sys_platform == 'win32'] -pytest>=3 - -[] -sphinx - -[dotenv:sys_platform == 'darwin'] -python-dotenv - "; - let meta = RequiresTxt::parse(s.as_bytes()).unwrap(); - assert_eq!( - meta.requires_dist, - vec![ - "Werkzeug>=0.14".parse().unwrap(), - "Jinja2>=2.10 ; extra == \"dev\"".parse().unwrap(), - "pytest>=3; sys_platform == 'win32'".parse().unwrap(), - "sphinx".parse().unwrap(), - "python-dotenv; sys_platform == 'darwin' and extra == \"dotenv\"" - .parse() - .unwrap(), - ] - ); - } -} diff --git a/crates/pypi-types/src/metadata/metadata10.rs b/crates/pypi-types/src/metadata/metadata10.rs new file mode 100644 index 000000000000..2e41f3ee8399 --- /dev/null +++ b/crates/pypi-types/src/metadata/metadata10.rs @@ -0,0 +1,31 @@ +use crate::metadata::Headers; +use crate::MetadataError; +use serde::Deserialize; +use uv_normalize::PackageName; + +/// A subset of the full core metadata specification, including only the +/// fields that have been consistent across all versions of the specification. +/// +/// Core Metadata 1.0 is specified in . +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub struct Metadata10 { + pub name: PackageName, + pub version: String, +} + +impl Metadata10 { + /// Parse the [`Metadata10`] from a `PKG-INFO` file, as included in a source distribution. + pub fn parse_pkg_info(content: &[u8]) -> Result { + let headers = Headers::parse(content)?; + let name = PackageName::new( + headers + .get_first_value("Name") + .ok_or(MetadataError::FieldNotFound("Name"))?, + )?; + let version = headers + .get_first_value("Version") + .ok_or(MetadataError::FieldNotFound("Version"))?; + Ok(Self { name, version }) + } +} diff --git a/crates/pypi-types/src/metadata/metadata12.rs b/crates/pypi-types/src/metadata/metadata12.rs new file mode 100644 index 000000000000..4ca5f8874c3d --- /dev/null +++ b/crates/pypi-types/src/metadata/metadata12.rs @@ -0,0 +1,64 @@ +use crate::metadata::{parse_version, Headers}; +use crate::{LenientVersionSpecifiers, MetadataError}; +use pep440_rs::{Version, VersionSpecifiers}; +use serde::Deserialize; +use std::str::FromStr; +use uv_normalize::PackageName; + +/// A subset of the full cure metadata specification, only including the +/// fields that have been consistent across all versions of the specification later than 1.2. +/// +/// Python Package Metadata 1.2 is specified in . +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub struct Metadata12 { + pub name: PackageName, + pub version: Version, + pub requires_python: Option, +} + +impl Metadata12 { + /// Parse the [`Metadata12`] from a `.dist-info/METADATA` file, as included in a built + /// distribution. + pub fn parse_metadata(content: &[u8]) -> Result { + let headers = Headers::parse(content)?; + + // To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be + // present and set to a value of at least `2.2`. + let metadata_version = headers + .get_first_value("Metadata-Version") + .ok_or(MetadataError::FieldNotFound("Metadata-Version"))?; + + // Parse the version into (major, minor). + let (major, minor) = parse_version(&metadata_version)?; + + // At time of writing: + // > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”. + if (major, minor) < (1, 0) || (major, minor) >= (3, 0) { + return Err(MetadataError::InvalidMetadataVersion(metadata_version)); + } + + let name = PackageName::new( + headers + .get_first_value("Name") + .ok_or(MetadataError::FieldNotFound("Name"))?, + )?; + let version = Version::from_str( + &headers + .get_first_value("Version") + .ok_or(MetadataError::FieldNotFound("Version"))?, + ) + .map_err(MetadataError::Pep440VersionError)?; + let requires_python = headers + .get_first_value("Requires-Python") + .map(|requires_python| LenientVersionSpecifiers::from_str(&requires_python)) + .transpose()? + .map(VersionSpecifiers::from); + + Ok(Self { + name, + version, + requires_python, + }) + } +} diff --git a/crates/pypi-types/src/metadata/metadata_resolver.rs b/crates/pypi-types/src/metadata/metadata_resolver.rs new file mode 100644 index 000000000000..255644dcc563 --- /dev/null +++ b/crates/pypi-types/src/metadata/metadata_resolver.rs @@ -0,0 +1,229 @@ +//! Derived from `pypi_types_crate`. + +use std::str::FromStr; + +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use tracing::warn; + +use pep440_rs::{Version, VersionSpecifiers}; +use pep508_rs::Requirement; +use uv_normalize::{ExtraName, PackageName}; + +use crate::lenient_requirement::LenientRequirement; +use crate::metadata::pyproject_toml::parse_pyproject_toml; +use crate::metadata::Headers; +use crate::{metadata, LenientVersionSpecifiers, MetadataError, VerbatimParsedUrl}; + +/// A subset of the full core metadata specification, including only the +/// fields that are relevant to dependency resolution. +/// +/// Core Metadata 2.3 is specified in . +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub struct MetadataResolver { + // Mandatory fields + pub name: PackageName, + pub version: Version, + // Optional fields + pub requires_dist: Vec>, + pub requires_python: Option, + pub provides_extras: Vec, +} + +/// From +impl MetadataResolver { + /// Parse the [`MetadataResolver`] from a `METADATA` file, as included in a built distribution (wheel). + pub fn parse_metadata(content: &[u8]) -> Result { + let headers = Headers::parse(content)?; + + let name = PackageName::new( + headers + .get_first_value("Name") + .ok_or(MetadataError::FieldNotFound("Name"))?, + )?; + let version = Version::from_str( + &headers + .get_first_value("Version") + .ok_or(MetadataError::FieldNotFound("Version"))?, + ) + .map_err(MetadataError::Pep440VersionError)?; + let requires_dist = headers + .get_all_values("Requires-Dist") + .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) + .map_ok(Requirement::from) + .collect::, _>>()?; + let requires_python = headers + .get_first_value("Requires-Python") + .map(|requires_python| LenientVersionSpecifiers::from_str(&requires_python)) + .transpose()? + .map(VersionSpecifiers::from); + let provides_extras = headers + .get_all_values("Provides-Extra") + .filter_map(|provides_extra| match ExtraName::new(provides_extra) { + Ok(extra_name) => Some(extra_name), + Err(err) => { + warn!("Ignoring invalid extra: {err}"); + None + } + }) + .collect::>(); + + Ok(Self { + name, + version, + requires_dist, + requires_python, + provides_extras, + }) + } + + /// Read the [`MetadataResolver`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2 + /// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and + /// `Provides-Extra`) are marked as dynamic. + pub fn parse_pkg_info(content: &[u8]) -> Result { + let headers = Headers::parse(content)?; + + // To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be + // present and set to a value of at least `2.2`. + let metadata_version = headers + .get_first_value("Metadata-Version") + .ok_or(MetadataError::FieldNotFound("Metadata-Version"))?; + + // Parse the version into (major, minor). + let (major, minor) = metadata::parse_version(&metadata_version)?; + if (major, minor) < (2, 2) || (major, minor) >= (3, 0) { + return Err(MetadataError::UnsupportedMetadataVersion(metadata_version)); + } + + // If any of the fields we need are marked as dynamic, we can't use the `PKG-INFO` file. + let dynamic = headers.get_all_values("Dynamic").collect::>(); + for field in dynamic { + match field.as_str() { + "Requires-Python" => return Err(MetadataError::DynamicField("Requires-Python")), + "Requires-Dist" => return Err(MetadataError::DynamicField("Requires-Dist")), + "Provides-Extra" => return Err(MetadataError::DynamicField("Provides-Extra")), + _ => (), + } + } + + // The `Name` and `Version` fields are required, and can't be dynamic. + let name = PackageName::new( + headers + .get_first_value("Name") + .ok_or(MetadataError::FieldNotFound("Name"))?, + )?; + let version = Version::from_str( + &headers + .get_first_value("Version") + .ok_or(MetadataError::FieldNotFound("Version"))?, + ) + .map_err(MetadataError::Pep440VersionError)?; + + // The remaining fields are required to be present. + let requires_dist = headers + .get_all_values("Requires-Dist") + .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) + .map_ok(Requirement::from) + .collect::, _>>()?; + let requires_python = headers + .get_first_value("Requires-Python") + .map(|requires_python| LenientVersionSpecifiers::from_str(&requires_python)) + .transpose()? + .map(VersionSpecifiers::from); + let provides_extras = headers + .get_all_values("Provides-Extra") + .filter_map(|provides_extra| match ExtraName::new(provides_extra) { + Ok(extra_name) => Some(extra_name), + Err(err) => { + warn!("Ignoring invalid extra: {err}"); + None + } + }) + .collect::>(); + + Ok(Self { + name, + version, + requires_dist, + requires_python, + provides_extras, + }) + } + + pub fn parse_pyproject_toml(toml: &str) -> Result { + parse_pyproject_toml(toml) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::MetadataError; + use pep440_rs::Version; + use std::str::FromStr; + use uv_normalize::PackageName; + + #[test] + fn test_parse_metadata() { + let s = "Metadata-Version: 1.0"; + let meta = MetadataResolver::parse_metadata(s.as_bytes()); + assert!(matches!(meta, Err(MetadataError::FieldNotFound("Name")))); + + let s = "Metadata-Version: 1.0\nName: asdf"; + let meta = MetadataResolver::parse_metadata(s.as_bytes()); + assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version")))); + + let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0"; + let meta = MetadataResolver::parse_metadata(s.as_bytes()).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + + let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0\nAuthor: 中文\n\n一个 Python 包"; + let meta = MetadataResolver::parse_metadata(s.as_bytes()).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + + let s = "Metadata-Version: 1.0\nName: =?utf-8?q?foobar?=\nVersion: 1.0"; + let meta = MetadataResolver::parse_metadata(s.as_bytes()).unwrap(); + assert_eq!(meta.name, PackageName::from_str("foobar").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + + let s = "Metadata-Version: 1.0\nName: =?utf-8?q?=C3=A4_space?= \nVersion: 1.0"; + let meta = MetadataResolver::parse_metadata(s.as_bytes()); + assert!(matches!(meta, Err(MetadataError::InvalidName(_)))); + } + + #[test] + fn test_parse_pkg_info() { + let s = "Metadata-Version: 2.1"; + let meta = MetadataResolver::parse_pkg_info(s.as_bytes()); + assert!(matches!( + meta, + Err(MetadataError::UnsupportedMetadataVersion(_)) + )); + + let s = "Metadata-Version: 2.2\nName: asdf"; + let meta = MetadataResolver::parse_pkg_info(s.as_bytes()); + assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version")))); + + let s = "Metadata-Version: 2.3\nName: asdf"; + let meta = MetadataResolver::parse_pkg_info(s.as_bytes()); + assert!(matches!(meta, Err(MetadataError::FieldNotFound("Version")))); + + let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0"; + let meta = MetadataResolver::parse_pkg_info(s.as_bytes()).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + + let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nDynamic: Requires-Dist"; + let meta = MetadataResolver::parse_pkg_info(s.as_bytes()).unwrap_err(); + assert!(matches!(meta, MetadataError::DynamicField("Requires-Dist"))); + + let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo"; + let meta = MetadataResolver::parse_pkg_info(s.as_bytes()).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); + } +} diff --git a/crates/pypi-types/src/metadata/mod.rs b/crates/pypi-types/src/metadata/mod.rs new file mode 100644 index 000000000000..cb0ea7cd017c --- /dev/null +++ b/crates/pypi-types/src/metadata/mod.rs @@ -0,0 +1,113 @@ +mod metadata10; +mod metadata12; +mod metadata_resolver; +mod pyproject_toml; +mod requires_txt; + +use crate::VerbatimParsedUrl; +use mailparse::{MailHeaderMap, MailParseError}; +use pep440_rs::{VersionParseError, VersionSpecifiersParseError}; +use pep508_rs::Pep508Error; +use std::str::Utf8Error; +use thiserror::Error; +use uv_normalize::InvalidNameError; + +pub use metadata10::Metadata10; +pub use metadata12::Metadata12; +pub use metadata_resolver::MetadataResolver; +pub use pyproject_toml::RequiresDist; +pub use requires_txt::RequiresTxt; + +/// +/// +/// The error type +#[derive(Error, Debug)] +pub enum MetadataError { + #[error(transparent)] + MailParse(#[from] MailParseError), + #[error("Invalid `pyproject.toml`")] + InvalidPyprojectTomlSyntax(#[source] toml_edit::TomlError), + #[error("`pyproject.toml` is using the `[project]` table, but the required `project.name` field is not set.")] + InvalidPyprojectTomlMissingName(#[source] toml_edit::de::Error), + #[error(transparent)] + InvalidPyprojectTomlSchema(toml_edit::de::Error), + #[error("Metadata field {0} not found")] + FieldNotFound(&'static str), + #[error("Invalid version: {0}")] + Pep440VersionError(VersionParseError), + #[error(transparent)] + Pep440Error(#[from] VersionSpecifiersParseError), + #[error(transparent)] + Pep508Error(#[from] Box>), + #[error(transparent)] + InvalidName(#[from] InvalidNameError), + #[error("Invalid `Metadata-Version` field: {0}")] + InvalidMetadataVersion(String), + #[error("Reading metadata from `PKG-INFO` requires Metadata 2.2 or later (found: {0})")] + UnsupportedMetadataVersion(String), + #[error("The following field was marked as dynamic: {0}")] + DynamicField(&'static str), + #[error("The project uses Poetry's syntax to declare its dependencies, despite including a `project` table in `pyproject.toml`")] + PoetrySyntax, + #[error("Failed to read `requires.txt` contents")] + RequiresTxtContents(#[from] std::io::Error), + #[error("The description is not valid utf-8")] + DescriptionEncoding(#[source] Utf8Error), +} + +impl From> for MetadataError { + fn from(error: Pep508Error) -> Self { + Self::Pep508Error(Box::new(error)) + } +} + +/// The headers of a distribution metadata file. +#[derive(Debug)] +struct Headers<'a> { + headers: Vec>, +} + +impl<'a> Headers<'a> { + /// Parse the headers from the given metadata file content. + fn parse(content: &'a [u8]) -> Result { + let (headers, _) = mailparse::parse_headers(content)?; + Ok(Self { headers }) + } + + /// Return the first value associated with the header with the given name. + fn get_first_value(&self, name: &str) -> Option { + self.headers.get_first_header(name).and_then(|header| { + let value = header.get_value(); + if value == "UNKNOWN" { + None + } else { + Some(value) + } + }) + } + + /// Return all values associated with the header with the given name. + fn get_all_values(&self, name: &str) -> impl Iterator { + self.headers + .get_all_values(name) + .into_iter() + .filter(|value| value != "UNKNOWN") + } +} + +/// Parse a `Metadata-Version` field into a (major, minor) tuple. +fn parse_version(metadata_version: &str) -> Result<(u8, u8), MetadataError> { + let (major, minor) = + metadata_version + .split_once('.') + .ok_or(MetadataError::InvalidMetadataVersion( + metadata_version.to_string(), + ))?; + let major = major + .parse::() + .map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?; + let minor = minor + .parse::() + .map_err(|_| MetadataError::InvalidMetadataVersion(metadata_version.to_string()))?; + Ok((major, minor)) +} diff --git a/crates/pypi-types/src/metadata/pyproject_toml.rs b/crates/pypi-types/src/metadata/pyproject_toml.rs new file mode 100644 index 000000000000..faac0746e17f --- /dev/null +++ b/crates/pypi-types/src/metadata/pyproject_toml.rs @@ -0,0 +1,314 @@ +use crate::{ + LenientRequirement, LenientVersionSpecifiers, MetadataError, MetadataResolver, + VerbatimParsedUrl, +}; +use indexmap::IndexMap; +use itertools::Itertools; +use pep440_rs::{Version, VersionSpecifiers}; +use pep508_rs::Requirement; +use serde::de::IntoDeserializer; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use uv_normalize::{ExtraName, PackageName}; + +/// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621. +pub(crate) fn parse_pyproject_toml(contents: &str) -> Result { + let pyproject_toml = PyProjectToml::from_toml(contents)?; + + let project = pyproject_toml + .project + .ok_or(MetadataError::FieldNotFound("project"))?; + + // If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file. + let dynamic = project.dynamic.unwrap_or_default(); + for field in dynamic { + match field.as_str() { + "dependencies" => return Err(MetadataError::DynamicField("dependencies")), + "optional-dependencies" => { + return Err(MetadataError::DynamicField("optional-dependencies")) + } + "requires-python" => return Err(MetadataError::DynamicField("requires-python")), + "version" => return Err(MetadataError::DynamicField("version")), + _ => (), + } + } + + // If dependencies are declared with Poetry, and `project.dependencies` is omitted, treat + // the dependencies as dynamic. The inclusion of a `project` table without defining + // `project.dependencies` is almost certainly an error. + if project.dependencies.is_none() && pyproject_toml.tool.and_then(|tool| tool.poetry).is_some() + { + return Err(MetadataError::PoetrySyntax); + } + + let name = project.name; + let version = project + .version + .ok_or(MetadataError::FieldNotFound("version"))?; + + // Parse the Python version requirements. + let requires_python = project + .requires_python + .map(|requires_python| { + LenientVersionSpecifiers::from_str(&requires_python).map(VersionSpecifiers::from) + }) + .transpose()?; + + // Extract the requirements. + let mut requires_dist = project + .dependencies + .unwrap_or_default() + .into_iter() + .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) + .map_ok(Requirement::from) + .collect::, _>>()?; + + // Extract the optional dependencies. + let mut provides_extras: Vec = Vec::new(); + for (extra, requirements) in project.optional_dependencies.unwrap_or_default() { + requires_dist.extend( + requirements + .into_iter() + .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) + .map_ok(Requirement::from) + .map_ok(|requirement| requirement.with_extra_marker(&extra)) + .collect::, _>>()?, + ); + provides_extras.push(extra); + } + + Ok(MetadataResolver { + name, + version, + requires_dist, + requires_python, + provides_extras, + }) +} + +/// A `pyproject.toml` as specified in PEP 517. +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +struct PyProjectToml { + project: Option, + tool: Option, +} + +impl PyProjectToml { + pub(crate) fn from_toml(toml: &str) -> Result { + let pyproject_toml: toml_edit::ImDocument<_> = toml_edit::ImDocument::from_str(toml) + .map_err(MetadataError::InvalidPyprojectTomlSyntax)?; + let pyproject_toml: Self = PyProjectToml::deserialize(pyproject_toml.into_deserializer()) + .map_err(|err| { + // TODO(konsti): A typed error would be nicer, this can break on toml upgrades. + if err.message().contains("missing field `name`") { + MetadataError::InvalidPyprojectTomlMissingName(err) + } else { + MetadataError::InvalidPyprojectTomlSchema(err) + } + })?; + Ok(pyproject_toml) + } +} + +/// PEP 621 project metadata. +/// +/// This is a subset of the full metadata specification, and only includes the fields that are +/// relevant for dependency resolution. +/// +/// See . +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +struct Project { + /// The name of the project + name: PackageName, + /// The version of the project as supported by PEP 440 + version: Option, + /// The Python version requirements of the project + requires_python: Option, + /// Project dependencies + dependencies: Option>, + /// Optional dependencies + optional_dependencies: Option>>, + /// Specifies which fields listed by PEP 621 were intentionally unspecified + /// so another tool can/will provide such metadata dynamically. + dynamic: Option>, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +struct Tool { + poetry: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +#[allow(clippy::empty_structs_with_brackets)] +struct ToolPoetry {} + +/// Python Package Metadata 2.3 as specified in +/// . +/// +/// This is a subset of [`MetadataResolver`]; specifically, it omits the `version` and `requires-python` +/// fields, which aren't necessary when extracting the requirements of a package without installing +/// the package itself. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub struct RequiresDist { + pub name: PackageName, + pub requires_dist: Vec>, + pub provides_extras: Vec, +} + +impl RequiresDist { + /// Extract the [`RequiresDist`] from a `pyproject.toml` file, as specified in PEP 621. + pub fn parse_pyproject_toml(contents: &str) -> Result { + let pyproject_toml = PyProjectToml::from_toml(contents)?; + + let project = pyproject_toml + .project + .ok_or(MetadataError::FieldNotFound("project"))?; + + // If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` + // file. + let dynamic = project.dynamic.unwrap_or_default(); + for field in dynamic { + match field.as_str() { + "dependencies" => return Err(MetadataError::DynamicField("dependencies")), + "optional-dependencies" => { + return Err(MetadataError::DynamicField("optional-dependencies")) + } + _ => (), + } + } + + // If dependencies are declared with Poetry, and `project.dependencies` is omitted, treat + // the dependencies as dynamic. The inclusion of a `project` table without defining + // `project.dependencies` is almost certainly an error. + if project.dependencies.is_none() + && pyproject_toml.tool.and_then(|tool| tool.poetry).is_some() + { + return Err(MetadataError::PoetrySyntax); + } + + let name = project.name; + + // Extract the requirements. + let mut requires_dist = project + .dependencies + .unwrap_or_default() + .into_iter() + .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) + .map_ok(Requirement::from) + .collect::, _>>()?; + + // Extract the optional dependencies. + let mut provides_extras: Vec = Vec::new(); + for (extra, requirements) in project.optional_dependencies.unwrap_or_default() { + requires_dist.extend( + requirements + .into_iter() + .map(|requires_dist| LenientRequirement::from_str(&requires_dist)) + .map_ok(Requirement::from) + .map_ok(|requirement| requirement.with_extra_marker(&extra)) + .collect::, _>>()?, + ); + provides_extras.push(extra); + } + + Ok(Self { + name, + requires_dist, + provides_extras, + }) + } +} + +#[cfg(test)] +mod tests { + use crate::metadata::pyproject_toml::parse_pyproject_toml; + use crate::MetadataError; + use pep440_rs::Version; + use std::str::FromStr; + use uv_normalize::PackageName; + + #[test] + fn test_parse_pyproject_toml() { + let s = r#" + [project] + name = "asdf" + "#; + let meta = parse_pyproject_toml(s); + assert!(matches!(meta, Err(MetadataError::FieldNotFound("version")))); + + let s = r#" + [project] + name = "asdf" + dynamic = ["version"] + "#; + let meta = parse_pyproject_toml(s); + assert!(matches!(meta, Err(MetadataError::DynamicField("version")))); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + "#; + let meta = parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert!(meta.requires_python.is_none()); + assert!(meta.requires_dist.is_empty()); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + "#; + let meta = parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert!(meta.requires_dist.is_empty()); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + dependencies = ["foo"] + "#; + let meta = parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]); + assert!(meta.provides_extras.is_empty()); + + let s = r#" + [project] + name = "asdf" + version = "1.0" + requires-python = ">=3.6" + dependencies = ["foo"] + + [project.optional-dependencies] + dotenv = ["bar"] + "#; + let meta = parse_pyproject_toml(s).unwrap(); + assert_eq!(meta.name, PackageName::from_str("asdf").unwrap()); + assert_eq!(meta.version, Version::new([1, 0])); + assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap())); + assert_eq!( + meta.requires_dist, + vec![ + "foo".parse().unwrap(), + "bar; extra == \"dotenv\"".parse().unwrap() + ] + ); + assert_eq!(meta.provides_extras, vec!["dotenv".parse().unwrap()]); + } +} diff --git a/crates/pypi-types/src/metadata/requires_txt.rs b/crates/pypi-types/src/metadata/requires_txt.rs new file mode 100644 index 000000000000..3ec3f61d734b --- /dev/null +++ b/crates/pypi-types/src/metadata/requires_txt.rs @@ -0,0 +1,170 @@ +use crate::{LenientRequirement, MetadataError, VerbatimParsedUrl}; +use pep508_rs::marker::MarkerValueExtra; +use pep508_rs::{ExtraOperator, MarkerExpression, MarkerTree, Requirement}; +use serde::Deserialize; +use std::io::BufRead; +use std::str::FromStr; +use uv_normalize::ExtraName; + +/// `requires.txt` metadata as defined in . +/// +/// This is a subset of the full metadata specification, and only includes the fields that are +/// included in the legacy `requires.txt` file. +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub struct RequiresTxt { + pub requires_dist: Vec>, + pub provides_extras: Vec, +} + +impl RequiresTxt { + /// Parse the [`RequiresTxt`] from a `requires.txt` file, as included in an `egg-info`. + /// + /// See: + pub fn parse(content: &[u8]) -> Result { + let mut requires_dist = vec![]; + let mut provides_extras = vec![]; + let mut current_marker = MarkerTree::default(); + + for line in content.lines() { + let line = line.map_err(MetadataError::RequiresTxtContents)?; + + let line = line.trim(); + if line.is_empty() { + continue; + } + + // When encountering a new section, parse the extra and marker from the header, e.g., + // `[:sys_platform == "win32"]` or `[dev]`. + if line.starts_with('[') { + let line = line.trim_start_matches('[').trim_end_matches(']'); + + // Split into extra and marker, both of which can be empty. + let (extra, marker) = { + let (extra, marker) = match line.split_once(':') { + Some((extra, marker)) => (Some(extra), Some(marker)), + None => (Some(line), None), + }; + let extra = extra.filter(|extra| !extra.is_empty()); + let marker = marker.filter(|marker| !marker.is_empty()); + (extra, marker) + }; + + // Parse the extra. + let extra = if let Some(extra) = extra { + if let Ok(extra) = ExtraName::from_str(extra) { + provides_extras.push(extra.clone()); + Some(MarkerValueExtra::Extra(extra)) + } else { + Some(MarkerValueExtra::Arbitrary(extra.to_string())) + } + } else { + None + }; + + // Parse the marker. + let marker = marker.map(MarkerTree::parse_str).transpose()?; + + // Create the marker tree. + match (extra, marker) { + (Some(extra), Some(mut marker)) => { + marker.and(MarkerTree::expression(MarkerExpression::Extra { + operator: ExtraOperator::Equal, + name: extra, + })); + current_marker = marker; + } + (Some(extra), None) => { + current_marker = MarkerTree::expression(MarkerExpression::Extra { + operator: ExtraOperator::Equal, + name: extra, + }); + } + (None, Some(marker)) => { + current_marker = marker; + } + (None, None) => { + current_marker = MarkerTree::default(); + } + } + + continue; + } + + // Parse the requirement. + let requirement = + Requirement::::from(LenientRequirement::from_str(line)?); + + // Add the markers and extra, if necessary. + requires_dist.push(Requirement { + marker: current_marker.clone(), + ..requirement + }); + } + + Ok(Self { + requires_dist, + provides_extras, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_requires_txt() { + let s = r" +Werkzeug>=0.14 +Jinja2>=2.10 + +[dev] +pytest>=3 +sphinx + +[dotenv] +python-dotenv + "; + let meta = RequiresTxt::parse(s.as_bytes()).unwrap(); + assert_eq!( + meta.requires_dist, + vec![ + "Werkzeug>=0.14".parse().unwrap(), + "Jinja2>=2.10".parse().unwrap(), + "pytest>=3; extra == \"dev\"".parse().unwrap(), + "sphinx; extra == \"dev\"".parse().unwrap(), + "python-dotenv; extra == \"dotenv\"".parse().unwrap(), + ] + ); + + let s = r" +Werkzeug>=0.14 + +[dev:] +Jinja2>=2.10 + +[:sys_platform == 'win32'] +pytest>=3 + +[] +sphinx + +[dotenv:sys_platform == 'darwin'] +python-dotenv + "; + let meta = RequiresTxt::parse(s.as_bytes()).unwrap(); + assert_eq!( + meta.requires_dist, + vec![ + "Werkzeug>=0.14".parse().unwrap(), + "Jinja2>=2.10 ; extra == \"dev\"".parse().unwrap(), + "pytest>=3; sys_platform == 'win32'".parse().unwrap(), + "sphinx".parse().unwrap(), + "python-dotenv; sys_platform == 'darwin' and extra == \"dotenv\"" + .parse() + .unwrap(), + ] + ); + } +} diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index bfcc822c403d..041476eeeddf 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -11,7 +11,7 @@ use tracing::debug; pub use archive::ArchiveId; use distribution_types::InstalledDist; -use pypi_types::Metadata23; +use pypi_types::MetadataResolver; use uv_cache_info::Timestamp; use uv_fs::{cachedir, directories}; use uv_normalize::PackageName; @@ -789,7 +789,7 @@ impl CacheBucket { let Ok(metadata) = fs_err::read(path.join("metadata.msgpack")) else { return false; }; - let Ok(metadata) = rmp_serde::from_slice::(&metadata) else { + let Ok(metadata) = rmp_serde::from_slice::(&metadata) else { return false; }; metadata.name == *name diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 5ad34b515ce2..ee825d290747 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -19,7 +19,7 @@ use distribution_types::{ use pep440_rs::Version; use pep508_rs::MarkerEnvironment; use platform_tags::Platform; -use pypi_types::{Metadata23, SimpleJson}; +use pypi_types::{MetadataResolver, SimpleJson}; use uv_cache::{Cache, CacheBucket, CacheEntry, WheelCache}; use uv_configuration::KeyringProviderType; use uv_configuration::{IndexStrategy, TrustedHost}; @@ -405,7 +405,7 @@ impl RegistryClient { &self, built_dist: &BuiltDist, capabilities: &IndexCapabilities, - ) -> Result { + ) -> Result { let metadata = match &built_dist { BuiltDist::Registry(wheels) => { #[derive(Debug, Clone)] @@ -455,7 +455,7 @@ impl RegistryClient { .map_err(|err| { ErrorKind::Metadata(path.to_string_lossy().to_string(), err) })?; - Metadata23::parse_metadata(&contents).map_err(|err| { + MetadataResolver::parse_metadata(&contents).map_err(|err| { ErrorKind::MetadataParseError( wheel.filename.clone(), built_dist.to_string(), @@ -489,7 +489,7 @@ impl RegistryClient { .map_err(|err| { ErrorKind::Metadata(wheel.install_path.to_string_lossy().to_string(), err) })?; - Metadata23::parse_metadata(&contents).map_err(|err| { + MetadataResolver::parse_metadata(&contents).map_err(|err| { ErrorKind::MetadataParseError( wheel.filename.clone(), built_dist.to_string(), @@ -516,7 +516,7 @@ impl RegistryClient { file: &File, url: &Url, capabilities: &IndexCapabilities, - ) -> Result { + ) -> Result { // If the metadata file is available at its own url (PEP 658), download it from there. let filename = WheelFilename::from_str(&file.filename).map_err(ErrorKind::WheelFilename)?; if file.dist_info_metadata { @@ -541,7 +541,7 @@ impl RegistryClient { let bytes = response.bytes().await.map_err(ErrorKind::from)?; info_span!("parse_metadata21") - .in_scope(|| Metadata23::parse_metadata(bytes.as_ref())) + .in_scope(|| MetadataResolver::parse_metadata(bytes.as_ref())) .map_err(|err| { Error::from(ErrorKind::MetadataParseError( filename, @@ -582,7 +582,7 @@ impl RegistryClient { index: Option<&'data IndexUrl>, cache_shard: WheelCache<'data>, capabilities: &'data IndexCapabilities, - ) -> Result { + ) -> Result { let cache_entry = self.cache.entry( CacheBucket::Wheels, cache_shard.wheel_dir(filename.name.as_ref()), @@ -629,14 +629,15 @@ impl RegistryClient { .map_err(ErrorKind::AsyncHttpRangeReader)?; trace!("Getting metadata for {filename} by range request"); let text = wheel_metadata_from_remote_zip(filename, url, &mut reader).await?; - let metadata = Metadata23::parse_metadata(text.as_bytes()).map_err(|err| { - Error::from(ErrorKind::MetadataParseError( - filename.clone(), - url.to_string(), - Box::new(err), - )) - })?; - Ok::>(metadata) + let metadata = + MetadataResolver::parse_metadata(text.as_bytes()).map_err(|err| { + Error::from(ErrorKind::MetadataParseError( + filename.clone(), + url.to_string(), + Box::new(err), + )) + })?; + Ok::>(metadata) } .boxed_local() .instrument(info_span!("read_metadata_range_request", wheel = %filename)) diff --git a/crates/uv-distribution/src/download.rs b/crates/uv-distribution/src/download.rs index 17229f9188d5..d0062d644f59 100644 --- a/crates/uv-distribution/src/download.rs +++ b/crates/uv-distribution/src/download.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use crate::Error; use distribution_filename::WheelFilename; use distribution_types::{CachedDist, Dist, Hashed}; -use pypi_types::{HashDigest, Metadata23}; +use pypi_types::{HashDigest, MetadataResolver}; use uv_metadata::read_flat_wheel_metadata; use uv_cache_info::CacheInfo; @@ -40,8 +40,8 @@ impl LocalWheel { &self.filename } - /// Read the [`Metadata23`] from a wheel. - pub fn metadata(&self) -> Result { + /// Read the [`MetadataResolver`] from a wheel. + pub fn metadata(&self) -> Result { read_flat_wheel_metadata(&self.filename, &self.archive) .map_err(|err| Error::WheelMetadata(self.archive.clone(), Box::new(err))) } diff --git a/crates/uv-distribution/src/metadata/mod.rs b/crates/uv-distribution/src/metadata/mod.rs index 999fabd17607..0fe66d6cb17b 100644 --- a/crates/uv-distribution/src/metadata/mod.rs +++ b/crates/uv-distribution/src/metadata/mod.rs @@ -4,7 +4,7 @@ use std::path::Path; use thiserror::Error; use pep440_rs::{Version, VersionSpecifiers}; -use pypi_types::{HashDigest, Metadata23}; +use pypi_types::{HashDigest, MetadataResolver}; use uv_configuration::SourceStrategy; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_workspace::WorkspaceError; @@ -39,7 +39,7 @@ pub struct Metadata { impl Metadata { /// Lower without considering `tool.uv` in `pyproject.toml`, used for index and other archive /// dependencies. - pub fn from_metadata23(metadata: Metadata23) -> Self { + pub fn from_metadata23(metadata: MetadataResolver) -> Self { Self { name: metadata.name, version: metadata.version, @@ -57,7 +57,7 @@ impl Metadata { /// Lower by considering `tool.uv` in `pyproject.toml` if present, used for Git and directory /// dependencies. pub async fn from_workspace( - metadata: Metadata23, + metadata: MetadataResolver, install_path: &Path, sources: SourceStrategy, ) -> Result { @@ -102,7 +102,7 @@ pub struct ArchiveMetadata { impl ArchiveMetadata { /// Lower without considering `tool.uv` in `pyproject.toml`, used for index and other archive /// dependencies. - pub fn from_metadata23(metadata: Metadata23) -> Self { + pub fn from_metadata23(metadata: MetadataResolver) -> Self { Self { metadata: Metadata::from_metadata23(metadata), hashes: vec![], diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 0a9c78d56fef..b69d27357415 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -20,7 +20,7 @@ use distribution_types::{ use fs_err::tokio as fs; use futures::{FutureExt, TryStreamExt}; use platform_tags::Tags; -use pypi_types::{HashDigest, Metadata12, Metadata23, RequiresTxt}; +use pypi_types::{HashDigest, Metadata12, MetadataResolver, RequiresTxt}; use reqwest::Response; use tokio_util::compat::FuturesAsyncReadCompatExt; use tracing::{debug, info_span, instrument, warn, Instrument}; @@ -1583,7 +1583,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { source_root: &Path, subdirectory: Option<&Path>, cache_shard: &CacheShard, - ) -> Result<(String, WheelFilename, Metadata23), Error> { + ) -> Result<(String, WheelFilename, MetadataResolver), Error> { debug!("Building: {source}"); // Guard against build of source distributions when disabled. @@ -1641,7 +1641,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { source: &BuildableSource<'_>, source_root: &Path, subdirectory: Option<&Path>, - ) -> Result, Error> { + ) -> Result, Error> { debug!("Preparing metadata for: {source}"); // Set up the builder. @@ -1673,7 +1673,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { let content = fs::read(dist_info.join("METADATA")) .await .map_err(Error::CacheRead)?; - let metadata = Metadata23::parse_metadata(&content)?; + let metadata = MetadataResolver::parse_metadata(&content)?; // Validate the metadata. validate(source, &metadata)?; @@ -1685,7 +1685,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { source: &BuildableSource<'_>, source_root: &Path, subdirectory: Option<&Path>, - ) -> Result, Error> { + ) -> Result, Error> { // Attempt to read static metadata from the `pyproject.toml`. match read_pyproject_toml(source_root, subdirectory).await { Ok(metadata) => { @@ -1858,7 +1858,7 @@ pub fn prune(cache: &Cache) -> Result { } /// Validate that the source distribution matches the built metadata. -fn validate(source: &BuildableSource<'_>, metadata: &Metadata23) -> Result<(), Error> { +fn validate(source: &BuildableSource<'_>, metadata: &MetadataResolver) -> Result<(), Error> { if let Some(name) = source.name() { if metadata.name != *name { return Err(Error::NameMismatch { @@ -1955,7 +1955,7 @@ impl LocalRevisionPointer { } } -/// Read the [`Metadata23`] by combining a source distribution's `PKG-INFO` file with a +/// Read the [`MetadataResolver`] by combining a source distribution's `PKG-INFO` file with a /// `requires.txt`. /// /// `requires.txt` is a legacy concept from setuptools. For example, here's @@ -1988,7 +1988,7 @@ impl LocalRevisionPointer { async fn read_egg_info( source_tree: &Path, subdirectory: Option<&Path>, -) -> Result { +) -> Result { fn find_egg_info(source_tree: &Path) -> std::io::Result> { for entry in fs_err::read_dir(source_tree)? { let entry = entry?; @@ -2048,7 +2048,7 @@ async fn read_egg_info( let metadata = Metadata12::parse_metadata(&content).map_err(Error::PkgInfo)?; // Combine the sources. - Ok(Metadata23 { + Ok(MetadataResolver { name: metadata.name, version: metadata.version, requires_python: metadata.requires_python, @@ -2057,13 +2057,13 @@ async fn read_egg_info( }) } -/// Read the [`Metadata23`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2 +/// Read the [`MetadataResolver`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2 /// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and /// `Provides-Extra`) are marked as dynamic. async fn read_pkg_info( source_tree: &Path, subdirectory: Option<&Path>, -) -> Result { +) -> Result { // Read the `PKG-INFO` file. let pkg_info = match subdirectory { Some(subdirectory) => source_tree.join(subdirectory).join("PKG-INFO"), @@ -2078,17 +2078,17 @@ async fn read_pkg_info( }; // Parse the metadata. - let metadata = Metadata23::parse_pkg_info(&content).map_err(Error::PkgInfo)?; + let metadata = MetadataResolver::parse_pkg_info(&content).map_err(Error::PkgInfo)?; Ok(metadata) } -/// Read the [`Metadata23`] from a source distribution's `pyproject.toml` file, if it defines static +/// Read the [`MetadataResolver`] from a source distribution's `pyproject.toml` file, if it defines static /// metadata consistent with PEP 621. async fn read_pyproject_toml( source_tree: &Path, subdirectory: Option<&Path>, -) -> Result { +) -> Result { // Read the `pyproject.toml` file. let pyproject_toml = match subdirectory { Some(subdirectory) => source_tree.join(subdirectory).join("pyproject.toml"), @@ -2103,7 +2103,8 @@ async fn read_pyproject_toml( }; // Parse the metadata. - let metadata = Metadata23::parse_pyproject_toml(&content).map_err(Error::PyprojectToml)?; + let metadata = + MetadataResolver::parse_pyproject_toml(&content).map_err(Error::PyprojectToml)?; Ok(metadata) } @@ -2127,8 +2128,8 @@ async fn read_requires_dist(project_root: &Path) -> Result Result, Error> { +/// Read an existing cached [`MetadataResolver`], if it exists. +async fn read_cached_metadata(cache_entry: &CacheEntry) -> Result, Error> { match fs::read(&cache_entry.path()).await { Ok(cached) => Ok(Some(rmp_serde::from_slice(&cached)?)), Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), @@ -2136,14 +2137,14 @@ async fn read_cached_metadata(cache_entry: &CacheEntry) -> Result Result { +/// Read the [`MetadataResolver`] from a built wheel. +fn read_wheel_metadata(filename: &WheelFilename, wheel: &Path) -> Result { let file = fs_err::File::open(wheel).map_err(Error::CacheRead)?; let reader = std::io::BufReader::new(file); let mut archive = ZipArchive::new(reader)?; let dist_info = read_archive_metadata(filename, &mut archive) .map_err(|err| Error::WheelMetadata(wheel.to_path_buf(), Box::new(err)))?; - Ok(Metadata23::parse_metadata(&dist_info)?) + Ok(MetadataResolver::parse_metadata(&dist_info)?) } /// Apply an advisory lock to a [`CacheShard`] to prevent concurrent builds. diff --git a/crates/uv-git/Cargo.toml b/crates/uv-git/Cargo.toml index 57a912e9cc75..df655253c360 100644 --- a/crates/uv-git/Cargo.toml +++ b/crates/uv-git/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] cache-key = { workspace = true } -uv-fs = { workspace = true } +uv-fs = { workspace = true, features = ["tokio"] } uv-auth = { workspace = true } anyhow = { workspace = true } diff --git a/crates/uv-metadata/src/lib.rs b/crates/uv-metadata/src/lib.rs index 2993e391b050..6da2901c395f 100644 --- a/crates/uv-metadata/src/lib.rs +++ b/crates/uv-metadata/src/lib.rs @@ -4,7 +4,7 @@ //! specification](https://packaging.python.org/en/latest/specifications/core-metadata/). use distribution_filename::WheelFilename; -use pypi_types::Metadata23; +use pypi_types::MetadataResolver; use std::io; use std::io::{Read, Seek}; use std::path::Path; @@ -233,7 +233,7 @@ pub async fn read_metadata_async_stream( filename: &WheelFilename, debug_path: &str, reader: R, -) -> Result { +) -> Result { let reader = futures::io::BufReader::with_capacity(128 * 1024, reader); let mut zip = async_zip::base::read::stream::ZipFileReader::new(reader); @@ -246,7 +246,7 @@ pub async fn read_metadata_async_stream( let mut contents = Vec::new(); reader.read_to_end(&mut contents).await.unwrap(); - let metadata = Metadata23::parse_metadata(&contents) + let metadata = MetadataResolver::parse_metadata(&contents) .map_err(|err| Error::InvalidMetadata(debug_path.to_string(), Box::new(err)))?; return Ok(metadata); } @@ -259,14 +259,14 @@ pub async fn read_metadata_async_stream( Err(Error::MissingDistInfo) } -/// Read the [`Metadata23`] from an unzipped wheel. +/// Read the [`MetadataResolver`] from an unzipped wheel. pub fn read_flat_wheel_metadata( filename: &WheelFilename, wheel: impl AsRef, -) -> Result { +) -> Result { let dist_info_prefix = find_flat_dist_info(filename, &wheel)?; let metadata = read_dist_info_metadata(&dist_info_prefix, &wheel)?; - Metadata23::parse_metadata(&metadata).map_err(|err| { + MetadataResolver::parse_metadata(&metadata).map_err(|err| { Error::InvalidMetadata( format!("{dist_info_prefix}.dist-info/METADATA"), Box::new(err), diff --git a/crates/uv-requirements/src/unnamed.rs b/crates/uv-requirements/src/unnamed.rs index 936f6442dffb..3a1603ea5e5d 100644 --- a/crates/uv-requirements/src/unnamed.rs +++ b/crates/uv-requirements/src/unnamed.rs @@ -15,8 +15,8 @@ use distribution_types::{ RemoteSource, SourceUrl, VersionId, }; use pep508_rs::{UnnamedRequirement, VersionOrUrl}; -use pypi_types::Requirement; -use pypi_types::{Metadata10, ParsedUrl, VerbatimParsedUrl}; +use pypi_types::{Metadata10, Requirement}; +use pypi_types::{ParsedUrl, VerbatimParsedUrl}; use uv_distribution::{DistributionDatabase, Reporter}; use uv_normalize::PackageName; use uv_resolver::{InMemoryIndex, MetadataResponse}; diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index d98b64674291..01315de92dc9 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -30,7 +30,7 @@ use locals::Locals; use pep440_rs::{Version, MIN_VERSION}; use pep508_rs::MarkerTree; use platform_tags::Tags; -use pypi_types::{Metadata23, Requirement, VerbatimParsedUrl}; +use pypi_types::{MetadataResolver, Requirement, VerbatimParsedUrl}; pub use resolver_markers::ResolverMarkers; pub(crate) use urls::Urls; use uv_configuration::{Constraints, Overrides}; @@ -2583,7 +2583,7 @@ enum Response { /// The returned metadata for an already-installed distribution. Installed { dist: InstalledDist, - metadata: Metadata23, + metadata: MetadataResolver, }, }