From 441f58ce82126118c09cd3f0524cb7c2bfb60ea3 Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 19 Sep 2024 14:30:29 +0200 Subject: [PATCH] Implement `uv publish` --- Cargo.lock | 144 +++++ Cargo.toml | 6 +- crates/distribution-filename/src/lib.rs | 8 + crates/uv-publish/Cargo.toml | 41 ++ crates/uv-publish/src/lib.rs | 789 ++++++++++++++++++++++++ crates/uv/Cargo.toml | 1 + crates/uv/src/commands/mod.rs | 2 + crates/uv/src/commands/publish.rs | 74 +++ crates/uv/src/lib.rs | 24 +- crates/uv/tests/common/mod.rs | 17 +- crates/uv/tests/publish.rs | 51 ++ 11 files changed, 1141 insertions(+), 16 deletions(-) create mode 100644 crates/uv-publish/Cargo.toml create mode 100644 crates/uv-publish/src/lib.rs create mode 100644 crates/uv/src/commands/publish.rs create mode 100644 crates/uv/tests/publish.rs diff --git a/Cargo.lock b/Cargo.lock index cf26eae9c296..717b71aee838 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,18 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -32,6 +44,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "anes" version = "0.1.6" @@ -581,6 +599,16 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "chumsky" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9" +dependencies = [ + "hashbrown", + "stacker", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -1453,6 +1481,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "heck" @@ -2131,6 +2163,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -2677,6 +2719,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" +dependencies = [ + "cc", +] + [[package]] name = "ptr_meta" version = "0.3.0" @@ -2808,6 +2859,21 @@ dependencies = [ "uv-normalize", ] +[[package]] +name = "python-pkginfo" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3f3f0d552c7efdde2b6898bf21b49c4e76b3e6071ff196dfe52109804db896" +dependencies = [ + "flate2", + "fs-err", + "mailparse", + "rfc2047-decoder", + "tar", + "thiserror", + "zip", +] + [[package]] name = "quinn" version = "0.11.5" @@ -3090,6 +3156,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "once_cell", "percent-encoding", "pin-project-lite", @@ -3179,6 +3246,20 @@ dependencies = [ "rand", ] +[[package]] +name = "rfc2047-decoder" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e90a668c463c412c3118ae1883e18b53d812c349f5af7a06de3ba4bb0c17cc73" +dependencies = [ + "base64 0.21.7", + "charset", + "chumsky", + "memchr", + "quoted_printable", + "thiserror", +] + [[package]] name = "rgb" version = "0.8.50" @@ -3685,6 +3766,19 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "strict-num" version = "0.1.1" @@ -3799,6 +3893,17 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddb6b06d20fba9ed21fca3d696ee1b6e870bca0bcf9fa2971f6ae2436de576a" +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -4290,6 +4395,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -4510,6 +4624,7 @@ dependencies = [ "uv-git", "uv-installer", "uv-normalize", + "uv-publish", "uv-python", "uv-requirements", "uv-resolver", @@ -4981,6 +5096,35 @@ dependencies = [ "thiserror", ] +[[package]] +name = "uv-publish" +version = "0.1.0" +dependencies = [ + "async-compression", + "base64 0.22.1", + "distribution-filename", + "fs-err", + "futures", + "glob", + "insta", + "itertools 0.13.0", + "krata-tokio-tar", + "python-pkginfo", + "reqwest", + "reqwest-middleware", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "tracing", + "url", + "uv-client", + "uv-fs", + "uv-metadata", + "uv-warnings", +] + [[package]] name = "uv-python" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index 09163af86cf9..9ad2b8f7c6a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ uv-metadata = { path = "crates/uv-metadata" } uv-normalize = { path = "crates/uv-normalize" } uv-options-metadata = { path = "crates/uv-options-metadata" } uv-pubgrub = { path = "crates/uv-pubgrub" } +uv-publish = { path = "crates/uv-publish" } uv-python = { path = "crates/uv-python" } uv-requirements = { path = "crates/uv-requirements" } uv-resolver = { path = "crates/uv-resolver" } @@ -119,12 +120,13 @@ proc-macro2 = { version = "1.0.86" } pubgrub = { git = "https://github.com/astral-sh/pubgrub", rev = "388685a8711092971930986644cfed152d1a1f6c" } pyo3 = { version = "0.21.2" } pyo3-log = { version = "0.10.0" } +python-pkginfo = { version = "0.6.3" } quote = { version = "1.0.37" } rayon = { version = "1.10.0" } reflink-copy = { version = "0.1.19" } regex = { version = "1.10.6" } -reqwest = { version = "0.12.7", default-features = false, features = ["json", "gzip", "stream", "rustls-tls", "rustls-tls-native-roots", "socks"] } -reqwest-middleware = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "5e3eaf254b5bd481c75d2710eed055f95b756913" } +reqwest = { version = "0.12.7", default-features = false, features = ["json", "gzip", "stream", "rustls-tls", "rustls-tls-native-roots", "socks", "multipart"] } +reqwest-middleware = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "5e3eaf254b5bd481c75d2710eed055f95b756913", features = ["multipart"] } reqwest-retry = { git = "https://github.com/astral-sh/reqwest-middleware", rev = "5e3eaf254b5bd481c75d2710eed055f95b756913" } rkyv = { version = "0.8.8", features = ["bytecheck"] } rmp-serde = { version = "1.3.0" } diff --git a/crates/distribution-filename/src/lib.rs b/crates/distribution-filename/src/lib.rs index dd06a89363b8..9c79895586d0 100644 --- a/crates/distribution-filename/src/lib.rs +++ b/crates/distribution-filename/src/lib.rs @@ -67,6 +67,14 @@ impl DistFilename { Self::WheelFilename(filename) => &filename.version, } } + + /// Whether the file is a `bdist_wheel` or an `sdist`. + pub fn filetype(&self) -> &'static str { + match self { + Self::SourceDistFilename(_) => "sdist", + Self::WheelFilename(_) => "bdist_wheel", + } + } } impl Display for DistFilename { diff --git a/crates/uv-publish/Cargo.toml b/crates/uv-publish/Cargo.toml new file mode 100644 index 000000000000..eb2ea28dd4e8 --- /dev/null +++ b/crates/uv-publish/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "uv-publish" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +distribution-filename = { workspace = true } +uv-client = { workspace = true } +uv-fs = { workspace = true } +uv-metadata = { workspace = true } +uv-warnings = { workspace = true } + +async-compression = { workspace = true } +base64 = { workspace = true } +fs-err = { workspace = true } +futures = { workspace = true } +glob = { workspace = true } +itertools = { workspace = true } +krata-tokio-tar = { workspace = true } +python-pkginfo = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +insta = { version = "1.36.1", features = ["json", "filters"] } + +[lints] +workspace = true diff --git a/crates/uv-publish/src/lib.rs b/crates/uv-publish/src/lib.rs new file mode 100644 index 000000000000..013d1f7d84dd --- /dev/null +++ b/crates/uv-publish/src/lib.rs @@ -0,0 +1,789 @@ +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use distribution_filename::{DistFilename, SourceDistExtension, SourceDistFilename}; +use fs_err::File; +use futures::TryStreamExt; +use glob::{glob, GlobError, PatternError}; +use itertools::Itertools; +use python_pkginfo::Metadata; +use reqwest::header::AUTHORIZATION; +use reqwest::multipart::Part; +use reqwest::{Body, Response, StatusCode}; +use reqwest_middleware::RequestBuilder; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::{fmt, io}; +use thiserror::Error; +use tokio::io::AsyncReadExt; +use tracing::{debug, enabled, trace, Level}; +use url::Url; +use uv_client::BaseClient; +use uv_fs::Simplified; +use uv_metadata::read_metadata_async_seek; +use uv_warnings::warn_user_once; + +#[derive(Error, Debug)] +pub enum PublishError { + #[error("Invalid publish paths")] + Pattern(#[from] PatternError), + /// [`GlobError`] is a wrapped io error. + #[error(transparent)] + Glob(#[from] GlobError), + #[error("Path patterns didn't match any wheels or source distributions")] + NoFiles, + #[error(transparent)] + Fmt(#[from] fmt::Error), + #[error("File is neither a wheel nor a source distribution: `{}`", _0.user_display())] + InvalidFilename(PathBuf), + #[error("Failed to publish: `{}`", _0.user_display())] + PublishPrepare(PathBuf, #[source] PublishPrepareError), + #[error("Failed to publish `{}` to `{}`", _0.user_display(), _1)] + PublishSend(PathBuf, Url, #[source] PublishSendError), +} + +/// Failure to get the metadata for a specific file. +#[derive(Error, Debug)] +pub enum PublishPrepareError { + #[error(transparent)] + PkgInfoError(#[from] python_pkginfo::Error), + #[error(transparent)] + Io(#[from] io::Error), + #[error("Failed to read metadata")] + Metadata(#[from] uv_metadata::Error), + #[error("Only files ending in `.tar.gz` are valid source distributions: `{0}`")] + InvalidExtension(SourceDistFilename), + #[error("No PKG-INFO file found")] + MissingPkgInfo, + #[error("Multiple PKG-INFO files found: `{0}`")] + MultiplePkgInfo(String), + #[error("Failed to read: `{0}`")] + Read(String, #[source] io::Error), +} + +/// Failure in or after (HTTP) transport for a specific file. +#[derive(Error, Debug)] +pub enum PublishSendError { + #[error("Failed to send POST request: `{0}`")] + ReqwestMiddleware(Url, #[source] reqwest_middleware::Error), + #[error("Upload failed with status {0}")] + StatusNoBody(StatusCode, #[source] reqwest::Error), + #[error("Upload failed with status code {0}: {1}")] + Status(StatusCode, String), + /// The registry returned a "403 Forbidden" + #[error("Incorrect credentials (status code {0}): {1}")] + IncorrectCredentials(StatusCode, String), +} + +impl PublishSendError { + /// Extract `code` from the PyPI json error response, if any. + /// + /// The error response from PyPI contains crucial context, such as the difference between + /// "Invalid or non-existent authentication information" and "The user 'konstin' isn't allowed + /// to upload to project 'dummy'". + /// + /// Twine uses the HTTP status reason for its error messages. In HTTP 2.0 and onward this field + /// is abolished, so reqwest doesn't expose it, see + /// . + /// PyPI does respect the content type for error responses and can return an error display as + /// HTML, JSON and plain. Since HTML and plain text are both overly verbose, we show the JSON + /// response. Examples are shown below, line breaks were inserted for readability. Of those, + /// the `code` seems to be the most helpful message, so we return it. If the response isn't a + /// JSON document with `code` we return the regular body + /// + /// ```json + /// {"message": "The server could not comply with the request since it is either malformed or + /// otherwise incorrect.\n\n\nError: Use 'source' as Python version for an sdist.\n\n", + /// "code": "400 Error: Use 'source' as Python version for an sdist.", + /// "title": "Bad Request"} + /// ``` + /// + /// ```json + /// {"message": "Access was denied to this resource.\n\n\nInvalid or non-existent authentication + /// information. See https://test.pypi.org/help/#invalid-auth for more information.\n\n", + /// "code": "403 Invalid or non-existent authentication information. See + /// https://test.pypi.org/help/#invalid-auth for more information.", + /// "title": "Forbidden"} + /// ``` + /// ```json + /// {"message": "Access was denied to this resource.\n\n\n\n\n", + /// "code": "403 Username/Password authentication is no longer supported. Migrate to API + /// Tokens or Trusted Publishers instead. See https://test.pypi.org/help/#apitoken and + /// https://test.pypi.org/help/#trusted-publishers", + /// "title": "Forbidden"} + /// ``` + /// + /// For context, for the last case twine shows: + /// ```text + /// WARNING Error during upload. Retry with the --verbose option for more details. + /// ERROR HTTPError: 403 Forbidden from https://test.pypi.org/legacy/ + /// Username/Password authentication is no longer supported. Migrate to API + /// Tokens or Trusted Publishers instead. See + /// https://test.pypi.org/help/#apitoken and + /// https://test.pypi.org/help/#trusted-publishers + /// ``` + /// + /// ```text + /// INFO Response from https://test.pypi.org/legacy/: + /// 403 Username/Password authentication is no longer supported. Migrate to + /// API Tokens or Trusted Publishers instead. See + /// https://test.pypi.org/help/#apitoken and + /// https://test.pypi.org/help/#trusted-publishers + /// INFO + /// + /// 403 Username/Password authentication is no longer supported. + /// Migrate to API Tokens or Trusted Publishers instead. See + /// https://test.pypi.org/help/#apitoken and + /// https://test.pypi.org/help/#trusted-publishers + /// + /// + ///

403 Username/Password authentication is no longer supported. + /// Migrate to API Tokens or Trusted Publishers instead. See + /// https://test.pypi.org/help/#apitoken and + /// https://test.pypi.org/help/#trusted-publishers

+ /// Access was denied to this resource.

+ /// ``` + /// + /// In comparison, we now show (line-wrapped for readability): + /// + /// ```text + /// error: Failed to publish `dist/astral_test_1-0.1.0-py3-none-any.whl` to `https://test.pypi.org/legacy/` + /// Caused by: Incorrect credentials (status code 403 Forbidden): 403 Username/Password + /// authentication is no longer supported. Migrate to API Tokens or Trusted Publishers + /// instead. See https://test.pypi.org/help/#apitoken and https://test.pypi.org/help/#trusted-publishers + /// ``` + fn extract_error_message(body: String, content_type: Option<&str>) -> String { + if content_type == Some("application/json") { + #[derive(Deserialize)] + struct ErrorBody { + code: String, + } + + if let Ok(structured) = serde_json::from_str::(&body) { + structured.code + } else { + body + } + } else { + body + } + } +} + +pub fn files_for_publishing( + paths: Vec, +) -> Result, PublishError> { + let mut seen = HashSet::new(); + let mut files = Vec::new(); + for path in paths { + for entry in glob(&path)? { + let entry = entry?; + if !seen.insert(entry.clone()) { + continue; + } + if !entry.is_file() { + continue; + } + let Some(filename) = entry.file_name().and_then(|filename| filename.to_str()) else { + continue; + }; + let filename = DistFilename::try_from_normalized_filename(filename) + .ok_or_else(|| PublishError::InvalidFilename(entry.clone()))?; + files.push((entry, filename)); + } + } + // TODO(konsti): Should we sort those files, e.g. wheels before sdists because they are more + // certain to have reliable metadata, even though the metadata in the upload API is unreliable + // in general? + Ok(files) +} + +/// Calculate the SHA256 of a file. +fn hash_file(path: impl AsRef) -> Result { + // Ideally, this would be async, but in case we actually want to make parallel uploads we should + // use `spawn_blocking` since sha256 is cpu intensive. + let mut file = BufReader::new(File::open(path.as_ref())?); + let mut hasher = Sha256::new(); + io::copy(&mut file, &mut hasher)?; + Ok(format!("{:x}", hasher.finalize())) +} + +// Not in `uv-metadata` because we only support tar files here. +async fn source_dist_pkg_info(file: &Path) -> Result, PublishPrepareError> { + let file = fs_err::tokio::File::open(&file).await?; + let reader = tokio::io::BufReader::new(file); + let decoded = async_compression::tokio::bufread::GzipDecoder::new(reader); + let mut archive = tokio_tar::Archive::new(decoded); + let mut pkg_infos: Vec<(PathBuf, Vec)> = archive + .entries()? + .map_err(PublishPrepareError::from) + .try_filter_map(|mut entry| async move { + let path = entry + .path() + .map_err(PublishPrepareError::from)? + .to_path_buf(); + let mut components = path.components(); + let Some(_top_level) = components.next() else { + return Ok(None); + }; + let Some(pkg_info) = components.next() else { + return Ok(None); + }; + if components.next().is_some() || pkg_info.as_os_str() != "PKG-INFO" { + return Ok(None); + } + let mut buffer = Vec::new(); + entry.read_to_end(&mut buffer).await.map_err(|err| { + PublishPrepareError::Read(path.to_string_lossy().to_string(), err) + })?; + Ok(Some((path, buffer))) + }) + .try_collect() + .await?; + match pkg_infos.len() { + 0 => Err(PublishPrepareError::MissingPkgInfo), + 1 => Ok(pkg_infos.remove(0).1), + _ => Err(PublishPrepareError::MultiplePkgInfo( + pkg_infos + .iter() + .map(|(path, _buffer)| path.to_string_lossy()) + .join(", "), + )), + } +} + +async fn metadata(file: &Path, filename: &DistFilename) -> Result { + let contents = match filename { + DistFilename::SourceDistFilename(source_dist) => { + if source_dist.extension != SourceDistExtension::TarGz { + // See PEP 625. While we support installing legacy source distributions, we don't + // support creating and uploading them. + return Err(PublishPrepareError::InvalidExtension(source_dist.clone())); + } + source_dist_pkg_info(file).await? + } + DistFilename::WheelFilename(wheel) => { + let file = fs_err::tokio::File::open(&file).await?; + let reader = tokio::io::BufReader::new(file); + read_metadata_async_seek(wheel, reader).await? + } + }; + Ok(Metadata::parse(&contents)?) +} + +/// Upload a file to a registry. +/// +/// Returns `true` if the file was newly uploaded and `false` if it already existed. +pub async fn upload( + file: &Path, + filename: &DistFilename, + registry: &Url, + client: &BaseClient, + username: Option<&str>, + password: Option<&str>, +) -> Result { + let form_metadata = form_metadata(file, filename) + .await + .map_err(|err| PublishError::PublishPrepare(file.to_path_buf(), err))?; + let request = build_request( + file, + filename, + registry, + client, + username, + password, + form_metadata, + ) + .await + .map_err(|err| PublishError::PublishPrepare(file.to_path_buf(), err))?; + + let response = request.send().await.map_err(|err| { + let send_err = PublishSendError::ReqwestMiddleware(registry.clone(), err); + PublishError::PublishSend(file.to_path_buf(), registry.clone(), send_err) + })?; + + handle_response(registry, response) + .await + .map_err(|err| PublishError::PublishSend(file.to_path_buf(), registry.clone(), err)) +} + +/// Collect the non-file field for the multipart request from the package METADATA. +async fn form_metadata( + file: &Path, + filename: &DistFilename, +) -> Result, PublishPrepareError> { + let hash_hex = hash_file(file)?; + + let metadata = metadata(file, filename).await?; + + let mut form_metadata = vec![ + (":action", "file_upload".to_string()), + ("sha256_digest", hash_hex), + ("protocol_version", "1".to_string()), + ("metadata_version", metadata.metadata_version.clone()), + // Twine transforms the name with `re.sub("[^A-Za-z0-9.]+", "-", name)` + // * + // * + // warehouse seems to call `packaging.utils.canonicalize_name` nowadays and has a separate + // `normalized_name`, so we'll start with this and we'll readjust if there are user reports. + ("name", metadata.name.clone()), + ("version", metadata.version.clone()), + ("filetype", filename.filetype().to_string()), + ]; + + if let DistFilename::WheelFilename(wheel) = filename { + form_metadata.push(("pyversion", wheel.python_tag.join("."))); + } else { + form_metadata.push(("pyversion", "source".to_string())); + } + + let mut add_option = |name, value: Option| { + if let Some(some) = value.clone() { + form_metadata.push((name, some)); + } + }; + + // https://github.com/pypi/warehouse/blob/d2c36d992cf9168e0518201d998b2707a3ef1e72/warehouse/forklift/legacy.py#L1376-L1430 + add_option("summary", metadata.summary); + add_option("description", metadata.description); + add_option( + "description_content_type", + metadata.description_content_type, + ); + add_option("author", metadata.author); + add_option("author_email", metadata.author_email); + add_option("maintainer", metadata.maintainer); + add_option("maintainer_email", metadata.maintainer_email); + add_option("license", metadata.license); + add_option("keywords", metadata.keywords); + add_option("home_page", metadata.home_page); + add_option("download_url", metadata.download_url); + + // GitLab PyPI repository API implementation requires this metadata field + // and twine always includes it in the request, even when it's empty. + form_metadata.push(( + "requires_python", + metadata.requires_python.unwrap_or(String::new()), + )); + + let mut add_vec = |name, values: Vec| { + for i in values { + form_metadata.push((name, i.clone())); + } + }; + + add_vec("classifiers", metadata.classifiers); + add_vec("platform", metadata.platforms); + add_vec("requires_dist", metadata.requires_dist); + add_vec("provides_dist", metadata.provides_dist); + add_vec("obsoletes_dist", metadata.obsoletes_dist); + add_vec("requires_external", metadata.requires_external); + add_vec("project_urls", metadata.project_urls); + + Ok(form_metadata) +} + +async fn build_request( + file: &Path, + filename: &DistFilename, + registry: &Url, + client: &BaseClient, + username: Option<&str>, + password: Option<&str>, + form_metadata: Vec<(&'static str, String)>, +) -> Result { + let mut form = reqwest::multipart::Form::new(); + for (key, value) in form_metadata { + form = form.text(key, value); + } + + let file: tokio::fs::File = fs_err::tokio::File::open(file).await?.into(); + let file_reader = Body::from(file); + form = form.part( + "content", + Part::stream(file_reader).file_name(filename.to_string()), + ); + + let url = if let Some(username) = username { + if password.is_none() { + // Attach the username to the URL so the authentication middleware can find the matching + // password. + let mut url = registry.clone(); + let _ = url.set_username(username); + url + } else { + // We set the authorization header below. + registry.clone() + } + } else { + registry.clone() + }; + + let mut request = client + .client() + .post(url) + .multipart(form) + // Ask PyPI for a structured error messages instead of HTML-markup error messages. + // For other registries, we ask them to return plain text over HTML. See + // [`PublishSendError::extract_remote_error`]. + .header( + reqwest::header::ACCEPT, + "application/json;q=0.9, text/plain;q=0.8, text/html;q=0.7", + ); + if let (Some(username), Some(password)) = (username, password) { + debug!("Using username/password basic auth"); + let credentials = BASE64_STANDARD.encode(format!("{username}:{password}")); + request = request.header(AUTHORIZATION, format!("Basic {credentials}")); + } + Ok(request) +} + +/// Returns `true` if the file was newly uploaded and `false` if it already existed. +async fn handle_response(registry: &Url, response: Response) -> Result { + let status_code = response.status(); + debug!("Response code for {registry}: {status_code}"); + trace!("Response headers for {registry}: {response:?}"); + + // TODO(konsti): There's some strange behavior here we're not handling yet: When I POST to + // https://test.pypi.org/simple/, I get a method not allowed error (as it should be), but when + // I post to https://test.pypi.org/simple (no slash) it returns a 200 with the content of the + // index that you should only GET. Logs for the latter case: + // ```text + // DEBUG redirecting 'https://test.pypi.org/simple' to 'https://test.pypi.org/simple/' + // DEBUG reuse idle connection for ("https", test.pypi.org) + // DEBUG Response code for https://test.pypi.org/simple: 200 OK + // TRACE Response headers for https://test.pypi.org/simple: Response { url: "https://test.pypi.org/simple/", [...] } + // ``` + // twine always errors on redirects (), + // which does not seem desirable either. + if response.url() != registry { + warn_user_once!( + "The request was redirected, please use the new URL for future requests: {}", + response.url() + ); + } + + if status_code.is_success() { + if enabled!(Level::TRACE) { + match response.text().await { + Ok(response_content) => { + trace!("Response content for {registry}: {response_content}"); + } + Err(err) => { + trace!("Failed to read response content for {registry}: {err}"); + } + } + } + return Ok(true); + } + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|content_type| content_type.to_str().ok()) + .map(ToString::to_string); + let upload_error = response + .bytes() + .await + .map_err(|err| PublishSendError::StatusNoBody(status_code, err))?; + let upload_error = String::from_utf8_lossy(&upload_error); + + trace!("Response content for non-200 for {registry}: {upload_error}"); + + debug!("Upload error response: {upload_error}"); + // Detect existing file errors the way twine does. + // https://github.com/pypa/twine/blob/c512bbf166ac38239e58545a39155285f8747a7b/twine/commands/upload.py#L34-L72 + if status_code == 403 { + if upload_error.contains("overwrite artifact") { + // Artifactory (https://jfrog.com/artifactory/) + Ok(false) + } else { + Err(PublishSendError::IncorrectCredentials( + status_code, + PublishSendError::extract_error_message( + upload_error.to_string(), + content_type.as_deref(), + ), + )) + } + } else if status_code == 409 { + // conflict, pypiserver (https://pypi.org/project/pypiserver) + Ok(false) + } else if status_code == 400 + && (upload_error.contains("updating asset") || upload_error.contains("already been taken")) + { + // Nexus Repository OSS (https://www.sonatype.com/nexus-repository-oss) + // and Gitlab Enterprise Edition (https://about.gitlab.com) + Ok(false) + } else { + Err(PublishSendError::Status( + status_code, + PublishSendError::extract_error_message( + upload_error.to_string(), + content_type.as_deref(), + ), + )) + } +} + +#[cfg(test)] +mod tests { + use crate::{build_request, form_metadata}; + use distribution_filename::DistFilename; + use insta::{assert_debug_snapshot, assert_snapshot}; + use itertools::Itertools; + use std::path::PathBuf; + use url::Url; + use uv_client::BaseClientBuilder; + + /// Snapshot the data we send for an upload request for a source distribution. + #[tokio::test] + async fn upload_request_source_dist() { + let filename = "tqdm-999.0.0.tar.gz"; + let file = PathBuf::from("../../scripts/links/").join(filename); + let filename = DistFilename::try_from_normalized_filename(filename).unwrap(); + + let form_metadata = form_metadata(&file, &filename).await.unwrap(); + + let formatted_metadata = form_metadata + .iter() + .map(|(k, v)| format!("{k}: {v}")) + .join("\n"); + assert_snapshot!(&formatted_metadata, @r###" + :action: file_upload + sha256_digest: 89fa05cffa7f457658373b85de302d24d0c205ceda2819a8739e324b75e9430b + protocol_version: 1 + metadata_version: 2.3 + name: tqdm + version: 999.0.0 + filetype: sdist + pyversion: source + description: # tqdm + + [![PyPI - Version](https://img.shields.io/pypi/v/tqdm.svg)](https://pypi.org/project/tqdm) + [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/tqdm.svg)](https://pypi.org/project/tqdm) + + ----- + + **Table of Contents** + + - [Installation](#installation) + - [License](#license) + + ## Installation + + ```console + pip install tqdm + ``` + + ## License + + `tqdm` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license. + + description_content_type: text/markdown + author_email: Charlie Marsh + requires_python: >=3.8 + classifiers: Development Status :: 4 - Beta + classifiers: Programming Language :: Python + classifiers: Programming Language :: Python :: 3.8 + classifiers: Programming Language :: Python :: 3.9 + classifiers: Programming Language :: Python :: 3.10 + classifiers: Programming Language :: Python :: 3.11 + classifiers: Programming Language :: Python :: 3.12 + classifiers: Programming Language :: Python :: Implementation :: CPython + classifiers: Programming Language :: Python :: Implementation :: PyPy + project_urls: Documentation, https://github.com/unknown/tqdm#readme + project_urls: Issues, https://github.com/unknown/tqdm/issues + project_urls: Source, https://github.com/unknown/tqdm + "###); + + let request = build_request( + &file, + &filename, + &Url::parse("https://example.org/upload").unwrap(), + &BaseClientBuilder::new().build(), + Some("ferris"), + Some("F3RR!S"), + form_metadata, + ) + .await + .unwrap(); + + insta::with_settings!({ + filters => [("boundary=[0-9a-f-]+", "boundary=[...]")], + }, { + assert_debug_snapshot!(&request, @r###" + RequestBuilder { + inner: RequestBuilder { + method: POST, + url: Url { + scheme: "https", + cannot_be_a_base: false, + username: "", + password: None, + host: Some( + Domain( + "example.org", + ), + ), + port: None, + path: "/upload", + query: None, + fragment: None, + }, + headers: { + "content-type": "multipart/form-data; boundary=[...]", + "accept": "application/json;q=0.9, text/plain;q=0.8, text/html;q=0.7", + "authorization": "Basic ZmVycmlzOkYzUlIhUw==", + }, + }, + .. + } + "###); + }); + } + + /// Snapshot the data we send for an upload request for a wheel. + #[tokio::test] + async fn upload_request_wheel() { + let filename = "tqdm-4.66.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl"; + let file = PathBuf::from("../../scripts/links/").join(filename); + let filename = DistFilename::try_from_normalized_filename(filename).unwrap(); + + let form_metadata = form_metadata(&file, &filename).await.unwrap(); + + let formatted_metadata = form_metadata + .iter() + .map(|(k, v)| format!("{k}: {v}")) + .join("\n"); + assert_snapshot!(&formatted_metadata, @r###" + :action: file_upload + sha256_digest: 0d88ca657bc6b64995ca416e0c59c71af85cc10015d940fa446c42a8b485ee1c + protocol_version: 1 + metadata_version: 2.1 + name: tqdm + version: 4.66.1 + filetype: bdist_wheel + pyversion: py3 + summary: Fast, Extensible Progress Meter + description_content_type: text/x-rst + maintainer_email: tqdm developers + license: MPL-2.0 AND MIT + keywords: progressbar,progressmeter,progress,bar,meter,rate,eta,console,terminal,time + requires_python: >=3.7 + classifiers: Development Status :: 5 - Production/Stable + classifiers: Environment :: Console + classifiers: Environment :: MacOS X + classifiers: Environment :: Other Environment + classifiers: Environment :: Win32 (MS Windows) + classifiers: Environment :: X11 Applications + classifiers: Framework :: IPython + classifiers: Framework :: Jupyter + classifiers: Intended Audience :: Developers + classifiers: Intended Audience :: Education + classifiers: Intended Audience :: End Users/Desktop + classifiers: Intended Audience :: Other Audience + classifiers: Intended Audience :: System Administrators + classifiers: License :: OSI Approved :: MIT License + classifiers: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) + classifiers: Operating System :: MacOS + classifiers: Operating System :: MacOS :: MacOS X + classifiers: Operating System :: Microsoft + classifiers: Operating System :: Microsoft :: MS-DOS + classifiers: Operating System :: Microsoft :: Windows + classifiers: Operating System :: POSIX + classifiers: Operating System :: POSIX :: BSD + classifiers: Operating System :: POSIX :: BSD :: FreeBSD + classifiers: Operating System :: POSIX :: Linux + classifiers: Operating System :: POSIX :: SunOS/Solaris + classifiers: Operating System :: Unix + classifiers: Programming Language :: Python + classifiers: Programming Language :: Python :: 3 + classifiers: Programming Language :: Python :: 3.7 + classifiers: Programming Language :: Python :: 3.8 + classifiers: Programming Language :: Python :: 3.9 + classifiers: Programming Language :: Python :: 3.10 + classifiers: Programming Language :: Python :: 3.11 + classifiers: Programming Language :: Python :: 3 :: Only + classifiers: Programming Language :: Python :: Implementation + classifiers: Programming Language :: Python :: Implementation :: IronPython + classifiers: Programming Language :: Python :: Implementation :: PyPy + classifiers: Programming Language :: Unix Shell + classifiers: Topic :: Desktop Environment + classifiers: Topic :: Education :: Computer Aided Instruction (CAI) + classifiers: Topic :: Education :: Testing + classifiers: Topic :: Office/Business + classifiers: Topic :: Other/Nonlisted Topic + classifiers: Topic :: Software Development :: Build Tools + classifiers: Topic :: Software Development :: Libraries + classifiers: Topic :: Software Development :: Libraries :: Python Modules + classifiers: Topic :: Software Development :: Pre-processors + classifiers: Topic :: Software Development :: User Interfaces + classifiers: Topic :: System :: Installation/Setup + classifiers: Topic :: System :: Logging + classifiers: Topic :: System :: Monitoring + classifiers: Topic :: System :: Shells + classifiers: Topic :: Terminals + classifiers: Topic :: Utilities + requires_dist: colorama ; platform_system == "Windows" + requires_dist: pytest >=6 ; extra == 'dev' + requires_dist: pytest-cov ; extra == 'dev' + requires_dist: pytest-timeout ; extra == 'dev' + requires_dist: pytest-xdist ; extra == 'dev' + requires_dist: ipywidgets >=6 ; extra == 'notebook' + requires_dist: slack-sdk ; extra == 'slack' + requires_dist: requests ; extra == 'telegram' + project_urls: homepage, https://tqdm.github.io + project_urls: repository, https://github.com/tqdm/tqdm + project_urls: changelog, https://tqdm.github.io/releases + project_urls: wiki, https://github.com/tqdm/tqdm/wiki + "###); + + let request = build_request( + &file, + &filename, + &Url::parse("https://example.org/upload").unwrap(), + &BaseClientBuilder::new().build(), + Some("ferris"), + Some("F3RR!S"), + form_metadata, + ) + .await + .unwrap(); + + insta::with_settings!({ + filters => [("boundary=[0-9a-f-]+", "boundary=[...]")], + }, { + assert_debug_snapshot!(&request, @r###" + RequestBuilder { + inner: RequestBuilder { + method: POST, + url: Url { + scheme: "https", + cannot_be_a_base: false, + username: "", + password: None, + host: Some( + Domain( + "example.org", + ), + ), + port: None, + path: "/upload", + query: None, + fragment: None, + }, + headers: { + "content-type": "multipart/form-data; boundary=[...]", + "accept": "application/json;q=0.9, text/plain;q=0.8, text/html;q=0.7", + "authorization": "Basic ZmVycmlzOkYzUlIhUw==", + }, + }, + .. + } + "###); + }); + } +} diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index b0d49bc402dd..7b39efaf12f7 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -35,6 +35,7 @@ uv-fs = { workspace = true } uv-git = { workspace = true } uv-installer = { workspace = true } uv-normalize = { workspace = true } +uv-publish = { workspace = true } uv-python = { workspace = true, features = ["schemars"]} uv-requirements = { workspace = true } uv-resolver = { workspace = true } diff --git a/crates/uv/src/commands/mod.rs b/crates/uv/src/commands/mod.rs index e9caaa99c415..48d72ab2de35 100644 --- a/crates/uv/src/commands/mod.rs +++ b/crates/uv/src/commands/mod.rs @@ -30,6 +30,7 @@ pub(crate) use project::remove::remove; pub(crate) use project::run::{run, RunCommand}; pub(crate) use project::sync::sync; pub(crate) use project::tree::tree; +pub(crate) use publish::publish; pub(crate) use python::dir::dir as python_dir; pub(crate) use python::find::find as python_find; pub(crate) use python::install::install as python_install; @@ -70,6 +71,7 @@ pub(crate) mod reporters; mod tool; mod build; +mod publish; #[cfg(feature = "self-update")] mod self_update; mod venv; diff --git a/crates/uv/src/commands/publish.rs b/crates/uv/src/commands/publish.rs new file mode 100644 index 000000000000..39073ff09225 --- /dev/null +++ b/crates/uv/src/commands/publish.rs @@ -0,0 +1,74 @@ +use crate::commands::{human_readable_bytes, ExitStatus}; +use crate::printer::Printer; +use anyhow::{bail, Result}; +use owo_colors::OwoColorize; +use std::fmt::Write; +use tracing::info; +use url::Url; +use uv_client::{BaseClientBuilder, Connectivity}; +use uv_configuration::{KeyringProviderType, TrustedHost}; +use uv_publish::{files_for_publishing, upload}; + +pub(crate) async fn publish( + paths: Vec, + publish_url: Url, + keyring_provider: KeyringProviderType, + allow_insecure_host: Vec, + username: Option, + password: Option, + connectivity: Connectivity, + native_tls: bool, + printer: Printer, +) -> Result { + if connectivity.is_offline() { + bail!("You cannot publish files in offline mode"); + } + + let files = files_for_publishing(paths)?; + match files.len() { + 0 => bail!("No files found to publish"), + 1 => writeln!(printer.stderr(), "Publishing 1 file")?, + n => writeln!(printer.stderr(), "Publishing {n} files")?, + } + + let client = BaseClientBuilder::new() + // https://github.com/seanmonstar/reqwest/issues/2416 + .retries(0) + .keyring(keyring_provider) + .native_tls(native_tls) + .allow_insecure_host(allow_insecure_host) + // Don't try cloning the request to make an unauthenticated request first. + // https://github.com/seanmonstar/reqwest/issues/2416 + .only_authenticated(true) + .build(); + + for (file, filename) in files { + let size = file.metadata()?.len(); + let (bytes, unit) = human_readable_bytes(size); + writeln!( + printer.stderr(), + "{} {filename} {}", + "Uploading".bold().green(), + format!("({bytes:.1}{unit})").dimmed() + )?; + let uploaded = upload( + &file, + &filename, + &publish_url, + &client, + username.as_deref(), + password.as_deref(), + ) + .await?; // Filename and/or URL are already attached, if applicable. + info!("Upload succeeded"); + if !uploaded { + writeln!( + printer.stderr(), + "{}", + "File already existed, skipping".dimmed() + )?; + } + } + + Ok(ExitStatus::Success) +} diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index 565219041a6d..4223e9e4ee51 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1076,20 +1076,18 @@ async fn run(cli: Cli) -> Result { allow_insecure_host, } = PublishSettings::resolve(args, filesystem); - todo!( - "{:?}", - ( - files, - publish_url, - keyring_provider, - allow_insecure_host, - username, - password, - globals.connectivity, - globals.native_tls, - printer, - ) + commands::publish( + files, + publish_url, + keyring_provider, + allow_insecure_host, + username, + password, + globals.connectivity, + globals.native_tls, + printer, ) + .await } } } diff --git a/crates/uv/tests/common/mod.rs b/crates/uv/tests/common/mod.rs index 22339af658ea..4da45471c810 100644 --- a/crates/uv/tests/common/mod.rs +++ b/crates/uv/tests/common/mod.rs @@ -52,7 +52,7 @@ pub const INSTA_FILTERS: &[(&str, &str)] = &[ (r"tv_sec: \d+", "tv_sec: [TIME]"), (r"tv_nsec: \d+", "tv_nsec: [TIME]"), // Rewrite Windows output to Unix output - (r"\\([\w\d])", "/$1"), + (r"\\([\w\d]|\.\.)", "/$1"), (r"uv.exe", "uv"), // uv version display ( @@ -579,6 +579,21 @@ impl TestContext { command } + /// Create a `uv publish` command with options shared across scenarios. + #[expect(clippy::unused_self)] // For consistency + pub fn publish(&self) -> Command { + let mut command = Command::new(get_bin()); + command.arg("publish"); + + if cfg!(all(windows, debug_assertions)) { + // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the + // default windows stack of 1MB + command.env("UV_STACK_SIZE", (4 * 1024 * 1024).to_string()); + } + + command + } + /// Create a `uv python find` command with options shared across scenarios. pub fn python_find(&self) -> Command { let mut command = Command::new(get_bin()); diff --git a/crates/uv/tests/publish.rs b/crates/uv/tests/publish.rs new file mode 100644 index 000000000000..d06221c41041 --- /dev/null +++ b/crates/uv/tests/publish.rs @@ -0,0 +1,51 @@ +#![cfg(feature = "pypi")] + +use common::{uv_snapshot, TestContext}; + +mod common; + +#[test] +fn username_password_no_longer_supported() { + let context = TestContext::new("3.12"); + + uv_snapshot!(context.filters(), context.publish() + .arg("-u") + .arg("dummy") + .arg("-p") + .arg("dummy") + .arg("../../scripts/links/ok-1.0.0-py3-none-any.whl"), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Publishing 1 file + Uploading ok-1.0.0-py3-none-any.whl ([SIZE]) + error: Failed to publish `../../scripts/links/ok-1.0.0-py3-none-any.whl` to `https://upload.pypi.org/legacy/` + Caused by: Incorrect credentials (status code 403 Forbidden): 403 Username/Password authentication is no longer supported. Migrate to API Tokens or Trusted Publishers instead. See https://pypi.org/help/#apitoken and https://pypi.org/help/#trusted-publishers + "### + ); +} + +#[test] +fn invalid_token() { + let context = TestContext::new("3.12"); + + uv_snapshot!(context.filters(), context.publish() + .arg("-u") + .arg("__token__") + .arg("-p") + .arg("dummy") + .arg("../../scripts/links/ok-1.0.0-py3-none-any.whl"), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Publishing 1 file + Uploading ok-1.0.0-py3-none-any.whl ([SIZE]) + error: Failed to publish `../../scripts/links/ok-1.0.0-py3-none-any.whl` to `https://upload.pypi.org/legacy/` + Caused by: Incorrect credentials (status code 403 Forbidden): 403 Invalid or non-existent authentication information. See https://pypi.org/help/#invalid-auth for more information. + "### + ); +}