Move more features to monotrail-utils

Cargo.toml

@@ -4,6 +4,7 @@ resolver = "2"
 
 [workspace.dependencies]
 anyhow = "1.0.75"
+cpufeatures = "0.2.11"
 fs-err = "2.9.0"
 fs2 = "0.4.3"
 indoc = "2.0.4"
@@ -15,15 +16,19 @@ regex = "1.9.5"
 serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1.0.107"
 sha2 = "0.10.7"
+tar = "0.4.40"
 target-lexicon = "0.12.11"
 tempfile = "3.8.0"
 thiserror = "1.0.48"
 toml = "0.8.0"
 tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
 unscanny = "0.1.0"
+ureq = { version = "2.7.1", features = ["json"] }
 walkdir = "2.4.0"
 which = "4.4.2"
+widestring = "1.0.2"
+zstd = "0.12.4"
 
 # Config for 'cargo dist'
 [workspace.metadata.dist]

crates/install-wheel-rs/src/wheel.rs

@@ -776,8 +776,7 @@ fn install_script(
     // (#!/usr/bin/env python) for injection monotrail as python into PATH later
     let placeholder_python = b"#!python";
     // scripts might be binaries, so we read an exact number of bytes instead of the first line as string
-    let mut start = Vec::new();
-    start.resize(placeholder_python.len(), 0);
+    let mut start = vec![0; placeholder_python.len()];
     script.read_exact(&mut start)?;
     let size_and_encoded_hash = if start == placeholder_python {
         let start = get_shebang(&location).as_bytes().to_vec();
@@ -1011,6 +1010,7 @@ pub fn parse_key_value_file(
 /// <https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl>
 ///
 /// Wheel 1.0: <https://www.python.org/dev/peps/pep-0427/>
+#[allow(clippy::too_many_arguments)]
 pub fn install_wheel(
     location: &InstallLocation<LockedDir>,
     reader: impl Read + Seek,

crates/monotrail-utils/Cargo.toml

@@ -5,15 +5,24 @@ edition = "2021"
 
 [dependencies]
 anyhow = { workspace = true }
+cpufeatures = { workspace = true }
 fs-err = { workspace = true }
+fs2 = { workspace = true }
 pep508_rs = { workspace = true }
+regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+tar = { workspace = true }
+target-lexicon = { workspace = true }
+tempfile = { workspace = true }
 toml = { workspace = true }
 tracing = { workspace = true }
 unscanny = { workspace = true }
+ureq = { workspace = true }
+zstd = { workspace = true }
 
 [dev-dependencies]
 indoc = { workspace = true }
 logtest = { workspace = true }
+mockito = { workspace = true }
 tempfile = { workspace = true }

crates/monotrail-utils/src/lib.rs

@@ -3,3 +3,5 @@
 pub use requirements_txt::RequirementsTxt;
 
 mod requirements_txt;
+pub mod parse_cpython_args;
+pub mod standalone_python;

crates/monotrail-utils/src/parse_cpython_args.rs

@@ -0,0 +1,41 @@
+/// python has idiosyncratic cli options that are hard to replicate with clap, so we roll our own.
+/// Takes args without the first-is-current-program (i.e. python) convention.
+///
+/// `usage: python [option] ... [-c cmd | -m mod | file | -] [arg] ...`
+///
+/// Returns the script, if any
+pub fn naive_python_arg_parser<T: AsRef<str>>(args: &[T]) -> Result<Option<String>, String> {
+    // These are hand collected from `python --help`
+    // See also https://docs.python.org/3/using/cmdline.html#command-line
+    let no_value_opts = [
+        "-b", "-B", "-d", "-E", "-h", "-i", "-I", "-O", "-OO", "-q", "-s", "-S", "-u", "-v", "-V",
+        "-x", "-?",
+    ];
+    let value_opts = ["--check-hash-based-pycs", "-W", "-X"];
+    let mut arg_iter = args.iter();
+    loop {
+        if let Some(arg) = arg_iter.next() {
+            if no_value_opts.contains(&arg.as_ref()) {
+                continue;
+            } else if value_opts.contains(&arg.as_ref()) {
+                // consume the value belonging to the options
+                let value = arg_iter.next();
+                if value.is_none() {
+                    return Err(format!("Missing argument for {}", arg.as_ref()));
+                }
+                continue;
+            } else if arg.as_ref() == "-c" || arg.as_ref() == "-m" {
+                let value = arg_iter.next();
+                if value.is_none() {
+                    return Err(format!("Missing argument for {}", arg.as_ref()));
+                }
+                return Ok(None);
+            } else {
+                return Ok(Some(arg.as_ref().to_string()));
+            }
+        } else {
+            // interactive python shell
+            return Ok(None);
+        }
+    }
+}

crates/monotrail/src/standalone_python.rs → crates/monotrail-utils/src/standalone_python.rs

@@ -1,9 +1,6 @@
 //! Download and install standalone python builds (PyOxy) from
 //! <https://github.com/indygreg/python-build-standalone>
 
-use crate::markers::marker_environment_from_python;
-use crate::monotrail::{LaunchType, PythonContext};
-use crate::utils::cache_dir;
 use anyhow::{bail, Context};
 use fs2::FileExt;
 use fs_err as fs;
@@ -117,7 +114,7 @@ fn download_and_unpack_python(url: &str, target_dir: &Path) -> anyhow::Result<()
         .into_reader();
     let tar = zstd::Decoder::new(tar_zstd)?;
     let mut archive = tar::Archive::new(tar);
-    fs::create_dir_all(&target_dir)?;
+    fs::create_dir_all(target_dir)?;
     archive.unpack(target_dir)?;
     Ok(())
 }
@@ -165,7 +162,7 @@ fn provision_python_inner(
         )
     })?;
     // atomic installation by tempdir & rename
-    let temp_dir = tempdir_in(&python_parent_dir)
+    let temp_dir = tempdir_in(python_parent_dir)
         .context("Failed to create temporary directory for unpacking")?;
     match download_and_unpack_python(&url, temp_dir.path()) {
         Ok(()) => {}
@@ -187,15 +184,13 @@ fn provision_python_inner(
         }
     }
     // we can use fs::rename here because we stay in the same directory
-    fs::rename(temp_dir, &unpack_dir).context("Failed to move installed python into place")?;
+    fs::rename(temp_dir, unpack_dir).context("Failed to move installed python into place")?;
     debug!("Installed python {}.{}", python_version.0, python_version.1);
     Ok(())
 }
 
-/// If a downloaded python version exists, return this, otherwise download and unpack a matching one
-/// from indygreg/python-build-standalone
-pub fn provision_python(python_version: (u8, u8)) -> anyhow::Result<(PythonContext, PathBuf)> {
-    let python_parent_dir = cache_dir()?.join("python-build-standalone");
+pub fn provision_python(python_version: (u8, u8), cache_dir: &Path) -> anyhow::Result<(PathBuf, PathBuf)> {
+    let python_parent_dir = cache_dir.join("python-build-standalone");
     // We need this here for the locking logic
     fs::create_dir_all(&python_parent_dir).context("Failed to create cache dir")?;
     let unpack_dir =
@@ -243,17 +238,8 @@ pub fn provision_python(python_version: (u8, u8)) -> anyhow::Result<(PythonConte
             .join("bin")
             .join("python3")
     };
-    // TODO: Already init and use libpython here
-    let pep508_env = marker_environment_from_python(&python_binary);
-    let python_context = PythonContext {
-        sys_executable: python_binary,
-        version: python_version,
-        pep508_env,
-        launch_type: LaunchType::Binary,
-    };
-
     let python_home = unpack_dir.join("python").join("install");
-    Ok((python_context, python_home))
+    Ok((python_binary, python_home))
 }
 
 /// Returns a regex matching a compatible optimized build from the indygreg/python-build-standalone
@@ -300,14 +286,27 @@ pub fn filename_regex(major: u8, minor: u8) -> Regex {
         .unwrap()
 }
 
+
 #[cfg(test)]
 mod test {
+    use std::path::PathBuf;
+    use tempfile::tempdir;
     use mockito::{Mock, ServerGuard};
-
     #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
     use crate::standalone_python::provision_python;
     use crate::standalone_python::{find_python, PYTHON_STANDALONE_LATEST_RELEASE};
-    use crate::utils::zstd_json_mock;
+
+    pub fn zstd_json_mock(url: &str, fixture: impl Into<PathBuf>) -> (ServerGuard, Mock) {
+        use fs_err::File;
+
+        let mut server = mockito::Server::new();
+        let mock = server
+            .mock("GET", url)
+            .with_header("content-type", "application/json")
+            .with_body(zstd::stream::decode_all(File::open(fixture).unwrap()).unwrap())
+            .create();
+        (server, mock)
+    }
 
     fn mock() -> (ServerGuard, Mock) {
         zstd_json_mock(
@@ -336,7 +335,8 @@ mod test {
     #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
     fn test_provision_nonexistent_version() {
         let _mocks = mock();
-        let err = provision_python((3, 0)).unwrap_err();
+        let tempdir = tempdir().unwrap();
+        let err = provision_python((3, 0), tempdir.path()).unwrap_err();
         let expected = vec![
             r"Couldn't find a matching python 3.0 to download",
             r"Failed to find a matching python-build-standalone download: /^cpython-3\.0\.(\d+)\+(\d+)-x86_64\-unknown\-linux\-gnu-pgo\+lto-full\.tar\.zst$/. Searched in https://github.com/indygreg/python-build-standalone/releases/latest and https://github.com/indygreg/python-build-standalone/releases/tag/20220502",

crates/monotrail/Cargo.toml

@@ -11,7 +11,7 @@ name = "monotrail"
 [dependencies]
 anyhow = { workspace = true }
 clap = { version = "4.4.4", features = ["derive"] }
-cpufeatures = "0.2.9"
+cpufeatures = { workspace = true }
 data-encoding = "2.4.0"
 dirs = "5.0.1"
 fs-err = { workspace = true }
@@ -21,8 +21,7 @@ indicatif = "0.17.7"
 install-wheel-rs = { version = "0.0.1", path = "../install-wheel-rs" }
 libc = "0.2.148"
 libloading = "0.8.0"
-# For the zig build
-libz-sys = { version = "1.1.12", features = ["static"] }
+libz-sys = { version = "1.1.12", features = ["static"] } # For the zig build
 monotrail-utils = { version = "0.0.1", path = "../monotrail-utils" }
 nix = { version = "0.27.1", features = ["process"] }
 pep440_rs = "0.3.11"
@@ -33,18 +32,18 @@ regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sha2 = { workspace = true }
-tar = "0.4.40"
+tar = { workspace = true }
 target-lexicon = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 toml = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 unscanny = { workspace = true }
-ureq = { version = "2.7.1", features = ["json"] }
+ureq = { workspace = true }
 walkdir = { workspace = true }
-widestring = "1.0.2"
-zstd = "0.12.4"
+widestring = { workspace = true }
+zstd = { workspace = true }
 
 [dev-dependencies]
 indoc = { workspace = true }

crates/monotrail/src/inject_and_run.rs

@@ -1,7 +1,8 @@
 //! Communication with libpython
 
+use monotrail_utils::parse_cpython_args::naive_python_arg_parser;
 use crate::monotrail::{find_scripts, install, load_specs, FinderData, InjectData, PythonContext};
-use crate::standalone_python::provision_python;
+use crate::monotrail::provision_python_env;
 use crate::DEFAULT_PYTHON_VERSION;
 use anyhow::{bail, format_err, Context};
 use fs_err as fs;
@@ -97,47 +98,6 @@ unsafe fn pre_init(lib: &Library) -> anyhow::Result<()> {
     Ok(())
 }
 
-/// python has idiosyncratic cli options that are hard to replicate with clap, so we roll our own.
-/// Takes args without the first-is-current-program (i.e. python) convention.
-///
-/// `usage: python [option] ... [-c cmd | -m mod | file | -] [arg] ...`
-///
-/// Returns the script, if any
-pub fn naive_python_arg_parser<T: AsRef<str>>(args: &[T]) -> Result<Option<String>, String> {
-    // These are hand collected from `python --help`
-    // See also https://docs.python.org/3/using/cmdline.html#command-line
-    let no_value_opts = [
-        "-b", "-B", "-d", "-E", "-h", "-i", "-I", "-O", "-OO", "-q", "-s", "-S", "-u", "-v", "-V",
-        "-x", "-?",
-    ];
-    let value_opts = ["--check-hash-based-pycs", "-W", "-X"];
-    let mut arg_iter = args.iter();
-    loop {
-        if let Some(arg) = arg_iter.next() {
-            if no_value_opts.contains(&arg.as_ref()) {
-                continue;
-            } else if value_opts.contains(&arg.as_ref()) {
-                // consume the value belonging to the options
-                let value = arg_iter.next();
-                if value.is_none() {
-                    return Err(format!("Missing argument for {}", arg.as_ref()));
-                }
-                continue;
-            } else if arg.as_ref() == "-c" || arg.as_ref() == "-m" {
-                let value = arg_iter.next();
-                if value.is_none() {
-                    return Err(format!("Missing argument for {}", arg.as_ref()));
-                }
-                return Ok(None);
-            } else {
-                return Ok(Some(arg.as_ref().to_string()));
-            }
-        } else {
-            // interactive python shell
-            return Ok(None);
-        }
-    }
-}
 
 /// The way we're using to load symbol by symbol with the type generic is really ugly and cumbersome
 /// If you know how to do this with `extern` or even pyo3-ffi directly please tell me.
@@ -355,7 +315,7 @@ pub fn run_python_args(
     extras: &[String],
 ) -> anyhow::Result<i32> {
     let (args, python_version) = determine_python_version(args, python_version)?;
-    let (python_context, python_home) = provision_python(python_version)?;
+    let (python_context, python_home) = provision_python_env(python_version)?;
 
     let script = if let Some(root) = root {
         Some(root.to_path_buf())

crates/monotrail/src/lib.rs

@@ -33,7 +33,6 @@ mod ppipx;
 mod python_bindings;
 mod source_distribution;
 mod spec;
-mod standalone_python;
 mod utils;
 mod venv_parser;
 mod verify_installation;