From 54a81fd6310699aacb5ca911dad957790e0c87bd Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 00:15:32 +0700 Subject: [PATCH 001/102] feat: pacquet-store-dir --- Cargo.lock | 4 ++++ Cargo.toml | 1 + crates/store-dir/Cargo.toml | 11 +++++++++++ crates/store-dir/src/lib.rs | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 crates/store-dir/Cargo.toml create mode 100644 crates/store-dir/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3eb1fe9a6..0cb566a76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1481,6 +1481,10 @@ dependencies = [ "tokio", ] +[[package]] +name = "pacquet-store-dir" +version = "0.0.1" + [[package]] name = "pacquet-tarball" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index c4a270e14..ea14ad8b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ pacquet-npmrc = { path = "crates/npmrc" } pacquet-executor = { path = "crates/executor" } pacquet-cafs = { path = "crates/cafs" } pacquet-diagnostics = { path = "crates/diagnostics" } +pacquet-store-dir = { path = "crates/diagnostics" } # Dependencies async-recursion = { version = "1.0.5" } diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml new file mode 100644 index 000000000..20ebc4a20 --- /dev/null +++ b/crates/store-dir/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "pacquet-store-dir" +description = "Abstraction over store-dir paths" +version = "0.0.1" +publish = false +authors.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +repository.workspace = true diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs new file mode 100644 index 000000000..47dba78ae --- /dev/null +++ b/crates/store-dir/src/lib.rs @@ -0,0 +1,35 @@ +use std::path::PathBuf; + +/// Represent a store directory. +/// +/// * The store directory stores all files that were acquired by installing packages with pacquet or pnpm. +/// * The files in `node_modules` directories are hardlinks or reflinks to the files in the store directory. +/// * The store directory can and often act as a global shared cache of all installation of different workspaces. +/// * The location of the store directory can be customized by `store-dir` field. +pub struct StoreDir { + /// Path to the root of the store directory from which all sub-paths are derived. + /// + /// Consumer of this struct should interact with the sub-paths instead of this path. + root: PathBuf, +} + +impl StoreDir { + /// Get `{store}/v3`. + fn v3(&self) -> PathBuf { + self.root.join("v3") + } + + /// The directory that contains all files from the once-installed packages. + fn files(&self) -> PathBuf { + self.v3().join("files") + } + + /// Path to a file in the store directory. + /// + /// **Parameters:** + /// * `head` is the first 2 hexadecimal digit of the file address. + /// * `tail` is the rest of the address and an optional suffix. + pub fn file_path_by_hash_str(&self, head: &str, tail: &str) -> PathBuf { + self.files().join(format!("{head}{tail}")) + } +} From 13471c35150e89d561256897fdd4931e3ea196b1 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 12:44:58 +0700 Subject: [PATCH 002/102] fix: file_path_by_hash_str --- crates/store-dir/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 47dba78ae..d9d692534 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -30,6 +30,6 @@ impl StoreDir { /// * `head` is the first 2 hexadecimal digit of the file address. /// * `tail` is the rest of the address and an optional suffix. pub fn file_path_by_hash_str(&self, head: &str, tail: &str) -> PathBuf { - self.files().join(format!("{head}{tail}")) + self.files().join(head).join(tail) } } From 30267c980c90b680f3eb8724a806b76fa0d89c11 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 13:09:47 +0700 Subject: [PATCH 003/102] feat: serde --- Cargo.lock | 3 +++ crates/store-dir/Cargo.toml | 3 +++ crates/store-dir/src/lib.rs | 3 +++ 3 files changed, 9 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 0cb566a76..0c12a7aed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1484,6 +1484,9 @@ dependencies = [ [[package]] name = "pacquet-store-dir" version = "0.0.1" +dependencies = [ + "serde", +] [[package]] name = "pacquet-tarball" diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index 20ebc4a20..71915272c 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -9,3 +9,6 @@ homepage.workspace = true keywords.workspace = true license.workspace = true repository.workspace = true + +[dependencies] +serde = { workspace = true } diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index d9d692534..ef0252fdd 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,3 +1,4 @@ +use serde::{Deserialize, Serialize}; use std::path::PathBuf; /// Represent a store directory. @@ -6,6 +7,8 @@ use std::path::PathBuf; /// * The files in `node_modules` directories are hardlinks or reflinks to the files in the store directory. /// * The store directory can and often act as a global shared cache of all installation of different workspaces. /// * The location of the store directory can be customized by `store-dir` field. +#[derive(Debug, Deserialize, Serialize)] +#[serde(transparent)] pub struct StoreDir { /// Path to the root of the store directory from which all sub-paths are derived. /// From b5ef876be1557bb101054a4d63165a8444ca74ed Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 13:17:38 +0700 Subject: [PATCH 004/102] feat: StoreDir::tmp --- crates/store-dir/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index ef0252fdd..ec15753cb 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -35,4 +35,9 @@ impl StoreDir { pub fn file_path_by_hash_str(&self, head: &str, tail: &str) -> PathBuf { self.files().join(head).join(tail) } + + /// Path to the temporary directory inside the store. + pub fn tmp(&self) -> PathBuf { + self.v3().join("tmp") + } } From a79c0a151a6780f79b88f3502df4cc09ea2d290d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 13:19:16 +0700 Subject: [PATCH 005/102] fix(cargo): path to a crate --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ea14ad8b6..9a9761ac6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ pacquet-npmrc = { path = "crates/npmrc" } pacquet-executor = { path = "crates/executor" } pacquet-cafs = { path = "crates/cafs" } pacquet-diagnostics = { path = "crates/diagnostics" } -pacquet-store-dir = { path = "crates/diagnostics" } +pacquet-store-dir = { path = "crates/store-dir" } # Dependencies async-recursion = { version = "1.0.5" } From ddfeeb4f2233f37f5276d2e2d752bc2c76509235 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 13:24:45 +0700 Subject: [PATCH 006/102] feat: impl From for StoreDir --- Cargo.lock | 1 + crates/store-dir/Cargo.toml | 3 ++- crates/store-dir/src/lib.rs | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c12a7aed..5f0a2b286 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1485,6 +1485,7 @@ dependencies = [ name = "pacquet-store-dir" version = "0.0.1" dependencies = [ + "derive_more", "serde", ] diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index 71915272c..9e414792e 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -11,4 +11,5 @@ license.workspace = true repository.workspace = true [dependencies] -serde = { workspace = true } +derive_more = { workspace = true } +serde = { workspace = true } diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index ec15753cb..400eb3379 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,3 +1,4 @@ +use derive_more::From; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -7,7 +8,7 @@ use std::path::PathBuf; /// * The files in `node_modules` directories are hardlinks or reflinks to the files in the store directory. /// * The store directory can and often act as a global shared cache of all installation of different workspaces. /// * The location of the store directory can be customized by `store-dir` field. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, From, Deserialize, Serialize)] #[serde(transparent)] pub struct StoreDir { /// Path to the root of the store directory from which all sub-paths are derived. From b122d5cf72aa75ee84fdc6da0b2da574c9b679c3 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 13:40:47 +0700 Subject: [PATCH 007/102] test: StoreDir --- Cargo.lock | 2 ++ crates/store-dir/Cargo.toml | 4 ++++ crates/store-dir/src/lib.rs | 29 +++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 5f0a2b286..470ff4750 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1486,6 +1486,8 @@ name = "pacquet-store-dir" version = "0.0.1" dependencies = [ "derive_more", + "pipe-trait", + "pretty_assertions", "serde", ] diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index 9e414792e..a84f4351f 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -13,3 +13,7 @@ repository.workspace = true [dependencies] derive_more = { workspace = true } serde = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +pipe-trait = { workspace = true } diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 400eb3379..532b9ae0e 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -42,3 +42,32 @@ impl StoreDir { self.v3().join("tmp") } } + +#[cfg(test)] +mod tests { + use super::*; + use pipe_trait::Pipe; + use pretty_assertions::assert_eq; + + #[cfg(unix)] + #[test] + fn file_path_by_hash_str() { + let received = "/home/user/.local/share/pnpm/store" + .pipe(PathBuf::from) + .pipe(StoreDir::from) + .file_path_by_hash_str("3e", "f722d37b016c63ac0126cfdcec"); + let expected = PathBuf::from( + "/home/user/.local/share/pnpm/store/v3/files/3e/f722d37b016c63ac0126cfdcec", + ); + assert_eq!(&received, &expected); + } + + #[cfg(unix)] + #[test] + fn tmp() { + let received = + "/home/user/.local/share/pnpm/store".pipe(PathBuf::from).pipe(StoreDir::from).tmp(); + let expected = PathBuf::from("/home/user/.local/share/pnpm/store/v3/tmp"); + assert_eq!(&received, &expected); + } +} From 58c3837bf858430f3241a8059f3bc939e5d61afb Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 13:48:03 +0700 Subject: [PATCH 008/102] feat: StoreDir::display --- crates/store-dir/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 532b9ae0e..d83bbb163 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,6 +1,6 @@ use derive_more::From; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; +use std::path::{self, PathBuf}; /// Represent a store directory. /// @@ -18,6 +18,11 @@ pub struct StoreDir { } impl StoreDir { + /// Create an object that [displays](std::fmt::Display) the root of the store directory. + pub fn display(&self) -> path::Display { + self.root.display() + } + /// Get `{store}/v3`. fn v3(&self) -> PathBuf { self.root.join("v3") From 68cfc31ffa220c87f4ff1bc0a383513c724a711f Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 13:52:24 +0700 Subject: [PATCH 009/102] feat: impl Eq for StoreDir --- crates/store-dir/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index d83bbb163..baa750662 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -8,7 +8,7 @@ use std::path::{self, PathBuf}; /// * The files in `node_modules` directories are hardlinks or reflinks to the files in the store directory. /// * The store directory can and often act as a global shared cache of all installation of different workspaces. /// * The location of the store directory can be customized by `store-dir` field. -#[derive(Debug, From, Deserialize, Serialize)] +#[derive(Debug, PartialEq, Eq, From, Deserialize, Serialize)] #[serde(transparent)] pub struct StoreDir { /// Path to the root of the store directory from which all sub-paths are derived. From 6e4ed8649bd60f0c02e67aedf0f2482a566beca0 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 14:33:29 +0700 Subject: [PATCH 010/102] feat: StoreDir::file_path_by_content_address --- Cargo.lock | 2 ++ crates/store-dir/Cargo.toml | 2 ++ crates/store-dir/src/lib.rs | 28 +++++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 470ff4750..b86151a56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1489,6 +1489,8 @@ dependencies = [ "pipe-trait", "pretty_assertions", "serde", + "ssri", + "strum", ] [[package]] diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index a84f4351f..b549404bd 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -13,6 +13,8 @@ repository.workspace = true [dependencies] derive_more = { workspace = true } serde = { workspace = true } +ssri = { workspace = true } +strum = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index baa750662..44493c92b 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,6 +1,17 @@ use derive_more::From; use serde::{Deserialize, Serialize}; +use ssri::{Algorithm, Integrity}; // TODO: use proper sha2::Sha512 to remove assert_eq use std::path::{self, PathBuf}; +use strum::IntoStaticStr; + +/// Optional suffix of a content address of a file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr)] +pub enum FileSuffix { + #[strum(serialize = "-exec")] + Exec, + #[strum(serialize = "-index.json")] + Index, +} /// Represent a store directory. /// @@ -38,10 +49,25 @@ impl StoreDir { /// **Parameters:** /// * `head` is the first 2 hexadecimal digit of the file address. /// * `tail` is the rest of the address and an optional suffix. - pub fn file_path_by_hash_str(&self, head: &str, tail: &str) -> PathBuf { + fn file_path_by_hash_str(&self, head: &str, tail: &str) -> PathBuf { self.files().join(head).join(tail) } + /// Path to a file in the store directory. + pub fn file_path_by_content_address( + &self, + integrity: Integrity, // TODO: use proper sha2::Sha512 to remove assert_eq + suffix: Option, + ) -> PathBuf { + let (algorithm, hex) = integrity.to_hex(); + assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 algorithm is supported"); // TODO: use proper sha2::Sha512 to remove assert_eq + let head = &hex[..2]; + let middle = &hex[2..]; + let suffix = suffix.map_or("", <&str>::from); + let tail = format!("{middle}{suffix}"); + self.file_path_by_hash_str(head, &tail) + } + /// Path to the temporary directory inside the store. pub fn tmp(&self) -> PathBuf { self.v3().join("tmp") From bb931d92da942becbb29392249d3160fabed128a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 14:37:17 +0700 Subject: [PATCH 011/102] fix: borrow --- crates/store-dir/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 44493c92b..d48de6996 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -56,7 +56,7 @@ impl StoreDir { /// Path to a file in the store directory. pub fn file_path_by_content_address( &self, - integrity: Integrity, // TODO: use proper sha2::Sha512 to remove assert_eq + integrity: &Integrity, // TODO: use proper sha2::Sha512 to remove assert_eq suffix: Option, ) -> PathBuf { let (algorithm, hex) = integrity.to_hex(); From 0ab7ff4f0d7522aab570a343a474cf1f0263bc29 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 14:53:34 +0700 Subject: [PATCH 012/102] feat: use StoreDir --- Cargo.lock | 4 ++ crates/cafs/Cargo.toml | 2 + crates/cafs/src/lib.rs | 14 +++--- crates/cli/src/cli_args/store.rs | 4 +- crates/npmrc/Cargo.toml | 2 + crates/npmrc/src/custom_deserializer.rs | 44 +++++++++++-------- crates/npmrc/src/lib.rs | 18 ++++---- crates/package-manager/src/install.rs | 2 +- .../src/install_package_from_registry.rs | 2 +- crates/tarball/Cargo.toml | 1 + crates/tarball/src/lib.rs | 14 +++--- tasks/micro-benchmark/Cargo.toml | 5 ++- tasks/micro-benchmark/src/main.rs | 6 ++- 13 files changed, 70 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b86151a56..d3bc7c099 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1303,6 +1303,7 @@ version = "0.0.1" dependencies = [ "derive_more", "miette", + "pacquet-store-dir", "pretty_assertions", "ssri", "tempfile", @@ -1401,6 +1402,7 @@ dependencies = [ "mockito", "node-semver", "pacquet-registry", + "pacquet-store-dir", "pacquet-tarball", "pipe-trait", "project-root", @@ -1414,6 +1416,7 @@ name = "pacquet-npmrc" version = "0.0.1" dependencies = [ "home", + "pacquet-store-dir", "pipe-trait", "pretty_assertions", "serde", @@ -1502,6 +1505,7 @@ dependencies = [ "miette", "pacquet-cafs", "pacquet-diagnostics", + "pacquet-store-dir", "pipe-trait", "pretty_assertions", "reqwest", diff --git a/crates/cafs/Cargo.toml b/crates/cafs/Cargo.toml index 02e0b9f5e..ad67d6b90 100644 --- a/crates/cafs/Cargo.toml +++ b/crates/cafs/Cargo.toml @@ -11,6 +11,8 @@ license.workspace = true repository.workspace = true [dependencies] +pacquet-store-dir = { workspace = true } + derive_more = { workspace = true } miette = { workspace = true } ssri = { workspace = true } diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index c55179e26..6da261109 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -7,6 +7,7 @@ use std::{ use derive_more::{Display, Error, From}; use miette::Diagnostic; +use pacquet_store_dir::StoreDir; use ssri::{Algorithm, IntegrityOpts}; #[derive(Debug, Display, Error, From, Diagnostic)] @@ -37,11 +38,9 @@ fn content_path_from_hex(file_type: FileType, hex: &str) -> PathBuf { Path::new(&hex[..2]).join(file_name) } -pub fn write_sync(store_dir: &Path, buffer: &[u8]) -> Result { - let hex_integrity = - IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(buffer).result().to_hex().1; - let content_path = content_path_from_hex(FileType::NonExec, &hex_integrity); - let file_path = store_dir.join(&content_path); +pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result { + let integrity = IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(buffer).result(); + let file_path = store_dir.file_path_by_content_address(&integrity, None); if !file_path.exists() { let parent_dir = file_path.parent().unwrap(); @@ -49,7 +48,7 @@ pub fn write_sync(store_dir: &Path, buffer: &[u8]) -> Result fs::write(&file_path, buffer)?; } - Ok(content_path) + Ok(file_path) } pub fn prune_sync(store_dir: &Path) -> Result<(), CafsError> { @@ -88,8 +87,7 @@ mod tests { fn should_write_and_clear() { let dir = tempdir().unwrap(); let buffer = vec![0, 1, 2, 3, 4, 5, 6]; - let saved_file_path = write_sync(dir.path(), &buffer).unwrap(); - let store_path = dir.path().join(saved_file_path); + let store_path = write_sync(&dir.path().to_path_buf().into(), &buffer).unwrap(); assert!(store_path.exists()); prune_sync(dir.path()).unwrap(); assert!(!store_path.exists()); diff --git a/crates/cli/src/cli_args/store.rs b/crates/cli/src/cli_args/store.rs index 4a908fe41..f6d2a99f6 100644 --- a/crates/cli/src/cli_args/store.rs +++ b/crates/cli/src/cli_args/store.rs @@ -1,5 +1,4 @@ use clap::Subcommand; -use miette::Context; use pacquet_npmrc::Npmrc; #[derive(Debug, Subcommand)] @@ -29,7 +28,8 @@ impl StoreCommand { panic!("Not implemented") } StoreCommand::Prune => { - pacquet_cafs::prune_sync(&config().store_dir).wrap_err("pruning store")?; + // pacquet_cafs::prune_sync(&config().store_dir).wrap_err("pruning store")?; + todo!("pruning the store is not yet implemented") } StoreCommand::Path => { println!("{}", config().store_dir.display()); diff --git a/crates/npmrc/Cargo.toml b/crates/npmrc/Cargo.toml index ec11d9ab4..98b7cff9e 100644 --- a/crates/npmrc/Cargo.toml +++ b/crates/npmrc/Cargo.toml @@ -11,6 +11,8 @@ license.workspace = true repository.workspace = true [dependencies] +pacquet-store-dir = { workspace = true } + home = { workspace = true } pipe-trait = { workspace = true } serde = { workspace = true } diff --git a/crates/npmrc/src/custom_deserializer.rs b/crates/npmrc/src/custom_deserializer.rs index aa0a41406..614c17458 100644 --- a/crates/npmrc/src/custom_deserializer.rs +++ b/crates/npmrc/src/custom_deserializer.rs @@ -1,3 +1,4 @@ +use pacquet_store_dir::StoreDir; use serde::{de, Deserialize, Deserializer}; use std::{env, path::PathBuf, str::FromStr}; @@ -45,19 +46,19 @@ fn default_store_dir_windows(home_dir: &Path, current_dir: &Path) -> PathBuf { PathBuf::from(format!("{current_drive}:\\.pacquet-store")) } -/// If the $PACQUET_HOME env variable is set, then $PACQUET_HOME/store +/// If the $PNPM_HOME env variable is set, then $PNPM_HOME/store /// If the $XDG_DATA_HOME env variable is set, then $XDG_DATA_HOME/pacquet/store /// On Windows: ~/AppData/Local/pacquet/store /// On macOS: ~/Library/pacquet/store /// On Linux: ~/.local/share/pacquet/store -pub fn default_store_dir() -> PathBuf { +pub fn default_store_dir() -> StoreDir { // TODO: If env variables start with ~, make sure to resolve it into home_dir. - if let Ok(pacquet_home) = env::var("PACQUET_HOME") { - return PathBuf::from(pacquet_home).join("store"); + if let Ok(pnpm_home) = env::var("PNPM_HOME") { + return PathBuf::from(pnpm_home).join("store").into(); } if let Ok(xdg_data_home) = env::var("XDG_DATA_HOME") { - return PathBuf::from(xdg_data_home).join("pacquet/store"); + return PathBuf::from(xdg_data_home).join("pnpm").join("store").into(); } // Using ~ (tilde) for defining home path is not supported in Rust and @@ -67,13 +68,13 @@ pub fn default_store_dir() -> PathBuf { #[cfg(windows)] if cfg!(windows) { let current_dir = env::current_dir().expect("current directory is unavailable"); - return default_store_dir_windows(&home_dir, ¤t_dir); + return default_store_dir_windows(&home_dir, ¤t_dir).into(); } // https://doc.rust-lang.org/std/env/consts/constant.OS.html match env::consts::OS { - "linux" => home_dir.join(".local/share/pacquet/store"), - "macos" => home_dir.join("Library/pacquet/store"), + "linux" => home_dir.join(".local/share/pacquet/store").into(), + "macos" => home_dir.join("Library/pacquet/store").into(), _ => panic!("unsupported operating system: {}", env::consts::OS), } } @@ -126,6 +127,13 @@ where Ok(env::current_dir().map_err(de::Error::custom)?.join(path)) } +pub fn deserialize_store_dir<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + deserialize_pathbuf(deserializer).map(StoreDir::from) +} + /// This deserializer adds a trailing "/" if not exist to make our life easier. pub fn deserialize_registry<'de, D>(deserializer: D) -> Result where @@ -142,23 +150,23 @@ where #[cfg(test)] mod tests { + use super::*; use pretty_assertions::assert_eq; - use std::{env, path::Path}; + use std::env; - use super::*; #[test] - fn test_default_store_dir_with_pac_env() { - env::set_var("PACQUET_HOME", "/tmp/pacquet_home"); - let store_dir = default_store_dir(); - assert_eq!(store_dir, Path::new("/tmp/pacquet_home/store")); - env::remove_var("PACQUET_HOME"); + fn test_default_store_dir_with_pnpm_home_env() { + env::set_var("PNPM_HOME", "/tmp/pnpm-home"); // TODO: change this to dependency injection + let store_dir = default_store_dir().display().to_string(); + assert_eq!(store_dir, "/tmp/pnpm-home/store"); + env::remove_var("PNPM_HOME"); } #[test] fn test_default_store_dir_with_xdg_env() { - env::set_var("XDG_DATA_HOME", "/tmp/xdg_data_home"); - let store_dir = default_store_dir(); - assert_eq!(store_dir, Path::new("/tmp/xdg_data_home/pacquet/store")); + env::set_var("XDG_DATA_HOME", "/tmp/xdg_data_home"); // TODO: change this to dependency injection + let store_dir = default_store_dir().display().to_string(); + assert_eq!(store_dir, "/tmp/xdg_data_home/pnpm/store"); env::remove_var("XDG_DATA_HOME"); } diff --git a/crates/npmrc/src/lib.rs b/crates/npmrc/src/lib.rs index d7e9323ea..6e77ef014 100644 --- a/crates/npmrc/src/lib.rs +++ b/crates/npmrc/src/lib.rs @@ -1,5 +1,6 @@ mod custom_deserializer; +use pacquet_store_dir::StoreDir; use pipe_trait::Pipe; use serde::Deserialize; use std::{fs, path::PathBuf}; @@ -7,7 +8,8 @@ use std::{fs, path::PathBuf}; use crate::custom_deserializer::{ bool_true, default_hoist_pattern, default_modules_cache_max_age, default_modules_dir, default_public_hoist_pattern, default_registry, default_store_dir, default_virtual_store_dir, - deserialize_bool, deserialize_pathbuf, deserialize_registry, deserialize_u64, + deserialize_bool, deserialize_pathbuf, deserialize_registry, deserialize_store_dir, + deserialize_u64, }; #[derive(Debug, Deserialize, Default, PartialEq)] @@ -79,8 +81,8 @@ pub struct Npmrc { pub shamefully_hoist: bool, /// The location where all the packages are saved on the disk. - #[serde(default = "default_store_dir", deserialize_with = "deserialize_pathbuf")] - pub store_dir: PathBuf, + #[serde(default = "default_store_dir", deserialize_with = "deserialize_store_dir")] + pub store_dir: StoreDir, /// The directory in which dependencies will be installed (instead of node_modules). #[serde(default = "default_modules_dir", deserialize_with = "deserialize_pathbuf")] @@ -246,18 +248,18 @@ mod tests { } #[test] - pub fn should_use_pacquet_home_env_var() { - env::set_var("PACQUET_HOME", "/hello"); + pub fn should_use_pnpm_home_env_var() { + env::set_var("PNPM_HOME", "/hello"); // TODO: change this to dependency injection let value: Npmrc = serde_ini::from_str("").unwrap(); - assert_eq!(value.store_dir, PathBuf::from_str("/hello/store").unwrap()); - env::remove_var("PACQUET_HOME"); + assert_eq!(value.store_dir.display().to_string(), "/hello/store"); + env::remove_var("PNPM_HOME"); } #[test] pub fn should_use_xdg_data_home_env_var() { env::set_var("XDG_DATA_HOME", "/hello"); let value: Npmrc = serde_ini::from_str("").unwrap(); - assert_eq!(value.store_dir, PathBuf::from_str("/hello/pacquet/store").unwrap()); + assert_eq!(value.store_dir.display().to_string(), "/hello/pnpm/store"); env::remove_var("XDG_DATA_HOME"); } diff --git a/crates/package-manager/src/install.rs b/crates/package-manager/src/install.rs index 2cb681be3..4f2a7dfae 100644 --- a/crates/package-manager/src/install.rs +++ b/crates/package-manager/src/install.rs @@ -102,7 +102,7 @@ mod tests { manifest.save().unwrap(); let mut config = Npmrc::new(); - config.store_dir = store_dir.to_path_buf(); + config.store_dir = store_dir.to_path_buf().into(); config.modules_dir = modules_dir.to_path_buf(); config.virtual_store_dir = virtual_store_dir.to_path_buf(); let config = config.leak(); diff --git a/crates/package-manager/src/install_package_from_registry.rs b/crates/package-manager/src/install_package_from_registry.rs index a2a01e87f..f341b6d3e 100644 --- a/crates/package-manager/src/install_package_from_registry.rs +++ b/crates/package-manager/src/install_package_from_registry.rs @@ -127,7 +127,7 @@ mod tests { hoist_pattern: vec![], public_hoist_pattern: vec![], shamefully_hoist: false, - store_dir: store_dir.to_path_buf(), + store_dir: store_dir.to_path_buf().into(), modules_dir: modules_dir.to_path_buf(), node_linker: Default::default(), symlink: false, diff --git a/crates/tarball/Cargo.toml b/crates/tarball/Cargo.toml index ae787be44..6e80d3165 100644 --- a/crates/tarball/Cargo.toml +++ b/crates/tarball/Cargo.toml @@ -13,6 +13,7 @@ repository.workspace = true [dependencies] pacquet-cafs = { workspace = true } pacquet-diagnostics = { workspace = true } +pacquet-store-dir = { workspace = true } dashmap = { workspace = true } derive_more = { workspace = true } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 09222a56c..9c5014692 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -2,13 +2,14 @@ use std::{ collections::HashMap, ffi::OsString, io::{Cursor, Read}, - path::{Path, PathBuf}, + path::PathBuf, sync::Arc, }; use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; +use pacquet_store_dir::StoreDir; use pipe_trait::Pipe; use reqwest::Client; use ssri::{Integrity, IntegrityChecker}; @@ -116,7 +117,7 @@ fn verify_checksum(data: &[u8], integrity: Integrity) -> Result { pub tarball_cache: &'a Cache, pub http_client: &'a Client, - pub store_dir: &'static Path, + pub store_dir: &'static StoreDir, pub package_integrity: &'a str, pub package_unpacked_size: Option, pub package_url: &'a str, @@ -217,10 +218,10 @@ impl<'a> DownloadTarballToStore<'a> { let entry_path = entry.path().unwrap(); let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); - let integrity = pacquet_cafs::write_sync(store_dir, &buffer) + let file_path = pacquet_cafs::write_sync(store_dir, &buffer) .map_err(TarballError::WriteCafs)?; - Ok((cleaned_entry_path, store_dir.join(integrity))) + Ok((cleaned_entry_path, file_path)) }) .collect::, TarballError>>() .map_err(TaskError::Other) @@ -259,9 +260,10 @@ mod tests { /// /// **Side effect:** /// The `'static` path becomes dangling outside the scope of [`TempDir`]. - fn tempdir_with_leaked_path() -> (TempDir, &'static Path) { + fn tempdir_with_leaked_path() -> (TempDir, &'static StoreDir) { let tempdir = tempdir().unwrap(); - let leaked_path = tempdir.path().to_path_buf().pipe(Box::new).pipe(Box::leak); + let leaked_path = + tempdir.path().to_path_buf().pipe(StoreDir::from).pipe(Box::new).pipe(Box::leak); (tempdir, leaked_path) } diff --git a/tasks/micro-benchmark/Cargo.toml b/tasks/micro-benchmark/Cargo.toml index cb620dee8..02057066d 100644 --- a/tasks/micro-benchmark/Cargo.toml +++ b/tasks/micro-benchmark/Cargo.toml @@ -15,8 +15,9 @@ name = "micro-benchmark" path = "src/main.rs" [dependencies] -pacquet-registry = { workspace = true } -pacquet-tarball = { workspace = true } +pacquet-registry = { workspace = true } +pacquet-store-dir = { workspace = true } +pacquet-tarball = { workspace = true } clap = { workspace = true } criterion = { workspace = true } diff --git a/tasks/micro-benchmark/src/main.rs b/tasks/micro-benchmark/src/main.rs index 96f1ae9a6..2b8fb5e92 100644 --- a/tasks/micro-benchmark/src/main.rs +++ b/tasks/micro-benchmark/src/main.rs @@ -3,6 +3,7 @@ use std::{fs, path::Path}; use clap::Parser; use criterion::{Criterion, Throughput}; use mockito::ServerGuard; +use pacquet_store_dir::StoreDir; use pacquet_tarball::DownloadTarballToStore; use pipe_trait::Pipe; use project_root::get_project_root; @@ -28,14 +29,15 @@ fn bench_tarball(c: &mut Criterion, server: &mut ServerGuard, fixtures_folder: & group.bench_function("download_dependency", |b| { b.to_async(&rt).iter(|| async { // NOTE: the tempdir is being leaked, meaning the cleanup would be postponed until the end of the benchmark - let dir = tempdir().unwrap().pipe(Box::new).pipe(Box::leak); + let dir = tempdir().unwrap(); + let store_dir = dir.path().to_path_buf().pipe(StoreDir::from).pipe(Box::new).pipe(Box::leak); let http_client = Client::new(); let cas_map = DownloadTarballToStore{ tarball_cache: &Default::default(), http_client: &http_client, - store_dir: dir.path(), + store_dir, package_integrity: "sha512-dj7vjIn1Ar8sVXj2yAXiMNCJDmS9MQ9XMlIecX2dIzzhjSHCyKo4DdXjXMs7wKW2kj6yvVRSpuQjOZ3YLrh56w==", package_unpacked_size: Some(16697), package_url: url, From c8526d0d33ad270bc499e65b7c48d0191ea47811 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 15:15:17 +0700 Subject: [PATCH 013/102] feat: remove incomplete prune implementation --- crates/cafs/src/lib.rs | 16 ++-------------- crates/cli/src/cli_args/store.rs | 3 +-- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 6da261109..303d88fa7 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -51,11 +51,9 @@ pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result Result<(), CafsError> { - // TODO: This should remove unreferenced packages, not all packages. +pub fn prune_sync(store_dir: &StoreDir) -> Result<(), CafsError> { // Ref: https://pnpm.io/cli/store#prune - fs::remove_dir_all(store_dir)?; - Ok(()) + todo!("remove orphaned files") } #[cfg(test)] @@ -82,14 +80,4 @@ mod tests { PathBuf::from("12/34567890abcdef1234567890abcdef12345678-index.json") ); } - - #[test] - fn should_write_and_clear() { - let dir = tempdir().unwrap(); - let buffer = vec![0, 1, 2, 3, 4, 5, 6]; - let store_path = write_sync(&dir.path().to_path_buf().into(), &buffer).unwrap(); - assert!(store_path.exists()); - prune_sync(dir.path()).unwrap(); - assert!(!store_path.exists()); - } } diff --git a/crates/cli/src/cli_args/store.rs b/crates/cli/src/cli_args/store.rs index f6d2a99f6..40f45d4e7 100644 --- a/crates/cli/src/cli_args/store.rs +++ b/crates/cli/src/cli_args/store.rs @@ -28,8 +28,7 @@ impl StoreCommand { panic!("Not implemented") } StoreCommand::Prune => { - // pacquet_cafs::prune_sync(&config().store_dir).wrap_err("pruning store")?; - todo!("pruning the store is not yet implemented") + pacquet_cafs::prune_sync(&config().store_dir).wrap_err("pruning store")?; } StoreCommand::Path => { println!("{}", config().store_dir.display()); From 1d716f29a51d1cb2341f2a753658240807839c87 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 15:17:22 +0700 Subject: [PATCH 014/102] fix: compile error --- crates/cli/src/cli_args/store.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cli/src/cli_args/store.rs b/crates/cli/src/cli_args/store.rs index 40f45d4e7..4a908fe41 100644 --- a/crates/cli/src/cli_args/store.rs +++ b/crates/cli/src/cli_args/store.rs @@ -1,4 +1,5 @@ use clap::Subcommand; +use miette::Context; use pacquet_npmrc::Npmrc; #[derive(Debug, Subcommand)] From e8c12fb7843cfe9125a38127cd57838ecf4f0d5a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 15:41:40 +0700 Subject: [PATCH 015/102] test(windows): fix --- crates/npmrc/src/custom_deserializer.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/npmrc/src/custom_deserializer.rs b/crates/npmrc/src/custom_deserializer.rs index 614c17458..e8734404e 100644 --- a/crates/npmrc/src/custom_deserializer.rs +++ b/crates/npmrc/src/custom_deserializer.rs @@ -154,6 +154,7 @@ mod tests { use pretty_assertions::assert_eq; use std::env; + #[cfg(unix)] #[test] fn test_default_store_dir_with_pnpm_home_env() { env::set_var("PNPM_HOME", "/tmp/pnpm-home"); // TODO: change this to dependency injection @@ -162,6 +163,7 @@ mod tests { env::remove_var("PNPM_HOME"); } + #[cfg(unix)] #[test] fn test_default_store_dir_with_xdg_env() { env::set_var("XDG_DATA_HOME", "/tmp/xdg_data_home"); // TODO: change this to dependency injection From 90da00b08d65fb8707a4d05c340baffe3db70dc4 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 20:20:47 +0700 Subject: [PATCH 016/102] test(windows): fix --- crates/npmrc/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/npmrc/src/lib.rs b/crates/npmrc/src/lib.rs index 6e77ef014..fe223e358 100644 --- a/crates/npmrc/src/lib.rs +++ b/crates/npmrc/src/lib.rs @@ -247,6 +247,7 @@ mod tests { assert_eq!(value.modules_cache_max_age, 1000); } + #[cfg(unix)] #[test] pub fn should_use_pnpm_home_env_var() { env::set_var("PNPM_HOME", "/hello"); // TODO: change this to dependency injection @@ -255,6 +256,7 @@ mod tests { env::remove_var("PNPM_HOME"); } + #[cfg(unix)] #[test] pub fn should_use_xdg_data_home_env_var() { env::set_var("XDG_DATA_HOME", "/hello"); From 6d43bce398e75146e9f4850d7cfbb0b2a6670f9a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 20:27:20 +0700 Subject: [PATCH 017/102] refactor: remove unused items --- crates/cafs/src/lib.rs | 57 ++---------------------------------------- 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 303d88fa7..31e7226e9 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -1,14 +1,8 @@ -#![allow(unused)] - -use std::{ - fs, - path::{Path, PathBuf}, -}; - use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::StoreDir; use ssri::{Algorithm, IntegrityOpts}; +use std::{fs, path::PathBuf}; #[derive(Debug, Display, Error, From, Diagnostic)] #[non_exhaustive] @@ -17,27 +11,6 @@ pub enum CafsError { Io(std::io::Error), // TODO: remove derive(From), split this variant } -enum FileType { - Exec, - NonExec, - Index, -} - -impl FileType { - fn file_name_suffix(&self) -> &'static str { - match self { - FileType::Exec => "-exec", - FileType::NonExec => "", - FileType::Index => "-index.json", - } - } -} - -fn content_path_from_hex(file_type: FileType, hex: &str) -> PathBuf { - let file_name = format!("{}{}", &hex[2..], file_type.file_name_suffix()); - Path::new(&hex[..2]).join(file_name) -} - pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result { let integrity = IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(buffer).result(); let file_path = store_dir.file_path_by_content_address(&integrity, None); @@ -51,33 +24,7 @@ pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result Result<(), CafsError> { +pub fn prune_sync(_store_dir: &StoreDir) -> Result<(), CafsError> { // Ref: https://pnpm.io/cli/store#prune todo!("remove orphaned files") } - -#[cfg(test)] -mod tests { - use std::{env, str::FromStr}; - - use pretty_assertions::assert_eq; - use tempfile::tempdir; - - use super::*; - - #[test] - fn create_content_path_from_hex() { - assert_eq!( - content_path_from_hex(FileType::NonExec, "1234567890abcdef1234567890abcdef12345678"), - PathBuf::from("12/34567890abcdef1234567890abcdef12345678") - ); - assert_eq!( - content_path_from_hex(FileType::Exec, "1234567890abcdef1234567890abcdef12345678"), - PathBuf::from("12/34567890abcdef1234567890abcdef12345678-exec") - ); - assert_eq!( - content_path_from_hex(FileType::Index, "1234567890abcdef1234567890abcdef12345678"), - PathBuf::from("12/34567890abcdef1234567890abcdef12345678-index.json") - ); - } -} From 44c4f49039812b3510f406d1016124fc5f11b85d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 20:40:01 +0700 Subject: [PATCH 018/102] refactor: remove unnecessary `.to_path_buf()` --- crates/package-manager/src/install.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/package-manager/src/install.rs b/crates/package-manager/src/install.rs index 4f2a7dfae..f5a334f4c 100644 --- a/crates/package-manager/src/install.rs +++ b/crates/package-manager/src/install.rs @@ -102,7 +102,7 @@ mod tests { manifest.save().unwrap(); let mut config = Npmrc::new(); - config.store_dir = store_dir.to_path_buf().into(); + config.store_dir = store_dir.into(); config.modules_dir = modules_dir.to_path_buf(); config.virtual_store_dir = virtual_store_dir.to_path_buf(); let config = config.leak(); From 185e4bf55d3a6546591198b5469a17163221f380 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 20:58:49 +0700 Subject: [PATCH 019/102] test: file_path_by_content_address --- crates/store-dir/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index d48de6996..638a4f20f 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -79,6 +79,7 @@ mod tests { use super::*; use pipe_trait::Pipe; use pretty_assertions::assert_eq; + use ssri::{Algorithm, IntegrityOpts}; #[cfg(unix)] #[test] @@ -93,6 +94,39 @@ mod tests { assert_eq!(&received, &expected); } + #[cfg(unix)] + #[test] + fn file_path_by_content_address() { + fn case(file_content: &str, suffix: Option, expected: &str) { + eprintln!("CASE: {file_content:?}, {suffix:?}"); + let store_dir = "STORE_DIR".pipe(PathBuf::from).pipe(StoreDir::from); + let integrity = + IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(file_content).result(); + dbg!(&integrity); + let received = store_dir.file_path_by_content_address(&integrity, suffix); + let expected = PathBuf::from(expected); + assert_eq!(&received, &expected); + } + + case( + "hello world", + None, + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", + ); + + case( + "hello world", + Some(FileSuffix::Exec), + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", + ); + + case( + "hello world", + Some(FileSuffix::Index), + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-index.json", + ); + } + #[cfg(unix)] #[test] fn tmp() { From 0c4e27c8713adf2d9882b140e70bc5157a47b740 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 21:05:21 +0700 Subject: [PATCH 020/102] refactor: StoreDir::new --- Cargo.lock | 1 + crates/package-manager/Cargo.toml | 1 + .../src/install_package_from_registry.rs | 3 ++- crates/store-dir/src/lib.rs | 13 ++++++++----- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3bc7c099..39a3be07c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1439,6 +1439,7 @@ dependencies = [ "pacquet-npmrc", "pacquet-package-manifest", "pacquet-registry", + "pacquet-store-dir", "pacquet-tarball", "pacquet-testing-utils", "pipe-trait", diff --git a/crates/package-manager/Cargo.toml b/crates/package-manager/Cargo.toml index 455092e60..b679b33b7 100644 --- a/crates/package-manager/Cargo.toml +++ b/crates/package-manager/Cargo.toml @@ -30,6 +30,7 @@ tracing = { workspace = true } miette = { workspace = true } [dev-dependencies] +pacquet-store-dir = { workspace = true } pacquet-testing-utils = { workspace = true } node-semver = { workspace = true } diff --git a/crates/package-manager/src/install_package_from_registry.rs b/crates/package-manager/src/install_package_from_registry.rs index f341b6d3e..6a0f31df8 100644 --- a/crates/package-manager/src/install_package_from_registry.rs +++ b/crates/package-manager/src/install_package_from_registry.rs @@ -115,6 +115,7 @@ mod tests { use super::*; use node_semver::Version; use pacquet_npmrc::Npmrc; + use pacquet_store_dir::StoreDir; use pipe_trait::Pipe; use pretty_assertions::assert_eq; use std::fs; @@ -127,7 +128,7 @@ mod tests { hoist_pattern: vec![], public_hoist_pattern: vec![], shamefully_hoist: false, - store_dir: store_dir.to_path_buf().into(), + store_dir: StoreDir::new(store_dir), modules_dir: modules_dir.to_path_buf(), node_linker: Default::default(), symlink: false, diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 638a4f20f..570508610 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -29,6 +29,11 @@ pub struct StoreDir { } impl StoreDir { + /// Construct an instance of [`StoreDir`]. + pub fn new(root: impl Into) -> Self { + root.into().into() + } + /// Create an object that [displays](std::fmt::Display) the root of the store directory. pub fn display(&self) -> path::Display { self.root.display() @@ -85,8 +90,7 @@ mod tests { #[test] fn file_path_by_hash_str() { let received = "/home/user/.local/share/pnpm/store" - .pipe(PathBuf::from) - .pipe(StoreDir::from) + .pipe(StoreDir::new) .file_path_by_hash_str("3e", "f722d37b016c63ac0126cfdcec"); let expected = PathBuf::from( "/home/user/.local/share/pnpm/store/v3/files/3e/f722d37b016c63ac0126cfdcec", @@ -99,7 +103,7 @@ mod tests { fn file_path_by_content_address() { fn case(file_content: &str, suffix: Option, expected: &str) { eprintln!("CASE: {file_content:?}, {suffix:?}"); - let store_dir = "STORE_DIR".pipe(PathBuf::from).pipe(StoreDir::from); + let store_dir = StoreDir::new("STORE_DIR"); let integrity = IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(file_content).result(); dbg!(&integrity); @@ -130,8 +134,7 @@ mod tests { #[cfg(unix)] #[test] fn tmp() { - let received = - "/home/user/.local/share/pnpm/store".pipe(PathBuf::from).pipe(StoreDir::from).tmp(); + let received = StoreDir::new("/home/user/.local/share/pnpm/store").tmp(); let expected = PathBuf::from("/home/user/.local/share/pnpm/store/v3/tmp"); assert_eq!(&received, &expected); } From 4e40463ce142cb998c06b916da5814b37fedd9fd Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 21:07:18 +0700 Subject: [PATCH 021/102] refactor: combine `#[cfg(unix)]` --- crates/store-dir/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 570508610..831f84e64 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -79,6 +79,7 @@ impl StoreDir { } } +#[cfg(unix)] // avoid test failure on windows due to forward/backward slash difference #[cfg(test)] mod tests { use super::*; @@ -86,7 +87,6 @@ mod tests { use pretty_assertions::assert_eq; use ssri::{Algorithm, IntegrityOpts}; - #[cfg(unix)] #[test] fn file_path_by_hash_str() { let received = "/home/user/.local/share/pnpm/store" @@ -98,7 +98,6 @@ mod tests { assert_eq!(&received, &expected); } - #[cfg(unix)] #[test] fn file_path_by_content_address() { fn case(file_content: &str, suffix: Option, expected: &str) { @@ -131,7 +130,6 @@ mod tests { ); } - #[cfg(unix)] #[test] fn tmp() { let received = StoreDir::new("/home/user/.local/share/pnpm/store").tmp(); From deb1681c38f516cb2b3c1ec3fc177901c300d3bc Mon Sep 17 00:00:00 2001 From: khai96_ Date: Mon, 30 Oct 2023 23:56:39 +0700 Subject: [PATCH 022/102] refactor: use sha2 directly --- Cargo.lock | 8 ++++---- Cargo.toml | 1 + crates/cafs/Cargo.toml | 2 +- crates/cafs/src/lib.rs | 9 ++++++--- crates/store-dir/Cargo.toml | 2 +- crates/store-dir/src/lib.rs | 21 ++++++++++++--------- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39a3be07c..1f905a4cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1305,7 +1305,7 @@ dependencies = [ "miette", "pacquet-store-dir", "pretty_assertions", - "ssri", + "sha2", "tempfile", ] @@ -1493,7 +1493,7 @@ dependencies = [ "pipe-trait", "pretty_assertions", "serde", - "ssri", + "sha2", "strum", ] @@ -2003,9 +2003,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", diff --git a/Cargo.toml b/Cargo.toml index 9a9761ac6..af23bcd11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ serde = { version = "1.0.188", features = ["derive"] } serde_ini = { version = "0.2.0" } serde_json = { version = "1.0.107", features = ["preserve_order"] } serde_yaml = { version = "0.9.1" } +sha2 = { version = "0.10.8" } split-first-char = { version = "0.0.0" } ssri = { version = "9.0.0" } strum = { version = "0.25.0", features = ["derive"] } diff --git a/crates/cafs/Cargo.toml b/crates/cafs/Cargo.toml index ad67d6b90..87fa92e74 100644 --- a/crates/cafs/Cargo.toml +++ b/crates/cafs/Cargo.toml @@ -15,7 +15,7 @@ pacquet-store-dir = { workspace = true } derive_more = { workspace = true } miette = { workspace = true } -ssri = { workspace = true } +sha2 = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 31e7226e9..aaee68527 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -1,7 +1,7 @@ use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::StoreDir; -use ssri::{Algorithm, IntegrityOpts}; +use sha2::{Digest, Sha512}; use std::{fs, path::PathBuf}; #[derive(Debug, Display, Error, From, Diagnostic)] @@ -12,8 +12,11 @@ pub enum CafsError { } pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result { - let integrity = IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(buffer).result(); - let file_path = store_dir.file_path_by_content_address(&integrity, None); + let mut hasher = Sha512::new(); + hasher.update(buffer); + let file_hash = hasher.finalize(); + + let file_path = store_dir.file_path_by_content_address(file_hash, None); if !file_path.exists() { let parent_dir = file_path.parent().unwrap(); diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index b549404bd..dfad2f929 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -13,7 +13,7 @@ repository.workspace = true [dependencies] derive_more = { workspace = true } serde = { workspace = true } -ssri = { workspace = true } +sha2 = { workspace = true } strum = { workspace = true } [dev-dependencies] diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 831f84e64..fa02456a9 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,9 +1,12 @@ use derive_more::From; use serde::{Deserialize, Serialize}; -use ssri::{Algorithm, Integrity}; // TODO: use proper sha2::Sha512 to remove assert_eq +use sha2::{digest, Sha512}; use std::path::{self, PathBuf}; use strum::IntoStaticStr; +/// Content hash of a file. +pub type FileHash = digest::Output; + /// Optional suffix of a content address of a file. #[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr)] pub enum FileSuffix { @@ -61,11 +64,10 @@ impl StoreDir { /// Path to a file in the store directory. pub fn file_path_by_content_address( &self, - integrity: &Integrity, // TODO: use proper sha2::Sha512 to remove assert_eq + hash: FileHash, suffix: Option, ) -> PathBuf { - let (algorithm, hex) = integrity.to_hex(); - assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 algorithm is supported"); // TODO: use proper sha2::Sha512 to remove assert_eq + let hex = format!("{hash:x}"); let head = &hex[..2]; let middle = &hex[2..]; let suffix = suffix.map_or("", <&str>::from); @@ -85,7 +87,7 @@ mod tests { use super::*; use pipe_trait::Pipe; use pretty_assertions::assert_eq; - use ssri::{Algorithm, IntegrityOpts}; + use sha2::{Digest, Sha512}; #[test] fn file_path_by_hash_str() { @@ -103,10 +105,11 @@ mod tests { fn case(file_content: &str, suffix: Option, expected: &str) { eprintln!("CASE: {file_content:?}, {suffix:?}"); let store_dir = StoreDir::new("STORE_DIR"); - let integrity = - IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(file_content).result(); - dbg!(&integrity); - let received = store_dir.file_path_by_content_address(&integrity, suffix); + let mut hasher = Sha512::new(); + hasher.update(file_content); + let file_hash = hasher.finalize(); + eprintln!("file_hash = {file_hash:x}"); + let received = store_dir.file_path_by_content_address(file_hash, suffix); let expected = PathBuf::from(expected); assert_eq!(&received, &expected); } From c76724a76699052936cf37bbba1df1a114bb06e7 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 00:09:14 +0700 Subject: [PATCH 023/102] refactor: use `new_with_prefix` --- crates/cafs/src/lib.rs | 4 +--- crates/store-dir/src/lib.rs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index aaee68527..7c73a730f 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -12,9 +12,7 @@ pub enum CafsError { } pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result { - let mut hasher = Sha512::new(); - hasher.update(buffer); - let file_hash = hasher.finalize(); + let file_hash = Sha512::new_with_prefix(buffer).finalize(); let file_path = store_dir.file_path_by_content_address(file_hash, None); diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index fa02456a9..6fac7f9fe 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -105,9 +105,7 @@ mod tests { fn case(file_content: &str, suffix: Option, expected: &str) { eprintln!("CASE: {file_content:?}, {suffix:?}"); let store_dir = StoreDir::new("STORE_DIR"); - let mut hasher = Sha512::new(); - hasher.update(file_content); - let file_hash = hasher.finalize(); + let file_hash = Sha512::new_with_prefix(file_content).finalize(); eprintln!("file_hash = {file_hash:x}"); let received = store_dir.file_path_by_content_address(file_hash, suffix); let expected = PathBuf::from(expected); From 1ee706340a138935903dee73924e879197ac52aa Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 00:28:54 +0700 Subject: [PATCH 024/102] refactor: use `digest` --- crates/cafs/src/lib.rs | 2 +- crates/store-dir/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 7c73a730f..e5381b353 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -12,7 +12,7 @@ pub enum CafsError { } pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result { - let file_hash = Sha512::new_with_prefix(buffer).finalize(); + let file_hash = Sha512::digest(buffer); let file_path = store_dir.file_path_by_content_address(file_hash, None); diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 6fac7f9fe..4d9895e30 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -105,7 +105,7 @@ mod tests { fn case(file_content: &str, suffix: Option, expected: &str) { eprintln!("CASE: {file_content:?}, {suffix:?}"); let store_dir = StoreDir::new("STORE_DIR"); - let file_hash = Sha512::new_with_prefix(file_content).finalize(); + let file_hash = Sha512::digest(file_content); eprintln!("file_hash = {file_hash:x}"); let received = store_dir.file_path_by_content_address(file_hash, suffix); let expected = PathBuf::from(expected); From 5610e59322b76e9847009b41b9ff737be6fee70d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 02:49:45 +0700 Subject: [PATCH 025/102] test: add back windows --- crates/npmrc/src/custom_deserializer.rs | 14 ++++++++------ crates/npmrc/src/lib.rs | 10 ++++++---- crates/store-dir/src/lib.rs | 3 +-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/npmrc/src/custom_deserializer.rs b/crates/npmrc/src/custom_deserializer.rs index e8734404e..c7ea76ded 100644 --- a/crates/npmrc/src/custom_deserializer.rs +++ b/crates/npmrc/src/custom_deserializer.rs @@ -154,21 +154,23 @@ mod tests { use pretty_assertions::assert_eq; use std::env; - #[cfg(unix)] + fn display_store_dir(store_dir: &StoreDir) -> String { + store_dir.display().to_string().replace('\\', "/") + } + #[test] fn test_default_store_dir_with_pnpm_home_env() { env::set_var("PNPM_HOME", "/tmp/pnpm-home"); // TODO: change this to dependency injection - let store_dir = default_store_dir().display().to_string(); - assert_eq!(store_dir, "/tmp/pnpm-home/store"); + let store_dir = default_store_dir(); + assert_eq!(display_store_dir(&store_dir), "/tmp/pnpm-home/store"); env::remove_var("PNPM_HOME"); } - #[cfg(unix)] #[test] fn test_default_store_dir_with_xdg_env() { env::set_var("XDG_DATA_HOME", "/tmp/xdg_data_home"); // TODO: change this to dependency injection - let store_dir = default_store_dir().display().to_string(); - assert_eq!(store_dir, "/tmp/xdg_data_home/pnpm/store"); + let store_dir = default_store_dir(); + assert_eq!(display_store_dir(&store_dir), "/tmp/xdg_data_home/pnpm/store"); env::remove_var("XDG_DATA_HOME"); } diff --git a/crates/npmrc/src/lib.rs b/crates/npmrc/src/lib.rs index fe223e358..ef56ba4ca 100644 --- a/crates/npmrc/src/lib.rs +++ b/crates/npmrc/src/lib.rs @@ -211,6 +211,10 @@ mod tests { use super::*; + fn display_store_dir(store_dir: &StoreDir) -> String { + store_dir.display().to_string().replace('\\', "/") + } + #[test] pub fn have_default_values() { let value = Npmrc::new(); @@ -247,21 +251,19 @@ mod tests { assert_eq!(value.modules_cache_max_age, 1000); } - #[cfg(unix)] #[test] pub fn should_use_pnpm_home_env_var() { env::set_var("PNPM_HOME", "/hello"); // TODO: change this to dependency injection let value: Npmrc = serde_ini::from_str("").unwrap(); - assert_eq!(value.store_dir.display().to_string(), "/hello/store"); + assert_eq!(display_store_dir(&value.store_dir), "/hello/store"); env::remove_var("PNPM_HOME"); } - #[cfg(unix)] #[test] pub fn should_use_xdg_data_home_env_var() { env::set_var("XDG_DATA_HOME", "/hello"); let value: Npmrc = serde_ini::from_str("").unwrap(); - assert_eq!(value.store_dir.display().to_string(), "/hello/pnpm/store"); + assert_eq!(display_store_dir(&value.store_dir), "/hello/pnpm/store"); env::remove_var("XDG_DATA_HOME"); } diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 4d9895e30..d158b619c 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -81,7 +81,6 @@ impl StoreDir { } } -#[cfg(unix)] // avoid test failure on windows due to forward/backward slash difference #[cfg(test)] mod tests { use super::*; @@ -108,7 +107,7 @@ mod tests { let file_hash = Sha512::digest(file_content); eprintln!("file_hash = {file_hash:x}"); let received = store_dir.file_path_by_content_address(file_hash, suffix); - let expected = PathBuf::from(expected); + let expected: PathBuf = expected.split('/').collect(); assert_eq!(&received, &expected); } From f1f431606e726b97e9c390e320bbac9d924681ee Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 02:50:27 +0700 Subject: [PATCH 026/102] docs: add missing TODO --- crates/npmrc/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/npmrc/src/lib.rs b/crates/npmrc/src/lib.rs index ef56ba4ca..d82e1dae8 100644 --- a/crates/npmrc/src/lib.rs +++ b/crates/npmrc/src/lib.rs @@ -261,7 +261,7 @@ mod tests { #[test] pub fn should_use_xdg_data_home_env_var() { - env::set_var("XDG_DATA_HOME", "/hello"); + env::set_var("XDG_DATA_HOME", "/hello"); // TODO: change this to dependency injection let value: Npmrc = serde_ini::from_str("").unwrap(); assert_eq!(display_store_dir(&value.store_dir), "/hello/pnpm/store"); env::remove_var("XDG_DATA_HOME"); From b01abcd423c25e0ff279f9f0071b9d5290f37396 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 03:09:15 +0700 Subject: [PATCH 027/102] refactor: pass suffix to write_sync --- crates/cafs/src/lib.rs | 10 +++++++--- crates/tarball/src/lib.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index e5381b353..f3898dccd 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -1,6 +1,6 @@ use derive_more::{Display, Error, From}; use miette::Diagnostic; -use pacquet_store_dir::StoreDir; +use pacquet_store_dir::{FileSuffix, StoreDir}; use sha2::{Digest, Sha512}; use std::{fs, path::PathBuf}; @@ -11,10 +11,14 @@ pub enum CafsError { Io(std::io::Error), // TODO: remove derive(From), split this variant } -pub fn write_sync(store_dir: &StoreDir, buffer: &[u8]) -> Result { +pub fn write_sync( + store_dir: &StoreDir, + buffer: &[u8], + suffix: Option, +) -> Result { let file_hash = Sha512::digest(buffer); - let file_path = store_dir.file_path_by_content_address(file_hash, None); + let file_path = store_dir.file_path_by_content_address(file_hash, suffix); if !file_path.exists() { let parent_dir = file_path.parent().unwrap(); diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 9c5014692..5b0cc04a9 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -218,7 +218,7 @@ impl<'a> DownloadTarballToStore<'a> { let entry_path = entry.path().unwrap(); let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); - let file_path = pacquet_cafs::write_sync(store_dir, &buffer) + let file_path = pacquet_cafs::write_sync(store_dir, &buffer, None) .map_err(TarballError::WriteCafs)?; Ok((cleaned_entry_path, file_path)) From 4dfa84e8c56d1b7868b9a328b6017ce1f8ce11ed Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 03:10:08 +0700 Subject: [PATCH 028/102] style: remove an empty line --- crates/cafs/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index f3898dccd..7ea613c5d 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -17,7 +17,6 @@ pub fn write_sync( suffix: Option, ) -> Result { let file_hash = Sha512::digest(buffer); - let file_path = store_dir.file_path_by_content_address(file_hash, suffix); if !file_path.exists() { From 31148d3e802b3ef426b7d12e8e1efdeb67b9f0fc Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 03:36:34 +0700 Subject: [PATCH 029/102] feat: executable bits --- Cargo.lock | 5 +++-- Cargo.toml | 1 + crates/tarball/Cargo.toml | 1 + crates/tarball/src/lib.rs | 9 +++++++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f905a4cd..9ffe81ad3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,9 +992,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "linked-hash-map" @@ -1503,6 +1503,7 @@ version = "0.0.1" dependencies = [ "dashmap", "derive_more", + "libc", "miette", "pacquet-cafs", "pacquet-diagnostics", diff --git a/Cargo.toml b/Cargo.toml index af23bcd11..7b87696a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ miette = { version = "5.9.0", features = ["fancy"] } os_display = { version = "0.1.3" } reflink-copy = { version = "0.1.9" } junction = { version = "1.0.0" } +libc = { version = "0.2.149" } reqwest = { version = "0.11", default-features = false, features = ["json", "native-tls-vendored"] } node-semver = { version = "2.1.0" } pipe-trait = { version = "0.4.0" } diff --git a/crates/tarball/Cargo.toml b/crates/tarball/Cargo.toml index 6e80d3165..98cac10f0 100644 --- a/crates/tarball/Cargo.toml +++ b/crates/tarball/Cargo.toml @@ -17,6 +17,7 @@ pacquet-store-dir = { workspace = true } dashmap = { workspace = true } derive_more = { workspace = true } +libc = { workspace = true } miette = { workspace = true } pipe-trait = { workspace = true } reqwest = { workspace = true } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 5b0cc04a9..ffe9b4438 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -8,8 +8,9 @@ use std::{ use dashmap::DashMap; use derive_more::{Display, Error, From}; +use libc::S_IXUSR; use miette::Diagnostic; -use pacquet_store_dir::StoreDir; +use pacquet_store_dir::{FileSuffix, StoreDir}; use pipe_trait::Pipe; use reqwest::Client; use ssri::{Integrity, IntegrityChecker}; @@ -211,6 +212,10 @@ impl<'a> DownloadTarballToStore<'a> { .map(|entry| -> Result<(OsString, PathBuf), TarballError> { let mut entry = entry.unwrap(); + let mode = entry.header().mode().expect("get mode"); // TODO: properly propagate this error + let is_executable = mode & S_IXUSR != 0; + let file_suffix = is_executable.then_some(FileSuffix::Exec); + // Read the contents of the entry let mut buffer = Vec::with_capacity(entry.size() as usize); entry.read_to_end(&mut buffer).unwrap(); @@ -218,7 +223,7 @@ impl<'a> DownloadTarballToStore<'a> { let entry_path = entry.path().unwrap(); let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); - let file_path = pacquet_cafs::write_sync(store_dir, &buffer, None) + let file_path = pacquet_cafs::write_sync(store_dir, &buffer, file_suffix) .map_err(TarballError::WriteCafs)?; Ok((cleaned_entry_path, file_path)) From 7b5d7bf87bbce6fd859dabb817bc4c95303042f7 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 03:42:21 +0700 Subject: [PATCH 030/102] refactor: use early return --- crates/cafs/src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 7ea613c5d..02266148f 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -19,12 +19,14 @@ pub fn write_sync( let file_hash = Sha512::digest(buffer); let file_path = store_dir.file_path_by_content_address(file_hash, suffix); - if !file_path.exists() { - let parent_dir = file_path.parent().unwrap(); - fs::create_dir_all(parent_dir)?; - fs::write(&file_path, buffer)?; + if file_path.exists() { + return Ok(file_path); } + let parent_dir = file_path.parent().unwrap(); + fs::create_dir_all(parent_dir)?; + fs::write(&file_path, buffer)?; + Ok(file_path) } From dd430c6de6f601e01bbe69b2f36bccc1bf3b4fff Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 03:44:58 +0700 Subject: [PATCH 031/102] feat: set executables as executable --- crates/cafs/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 02266148f..3ef247faf 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -27,6 +27,15 @@ pub fn write_sync( fs::create_dir_all(parent_dir)?; fs::write(&file_path, buffer)?; + #[cfg(unix)] + { + use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + if suffix == Some(FileSuffix::Exec) { + let permissions = Permissions::from_mode(0o777); + fs::set_permissions(&file_path, permissions).expect("make the file executable"); + } + } + Ok(file_path) } From 8d9eb49bf2670478c97fd9f4f21dec3063f8acde Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 03:55:46 +0700 Subject: [PATCH 032/102] fix: macos and windows --- Cargo.lock | 1 - Cargo.toml | 1 - crates/tarball/Cargo.toml | 1 - crates/tarball/src/lib.rs | 8 ++++++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ffe81ad3..a3303c291 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1503,7 +1503,6 @@ version = "0.0.1" dependencies = [ "dashmap", "derive_more", - "libc", "miette", "pacquet-cafs", "pacquet-diagnostics", diff --git a/Cargo.toml b/Cargo.toml index 7b87696a8..af23bcd11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,6 @@ miette = { version = "5.9.0", features = ["fancy"] } os_display = { version = "0.1.3" } reflink-copy = { version = "0.1.9" } junction = { version = "1.0.0" } -libc = { version = "0.2.149" } reqwest = { version = "0.11", default-features = false, features = ["json", "native-tls-vendored"] } node-semver = { version = "2.1.0" } pipe-trait = { version = "0.4.0" } diff --git a/crates/tarball/Cargo.toml b/crates/tarball/Cargo.toml index 98cac10f0..6e80d3165 100644 --- a/crates/tarball/Cargo.toml +++ b/crates/tarball/Cargo.toml @@ -17,7 +17,6 @@ pacquet-store-dir = { workspace = true } dashmap = { workspace = true } derive_more = { workspace = true } -libc = { workspace = true } miette = { workspace = true } pipe-trait = { workspace = true } reqwest = { workspace = true } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index ffe9b4438..ab3d5007b 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -8,7 +8,6 @@ use std::{ use dashmap::DashMap; use derive_more::{Display, Error, From}; -use libc::S_IXUSR; use miette::Diagnostic; use pacquet_store_dir::{FileSuffix, StoreDir}; use pipe_trait::Pipe; @@ -19,6 +18,11 @@ use tokio::sync::{Notify, RwLock}; use tracing::instrument; use zune_inflate::{errors::InflateDecodeErrors, DeflateDecoder, DeflateOptions}; +/// Mask of the executable bits. +/// +/// This value is equal to `S_IXUSR` in libc. +const EXEC_MASK: u32 = 64; + #[derive(Debug, Display, Error, Diagnostic)] #[display("Failed to fetch {url}: {error}")] pub struct NetworkError { @@ -213,7 +217,7 @@ impl<'a> DownloadTarballToStore<'a> { let mut entry = entry.unwrap(); let mode = entry.header().mode().expect("get mode"); // TODO: properly propagate this error - let is_executable = mode & S_IXUSR != 0; + let is_executable = mode & EXEC_MASK != 0; let file_suffix = is_executable.then_some(FileSuffix::Exec); // Read the contents of the entry From 3b6e68a48b0711a2666bbe4964b9fd485a9842d3 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 15:21:00 +0700 Subject: [PATCH 033/102] feat: index files --- Cargo.lock | 7 ++- Cargo.toml | 1 + crates/cafs/src/lib.rs | 8 +-- crates/tarball/Cargo.toml | 3 ++ crates/tarball/src/lib.rs | 100 ++++++++++++++++++++++++++++---------- 5 files changed, 88 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3303c291..e95821552 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bitflags" @@ -1501,6 +1501,7 @@ dependencies = [ name = "pacquet-tarball" version = "0.0.1" dependencies = [ + "base64", "dashmap", "derive_more", "miette", @@ -1510,6 +1511,8 @@ dependencies = [ "pipe-trait", "pretty_assertions", "reqwest", + "serde", + "serde_json", "ssri", "tar", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index af23bcd11..068b515c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ pacquet-store-dir = { path = "crates/store-dir" } async-recursion = { version = "1.0.5" } clap = { version = "4", features = ["derive", "string"] } command-extra = { version = "1.0.0" } +base64 = { version = "0.21.5" } dashmap = { version = "5.5.3" } derive_more = { version = "1.0.0-beta.3", features = ["full"] } dunce = { version = "1.0.4" } diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 3ef247faf..23be0dbc3 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -1,6 +1,6 @@ use derive_more::{Display, Error, From}; use miette::Diagnostic; -use pacquet_store_dir::{FileSuffix, StoreDir}; +use pacquet_store_dir::{FileHash, FileSuffix, StoreDir}; use sha2::{Digest, Sha512}; use std::{fs, path::PathBuf}; @@ -15,12 +15,12 @@ pub fn write_sync( store_dir: &StoreDir, buffer: &[u8], suffix: Option, -) -> Result { +) -> Result<(PathBuf, FileHash), CafsError> { let file_hash = Sha512::digest(buffer); let file_path = store_dir.file_path_by_content_address(file_hash, suffix); if file_path.exists() { - return Ok(file_path); + return Ok((file_path, file_hash)); } let parent_dir = file_path.parent().unwrap(); @@ -36,7 +36,7 @@ pub fn write_sync( } } - Ok(file_path) + Ok((file_path, file_hash)) } pub fn prune_sync(_store_dir: &StoreDir) -> Result<(), CafsError> { diff --git a/crates/tarball/Cargo.toml b/crates/tarball/Cargo.toml index 6e80d3165..f60b0e460 100644 --- a/crates/tarball/Cargo.toml +++ b/crates/tarball/Cargo.toml @@ -15,11 +15,14 @@ pacquet-cafs = { workspace = true } pacquet-diagnostics = { workspace = true } pacquet-store-dir = { workspace = true } +base64 = { workspace = true } dashmap = { workspace = true } derive_more = { workspace = true } miette = { workspace = true } pipe-trait = { workspace = true } reqwest = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } ssri = { workspace = true } tar = { workspace = true } tokio = { workspace = true } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index ab3d5007b..5618dcc2c 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -6,12 +6,14 @@ use std::{ sync::Arc, }; +use base64::{engine::general_purpose::STANDARD as BASE64_STD, Engine}; use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::{FileSuffix, StoreDir}; use pipe_trait::Pipe; use reqwest::Client; +use serde::{Deserialize, Serialize}; use ssri::{Integrity, IntegrityChecker}; use tar::Archive; use tokio::sync::{Notify, RwLock}; @@ -206,34 +208,65 @@ impl<'a> DownloadTarballToStore<'a> { } let cas_paths = tokio::task::spawn(async move { verify_checksum(&response, package_integrity).map_err(TaskError::Checksum)?; - let data = - decompress_gzip(&response, package_unpacked_size).map_err(TaskError::Other)?; - Archive::new(Cursor::new(data)) + + let mut archive = decompress_gzip(&response, package_unpacked_size) + .map_err(TaskError::Other)? + .pipe(Cursor::new) + .pipe(Archive::new); + + let entries = archive .entries() .map_err(TarballError::ReadTarballEntries) .map_err(TaskError::Other)? - .filter(|entry| !entry.as_ref().unwrap().header().entry_type().is_dir()) - .map(|entry| -> Result<(OsString, PathBuf), TarballError> { - let mut entry = entry.unwrap(); - - let mode = entry.header().mode().expect("get mode"); // TODO: properly propagate this error - let is_executable = mode & EXEC_MASK != 0; - let file_suffix = is_executable.then_some(FileSuffix::Exec); - - // Read the contents of the entry - let mut buffer = Vec::with_capacity(entry.size() as usize); - entry.read_to_end(&mut buffer).unwrap(); - - let entry_path = entry.path().unwrap(); - let cleaned_entry_path = - entry_path.components().skip(1).collect::().into_os_string(); - let file_path = pacquet_cafs::write_sync(store_dir, &buffer, file_suffix) - .map_err(TarballError::WriteCafs)?; - - Ok((cleaned_entry_path, file_path)) - }) - .collect::, TarballError>>() - .map_err(TaskError::Other) + .filter(|entry| !entry.as_ref().unwrap().header().entry_type().is_dir()); + + let ((_, Some(capacity)) | (capacity, None)) = entries.size_hint(); + let mut cas_paths = HashMap::::with_capacity(capacity); + let mut tarball_index = TarballIndex { files: HashMap::with_capacity(capacity) }; + + for entry in entries { + let mut entry = entry.unwrap(); + + let file_mode = entry.header().mode().expect("get mode"); // TODO: properly propagate this error + let is_executable = file_mode & EXEC_MASK != 0; + let file_suffix = is_executable.then_some(FileSuffix::Exec); + + // Read the contents of the entry + let mut buffer = Vec::with_capacity(entry.size() as usize); + entry.read_to_end(&mut buffer).unwrap(); + + let entry_path = entry.path().unwrap(); + let cleaned_entry_path = + entry_path.components().skip(1).collect::().into_os_string(); + let (file_path, file_hash) = + pacquet_cafs::write_sync(store_dir, &buffer, file_suffix) + .map_err(TarballError::WriteCafs) + .map_err(TaskError::Other)?; + + if let Some(previous) = cas_paths.insert(cleaned_entry_path.clone(), file_path) { + panic!( + "Unexpected error: {previous:?} already exists at {cleaned_entry_path:?}" + ); + } + + let file_size = entry.header().size().ok(); + let file_integrity = format!("sha512-{}", BASE64_STD.encode(file_hash)); + let file_attrs = TarballIndexFileAttrs { + integrity: file_integrity, + mode: file_mode, + size: file_size, + }; + + tarball_index.files.insert(cleaned_entry_path, file_attrs); + } + + let tarball_index = + serde_json::to_string(&tarball_index).expect("convert a TarballIndex to JSON"); + pacquet_cafs::write_sync(store_dir, tarball_index.as_bytes(), Some(FileSuffix::Index)) + .map_err(TarballError::WriteCafs) + .map_err(TaskError::Other)?; + + Ok(cas_paths) }) .await .expect("no join error") @@ -251,6 +284,23 @@ impl<'a> DownloadTarballToStore<'a> { } } +/// Content of an index file (`$STORE_DIR/v3/files/*/*-index.json`). +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct TarballIndex { + pub files: HashMap, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +struct TarballIndexFileAttrs { + // pub checked_at: ??? + pub integrity: String, + pub mode: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, +} + #[cfg(test)] mod tests { use pipe_trait::Pipe; From 871c8b6348d7a59f2a250c04bdc2287b0bade343 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 15:33:44 +0700 Subject: [PATCH 034/102] fix: serde panics --- crates/tarball/src/lib.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 5618dcc2c..888f3c5fd 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -243,10 +243,13 @@ impl<'a> DownloadTarballToStore<'a> { .map_err(TarballError::WriteCafs) .map_err(TaskError::Other)?; - if let Some(previous) = cas_paths.insert(cleaned_entry_path.clone(), file_path) { - panic!( - "Unexpected error: {previous:?} already exists at {cleaned_entry_path:?}" - ); + let tarball_index_key = cleaned_entry_path + .to_str() + .expect("entry path must be valid UTF-8") // TODO: propagate this error, provide more information + .to_string(); // TODO: convert cleaned_entry_path to String too. + + if let Some(previous) = cas_paths.insert(cleaned_entry_path, file_path) { + panic!("Unexpected error: {previous:?} shouldn't collide"); } let file_size = entry.header().size().ok(); @@ -257,7 +260,9 @@ impl<'a> DownloadTarballToStore<'a> { size: file_size, }; - tarball_index.files.insert(cleaned_entry_path, file_attrs); + if let Some(previous) = tarball_index.files.insert(tarball_index_key, file_attrs) { + panic!("Unexpected error: {previous:?} shouldn't collide"); + } } let tarball_index = @@ -288,7 +293,7 @@ impl<'a> DownloadTarballToStore<'a> { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] struct TarballIndex { - pub files: HashMap, + pub files: HashMap, } #[derive(Debug, Deserialize, Serialize)] From 09c422228664afc207cf544754c83f2bdd15f424 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 15:34:02 +0700 Subject: [PATCH 035/102] docs: existing bugs --- crates/tarball/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 888f3c5fd..b2b202935 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -267,6 +267,8 @@ impl<'a> DownloadTarballToStore<'a> { let tarball_index = serde_json::to_string(&tarball_index).expect("convert a TarballIndex to JSON"); + + // TODO: fix bug here pacquet_cafs::write_sync(store_dir, tarball_index.as_bytes(), Some(FileSuffix::Index)) .map_err(TarballError::WriteCafs) .map_err(TaskError::Other)?; From 9ea30e26cbb95021320dd5f584ab1b4bc200aef1 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 15:51:19 +0700 Subject: [PATCH 036/102] refactor: rename a function --- crates/store-dir/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index d158b619c..5e37b1c4f 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -57,7 +57,7 @@ impl StoreDir { /// **Parameters:** /// * `head` is the first 2 hexadecimal digit of the file address. /// * `tail` is the rest of the address and an optional suffix. - fn file_path_by_hash_str(&self, head: &str, tail: &str) -> PathBuf { + fn file_path_by_head_tail(&self, head: &str, tail: &str) -> PathBuf { self.files().join(head).join(tail) } @@ -72,7 +72,7 @@ impl StoreDir { let middle = &hex[2..]; let suffix = suffix.map_or("", <&str>::from); let tail = format!("{middle}{suffix}"); - self.file_path_by_hash_str(head, &tail) + self.file_path_by_head_tail(head, &tail) } /// Path to the temporary directory inside the store. @@ -89,10 +89,10 @@ mod tests { use sha2::{Digest, Sha512}; #[test] - fn file_path_by_hash_str() { + fn file_path_by_head_tail() { let received = "/home/user/.local/share/pnpm/store" .pipe(StoreDir::new) - .file_path_by_hash_str("3e", "f722d37b016c63ac0126cfdcec"); + .file_path_by_head_tail("3e", "f722d37b016c63ac0126cfdcec"); let expected = PathBuf::from( "/home/user/.local/share/pnpm/store/v3/files/3e/f722d37b016c63ac0126cfdcec", ); From 4fee0a3ded0758792cd1244fe6a988071c80a15d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 15:53:26 +0700 Subject: [PATCH 037/102] refactor: StoreDir::file_path_by_hex_str --- crates/store-dir/src/lib.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 5e37b1c4f..fbe436269 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -61,6 +61,15 @@ impl StoreDir { self.files().join(head).join(tail) } + /// Path to a file in the store directory. + pub fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { + let head = &hex[..2]; + let middle = &hex[2..]; + let suffix = suffix.map_or("", <&str>::from); + let tail = format!("{middle}{suffix}"); + self.file_path_by_head_tail(head, &tail) + } + /// Path to a file in the store directory. pub fn file_path_by_content_address( &self, @@ -68,11 +77,7 @@ impl StoreDir { suffix: Option, ) -> PathBuf { let hex = format!("{hash:x}"); - let head = &hex[..2]; - let middle = &hex[2..]; - let suffix = suffix.map_or("", <&str>::from); - let tail = format!("{middle}{suffix}"); - self.file_path_by_head_tail(head, &tail) + self.file_path_by_hex_str(&hex, suffix) } /// Path to the temporary directory inside the store. From 8f98e730c1c35e2371b1d5ebd7f9e3f86b99104e Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 16:21:48 +0700 Subject: [PATCH 038/102] fix: content address of the index files --- Cargo.lock | 2 ++ crates/cafs/Cargo.toml | 1 + crates/cafs/src/lib.rs | 36 +++++++++++++++++++++++++++--------- crates/store-dir/Cargo.toml | 1 + crates/store-dir/src/lib.rs | 10 +++++++++- crates/tarball/src/lib.rs | 7 +++---- 6 files changed, 43 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e95821552..fb4f175b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1306,6 +1306,7 @@ dependencies = [ "pacquet-store-dir", "pretty_assertions", "sha2", + "ssri", "tempfile", ] @@ -1494,6 +1495,7 @@ dependencies = [ "pretty_assertions", "serde", "sha2", + "ssri", "strum", ] diff --git a/crates/cafs/Cargo.toml b/crates/cafs/Cargo.toml index 87fa92e74..8018a2a49 100644 --- a/crates/cafs/Cargo.toml +++ b/crates/cafs/Cargo.toml @@ -16,6 +16,7 @@ pacquet-store-dir = { workspace = true } derive_more = { workspace = true } miette = { workspace = true } sha2 = { workspace = true } +ssri = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs index 23be0dbc3..ff48a6bcf 100644 --- a/crates/cafs/src/lib.rs +++ b/crates/cafs/src/lib.rs @@ -2,7 +2,11 @@ use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::{FileHash, FileSuffix, StoreDir}; use sha2::{Digest, Sha512}; -use std::{fs, path::PathBuf}; +use ssri::Integrity; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; #[derive(Debug, Display, Error, From, Diagnostic)] #[non_exhaustive] @@ -11,7 +15,17 @@ pub enum CafsError { Io(std::io::Error), // TODO: remove derive(From), split this variant } -pub fn write_sync( +fn write_file_if_not_exist(file_path: &Path, content: &[u8]) -> io::Result<()> { + if file_path.exists() { + return Ok(()); + } + + let parent_dir = file_path.parent().unwrap(); + fs::create_dir_all(parent_dir)?; + fs::write(file_path, content) +} + +pub fn write_non_index_file( store_dir: &StoreDir, buffer: &[u8], suffix: Option, @@ -19,13 +33,7 @@ pub fn write_sync( let file_hash = Sha512::digest(buffer); let file_path = store_dir.file_path_by_content_address(file_hash, suffix); - if file_path.exists() { - return Ok((file_path, file_hash)); - } - - let parent_dir = file_path.parent().unwrap(); - fs::create_dir_all(parent_dir)?; - fs::write(&file_path, buffer)?; + write_file_if_not_exist(&file_path, buffer)?; #[cfg(unix)] { @@ -39,6 +47,16 @@ pub fn write_sync( Ok((file_path, file_hash)) } +pub fn write_tarball_index_file( + store_dir: &StoreDir, + tarball_integrity: &Integrity, + index_content: &str, +) -> Result<(), CafsError> { + let file_path = store_dir.tarball_index_file_path(tarball_integrity); + write_file_if_not_exist(&file_path, index_content.as_bytes())?; + Ok(()) +} + pub fn prune_sync(_store_dir: &StoreDir) -> Result<(), CafsError> { // Ref: https://pnpm.io/cli/store#prune todo!("remove orphaned files") diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index dfad2f929..38bd1cefb 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -14,6 +14,7 @@ repository.workspace = true derive_more = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } +ssri = { workspace = true } strum = { workspace = true } [dev-dependencies] diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index fbe436269..5487fdb67 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,6 +1,7 @@ use derive_more::From; use serde::{Deserialize, Serialize}; use sha2::{digest, Sha512}; +use ssri::{Algorithm, Integrity}; use std::path::{self, PathBuf}; use strum::IntoStaticStr; @@ -62,7 +63,7 @@ impl StoreDir { } /// Path to a file in the store directory. - pub fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { + fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { let head = &hex[..2]; let middle = &hex[2..]; let suffix = suffix.map_or("", <&str>::from); @@ -80,6 +81,13 @@ impl StoreDir { self.file_path_by_hex_str(&hex, suffix) } + /// Path to an index file of a tarball. + pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { + let (algorithm, hex) = tarball_integrity.to_hex(); + assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error + self.file_path_by_hex_str(&hex, Some(FileSuffix::Index)) + } + /// Path to the temporary directory inside the store. pub fn tmp(&self) -> PathBuf { self.v3().join("tmp") diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index b2b202935..26c60278d 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -207,7 +207,7 @@ impl<'a> DownloadTarballToStore<'a> { Other(TarballError), } let cas_paths = tokio::task::spawn(async move { - verify_checksum(&response, package_integrity).map_err(TaskError::Checksum)?; + verify_checksum(&response, package_integrity.clone()).map_err(TaskError::Checksum)?; let mut archive = decompress_gzip(&response, package_unpacked_size) .map_err(TaskError::Other)? @@ -239,7 +239,7 @@ impl<'a> DownloadTarballToStore<'a> { let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); let (file_path, file_hash) = - pacquet_cafs::write_sync(store_dir, &buffer, file_suffix) + pacquet_cafs::write_non_index_file(store_dir, &buffer, file_suffix) .map_err(TarballError::WriteCafs) .map_err(TaskError::Other)?; @@ -268,8 +268,7 @@ impl<'a> DownloadTarballToStore<'a> { let tarball_index = serde_json::to_string(&tarball_index).expect("convert a TarballIndex to JSON"); - // TODO: fix bug here - pacquet_cafs::write_sync(store_dir, tarball_index.as_bytes(), Some(FileSuffix::Index)) + pacquet_cafs::write_tarball_index_file(store_dir, &package_integrity, &tarball_index) .map_err(TarballError::WriteCafs) .map_err(TaskError::Other)?; From 209e197dcc0b9a626eefb68cdb9fc75b649f225d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 19:17:22 +0700 Subject: [PATCH 039/102] refactor: derive From for TaskError --- crates/tarball/src/lib.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 26c60278d..42d0aea15 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -202,6 +202,7 @@ impl<'a> DownloadTarballToStore<'a> { integrity: package_integrity.to_string(), error, })?; + #[derive(Debug, From)] enum TaskError { Checksum(ssri::Error), Other(TarballError), @@ -240,8 +241,7 @@ impl<'a> DownloadTarballToStore<'a> { entry_path.components().skip(1).collect::().into_os_string(); let (file_path, file_hash) = pacquet_cafs::write_non_index_file(store_dir, &buffer, file_suffix) - .map_err(TarballError::WriteCafs) - .map_err(TaskError::Other)?; + .map_err(TarballError::WriteCafs)?; let tarball_index_key = cleaned_entry_path .to_str() @@ -269,8 +269,7 @@ impl<'a> DownloadTarballToStore<'a> { serde_json::to_string(&tarball_index).expect("convert a TarballIndex to JSON"); pacquet_cafs::write_tarball_index_file(store_dir, &package_integrity, &tarball_index) - .map_err(TarballError::WriteCafs) - .map_err(TaskError::Other)?; + .map_err(TarballError::WriteCafs)?; Ok(cas_paths) }) From 29e6901998397ab19781bba93661c2123829fa71 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 19:36:22 +0700 Subject: [PATCH 040/102] refactor: move StoreDir to its own mod --- crates/store-dir/src/lib.rs | 153 +----------------------------- crates/store-dir/src/store_dir.rs | 152 +++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 151 deletions(-) create mode 100644 crates/store-dir/src/store_dir.rs diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 5487fdb67..1aa6f6083 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,152 +1,3 @@ -use derive_more::From; -use serde::{Deserialize, Serialize}; -use sha2::{digest, Sha512}; -use ssri::{Algorithm, Integrity}; -use std::path::{self, PathBuf}; -use strum::IntoStaticStr; +mod store_dir; -/// Content hash of a file. -pub type FileHash = digest::Output; - -/// Optional suffix of a content address of a file. -#[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr)] -pub enum FileSuffix { - #[strum(serialize = "-exec")] - Exec, - #[strum(serialize = "-index.json")] - Index, -} - -/// Represent a store directory. -/// -/// * The store directory stores all files that were acquired by installing packages with pacquet or pnpm. -/// * The files in `node_modules` directories are hardlinks or reflinks to the files in the store directory. -/// * The store directory can and often act as a global shared cache of all installation of different workspaces. -/// * The location of the store directory can be customized by `store-dir` field. -#[derive(Debug, PartialEq, Eq, From, Deserialize, Serialize)] -#[serde(transparent)] -pub struct StoreDir { - /// Path to the root of the store directory from which all sub-paths are derived. - /// - /// Consumer of this struct should interact with the sub-paths instead of this path. - root: PathBuf, -} - -impl StoreDir { - /// Construct an instance of [`StoreDir`]. - pub fn new(root: impl Into) -> Self { - root.into().into() - } - - /// Create an object that [displays](std::fmt::Display) the root of the store directory. - pub fn display(&self) -> path::Display { - self.root.display() - } - - /// Get `{store}/v3`. - fn v3(&self) -> PathBuf { - self.root.join("v3") - } - - /// The directory that contains all files from the once-installed packages. - fn files(&self) -> PathBuf { - self.v3().join("files") - } - - /// Path to a file in the store directory. - /// - /// **Parameters:** - /// * `head` is the first 2 hexadecimal digit of the file address. - /// * `tail` is the rest of the address and an optional suffix. - fn file_path_by_head_tail(&self, head: &str, tail: &str) -> PathBuf { - self.files().join(head).join(tail) - } - - /// Path to a file in the store directory. - fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { - let head = &hex[..2]; - let middle = &hex[2..]; - let suffix = suffix.map_or("", <&str>::from); - let tail = format!("{middle}{suffix}"); - self.file_path_by_head_tail(head, &tail) - } - - /// Path to a file in the store directory. - pub fn file_path_by_content_address( - &self, - hash: FileHash, - suffix: Option, - ) -> PathBuf { - let hex = format!("{hash:x}"); - self.file_path_by_hex_str(&hex, suffix) - } - - /// Path to an index file of a tarball. - pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { - let (algorithm, hex) = tarball_integrity.to_hex(); - assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error - self.file_path_by_hex_str(&hex, Some(FileSuffix::Index)) - } - - /// Path to the temporary directory inside the store. - pub fn tmp(&self) -> PathBuf { - self.v3().join("tmp") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pipe_trait::Pipe; - use pretty_assertions::assert_eq; - use sha2::{Digest, Sha512}; - - #[test] - fn file_path_by_head_tail() { - let received = "/home/user/.local/share/pnpm/store" - .pipe(StoreDir::new) - .file_path_by_head_tail("3e", "f722d37b016c63ac0126cfdcec"); - let expected = PathBuf::from( - "/home/user/.local/share/pnpm/store/v3/files/3e/f722d37b016c63ac0126cfdcec", - ); - assert_eq!(&received, &expected); - } - - #[test] - fn file_path_by_content_address() { - fn case(file_content: &str, suffix: Option, expected: &str) { - eprintln!("CASE: {file_content:?}, {suffix:?}"); - let store_dir = StoreDir::new("STORE_DIR"); - let file_hash = Sha512::digest(file_content); - eprintln!("file_hash = {file_hash:x}"); - let received = store_dir.file_path_by_content_address(file_hash, suffix); - let expected: PathBuf = expected.split('/').collect(); - assert_eq!(&received, &expected); - } - - case( - "hello world", - None, - "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", - ); - - case( - "hello world", - Some(FileSuffix::Exec), - "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", - ); - - case( - "hello world", - Some(FileSuffix::Index), - "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-index.json", - ); - } - - #[test] - fn tmp() { - let received = StoreDir::new("/home/user/.local/share/pnpm/store").tmp(); - let expected = PathBuf::from("/home/user/.local/share/pnpm/store/v3/tmp"); - assert_eq!(&received, &expected); - } -} +pub use store_dir::*; diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs new file mode 100644 index 000000000..67e6a3025 --- /dev/null +++ b/crates/store-dir/src/store_dir.rs @@ -0,0 +1,152 @@ +use derive_more::From; +use serde::{Deserialize, Serialize}; +use sha2::{digest, Sha512}; +use ssri::{Algorithm, Integrity}; +use std::path::{self, PathBuf}; +use strum::IntoStaticStr; + +/// Content hash of a file. +pub type FileHash = digest::Output; + +/// Optional suffix of a content address of a file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr)] +pub enum FileSuffix { + #[strum(serialize = "-exec")] + Exec, + #[strum(serialize = "-index.json")] + Index, +} + +/// Represent a store directory. +/// +/// * The store directory stores all files that were acquired by installing packages with pacquet or pnpm. +/// * The files in `node_modules` directories are hardlinks or reflinks to the files in the store directory. +/// * The store directory can and often act as a global shared cache of all installation of different workspaces. +/// * The location of the store directory can be customized by `store-dir` field. +#[derive(Debug, PartialEq, Eq, From, Deserialize, Serialize)] +#[serde(transparent)] +pub struct StoreDir { + /// Path to the root of the store directory from which all sub-paths are derived. + /// + /// Consumer of this struct should interact with the sub-paths instead of this path. + root: PathBuf, +} + +impl StoreDir { + /// Construct an instance of [`StoreDir`]. + pub fn new(root: impl Into) -> Self { + root.into().into() + } + + /// Create an object that [displays](std::fmt::Display) the root of the store directory. + pub fn display(&self) -> path::Display { + self.root.display() + } + + /// Get `{store}/v3`. + fn v3(&self) -> PathBuf { + self.root.join("v3") + } + + /// The directory that contains all files from the once-installed packages. + fn files(&self) -> PathBuf { + self.v3().join("files") + } + + /// Path to a file in the store directory. + /// + /// **Parameters:** + /// * `head` is the first 2 hexadecimal digit of the file address. + /// * `tail` is the rest of the address and an optional suffix. + fn file_path_by_head_tail(&self, head: &str, tail: &str) -> PathBuf { + self.files().join(head).join(tail) + } + + /// Path to a file in the store directory. + pub(crate) fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { + let head = &hex[..2]; + let middle = &hex[2..]; + let suffix = suffix.map_or("", <&str>::from); + let tail = format!("{middle}{suffix}"); + self.file_path_by_head_tail(head, &tail) + } + + /// Path to a file in the store directory. + pub fn file_path_by_content_address( + &self, + hash: FileHash, + suffix: Option, + ) -> PathBuf { + let hex = format!("{hash:x}"); + self.file_path_by_hex_str(&hex, suffix) + } + + /// Path to an index file of a tarball. + pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { + let (algorithm, hex) = tarball_integrity.to_hex(); + assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error + self.file_path_by_hex_str(&hex, Some(FileSuffix::Index)) + } + + /// Path to the temporary directory inside the store. + pub fn tmp(&self) -> PathBuf { + self.v3().join("tmp") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pipe_trait::Pipe; + use pretty_assertions::assert_eq; + use sha2::{Digest, Sha512}; + + #[test] + fn file_path_by_head_tail() { + let received = "/home/user/.local/share/pnpm/store" + .pipe(StoreDir::new) + .file_path_by_head_tail("3e", "f722d37b016c63ac0126cfdcec"); + let expected = PathBuf::from( + "/home/user/.local/share/pnpm/store/v3/files/3e/f722d37b016c63ac0126cfdcec", + ); + assert_eq!(&received, &expected); + } + + #[test] + fn file_path_by_content_address() { + fn case(file_content: &str, suffix: Option, expected: &str) { + eprintln!("CASE: {file_content:?}, {suffix:?}"); + let store_dir = StoreDir::new("STORE_DIR"); + let file_hash = Sha512::digest(file_content); + eprintln!("file_hash = {file_hash:x}"); + let received = store_dir.file_path_by_content_address(file_hash, suffix); + let expected: PathBuf = expected.split('/').collect(); + assert_eq!(&received, &expected); + } + + case( + "hello world", + None, + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", + ); + + case( + "hello world", + Some(FileSuffix::Exec), + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", + ); + + case( + "hello world", + Some(FileSuffix::Index), + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-index.json", + ); + } + + #[test] + fn tmp() { + let received = StoreDir::new("/home/user/.local/share/pnpm/store").tmp(); + let expected = PathBuf::from("/home/user/.local/share/pnpm/store/v3/tmp"); + assert_eq!(&received, &expected); + } +} From db673efdc933a27a10eef3ae46b2f3f5870fcc96 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 19:59:10 +0700 Subject: [PATCH 041/102] refactor: move cafs into store-dir --- Cargo.lock | 16 +------ Cargo.toml | 1 - crates/cafs/Cargo.toml | 23 ---------- crates/cafs/src/lib.rs | 63 --------------------------- crates/cli/Cargo.toml | 1 - crates/cli/src/cli_args/store.rs | 2 +- crates/store-dir/Cargo.toml | 1 + crates/store-dir/src/lib.rs | 4 ++ crates/store-dir/src/prune.rs | 14 ++++++ crates/store-dir/src/write_file.rs | 68 ++++++++++++++++++++++++++++++ crates/tarball/Cargo.toml | 1 - crates/tarball/src/lib.rs | 21 +++++---- 12 files changed, 102 insertions(+), 113 deletions(-) delete mode 100644 crates/cafs/Cargo.toml delete mode 100644 crates/cafs/src/lib.rs create mode 100644 crates/store-dir/src/prune.rs create mode 100644 crates/store-dir/src/write_file.rs diff --git a/Cargo.lock b/Cargo.lock index fb4f175b7..bbc4a4fa4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1297,19 +1297,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "pacquet-cafs" -version = "0.0.1" -dependencies = [ - "derive_more", - "miette", - "pacquet-store-dir", - "pretty_assertions", - "sha2", - "ssri", - "tempfile", -] - [[package]] name = "pacquet-cli" version = "0.0.1" @@ -1322,7 +1309,6 @@ dependencies = [ "home", "insta", "miette", - "pacquet-cafs", "pacquet-diagnostics", "pacquet-executor", "pacquet-fs", @@ -1491,6 +1477,7 @@ name = "pacquet-store-dir" version = "0.0.1" dependencies = [ "derive_more", + "miette", "pipe-trait", "pretty_assertions", "serde", @@ -1507,7 +1494,6 @@ dependencies = [ "dashmap", "derive_more", "miette", - "pacquet-cafs", "pacquet-diagnostics", "pacquet-store-dir", "pipe-trait", diff --git a/Cargo.toml b/Cargo.toml index 068b515c4..2b2109f7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ pacquet-package-manager = { path = "crates/package-manager" } pacquet-lockfile = { path = "crates/lockfile" } pacquet-npmrc = { path = "crates/npmrc" } pacquet-executor = { path = "crates/executor" } -pacquet-cafs = { path = "crates/cafs" } pacquet-diagnostics = { path = "crates/diagnostics" } pacquet-store-dir = { path = "crates/store-dir" } diff --git a/crates/cafs/Cargo.toml b/crates/cafs/Cargo.toml deleted file mode 100644 index 8018a2a49..000000000 --- a/crates/cafs/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "pacquet-cafs" -version = "0.0.1" -publish = false -authors.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -pacquet-store-dir = { workspace = true } - -derive_more = { workspace = true } -miette = { workspace = true } -sha2 = { workspace = true } -ssri = { workspace = true } - -[dev-dependencies] -tempfile = { workspace = true } -pretty_assertions = { workspace = true } diff --git a/crates/cafs/src/lib.rs b/crates/cafs/src/lib.rs deleted file mode 100644 index ff48a6bcf..000000000 --- a/crates/cafs/src/lib.rs +++ /dev/null @@ -1,63 +0,0 @@ -use derive_more::{Display, Error, From}; -use miette::Diagnostic; -use pacquet_store_dir::{FileHash, FileSuffix, StoreDir}; -use sha2::{Digest, Sha512}; -use ssri::Integrity; -use std::{ - fs, io, - path::{Path, PathBuf}, -}; - -#[derive(Debug, Display, Error, From, Diagnostic)] -#[non_exhaustive] -pub enum CafsError { - #[diagnostic(code(pacquet_cafs::io_error))] - Io(std::io::Error), // TODO: remove derive(From), split this variant -} - -fn write_file_if_not_exist(file_path: &Path, content: &[u8]) -> io::Result<()> { - if file_path.exists() { - return Ok(()); - } - - let parent_dir = file_path.parent().unwrap(); - fs::create_dir_all(parent_dir)?; - fs::write(file_path, content) -} - -pub fn write_non_index_file( - store_dir: &StoreDir, - buffer: &[u8], - suffix: Option, -) -> Result<(PathBuf, FileHash), CafsError> { - let file_hash = Sha512::digest(buffer); - let file_path = store_dir.file_path_by_content_address(file_hash, suffix); - - write_file_if_not_exist(&file_path, buffer)?; - - #[cfg(unix)] - { - use std::{fs::Permissions, os::unix::fs::PermissionsExt}; - if suffix == Some(FileSuffix::Exec) { - let permissions = Permissions::from_mode(0o777); - fs::set_permissions(&file_path, permissions).expect("make the file executable"); - } - } - - Ok((file_path, file_hash)) -} - -pub fn write_tarball_index_file( - store_dir: &StoreDir, - tarball_integrity: &Integrity, - index_content: &str, -) -> Result<(), CafsError> { - let file_path = store_dir.tarball_index_file_path(tarball_integrity); - write_file_if_not_exist(&file_path, index_content.as_bytes())?; - Ok(()) -} - -pub fn prune_sync(_store_dir: &StoreDir) -> Result<(), CafsError> { - // Ref: https://pnpm.io/cli/store#prune - todo!("remove orphaned files") -} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d8932cd1b..be90dc131 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -15,7 +15,6 @@ name = "pacquet" path = "src/bin/main.rs" [dependencies] -pacquet-cafs = { workspace = true } pacquet-executor = { workspace = true } pacquet-fs = { workspace = true } pacquet-lockfile = { workspace = true } diff --git a/crates/cli/src/cli_args/store.rs b/crates/cli/src/cli_args/store.rs index 4a908fe41..5f7e78757 100644 --- a/crates/cli/src/cli_args/store.rs +++ b/crates/cli/src/cli_args/store.rs @@ -29,7 +29,7 @@ impl StoreCommand { panic!("Not implemented") } StoreCommand::Prune => { - pacquet_cafs::prune_sync(&config().store_dir).wrap_err("pruning store")?; + config().store_dir.prune().wrap_err("pruning store")?; } StoreCommand::Path => { println!("{}", config().store_dir.display()); diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index 38bd1cefb..1ea80423f 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true [dependencies] derive_more = { workspace = true } +miette = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } ssri = { workspace = true } diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 1aa6f6083..61aab3ac8 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,3 +1,7 @@ +mod prune; mod store_dir; +mod write_file; +pub use prune::*; pub use store_dir::*; +pub use write_file::*; diff --git a/crates/store-dir/src/prune.rs b/crates/store-dir/src/prune.rs new file mode 100644 index 000000000..ee1514c25 --- /dev/null +++ b/crates/store-dir/src/prune.rs @@ -0,0 +1,14 @@ +use crate::StoreDir; +use derive_more::{Display, Error}; +use miette::Diagnostic; + +#[derive(Debug, Display, Error, Diagnostic)] +pub enum PruneError {} + +impl StoreDir { + /// Remove all files in the store that don't have reference elsewhere. + pub fn prune(&self) -> Result<(), PruneError> { + // Ref: https://pnpm.io/cli/store#prune + todo!("remove orphaned files") + } +} diff --git a/crates/store-dir/src/write_file.rs b/crates/store-dir/src/write_file.rs new file mode 100644 index 000000000..7c4f83525 --- /dev/null +++ b/crates/store-dir/src/write_file.rs @@ -0,0 +1,68 @@ +use crate::{FileHash, FileSuffix, StoreDir}; +use derive_more::{Display, Error}; +use miette::Diagnostic; +use sha2::{Digest, Sha512}; +use ssri::Integrity; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +// TODO: separate 2 error cases and add more details +fn write_file_if_not_exist(file_path: &Path, content: &[u8]) -> io::Result<()> { + if file_path.exists() { + return Ok(()); + } + + let parent_dir = file_path.parent().unwrap(); + fs::create_dir_all(parent_dir)?; + fs::write(file_path, content) +} + +#[derive(Debug, Display, Error, Diagnostic)] +pub enum WriteNonIndexFileError { + WriteFile(io::Error), // TODO: add more details +} + +impl StoreDir { + /// Write a file from an npm package to the store directory. + pub fn write_non_index_file( + &self, + buffer: &[u8], + suffix: Option, + ) -> Result<(PathBuf, FileHash), WriteNonIndexFileError> { + let file_hash = Sha512::digest(buffer); + let file_path = self.file_path_by_content_address(file_hash, suffix); + + write_file_if_not_exist(&file_path, buffer).map_err(WriteNonIndexFileError::WriteFile)?; + + #[cfg(unix)] + { + use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + if suffix == Some(FileSuffix::Exec) { + let permissions = Permissions::from_mode(0o777); + fs::set_permissions(&file_path, permissions).expect("make the file executable"); + } + } + + Ok((file_path, file_hash)) + } +} + +#[derive(Debug, Display, Error, Diagnostic)] +pub enum WriteTarballIndexFileError { + WriteFile(io::Error), // TODO: add more details +} + +impl StoreDir { + /// Write a JSON file that indexes files in a tarball to the store directory. + pub fn write_tarball_index_file( + &self, + tarball_integrity: &Integrity, + index_content: &str, + ) -> Result<(), WriteTarballIndexFileError> { + let file_path = self.tarball_index_file_path(tarball_integrity); + write_file_if_not_exist(&file_path, index_content.as_bytes()) + .map_err(WriteTarballIndexFileError::WriteFile) + } +} diff --git a/crates/tarball/Cargo.toml b/crates/tarball/Cargo.toml index f60b0e460..2084002d7 100644 --- a/crates/tarball/Cargo.toml +++ b/crates/tarball/Cargo.toml @@ -11,7 +11,6 @@ license.workspace = true repository.workspace = true [dependencies] -pacquet-cafs = { workspace = true } pacquet-diagnostics = { workspace = true } pacquet-store-dir = { workspace = true } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 42d0aea15..b75ecfeff 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -10,7 +10,7 @@ use base64::{engine::general_purpose::STANDARD as BASE64_STD, Engine}; use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; -use pacquet_store_dir::{FileSuffix, StoreDir}; +use pacquet_store_dir::{FileSuffix, StoreDir, WriteNonIndexFileError, WriteTarballIndexFileError}; use pipe_trait::Pipe; use reqwest::Client; use serde::{Deserialize, Serialize}; @@ -78,7 +78,12 @@ pub enum TarballError { #[from(ignore)] #[display("Failed to write cafs: {_0}")] #[diagnostic(transparent)] - WriteCafs(pacquet_cafs::CafsError), + WriteNonIndexFile(WriteNonIndexFileError), + + #[from(ignore)] + #[display("Failed to write tarball index: {_0}")] + #[diagnostic(transparent)] + WriteTarballIndexFile(WriteTarballIndexFileError), #[from(ignore)] #[diagnostic(code(pacquet_tarball::task_join_error))] @@ -239,9 +244,9 @@ impl<'a> DownloadTarballToStore<'a> { let entry_path = entry.path().unwrap(); let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); - let (file_path, file_hash) = - pacquet_cafs::write_non_index_file(store_dir, &buffer, file_suffix) - .map_err(TarballError::WriteCafs)?; + let (file_path, file_hash) = store_dir + .write_non_index_file(&buffer, file_suffix) + .map_err(TarballError::WriteNonIndexFile)?; let tarball_index_key = cleaned_entry_path .to_str() @@ -267,9 +272,9 @@ impl<'a> DownloadTarballToStore<'a> { let tarball_index = serde_json::to_string(&tarball_index).expect("convert a TarballIndex to JSON"); - - pacquet_cafs::write_tarball_index_file(store_dir, &package_integrity, &tarball_index) - .map_err(TarballError::WriteCafs)?; + store_dir + .write_tarball_index_file(&package_integrity, &tarball_index) + .map_err(TarballError::WriteTarballIndexFile)?; Ok(cas_paths) }) From 593dcdb9586dbcc9e7c92e2b8650f98f6b49b72c Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:02:06 +0700 Subject: [PATCH 042/102] docs(store-dir): error type --- crates/store-dir/src/prune.rs | 1 + crates/store-dir/src/write_file.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/crates/store-dir/src/prune.rs b/crates/store-dir/src/prune.rs index ee1514c25..0991feaef 100644 --- a/crates/store-dir/src/prune.rs +++ b/crates/store-dir/src/prune.rs @@ -2,6 +2,7 @@ use crate::StoreDir; use derive_more::{Display, Error}; use miette::Diagnostic; +/// Error type of [`StoreDir::prune`]. #[derive(Debug, Display, Error, Diagnostic)] pub enum PruneError {} diff --git a/crates/store-dir/src/write_file.rs b/crates/store-dir/src/write_file.rs index 7c4f83525..b539d42cf 100644 --- a/crates/store-dir/src/write_file.rs +++ b/crates/store-dir/src/write_file.rs @@ -19,6 +19,7 @@ fn write_file_if_not_exist(file_path: &Path, content: &[u8]) -> io::Result<()> { fs::write(file_path, content) } +/// Error type of [`StoreDir::write_non_index_file`]. #[derive(Debug, Display, Error, Diagnostic)] pub enum WriteNonIndexFileError { WriteFile(io::Error), // TODO: add more details @@ -49,6 +50,7 @@ impl StoreDir { } } +/// Error type of [`StoreDir::write_tarball_index_file`]. #[derive(Debug, Display, Error, Diagnostic)] pub enum WriteTarballIndexFileError { WriteFile(io::Error), // TODO: add more details From 4c2141ac5c212ff2d9d5c24641d4fe7727dee65c Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:13:45 +0700 Subject: [PATCH 043/102] refactor: move TarballIndex into store-dir --- Cargo.lock | 1 + crates/store-dir/Cargo.toml | 1 + crates/store-dir/src/index.rs | 20 ++++++++++++++++++++ crates/store-dir/src/lib.rs | 2 ++ crates/store-dir/src/write_file.rs | 6 ++++-- crates/tarball/src/lib.rs | 25 ++++--------------------- 6 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 crates/store-dir/src/index.rs diff --git a/Cargo.lock b/Cargo.lock index bbc4a4fa4..8511e0487 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1481,6 +1481,7 @@ dependencies = [ "pipe-trait", "pretty_assertions", "serde", + "serde_json", "sha2", "ssri", "strum", diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index 1ea80423f..eafc277fd 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -14,6 +14,7 @@ repository.workspace = true derive_more = { workspace = true } miette = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } sha2 = { workspace = true } ssri = { workspace = true } strum = { workspace = true } diff --git a/crates/store-dir/src/index.rs b/crates/store-dir/src/index.rs new file mode 100644 index 000000000..f73699d9d --- /dev/null +++ b/crates/store-dir/src/index.rs @@ -0,0 +1,20 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Content of an index file (`$STORE_DIR/v3/files/*/*-index.json`). +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TarballIndex { + pub files: HashMap, +} + +/// Value of the [`files`](TarballIndex::files) map. +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TarballIndexFileAttrs { + // pub checked_at: ??? + pub integrity: String, + pub mode: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, +} diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 61aab3ac8..516d250d7 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,7 +1,9 @@ +mod index; mod prune; mod store_dir; mod write_file; +pub use index::*; pub use prune::*; pub use store_dir::*; pub use write_file::*; diff --git a/crates/store-dir/src/write_file.rs b/crates/store-dir/src/write_file.rs index b539d42cf..ac5ada45c 100644 --- a/crates/store-dir/src/write_file.rs +++ b/crates/store-dir/src/write_file.rs @@ -1,4 +1,4 @@ -use crate::{FileHash, FileSuffix, StoreDir}; +use crate::{FileHash, FileSuffix, StoreDir, TarballIndex}; use derive_more::{Display, Error}; use miette::Diagnostic; use sha2::{Digest, Sha512}; @@ -61,9 +61,11 @@ impl StoreDir { pub fn write_tarball_index_file( &self, tarball_integrity: &Integrity, - index_content: &str, + index_content: &TarballIndex, ) -> Result<(), WriteTarballIndexFileError> { let file_path = self.tarball_index_file_path(tarball_integrity); + let index_content = + serde_json::to_string(&index_content).expect("convert a TarballIndex to JSON"); write_file_if_not_exist(&file_path, index_content.as_bytes()) .map_err(WriteTarballIndexFileError::WriteFile) } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index b75ecfeff..1b9edc4ff 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -10,10 +10,12 @@ use base64::{engine::general_purpose::STANDARD as BASE64_STD, Engine}; use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; -use pacquet_store_dir::{FileSuffix, StoreDir, WriteNonIndexFileError, WriteTarballIndexFileError}; +use pacquet_store_dir::{ + FileSuffix, StoreDir, TarballIndex, TarballIndexFileAttrs, WriteNonIndexFileError, + WriteTarballIndexFileError, +}; use pipe_trait::Pipe; use reqwest::Client; -use serde::{Deserialize, Serialize}; use ssri::{Integrity, IntegrityChecker}; use tar::Archive; use tokio::sync::{Notify, RwLock}; @@ -270,8 +272,6 @@ impl<'a> DownloadTarballToStore<'a> { } } - let tarball_index = - serde_json::to_string(&tarball_index).expect("convert a TarballIndex to JSON"); store_dir .write_tarball_index_file(&package_integrity, &tarball_index) .map_err(TarballError::WriteTarballIndexFile)?; @@ -294,23 +294,6 @@ impl<'a> DownloadTarballToStore<'a> { } } -/// Content of an index file (`$STORE_DIR/v3/files/*/*-index.json`). -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -struct TarballIndex { - pub files: HashMap, -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -struct TarballIndexFileAttrs { - // pub checked_at: ??? - pub integrity: String, - pub mode: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub size: Option, -} - #[cfg(test)] mod tests { use pipe_trait::Pipe; From c7e8d9a1208a6022120e1ab66a9552821d44b152 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:24:49 +0700 Subject: [PATCH 044/102] refactor: move a function to pacquet-fs --- Cargo.lock | 1 + crates/fs/src/lib.rs | 17 ++++++++++++++++- crates/store-dir/Cargo.toml | 2 ++ crates/store-dir/src/write_file.rs | 21 ++++----------------- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8511e0487..f3cf0fbf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1478,6 +1478,7 @@ version = "0.0.1" dependencies = [ "derive_more", "miette", + "pacquet-fs", "pipe-trait", "pretty_assertions", "serde", diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 08c68721e..5bea31529 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -1,4 +1,4 @@ -use std::{io, path::Path}; +use std::{fs, io, path::Path}; /// Create a symlink to a directory. /// @@ -9,3 +9,18 @@ pub fn symlink_dir(original: &Path, link: &Path) -> io::Result<()> { #[cfg(windows)] return junction::create(original, link); // junctions instead of symlinks because symlinks may require elevated privileges. } + +/// Write `content` to `file_path` unless it already exists. +/// +/// Ancestor directories will be created if they don't already exist. +/// +/// **TODO:** separate 2 error cases and add more details +pub fn ensure_file(file_path: &Path, content: &[u8]) -> io::Result<()> { + if file_path.exists() { + return Ok(()); + } + + let parent_dir = file_path.parent().unwrap(); + fs::create_dir_all(parent_dir)?; + fs::write(file_path, content) +} diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index eafc277fd..7f1b9a577 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -11,6 +11,8 @@ license.workspace = true repository.workspace = true [dependencies] +pacquet-fs = { workspace = true } + derive_more = { workspace = true } miette = { workspace = true } serde = { workspace = true } diff --git a/crates/store-dir/src/write_file.rs b/crates/store-dir/src/write_file.rs index ac5ada45c..b6e47bac4 100644 --- a/crates/store-dir/src/write_file.rs +++ b/crates/store-dir/src/write_file.rs @@ -1,23 +1,10 @@ use crate::{FileHash, FileSuffix, StoreDir, TarballIndex}; use derive_more::{Display, Error}; use miette::Diagnostic; +use pacquet_fs::ensure_file; use sha2::{Digest, Sha512}; use ssri::Integrity; -use std::{ - fs, io, - path::{Path, PathBuf}, -}; - -// TODO: separate 2 error cases and add more details -fn write_file_if_not_exist(file_path: &Path, content: &[u8]) -> io::Result<()> { - if file_path.exists() { - return Ok(()); - } - - let parent_dir = file_path.parent().unwrap(); - fs::create_dir_all(parent_dir)?; - fs::write(file_path, content) -} +use std::{fs, io, path::PathBuf}; /// Error type of [`StoreDir::write_non_index_file`]. #[derive(Debug, Display, Error, Diagnostic)] @@ -35,7 +22,7 @@ impl StoreDir { let file_hash = Sha512::digest(buffer); let file_path = self.file_path_by_content_address(file_hash, suffix); - write_file_if_not_exist(&file_path, buffer).map_err(WriteNonIndexFileError::WriteFile)?; + ensure_file(&file_path, buffer).map_err(WriteNonIndexFileError::WriteFile)?; #[cfg(unix)] { @@ -66,7 +53,7 @@ impl StoreDir { let file_path = self.tarball_index_file_path(tarball_integrity); let index_content = serde_json::to_string(&index_content).expect("convert a TarballIndex to JSON"); - write_file_if_not_exist(&file_path, index_content.as_bytes()) + ensure_file(&file_path, index_content.as_bytes()) .map_err(WriteTarballIndexFileError::WriteFile) } } From b34a3f0cfe13b7ce1bd762d996ec545f7fd2006a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:32:44 +0700 Subject: [PATCH 045/102] feat(fs): divide error variants into 2 --- Cargo.lock | 2 ++ crates/fs/Cargo.toml | 4 ++++ crates/fs/src/lib.rs | 34 +++++++++++++++++++++++++----- crates/store-dir/src/write_file.rs | 8 +++---- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3cf0fbf8..faaaf458f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,7 +1348,9 @@ dependencies = [ name = "pacquet-fs" version = "0.0.1" dependencies = [ + "derive_more", "junction", + "miette", ] [[package]] diff --git a/crates/fs/Cargo.toml b/crates/fs/Cargo.toml index 1a4e38552..6d75761de 100644 --- a/crates/fs/Cargo.toml +++ b/crates/fs/Cargo.toml @@ -10,5 +10,9 @@ keywords.workspace = true license.workspace = true repository.workspace = true +[dependencies] +derive_more = { workspace = true } +miette = { workspace = true } + [target.'cfg(windows)'.dependencies] junction = { workspace = true } diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 5bea31529..466a4bf63 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -1,4 +1,9 @@ -use std::{fs, io, path::Path}; +use derive_more::{Display, Error}; +use miette::Diagnostic; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; /// Create a symlink to a directory. /// @@ -10,17 +15,36 @@ pub fn symlink_dir(original: &Path, link: &Path) -> io::Result<()> { return junction::create(original, link); // junctions instead of symlinks because symlinks may require elevated privileges. } +/// Error type of [`ensure_file`]. +#[derive(Debug, Display, Error, Diagnostic)] +pub enum EnsureFileError { + #[display("Failed to create the parent directory at {parent_dir:?}: {error}")] + CreateDir { + parent_dir: PathBuf, + #[error(source)] + error: io::Error, + }, + #[display("Failed to write to file at {file_path:?}: {error}")] + WriteFile { + file_path: PathBuf, + #[error(source)] + error: io::Error, + }, +} + /// Write `content` to `file_path` unless it already exists. /// /// Ancestor directories will be created if they don't already exist. -/// -/// **TODO:** separate 2 error cases and add more details -pub fn ensure_file(file_path: &Path, content: &[u8]) -> io::Result<()> { +pub fn ensure_file(file_path: &Path, content: &[u8]) -> Result<(), EnsureFileError> { if file_path.exists() { return Ok(()); } let parent_dir = file_path.parent().unwrap(); - fs::create_dir_all(parent_dir)?; + fs::create_dir_all(parent_dir).map_err(|error| EnsureFileError::CreateDir { + parent_dir: parent_dir.to_path_buf(), + error, + })?; fs::write(file_path, content) + .map_err(|error| EnsureFileError::WriteFile { file_path: file_path.to_path_buf(), error }) } diff --git a/crates/store-dir/src/write_file.rs b/crates/store-dir/src/write_file.rs index b6e47bac4..dc3f53e73 100644 --- a/crates/store-dir/src/write_file.rs +++ b/crates/store-dir/src/write_file.rs @@ -1,15 +1,15 @@ use crate::{FileHash, FileSuffix, StoreDir, TarballIndex}; use derive_more::{Display, Error}; use miette::Diagnostic; -use pacquet_fs::ensure_file; +use pacquet_fs::{ensure_file, EnsureFileError}; use sha2::{Digest, Sha512}; use ssri::Integrity; -use std::{fs, io, path::PathBuf}; +use std::{fs, path::PathBuf}; /// Error type of [`StoreDir::write_non_index_file`]. #[derive(Debug, Display, Error, Diagnostic)] pub enum WriteNonIndexFileError { - WriteFile(io::Error), // TODO: add more details + WriteFile(EnsureFileError), } impl StoreDir { @@ -40,7 +40,7 @@ impl StoreDir { /// Error type of [`StoreDir::write_tarball_index_file`]. #[derive(Debug, Display, Error, Diagnostic)] pub enum WriteTarballIndexFileError { - WriteFile(io::Error), // TODO: add more details + WriteFile(EnsureFileError), } impl StoreDir { From a77b77582b125b9b7545f19e164f2ae8c7409dac Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:37:37 +0700 Subject: [PATCH 046/102] refactor: rename --- crates/store-dir/src/write_file.rs | 10 +++++----- crates/tarball/src/lib.rs | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/store-dir/src/write_file.rs b/crates/store-dir/src/write_file.rs index dc3f53e73..73f8f91ed 100644 --- a/crates/store-dir/src/write_file.rs +++ b/crates/store-dir/src/write_file.rs @@ -6,23 +6,23 @@ use sha2::{Digest, Sha512}; use ssri::Integrity; use std::{fs, path::PathBuf}; -/// Error type of [`StoreDir::write_non_index_file`]. +/// Error type of [`StoreDir::write_cas_file`]. #[derive(Debug, Display, Error, Diagnostic)] -pub enum WriteNonIndexFileError { +pub enum WriteCasFileError { WriteFile(EnsureFileError), } impl StoreDir { /// Write a file from an npm package to the store directory. - pub fn write_non_index_file( + pub fn write_cas_file( &self, buffer: &[u8], suffix: Option, - ) -> Result<(PathBuf, FileHash), WriteNonIndexFileError> { + ) -> Result<(PathBuf, FileHash), WriteCasFileError> { let file_hash = Sha512::digest(buffer); let file_path = self.file_path_by_content_address(file_hash, suffix); - ensure_file(&file_path, buffer).map_err(WriteNonIndexFileError::WriteFile)?; + ensure_file(&file_path, buffer).map_err(WriteCasFileError::WriteFile)?; #[cfg(unix)] { diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 1b9edc4ff..e07f3bfaf 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -11,7 +11,7 @@ use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::{ - FileSuffix, StoreDir, TarballIndex, TarballIndexFileAttrs, WriteNonIndexFileError, + FileSuffix, StoreDir, TarballIndex, TarballIndexFileAttrs, WriteCasFileError, WriteTarballIndexFileError, }; use pipe_trait::Pipe; @@ -80,7 +80,7 @@ pub enum TarballError { #[from(ignore)] #[display("Failed to write cafs: {_0}")] #[diagnostic(transparent)] - WriteNonIndexFile(WriteNonIndexFileError), + WriteCasFile(WriteCasFileError), #[from(ignore)] #[display("Failed to write tarball index: {_0}")] @@ -247,8 +247,8 @@ impl<'a> DownloadTarballToStore<'a> { let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); let (file_path, file_hash) = store_dir - .write_non_index_file(&buffer, file_suffix) - .map_err(TarballError::WriteNonIndexFile)?; + .write_cas_file(&buffer, file_suffix) + .map_err(TarballError::WriteCasFile)?; let tarball_index_key = cleaned_entry_path .to_str() From e3821c193dd2991481de6257b44ece9ad26aa581 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:39:55 +0700 Subject: [PATCH 047/102] refactor: correct categories --- .../src/{write_file.rs => cas_file.rs} | 24 +--------- crates/store-dir/src/index.rs | 20 -------- crates/store-dir/src/index_file.rs | 46 +++++++++++++++++++ crates/store-dir/src/lib.rs | 8 ++-- 4 files changed, 51 insertions(+), 47 deletions(-) rename crates/store-dir/src/{write_file.rs => cas_file.rs} (57%) delete mode 100644 crates/store-dir/src/index.rs create mode 100644 crates/store-dir/src/index_file.rs diff --git a/crates/store-dir/src/write_file.rs b/crates/store-dir/src/cas_file.rs similarity index 57% rename from crates/store-dir/src/write_file.rs rename to crates/store-dir/src/cas_file.rs index 73f8f91ed..49fe759ba 100644 --- a/crates/store-dir/src/write_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -1,9 +1,8 @@ -use crate::{FileHash, FileSuffix, StoreDir, TarballIndex}; +use crate::{FileHash, FileSuffix, StoreDir}; use derive_more::{Display, Error}; use miette::Diagnostic; use pacquet_fs::{ensure_file, EnsureFileError}; use sha2::{Digest, Sha512}; -use ssri::Integrity; use std::{fs, path::PathBuf}; /// Error type of [`StoreDir::write_cas_file`]. @@ -36,24 +35,3 @@ impl StoreDir { Ok((file_path, file_hash)) } } - -/// Error type of [`StoreDir::write_tarball_index_file`]. -#[derive(Debug, Display, Error, Diagnostic)] -pub enum WriteTarballIndexFileError { - WriteFile(EnsureFileError), -} - -impl StoreDir { - /// Write a JSON file that indexes files in a tarball to the store directory. - pub fn write_tarball_index_file( - &self, - tarball_integrity: &Integrity, - index_content: &TarballIndex, - ) -> Result<(), WriteTarballIndexFileError> { - let file_path = self.tarball_index_file_path(tarball_integrity); - let index_content = - serde_json::to_string(&index_content).expect("convert a TarballIndex to JSON"); - ensure_file(&file_path, index_content.as_bytes()) - .map_err(WriteTarballIndexFileError::WriteFile) - } -} diff --git a/crates/store-dir/src/index.rs b/crates/store-dir/src/index.rs deleted file mode 100644 index f73699d9d..000000000 --- a/crates/store-dir/src/index.rs +++ /dev/null @@ -1,20 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// Content of an index file (`$STORE_DIR/v3/files/*/*-index.json`). -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct TarballIndex { - pub files: HashMap, -} - -/// Value of the [`files`](TarballIndex::files) map. -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct TarballIndexFileAttrs { - // pub checked_at: ??? - pub integrity: String, - pub mode: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub size: Option, -} diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs new file mode 100644 index 000000000..57a8768ff --- /dev/null +++ b/crates/store-dir/src/index_file.rs @@ -0,0 +1,46 @@ +use crate::StoreDir; +use derive_more::{Display, Error}; +use miette::Diagnostic; +use pacquet_fs::{ensure_file, EnsureFileError}; +use serde::{Deserialize, Serialize}; +use ssri::Integrity; +use std::collections::HashMap; + +/// Content of an index file (`$STORE_DIR/v3/files/*/*-index.json`). +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TarballIndex { + pub files: HashMap, +} + +/// Value of the [`files`](TarballIndex::files) map. +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TarballIndexFileAttrs { + // pub checked_at: ??? + pub integrity: String, + pub mode: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, +} + +/// Error type of [`StoreDir::write_tarball_index_file`]. +#[derive(Debug, Display, Error, Diagnostic)] +pub enum WriteTarballIndexFileError { + WriteFile(EnsureFileError), +} + +impl StoreDir { + /// Write a JSON file that indexes files in a tarball to the store directory. + pub fn write_tarball_index_file( + &self, + tarball_integrity: &Integrity, + index_content: &TarballIndex, + ) -> Result<(), WriteTarballIndexFileError> { + let file_path = self.tarball_index_file_path(tarball_integrity); + let index_content = + serde_json::to_string(&index_content).expect("convert a TarballIndex to JSON"); + ensure_file(&file_path, index_content.as_bytes()) + .map_err(WriteTarballIndexFileError::WriteFile) + } +} diff --git a/crates/store-dir/src/lib.rs b/crates/store-dir/src/lib.rs index 516d250d7..657a69c72 100644 --- a/crates/store-dir/src/lib.rs +++ b/crates/store-dir/src/lib.rs @@ -1,9 +1,9 @@ -mod index; +mod cas_file; +mod index_file; mod prune; mod store_dir; -mod write_file; -pub use index::*; +pub use cas_file::*; +pub use index_file::*; pub use prune::*; pub use store_dir::*; -pub use write_file::*; From 1ef6d6ffc5d701488ac8249b53da853aecb85c17 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:42:55 +0700 Subject: [PATCH 048/102] refactor: remove unnecessary `pub(crate)` --- crates/store-dir/src/store_dir.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs index 67e6a3025..5487fdb67 100644 --- a/crates/store-dir/src/store_dir.rs +++ b/crates/store-dir/src/store_dir.rs @@ -63,7 +63,7 @@ impl StoreDir { } /// Path to a file in the store directory. - pub(crate) fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { + fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { let head = &hex[..2]; let middle = &hex[2..]; let suffix = suffix.map_or("", <&str>::from); From d9270f4d5f85019ce2c9c172a6594595f5f41940 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:47:08 +0700 Subject: [PATCH 049/102] refactor: pass `executable` instead of `suffix` --- crates/store-dir/src/cas_file.rs | 8 ++++---- crates/store-dir/src/store_dir.rs | 25 ++++++++----------------- crates/tarball/src/lib.rs | 6 ++---- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index 49fe759ba..4328d037a 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -1,4 +1,4 @@ -use crate::{FileHash, FileSuffix, StoreDir}; +use crate::{FileHash, StoreDir}; use derive_more::{Display, Error}; use miette::Diagnostic; use pacquet_fs::{ensure_file, EnsureFileError}; @@ -16,17 +16,17 @@ impl StoreDir { pub fn write_cas_file( &self, buffer: &[u8], - suffix: Option, + executable: bool, ) -> Result<(PathBuf, FileHash), WriteCasFileError> { let file_hash = Sha512::digest(buffer); - let file_path = self.file_path_by_content_address(file_hash, suffix); + let file_path = self.file_path_by_content_address(file_hash, executable); ensure_file(&file_path, buffer).map_err(WriteCasFileError::WriteFile)?; #[cfg(unix)] { use std::{fs::Permissions, os::unix::fs::PermissionsExt}; - if suffix == Some(FileSuffix::Exec) { + if executable { let permissions = Permissions::from_mode(0o777); fs::set_permissions(&file_path, permissions).expect("make the file executable"); } diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs index 5487fdb67..c1e94a36b 100644 --- a/crates/store-dir/src/store_dir.rs +++ b/crates/store-dir/src/store_dir.rs @@ -10,7 +10,7 @@ pub type FileHash = digest::Output; /// Optional suffix of a content address of a file. #[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr)] -pub enum FileSuffix { +enum FileSuffix { #[strum(serialize = "-exec")] Exec, #[strum(serialize = "-index.json")] @@ -72,12 +72,9 @@ impl StoreDir { } /// Path to a file in the store directory. - pub fn file_path_by_content_address( - &self, - hash: FileHash, - suffix: Option, - ) -> PathBuf { + pub fn file_path_by_content_address(&self, hash: FileHash, executable: bool) -> PathBuf { let hex = format!("{hash:x}"); + let suffix = executable.then_some(FileSuffix::Exec); self.file_path_by_hex_str(&hex, suffix) } @@ -114,33 +111,27 @@ mod tests { #[test] fn file_path_by_content_address() { - fn case(file_content: &str, suffix: Option, expected: &str) { - eprintln!("CASE: {file_content:?}, {suffix:?}"); + fn case(file_content: &str, executable: bool, expected: &str) { + eprintln!("CASE: {file_content:?}, {executable:?}"); let store_dir = StoreDir::new("STORE_DIR"); let file_hash = Sha512::digest(file_content); eprintln!("file_hash = {file_hash:x}"); - let received = store_dir.file_path_by_content_address(file_hash, suffix); + let received = store_dir.file_path_by_content_address(file_hash, executable); let expected: PathBuf = expected.split('/').collect(); assert_eq!(&received, &expected); } case( "hello world", - None, + false, "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", ); case( "hello world", - Some(FileSuffix::Exec), + true, "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", ); - - case( - "hello world", - Some(FileSuffix::Index), - "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-index.json", - ); } #[test] diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index e07f3bfaf..49af7697e 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -11,8 +11,7 @@ use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::{ - FileSuffix, StoreDir, TarballIndex, TarballIndexFileAttrs, WriteCasFileError, - WriteTarballIndexFileError, + StoreDir, TarballIndex, TarballIndexFileAttrs, WriteCasFileError, WriteTarballIndexFileError, }; use pipe_trait::Pipe; use reqwest::Client; @@ -237,7 +236,6 @@ impl<'a> DownloadTarballToStore<'a> { let file_mode = entry.header().mode().expect("get mode"); // TODO: properly propagate this error let is_executable = file_mode & EXEC_MASK != 0; - let file_suffix = is_executable.then_some(FileSuffix::Exec); // Read the contents of the entry let mut buffer = Vec::with_capacity(entry.size() as usize); @@ -247,7 +245,7 @@ impl<'a> DownloadTarballToStore<'a> { let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); let (file_path, file_hash) = store_dir - .write_cas_file(&buffer, file_suffix) + .write_cas_file(&buffer, is_executable) .map_err(TarballError::WriteCasFile)?; let tarball_index_key = cleaned_entry_path From 7d771db1c428913e4202919e0c8d568009399e4d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:50:28 +0700 Subject: [PATCH 050/102] refactor: remove FileSuffix and strum --- Cargo.lock | 1 - crates/store-dir/Cargo.toml | 1 - crates/store-dir/src/store_dir.rs | 18 ++++-------------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index faaaf458f..8c62861db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1487,7 +1487,6 @@ dependencies = [ "serde_json", "sha2", "ssri", - "strum", ] [[package]] diff --git a/crates/store-dir/Cargo.toml b/crates/store-dir/Cargo.toml index 7f1b9a577..fef784cb0 100644 --- a/crates/store-dir/Cargo.toml +++ b/crates/store-dir/Cargo.toml @@ -19,7 +19,6 @@ serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } ssri = { workspace = true } -strum = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs index c1e94a36b..591a3e3cc 100644 --- a/crates/store-dir/src/store_dir.rs +++ b/crates/store-dir/src/store_dir.rs @@ -3,20 +3,10 @@ use serde::{Deserialize, Serialize}; use sha2::{digest, Sha512}; use ssri::{Algorithm, Integrity}; use std::path::{self, PathBuf}; -use strum::IntoStaticStr; /// Content hash of a file. pub type FileHash = digest::Output; -/// Optional suffix of a content address of a file. -#[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr)] -enum FileSuffix { - #[strum(serialize = "-exec")] - Exec, - #[strum(serialize = "-index.json")] - Index, -} - /// Represent a store directory. /// /// * The store directory stores all files that were acquired by installing packages with pacquet or pnpm. @@ -63,10 +53,10 @@ impl StoreDir { } /// Path to a file in the store directory. - fn file_path_by_hex_str(&self, hex: &str, suffix: Option) -> PathBuf { + fn file_path_by_hex_str(&self, hex: &str, suffix: Option<&'static str>) -> PathBuf { let head = &hex[..2]; let middle = &hex[2..]; - let suffix = suffix.map_or("", <&str>::from); + let suffix = suffix.unwrap_or(""); let tail = format!("{middle}{suffix}"); self.file_path_by_head_tail(head, &tail) } @@ -74,7 +64,7 @@ impl StoreDir { /// Path to a file in the store directory. pub fn file_path_by_content_address(&self, hash: FileHash, executable: bool) -> PathBuf { let hex = format!("{hash:x}"); - let suffix = executable.then_some(FileSuffix::Exec); + let suffix = executable.then_some("-exec"); self.file_path_by_hex_str(&hex, suffix) } @@ -82,7 +72,7 @@ impl StoreDir { pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { let (algorithm, hex) = tarball_integrity.to_hex(); assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error - self.file_path_by_hex_str(&hex, Some(FileSuffix::Index)) + self.file_path_by_hex_str(&hex, Some("-index.json")) } /// Path to the temporary directory inside the store. From 5f3050ad0d1d810df1237d5857f3134f8427efa2 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:53:37 +0700 Subject: [PATCH 051/102] refactor: remove `Option` for `suffix` --- crates/store-dir/src/store_dir.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs index 591a3e3cc..ef5df6e25 100644 --- a/crates/store-dir/src/store_dir.rs +++ b/crates/store-dir/src/store_dir.rs @@ -53,10 +53,9 @@ impl StoreDir { } /// Path to a file in the store directory. - fn file_path_by_hex_str(&self, hex: &str, suffix: Option<&'static str>) -> PathBuf { + fn file_path_by_hex_str(&self, hex: &str, suffix: &'static str) -> PathBuf { let head = &hex[..2]; let middle = &hex[2..]; - let suffix = suffix.unwrap_or(""); let tail = format!("{middle}{suffix}"); self.file_path_by_head_tail(head, &tail) } @@ -64,7 +63,7 @@ impl StoreDir { /// Path to a file in the store directory. pub fn file_path_by_content_address(&self, hash: FileHash, executable: bool) -> PathBuf { let hex = format!("{hash:x}"); - let suffix = executable.then_some("-exec"); + let suffix = if executable { "-exec" } else { "" }; self.file_path_by_hex_str(&hex, suffix) } @@ -72,7 +71,7 @@ impl StoreDir { pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { let (algorithm, hex) = tarball_integrity.to_hex(); assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error - self.file_path_by_hex_str(&hex, Some("-index.json")) + self.file_path_by_hex_str(&hex, "-index.json") } /// Path to the temporary directory inside the store. From 51c0672f7922cb0acc73089d8c0b708c09d4123a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 20:54:28 +0700 Subject: [PATCH 052/102] refactor: rename a function --- crates/store-dir/src/cas_file.rs | 2 +- crates/store-dir/src/store_dir.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index 4328d037a..1029543e8 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -19,7 +19,7 @@ impl StoreDir { executable: bool, ) -> Result<(PathBuf, FileHash), WriteCasFileError> { let file_hash = Sha512::digest(buffer); - let file_path = self.file_path_by_content_address(file_hash, executable); + let file_path = self.cas_file_path(file_hash, executable); ensure_file(&file_path, buffer).map_err(WriteCasFileError::WriteFile)?; diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs index ef5df6e25..eb96610db 100644 --- a/crates/store-dir/src/store_dir.rs +++ b/crates/store-dir/src/store_dir.rs @@ -61,7 +61,7 @@ impl StoreDir { } /// Path to a file in the store directory. - pub fn file_path_by_content_address(&self, hash: FileHash, executable: bool) -> PathBuf { + pub fn cas_file_path(&self, hash: FileHash, executable: bool) -> PathBuf { let hex = format!("{hash:x}"); let suffix = if executable { "-exec" } else { "" }; self.file_path_by_hex_str(&hex, suffix) @@ -105,7 +105,7 @@ mod tests { let store_dir = StoreDir::new("STORE_DIR"); let file_hash = Sha512::digest(file_content); eprintln!("file_hash = {file_hash:x}"); - let received = store_dir.file_path_by_content_address(file_hash, executable); + let received = store_dir.cas_file_path(file_hash, executable); let expected: PathBuf = expected.split('/').collect(); assert_eq!(&received, &expected); } From fb1808743b94388a527f9c28f40ff4b68219409d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:03:14 +0700 Subject: [PATCH 053/102] refactor: make_file_executable --- crates/fs/src/lib.rs | 14 ++++++++++++++ crates/store-dir/src/cas_file.rs | 14 +++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 466a4bf63..4de6909a4 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -48,3 +48,17 @@ pub fn ensure_file(file_path: &Path, content: &[u8]) -> Result<(), EnsureFileErr fs::write(file_path, content) .map_err(|error| EnsureFileError::WriteFile { file_path: file_path.to_path_buf(), error }) } + +/// Set file mode to 777 on POSIX platforms such as Linux or macOS, +/// or do nothing on Windows. +pub fn make_file_executable(file_path: &Path) -> io::Result<()> { + #[cfg(unix)] + return { + use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + let permissions = Permissions::from_mode(0o777); + fs::set_permissions(file_path, permissions) + }; + + #[cfg(windows)] + return Ok(()); +} diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index 1029543e8..adfdf16ad 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -1,9 +1,9 @@ use crate::{FileHash, StoreDir}; use derive_more::{Display, Error}; use miette::Diagnostic; -use pacquet_fs::{ensure_file, EnsureFileError}; +use pacquet_fs::{ensure_file, make_file_executable, EnsureFileError}; use sha2::{Digest, Sha512}; -use std::{fs, path::PathBuf}; +use std::path::PathBuf; /// Error type of [`StoreDir::write_cas_file`]. #[derive(Debug, Display, Error, Diagnostic)] @@ -22,15 +22,7 @@ impl StoreDir { let file_path = self.cas_file_path(file_hash, executable); ensure_file(&file_path, buffer).map_err(WriteCasFileError::WriteFile)?; - - #[cfg(unix)] - { - use std::{fs::Permissions, os::unix::fs::PermissionsExt}; - if executable { - let permissions = Permissions::from_mode(0o777); - fs::set_permissions(&file_path, permissions).expect("make the file executable"); - } - } + make_file_executable(&file_path).expect("make the file executable"); // TODO: propagate this error Ok((file_path, file_hash)) } From 3ba598679c784b2a961363a12193f09a092581b8 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:03:14 +0700 Subject: [PATCH 054/102] feat(fs): make_file_executable --- crates/fs/src/lib.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 08c68721e..9fdac655d 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -1,4 +1,4 @@ -use std::{io, path::Path}; +use std::{fs, io, path::Path}; /// Create a symlink to a directory. /// @@ -9,3 +9,17 @@ pub fn symlink_dir(original: &Path, link: &Path) -> io::Result<()> { #[cfg(windows)] return junction::create(original, link); // junctions instead of symlinks because symlinks may require elevated privileges. } + +/// Set file mode to 777 on POSIX platforms such as Linux or macOS, +/// or do nothing on Windows. +pub fn make_file_executable(file_path: &Path) -> io::Result<()> { + #[cfg(unix)] + return { + use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + let permissions = Permissions::from_mode(0o777); + fs::set_permissions(file_path, permissions) + }; + + #[cfg(windows)] + return Ok(()); +} From 67821530600730a5ad68790ee90197c8ae95a65a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:08:45 +0700 Subject: [PATCH 055/102] refactor: re-use make_file_executable --- Cargo.lock | 1 + tasks/integrated-benchmark/Cargo.toml | 2 ++ tasks/integrated-benchmark/src/work_env.rs | 8 ++------ 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3eb1fe9a6..7d5675749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1370,6 +1370,7 @@ dependencies = [ "clap", "itertools 0.11.0", "os_display", + "pacquet-fs", "pipe-trait", "reqwest", "tokio", diff --git a/tasks/integrated-benchmark/Cargo.toml b/tasks/integrated-benchmark/Cargo.toml index c595a6ff2..06485ecc5 100644 --- a/tasks/integrated-benchmark/Cargo.toml +++ b/tasks/integrated-benchmark/Cargo.toml @@ -15,6 +15,8 @@ name = "integrated-benchmark" path = "src/main.rs" [dependencies] +pacquet-fs = { workspace = true } + clap = { workspace = true } itertools = { workspace = true } os_display = { workspace = true } diff --git a/tasks/integrated-benchmark/src/work_env.rs b/tasks/integrated-benchmark/src/work_env.rs index 3073c50e6..f689923db 100644 --- a/tasks/integrated-benchmark/src/work_env.rs +++ b/tasks/integrated-benchmark/src/work_env.rs @@ -4,6 +4,7 @@ use crate::{ }; use itertools::Itertools; use os_display::Quotable; +use pacquet_fs::make_file_executable; use pipe_trait::Pipe; use std::{ fmt, @@ -250,12 +251,7 @@ fn create_install_script(dir: &Path, scenario: BenchmarkScenario, for_pnpm: bool } writeln!(file).unwrap(); - #[cfg(unix)] - { - use std::{fs::Permissions, os::unix::fs::PermissionsExt}; - let permissions = Permissions::from_mode(0o777); - fs::set_permissions(path, permissions).expect("make the script executable"); - } + make_file_executable(&path).expect("make the script executable"); } fn executor<'a>(message: &'a str) -> impl FnOnce(&'a mut Command) { From b1c9e066750bc7838bb69f31ac43b7031d715773 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:15:27 +0700 Subject: [PATCH 056/102] test: rename --- crates/store-dir/src/store_dir.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs index eb96610db..0f75ba210 100644 --- a/crates/store-dir/src/store_dir.rs +++ b/crates/store-dir/src/store_dir.rs @@ -99,7 +99,7 @@ mod tests { } #[test] - fn file_path_by_content_address() { + fn cas_file_path() { fn case(file_content: &str, executable: bool, expected: &str) { eprintln!("CASE: {file_content:?}, {executable:?}"); let store_dir = StoreDir::new("STORE_DIR"); From af24bd51f9c1f02162e6bfe5551844765586f4d2 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:21:26 +0700 Subject: [PATCH 057/102] refactor: move some functions --- crates/store-dir/src/cas_file.rs | 39 +++++++++++++++++++++++++++ crates/store-dir/src/index_file.rs | 13 +++++++-- crates/store-dir/src/store_dir.rs | 43 +----------------------------- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index adfdf16ad..b7b1be964 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -5,6 +5,15 @@ use pacquet_fs::{ensure_file, make_file_executable, EnsureFileError}; use sha2::{Digest, Sha512}; use std::path::PathBuf; +impl StoreDir { + /// Path to a file in the store directory. + pub fn cas_file_path(&self, hash: FileHash, executable: bool) -> PathBuf { + let hex = format!("{hash:x}"); + let suffix = if executable { "-exec" } else { "" }; + self.file_path_by_hex_str(&hex, suffix) + } +} + /// Error type of [`StoreDir::write_cas_file`]. #[derive(Debug, Display, Error, Diagnostic)] pub enum WriteCasFileError { @@ -27,3 +36,33 @@ impl StoreDir { Ok((file_path, file_hash)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cas_file_path() { + fn case(file_content: &str, executable: bool, expected: &str) { + eprintln!("CASE: {file_content:?}, {executable:?}"); + let store_dir = StoreDir::new("STORE_DIR"); + let file_hash = Sha512::digest(file_content); + eprintln!("file_hash = {file_hash:x}"); + let received = store_dir.cas_file_path(file_hash, executable); + let expected: PathBuf = expected.split('/').collect(); + assert_eq!(&received, &expected); + } + + case( + "hello world", + false, + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", + ); + + case( + "hello world", + true, + "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", + ); + } +} diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index 57a8768ff..3d7498252 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -3,8 +3,17 @@ use derive_more::{Display, Error}; use miette::Diagnostic; use pacquet_fs::{ensure_file, EnsureFileError}; use serde::{Deserialize, Serialize}; -use ssri::Integrity; -use std::collections::HashMap; +use ssri::{Algorithm, Integrity}; +use std::{collections::HashMap, path::PathBuf}; + +impl StoreDir { + /// Path to an index file of a tarball. + pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { + let (algorithm, hex) = tarball_integrity.to_hex(); + assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error + self.file_path_by_hex_str(&hex, "-index.json") + } +} /// Content of an index file (`$STORE_DIR/v3/files/*/*-index.json`). #[derive(Debug, Deserialize, Serialize)] diff --git a/crates/store-dir/src/store_dir.rs b/crates/store-dir/src/store_dir.rs index 0f75ba210..78ea01f2e 100644 --- a/crates/store-dir/src/store_dir.rs +++ b/crates/store-dir/src/store_dir.rs @@ -1,7 +1,6 @@ use derive_more::From; use serde::{Deserialize, Serialize}; use sha2::{digest, Sha512}; -use ssri::{Algorithm, Integrity}; use std::path::{self, PathBuf}; /// Content hash of a file. @@ -53,27 +52,13 @@ impl StoreDir { } /// Path to a file in the store directory. - fn file_path_by_hex_str(&self, hex: &str, suffix: &'static str) -> PathBuf { + pub(crate) fn file_path_by_hex_str(&self, hex: &str, suffix: &'static str) -> PathBuf { let head = &hex[..2]; let middle = &hex[2..]; let tail = format!("{middle}{suffix}"); self.file_path_by_head_tail(head, &tail) } - /// Path to a file in the store directory. - pub fn cas_file_path(&self, hash: FileHash, executable: bool) -> PathBuf { - let hex = format!("{hash:x}"); - let suffix = if executable { "-exec" } else { "" }; - self.file_path_by_hex_str(&hex, suffix) - } - - /// Path to an index file of a tarball. - pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { - let (algorithm, hex) = tarball_integrity.to_hex(); - assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error - self.file_path_by_hex_str(&hex, "-index.json") - } - /// Path to the temporary directory inside the store. pub fn tmp(&self) -> PathBuf { self.v3().join("tmp") @@ -85,7 +70,6 @@ mod tests { use super::*; use pipe_trait::Pipe; use pretty_assertions::assert_eq; - use sha2::{Digest, Sha512}; #[test] fn file_path_by_head_tail() { @@ -98,31 +82,6 @@ mod tests { assert_eq!(&received, &expected); } - #[test] - fn cas_file_path() { - fn case(file_content: &str, executable: bool, expected: &str) { - eprintln!("CASE: {file_content:?}, {executable:?}"); - let store_dir = StoreDir::new("STORE_DIR"); - let file_hash = Sha512::digest(file_content); - eprintln!("file_hash = {file_hash:x}"); - let received = store_dir.cas_file_path(file_hash, executable); - let expected: PathBuf = expected.split('/').collect(); - assert_eq!(&received, &expected); - } - - case( - "hello world", - false, - "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", - ); - - case( - "hello world", - true, - "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", - ); - } - #[test] fn tmp() { let received = StoreDir::new("/home/user/.local/share/pnpm/store").tmp(); From 4bc61d02ebd327db1d02d6c4c46245e582ad5421 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:24:10 +0700 Subject: [PATCH 058/102] fix: clippy on windows --- crates/fs/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 9fdac655d..79c482596 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -1,4 +1,4 @@ -use std::{fs, io, path::Path}; +use std::{io, path::Path}; /// Create a symlink to a directory. /// @@ -15,9 +15,12 @@ pub fn symlink_dir(original: &Path, link: &Path) -> io::Result<()> { pub fn make_file_executable(file_path: &Path) -> io::Result<()> { #[cfg(unix)] return { - use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + use std::{ + fs::{set_permissions, Permissions}, + os::unix::fs::PermissionsExt, + }; let permissions = Permissions::from_mode(0o777); - fs::set_permissions(file_path, permissions) + set_permissions(file_path, permissions) }; #[cfg(windows)] From 184cf1385efeac1fedeb9825f6701f0c8d0b5d86 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:32:38 +0700 Subject: [PATCH 059/102] test: tarball_index_file_path --- crates/store-dir/src/index_file.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index 3d7498252..87b589f5f 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -53,3 +53,20 @@ impl StoreDir { .map_err(WriteTarballIndexFileError::WriteFile) } } + +#[cfg(test)] +mod tests { + use super::*; + use ssri::IntegrityOpts; + + #[test] + fn tarball_index_file_path() { + let store_dir = StoreDir::new("STORE_DIR"); + let tarball_integrity = + IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(b"TARBALL CONTENT").result(); + let received = store_dir.tarball_index_file_path(&tarball_integrity); + let expected = "STORE_DIR/v3/files/bc/d60799116ebef60071b9f2c7dafd7e2a4e1b366e341f750b2de52dd6995ab409b530f31b2b0a56c168a808a977156c3f5f13b026fb117d36314d8077f8733f-index.json"; + let expected: PathBuf = expected.split('/').collect(); + assert_eq!(&received, &expected); + } +} From d31bd5ed7cb4bdffd84fccb164d4a02036e4e451 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:35:03 +0700 Subject: [PATCH 060/102] fix: clippy on windows --- crates/fs/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 79c482596..6a75b5379 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -24,5 +24,8 @@ pub fn make_file_executable(file_path: &Path) -> io::Result<()> { }; #[cfg(windows)] - return Ok(()); + return { + drop(file_path); + Ok(()) + }; } From ac01df08cf94c123351bec166272b962779f37ba Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:38:01 +0700 Subject: [PATCH 061/102] clippy: fix on windows --- crates/fs/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 6a75b5379..2b15c490c 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -12,6 +12,7 @@ pub fn symlink_dir(original: &Path, link: &Path) -> io::Result<()> { /// Set file mode to 777 on POSIX platforms such as Linux or macOS, /// or do nothing on Windows. +#[cfg_attr(windows, allow(unused))] pub fn make_file_executable(file_path: &Path) -> io::Result<()> { #[cfg(unix)] return { @@ -24,8 +25,5 @@ pub fn make_file_executable(file_path: &Path) -> io::Result<()> { }; #[cfg(windows)] - return { - drop(file_path); - Ok(()) - }; + return Ok(()); } From a757a85f36a23621f97a5dd3461eeebe9f4c8147 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Tue, 31 Oct 2023 21:51:59 +0700 Subject: [PATCH 062/102] fix: only make file executable when executable --- crates/store-dir/src/cas_file.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index b7b1be964..a258d06a7 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -31,7 +31,10 @@ impl StoreDir { let file_path = self.cas_file_path(file_hash, executable); ensure_file(&file_path, buffer).map_err(WriteCasFileError::WriteFile)?; - make_file_executable(&file_path).expect("make the file executable"); // TODO: propagate this error + if executable { + // TODO: propagate this error + make_file_executable(&file_path).expect("make the file executable"); + } Ok((file_path, file_hash)) } From 2156131c75edd7505668604e357f586767c4dec5 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 01:19:44 +0700 Subject: [PATCH 063/102] feat: TarballIndexFileAttrs::checked_at --- crates/store-dir/src/index_file.rs | 3 ++- crates/tarball/src/lib.rs | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index 87b589f5f..fbca57f38 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -26,7 +26,8 @@ pub struct TarballIndex { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct TarballIndexFileAttrs { - // pub checked_at: ??? + #[serde(skip_serializing_if = "Option::is_none")] + pub checked_at: Option, pub integrity: String, pub mode: u32, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 49af7697e..164434fa0 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -4,6 +4,7 @@ use std::{ io::{Cursor, Read}, path::PathBuf, sync::Arc, + time::UNIX_EPOCH, }; use base64::{engine::general_purpose::STANDARD as BASE64_STD, Engine}; @@ -257,9 +258,11 @@ impl<'a> DownloadTarballToStore<'a> { panic!("Unexpected error: {previous:?} shouldn't collide"); } + let checked_at = UNIX_EPOCH.elapsed().ok().map(|x| x.as_millis()); let file_size = entry.header().size().ok(); let file_integrity = format!("sha512-{}", BASE64_STD.encode(file_hash)); let file_attrs = TarballIndexFileAttrs { + checked_at, integrity: file_integrity, mode: file_mode, size: file_size, From d8c09daf0a5ec2ba18f20f29fd7442bf227b9148 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 02:02:37 +0700 Subject: [PATCH 064/102] test(cli): change dir to a sub dir --- crates/cli/tests/_utils.rs | 10 +++++----- crates/cli/tests/add.rs | 35 ++++++++++++++++++++------------- crates/cli/tests/init.rs | 16 +++++++++------ crates/cli/tests/install.rs | 22 +++++++++++---------- crates/cli/tests/store.rs | 10 ++++++---- crates/testing-utils/src/bin.rs | 12 ++++++++++- 6 files changed, 65 insertions(+), 40 deletions(-) diff --git a/crates/cli/tests/_utils.rs b/crates/cli/tests/_utils.rs index 60a7de59d..cd5859fe2 100644 --- a/crates/cli/tests/_utils.rs +++ b/crates/cli/tests/_utils.rs @@ -1,15 +1,15 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; -use pacquet_testing_utils::bin::pacquet_with_temp_cwd; -use std::ffi::OsStr; +use pacquet_testing_utils::bin::pacquet_with_temp_sub_cwd; +use std::{ffi::OsStr, path::PathBuf}; use tempfile::TempDir; -pub fn exec_pacquet_in_temp_cwd(args: Args) -> TempDir +pub fn exec_pacquet_in_temp_cwd(args: Args) -> (TempDir, PathBuf) where Args: IntoIterator, Args::Item: AsRef, { - let (command, current_dir) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_sub_cwd(); command.with_args(args).assert().success(); - current_dir + (root, workspace) } diff --git a/crates/cli/tests/add.rs b/crates/cli/tests/add.rs index 22e7f28bf..0a5d5f9a8 100644 --- a/crates/cli/tests/add.rs +++ b/crates/cli/tests/add.rs @@ -8,17 +8,17 @@ use std::{env, fs}; #[test] fn should_install_all_dependencies() { - let dir = exec_pacquet_in_temp_cwd(["add", "is-even"]); + let (root, workspace) = exec_pacquet_in_temp_cwd(["add", "is-even"]); eprintln!("Directory list"); - insta::assert_debug_snapshot!(get_all_folders(dir.path())); + insta::assert_debug_snapshot!(get_all_folders(&workspace)); - let manifest_path = dir.path().join("package.json"); + let manifest_path = workspace.join("package.json"); eprintln!("Ensure the manifest file ({manifest_path:?}) exists"); assert!(manifest_path.exists()); - let virtual_store_dir = dir.path().join("node_modules").join(".pacquet"); + let virtual_store_dir = workspace.join("node_modules").join(".pacquet"); eprintln!("Ensure virtual store dir ({virtual_store_dir:?}) exists"); assert!(virtual_store_dir.exists()); @@ -34,22 +34,24 @@ fn should_install_all_dependencies() { eprintln!("Ensure that is-number does not have any dependencies"); let is_number_path = virtual_store_dir.join("is-number@3.0.0/node_modules"); assert_eq!(get_filenames_in_folder(&is_number_path), vec!["is-number", "kind-of"]); + + drop(root); // cleanup } #[test] #[cfg(unix)] pub fn should_symlink_correctly() { - let dir = exec_pacquet_in_temp_cwd(["add", "is-odd"]); + let (root, workspace) = exec_pacquet_in_temp_cwd(["add", "is-odd"]); eprintln!("Directory list"); - insta::assert_debug_snapshot!(get_all_folders(dir.path())); + insta::assert_debug_snapshot!(get_all_folders(&workspace)); - let manifest_path = dir.path().join("package.json"); + let manifest_path = workspace.join("package.json"); eprintln!("Ensure the manifest file ({manifest_path:?}) exists"); assert!(manifest_path.exists()); - let virtual_store_dir = dir.path().join("node_modules").join(".pacquet"); + let virtual_store_dir = workspace.join("node_modules").join(".pacquet"); eprintln!("Ensure virtual store dir ({virtual_store_dir:?}) exists"); assert!(virtual_store_dir.exists()); @@ -59,30 +61,35 @@ pub fn should_symlink_correctly() { fs::read_link(virtual_store_dir.join("is-odd@3.0.1/node_modules/is-number")).unwrap(), fs::canonicalize(virtual_store_dir.join("is-number@6.0.0/node_modules/is-number")).unwrap(), ); + + drop(root); // cleanup } #[test] fn should_add_to_package_json() { - let dir = exec_pacquet_in_temp_cwd(["add", "is-odd"]); - let file = PackageManifest::from_path(dir.path().join("package.json")).unwrap(); + let (root, dir) = exec_pacquet_in_temp_cwd(["add", "is-odd"]); + let file = PackageManifest::from_path(dir.join("package.json")).unwrap(); eprintln!("Ensure is-odd is added to package.json#dependencies"); assert!(file.dependencies([DependencyGroup::Prod]).any(|(k, _)| k == "is-odd")); + drop(root); // cleanup } #[test] fn should_add_dev_dependency() { - let dir = exec_pacquet_in_temp_cwd(["add", "is-odd", "--save-dev"]); - let file = PackageManifest::from_path(dir.path().join("package.json")).unwrap(); + let (root, dir) = exec_pacquet_in_temp_cwd(["add", "is-odd", "--save-dev"]); + let file = PackageManifest::from_path(dir.join("package.json")).unwrap(); eprintln!("Ensure is-odd is added to package.json#devDependencies"); assert!(file.dependencies([DependencyGroup::Dev]).any(|(k, _)| k == "is-odd")); + drop(root); // cleanup } #[test] fn should_add_peer_dependency() { - let dir = exec_pacquet_in_temp_cwd(["add", "is-odd", "--save-peer"]); - let file = PackageManifest::from_path(dir.path().join("package.json")).unwrap(); + let (root, dir) = exec_pacquet_in_temp_cwd(["add", "is-odd", "--save-peer"]); + let file = PackageManifest::from_path(dir.join("package.json")).unwrap(); eprintln!("Ensure is-odd is added to package.json#devDependencies"); assert!(file.dependencies([DependencyGroup::Dev]).any(|(k, _)| k == "is-odd")); eprintln!("Ensure is-odd is added to package.json#peerDependencies"); assert!(file.dependencies([DependencyGroup::Peer]).any(|(k, _)| k == "is-odd")); + drop(root); // cleanup } diff --git a/crates/cli/tests/init.rs b/crates/cli/tests/init.rs index 56d12b28c..867e34205 100644 --- a/crates/cli/tests/init.rs +++ b/crates/cli/tests/init.rs @@ -2,15 +2,15 @@ pub mod _utils; pub use _utils::*; use command_extra::CommandExtra; -use pacquet_testing_utils::{bin::pacquet_with_temp_cwd, fs::get_filenames_in_folder}; +use pacquet_testing_utils::{bin::pacquet_with_temp_sub_cwd, fs::get_filenames_in_folder}; use pretty_assertions::assert_eq; use std::{env, fs}; #[test] fn should_create_package_json() { - let dir = exec_pacquet_in_temp_cwd(["init"]); + let (root, workspace) = exec_pacquet_in_temp_cwd(["init"]); - let manifest_path = dir.path().join("package.json"); + let manifest_path = workspace.join("package.json"); dbg!(&manifest_path); eprintln!("Content of package.json"); @@ -18,14 +18,16 @@ fn should_create_package_json() { insta::assert_snapshot!(package_json_content); eprintln!("Created files"); - assert_eq!(get_filenames_in_folder(dir.path()), ["package.json"]); + assert_eq!(get_filenames_in_folder(&workspace), ["package.json"]); + + drop(root); // cleanup } #[test] fn should_throw_on_existing_file() { - let (command, dir) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_sub_cwd(); - let manifest_path = dir.path().join("package.json"); + let manifest_path = workspace.join("package.json"); dbg!(&manifest_path); eprintln!("Creating package.json..."); @@ -40,4 +42,6 @@ fn should_throw_on_existing_file() { eprintln!("Stderr"); insta::assert_snapshot!(String::from_utf8_lossy(&output.stderr).trim_end()); + + drop(root); // cleanup } diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 21bf5af1f..d19784437 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -1,17 +1,17 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; use pacquet_testing_utils::{ - bin::pacquet_with_temp_cwd, + bin::pacquet_with_temp_sub_cwd, fs::{get_all_folders, is_symlink_or_junction}, }; use std::fs; #[test] fn should_install_dependencies() { - let (command, dir) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_sub_cwd(); eprintln!("Creating package.json..."); - let manifest_path = dir.path().join("package.json"); + let manifest_path = workspace.join("package.json"); let package_json_content = serde_json::json!({ "dependencies": { "is-odd": "3.0.1", @@ -26,19 +26,21 @@ fn should_install_dependencies() { command.with_arg("install").assert().success(); eprintln!("Make sure the package is installed"); - assert!(is_symlink_or_junction(&dir.path().join("node_modules/is-odd")).unwrap()); - assert!(dir.path().join("node_modules/.pacquet/is-odd@3.0.1").exists()); + assert!(is_symlink_or_junction(&workspace.join("node_modules/is-odd")).unwrap()); + assert!(workspace.join("node_modules/.pacquet/is-odd@3.0.1").exists()); eprintln!("Make sure it installs direct dependencies"); - assert!(!dir.path().join("node_modules/is-number").exists()); - assert!(dir.path().join("node_modules/.pacquet/is-number@6.0.0").exists()); + assert!(!workspace.join("node_modules/is-number").exists()); + assert!(workspace.join("node_modules/.pacquet/is-number@6.0.0").exists()); eprintln!("Make sure we install dev-dependencies as well"); assert!( - is_symlink_or_junction(&dir.path().join("node_modules/fast-decode-uri-component")).unwrap() + is_symlink_or_junction(&workspace.join("node_modules/fast-decode-uri-component")).unwrap() ); - assert!(dir.path().join("node_modules/.pacquet/fast-decode-uri-component@1.0.1").is_dir()); + assert!(workspace.join("node_modules/.pacquet/fast-decode-uri-component@1.0.1").is_dir()); eprintln!("Directory list"); - insta::assert_debug_snapshot!(get_all_folders(dir.path())); + insta::assert_debug_snapshot!(get_all_folders(&workspace)); + + drop(root); // cleanup } diff --git a/crates/cli/tests/store.rs b/crates/cli/tests/store.rs index fabd48022..e313da219 100644 --- a/crates/cli/tests/store.rs +++ b/crates/cli/tests/store.rs @@ -1,5 +1,5 @@ use command_extra::CommandExtra; -use pacquet_testing_utils::bin::pacquet_with_temp_cwd; +use pacquet_testing_utils::bin::pacquet_with_temp_sub_cwd; use pipe_trait::Pipe; use pretty_assertions::assert_eq; use std::{ @@ -20,10 +20,10 @@ fn canonicalize(path: &Path) -> PathBuf { #[test] fn store_path_should_return_store_dir_from_npmrc() { - let (command, dir) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_sub_cwd(); eprintln!("Creating .npmrc..."); - fs::write(dir.path().join(".npmrc"), "store-dir=foo/bar").expect("write to .npmrc"); + fs::write(workspace.join(".npmrc"), "store-dir=foo/bar").expect("write to .npmrc"); eprintln!("Executing pacquet store path..."); let output = command.with_args(["store", "path"]).output().expect("run pacquet store path"); @@ -36,6 +36,8 @@ fn store_path_should_return_store_dir_from_npmrc() { let normalize = |path: &str| path.replace('\\', "/"); assert_eq!( String::from_utf8_lossy(&output.stdout).trim_end().pipe(normalize), - dir.path().pipe(canonicalize).join("foo/bar").to_string_lossy().pipe_as_ref(normalize), + canonicalize(&workspace).join("foo/bar").to_string_lossy().pipe_as_ref(normalize), ); + + drop(root); // cleanup } diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index c51ceb86e..1cf162bb0 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -1,6 +1,6 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; -use std::process::Command; +use std::{fs, path::PathBuf, process::Command}; use tempfile::{tempdir, TempDir}; pub fn pacquet_with_temp_cwd() -> (Command, TempDir) { @@ -10,3 +10,13 @@ pub fn pacquet_with_temp_cwd() -> (Command, TempDir) { .with_current_dir(current_dir.path()); (command, current_dir) } + +pub fn pacquet_with_temp_sub_cwd() -> (Command, TempDir, PathBuf) { + let root = tempdir().expect("create temporary directory"); + let workspace = root.path().join("workspace"); + fs::create_dir(&workspace).expect("create temporary workspace for pacquet"); + let command = Command::cargo_bin("pacquet") + .expect("find the pacquet binary") + .with_current_dir(&workspace); + (command, root, workspace) +} From 0cae50e438ad4792eb425cb6b7bbb57157bb2d3b Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 02:11:12 +0700 Subject: [PATCH 065/102] feat: add .npmrc to every integration tests --- Cargo.lock | 1 + crates/cli/tests/_utils.rs | 4 ++-- crates/cli/tests/init.rs | 6 +++--- crates/cli/tests/install.rs | 4 ++-- crates/cli/tests/store.rs | 4 ++-- crates/testing-utils/Cargo.toml | 9 +++++---- crates/testing-utils/src/bin.rs | 11 ++++++++++- 7 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d5675749..58a5cd4f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1510,6 +1510,7 @@ dependencies = [ "command-extra", "junction", "tempfile", + "text-block-macros", "walkdir", ] diff --git a/crates/cli/tests/_utils.rs b/crates/cli/tests/_utils.rs index cd5859fe2..3ffcacb60 100644 --- a/crates/cli/tests/_utils.rs +++ b/crates/cli/tests/_utils.rs @@ -1,6 +1,6 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; -use pacquet_testing_utils::bin::pacquet_with_temp_sub_cwd; +use pacquet_testing_utils::bin::pacquet_with_temp_npmrc; use std::{ffi::OsStr, path::PathBuf}; use tempfile::TempDir; @@ -9,7 +9,7 @@ where Args: IntoIterator, Args::Item: AsRef, { - let (command, root, workspace) = pacquet_with_temp_sub_cwd(); + let (command, root, workspace) = pacquet_with_temp_npmrc(); command.with_args(args).assert().success(); (root, workspace) } diff --git a/crates/cli/tests/init.rs b/crates/cli/tests/init.rs index 867e34205..cec4e6689 100644 --- a/crates/cli/tests/init.rs +++ b/crates/cli/tests/init.rs @@ -2,7 +2,7 @@ pub mod _utils; pub use _utils::*; use command_extra::CommandExtra; -use pacquet_testing_utils::{bin::pacquet_with_temp_sub_cwd, fs::get_filenames_in_folder}; +use pacquet_testing_utils::{bin::pacquet_with_temp_npmrc, fs::get_filenames_in_folder}; use pretty_assertions::assert_eq; use std::{env, fs}; @@ -18,14 +18,14 @@ fn should_create_package_json() { insta::assert_snapshot!(package_json_content); eprintln!("Created files"); - assert_eq!(get_filenames_in_folder(&workspace), ["package.json"]); + assert_eq!(get_filenames_in_folder(&workspace), [".npmrc", "package.json"]); drop(root); // cleanup } #[test] fn should_throw_on_existing_file() { - let (command, root, workspace) = pacquet_with_temp_sub_cwd(); + let (command, root, workspace) = pacquet_with_temp_npmrc(); let manifest_path = workspace.join("package.json"); dbg!(&manifest_path); diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index d19784437..7c7de8220 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -1,14 +1,14 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; use pacquet_testing_utils::{ - bin::pacquet_with_temp_sub_cwd, + bin::pacquet_with_temp_npmrc, fs::{get_all_folders, is_symlink_or_junction}, }; use std::fs; #[test] fn should_install_dependencies() { - let (command, root, workspace) = pacquet_with_temp_sub_cwd(); + let (command, root, workspace) = pacquet_with_temp_npmrc(); eprintln!("Creating package.json..."); let manifest_path = workspace.join("package.json"); diff --git a/crates/cli/tests/store.rs b/crates/cli/tests/store.rs index e313da219..4ad04997a 100644 --- a/crates/cli/tests/store.rs +++ b/crates/cli/tests/store.rs @@ -1,5 +1,5 @@ use command_extra::CommandExtra; -use pacquet_testing_utils::bin::pacquet_with_temp_sub_cwd; +use pacquet_testing_utils::bin::pacquet_with_temp_npmrc; use pipe_trait::Pipe; use pretty_assertions::assert_eq; use std::{ @@ -20,7 +20,7 @@ fn canonicalize(path: &Path) -> PathBuf { #[test] fn store_path_should_return_store_dir_from_npmrc() { - let (command, root, workspace) = pacquet_with_temp_sub_cwd(); + let (command, root, workspace) = pacquet_with_temp_npmrc(); eprintln!("Creating .npmrc..."); fs::write(workspace.join(".npmrc"), "store-dir=foo/bar").expect("write to .npmrc"); diff --git a/crates/testing-utils/Cargo.toml b/crates/testing-utils/Cargo.toml index 3566c60a7..db0a226b7 100644 --- a/crates/testing-utils/Cargo.toml +++ b/crates/testing-utils/Cargo.toml @@ -11,10 +11,11 @@ license.workspace = true repository.workspace = true [dependencies] -assert_cmd = { workspace = true } -command-extra = { workspace = true } -tempfile = { workspace = true } -walkdir = { workspace = true } +assert_cmd = { workspace = true } +command-extra = { workspace = true } +tempfile = { workspace = true } +walkdir = { workspace = true } +text-block-macros = { workspace = true } [target.'cfg(windows)'.dependencies] junction = { workspace = true } diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index 1cf162bb0..e6d1e012a 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -2,6 +2,7 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; use std::{fs, path::PathBuf, process::Command}; use tempfile::{tempdir, TempDir}; +use text_block_macros::text_block_fnl; pub fn pacquet_with_temp_cwd() -> (Command, TempDir) { let current_dir = tempdir().expect("create temporary working directory for pacquet"); @@ -11,10 +12,18 @@ pub fn pacquet_with_temp_cwd() -> (Command, TempDir) { (command, current_dir) } -pub fn pacquet_with_temp_sub_cwd() -> (Command, TempDir, PathBuf) { +pub fn pacquet_with_temp_npmrc() -> (Command, TempDir, PathBuf) { let root = tempdir().expect("create temporary directory"); let workspace = root.path().join("workspace"); fs::create_dir(&workspace).expect("create temporary workspace for pacquet"); + fs::write( + workspace.join(".npmrc"), + text_block_fnl! { + "store-dir=../pacquet-store" + "cache-dir=../pacquet-cache" + }, + ) + .expect("write to .npmrc"); let command = Command::cargo_bin("pacquet") .expect("find the pacquet binary") .with_current_dir(&workspace); From 1c2e14a2b8b890a8210f13d0d8c4b3b8fd304bb6 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 02:13:04 +0700 Subject: [PATCH 066/102] refactor: remove unused, reuse available name --- crates/cli/tests/_utils.rs | 4 ++-- crates/cli/tests/init.rs | 4 ++-- crates/cli/tests/install.rs | 4 ++-- crates/cli/tests/store.rs | 4 ++-- crates/testing-utils/src/bin.rs | 10 +--------- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/crates/cli/tests/_utils.rs b/crates/cli/tests/_utils.rs index 3ffcacb60..18d926963 100644 --- a/crates/cli/tests/_utils.rs +++ b/crates/cli/tests/_utils.rs @@ -1,6 +1,6 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; -use pacquet_testing_utils::bin::pacquet_with_temp_npmrc; +use pacquet_testing_utils::bin::pacquet_with_temp_cwd; use std::{ffi::OsStr, path::PathBuf}; use tempfile::TempDir; @@ -9,7 +9,7 @@ where Args: IntoIterator, Args::Item: AsRef, { - let (command, root, workspace) = pacquet_with_temp_npmrc(); + let (command, root, workspace) = pacquet_with_temp_cwd(); command.with_args(args).assert().success(); (root, workspace) } diff --git a/crates/cli/tests/init.rs b/crates/cli/tests/init.rs index cec4e6689..fc8a15d1c 100644 --- a/crates/cli/tests/init.rs +++ b/crates/cli/tests/init.rs @@ -2,7 +2,7 @@ pub mod _utils; pub use _utils::*; use command_extra::CommandExtra; -use pacquet_testing_utils::{bin::pacquet_with_temp_npmrc, fs::get_filenames_in_folder}; +use pacquet_testing_utils::{bin::pacquet_with_temp_cwd, fs::get_filenames_in_folder}; use pretty_assertions::assert_eq; use std::{env, fs}; @@ -25,7 +25,7 @@ fn should_create_package_json() { #[test] fn should_throw_on_existing_file() { - let (command, root, workspace) = pacquet_with_temp_npmrc(); + let (command, root, workspace) = pacquet_with_temp_cwd(); let manifest_path = workspace.join("package.json"); dbg!(&manifest_path); diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 7c7de8220..f55ff1681 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -1,14 +1,14 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; use pacquet_testing_utils::{ - bin::pacquet_with_temp_npmrc, + bin::pacquet_with_temp_cwd, fs::{get_all_folders, is_symlink_or_junction}, }; use std::fs; #[test] fn should_install_dependencies() { - let (command, root, workspace) = pacquet_with_temp_npmrc(); + let (command, root, workspace) = pacquet_with_temp_cwd(); eprintln!("Creating package.json..."); let manifest_path = workspace.join("package.json"); diff --git a/crates/cli/tests/store.rs b/crates/cli/tests/store.rs index 4ad04997a..fa280d8ec 100644 --- a/crates/cli/tests/store.rs +++ b/crates/cli/tests/store.rs @@ -1,5 +1,5 @@ use command_extra::CommandExtra; -use pacquet_testing_utils::bin::pacquet_with_temp_npmrc; +use pacquet_testing_utils::bin::pacquet_with_temp_cwd; use pipe_trait::Pipe; use pretty_assertions::assert_eq; use std::{ @@ -20,7 +20,7 @@ fn canonicalize(path: &Path) -> PathBuf { #[test] fn store_path_should_return_store_dir_from_npmrc() { - let (command, root, workspace) = pacquet_with_temp_npmrc(); + let (command, root, workspace) = pacquet_with_temp_cwd(); eprintln!("Creating .npmrc..."); fs::write(workspace.join(".npmrc"), "store-dir=foo/bar").expect("write to .npmrc"); diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index e6d1e012a..3f1eff5da 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -4,15 +4,7 @@ use std::{fs, path::PathBuf, process::Command}; use tempfile::{tempdir, TempDir}; use text_block_macros::text_block_fnl; -pub fn pacquet_with_temp_cwd() -> (Command, TempDir) { - let current_dir = tempdir().expect("create temporary working directory for pacquet"); - let command = Command::cargo_bin("pacquet") - .expect("find the pacquet binary") - .with_current_dir(current_dir.path()); - (command, current_dir) -} - -pub fn pacquet_with_temp_npmrc() -> (Command, TempDir, PathBuf) { +pub fn pacquet_with_temp_cwd() -> (Command, TempDir, PathBuf) { let root = tempdir().expect("create temporary directory"); let workspace = root.path().join("workspace"); fs::create_dir(&workspace).expect("create temporary workspace for pacquet"); From bc55d2de50fe56be5278fee1696698f1f9e9c1d8 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 02:16:54 +0700 Subject: [PATCH 067/102] feat(test): create_npmrc parameter --- crates/cli/tests/_utils.rs | 4 ++-- crates/cli/tests/add.rs | 10 +++++----- crates/cli/tests/init.rs | 6 +++--- crates/cli/tests/install.rs | 2 +- crates/cli/tests/store.rs | 2 +- crates/testing-utils/src/bin.rs | 20 +++++++++++--------- 6 files changed, 23 insertions(+), 21 deletions(-) diff --git a/crates/cli/tests/_utils.rs b/crates/cli/tests/_utils.rs index 18d926963..c9079d736 100644 --- a/crates/cli/tests/_utils.rs +++ b/crates/cli/tests/_utils.rs @@ -4,12 +4,12 @@ use pacquet_testing_utils::bin::pacquet_with_temp_cwd; use std::{ffi::OsStr, path::PathBuf}; use tempfile::TempDir; -pub fn exec_pacquet_in_temp_cwd(args: Args) -> (TempDir, PathBuf) +pub fn exec_pacquet_in_temp_cwd(create_npmrc: bool, args: Args) -> (TempDir, PathBuf) where Args: IntoIterator, Args::Item: AsRef, { - let (command, root, workspace) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_cwd(create_npmrc); command.with_args(args).assert().success(); (root, workspace) } diff --git a/crates/cli/tests/add.rs b/crates/cli/tests/add.rs index 0a5d5f9a8..2f041cebc 100644 --- a/crates/cli/tests/add.rs +++ b/crates/cli/tests/add.rs @@ -8,7 +8,7 @@ use std::{env, fs}; #[test] fn should_install_all_dependencies() { - let (root, workspace) = exec_pacquet_in_temp_cwd(["add", "is-even"]); + let (root, workspace) = exec_pacquet_in_temp_cwd(true, ["add", "is-even"]); eprintln!("Directory list"); insta::assert_debug_snapshot!(get_all_folders(&workspace)); @@ -41,7 +41,7 @@ fn should_install_all_dependencies() { #[test] #[cfg(unix)] pub fn should_symlink_correctly() { - let (root, workspace) = exec_pacquet_in_temp_cwd(["add", "is-odd"]); + let (root, workspace) = exec_pacquet_in_temp_cwd(true, ["add", "is-odd"]); eprintln!("Directory list"); insta::assert_debug_snapshot!(get_all_folders(&workspace)); @@ -67,7 +67,7 @@ pub fn should_symlink_correctly() { #[test] fn should_add_to_package_json() { - let (root, dir) = exec_pacquet_in_temp_cwd(["add", "is-odd"]); + let (root, dir) = exec_pacquet_in_temp_cwd(true, ["add", "is-odd"]); let file = PackageManifest::from_path(dir.join("package.json")).unwrap(); eprintln!("Ensure is-odd is added to package.json#dependencies"); assert!(file.dependencies([DependencyGroup::Prod]).any(|(k, _)| k == "is-odd")); @@ -76,7 +76,7 @@ fn should_add_to_package_json() { #[test] fn should_add_dev_dependency() { - let (root, dir) = exec_pacquet_in_temp_cwd(["add", "is-odd", "--save-dev"]); + let (root, dir) = exec_pacquet_in_temp_cwd(true, ["add", "is-odd", "--save-dev"]); let file = PackageManifest::from_path(dir.join("package.json")).unwrap(); eprintln!("Ensure is-odd is added to package.json#devDependencies"); assert!(file.dependencies([DependencyGroup::Dev]).any(|(k, _)| k == "is-odd")); @@ -85,7 +85,7 @@ fn should_add_dev_dependency() { #[test] fn should_add_peer_dependency() { - let (root, dir) = exec_pacquet_in_temp_cwd(["add", "is-odd", "--save-peer"]); + let (root, dir) = exec_pacquet_in_temp_cwd(true, ["add", "is-odd", "--save-peer"]); let file = PackageManifest::from_path(dir.join("package.json")).unwrap(); eprintln!("Ensure is-odd is added to package.json#devDependencies"); assert!(file.dependencies([DependencyGroup::Dev]).any(|(k, _)| k == "is-odd")); diff --git a/crates/cli/tests/init.rs b/crates/cli/tests/init.rs index fc8a15d1c..2e93a3520 100644 --- a/crates/cli/tests/init.rs +++ b/crates/cli/tests/init.rs @@ -8,7 +8,7 @@ use std::{env, fs}; #[test] fn should_create_package_json() { - let (root, workspace) = exec_pacquet_in_temp_cwd(["init"]); + let (root, workspace) = exec_pacquet_in_temp_cwd(false, ["init"]); let manifest_path = workspace.join("package.json"); dbg!(&manifest_path); @@ -18,14 +18,14 @@ fn should_create_package_json() { insta::assert_snapshot!(package_json_content); eprintln!("Created files"); - assert_eq!(get_filenames_in_folder(&workspace), [".npmrc", "package.json"]); + assert_eq!(get_filenames_in_folder(&workspace), ["package.json"]); drop(root); // cleanup } #[test] fn should_throw_on_existing_file() { - let (command, root, workspace) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_cwd(false); let manifest_path = workspace.join("package.json"); dbg!(&manifest_path); diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index f55ff1681..f0f2412f8 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -8,7 +8,7 @@ use std::fs; #[test] fn should_install_dependencies() { - let (command, root, workspace) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_cwd(true); eprintln!("Creating package.json..."); let manifest_path = workspace.join("package.json"); diff --git a/crates/cli/tests/store.rs b/crates/cli/tests/store.rs index fa280d8ec..ed101269f 100644 --- a/crates/cli/tests/store.rs +++ b/crates/cli/tests/store.rs @@ -20,7 +20,7 @@ fn canonicalize(path: &Path) -> PathBuf { #[test] fn store_path_should_return_store_dir_from_npmrc() { - let (command, root, workspace) = pacquet_with_temp_cwd(); + let (command, root, workspace) = pacquet_with_temp_cwd(false); eprintln!("Creating .npmrc..."); fs::write(workspace.join(".npmrc"), "store-dir=foo/bar").expect("write to .npmrc"); diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index 3f1eff5da..a81cbfcb5 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -4,18 +4,20 @@ use std::{fs, path::PathBuf, process::Command}; use tempfile::{tempdir, TempDir}; use text_block_macros::text_block_fnl; -pub fn pacquet_with_temp_cwd() -> (Command, TempDir, PathBuf) { +pub fn pacquet_with_temp_cwd(create_npmrc: bool) -> (Command, TempDir, PathBuf) { let root = tempdir().expect("create temporary directory"); let workspace = root.path().join("workspace"); fs::create_dir(&workspace).expect("create temporary workspace for pacquet"); - fs::write( - workspace.join(".npmrc"), - text_block_fnl! { - "store-dir=../pacquet-store" - "cache-dir=../pacquet-cache" - }, - ) - .expect("write to .npmrc"); + if create_npmrc { + fs::write( + workspace.join(".npmrc"), + text_block_fnl! { + "store-dir=../pacquet-store" + "cache-dir=../pacquet-cache" + }, + ) + .expect("write to .npmrc"); + } let command = Command::cargo_bin("pacquet") .expect("find the pacquet binary") .with_current_dir(&workspace); From 81668839408938980e703f13fca865f372a755c9 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 02:27:12 +0700 Subject: [PATCH 068/102] test: snapshot the store --- crates/cli/tests/install.rs | 8 ++- .../install__should_install_dependencies.snap | 70 ++++++++++++++----- crates/testing-utils/src/fs.rs | 24 +++++++ 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index f0f2412f8..28855e499 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -2,7 +2,7 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; use pacquet_testing_utils::{ bin::pacquet_with_temp_cwd, - fs::{get_all_folders, is_symlink_or_junction}, + fs::{get_all_files, get_all_folders, is_symlink_or_junction}, }; use std::fs; @@ -39,8 +39,10 @@ fn should_install_dependencies() { ); assert!(workspace.join("node_modules/.pacquet/fast-decode-uri-component@1.0.1").is_dir()); - eprintln!("Directory list"); - insta::assert_debug_snapshot!(get_all_folders(&workspace)); + eprintln!("Snapshot"); + let workspace_folders = get_all_folders(&workspace); + let store_files = get_all_files(&root.path().join("pacquet-store")); + insta::assert_debug_snapshot!((workspace_folders, store_files)); drop(root); // cleanup } diff --git a/crates/cli/tests/snapshots/install__should_install_dependencies.snap b/crates/cli/tests/snapshots/install__should_install_dependencies.snap index a059e4940..e6000a49f 100644 --- a/crates/cli/tests/snapshots/install__should_install_dependencies.snap +++ b/crates/cli/tests/snapshots/install__should_install_dependencies.snap @@ -1,21 +1,55 @@ --- source: crates/cli/tests/install.rs -assertion_line: 74 -expression: get_all_folders(dir.path()) +assertion_line: 45 +expression: "(workspace_folders, store_files)" --- -[ - "node_modules", - "node_modules/.pacquet", - "node_modules/.pacquet/fast-decode-uri-component@1.0.1", - "node_modules/.pacquet/fast-decode-uri-component@1.0.1/node_modules", - "node_modules/.pacquet/fast-decode-uri-component@1.0.1/node_modules/fast-decode-uri-component", - "node_modules/.pacquet/is-number@6.0.0", - "node_modules/.pacquet/is-number@6.0.0/node_modules", - "node_modules/.pacquet/is-number@6.0.0/node_modules/is-number", - "node_modules/.pacquet/is-odd@3.0.1", - "node_modules/.pacquet/is-odd@3.0.1/node_modules", - "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-number", - "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-odd", - "node_modules/fast-decode-uri-component", - "node_modules/is-odd", -] +( + [ + "node_modules", + "node_modules/.pacquet", + "node_modules/.pacquet/fast-decode-uri-component@1.0.1", + "node_modules/.pacquet/fast-decode-uri-component@1.0.1/node_modules", + "node_modules/.pacquet/fast-decode-uri-component@1.0.1/node_modules/fast-decode-uri-component", + "node_modules/.pacquet/is-number@6.0.0", + "node_modules/.pacquet/is-number@6.0.0/node_modules", + "node_modules/.pacquet/is-number@6.0.0/node_modules/is-number", + "node_modules/.pacquet/is-odd@3.0.1", + "node_modules/.pacquet/is-odd@3.0.1/node_modules", + "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-number", + "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-odd", + "node_modules/fast-decode-uri-component", + "node_modules/is-odd", + ], + [ + "01", + "01/d306b80f4b1678c655f3b0685db1f7a741fc746d251c00e925298ab89630a3167bde8c7d8d779b72770c41f9f302bb1e3f312672ce7aa8e6c156c319eeeada", + "17", + "17/54b7271ca0e30dc7d775986711a2b61b298200c12f867bde6f7e19d56047c9ab77a93304520fa1a31b0c3a4e6fb6b310f89dd8cb4d95a5c662807ef733f85f", + "1a", + "1a/5c01a1321e71a1c263fb83878cbba2686f71f571b42c2d40d93858bf1f4c6ff7f625125a68932c243e75ead4a9d2f18d9b22b3fe55c39fe6605a42b2e70620", + "37", + "37/8eb16c230fce354ef7e72722ee708cbefea9a481aa597b06fefc12f665ab60176cbf04d6bc159be42ccdbc51764d71516f86bb28a01b0e60c76a2753f1cf44", + "40", + "40/fe18b2c1f241075b732727110323f3e29f9d6c8765ff0222bccfa34784caa8abcd18ffbd4bbfab979e5478ace273a74ed3a0a22f0261334fa7c59e032edbde", + "54", + "54/7bcd683e2f9f7b0c2790c637b9231539947be923479534592d91886a56d6262fdc1d8ced24ac2cc3adfda139fe399dc7502d382c7afc536e3057a7cf11e6c0", + "55", + "55/dcbc7fff1ac1c4755f4cbf24ba21e0dcae062c802d3db937d2b6182f4554dd85d14529a0c9688eac825a345b63ab8b2e5d7699e2be07178df5bb8a072235eb", + "58", + "58/01e592e94aa4076c179180956f33f8262fa7157a06bfe636a6854bd3b9b2ddf44780ffa1405642ad8ece2ed6514e19a28178cbecdcf5608a3be710bbf3c0d4", + "60", + "60/ab54849d6cfd7b05076c3bc4d9baffe1f800f4d5e2d67c39c521a7a3d93b347ce5a8e45397102e40e2103652c543c8e7bb161c15d0a6e34ec1a9d11c168d2a", + "88", + "88/1f101d2f1886fc5f9ac5c8d25e5540908e140031169ea41a4a9941cf95fe2711c1292c1995609c6f174b3815cc0dcc6cd2a10339a348d39fe75bc77b53ba9e", + "9f", + "9f/cbe5dc86bc33bdb4f125559e767c3aaad94c7c4b21b80b694d9b0158ee0f10ee53b127c2bf05db9768afdaf1fb6778fd08a19412fb3c1adad655863e6c351d", + "ca", + "ca/65df16eb450ab63f90f107a8cc9ac82da9635408e813a6f6d3e13cfba78917b31b66652479ebba54b612566600b2ab1bc16b82d082c5f87803961a8e9e3931", + "e5", + "e5/c2514d2b0d3d6bedad24fdd75c55e780211cf0692c21f401201c2d7dfa990ad00188bd68b30c826aaa2a72ecad12a74bc03430384f933b1751035dd1b18ca2", + "f7", + "f7/02180f4465a777127ad6a211306dd564993c17f7d3d9401604df4bc657d8d888902de632a4cf55b551ddfb9f84ba8a900788773f03c77836143ef94e182c7c", + "ff", + "ff/d3f3fed65d7b489216916772c6180b5c1714ddde16e7e2c99ff5f8a4214bb8f884a0c1d06a1b1a8a4b0d6252ad85b533b1626f0dd8d474f2283f6cdfd69f6c", + ], +) diff --git a/crates/testing-utils/src/fs.rs b/crates/testing-utils/src/fs.rs index f6b029021..2f9233824 100644 --- a/crates/testing-utils/src/fs.rs +++ b/crates/testing-utils/src/fs.rs @@ -35,6 +35,30 @@ pub fn get_all_folders(root: &std::path::Path) -> Vec { files } +pub fn get_all_files(root: &std::path::Path) -> Vec { + let mut files = Vec::new(); + for entry in walkdir::WalkDir::new(root) { + let entry = entry.unwrap(); + let entry_path = entry.path(); + + // We need this mutation to ensure that both Unix and Windows paths resolves the same. + // TODO: Find a better way to do this? + let simple_path = entry_path + .strip_prefix(root) + .unwrap() + .components() + .map(|c| c.as_os_str().to_str().expect("invalid UTF-8")) + .collect::>() + .join("/"); + + if !simple_path.is_empty() { + files.push(simple_path); + } + } + files.sort(); + files +} + // Helper function to check if a path is a symlink or junction pub fn is_symlink_or_junction(path: &Path) -> io::Result { #[cfg(windows)] From ca61973502a416d7844b2226adf7c506e1ebc79f Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 02:33:12 +0700 Subject: [PATCH 069/102] test(cli/install): correct snapshot --- .../install__should_install_dependencies.snap | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/crates/cli/tests/snapshots/install__should_install_dependencies.snap b/crates/cli/tests/snapshots/install__should_install_dependencies.snap index e6000a49f..4a1e7e0bd 100644 --- a/crates/cli/tests/snapshots/install__should_install_dependencies.snap +++ b/crates/cli/tests/snapshots/install__should_install_dependencies.snap @@ -21,35 +21,42 @@ expression: "(workspace_folders, store_files)" "node_modules/is-odd", ], [ - "01", - "01/d306b80f4b1678c655f3b0685db1f7a741fc746d251c00e925298ab89630a3167bde8c7d8d779b72770c41f9f302bb1e3f312672ce7aa8e6c156c319eeeada", - "17", - "17/54b7271ca0e30dc7d775986711a2b61b298200c12f867bde6f7e19d56047c9ab77a93304520fa1a31b0c3a4e6fb6b310f89dd8cb4d95a5c662807ef733f85f", - "1a", - "1a/5c01a1321e71a1c263fb83878cbba2686f71f571b42c2d40d93858bf1f4c6ff7f625125a68932c243e75ead4a9d2f18d9b22b3fe55c39fe6605a42b2e70620", - "37", - "37/8eb16c230fce354ef7e72722ee708cbefea9a481aa597b06fefc12f665ab60176cbf04d6bc159be42ccdbc51764d71516f86bb28a01b0e60c76a2753f1cf44", - "40", - "40/fe18b2c1f241075b732727110323f3e29f9d6c8765ff0222bccfa34784caa8abcd18ffbd4bbfab979e5478ace273a74ed3a0a22f0261334fa7c59e032edbde", - "54", - "54/7bcd683e2f9f7b0c2790c637b9231539947be923479534592d91886a56d6262fdc1d8ced24ac2cc3adfda139fe399dc7502d382c7afc536e3057a7cf11e6c0", - "55", - "55/dcbc7fff1ac1c4755f4cbf24ba21e0dcae062c802d3db937d2b6182f4554dd85d14529a0c9688eac825a345b63ab8b2e5d7699e2be07178df5bb8a072235eb", - "58", - "58/01e592e94aa4076c179180956f33f8262fa7157a06bfe636a6854bd3b9b2ddf44780ffa1405642ad8ece2ed6514e19a28178cbecdcf5608a3be710bbf3c0d4", - "60", - "60/ab54849d6cfd7b05076c3bc4d9baffe1f800f4d5e2d67c39c521a7a3d93b347ce5a8e45397102e40e2103652c543c8e7bb161c15d0a6e34ec1a9d11c168d2a", - "88", - "88/1f101d2f1886fc5f9ac5c8d25e5540908e140031169ea41a4a9941cf95fe2711c1292c1995609c6f174b3815cc0dcc6cd2a10339a348d39fe75bc77b53ba9e", - "9f", - "9f/cbe5dc86bc33bdb4f125559e767c3aaad94c7c4b21b80b694d9b0158ee0f10ee53b127c2bf05db9768afdaf1fb6778fd08a19412fb3c1adad655863e6c351d", - "ca", - "ca/65df16eb450ab63f90f107a8cc9ac82da9635408e813a6f6d3e13cfba78917b31b66652479ebba54b612566600b2ab1bc16b82d082c5f87803961a8e9e3931", - "e5", - "e5/c2514d2b0d3d6bedad24fdd75c55e780211cf0692c21f401201c2d7dfa990ad00188bd68b30c826aaa2a72ecad12a74bc03430384f933b1751035dd1b18ca2", - "f7", - "f7/02180f4465a777127ad6a211306dd564993c17f7d3d9401604df4bc657d8d888902de632a4cf55b551ddfb9f84ba8a900788773f03c77836143ef94e182c7c", - "ff", - "ff/d3f3fed65d7b489216916772c6180b5c1714ddde16e7e2c99ff5f8a4214bb8f884a0c1d06a1b1a8a4b0d6252ad85b533b1626f0dd8d474f2283f6cdfd69f6c", + "v3", + "v3/files", + "v3/files/01", + "v3/files/01/d306b80f4b1678c655f3b0685db1f7a741fc746d251c00e925298ab89630a3167bde8c7d8d779b72770c41f9f302bb1e3f312672ce7aa8e6c156c319eeeada", + "v3/files/09", + "v3/files/09/0a6758fac3c263f5f923075d986d2ed26ff74ca2c957e5b86b17e62342564ae142d5374d01ec5163c6f7091d93d2e0779c8da4083d8d0128f7408169781630-index.json", + "v3/files/17", + "v3/files/17/54b7271ca0e30dc7d775986711a2b61b298200c12f867bde6f7e19d56047c9ab77a93304520fa1a31b0c3a4e6fb6b310f89dd8cb4d95a5c662807ef733f85f", + "v3/files/1a", + "v3/files/1a/5c01a1321e71a1c263fb83878cbba2686f71f571b42c2d40d93858bf1f4c6ff7f625125a68932c243e75ead4a9d2f18d9b22b3fe55c39fe6605a42b2e70620", + "v3/files/37", + "v3/files/37/8eb16c230fce354ef7e72722ee708cbefea9a481aa597b06fefc12f665ab60176cbf04d6bc159be42ccdbc51764d71516f86bb28a01b0e60c76a2753f1cf44", + "v3/files/40", + "v3/files/40/fe18b2c1f241075b732727110323f3e29f9d6c8765ff0222bccfa34784caa8abcd18ffbd4bbfab979e5478ace273a74ed3a0a22f0261334fa7c59e032edbde", + "v3/files/54", + "v3/files/54/7bcd683e2f9f7b0c2790c637b9231539947be923479534592d91886a56d6262fdc1d8ced24ac2cc3adfda139fe399dc7502d382c7afc536e3057a7cf11e6c0", + "v3/files/55", + "v3/files/55/dcbc7fff1ac1c4755f4cbf24ba21e0dcae062c802d3db937d2b6182f4554dd85d14529a0c9688eac825a345b63ab8b2e5d7699e2be07178df5bb8a072235eb", + "v3/files/58", + "v3/files/58/01e592e94aa4076c179180956f33f8262fa7157a06bfe636a6854bd3b9b2ddf44780ffa1405642ad8ece2ed6514e19a28178cbecdcf5608a3be710bbf3c0d4", + "v3/files/58/a80a5a0e5e531bd1646c16f05bdf6da1fb0174a1d9c2fede3e5f306cd4302c56049ddd5776bb5b3f32d9ffee4347b707a5a6a1d0f7a12e788d343d1d54c822-index.json", + "v3/files/5a", + "v3/files/5a/ed551de20b04af0a016254022499417f781a6384e39460ebfe77f1f2b08a5a14bb6d4a9dc1246063eaa1bda8499e66513ef7bcb1ab1d08cac3728c3f07c3ce-index.json", + "v3/files/60", + "v3/files/60/ab54849d6cfd7b05076c3bc4d9baffe1f800f4d5e2d67c39c521a7a3d93b347ce5a8e45397102e40e2103652c543c8e7bb161c15d0a6e34ec1a9d11c168d2a", + "v3/files/88", + "v3/files/88/1f101d2f1886fc5f9ac5c8d25e5540908e140031169ea41a4a9941cf95fe2711c1292c1995609c6f174b3815cc0dcc6cd2a10339a348d39fe75bc77b53ba9e", + "v3/files/9f", + "v3/files/9f/cbe5dc86bc33bdb4f125559e767c3aaad94c7c4b21b80b694d9b0158ee0f10ee53b127c2bf05db9768afdaf1fb6778fd08a19412fb3c1adad655863e6c351d", + "v3/files/ca", + "v3/files/ca/65df16eb450ab63f90f107a8cc9ac82da9635408e813a6f6d3e13cfba78917b31b66652479ebba54b612566600b2ab1bc16b82d082c5f87803961a8e9e3931", + "v3/files/e5", + "v3/files/e5/c2514d2b0d3d6bedad24fdd75c55e780211cf0692c21f401201c2d7dfa990ad00188bd68b30c826aaa2a72ecad12a74bc03430384f933b1751035dd1b18ca2", + "v3/files/f7", + "v3/files/f7/02180f4465a777127ad6a211306dd564993c17f7d3d9401604df4bc657d8d888902de632a4cf55b551ddfb9f84ba8a900788773f03c77836143ef94e182c7c", + "v3/files/ff", + "v3/files/ff/d3f3fed65d7b489216916772c6180b5c1714ddde16e7e2c99ff5f8a4214bb8f884a0c1d06a1b1a8a4b0d6252ad85b533b1626f0dd8d474f2283f6cdfd69f6c", ], ) From bc80817ee553c6cc41bc832370a509de59c02a1f Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 03:15:15 +0700 Subject: [PATCH 070/102] test: remove dirs from `get_all_files` --- .../install__should_install_dependencies.snap | 21 +------------------ crates/testing-utils/src/fs.rs | 4 ++++ 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/crates/cli/tests/snapshots/install__should_install_dependencies.snap b/crates/cli/tests/snapshots/install__should_install_dependencies.snap index 4a1e7e0bd..a1b28283d 100644 --- a/crates/cli/tests/snapshots/install__should_install_dependencies.snap +++ b/crates/cli/tests/snapshots/install__should_install_dependencies.snap @@ -1,6 +1,6 @@ --- source: crates/cli/tests/install.rs -assertion_line: 45 +assertion_line: 46 expression: "(workspace_folders, store_files)" --- ( @@ -21,42 +21,23 @@ expression: "(workspace_folders, store_files)" "node_modules/is-odd", ], [ - "v3", - "v3/files", - "v3/files/01", "v3/files/01/d306b80f4b1678c655f3b0685db1f7a741fc746d251c00e925298ab89630a3167bde8c7d8d779b72770c41f9f302bb1e3f312672ce7aa8e6c156c319eeeada", - "v3/files/09", "v3/files/09/0a6758fac3c263f5f923075d986d2ed26ff74ca2c957e5b86b17e62342564ae142d5374d01ec5163c6f7091d93d2e0779c8da4083d8d0128f7408169781630-index.json", - "v3/files/17", "v3/files/17/54b7271ca0e30dc7d775986711a2b61b298200c12f867bde6f7e19d56047c9ab77a93304520fa1a31b0c3a4e6fb6b310f89dd8cb4d95a5c662807ef733f85f", - "v3/files/1a", "v3/files/1a/5c01a1321e71a1c263fb83878cbba2686f71f571b42c2d40d93858bf1f4c6ff7f625125a68932c243e75ead4a9d2f18d9b22b3fe55c39fe6605a42b2e70620", - "v3/files/37", "v3/files/37/8eb16c230fce354ef7e72722ee708cbefea9a481aa597b06fefc12f665ab60176cbf04d6bc159be42ccdbc51764d71516f86bb28a01b0e60c76a2753f1cf44", - "v3/files/40", "v3/files/40/fe18b2c1f241075b732727110323f3e29f9d6c8765ff0222bccfa34784caa8abcd18ffbd4bbfab979e5478ace273a74ed3a0a22f0261334fa7c59e032edbde", - "v3/files/54", "v3/files/54/7bcd683e2f9f7b0c2790c637b9231539947be923479534592d91886a56d6262fdc1d8ced24ac2cc3adfda139fe399dc7502d382c7afc536e3057a7cf11e6c0", - "v3/files/55", "v3/files/55/dcbc7fff1ac1c4755f4cbf24ba21e0dcae062c802d3db937d2b6182f4554dd85d14529a0c9688eac825a345b63ab8b2e5d7699e2be07178df5bb8a072235eb", - "v3/files/58", "v3/files/58/01e592e94aa4076c179180956f33f8262fa7157a06bfe636a6854bd3b9b2ddf44780ffa1405642ad8ece2ed6514e19a28178cbecdcf5608a3be710bbf3c0d4", "v3/files/58/a80a5a0e5e531bd1646c16f05bdf6da1fb0174a1d9c2fede3e5f306cd4302c56049ddd5776bb5b3f32d9ffee4347b707a5a6a1d0f7a12e788d343d1d54c822-index.json", - "v3/files/5a", "v3/files/5a/ed551de20b04af0a016254022499417f781a6384e39460ebfe77f1f2b08a5a14bb6d4a9dc1246063eaa1bda8499e66513ef7bcb1ab1d08cac3728c3f07c3ce-index.json", - "v3/files/60", "v3/files/60/ab54849d6cfd7b05076c3bc4d9baffe1f800f4d5e2d67c39c521a7a3d93b347ce5a8e45397102e40e2103652c543c8e7bb161c15d0a6e34ec1a9d11c168d2a", - "v3/files/88", "v3/files/88/1f101d2f1886fc5f9ac5c8d25e5540908e140031169ea41a4a9941cf95fe2711c1292c1995609c6f174b3815cc0dcc6cd2a10339a348d39fe75bc77b53ba9e", - "v3/files/9f", "v3/files/9f/cbe5dc86bc33bdb4f125559e767c3aaad94c7c4b21b80b694d9b0158ee0f10ee53b127c2bf05db9768afdaf1fb6778fd08a19412fb3c1adad655863e6c351d", - "v3/files/ca", "v3/files/ca/65df16eb450ab63f90f107a8cc9ac82da9635408e813a6f6d3e13cfba78917b31b66652479ebba54b612566600b2ab1bc16b82d082c5f87803961a8e9e3931", - "v3/files/e5", "v3/files/e5/c2514d2b0d3d6bedad24fdd75c55e780211cf0692c21f401201c2d7dfa990ad00188bd68b30c826aaa2a72ecad12a74bc03430384f933b1751035dd1b18ca2", - "v3/files/f7", "v3/files/f7/02180f4465a777127ad6a211306dd564993c17f7d3d9401604df4bc657d8d888902de632a4cf55b551ddfb9f84ba8a900788773f03c77836143ef94e182c7c", - "v3/files/ff", "v3/files/ff/d3f3fed65d7b489216916772c6180b5c1714ddde16e7e2c99ff5f8a4214bb8f884a0c1d06a1b1a8a4b0d6252ad85b533b1626f0dd8d474f2283f6cdfd69f6c", ], ) diff --git a/crates/testing-utils/src/fs.rs b/crates/testing-utils/src/fs.rs index 2f9233824..8b3f126bd 100644 --- a/crates/testing-utils/src/fs.rs +++ b/crates/testing-utils/src/fs.rs @@ -41,6 +41,10 @@ pub fn get_all_files(root: &std::path::Path) -> Vec { let entry = entry.unwrap(); let entry_path = entry.path(); + if entry.file_type().is_dir() { + continue; + } + // We need this mutation to ensure that both Unix and Windows paths resolves the same. // TODO: Find a better way to do this? let simple_path = entry_path From b4c9edc8c07729770a79382e30289991c3f99cda Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 03:17:40 +0700 Subject: [PATCH 071/102] test(cli/install): exec files in the store --- crates/cli/tests/install.rs | 45 ++++++++++++++ .../install__should_install_exec_files.snap | 58 +++++++++++++++++++ crates/testing-utils/src/fs.rs | 12 ++++ 3 files changed, 115 insertions(+) create mode 100644 crates/cli/tests/snapshots/install__should_install_exec_files.snap diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 28855e499..2ffa31880 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -4,6 +4,7 @@ use pacquet_testing_utils::{ bin::pacquet_with_temp_cwd, fs::{get_all_files, get_all_folders, is_symlink_or_junction}, }; +use pipe_trait::Pipe; use std::fs; #[test] @@ -46,3 +47,47 @@ fn should_install_dependencies() { drop(root); // cleanup } + +#[test] +fn should_install_exec_files() { + let (command, root, workspace) = pacquet_with_temp_cwd(true); + + eprintln!("Creating package.json..."); + let manifest_path = workspace.join("package.json"); + let package_json_content = serde_json::json!({ + "dependencies": { + "pretty-exec": "0.3.10", + }, + }); + fs::write(&manifest_path, package_json_content.to_string()).expect("write to package.json"); + + eprintln!("Executing command..."); + command.with_arg("install").assert().success(); + + eprintln!("Listing all files in the store..."); + let store_files = root.path().join("pacquet-store").pipe_as_ref(get_all_files); + let (exec_files, non_exec_files): (Vec<_>, Vec<_>) = + store_files.iter().partition(|path| path.ends_with("-exec")); + + eprintln!("Snapshot"); + insta::assert_debug_snapshot!(exec_files); + + #[cfg(unix)] + { + use pacquet_testing_utils::fs::is_path_executable; + use pretty_assertions::assert_eq; + + eprintln!("All files that end with '-exec' are executable, others not"); + assert_eq!( + store_files.iter().partition(|name| { + root.path().join("pacquet-store").join(name).pipe_as_ref(is_path_executable) + }), + (exec_files, non_exec_files), + ); + } + + #[cfg(windows)] + let _ = non_exec_files; + + drop(root); // cleanup +} diff --git a/crates/cli/tests/snapshots/install__should_install_exec_files.snap b/crates/cli/tests/snapshots/install__should_install_exec_files.snap new file mode 100644 index 000000000..ee91813cb --- /dev/null +++ b/crates/cli/tests/snapshots/install__should_install_exec_files.snap @@ -0,0 +1,58 @@ +--- +source: crates/cli/tests/install.rs +assertion_line: 77 +expression: exec_files +--- +[ + "v3/files/06/dd9d01a8428becedfb67b9ee3747123a3111423eb66fa16330bd8b9723fcf0a76d46881f21d821fb3bc97d09107948676bb64913d0978e25444195ca874b52-exec", + "v3/files/0d/7059d65c4edb99ee6458b9a1b7de7c767f176341b0492ad213c975a7930d61627457a8bff08ee299d3d83b264ff6d2e39c6227f2bd2d1ab655b7bcdeda2216-exec", + "v3/files/16/12f5046e12d3e734174f9a81280e742bec42f06f948faf57c4c762ab9233e3c90b18be80fbced6c34e7d041066c70caed452c50ddb36b2ab1f3fb5af25d52f-exec", + "v3/files/1c/90670b152f9adf41c4bbf08a94e0f5258296a6bc2689c7db5c77ac31731d441d14cce0de164584937a1ff25dcc158f4148efe5db5a2fea9cbabb47a90db227-exec", + "v3/files/1d/0688424f69c0e7322aeb720e4e28d9af3b5a7a2dc18b8b198156e377a61a6e05bc824528fca0f8e61ac39b137a028029ff82e5229ad400a3cc22e2bdb687ad-exec", + "v3/files/2f/6d3fc9c5874a35976415e7d60bde746579a35d36a57fe379d562fcc84ebde3caa76174b57777317c445bdfaefe2ab44d0b60aad9da53252f900dce334edf39-exec", + "v3/files/35/eaad53a9a1909bd29444761da2fbb3e33ff865e784f1a48d4997f2e8d442eb613269d101fd494cd31b83abc2c1d2a8b13776be0027eaf330f86076e9f5e1a1-exec", + "v3/files/37/3874f9423a505c6f5a0255bfda9d7adc8917a5f40655e66ec4e70c8093e986dee47f1fbc2db07c08d420029723b1cf36a631042844c172dcc85914369744e4-exec", + "v3/files/37/ace9824bdf57e0a823e159e28c50498ba1751eabb2d6721d79c65891c11637fb4a6e5640de3fa717dcbe052c062888957b91b4ccb50a87f638aff0c0f7a3d2-exec", + "v3/files/3d/4b41fed5868cef500302273fcdcf094294b3fe3c0f6337006d6a679ae82c717f8cd7b4beff4175f6b758295bbbf8d500aca8a185c728d91849cf3687420ec4-exec", + "v3/files/40/2757b9075dc4fe7a4467692d7110cf52efce013762a3cbcaa5e9f53f7982a47e0c6219e96ae2b2b9f56de82ec0317571c69f08d24f115c0f5f41a2338a4556-exec", + "v3/files/42/b9aaa6efb15bd57f355f71b600b3f3fcb5bf831c8244eabccae7cac6bf23f538137e480196ac0bbc1962fc316b74ffa7f45fcc2d35cf426892c5a4c2595311-exec", + "v3/files/4b/c429f636ec1587b9fdbf718f643fb56084dbd8354a028869586044d7ee9f9cc88cd3e076d8548dc33002ed902dbab8f4e9ddc375773ad8fd36b704c7cbc127-exec", + "v3/files/4b/d139044276d889c1130e6bfd0baaf3553ed57d18e6b00b129da4917e8037775ca118d5c7503863415c2802dace580aa2a8e4539bbeb796fbea889d85805bb6-exec", + "v3/files/56/1c720a44e1f312ce669f6e442c71fe2c653b47b374811951820743098d98a179e34b29da9803bae82ed330477605f406a9050cbfe48760a97969e9d819a9ad-exec", + "v3/files/56/2af5ef79c1ebd3a4682b82a82019217500e17a65f00ac491755d1db3bd1b8beade4668756801ea753227e124b9f92352f6d56ef0ece29eda18ee61a4fc6fc3-exec", + "v3/files/56/d15758cfdca3ac1f606b6c65a83880e8c7781737a95c613f66cd1c1d258220f9accc963e781f2c52811e606d2709231f265f891917316a168e4eed3d65aebc-exec", + "v3/files/58/903154cb5a895f14ce6e72100d06e5e7dd71c4b720d0765bee818c35f3002c552d52b1b2c53b505ea8d9880592a1e265234b2dec6a0bc0bb1c01f7cef0b6af-exec", + "v3/files/59/cc0bee5bc81251374911b5a241081796d45ea07324f13ec753bab1d5731a5d01b8e6ec023622e4db42db100c96547f75570a82e87293adc19324a4dd8f2be1-exec", + "v3/files/5c/c390a443f845d6bf08545aac1885e0c8bce82f7d983bd313c98f688e31d03b0d97ee66717a013e14d1af0fac3ef3ecf9b0a5f13db328aa45e29776bea3eece-exec", + "v3/files/61/c4590bd0ec5fa0e11e4a6f080b22b661b18043d38eb0b40b03d16e425ba2eac6a2c884cb687b0169fba3b2c8b155d86525316a783633367d474e3d5f9bd7ea-exec", + "v3/files/62/d42979c24a181b63ebf849efe07bc1d277db4aee36e4b8755f42387329d8cfd93bb190478141ac212bd90e3fda0a3c204213ccaa13fa37028f098cb8463982-exec", + "v3/files/64/504f4c713fb006acbe01eedf3b374e7d2066140e85e6cd06d8a9bfb33df921c2424b6ceb06ce82b62403754f61efdb94a38c50ea070a4becbdfce407ebfb88-exec", + "v3/files/68/7a55d92a6fa64a792b3dd04ec2faede8ce4c83363d91d4a5561d469fbb97a8ce18d695f4e6a0f1f82522be82053cc52b3746a6d124bf24c43b8445bcb01165-exec", + "v3/files/6f/294a4e02d24497ca0bf6670f9dec40755b5c8466873c0ac8bb6a28f4cb9b9397dfb1b071c49cf20707c32894aa319a8941d9221a3f352d3f25761d7f526951-exec", + "v3/files/77/39d52696300e9b9b8f7c9663c8a02533c16621790eab060b34429f35d2857fafeda6034665dc675a73838b3b060fb3cc3319de60d58120d945343e3a70b71c-exec", + "v3/files/77/dc603406d1f4afd81568fc4e9f04981249f10212938e3c436e9dcb3b5d640b45b01139560c6d09b84e70eb9d85331c4a912ec8aad2ade3f677470fde2c880c-exec", + "v3/files/7b/0916d2add24d1616e7b6c611f9076832f190dc5074669a03698520c21baa28f7b2f6514216c64a453aea37c5c5ac89845e55b2ffd51bf9d3c65ec208401b8b-exec", + "v3/files/7c/045a8b58e40d57794462e7041249e9b262d71e7d765f35eadff05a849ae4ac7f3f6ea41eb55ce670195e4326cba42c9e39aa8421269a3c87946d4ec8f2a67e-exec", + "v3/files/85/2a6000d178c83e05c233be924f49f276aac9dcd8a93c79104da05bb45edb2232f88d7ae016f412bd239774f5b3186de452a3e6207b55a5d85b61eadb9bc066-exec", + "v3/files/8a/029e32830002bc6b920148ef245ccb291b1b349b02848aa5ce99ca75c6d3a74e1db0bffdea1502a35925737bceb4c08223fa27850912e7a65b7da9788577a4-exec", + "v3/files/94/33bc9e6d478cfedd62c83ad0bd674514f3920edd1833569de1018dbeeacbb159726280fd5a9fbb79e703d1522d8e0a348ab94cb4b649df8e9effc9735fef1e-exec", + "v3/files/9b/9b2bdd63c75e4d374106d78af27f7a87525ff3ee9b282be103cc3ce98171f4e17cd9a28eecf8e7004868103f609c453c7e777e3e9083c4a4a88e0c72a0fcba-exec", + "v3/files/a0/09e43d84d4781d35163d895ca635ead87b2d2f8d9b1689ed09ab64e0536195764b067edbc893a1fed95367d6a99e382455cd8331c8d09cfe173b0fe1126c24-exec", + "v3/files/a2/a9cb7d3de319d0f058f6af6f212c6d48d388cb17eaa833df307c81b9cd5a384ccbdd14dbe6616a401292e77c67f4ee311af75ec026ff2cb2bafc6449decdbd-exec", + "v3/files/a2/bbd0fc27dcb18788968a58f1c9d38fa49e27b32d6b942d1ea0f8d9d81a056400478137dcec81b6ddeda34addd14574690c5bef79b1b4d1145f0df7418d3abe-exec", + "v3/files/a8/1983c1c62d075b3c76b1ddc412ed3ae745a30f6f11f580dcc64dcc2b088110e2b55e9dbaa9fbbaaadb3c2d81108773408658b653fed018c50804fe809b0297-exec", + "v3/files/af/93b76be9167844f6d773f3e20038c6bcc415754e5402a5758e6b6dc59e29602997e2c831c17cf3a61e7b800a33f0eadee2a55708c6d7f2db1479c37ddb9780-exec", + "v3/files/b2/9f0073f86f7e9aed174fce942493e1e4a8fa5dac6afd4ff2108c74056f618e9e83cf6acf8fac63def0686fc46e266fa0bc90da004be5fb34df4f7f061e80a3-exec", + "v3/files/b7/f1e5dc2e73938d0dc0a7a1349c4c0e88ca1b5dff93164f254c4d03240ee7d58984b0f19fc42050ef51f79815ab41f8235953c87d46a894b62930f3ca32ccc8-exec", + "v3/files/b9/acf9e03ebcc7627deb8b26a0029e0b51fe3ff6b6d3a8601dd1fc684235fe0b527a0da83bb913a15fac0b86ee6776093ca69610e36774ad29f221969a23a766-exec", + "v3/files/bc/16419489153aa19e5a0c2989286941310a41478678fa91249878b232f9ffbfa5f9935846dc9ec57e98be60a489dd679de4ed9990f843570aa0fa3ba81e5356-exec", + "v3/files/c2/65c48634b48a6a4b70c9cfcdae4c44b2eaf59ab39152ef1fc40cd48c96c8a9da0b3549dcc3be6298edd9e75f891cd2cc3390854fb5c95a168165d0222876d2-exec", + "v3/files/c6/703e2577c586d276c218902674852f8bfd6ca4b07f018512e255e1f20853ca182ee85769961e78431ea9812bfeb09f031ad581f9e31533f12b9d43e587e757-exec", + "v3/files/d6/f84a85f42a7c7971d607ee2df1952351246b079278bcbafda757f643094c9bbafb430e1aca63af939433b2216232e30aed4d5ea7854fb1ff824051e171d91e-exec", + "v3/files/dd/8a7b4d525f8cba564e94846ed484f960663b4df324cbedccc422cd899b2139e88317e811a5e3c269baa641a7348de6359314ef3a5c1e6cc3f7ff5f23b8e1fe-exec", + "v3/files/e4/39669d73b83f43eeaabee13f4a06190c43b831544e44617fd955d5dddfefe0073d0d84edc0a614387a5cfce618668041ef0238234f2be99f48d35f7efe23c0-exec", + "v3/files/e5/3dee301557be2befd6403b92ad496e2473c0398c67072ae929675626e0f9bba070459ed772735fdc105a594a87f07900c68e745a5e636be753e0ba6b9c315a-exec", + "v3/files/f0/3e0687f9a3d13087f7b966cd0ef1a143e5e70216be4209073bb15738e6c45a0864f619ca3274269d658020c9837b6c5584601fc1dbea2601fd2314df1830c6-exec", + "v3/files/f1/eac3da4b2d7d1965a923a7fd9d09fefccd711d7f8438d1156fcf0fbd46e678ccb4c60262a80ca9eb9380df498812d7b5ec893d82bd68f8c0148c71e1df0530-exec", + "v3/files/ff/00a34fb9c309279edf249bca6d5e54c22d2151b118f358f4e737875bb6e8162e2af9cf5f6f60693f0f3c481b0c2f4a3d28e6f91dcc5c9db7cfe3c527efabbc-exec", +] diff --git a/crates/testing-utils/src/fs.rs b/crates/testing-utils/src/fs.rs index 8b3f126bd..1fd16fe67 100644 --- a/crates/testing-utils/src/fs.rs +++ b/crates/testing-utils/src/fs.rs @@ -71,3 +71,15 @@ pub fn is_symlink_or_junction(path: &Path) -> io::Result { #[cfg(not(windows))] return Ok(path.is_symlink()); } + +/// Check if a file is executable. +#[cfg(unix)] +pub fn is_path_executable(path: &Path) -> bool { + use std::{fs::File, os::unix::prelude::*}; + let mode = File::open(path) + .expect("open the file") + .metadata() + .expect("get metadata of the file") + .mode(); + mode & 0o111 != 0 +} From b3f9d8c0da97326b966dc1b05afc543c28202375 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 03:15:15 +0700 Subject: [PATCH 072/102] test: remove dirs from `get_all_files` --- .../install__should_install_dependencies.snap | 15 --------------- crates/testing-utils/src/fs.rs | 4 ++++ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/crates/cli/tests/snapshots/install__should_install_dependencies.snap b/crates/cli/tests/snapshots/install__should_install_dependencies.snap index e6000a49f..d51501862 100644 --- a/crates/cli/tests/snapshots/install__should_install_dependencies.snap +++ b/crates/cli/tests/snapshots/install__should_install_dependencies.snap @@ -21,35 +21,20 @@ expression: "(workspace_folders, store_files)" "node_modules/is-odd", ], [ - "01", "01/d306b80f4b1678c655f3b0685db1f7a741fc746d251c00e925298ab89630a3167bde8c7d8d779b72770c41f9f302bb1e3f312672ce7aa8e6c156c319eeeada", - "17", "17/54b7271ca0e30dc7d775986711a2b61b298200c12f867bde6f7e19d56047c9ab77a93304520fa1a31b0c3a4e6fb6b310f89dd8cb4d95a5c662807ef733f85f", - "1a", "1a/5c01a1321e71a1c263fb83878cbba2686f71f571b42c2d40d93858bf1f4c6ff7f625125a68932c243e75ead4a9d2f18d9b22b3fe55c39fe6605a42b2e70620", - "37", "37/8eb16c230fce354ef7e72722ee708cbefea9a481aa597b06fefc12f665ab60176cbf04d6bc159be42ccdbc51764d71516f86bb28a01b0e60c76a2753f1cf44", - "40", "40/fe18b2c1f241075b732727110323f3e29f9d6c8765ff0222bccfa34784caa8abcd18ffbd4bbfab979e5478ace273a74ed3a0a22f0261334fa7c59e032edbde", - "54", "54/7bcd683e2f9f7b0c2790c637b9231539947be923479534592d91886a56d6262fdc1d8ced24ac2cc3adfda139fe399dc7502d382c7afc536e3057a7cf11e6c0", - "55", "55/dcbc7fff1ac1c4755f4cbf24ba21e0dcae062c802d3db937d2b6182f4554dd85d14529a0c9688eac825a345b63ab8b2e5d7699e2be07178df5bb8a072235eb", - "58", "58/01e592e94aa4076c179180956f33f8262fa7157a06bfe636a6854bd3b9b2ddf44780ffa1405642ad8ece2ed6514e19a28178cbecdcf5608a3be710bbf3c0d4", - "60", "60/ab54849d6cfd7b05076c3bc4d9baffe1f800f4d5e2d67c39c521a7a3d93b347ce5a8e45397102e40e2103652c543c8e7bb161c15d0a6e34ec1a9d11c168d2a", - "88", "88/1f101d2f1886fc5f9ac5c8d25e5540908e140031169ea41a4a9941cf95fe2711c1292c1995609c6f174b3815cc0dcc6cd2a10339a348d39fe75bc77b53ba9e", - "9f", "9f/cbe5dc86bc33bdb4f125559e767c3aaad94c7c4b21b80b694d9b0158ee0f10ee53b127c2bf05db9768afdaf1fb6778fd08a19412fb3c1adad655863e6c351d", - "ca", "ca/65df16eb450ab63f90f107a8cc9ac82da9635408e813a6f6d3e13cfba78917b31b66652479ebba54b612566600b2ab1bc16b82d082c5f87803961a8e9e3931", - "e5", "e5/c2514d2b0d3d6bedad24fdd75c55e780211cf0692c21f401201c2d7dfa990ad00188bd68b30c826aaa2a72ecad12a74bc03430384f933b1751035dd1b18ca2", - "f7", "f7/02180f4465a777127ad6a211306dd564993c17f7d3d9401604df4bc657d8d888902de632a4cf55b551ddfb9f84ba8a900788773f03c77836143ef94e182c7c", - "ff", "ff/d3f3fed65d7b489216916772c6180b5c1714ddde16e7e2c99ff5f8a4214bb8f884a0c1d06a1b1a8a4b0d6252ad85b533b1626f0dd8d474f2283f6cdfd69f6c", ], ) diff --git a/crates/testing-utils/src/fs.rs b/crates/testing-utils/src/fs.rs index 2f9233824..8b3f126bd 100644 --- a/crates/testing-utils/src/fs.rs +++ b/crates/testing-utils/src/fs.rs @@ -41,6 +41,10 @@ pub fn get_all_files(root: &std::path::Path) -> Vec { let entry = entry.unwrap(); let entry_path = entry.path(); + if entry.file_type().is_dir() { + continue; + } + // We need this mutation to ensure that both Unix and Windows paths resolves the same. // TODO: Find a better way to do this? let simple_path = entry_path From e4e177332d8d404d6c7a2ea696e097a534bdd413 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 03:25:07 +0700 Subject: [PATCH 073/102] style: use qualified --- crates/testing-utils/src/fs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/testing-utils/src/fs.rs b/crates/testing-utils/src/fs.rs index 8b3f126bd..7b2033168 100644 --- a/crates/testing-utils/src/fs.rs +++ b/crates/testing-utils/src/fs.rs @@ -10,7 +10,7 @@ pub fn get_filenames_in_folder(path: &Path) -> Vec { files } -pub fn get_all_folders(root: &std::path::Path) -> Vec { +pub fn get_all_folders(root: &Path) -> Vec { let mut files = Vec::new(); for entry in walkdir::WalkDir::new(root) { let entry = entry.unwrap(); @@ -35,7 +35,7 @@ pub fn get_all_folders(root: &std::path::Path) -> Vec { files } -pub fn get_all_files(root: &std::path::Path) -> Vec { +pub fn get_all_files(root: &Path) -> Vec { let mut files = Vec::new(); for entry in walkdir::WalkDir::new(root) { let entry = entry.unwrap(); From d91a3556e919acd53ec6e9d0aa546007fbade80d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 03:54:00 +0700 Subject: [PATCH 074/102] docs: add a comment --- crates/cli/tests/install.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 2ffa31880..13fc5deb6 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -87,7 +87,7 @@ fn should_install_exec_files() { } #[cfg(windows)] - let _ = non_exec_files; + let _ = non_exec_files; // suppress clippy complaint on windows drop(root); // cleanup } From 7966f5d1da6a99d8510b494a9e6520c6ac1653a8 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 13:27:02 +0700 Subject: [PATCH 075/102] test(cli/install): snapshot all files --- crates/cli/tests/install.rs | 11 +- .../install__should_install_exec_files.snap | 157 +++++++++++++++++- 2 files changed, 158 insertions(+), 10 deletions(-) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 13fc5deb6..b31a3984b 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -66,11 +66,9 @@ fn should_install_exec_files() { eprintln!("Listing all files in the store..."); let store_files = root.path().join("pacquet-store").pipe_as_ref(get_all_files); - let (exec_files, non_exec_files): (Vec<_>, Vec<_>) = - store_files.iter().partition(|path| path.ends_with("-exec")); eprintln!("Snapshot"); - insta::assert_debug_snapshot!(exec_files); + insta::assert_debug_snapshot!(store_files); #[cfg(unix)] { @@ -79,15 +77,12 @@ fn should_install_exec_files() { eprintln!("All files that end with '-exec' are executable, others not"); assert_eq!( - store_files.iter().partition(|name| { + store_files.iter().partition::, _>(|name| { root.path().join("pacquet-store").join(name).pipe_as_ref(is_path_executable) }), - (exec_files, non_exec_files), + store_files.iter().partition::, _>(|path| path.ends_with("-exec")), ); } - #[cfg(windows)] - let _ = non_exec_files; // suppress clippy complaint on windows - drop(root); // cleanup } diff --git a/crates/cli/tests/snapshots/install__should_install_exec_files.snap b/crates/cli/tests/snapshots/install__should_install_exec_files.snap index ee91813cb..2752ca803 100644 --- a/crates/cli/tests/snapshots/install__should_install_exec_files.snap +++ b/crates/cli/tests/snapshots/install__should_install_exec_files.snap @@ -1,58 +1,211 @@ --- source: crates/cli/tests/install.rs -assertion_line: 77 -expression: exec_files +assertion_line: 71 +expression: store_files --- [ + "v3/files/00/1bd318b4f0e816c9ded28091d256ae7ff7d3b0e9ff3b262376377857e0801e53b59d3ea434c2e032cd746ed135c655b1479cafe5e4473c7fbb224ae36f4ae8", + "v3/files/00/46311fdde31853e7fdada2540c16f3b56e508911d45554281efb370305ee70530e40ebad3fc7a6dfc8ac2274417856dbb8d304371fe5963bc3a462a93330d9-index.json", + "v3/files/00/deb738ab63d081d3203475b2e1808204a4c84055a47deec42686a254f056e913b674f93421d29d110f38f44cec68dac711a5cc34b0b2353381b16d9fd8045d", + "v3/files/03/07a3d034d619f4687f8284d16bdb383888d588b568d69b9b072e184e714b078a3cef082b4570f168cce6664b5e56967385d2a80d9db03197d1a87efcb5ac7f", + "v3/files/04/2090ccf97b5cf02090e4e356ee48a07468d0a6c80c7085f1622c6dfde1da4eb9f54f2b97e3d17d9998b990804abeb0741c24b3d984a9102ed186dffdb9f53a", "v3/files/06/dd9d01a8428becedfb67b9ee3747123a3111423eb66fa16330bd8b9723fcf0a76d46881f21d821fb3bc97d09107948676bb64913d0978e25444195ca874b52-exec", + "v3/files/07/63570393a082d18dbdfc05b88a99cfac9933976e13eaec2c2ff6cf4a05cf97bb9a7f67030339afb935e0f19b47d56ee0420a99d3726f1c65429e8635b537fe", + "v3/files/0b/49b251d76752186967f51c0fdc99c8c5a87505787d3db0c5c7ea6ff4e80f467df2707d59f58197a928355589fbc8f378658feb0423cfdcdd5fb527b9ad3f49", "v3/files/0d/7059d65c4edb99ee6458b9a1b7de7c767f176341b0492ad213c975a7930d61627457a8bff08ee299d3d83b264ff6d2e39c6227f2bd2d1ab655b7bcdeda2216-exec", + "v3/files/0e/1fdd6e8f2e01d320901acfa388d0c9b30ee57303c7cec769b9f959ea4920f87a63f3392e362757106fd1a513dbf9037698adf291d604188cb44712d7afbafc", + "v3/files/0e/5dc92f9f523fa072574ccebb28df5b71f8d7ee7a3cfef6eddc92a67bf2a5285ec7ad7f9624b1ca89b9ec3a0105f288df826dcbc34d24f632d6721c3157615e", + "v3/files/10/9f64d9f1bc268b7e437e997251a1e3f61ae2ead5de65afbeb9322047115418524dfccbc0ce57de7f3f0b35004358b9f251c2ff60ba7cfafbcc89e57a62a7df", + "v3/files/10/a55f947076d0a3c6971b374e0579968ee67348344f4d057e1f8c22c3eafa82566ea8bf7710368ee3ec1ac69521761ae1ec698dc360303ad0c2b6284a0980c9", + "v3/files/10/c1a5869f7cba07590443f48fa421bd8b0a8f8b428f36865a3af68d1c87c745b98ab55e30379e15971f8e584a4efb84a05505a9b133ab5630d5ec745e3cedea", + "v3/files/10/fb8645d46738c3b242ae4689cb2c763515cf1ac3b8dcfced8be40ff94d60bfbd5737acd22596a1f971a3e30e2a5baf25c5123c65623edd190639e73c60a8a1", + "v3/files/12/06b1fa1dcb049bc96a0e252b4abc21c58975932a0e149c686964d4fcbc11702536d34de2fa6ab89c2384a620ac921b0d556185c7e856985c4ab606d9200dd4", + "v3/files/13/29094ff4352a34d672da698080207d23b4b4a56e6548e180caf5ee4a93ba6325e807efdc421295e53ba99533a170c54c01d30c2e0d3a81bf67153712f94c3d-index.json", + "v3/files/13/90ab3de4cd21a6407edc2a309a644fc3c335a994254aee6c72d367a4639f797d46f24a48bc3a3065d3e9201c44757796d2ce49339ad47be443bfc650ea1a1f", + "v3/files/14/1f74f51ee662fc5a263e0cb193c47c8eb66201a27dd1a146d253efb413684c7107e3910a02167de8c649693929fe1781f79a6783d6115e2ca17b7adef9c594", + "v3/files/16/053da13599a77fd7657897242f1e8a117dbfeb05126b21ae2bb2592da1b14fd2303991ba6759d6730651773b775c9e2c501e7b0e6f3ab9e25821192279ccf8", "v3/files/16/12f5046e12d3e734174f9a81280e742bec42f06f948faf57c4c762ab9233e3c90b18be80fbced6c34e7d041066c70caed452c50ddb36b2ab1f3fb5af25d52f-exec", + "v3/files/17/260f7abd7803c7a179e485397421ec4f935ab216d8020610c923fbac0b4453ed4064df70d59c85ab09793b954c062a66b40faa975d95fa4aacc1f4f07ada81", + "v3/files/17/4b5d7ec4108f929bfe28012503ef50a56e823a3dd8133d58b9f5fa18870d0f8b890edb9ab4d654e27cf6cf659a09ff4b005ee1edf5bfe1afbc89f27bc4e2c0", + "v3/files/1c/90670b152f9adf41c4bbf08a94e0f5258296a6bc2689c7db5c77ac31731d441d14cce0de164584937a1ff25dcc158f4148efe5db5a2fea9cbabb47a90db227", "v3/files/1c/90670b152f9adf41c4bbf08a94e0f5258296a6bc2689c7db5c77ac31731d441d14cce0de164584937a1ff25dcc158f4148efe5db5a2fea9cbabb47a90db227-exec", + "v3/files/1d/0688424f69c0e7322aeb720e4e28d9af3b5a7a2dc18b8b198156e377a61a6e05bc824528fca0f8e61ac39b137a028029ff82e5229ad400a3cc22e2bdb687ad", "v3/files/1d/0688424f69c0e7322aeb720e4e28d9af3b5a7a2dc18b8b198156e377a61a6e05bc824528fca0f8e61ac39b137a028029ff82e5229ad400a3cc22e2bdb687ad-exec", + "v3/files/1d/7fd1d7ae2c1795a101f3fe95deb928c9d90019ff959c3f193895f0ac305f909fb93cf9d241d64ff7953a548b73cec21f1927ee6bba20d1915b9e21f0190ae3", + "v3/files/1d/c5fb24099f33d6874f65c424d9f53ee70c0dc3166b2493cbceb51372efe393a42fe2d083bc784e541a35b2e6472e31919425724d803185837d653a17e2095e", + "v3/files/27/b172c27f5f7b71e57a2b3aeda876e6b05f650f0b847b7916558e641eb93f8801664e936a62b13fbbd1fa91454fb2fc5b21dbcff7c25c92a9980b02eacc892d", + "v3/files/28/cf9314df893197ac3d1d263e761046cd8cea6ca77b9c761149792f6a9636a165c2fe132e676de5b9edc574482bf60c3354cd32419b22e3a91ffc9c626fd887", + "v3/files/2c/da59f0a6854323c1dc93f51ee88611c70d92fc28d3c8a32816d74d485dcd5ce828e5d020ec0a317a838b67da6a4459ceed4837cd73b6b85f7199c6d6545d1b", "v3/files/2f/6d3fc9c5874a35976415e7d60bde746579a35d36a57fe379d562fcc84ebde3caa76174b57777317c445bdfaefe2ab44d0b60aad9da53252f900dce334edf39-exec", + "v3/files/31/46b9dfb4bec15e2dd76608b98911757a7ce619438d16a5ad6ca17a60dd79c8cdfe43adf2456b6413ed772097c9e6e0e726174758564459fb1153ee8563ab15", + "v3/files/34/ddb9208914ac53ed7c0e7162f74d0313a8f348f34db824414028313c03de674995ac98bbf856f5219d44d1af1455fa41678eb14dbc4639567b9227ef11ca31", + "v3/files/35/4bc6788ba428a39568ed879ffceb19bffc254068e3eeef3901bbc18ef78ae597c7c4963c0dfeec660629b7fa10a49d3a0e56a208741dd716d8827e0404c0ab", + "v3/files/35/7909a22fc6ae96328f87dda34b6d774e020b3fd1f5ecc513600dfb04c94c8b82e3558bd7907ef5c914a14d9857f7a04b92243c5e168f3cf1a3d47f1b9d956d", "v3/files/35/eaad53a9a1909bd29444761da2fbb3e33ff865e784f1a48d4997f2e8d442eb613269d101fd494cd31b83abc2c1d2a8b13776be0027eaf330f86076e9f5e1a1-exec", + "v3/files/36/628b538527c1694a32d221d14ef43df9e4b129e6fa5740b5dda9ccb6fe8550ccb54a30a49565d665c6d93a23753ceab476eb488ed1b89f7c76a3b5b823c84c", "v3/files/37/3874f9423a505c6f5a0255bfda9d7adc8917a5f40655e66ec4e70c8093e986dee47f1fbc2db07c08d420029723b1cf36a631042844c172dcc85914369744e4-exec", "v3/files/37/ace9824bdf57e0a823e159e28c50498ba1751eabb2d6721d79c65891c11637fb4a6e5640de3fa717dcbe052c062888957b91b4ccb50a87f638aff0c0f7a3d2-exec", + "v3/files/3a/b5d1d90855d80fa4aca46de6de6926adbcd4c5cdfc8f893410fc9f4c4ad5792623e4e3da51f9f37f5ef5d99d41298c7bc5f6b8f13acc03ef472ab5018c1df4", "v3/files/3d/4b41fed5868cef500302273fcdcf094294b3fe3c0f6337006d6a679ae82c717f8cd7b4beff4175f6b758295bbbf8d500aca8a185c728d91849cf3687420ec4-exec", + "v3/files/3e/cd249df6b3ae458d0a95123703e1637f56bddcaff48c3540425464cd174ff6a74342cf26a97b753d74f1381b8fc02eee457bcbe46c7d1f5e2f5638b2160579", + "v3/files/40/1949f50c5f620d8a88e5a85cd0ed52858bae660a0bd92eb2f03618ff90af19f1b527fb9bd8a003cfddf6f035a8e01fd2a9ca3fdbc34dc7c2582fd1a85a8aa4", "v3/files/40/2757b9075dc4fe7a4467692d7110cf52efce013762a3cbcaa5e9f53f7982a47e0c6219e96ae2b2b9f56de82ec0317571c69f08d24f115c0f5f41a2338a4556-exec", + "v3/files/40/4846e68fa46356f1bace923489e6f10556da4f0d668447fafb6139b8ca4654754c915993f3fd947289a02be4f358620911f936e40573c6ecfa97ee33b9cc59", "v3/files/42/b9aaa6efb15bd57f355f71b600b3f3fcb5bf831c8244eabccae7cac6bf23f538137e480196ac0bbc1962fc316b74ffa7f45fcc2d35cf426892c5a4c2595311-exec", + "v3/files/43/9a9061360cc18785d355bb18f07e186c55d114358b0d13b92ad479e826b0664021dfa1328613dfe9743ca528c79812e40d4f8594b26c09a04e7dc37d557d14", + "v3/files/44/9fbdf7888a5b9088b5f84aa6d1a42cf951782a062079f63fe5e1e797e709ed4737c3e19300d0a98a01013431e73652c5b81438913ba952ff1fb63bce460e5b", + "v3/files/45/11023ec8fb8aeff16f9a0a61cb051d2a6914d9ec8ffe763954d129be333f9a275f0545df3566993a0d70e7c60be0910e97cafd4e7ce1f320dfc64709a12529-index.json", + "v3/files/49/7414c7053618e078df0c9047828041dfb0e656951a8fc3177cf12bac50232405c5eb37d897a105a861bc8ff896b4d1a2c699fd7ad91c6df22f71a33bc44578", + "v3/files/4a/74a235678af320bef5ca5746d31f31fb9c5c01ac3da499018c2f4216834a53096e104300911b22b80e16c2bb25d673cade6d73c4e8807ef17f12e7569d3e26", "v3/files/4b/c429f636ec1587b9fdbf718f643fb56084dbd8354a028869586044d7ee9f9cc88cd3e076d8548dc33002ed902dbab8f4e9ddc375773ad8fd36b704c7cbc127-exec", "v3/files/4b/d139044276d889c1130e6bfd0baaf3553ed57d18e6b00b129da4917e8037775ca118d5c7503863415c2802dace580aa2a8e4539bbeb796fbea889d85805bb6-exec", + "v3/files/4d/912cb78467a9b25e5fd1be0ea9371f6a5250a8ed5cecb42781f981ba1ee13763d229de2d2c33fb4361854a318f236f2fca0553bb5871ce6ba8cc6d670ae4ee", + "v3/files/4d/99655ebd3ac09430ab6beb431d4f95f71bac48c87f67d10cfe2614f77b20655a47eecb973da1355e15104344dc4688a6c7df128514005d9bd5462c8edc62c3", + "v3/files/4d/d0650cd0c02e62242867e80abb87a618f8d48fd25c16e969bb45a8157d6910ae9d798b95ecc715e1db88ee21e39eb857fd302ff075f583faf8e8b69b55c54f", + "v3/files/4e/e1c88f8c3f4e4cd34cb6c00339bf9d6d036ff4ade3af49e871cc8966b84c729d8b75492acc6413c9a664ac00a57958223ac13c4229da8c62ebe6a53e4f783f", + "v3/files/52/7e009073351c135b4b3d04d776d01dddd136f87c43804b6fff499ff7d63f0d698fcd3ee05fe337480c20e496776765cc8ae64f5c0449eb51d0c8e8a40a2e91", + "v3/files/53/a6c263fb42e0c2152aa4dba9117b14cc64b4bdc5b74f81895e0b50f141473abae3a491a198d3945a4c3edeeb3f957f26656e04132992b452e2574d47080f5c", + "v3/files/55/1f0f90d2a325182538c29ab593604aba75c93d20c5c29d6bb954f015bbd3b504472f2fa40a4ef78567ecec06d2b80db58fe99b50fdacc8b223ef2429ab4f82", "v3/files/56/1c720a44e1f312ce669f6e442c71fe2c653b47b374811951820743098d98a179e34b29da9803bae82ed330477605f406a9050cbfe48760a97969e9d819a9ad-exec", "v3/files/56/2af5ef79c1ebd3a4682b82a82019217500e17a65f00ac491755d1db3bd1b8beade4668756801ea753227e124b9f92352f6d56ef0ece29eda18ee61a4fc6fc3-exec", "v3/files/56/d15758cfdca3ac1f606b6c65a83880e8c7781737a95c613f66cd1c1d258220f9accc963e781f2c52811e606d2709231f265f891917316a168e4eed3d65aebc-exec", "v3/files/58/903154cb5a895f14ce6e72100d06e5e7dd71c4b720d0765bee818c35f3002c552d52b1b2c53b505ea8d9880592a1e265234b2dec6a0bc0bb1c01f7cef0b6af-exec", + "v3/files/59/24989d54f63ae18d676a3fab8d8b988f5a1e16d925054a38a69066b615fdd842bc4f1eddbafacfbd54e71508470521af0956aaa49d8e742b3b77543b27a1b4", + "v3/files/59/b2b098e162b61370f317948f7bc671fb412bfa0c07977634f07755e663ed0c2d683aafae8eba41358d11f65074ea46891b40d852e83ad6bbd2951c7578cfee", "v3/files/59/cc0bee5bc81251374911b5a241081796d45ea07324f13ec753bab1d5731a5d01b8e6ec023622e4db42db100c96547f75570a82e87293adc19324a4dd8f2be1-exec", + "v3/files/5b/335edf6056f57c261cc03eb1229799e64d2bbf2620f39182ffe7c87db5b39627c71fb7f10a22f0445ab66cc5b6d6f06689209dea5ea4f6265efad993104ac7", + "v3/files/5b/a65521cbda0287e641d44ae987a2736198f568385289347e1d47aba6e12b9979da56e7ca8853876e6125f469ed45dd576fd69dbad58cddbc354906347789a2", "v3/files/5c/c390a443f845d6bf08545aac1885e0c8bce82f7d983bd313c98f688e31d03b0d97ee66717a013e14d1af0fac3ef3ecf9b0a5f13db328aa45e29776bea3eece-exec", + "v3/files/5e/78b7e4d2b38e032bc1ebf2b074c202bb4b0e93efc9ef3357fd04e04c989f8dcfeffeeabd0c0f87d0469077b06ccba5567b5b8a099c4fbadd5f704da3dc1126-index.json", + "v3/files/5f/a11ad79ad227c77d1f57c7ec9f736073f2919b8acfdd23ec743b500bdb335f97c1c1ffe721f0eff116502455f31f99a4344a0d9ca97c5bb5c7e2ebd54108bb", + "v3/files/60/ae951206f3444a5190c888d4e450d5a2ece233bb190551d0afcc40b7a68f11fdd0be68aab6249ea32f2155739f392b0e173c679747989a110c5d6d90edf17e", "v3/files/61/c4590bd0ec5fa0e11e4a6f080b22b661b18043d38eb0b40b03d16e425ba2eac6a2c884cb687b0169fba3b2c8b155d86525316a783633367d474e3d5f9bd7ea-exec", + "v3/files/62/5122568344763ad0d6d84d788d52a3322adbb4cf6f26e48520c8a92e971d82968b799fe5b700f9cda268978204d5b93028c3c10808c1700ffeb1cdaa16ee09", "v3/files/62/d42979c24a181b63ebf849efe07bc1d277db4aee36e4b8755f42387329d8cfd93bb190478141ac212bd90e3fda0a3c204213ccaa13fa37028f098cb8463982-exec", + "v3/files/63/159b500cb660028bb869598a6f9a0e7a5482dadb536f27381118b40fe230992fa156db1cabb150223689e5add0347b3fb318072ee77169078d93fc5bced2b3", "v3/files/64/504f4c713fb006acbe01eedf3b374e7d2066140e85e6cd06d8a9bfb33df921c2424b6ceb06ce82b62403754f61efdb94a38c50ea070a4becbdfce407ebfb88-exec", + "v3/files/65/572f3376372b45922adbfa3354fca4e0e6efd320041dd3c9af40c52a1c28673a27d96c05ae195b35530fa0d64a7ca9445c81bbd31ca0c14db4d846438caeab-index.json", "v3/files/68/7a55d92a6fa64a792b3dd04ec2faede8ce4c83363d91d4a5561d469fbb97a8ce18d695f4e6a0f1f82522be82053cc52b3746a6d124bf24c43b8445bcb01165-exec", + "v3/files/68/c87ffa98edeb3d89e8505fe5eaf94977591f66b97a258581ac82332e8742013ba9479426fa713ec660444ebb5b42dd8ae9c992904c028f91a55c0aef64ccc9", + "v3/files/68/fcef5ff77884bb9aed8629affcb16058b7b96606a0c4cea59096ea90182f17ac4edfbefdd9c1b11c5c3de71778f2c43e5de732d83a5333f8a96cf089164367", + "v3/files/69/959d580d921ed0d1294e4a49904df9bd680d072aaafda4c7b656d4cd255e8682ee83b7c14ed658a55b443b5406b1211149c1fe17045626386699cd8aeb68e0", + "v3/files/6a/8011db36b113d52d7173b8904f456cf06021911593bbb767e9c43e60e8eabc1c62d874ecac6b5597a3544772d9798abe491c6fc3bf5ccf542b76cf73588c3f", + "v3/files/6c/3ce97565f2f9cbaf9d42c05f0154ef002b6836e3fee42b1e2f96ee3c42a8e54009f4a47f40f6fde196ebf3d9a635f880a093112013a75a01389fa3f2f15644", + "v3/files/6e/6f6bd9fc4f1b9be9c1df2a866dbec68cdc04169a42c7da4667fe4cd69b68647bd27572d5dd8fbc139ec9a4606ebdaffd9b23ac439b6c0e9f36d0d021a58cf3", "v3/files/6f/294a4e02d24497ca0bf6670f9dec40755b5c8466873c0ac8bb6a28f4cb9b9397dfb1b071c49cf20707c32894aa319a8941d9221a3f352d3f25761d7f526951-exec", + "v3/files/71/794c245ed02030a6151b74239b44307a55a8f6f1eb399c7521b3c6d82349c8420d1195be9c78587c32633cfaaf77a3a71d64de3c6305ecf217722a5c35dc9f", + "v3/files/74/ecbedc0b96ddadb035b64722e319a537208c6b8b53fb812ffb9b71917d3976c3a3c7dfe0ef32569e417f479f4bcb84a18a39ab8171edd63d3a04065e002c40-index.json", + "v3/files/74/fc7841bb52c17b410f7f332c93320097507a273e05e43e00e9ff6095a831a63b0a24e3078a255cebc99f352d41ad424b12f421f657f3c92e7ba5e23421429c", + "v3/files/75/71ea25152542ffc2cc5a6ea5ece32fe0afb7b4658508cee10e3c5f456bf540e14c8a7c2325451ae1dce419feb858985530f52578c79667d75d9d33ae79c442", + "v3/files/76/5f0c1cdb710b9cf2ffe714c82cbd42cea740c8d5340fcb1aff5cae70c156598377f5ebac0925ec34252414e99a53d3fa5066ec90fa438913ef0e6f1a9874bf", "v3/files/77/39d52696300e9b9b8f7c9663c8a02533c16621790eab060b34429f35d2857fafeda6034665dc675a73838b3b060fb3cc3319de60d58120d945343e3a70b71c-exec", "v3/files/77/dc603406d1f4afd81568fc4e9f04981249f10212938e3c436e9dcb3b5d640b45b01139560c6d09b84e70eb9d85331c4a912ec8aad2ade3f677470fde2c880c-exec", + "v3/files/78/31cba562e1f7e03a9aba93441b496b79cd26ae5cae788daefdcf02d9de03f843a46774f1a43b55e997d5857abf7d4bfc3ea01ee2c7ae23e41eafbbe829c72f", + "v3/files/78/b1155ca28b8e1143f7d5fd497976349da9baffced3e614c6acb6a68e35841130b4325f004605ab33591d93d2387416a26880439cf16dfa8d84954a1b878fa1", + "v3/files/7a/df42cfe61a270231d0778a13715cad50e769aa632c86e978098deb07ddc665e67ed02ee830b072daab691aad71a825dfbe5e0a23b0e59736f6def8f45db000", "v3/files/7b/0916d2add24d1616e7b6c611f9076832f190dc5074669a03698520c21baa28f7b2f6514216c64a453aea37c5c5ac89845e55b2ffd51bf9d3c65ec208401b8b-exec", "v3/files/7c/045a8b58e40d57794462e7041249e9b262d71e7d765f35eadff05a849ae4ac7f3f6ea41eb55ce670195e4326cba42c9e39aa8421269a3c87946d4ec8f2a67e-exec", + "v3/files/7c/0b427efe23260f84c6d5e1bc2902a94299fb8a24fc9424d35a832a12f90430e84b7031297855e7f300554672427484776193f767e4819974a73d10d00cdc21-index.json", + "v3/files/7d/d4c7864159636d2e046813f76af1e6f14aa77914c04c7737ca1af3867807c49ca39955e43200716bb5f3b0d87bc69e8627e1dd7dd051927080a73a203ddb29", + "v3/files/7f/6f0047fd5ee304d158c34cd9d97a1aaf22a68e0dcf97bdb01e3d22efb2825f8e03ff28c72084dd0e6e9fa9e5d831b0c12d25819634be30bf6e03e8369a2746", + "v3/files/7f/9827a08004469d57872dfb5e541bc456eba6685d7e65044797df3554c0ba46b4d7b8549d94178f3a62b16baeb8b3827f71eeba5521cbe6a4effa4ef9085ca5", + "v3/files/80/5749e65c8acc1a444c249349a90f45f79ecadf4e0c847fe8e48740cc83391dc2f854d4f1b2c562f6762b0479f31b1281201baeaa31805c5cdd4e49bc110161", + "v3/files/81/03f6c7554de6667b68a121dce9482a3a052d4c4286a09e4e183740c25a70b3e7ae851d784b182f1473a0a94de90fc6cf26fbad3521e5c21207afc2ddc97bcd", + "v3/files/81/cf988f62c7bd603e3a6ed9c3ecff7476f48de8f63bf022df1bace6b920ef77fe715b74f6b5740a9ca1dcd429d2867d73a8d32cf7f9c85eb2d16ac3d35078e4", + "v3/files/84/66503ca6efa48cbf66a56f8578eeca93ab2dd1427227d0561c0e97aa3ec7c398324f0a85ed5f2dbe41bbadc1629aed864aca96e90c0a28e0ac54662adf2ff7", "v3/files/85/2a6000d178c83e05c233be924f49f276aac9dcd8a93c79104da05bb45edb2232f88d7ae016f412bd239774f5b3186de452a3e6207b55a5d85b61eadb9bc066-exec", + "v3/files/85/42c50c2a195d857cb8e8d6e0e0ab50aa3179f7ca90744bb9926743a045b50b6959c658c656fffd8793051fcb8e4abac616b1677c47c93e9207b0df14741033", + "v3/files/85/ef15beae6788605513edd6ce46ea488a31ca1f4fcd34a9b2af0ee356e59885b089891c78bac98a744b69a249b00b9320f60227522fe3fb143e857778c44a1f", + "v3/files/87/8fd723b82a79c6867ef0ffb8d6dff4235a28176a5b547c59db4cf26d2e23a1c8fc93e45c55aebb7ed54a5aff2fb390cc9578b28c7c94fe0521f4994163b925", + "v3/files/89/927216209b9f9453a9d11d72e6479feecdd0e25f523e0dc5e471708081352728a7f0ad74e93d2e00e29f43bc36828fb4807d612eb81c431efc24d44a1b6fc7", "v3/files/8a/029e32830002bc6b920148ef245ccb291b1b349b02848aa5ce99ca75c6d3a74e1db0bffdea1502a35925737bceb4c08223fa27850912e7a65b7da9788577a4-exec", + "v3/files/8b/1df454135be1140b6b93368e07c3e8827dfb55ca99e4442ab398bfbac3a161990a29f4eb4e6f131ded4c055be6436fb5e0b0670042fa71067d0e231cb83d17", + "v3/files/8b/b946cb640c1182d98c0ad15fcd1949acd4ccbe83e402efc0b240f8d306006b1ed9505641c7b80f53230b895bf4c2ed62d89ff52b79496e0025e82a6ec8bc02", + "v3/files/8d/a6fa05e41b42b07bce045f7c445ae055d260d9d489746b4991a015307a6fe177d0b3816e3cb77dc8f85384a688c9c51294df5643e46ffda3b451ec11e6bb30-index.json", + "v3/files/8d/bc3567b9616a81488e0da1b07a6de9017bee3ff19e009658d9214d1024002dc40d1cf5088470c420fb518aafd6d12e06a7230b91a951551b9624280a5554bf", + "v3/files/90/21f610d623865c7023b69ef3dca63083f6b5978d70924e72a9df89f9e61d088778ae265a719ce30f328288942291b5ab26aead99114ab62dc0708420c3616c", + "v3/files/90/593a81296a4f848dcd1302eea531f848fb574232dc289230c4302ae535552ae6437bda0684ca595857e48dc5fe46ecabd82bab63925a05c63b270af4a84236", + "v3/files/92/ca2a8d9a21324f0dbee51c50e2c953fb60ea681fd53251ce68fd21bda4ba03be900d20b334cff8cfaaa9bb10faad4b2b04a3cdec0184ab4131f7772b8482ea", "v3/files/94/33bc9e6d478cfedd62c83ad0bd674514f3920edd1833569de1018dbeeacbb159726280fd5a9fbb79e703d1522d8e0a348ab94cb4b649df8e9effc9735fef1e-exec", + "v3/files/94/db0ff9a334a7f9a2a6540d6976023ea4f420aaf890284b3e5882e8e00d3a7905d2fe211517ebf563fa15f6f43d9368025276c4e5b375035b6290c06c7e781e", + "v3/files/98/d8250cb1b661359ada8be5d2a35bb7056856936ba21ab77e0c059a179db262b13a7ee11f5566907f744a780994578afeced2b1805a9c8a85ffe311caa63157", + "v3/files/9a/b96df11753b3923c1a191c05de6c6d3ae35f1f09052fffbc89837b44838c6547c57cf56dc176d44cf4a207275cfabf9bae02eed92028879a357c27044d9379", "v3/files/9b/9b2bdd63c75e4d374106d78af27f7a87525ff3ee9b282be103cc3ce98171f4e17cd9a28eecf8e7004868103f609c453c7e777e3e9083c4a4a88e0c72a0fcba-exec", + "v3/files/9b/ad5f7b6f44db3f82215d60d462c4d28b00cc8c743e8167ceec9f112ea38ff5570726c3a79d49ea3ef70f34f0ed138e2b5664252b894f024564ef62054717c4", + "v3/files/9c/8b2def76ae5ffe4d636166bf9635d7abd69cdac4bf819a2145f7969646d39ae95c96364bc117f9fa544b98518c294233455d4f665af430c75d70798dd4ab13", + "v3/files/9d/e93ee7b458a9d6b97664022909ad25a7cb89c2cfdd8ee19aa2e126566b7a7a930b24143a2a76f83dbff19f1a67b0a71de93e8ab248720c2ee243396e869451", + "v3/files/9e/2449cf9cd108e7d15bc68c57b45ab4ecbb84b8ec7d6c2e7a367a47aa4ede65c7c4aa570b39940be77214c666730c1fa5250ac76740345b6e0eef852ccf8a52", "v3/files/a0/09e43d84d4781d35163d895ca635ead87b2d2f8d9b1689ed09ab64e0536195764b067edbc893a1fed95367d6a99e382455cd8331c8d09cfe173b0fe1126c24-exec", + "v3/files/a0/680b324fe0a2ac774c6b55ae40c82ec51f49e42f59718c1fa708ff46f7861f3889bea1be376b684321788747fefabf9207469f3e46467ffc8cd2f0e6b9486e", + "v3/files/a0/a9db845c91217a54b9ecfc881326c846b89db8f820e432ba173fc32f6463bfd654f73020ef5503aebc3eef1190eefed06efa48b44e7b2c3d0a9434eb58b898-index.json", + "v3/files/a1/c6cb0a6a96eac16d92b1b2b0563c03339d484429b2f4d543bded0481f0bae4dec7944dc51a5948b22cf3fa63205424c7f745ca2181b728106508292a66cf60", + "v3/files/a1/e2fc211820624ccf43443c0db06e9161ac1bfe2faf8d1b0fcbad391cbd7dfd65ba15dcab537ea42a35b297127d2513509fbb809d06bd666b3b8e9ab0d0b910", "v3/files/a2/a9cb7d3de319d0f058f6af6f212c6d48d388cb17eaa833df307c81b9cd5a384ccbdd14dbe6616a401292e77c67f4ee311af75ec026ff2cb2bafc6449decdbd-exec", "v3/files/a2/bbd0fc27dcb18788968a58f1c9d38fa49e27b32d6b942d1ea0f8d9d81a056400478137dcec81b6ddeda34addd14574690c5bef79b1b4d1145f0df7418d3abe-exec", + "v3/files/a3/253c94a2a6ebcf1f26a558a5cb24475d03c3ed2e52f6429aa4d23103bb959016d6631308a4094ca334cda03a7e4c50b5eee74e8846db457f079760e5c669c3", + "v3/files/a4/a9e2494e1e0da2eea001379babc849ecd0349e3c00c7b0a1cb01fef1d387a9c7113631a156778fd6092fb5c1c75eb9afafabd80e898e8fc4abf1eb8094a216", + "v3/files/a6/e92c60a1d072f4572d295a8a3e7a0a105c02c9e7ab824dc5e7d41ecf1bed3e7b2ea56a944e8c11742d163070ca819dbd338119c1944343c0b28a6eef65bc1b", + "v3/files/a7/8bf25ca4359bdc9c5ae6f1f69c21dfe6c3bba4628fb2bbcbb8ced3440d43e77f3a46a2b85432fcffb75c320e9ccbef7a897b8218f087e5eeca58cc494b17b0", "v3/files/a8/1983c1c62d075b3c76b1ddc412ed3ae745a30f6f11f580dcc64dcc2b088110e2b55e9dbaa9fbbaaadb3c2d81108773408658b653fed018c50804fe809b0297-exec", + "v3/files/a9/733d4214604b7aac76fb98b2d9cbb3109ee5142d37f2136672f2e2815ad5a826b36c5a8d5e8cfb39b4492bf7f51d1fcbd3a3193acd8adce9340a41f9035150", + "v3/files/aa/9080bd197db2db8e1ef78ab27ec79dc251befe74d6a21a70acd094effe2f0c5cf7ed2adb02f2bf80dfbedf34fc33e7da9a8e06c25d0e2a205c647df8ebf047-index.json", + "v3/files/ae/2f05ef38831679089b850b72dd72245ea3e51d65597dc11fafde5f6a991dde21f5f413f63c61a02cfdcb09417bcac06db5c006222c602d785a43b5a1f13032", "v3/files/af/93b76be9167844f6d773f3e20038c6bcc415754e5402a5758e6b6dc59e29602997e2c831c17cf3a61e7b800a33f0eadee2a55708c6d7f2db1479c37ddb9780-exec", "v3/files/b2/9f0073f86f7e9aed174fce942493e1e4a8fa5dac6afd4ff2108c74056f618e9e83cf6acf8fac63def0686fc46e266fa0bc90da004be5fb34df4f7f061e80a3-exec", + "v3/files/b3/79ee82766e9820bffe7e26a0619eead90d33260475a0c18ad563c0fc57991e659cc1b974778f8a71f611dc641b7461d608451c78b1006e0d261a495b5aed85-index.json", + "v3/files/b4/6d960ff0ac8944ee6b1fd0306872c53a77640f5fb98ea7c66c352d21641441d8fc4bceb5d384aff7779feb5603a08488cd446e7a561ba408377db200815999", + "v3/files/b6/fc0809956c33c1eb37a0c6ad44c109f2c1c77a9b161c8b3df0562bc302391ded52c853033749fd9cd8af0c7c24fd6da57144f11c6dc4eadb5edcede5ffd877", "v3/files/b7/f1e5dc2e73938d0dc0a7a1349c4c0e88ca1b5dff93164f254c4d03240ee7d58984b0f19fc42050ef51f79815ab41f8235953c87d46a894b62930f3ca32ccc8-exec", + "v3/files/b8/a7cb7d0dbfe703933a77aa6d41c78aac3c7f74bb0aa7d4ccb412b222558f8d7ca1ce361aecaa4e3713b0366bd999ca14c8d1dd0c3d9890badc208a0e436433", + "v3/files/b9/14414f631f10ecb1b840ac78265ddbc8c2d4027c866aaa9de590e155944e008016f6f7c0a5358cbaa3e9e802e04c6bafcf28820ddda7eceba07e3026e697cf-index.json", "v3/files/b9/acf9e03ebcc7627deb8b26a0029e0b51fe3ff6b6d3a8601dd1fc684235fe0b527a0da83bb913a15fac0b86ee6776093ca69610e36774ad29f221969a23a766-exec", "v3/files/bc/16419489153aa19e5a0c2989286941310a41478678fa91249878b232f9ffbfa5f9935846dc9ec57e98be60a489dd679de4ed9990f843570aa0fa3ba81e5356-exec", + "v3/files/bd/a3d5a1409e6cd87f61d971ddf347c11825130591ad261b6b92ec93e64872d146731283e14f497fc375c676298bed3446a0de53406b156781fb4c5342c33ade", + "v3/files/bd/acd8bef8171e84e07964785d04e504cc2017167cbc3cb6e284b6db802454e10a6797f4d02d1cfc0ff7dc44c9d63ac84157d22410ead7ca9c28cd50c4a8b065", + "v3/files/be/6f21427e4cba83248c962f0411d302db1dadcb97995aca6702f7144ab7da81a2285858e7ad85a144136d7bf6e30934af0255a60ae26d408e214f7c21a4bd32", + "v3/files/c0/997399d1b86a27a8e3d8bc4a285bfadc1243b9337381c9c13ea8e59b5709e0769a9863ec18656c1f6dab2ff6e29da5b0b008ad99c6e2b0fb4dec4de543a018", + "v3/files/c0/cc33d9cad1836b35a6c34061a9faf44a7e2d10633db1bbc0f16cb1ca5754c7098c8393ebe736e1555bf18f68b52c667f98661d85ef4a54db7f07d2d274709c-index.json", "v3/files/c2/65c48634b48a6a4b70c9cfcdae4c44b2eaf59ab39152ef1fc40cd48c96c8a9da0b3549dcc3be6298edd9e75f891cd2cc3390854fb5c95a168165d0222876d2-exec", + "v3/files/c3/ee6434ab4d901f15830939ca719016c24f7ad4d3809214a7682c571a67b0b4546868e662a21b12d74be6416012d2ebc9f2f45e8c909dceb29e9d123b8d3536", + "v3/files/c5/57dce947b9939088714d013c9a417e8d38d08b63efe74ec1e309f7d08302ebe27a4931078c131a40ce223315d4f679dd6241f9a8a98f6c48e6b71789a4fc70", "v3/files/c6/703e2577c586d276c218902674852f8bfd6ca4b07f018512e255e1f20853ca182ee85769961e78431ea9812bfeb09f031ad581f9e31533f12b9d43e587e757-exec", + "v3/files/c9/a6d4755cb30105d20d11099b30b209c9b923b90aa928da22a4b2532398931bedde1b7da8f7fd6ca72d20241b2684d7e200d38da79c024bab9d646540e1154e", + "v3/files/cc/6ef388a93c5c4bbb57078ad84343df773366871a5239468ec1c1757b6dcea5e14d571261cd017a935bc493def5d5fc61009dfe680db01c55595a1ff2b9bc40", + "v3/files/cd/02624dcc73c24ee0b488b1798a60de203d3487d6b0f75a869bd0bc9845f30e907a90ee727d1f64ffc3c6824fb2dfc900f13406ab63c827da719babc3407c9e", + "v3/files/cd/b07dac22404f5adb8e25436f686a2851cd60bc60b64f0d511c59dc86700f717a36dc5b5d94029e74a2d4b931f880e885d3e5169db6db05402c885e64941212-index.json", + "v3/files/d1/c303436fde8d31686f2375b30b31e6b107c7b4f3cb2566194347049e385ebfb241aa56491516a815f8a5395d256ba121b71cd2a5032cc3c94c2755ca4fb29b", + "v3/files/d1/dbaab34145159f6b9cdf552f24a4e817e98369d330b7cad8d28d9a71dde33601d57f36e0e6cbadafee8a3df4dac525f7a47d164f262fe8afdf0dd1f0847abc", + "v3/files/d2/fa1afc9873a0f4de45cd958ed20fbcfb11096759be6eee582f4de939e03bb6a07c8ae7cfee86a50d57949dd3f9fe3b053a95947ef44ff06f7a3d02a4f1eb9b", + "v3/files/d5/b90e7a534ce77c513674a4a09c8fa9908601060f8bd44bba7ac40243a7fc011a6c65be7e72134202872698103451c8b979bba2f15e6b6147c6e70008007f9c", "v3/files/d6/f84a85f42a7c7971d607ee2df1952351246b079278bcbafda757f643094c9bbafb430e1aca63af939433b2216232e30aed4d5ea7854fb1ff824051e171d91e-exec", + "v3/files/d9/9fbd174c3f4bcefcbcf765b2e8f76211f7867059fd423580cef58ad3012e6fcbbdc87f3d849ed0d3c88c3e52304e59b6eb27c10307c514064807af43893877", + "v3/files/d9/b1f8849e09ebe0ccce0995993ccf7f83ee38f9414cefbf0ef34dcae21d42f315ccad4032a40eebdd7751765e338423fe7fb4e5b85b71d5debbe3bba894d9f8", + "v3/files/da/2d40a90eb81ee2fd0f2add4293f43902903711af0a64c16a7d78e20913842c4fb0ca62c04c4d92ceb2703a966423d962fa60fb4181fc213d99f1a0b4339297", "v3/files/dd/8a7b4d525f8cba564e94846ed484f960663b4df324cbedccc422cd899b2139e88317e811a5e3c269baa641a7348de6359314ef3a5c1e6cc3f7ff5f23b8e1fe-exec", + "v3/files/de/3d6710a6195ac143a35a4688c78ddd8154ab002733f8c9cc1a4d9bd5cd77c77fcae53e97fd04c9f2a15b6ae245c8f4ad0b7abf13f064efe028f77635ba93fc", + "v3/files/e1/d0af67959826971b20963844f5213816c5b9dd75e7a46bed1a61b91d76ffe997294788a42c68976fee58be160c534d9521fdd3d336018e1f88b589a3cf9f4f", + "v3/files/e3/b6870966beed0a421519f885a8e6a9f69b7cd06319ed7250276913f1863e5f8b5c967895d205ca8d66f11da1f0ce631c2ed7506391132d12001db7bf8fa1bc", "v3/files/e4/39669d73b83f43eeaabee13f4a06190c43b831544e44617fd955d5dddfefe0073d0d84edc0a614387a5cfce618668041ef0238234f2be99f48d35f7efe23c0-exec", "v3/files/e5/3dee301557be2befd6403b92ad496e2473c0398c67072ae929675626e0f9bba070459ed772735fdc105a594a87f07900c68e745a5e636be753e0ba6b9c315a-exec", + "v3/files/e7/61df28bc8959531918e3b4a245629170909f6effd65f7e9db86a7bb81c9c83dd806ff3fb81426256dfaef29eb1588f49f4531952d54de35de431be6b914da8", + "v3/files/e7/787d498e4bc304540dc76364d498e72b750c870a4550b5bf1d4cb1bd0eccac00d2817482b834dfac86c1160790d772041d1a384e6a0545ef26f95e8bdada94", + "v3/files/ea/e061696e3726f2ce3c01f38e142982820ea7076590c8825dc7ba721a4e66af2742e6e59d90532dcec3a6c3647dc8a6e207d109b371b4905674409fe4ac4923", + "v3/files/eb/3773b87d3b91f4cd2d92a80c21ddff13189c501f1e17c06183f3e0b31cc95af66c99c5b7e36b393d9f7b23faf566d9c1cd623794c36e29de1df96708e19abd", + "v3/files/ed/41899967bb0874f42354d98be31d21cb9b6ce94294a994eef5214deff81fe8a9e287f926d0b54405e1ac2737ede914fdd4e67fb6461e874306cd22dcdbddb7", + "v3/files/ee/ae99bb2392ca88e1e3dfc2d23960c3c5c3b38870773c5a6e1bfce88de2b8989b2fa1b0ecf68a26e06cb4c9c46efe840c86be78b62bbf5c2abca2023ba7f536", "v3/files/f0/3e0687f9a3d13087f7b966cd0ef1a143e5e70216be4209073bb15738e6c45a0864f619ca3274269d658020c9837b6c5584601fc1dbea2601fd2314df1830c6-exec", + "v3/files/f1/3a40e1bc634f7d71b88cff6c2a94cc32507508bf6c9dd5a5122e5dfeeaf7e8b56148b2ad4acfcc2ca4866a27315d4d1d59d4482cdb948692e0226982833a80", "v3/files/f1/eac3da4b2d7d1965a923a7fd9d09fefccd711d7f8438d1156fcf0fbd46e678ccb4c60262a80ca9eb9380df498812d7b5ec893d82bd68f8c0148c71e1df0530-exec", + "v3/files/f2/5d079a1627100d33f0d252fc22bd535b4393dc7720b0abc77f167c98a5c88f81f571a06010fbdcd3fbe2f25e0b5ce0ed8242c19987bd922b47580c9ca2d8f2", + "v3/files/f2/9cacfaa15744fc96690a101e4224d95fe85e119fd93fee568a040b3cf80062b9876e9335e5fce60950ce95a7f8cd4f140e3dc9dc34919192b8c326677c84bc", + "v3/files/f4/a393da4150b41f91fa7f3206945627f98a728a65379c6f32c33b074fc58c7219d0ab2c84c8d9d91e54e958c05601589713f9ccfc34070f7e1cda87d86dd475", + "v3/files/f5/cb35226d3afdf2c03c1a57398f1f1cc0f62e050cde444468a94e5fb4ba7cb5b06c6e46f94aa552ba9d9132a5a113de0e084409ecf13a251fcef5ba9671230e", + "v3/files/f5/e89cbeefdbbfe6152bbf665b784ce61b88406b5a77354583bf88600731ba8b33f2d259ccfc519fe2ecfd7aa72cd0c955b322dc509f334ae4aed9d3895f0205", + "v3/files/f7/9436e735ef8c9de2498ac5febc9a9544b2930345686a982ce3ce9c85492e1b15d0fa4a51d016db4752c2e9a8c15ac54d19e55c943f5176292594b3f7a5adb8", + "v3/files/fa/a9e95cb8bef48d68623066d67d307a9f9a9606d88dfc7aca7780a5eba852ba395bbc48bcc4340cc07575b6661658ec6c993f30cf7077b1a0dcb381d1c0279b", + "v3/files/fb/2bd0916b04c64593856912b1a45034d575a7619df1e2f495712b11dfdd9a78f7d8a290dfc8785ddc1978c623057687836c6e460dbe62ab8c2a9874452ada59", + "v3/files/fc/1d65352c114c7594c9bedf5be432ba39d426feaf50bf8f7c52d32781323c84bfc9a68531aefb558c97ebe46e712e1d35d860ba1e1a6ab48b4a79b894092540", + "v3/files/fd/310b991968382a88f0cd0490ee6db22c2199a0133b6926972ee620e1b62ea45606a10c82d0c2c0367e732f261d9f26f7bbfcae71d6e72c80d9273706cafec1", + "v3/files/fe/198650f9a9d205e0c7f3afe68196e83014ddce21e1a2cc6bf6c4d539296473d4edd0b1e49152f61a18338ffef75f4458268fb41856ccafbaa46ffac5dc7e36", "v3/files/ff/00a34fb9c309279edf249bca6d5e54c22d2151b118f358f4e737875bb6e8162e2af9cf5f6f60693f0f3c481b0c2f4a3d28e6f91dcc5c9db7cfe3c527efabbc-exec", ] From aac7505d8160af825c9bd199ba580adca1021871 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 13:27:36 +0700 Subject: [PATCH 076/102] test(cli/install): snapshot last --- crates/cli/tests/install.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index b31a3984b..141729927 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -67,9 +67,6 @@ fn should_install_exec_files() { eprintln!("Listing all files in the store..."); let store_files = root.path().join("pacquet-store").pipe_as_ref(get_all_files); - eprintln!("Snapshot"); - insta::assert_debug_snapshot!(store_files); - #[cfg(unix)] { use pacquet_testing_utils::fs::is_path_executable; @@ -84,5 +81,8 @@ fn should_install_exec_files() { ); } + eprintln!("Snapshot"); + insta::assert_debug_snapshot!(store_files); + drop(root); // cleanup } From 27502716d904caeae5dc4d259884cdc53a593d08 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:10:40 +0700 Subject: [PATCH 077/102] feat(config): default store dir to pnpm Resolves https://github.com/pnpm/pacquet/pull/166#discussion_r1378224352 --- crates/cli/tests/add.rs | 4 +- crates/cli/tests/install.rs | 6 +-- .../add__should_install_all_dependencies.snap | 46 +++++++++---------- .../add__should_symlink_correctly.snap | 20 ++++---- .../install__should_install_dependencies.snap | 22 ++++----- crates/npmrc/src/custom_deserializer.rs | 22 ++++----- 6 files changed, 60 insertions(+), 60 deletions(-) diff --git a/crates/cli/tests/add.rs b/crates/cli/tests/add.rs index 2f041cebc..649488091 100644 --- a/crates/cli/tests/add.rs +++ b/crates/cli/tests/add.rs @@ -18,7 +18,7 @@ fn should_install_all_dependencies() { eprintln!("Ensure the manifest file ({manifest_path:?}) exists"); assert!(manifest_path.exists()); - let virtual_store_dir = workspace.join("node_modules").join(".pacquet"); + let virtual_store_dir = workspace.join("node_modules").join(".pnpm"); eprintln!("Ensure virtual store dir ({virtual_store_dir:?}) exists"); assert!(virtual_store_dir.exists()); @@ -51,7 +51,7 @@ pub fn should_symlink_correctly() { eprintln!("Ensure the manifest file ({manifest_path:?}) exists"); assert!(manifest_path.exists()); - let virtual_store_dir = workspace.join("node_modules").join(".pacquet"); + let virtual_store_dir = workspace.join("node_modules").join(".pnpm"); eprintln!("Ensure virtual store dir ({virtual_store_dir:?}) exists"); assert!(virtual_store_dir.exists()); diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 141729927..d38e7e917 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -28,17 +28,17 @@ fn should_install_dependencies() { eprintln!("Make sure the package is installed"); assert!(is_symlink_or_junction(&workspace.join("node_modules/is-odd")).unwrap()); - assert!(workspace.join("node_modules/.pacquet/is-odd@3.0.1").exists()); + assert!(workspace.join("node_modules/.pnpm/is-odd@3.0.1").exists()); eprintln!("Make sure it installs direct dependencies"); assert!(!workspace.join("node_modules/is-number").exists()); - assert!(workspace.join("node_modules/.pacquet/is-number@6.0.0").exists()); + assert!(workspace.join("node_modules/.pnpm/is-number@6.0.0").exists()); eprintln!("Make sure we install dev-dependencies as well"); assert!( is_symlink_or_junction(&workspace.join("node_modules/fast-decode-uri-component")).unwrap() ); - assert!(workspace.join("node_modules/.pacquet/fast-decode-uri-component@1.0.1").is_dir()); + assert!(workspace.join("node_modules/.pnpm/fast-decode-uri-component@1.0.1").is_dir()); eprintln!("Snapshot"); let workspace_folders = get_all_folders(&workspace); diff --git a/crates/cli/tests/snapshots/add__should_install_all_dependencies.snap b/crates/cli/tests/snapshots/add__should_install_all_dependencies.snap index 0591e544f..93fc2c04a 100644 --- a/crates/cli/tests/snapshots/add__should_install_all_dependencies.snap +++ b/crates/cli/tests/snapshots/add__should_install_all_dependencies.snap @@ -1,30 +1,30 @@ --- source: crates/cli/tests/add.rs -assertion_line: 19 -expression: get_all_folders(dir.path()) +assertion_line: 14 +expression: get_all_folders(&workspace) --- [ "node_modules", - "node_modules/.pacquet", - "node_modules/.pacquet/is-buffer@1.1.6", - "node_modules/.pacquet/is-buffer@1.1.6/node_modules", - "node_modules/.pacquet/is-buffer@1.1.6/node_modules/is-buffer", - "node_modules/.pacquet/is-buffer@1.1.6/node_modules/is-buffer/test", - "node_modules/.pacquet/is-even@1.0.0", - "node_modules/.pacquet/is-even@1.0.0/node_modules", - "node_modules/.pacquet/is-even@1.0.0/node_modules/is-even", - "node_modules/.pacquet/is-even@1.0.0/node_modules/is-odd", - "node_modules/.pacquet/is-number@3.0.0", - "node_modules/.pacquet/is-number@3.0.0/node_modules", - "node_modules/.pacquet/is-number@3.0.0/node_modules/is-number", - "node_modules/.pacquet/is-number@3.0.0/node_modules/kind-of", - "node_modules/.pacquet/is-odd@0.1.2", - "node_modules/.pacquet/is-odd@0.1.2/node_modules", - "node_modules/.pacquet/is-odd@0.1.2/node_modules/is-number", - "node_modules/.pacquet/is-odd@0.1.2/node_modules/is-odd", - "node_modules/.pacquet/kind-of@3.2.2", - "node_modules/.pacquet/kind-of@3.2.2/node_modules", - "node_modules/.pacquet/kind-of@3.2.2/node_modules/is-buffer", - "node_modules/.pacquet/kind-of@3.2.2/node_modules/kind-of", + "node_modules/.pnpm", + "node_modules/.pnpm/is-buffer@1.1.6", + "node_modules/.pnpm/is-buffer@1.1.6/node_modules", + "node_modules/.pnpm/is-buffer@1.1.6/node_modules/is-buffer", + "node_modules/.pnpm/is-buffer@1.1.6/node_modules/is-buffer/test", + "node_modules/.pnpm/is-even@1.0.0", + "node_modules/.pnpm/is-even@1.0.0/node_modules", + "node_modules/.pnpm/is-even@1.0.0/node_modules/is-even", + "node_modules/.pnpm/is-even@1.0.0/node_modules/is-odd", + "node_modules/.pnpm/is-number@3.0.0", + "node_modules/.pnpm/is-number@3.0.0/node_modules", + "node_modules/.pnpm/is-number@3.0.0/node_modules/is-number", + "node_modules/.pnpm/is-number@3.0.0/node_modules/kind-of", + "node_modules/.pnpm/is-odd@0.1.2", + "node_modules/.pnpm/is-odd@0.1.2/node_modules", + "node_modules/.pnpm/is-odd@0.1.2/node_modules/is-number", + "node_modules/.pnpm/is-odd@0.1.2/node_modules/is-odd", + "node_modules/.pnpm/kind-of@3.2.2", + "node_modules/.pnpm/kind-of@3.2.2/node_modules", + "node_modules/.pnpm/kind-of@3.2.2/node_modules/is-buffer", + "node_modules/.pnpm/kind-of@3.2.2/node_modules/kind-of", "node_modules/is-even", ] diff --git a/crates/cli/tests/snapshots/add__should_symlink_correctly.snap b/crates/cli/tests/snapshots/add__should_symlink_correctly.snap index 04f3d30f4..fde599375 100644 --- a/crates/cli/tests/snapshots/add__should_symlink_correctly.snap +++ b/crates/cli/tests/snapshots/add__should_symlink_correctly.snap @@ -1,17 +1,17 @@ --- source: crates/cli/tests/add.rs -assertion_line: 57 -expression: get_all_folders(dir.path()) +assertion_line: 47 +expression: get_all_folders(&workspace) --- [ "node_modules", - "node_modules/.pacquet", - "node_modules/.pacquet/is-number@6.0.0", - "node_modules/.pacquet/is-number@6.0.0/node_modules", - "node_modules/.pacquet/is-number@6.0.0/node_modules/is-number", - "node_modules/.pacquet/is-odd@3.0.1", - "node_modules/.pacquet/is-odd@3.0.1/node_modules", - "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-number", - "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-odd", + "node_modules/.pnpm", + "node_modules/.pnpm/is-number@6.0.0", + "node_modules/.pnpm/is-number@6.0.0/node_modules", + "node_modules/.pnpm/is-number@6.0.0/node_modules/is-number", + "node_modules/.pnpm/is-odd@3.0.1", + "node_modules/.pnpm/is-odd@3.0.1/node_modules", + "node_modules/.pnpm/is-odd@3.0.1/node_modules/is-number", + "node_modules/.pnpm/is-odd@3.0.1/node_modules/is-odd", "node_modules/is-odd", ] diff --git a/crates/cli/tests/snapshots/install__should_install_dependencies.snap b/crates/cli/tests/snapshots/install__should_install_dependencies.snap index a1b28283d..a4e231e9f 100644 --- a/crates/cli/tests/snapshots/install__should_install_dependencies.snap +++ b/crates/cli/tests/snapshots/install__should_install_dependencies.snap @@ -6,17 +6,17 @@ expression: "(workspace_folders, store_files)" ( [ "node_modules", - "node_modules/.pacquet", - "node_modules/.pacquet/fast-decode-uri-component@1.0.1", - "node_modules/.pacquet/fast-decode-uri-component@1.0.1/node_modules", - "node_modules/.pacquet/fast-decode-uri-component@1.0.1/node_modules/fast-decode-uri-component", - "node_modules/.pacquet/is-number@6.0.0", - "node_modules/.pacquet/is-number@6.0.0/node_modules", - "node_modules/.pacquet/is-number@6.0.0/node_modules/is-number", - "node_modules/.pacquet/is-odd@3.0.1", - "node_modules/.pacquet/is-odd@3.0.1/node_modules", - "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-number", - "node_modules/.pacquet/is-odd@3.0.1/node_modules/is-odd", + "node_modules/.pnpm", + "node_modules/.pnpm/fast-decode-uri-component@1.0.1", + "node_modules/.pnpm/fast-decode-uri-component@1.0.1/node_modules", + "node_modules/.pnpm/fast-decode-uri-component@1.0.1/node_modules/fast-decode-uri-component", + "node_modules/.pnpm/is-number@6.0.0", + "node_modules/.pnpm/is-number@6.0.0/node_modules", + "node_modules/.pnpm/is-number@6.0.0/node_modules/is-number", + "node_modules/.pnpm/is-odd@3.0.1", + "node_modules/.pnpm/is-odd@3.0.1/node_modules", + "node_modules/.pnpm/is-odd@3.0.1/node_modules/is-number", + "node_modules/.pnpm/is-odd@3.0.1/node_modules/is-odd", "node_modules/fast-decode-uri-component", "node_modules/is-odd", ], diff --git a/crates/npmrc/src/custom_deserializer.rs b/crates/npmrc/src/custom_deserializer.rs index c7ea76ded..6066ba9ed 100644 --- a/crates/npmrc/src/custom_deserializer.rs +++ b/crates/npmrc/src/custom_deserializer.rs @@ -40,17 +40,17 @@ fn default_store_dir_windows(home_dir: &Path, current_dir: &Path) -> PathBuf { get_drive_letter(home_dir).expect("home dir is an absolute path with drive letter"); if current_drive == home_drive { - return home_dir.join("AppData/Local/pacquet/store"); + return home_dir.join("AppData/Local/pnpm/store"); } - PathBuf::from(format!("{current_drive}:\\.pacquet-store")) + PathBuf::from(format!("{current_drive}:\\.pnpm-store")) } /// If the $PNPM_HOME env variable is set, then $PNPM_HOME/store -/// If the $XDG_DATA_HOME env variable is set, then $XDG_DATA_HOME/pacquet/store -/// On Windows: ~/AppData/Local/pacquet/store -/// On macOS: ~/Library/pacquet/store -/// On Linux: ~/.local/share/pacquet/store +/// If the $XDG_DATA_HOME env variable is set, then $XDG_DATA_HOME/pnpm/store +/// On Windows: ~/AppData/Local/pnpm/store +/// On macOS: ~/Library/pnpm/store +/// On Linux: ~/.local/share/pnpm/store pub fn default_store_dir() -> StoreDir { // TODO: If env variables start with ~, make sure to resolve it into home_dir. if let Ok(pnpm_home) = env::var("PNPM_HOME") { @@ -73,8 +73,8 @@ pub fn default_store_dir() -> StoreDir { // https://doc.rust-lang.org/std/env/consts/constant.OS.html match env::consts::OS { - "linux" => home_dir.join(".local/share/pacquet/store").into(), - "macos" => home_dir.join("Library/pacquet/store").into(), + "linux" => home_dir.join(".local/share/pnpm/store").into(), + "macos" => home_dir.join("Library/pnpm/store").into(), _ => panic!("unsupported operating system: {}", env::consts::OS), } } @@ -86,7 +86,7 @@ pub fn default_modules_dir() -> PathBuf { pub fn default_virtual_store_dir() -> PathBuf { // TODO: find directory with package.json - env::current_dir().expect("current directory is unavailable").join("node_modules/.pacquet") + env::current_dir().expect("current directory is unavailable").join("node_modules/.pnpm") } pub fn default_registry() -> String { @@ -189,7 +189,7 @@ mod tests { let home_dir = Path::new("C:\\Users\\user"); let store_dir = default_store_dir_windows(&home_dir, ¤t_dir); - assert_eq!(store_dir, Path::new("D:\\.pacquet-store")); + assert_eq!(store_dir, Path::new("D:\\.pnpm-store")); } #[cfg(windows)] @@ -199,6 +199,6 @@ mod tests { let home_dir = Path::new("C:\\Users\\user"); let store_dir = default_store_dir_windows(&home_dir, ¤t_dir); - assert_eq!(store_dir, Path::new("C:\\Users\\user\\AppData\\Local\\pacquet\\store")); + assert_eq!(store_dir, Path::new("C:\\Users\\user\\AppData\\Local\\pnpm\\store")); } } From eea24593f94f78dc4c869e1452bd24323e9a4cb0 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:07:19 +0700 Subject: [PATCH 078/102] perf: set mode at the same time as writing files --- crates/fs/src/lib.rs | 54 ++++++++++++++++++++++++++++-- crates/store-dir/src/cas_file.rs | 29 +++++++--------- crates/store-dir/src/index_file.rs | 2 +- crates/tarball/src/lib.rs | 8 +---- 4 files changed, 65 insertions(+), 28 deletions(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 05bf8a469..65334eec4 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -1,10 +1,33 @@ use derive_more::{Display, Error}; use miette::Diagnostic; use std::{ - fs, io, + fs::{self, OpenOptions}, + io::{self, Write}, path::{Path, PathBuf}, }; +pub mod file_mode { + /// Bit mask to filter readable bits (`r--r--r--`). + pub const READ_MASK: u32 = 0b100_100_100; + + /// Bit mask to filter executable bits (`-w--w--w-`). + pub const WRITE_MASK: u32 = 0b010_010_010; + + /// Bit mask to filter executable bits (`--x--x--x`). + pub const EXEC_MASK: u32 = 0b001_001_001; + + /// All can read, write, and execute (`rwxrwxrwx`). + pub const ALL_RWX: u32 = 0b111_111_111; + + /// All can read and write, but not execute (`rw-rw-rw-`). + pub const ALL_RW: u32 = 0b110_110_110; + + /// Whether a file mode has any executable bit. + pub fn is_executable(mode: u32) -> bool { + mode & EXEC_MASK != 0 + } +} + /// Create a symlink to a directory. /// /// The `link` path will be a symbolic link pointing to `original`. @@ -24,6 +47,12 @@ pub enum EnsureFileError { #[error(source)] error: io::Error, }, + #[display("Failed to create file at {file_path:?}: {error}")] + CreateFile { + file_path: PathBuf, + #[error(source)] + error: io::Error, + }, #[display("Failed to write to file at {file_path:?}: {error}")] WriteFile { file_path: PathBuf, @@ -35,7 +64,11 @@ pub enum EnsureFileError { /// Write `content` to `file_path` unless it already exists. /// /// Ancestor directories will be created if they don't already exist. -pub fn ensure_file(file_path: &Path, content: &[u8]) -> Result<(), EnsureFileError> { +pub fn ensure_file( + file_path: &Path, + content: &[u8], + #[cfg_attr(windows, allow(unused))] mode: Option, +) -> Result<(), EnsureFileError> { if file_path.exists() { return Ok(()); } @@ -45,7 +78,22 @@ pub fn ensure_file(file_path: &Path, content: &[u8]) -> Result<(), EnsureFileErr parent_dir: parent_dir.to_path_buf(), error, })?; - fs::write(file_path, content) + + let mut options = OpenOptions::new(); + options.write(true).create(true); + + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + if let Some(mode) = mode { + options.mode(mode); + } + } + + options + .open(file_path) + .map_err(|error| EnsureFileError::CreateFile { file_path: file_path.to_path_buf(), error })? + .write_all(content) .map_err(|error| EnsureFileError::WriteFile { file_path: file_path.to_path_buf(), error }) } diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index a258d06a7..63c93decc 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -1,15 +1,15 @@ use crate::{FileHash, StoreDir}; use derive_more::{Display, Error}; use miette::Diagnostic; -use pacquet_fs::{ensure_file, make_file_executable, EnsureFileError}; +use pacquet_fs::{ensure_file, file_mode::is_executable, EnsureFileError}; use sha2::{Digest, Sha512}; use std::path::PathBuf; impl StoreDir { /// Path to a file in the store directory. - pub fn cas_file_path(&self, hash: FileHash, executable: bool) -> PathBuf { + pub fn cas_file_path(&self, hash: FileHash, mode: u32) -> PathBuf { let hex = format!("{hash:x}"); - let suffix = if executable { "-exec" } else { "" }; + let suffix = if is_executable(mode) { "-exec" } else { "" }; self.file_path_by_hex_str(&hex, suffix) } } @@ -25,17 +25,11 @@ impl StoreDir { pub fn write_cas_file( &self, buffer: &[u8], - executable: bool, + mode: u32, ) -> Result<(PathBuf, FileHash), WriteCasFileError> { let file_hash = Sha512::digest(buffer); - let file_path = self.cas_file_path(file_hash, executable); - - ensure_file(&file_path, buffer).map_err(WriteCasFileError::WriteFile)?; - if executable { - // TODO: propagate this error - make_file_executable(&file_path).expect("make the file executable"); - } - + let file_path = self.cas_file_path(file_hash, mode); + ensure_file(&file_path, buffer, Some(mode)).map_err(WriteCasFileError::WriteFile)?; Ok((file_path, file_hash)) } } @@ -43,28 +37,29 @@ impl StoreDir { #[cfg(test)] mod tests { use super::*; + use pacquet_fs::file_mode::{ALL_RW, ALL_RWX}; #[test] fn cas_file_path() { - fn case(file_content: &str, executable: bool, expected: &str) { - eprintln!("CASE: {file_content:?}, {executable:?}"); + fn case(file_content: &str, mode: u32, expected: &str) { + eprintln!("CASE: {file_content:?}, {mode:?}"); let store_dir = StoreDir::new("STORE_DIR"); let file_hash = Sha512::digest(file_content); eprintln!("file_hash = {file_hash:x}"); - let received = store_dir.cas_file_path(file_hash, executable); + let received = store_dir.cas_file_path(file_hash, mode); let expected: PathBuf = expected.split('/').collect(); assert_eq!(&received, &expected); } case( "hello world", - false, + ALL_RW, "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", ); case( "hello world", - true, + ALL_RWX, "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", ); } diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index fbca57f38..cb4f331d5 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -50,7 +50,7 @@ impl StoreDir { let file_path = self.tarball_index_file_path(tarball_integrity); let index_content = serde_json::to_string(&index_content).expect("convert a TarballIndex to JSON"); - ensure_file(&file_path, index_content.as_bytes()) + ensure_file(&file_path, index_content.as_bytes(), Some(0o666)) .map_err(WriteTarballIndexFileError::WriteFile) } } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 164434fa0..8f085b636 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -22,11 +22,6 @@ use tokio::sync::{Notify, RwLock}; use tracing::instrument; use zune_inflate::{errors::InflateDecodeErrors, DeflateDecoder, DeflateOptions}; -/// Mask of the executable bits. -/// -/// This value is equal to `S_IXUSR` in libc. -const EXEC_MASK: u32 = 64; - #[derive(Debug, Display, Error, Diagnostic)] #[display("Failed to fetch {url}: {error}")] pub struct NetworkError { @@ -236,7 +231,6 @@ impl<'a> DownloadTarballToStore<'a> { let mut entry = entry.unwrap(); let file_mode = entry.header().mode().expect("get mode"); // TODO: properly propagate this error - let is_executable = file_mode & EXEC_MASK != 0; // Read the contents of the entry let mut buffer = Vec::with_capacity(entry.size() as usize); @@ -246,7 +240,7 @@ impl<'a> DownloadTarballToStore<'a> { let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); let (file_path, file_hash) = store_dir - .write_cas_file(&buffer, is_executable) + .write_cas_file(&buffer, file_mode) .map_err(TarballError::WriteCasFile)?; let tarball_index_key = cleaned_entry_path From 50fb06d5bf03ca44671791cf12adc84ab9ca7819 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:14:25 +0700 Subject: [PATCH 079/102] refactor: clearer bit mask --- crates/testing-utils/src/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/testing-utils/src/fs.rs b/crates/testing-utils/src/fs.rs index 530672a61..d76a82d90 100644 --- a/crates/testing-utils/src/fs.rs +++ b/crates/testing-utils/src/fs.rs @@ -81,5 +81,5 @@ pub fn is_path_executable(path: &Path) -> bool { .metadata() .expect("get metadata of the file") .mode(); - mode & 0o111 != 0 + mode & 0b001_001_001 != 0 } From 4f8752294282aa5a1033a862ba62284a9a514215 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:19:50 +0700 Subject: [PATCH 080/102] refactor: rename TarballIndex -> PackageFilesIndex Resolves https://github.com/pnpm/pacquet/pull/166#discussion_r1378232144 --- crates/store-dir/src/index_file.rs | 6 +++--- crates/tarball/src/lib.rs | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index cb4f331d5..f545433bb 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -18,11 +18,11 @@ impl StoreDir { /// Content of an index file (`$STORE_DIR/v3/files/*/*-index.json`). #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -pub struct TarballIndex { +pub struct PackageFilesIndex { pub files: HashMap, } -/// Value of the [`files`](TarballIndex::files) map. +/// Value of the [`files`](PackageFilesIndex::files) map. #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct TarballIndexFileAttrs { @@ -45,7 +45,7 @@ impl StoreDir { pub fn write_tarball_index_file( &self, tarball_integrity: &Integrity, - index_content: &TarballIndex, + index_content: &PackageFilesIndex, ) -> Result<(), WriteTarballIndexFileError> { let file_path = self.tarball_index_file_path(tarball_integrity); let index_content = diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 8f085b636..750861727 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -12,7 +12,8 @@ use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::{ - StoreDir, TarballIndex, TarballIndexFileAttrs, WriteCasFileError, WriteTarballIndexFileError, + PackageFilesIndex, StoreDir, TarballIndexFileAttrs, WriteCasFileError, + WriteTarballIndexFileError, }; use pipe_trait::Pipe; use reqwest::Client; @@ -225,7 +226,7 @@ impl<'a> DownloadTarballToStore<'a> { let ((_, Some(capacity)) | (capacity, None)) = entries.size_hint(); let mut cas_paths = HashMap::::with_capacity(capacity); - let mut tarball_index = TarballIndex { files: HashMap::with_capacity(capacity) }; + let mut pkg_files_idx = PackageFilesIndex { files: HashMap::with_capacity(capacity) }; for entry in entries { let mut entry = entry.unwrap(); @@ -262,13 +263,13 @@ impl<'a> DownloadTarballToStore<'a> { size: file_size, }; - if let Some(previous) = tarball_index.files.insert(tarball_index_key, file_attrs) { + if let Some(previous) = pkg_files_idx.files.insert(tarball_index_key, file_attrs) { panic!("Unexpected error: {previous:?} shouldn't collide"); } } store_dir - .write_tarball_index_file(&package_integrity, &tarball_index) + .write_tarball_index_file(&package_integrity, &pkg_files_idx) .map_err(TarballError::WriteTarballIndexFile)?; Ok(cas_paths) From 3883db010dfdb4c101a7fed0d6f303317e7f000a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:21:42 +0700 Subject: [PATCH 081/102] refactor: rename some functions --- crates/store-dir/src/index_file.rs | 10 +++++----- crates/tarball/src/lib.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index f545433bb..ad1aaad00 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -8,7 +8,7 @@ use std::{collections::HashMap, path::PathBuf}; impl StoreDir { /// Path to an index file of a tarball. - pub fn tarball_index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { + pub fn index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { let (algorithm, hex) = tarball_integrity.to_hex(); assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error self.file_path_by_hex_str(&hex, "-index.json") @@ -42,12 +42,12 @@ pub enum WriteTarballIndexFileError { impl StoreDir { /// Write a JSON file that indexes files in a tarball to the store directory. - pub fn write_tarball_index_file( + pub fn write_index_file( &self, tarball_integrity: &Integrity, index_content: &PackageFilesIndex, ) -> Result<(), WriteTarballIndexFileError> { - let file_path = self.tarball_index_file_path(tarball_integrity); + let file_path = self.index_file_path(tarball_integrity); let index_content = serde_json::to_string(&index_content).expect("convert a TarballIndex to JSON"); ensure_file(&file_path, index_content.as_bytes(), Some(0o666)) @@ -61,11 +61,11 @@ mod tests { use ssri::IntegrityOpts; #[test] - fn tarball_index_file_path() { + fn index_file_path() { let store_dir = StoreDir::new("STORE_DIR"); let tarball_integrity = IntegrityOpts::new().algorithm(Algorithm::Sha512).chain(b"TARBALL CONTENT").result(); - let received = store_dir.tarball_index_file_path(&tarball_integrity); + let received = store_dir.index_file_path(&tarball_integrity); let expected = "STORE_DIR/v3/files/bc/d60799116ebef60071b9f2c7dafd7e2a4e1b366e341f750b2de52dd6995ab409b530f31b2b0a56c168a808a977156c3f5f13b026fb117d36314d8077f8733f-index.json"; let expected: PathBuf = expected.split('/').collect(); assert_eq!(&received, &expected); diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 750861727..ee2228032 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -269,7 +269,7 @@ impl<'a> DownloadTarballToStore<'a> { } store_dir - .write_tarball_index_file(&package_integrity, &pkg_files_idx) + .write_index_file(&package_integrity, &pkg_files_idx) .map_err(TarballError::WriteTarballIndexFile)?; Ok(cas_paths) From 2d54d459f349b9ef2ddc1d0340d79393aea644c1 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:23:30 +0700 Subject: [PATCH 082/102] refactor: rename to PackageFileInfo Resolves https://github.com/pnpm/pacquet/pull/166#discussion_r1378233204 --- crates/store-dir/src/index_file.rs | 4 ++-- crates/tarball/src/lib.rs | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index ad1aaad00..81f237847 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -19,13 +19,13 @@ impl StoreDir { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct PackageFilesIndex { - pub files: HashMap, + pub files: HashMap, } /// Value of the [`files`](PackageFilesIndex::files) map. #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -pub struct TarballIndexFileAttrs { +pub struct PackageFileInfo { #[serde(skip_serializing_if = "Option::is_none")] pub checked_at: Option, pub integrity: String, diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index ee2228032..fa7018ab1 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -12,8 +12,7 @@ use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; use pacquet_store_dir::{ - PackageFilesIndex, StoreDir, TarballIndexFileAttrs, WriteCasFileError, - WriteTarballIndexFileError, + PackageFileInfo, PackageFilesIndex, StoreDir, WriteCasFileError, WriteTarballIndexFileError, }; use pipe_trait::Pipe; use reqwest::Client; @@ -256,7 +255,7 @@ impl<'a> DownloadTarballToStore<'a> { let checked_at = UNIX_EPOCH.elapsed().ok().map(|x| x.as_millis()); let file_size = entry.header().size().ok(); let file_integrity = format!("sha512-{}", BASE64_STD.encode(file_hash)); - let file_attrs = TarballIndexFileAttrs { + let file_attrs = PackageFileInfo { checked_at, integrity: file_integrity, mode: file_mode, From 3ab34b855644f2ed22ecb87a35aa8a87922f1b9c Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:34:23 +0700 Subject: [PATCH 083/102] feat(tarball): don't panic on duplication --- crates/tarball/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index fa7018ab1..798afeb8c 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -249,7 +249,7 @@ impl<'a> DownloadTarballToStore<'a> { .to_string(); // TODO: convert cleaned_entry_path to String too. if let Some(previous) = cas_paths.insert(cleaned_entry_path, file_path) { - panic!("Unexpected error: {previous:?} shouldn't collide"); + tracing::warn!(?previous, "Duplication detected. Old entry has been ejected"); } let checked_at = UNIX_EPOCH.elapsed().ok().map(|x| x.as_millis()); @@ -263,7 +263,7 @@ impl<'a> DownloadTarballToStore<'a> { }; if let Some(previous) = pkg_files_idx.files.insert(tarball_index_key, file_attrs) { - panic!("Unexpected error: {previous:?} shouldn't collide"); + tracing::warn!(?previous, "Duplication detected. Old entry has been ejected"); } } From 7e0748d150138ab7128737a872cfa93c42503489 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:38:07 +0700 Subject: [PATCH 084/102] docs: future tasks --- crates/tarball/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index 798afeb8c..ddfb7a131 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -212,6 +212,10 @@ impl<'a> DownloadTarballToStore<'a> { let cas_paths = tokio::task::spawn(async move { verify_checksum(&response, package_integrity.clone()).map_err(TaskError::Checksum)?; + // TODO: move tarball extraction to its own function + // TODO: test it + // TODO: test the duplication of entries + let mut archive = decompress_gzip(&response, package_unpacked_size) .map_err(TaskError::Other)? .pipe(Cursor::new) From 417e8dc68fe322c956d02e8142e54acece3de1eb Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 15:39:35 +0700 Subject: [PATCH 085/102] fix(docs): correct a link --- crates/store-dir/src/index_file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index 81f237847..2497f2254 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -34,7 +34,7 @@ pub struct PackageFileInfo { pub size: Option, } -/// Error type of [`StoreDir::write_tarball_index_file`]. +/// Error type of [`StoreDir::write_index_file`]. #[derive(Debug, Display, Error, Diagnostic)] pub enum WriteTarballIndexFileError { WriteFile(EnsureFileError), From 354b344f9ec793291bfb6df4d6228a1dc81c38ce Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 19:08:47 +0700 Subject: [PATCH 086/102] feat: only appends `-exec` when exec bits --- crates/fs/src/lib.rs | 6 +++--- crates/store-dir/src/cas_file.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 65334eec4..bcf29371f 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -22,9 +22,9 @@ pub mod file_mode { /// All can read and write, but not execute (`rw-rw-rw-`). pub const ALL_RW: u32 = 0b110_110_110; - /// Whether a file mode has any executable bit. - pub fn is_executable(mode: u32) -> bool { - mode & EXEC_MASK != 0 + /// Whether a file mode has all executable bits. + pub fn is_all_exec(mode: u32) -> bool { + mode & EXEC_MASK == EXEC_MASK } } diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index 63c93decc..124c398a6 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -1,7 +1,7 @@ use crate::{FileHash, StoreDir}; use derive_more::{Display, Error}; use miette::Diagnostic; -use pacquet_fs::{ensure_file, file_mode::is_executable, EnsureFileError}; +use pacquet_fs::{ensure_file, file_mode::is_all_exec, EnsureFileError}; use sha2::{Digest, Sha512}; use std::path::PathBuf; @@ -9,7 +9,7 @@ impl StoreDir { /// Path to a file in the store directory. pub fn cas_file_path(&self, hash: FileHash, mode: u32) -> PathBuf { let hex = format!("{hash:x}"); - let suffix = if is_executable(mode) { "-exec" } else { "" }; + let suffix = if is_all_exec(mode) { "-exec" } else { "" }; self.file_path_by_hex_str(&hex, suffix) } } From 701479975c36b1f68ab4830ec24b4fa3b23faff7 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 19:20:01 +0700 Subject: [PATCH 087/102] feat: set mode of all exec files to 755 --- Cargo.lock | 1 + crates/fs/src/lib.rs | 3 +++ crates/store-dir/src/cas_file.rs | 24 ++++++++++++------------ crates/tarball/Cargo.toml | 1 + crates/tarball/src/lib.rs | 4 +++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e104dcdc..13c91e210 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1499,6 +1499,7 @@ dependencies = [ "derive_more", "miette", "pacquet-diagnostics", + "pacquet-fs", "pacquet-store-dir", "pipe-trait", "pretty_assertions", diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index bcf29371f..9058cde7b 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -22,6 +22,9 @@ pub mod file_mode { /// All can read and write, but not execute (`rw-rw-rw-`). pub const ALL_RW: u32 = 0b110_110_110; + /// All can read and execute, but only owner can write (`rwxr-xr-x`). + pub const EXEC_MODE: u32 = 0b111_101_101; + /// Whether a file mode has all executable bits. pub fn is_all_exec(mode: u32) -> bool { mode & EXEC_MASK == EXEC_MASK diff --git a/crates/store-dir/src/cas_file.rs b/crates/store-dir/src/cas_file.rs index 124c398a6..f53d4baea 100644 --- a/crates/store-dir/src/cas_file.rs +++ b/crates/store-dir/src/cas_file.rs @@ -1,15 +1,15 @@ use crate::{FileHash, StoreDir}; use derive_more::{Display, Error}; use miette::Diagnostic; -use pacquet_fs::{ensure_file, file_mode::is_all_exec, EnsureFileError}; +use pacquet_fs::{ensure_file, file_mode::EXEC_MODE, EnsureFileError}; use sha2::{Digest, Sha512}; use std::path::PathBuf; impl StoreDir { /// Path to a file in the store directory. - pub fn cas_file_path(&self, hash: FileHash, mode: u32) -> PathBuf { + pub fn cas_file_path(&self, hash: FileHash, executable: bool) -> PathBuf { let hex = format!("{hash:x}"); - let suffix = if is_all_exec(mode) { "-exec" } else { "" }; + let suffix = if executable { "-exec" } else { "" }; self.file_path_by_hex_str(&hex, suffix) } } @@ -25,11 +25,12 @@ impl StoreDir { pub fn write_cas_file( &self, buffer: &[u8], - mode: u32, + executable: bool, ) -> Result<(PathBuf, FileHash), WriteCasFileError> { let file_hash = Sha512::digest(buffer); - let file_path = self.cas_file_path(file_hash, mode); - ensure_file(&file_path, buffer, Some(mode)).map_err(WriteCasFileError::WriteFile)?; + let file_path = self.cas_file_path(file_hash, executable); + let mode = executable.then_some(EXEC_MODE); + ensure_file(&file_path, buffer, mode).map_err(WriteCasFileError::WriteFile)?; Ok((file_path, file_hash)) } } @@ -37,29 +38,28 @@ impl StoreDir { #[cfg(test)] mod tests { use super::*; - use pacquet_fs::file_mode::{ALL_RW, ALL_RWX}; #[test] fn cas_file_path() { - fn case(file_content: &str, mode: u32, expected: &str) { - eprintln!("CASE: {file_content:?}, {mode:?}"); + fn case(file_content: &str, executable: bool, expected: &str) { + eprintln!("CASE: {file_content:?}, {executable:?}"); let store_dir = StoreDir::new("STORE_DIR"); let file_hash = Sha512::digest(file_content); eprintln!("file_hash = {file_hash:x}"); - let received = store_dir.cas_file_path(file_hash, mode); + let received = store_dir.cas_file_path(file_hash, executable); let expected: PathBuf = expected.split('/').collect(); assert_eq!(&received, &expected); } case( "hello world", - ALL_RW, + false, "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f", ); case( "hello world", - ALL_RWX, + true, "STORE_DIR/v3/files/30/9ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f-exec", ); } diff --git a/crates/tarball/Cargo.toml b/crates/tarball/Cargo.toml index 2084002d7..41e675cee 100644 --- a/crates/tarball/Cargo.toml +++ b/crates/tarball/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true [dependencies] pacquet-diagnostics = { workspace = true } +pacquet-fs = { workspace = true } pacquet-store-dir = { workspace = true } base64 = { workspace = true } diff --git a/crates/tarball/src/lib.rs b/crates/tarball/src/lib.rs index ddfb7a131..26adfa7f9 100644 --- a/crates/tarball/src/lib.rs +++ b/crates/tarball/src/lib.rs @@ -11,6 +11,7 @@ use base64::{engine::general_purpose::STANDARD as BASE64_STD, Engine}; use dashmap::DashMap; use derive_more::{Display, Error, From}; use miette::Diagnostic; +use pacquet_fs::file_mode; use pacquet_store_dir::{ PackageFileInfo, PackageFilesIndex, StoreDir, WriteCasFileError, WriteTarballIndexFileError, }; @@ -235,6 +236,7 @@ impl<'a> DownloadTarballToStore<'a> { let mut entry = entry.unwrap(); let file_mode = entry.header().mode().expect("get mode"); // TODO: properly propagate this error + let file_is_executable = file_mode::is_all_exec(file_mode); // Read the contents of the entry let mut buffer = Vec::with_capacity(entry.size() as usize); @@ -244,7 +246,7 @@ impl<'a> DownloadTarballToStore<'a> { let cleaned_entry_path = entry_path.components().skip(1).collect::().into_os_string(); let (file_path, file_hash) = store_dir - .write_cas_file(&buffer, file_mode) + .write_cas_file(&buffer, file_is_executable) .map_err(TarballError::WriteCasFile)?; let tarball_index_key = cleaned_entry_path From 09807515bf181e167112fc58bcfd39e343de87e2 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 19:28:52 +0700 Subject: [PATCH 088/102] refactor: extract some expressions --- crates/cli/tests/install.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index d38e7e917..db9ed743f 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -72,13 +72,14 @@ fn should_install_exec_files() { use pacquet_testing_utils::fs::is_path_executable; use pretty_assertions::assert_eq; + let (suffix_exec, suffix_other) = + store_files.iter().partition::, _>(|path| path.ends_with("-exec")); + let (mode_exec, mode_other) = store_files.iter().partition::, _>(|name| { + root.path().join("pacquet-store").join(name).pipe_as_ref(is_path_executable) + }); + eprintln!("All files that end with '-exec' are executable, others not"); - assert_eq!( - store_files.iter().partition::, _>(|name| { - root.path().join("pacquet-store").join(name).pipe_as_ref(is_path_executable) - }), - store_files.iter().partition::, _>(|path| path.ends_with("-exec")), - ); + assert_eq!((suffix_exec, suffix_other), (mode_exec, mode_other)); } eprintln!("Snapshot"); From 499ba6e8a85f173c26ad490f0e0e1e276594df8c Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 19:43:27 +0700 Subject: [PATCH 089/102] test: make sure executable modes are 755 --- crates/cli/tests/install.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index db9ed743f..028344ead 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -71,15 +71,36 @@ fn should_install_exec_files() { { use pacquet_testing_utils::fs::is_path_executable; use pretty_assertions::assert_eq; + use std::{fs::File, iter::repeat, os::unix::fs::MetadataExt}; let (suffix_exec, suffix_other) = store_files.iter().partition::, _>(|path| path.ends_with("-exec")); let (mode_exec, mode_other) = store_files.iter().partition::, _>(|name| { root.path().join("pacquet-store").join(name).pipe_as_ref(is_path_executable) }); + let actual_modes: Vec<_> = mode_exec + .iter() + .map(|name| { + let mode = root + .path() + .join("pacquet-store") + .join(name) + .pipe(File::open) + .expect("open file to get mode") + .metadata() + .expect("get metadata") + .mode(); + (name.as_str(), mode & 0o777) + }) + .collect(); + let expected_modes: Vec<_> = + mode_exec.iter().map(|name| name.as_str()).zip(repeat(0o755)).collect(); eprintln!("All files that end with '-exec' are executable, others not"); assert_eq!((suffix_exec, suffix_other), (mode_exec, mode_other)); + + eprintln!("All executable files have mode 755"); + assert_eq!(actual_modes, expected_modes); } eprintln!("Snapshot"); From d31b6aea7555ae7b5e7a00c96f7f265d5e4a4e02 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 19:45:12 +0700 Subject: [PATCH 090/102] refactor: rearrange code --- crates/cli/tests/install.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 028344ead..32f2aaa84 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -73,11 +73,15 @@ fn should_install_exec_files() { use pretty_assertions::assert_eq; use std::{fs::File, iter::repeat, os::unix::fs::MetadataExt}; + eprintln!("All files that end with '-exec' are executable, others not"); let (suffix_exec, suffix_other) = store_files.iter().partition::, _>(|path| path.ends_with("-exec")); let (mode_exec, mode_other) = store_files.iter().partition::, _>(|name| { root.path().join("pacquet-store").join(name).pipe_as_ref(is_path_executable) }); + assert_eq!((&suffix_exec, &suffix_other), (&mode_exec, &mode_other)); + + eprintln!("All executable files have mode 755"); let actual_modes: Vec<_> = mode_exec .iter() .map(|name| { @@ -95,12 +99,7 @@ fn should_install_exec_files() { .collect(); let expected_modes: Vec<_> = mode_exec.iter().map(|name| name.as_str()).zip(repeat(0o755)).collect(); - - eprintln!("All files that end with '-exec' are executable, others not"); - assert_eq!((suffix_exec, suffix_other), (mode_exec, mode_other)); - - eprintln!("All executable files have mode 755"); - assert_eq!(actual_modes, expected_modes); + assert_eq!(&actual_modes, &expected_modes); } eprintln!("Snapshot"); From b847ecf351b7d4354ad7651c05836cee0dba1ad4 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 19:50:06 +0700 Subject: [PATCH 091/102] refactor: deduplicate --- crates/cli/tests/install.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 32f2aaa84..0f0bfc584 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -73,22 +73,21 @@ fn should_install_exec_files() { use pretty_assertions::assert_eq; use std::{fs::File, iter::repeat, os::unix::fs::MetadataExt}; + let resolve = |name: &str| root.path().join("pacquet-store").join(name); + eprintln!("All files that end with '-exec' are executable, others not"); let (suffix_exec, suffix_other) = store_files.iter().partition::, _>(|path| path.ends_with("-exec")); - let (mode_exec, mode_other) = store_files.iter().partition::, _>(|name| { - root.path().join("pacquet-store").join(name).pipe_as_ref(is_path_executable) - }); + let (mode_exec, mode_other) = store_files + .iter() + .partition::, _>(|name| resolve(name).as_path().pipe(is_path_executable)); assert_eq!((&suffix_exec, &suffix_other), (&mode_exec, &mode_other)); eprintln!("All executable files have mode 755"); let actual_modes: Vec<_> = mode_exec .iter() .map(|name| { - let mode = root - .path() - .join("pacquet-store") - .join(name) + let mode = resolve(name) .pipe(File::open) .expect("open file to get mode") .metadata() From c804271d2bc2a4414e99165692ee859e84be8624 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Wed, 1 Nov 2023 22:00:07 +0700 Subject: [PATCH 092/102] refactor: remove unused constants --- crates/fs/src/lib.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/crates/fs/src/lib.rs b/crates/fs/src/lib.rs index 9058cde7b..9b7af335c 100644 --- a/crates/fs/src/lib.rs +++ b/crates/fs/src/lib.rs @@ -7,21 +7,9 @@ use std::{ }; pub mod file_mode { - /// Bit mask to filter readable bits (`r--r--r--`). - pub const READ_MASK: u32 = 0b100_100_100; - - /// Bit mask to filter executable bits (`-w--w--w-`). - pub const WRITE_MASK: u32 = 0b010_010_010; - /// Bit mask to filter executable bits (`--x--x--x`). pub const EXEC_MASK: u32 = 0b001_001_001; - /// All can read, write, and execute (`rwxrwxrwx`). - pub const ALL_RWX: u32 = 0b111_111_111; - - /// All can read and write, but not execute (`rw-rw-rw-`). - pub const ALL_RW: u32 = 0b110_110_110; - /// All can read and execute, but only owner can write (`rwxr-xr-x`). pub const EXEC_MODE: u32 = 0b111_101_101; From 14a312589476bf380537a3157d57a48d66e5d37b Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 00:41:58 +0700 Subject: [PATCH 093/102] refactor: extract default npmrc --- crates/testing-utils/src/bin.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index a81cbfcb5..4abf82f46 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -4,19 +4,17 @@ use std::{fs, path::PathBuf, process::Command}; use tempfile::{tempdir, TempDir}; use text_block_macros::text_block_fnl; +const DEFAULT_NPMRC: &str = text_block_fnl! { + "store-dir=../pacquet-store" + "cache-dir=../pacquet-cache" +}; + pub fn pacquet_with_temp_cwd(create_npmrc: bool) -> (Command, TempDir, PathBuf) { let root = tempdir().expect("create temporary directory"); let workspace = root.path().join("workspace"); fs::create_dir(&workspace).expect("create temporary workspace for pacquet"); if create_npmrc { - fs::write( - workspace.join(".npmrc"), - text_block_fnl! { - "store-dir=../pacquet-store" - "cache-dir=../pacquet-cache" - }, - ) - .expect("write to .npmrc"); + fs::write(workspace.join(".npmrc"), DEFAULT_NPMRC).expect("write to .npmrc"); } let command = Command::cargo_bin("pacquet") .expect("find the pacquet binary") From ae2b85df7486560938ed6d208285a67fe186c105 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 00:44:54 +0700 Subject: [PATCH 094/102] refactor: extract create_default_npmrc --- crates/testing-utils/src/bin.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index 4abf82f46..dd523c5cf 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -1,6 +1,10 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; -use std::{fs, path::PathBuf, process::Command}; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; use tempfile::{tempdir, TempDir}; use text_block_macros::text_block_fnl; @@ -9,12 +13,16 @@ const DEFAULT_NPMRC: &str = text_block_fnl! { "cache-dir=../pacquet-cache" }; +fn create_default_npmrc(workspace: &Path) { + fs::write(workspace.join(".npmrc"), DEFAULT_NPMRC).expect("write to .npmrc"); +} + pub fn pacquet_with_temp_cwd(create_npmrc: bool) -> (Command, TempDir, PathBuf) { let root = tempdir().expect("create temporary directory"); let workspace = root.path().join("workspace"); fs::create_dir(&workspace).expect("create temporary workspace for pacquet"); if create_npmrc { - fs::write(workspace.join(".npmrc"), DEFAULT_NPMRC).expect("write to .npmrc"); + create_default_npmrc(&workspace) } let command = Command::cargo_bin("pacquet") .expect("find the pacquet binary") From c2656858454632c7ca0b2d7e94715091884ae7f2 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 01:20:14 +0700 Subject: [PATCH 095/102] test: check against pnpm --- .github/workflows/ci.yml | 7 +++ crates/cli/tests/pnpm_compatibility.rs | 74 ++++++++++++++++++++++++++ crates/testing-utils/src/bin.rs | 14 +++++ 3 files changed, 95 insertions(+) create mode 100644 crates/cli/tests/pnpm_compatibility.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7469074e2..ba33a0a7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,13 @@ jobs: clippy: true save-cache: ${{ github.ref_name == 'main' }} + - name: Install pnpm (for compatibility check) + uses: pnpm/action-setup@v2 + with: + version: 8.9.2 + run_install: false + standalone: true + - name: Clippy run: cargo clippy --locked -- -D warnings diff --git a/crates/cli/tests/pnpm_compatibility.rs b/crates/cli/tests/pnpm_compatibility.rs new file mode 100644 index 000000000..5cb949156 --- /dev/null +++ b/crates/cli/tests/pnpm_compatibility.rs @@ -0,0 +1,74 @@ +use assert_cmd::prelude::*; +use command_extra::CommandExtra; +use pacquet_testing_utils::{bin::pacquet_and_pnpm_with_temp_cwd, fs::get_all_files}; +use pretty_assertions::assert_eq; +use std::fs; + +#[test] +#[ignore = "requires metadata cache feature which pacquet doesn't yet have"] +fn store_usable_by_pnpm_offline() { + let (pacquet, pnpm, root, workspace) = pacquet_and_pnpm_with_temp_cwd(true); + + eprintln!("Creating package.json..."); + let manifest_path = workspace.join("package.json"); + let package_json_content = serde_json::json!({ + "dependencies": { + "is-odd": "3.0.1", + }, + "devDependencies": { + "pretty-exec": "0.3.10", + }, + }); + fs::write(manifest_path, package_json_content.to_string()).expect("write to package.json"); + + eprintln!("Using pacquet to populate the store..."); + pacquet.with_arg("install").assert().success(); + fs::remove_dir_all(workspace.join("node_modules")).expect("delete node_modules"); + + eprintln!("pnpm install --offline --ignore-scripts"); + pnpm.with_args(["install", "--offline", "--ignore-scripts"]).assert().success(); + + drop(root); // cleanup +} + +#[test] +fn same_file_structure() { + let (pacquet, pnpm, root, workspace) = pacquet_and_pnpm_with_temp_cwd(true); + + let store_dir = root.path().join("pacquet-store"); + let modules_dir = workspace.join("node_modules"); + let cleanup = || { + eprintln!("Cleaning up..."); + fs::remove_dir_all(&store_dir).expect("delete store dir"); + fs::remove_dir_all(&modules_dir).expect("delete node_modules"); + }; + + eprintln!("Creating package.json..."); + let manifest_path = workspace.join("package.json"); + let package_json_content = serde_json::json!({ + "dependencies": { + "is-odd": "3.0.1", + }, + "devDependencies": { + "pretty-exec": "0.3.10", + }, + }); + fs::write(manifest_path, package_json_content.to_string()).expect("write to package.json"); + + eprintln!("Installing with pacquet..."); + pacquet.with_arg("install").assert().success(); + let pacquet_store_files = get_all_files(&store_dir); + + cleanup(); + + eprintln!("Installing with pnpm..."); + pnpm.with_args(["install", "--ignore-scripts"]).assert().success(); + let pnpm_store_files = get_all_files(&store_dir); + + cleanup(); + + eprintln!("Produce the same store dir structure"); + assert_eq!(&pacquet_store_files, &pnpm_store_files); + + drop(root); // cleanup +} diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index dd523c5cf..10f1ff374 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -29,3 +29,17 @@ pub fn pacquet_with_temp_cwd(create_npmrc: bool) -> (Command, TempDir, PathBuf) .with_current_dir(&workspace); (command, root, workspace) } + +pub fn pacquet_and_pnpm_with_temp_cwd(create_npmrc: bool) -> (Command, Command, TempDir, PathBuf) { + let root = tempdir().expect("create temporary directory"); + let workspace = root.path().join("workspace"); + fs::create_dir(&workspace).expect("create temporary workspace for pacquet"); + if create_npmrc { + create_default_npmrc(&workspace) + } + let pacquet = Command::cargo_bin("pacquet") + .expect("find the pacquet binary") + .with_current_dir(&workspace); + let pnpm = Command::new("pnpm").with_current_dir(&workspace); + (pacquet, pnpm, root, workspace) +} From 27f8a3fd694aaf970891d8c1d06576300a287b02 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 01:41:48 +0700 Subject: [PATCH 096/102] ci: workaround --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba33a0a7a..c8a13f522 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,15 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@cargo-nextest - - run: cargo nextest run + - name: Test + shell: bash + run: | + # removing env vars is a temporary workaround for unit tests in pacquet relying on external environment + # this should be removed in the future + unset PNPM_HOME + unset XDG_DATA_HOME + + cargo nextest run typos: name: Spell Check From 9fb71f65bd8ff341cc48ebde694f7682baade6c0 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 01:49:22 +0700 Subject: [PATCH 097/102] ci: fix codecov --- .github/workflows/codecov.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 6e03108e9..5f9f4d41b 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -40,8 +40,21 @@ jobs: - name: Install llvm-tools-preview for llvm-cov run: rustup component add llvm-tools-preview + - name: Install pnpm (for compatibility check) + uses: pnpm/action-setup@v2 + with: + version: 8.9.2 + run_install: false + standalone: true + - name: Run - run: cargo codecov --lcov --output-path lcov.info + run: | + # removing env vars is a temporary workaround for unit tests in pacquet relying on external environment + # this should be removed in the future + unset PNPM_HOME + unset XDG_DATA_HOME + + cargo codecov --lcov --output-path lcov.info - name: Upload Artifact uses: actions/upload-artifact@v3 From 95741efac3224947de52e0800c42dde695d6fca5 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 02:02:15 +0700 Subject: [PATCH 098/102] test(windows): skip pnpm_compatibility --- crates/cli/tests/pnpm_compatibility.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cli/tests/pnpm_compatibility.rs b/crates/cli/tests/pnpm_compatibility.rs index 5cb949156..5948d5c68 100644 --- a/crates/cli/tests/pnpm_compatibility.rs +++ b/crates/cli/tests/pnpm_compatibility.rs @@ -1,3 +1,4 @@ +#![cfg(unix)] // running this on windows result in 'program not found' use assert_cmd::prelude::*; use command_extra::CommandExtra; use pacquet_testing_utils::{bin::pacquet_and_pnpm_with_temp_cwd, fs::get_all_files}; From bbdfc2f32f249be5acb4525d13b4690212d1e5cb Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 10:13:51 +0700 Subject: [PATCH 099/102] feat: support sha1 index files Resolves https://github.com/pnpm/pacquet/pull/166#discussion_r1378230529 --- crates/store-dir/src/index_file.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/store-dir/src/index_file.rs b/crates/store-dir/src/index_file.rs index 2497f2254..a7925912c 100644 --- a/crates/store-dir/src/index_file.rs +++ b/crates/store-dir/src/index_file.rs @@ -10,7 +10,10 @@ impl StoreDir { /// Path to an index file of a tarball. pub fn index_file_path(&self, tarball_integrity: &Integrity) -> PathBuf { let (algorithm, hex) = tarball_integrity.to_hex(); - assert_eq!(algorithm, Algorithm::Sha512, "Only Sha512 is supported"); // TODO: propagate this error + assert!( + matches!(algorithm, Algorithm::Sha512 | Algorithm::Sha1), + "Only Sha1 and Sha512 are supported. {algorithm} isn't", + ); // TODO: propagate this error self.file_path_by_hex_str(&hex, "-index.json") } } From a3b0e4b2c7b4d9d113315f300a6a7e12971d853a Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 11:24:25 +0700 Subject: [PATCH 100/102] test(cli/install): index files --- Cargo.lock | 2 + crates/cli/Cargo.toml | 2 + crates/cli/tests/_utils.rs | 53 +++++++++++++- crates/cli/tests/install.rs | 27 ++++++++ .../install__should_install_index_files.snap | 69 +++++++++++++++++++ 5 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 crates/cli/tests/snapshots/install__should_install_index_files.snap diff --git a/Cargo.lock b/Cargo.lock index 13c91e210..071bc4041 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1317,6 +1317,7 @@ dependencies = [ "pacquet-package-manager", "pacquet-package-manifest", "pacquet-registry", + "pacquet-store-dir", "pacquet-tarball", "pacquet-testing-utils", "pipe-trait", @@ -1325,6 +1326,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "walkdir", ] [[package]] diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index be90dc131..a5daff4e9 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -34,6 +34,7 @@ pipe-trait = { workspace = true } tokio = { workspace = true } [dev-dependencies] +pacquet-store-dir = { workspace = true } pacquet-testing-utils = { workspace = true } assert_cmd = { workspace = true } @@ -43,3 +44,4 @@ insta = { workspace = true } pretty_assertions = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } +walkdir = { workspace = true } diff --git a/crates/cli/tests/_utils.rs b/crates/cli/tests/_utils.rs index c9079d736..6c604a4df 100644 --- a/crates/cli/tests/_utils.rs +++ b/crates/cli/tests/_utils.rs @@ -1,8 +1,16 @@ use assert_cmd::prelude::*; use command_extra::CommandExtra; +use pacquet_store_dir::{PackageFileInfo, PackageFilesIndex}; use pacquet_testing_utils::bin::pacquet_with_temp_cwd; -use std::{ffi::OsStr, path::PathBuf}; +use pipe_trait::Pipe; +use std::{ + collections::BTreeMap, + ffi::OsStr, + fs::File, + path::{Path, PathBuf}, +}; use tempfile::TempDir; +use walkdir::{DirEntry, WalkDir}; pub fn exec_pacquet_in_temp_cwd(create_npmrc: bool, args: Args) -> (TempDir, PathBuf) where @@ -13,3 +21,46 @@ where command.with_args(args).assert().success(); (root, workspace) } + +pub fn index_file_contents( + store_dir: &Path, +) -> BTreeMap> { + // TODO: refactor the functions in pacquet_testing_utils::fs to be more functional + // TODO: this function and ones from pacquet_testing_utils::fs can share the suffix code + + let suffix = |entry: &DirEntry| -> String { + entry + .path() + .strip_prefix(store_dir) + .expect("strip store dir prefix from entry path to create suffix") + .to_str() + .expect("convert entry suffix to UTF-8") + .replace('\\', "/") + }; + + let sanitize = |mut value: PackageFileInfo| { + value.checked_at = None; // this value depends on time, therefore not deterministic + value + }; + + let content = |entry: &DirEntry| -> BTreeMap<_, _> { + entry + .path() + .pipe(File::open) + .expect("open file to read") + .pipe(serde_json::from_reader::<_, PackageFilesIndex>) + .expect("read and parse file") + .files + .into_iter() + .map(|(key, value)| (key, sanitize(value))) + .collect() + }; + + WalkDir::new(store_dir) + .into_iter() + .map(|entry| entry.expect("get entry")) + .filter(|entry| entry.file_name().to_string_lossy().ends_with("-index.json")) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| (suffix(&entry), content(&entry))) + .collect() +} diff --git a/crates/cli/tests/install.rs b/crates/cli/tests/install.rs index 0f0bfc584..eedac875e 100644 --- a/crates/cli/tests/install.rs +++ b/crates/cli/tests/install.rs @@ -1,3 +1,6 @@ +pub mod _utils; +pub use _utils::*; + use assert_cmd::prelude::*; use command_extra::CommandExtra; use pacquet_testing_utils::{ @@ -106,3 +109,27 @@ fn should_install_exec_files() { drop(root); // cleanup } + +#[test] +fn should_install_index_files() { + let (command, root, workspace) = pacquet_with_temp_cwd(true); + + eprintln!("Creating package.json..."); + let manifest_path = workspace.join("package.json"); + let package_json_content = serde_json::json!({ + "dependencies": { + "is-odd": "3.0.1", + }, + "devDependencies": { + "fast-decode-uri-component": "1.0.1", + }, + }); + fs::write(&manifest_path, package_json_content.to_string()).expect("write to package.json"); + + eprintln!("Executing command..."); + command.with_arg("install").assert().success(); + + eprintln!("Snapshot"); + let index_file_contents = root.path().join("pacquet-store").as_path().pipe(index_file_contents); + insta::assert_yaml_snapshot!(index_file_contents); +} diff --git a/crates/cli/tests/snapshots/install__should_install_index_files.snap b/crates/cli/tests/snapshots/install__should_install_index_files.snap new file mode 100644 index 000000000..27b535e27 --- /dev/null +++ b/crates/cli/tests/snapshots/install__should_install_index_files.snap @@ -0,0 +1,69 @@ +--- +source: crates/cli/tests/install.rs +assertion_line: 134 +expression: index_file_contents +--- +v3/files/09/0a6758fac3c263f5f923075d986d2ed26ff74ca2c957e5b86b17e62342564ae142d5374d01ec5163c6f7091d93d2e0779c8da4083d8d0128f7408169781630-index.json: + LICENSE: + integrity: sha512-Vdy8f/8awcR1X0y/JLoh4NyuBiyALT25N9K2GC9FVN2F0UUpoMlojqyCWjRbY6uLLl12meK+BxeN9buKByI16w== + mode: 420 + size: 1091 + README.md: + integrity: sha512-QP4YssHyQQdbcycnEQMj8+KfnWyHZf8CIrzPo0eEyqirzRj/vUu/q5eeVHis4nOnTtOgoi8CYTNPp8WeAy7b3g== + mode: 420 + size: 3495 + index.js: + integrity: sha512-iB8QHS8YhvxfmsXI0l5VQJCOFAAxFp6kGkqZQc+V/icRwSksGZVgnG8XSzgVzA3MbNKhAzmjSNOf51vHe1O6ng== + mode: 420 + size: 543 + package.json: + integrity: sha512-VHvNaD4vn3sMJ5DGN7kjFTmUe+kjR5U0WS2RiGpW1iYv3B2M7SSsLMOt/aE5/jmdx1AtOCx6/FNuMFenzxHmwA== + mode: 420 + size: 1381 +v3/files/58/a80a5a0e5e531bd1646c16f05bdf6da1fb0174a1d9c2fede3e5f306cd4302c56049ddd5776bb5b3f32d9ffee4347b707a5a6a1d0f7a12e788d343d1d54c822-index.json: + ".travis.yml": + integrity: sha512-N46xbCMPzjVO9+cnIu5wjL7+qaSBqll7Bv78EvZlq2AXbL8E1rwVm+QszbxRdk1xUW+GuyigGw5gx2onU/HPRA== + mode: 420 + size: 132 + LICENSE: + integrity: sha512-5cJRTSsNPWvtrST911xV54AhHPBpLCH0ASAcLX36mQrQAYi9aLMMgmqqKnLsrRKnS8A0MDhPkzsXUQNd0bGMog== + mode: 420 + size: 1174 + README.md: + integrity: sha512-ymXfFutFCrY/kPEHqMyayC2pY1QI6BOm9tPhPPuniRezG2ZlJHnrulS2ElZmALKrG8FrgtCCxfh4A5Yajp45MQ== + mode: 420 + size: 1941 + bench.js: + integrity: sha512-YKtUhJ1s/XsFB2w7xNm6/+H4APTV4tZ8OcUhp6PZOzR85ajkU5cQLkDiEDZSxUPI57sWHBXQpuNOwanRHBaNKg== + mode: 420 + size: 941 + index.js: + integrity: sha512-GlwBoTIecaHCY/uDh4y7omhvcfVxtCwtQNk4WL8fTG/39iUSWmiTLCQ+derUqdLxjZsis/5Vw5/mYFpCsucGIA== + mode: 420 + size: 3219 + package.json: + integrity: sha512-/9Pz/tZde0iSFpFncsYYC1wXFN3eFufiyZ/1+KQhS7j4hKDB0GobGopLDWJSrYW1M7Fibw3Y1HTyKD9s39afbA== + mode: 420 + size: 818 + test.js: + integrity: sha512-9wIYD0Rlp3cSetaiETBt1WSZPBf309lAFgTfS8ZX2NiIkC3mMqTPVbVR3fufhLqKkAeIdz8Dx3g2FD75ThgsfA== + mode: 420 + size: 1007 +v3/files/5a/ed551de20b04af0a016254022499417f781a6384e39460ebfe77f1f2b08a5a14bb6d4a9dc1246063eaa1bda8499e66513ef7bcb1ab1d08cac3728c3f07c3ce-index.json: + LICENSE: + integrity: sha512-n8vl3Ia8M7208SVVnnZ8OqrZTHxLIbgLaU2bAVjuDxDuU7Enwr8F25dor9rx+2d4/QihlBL7PBra1lWGPmw1HQ== + mode: 420 + size: 1088 + README.md: + integrity: sha512-WAHlkulKpAdsF5GAlW8z+CYvpxV6Br/mNqaFS9O5st30R4D/oUBWQq2Ozi7WUU4ZooF4y+zc9WCKO+cQu/PA1A== + mode: 420 + size: 5766 + index.js: + integrity: sha512-F1S3Jxyg4w3H13WYZxGithspggDBL4Z73m9+GdVgR8mrd6kzBFIPoaMbDDpOb7azEPid2MtNlaXGYoB+9zP4Xw== + mode: 420 + size: 662 + package.json: + integrity: sha512-AdMGuA9LFnjGVfOwaF2x96dB/HRtJRwA6SUpiriWMKMWe96MfY13m3J3DEH58wK7Hj8xJnLOeqjmwVbDGe7q2g== + mode: 420 + size: 1444 + From 3c505bae5e99ff1f8edab6d0e3cd2f4a776b479d Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 11:33:42 +0700 Subject: [PATCH 101/102] test(cli/install): same index files as pnpm --- crates/cli/tests/pnpm_compatibility.rs | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/crates/cli/tests/pnpm_compatibility.rs b/crates/cli/tests/pnpm_compatibility.rs index 5948d5c68..8719967f7 100644 --- a/crates/cli/tests/pnpm_compatibility.rs +++ b/crates/cli/tests/pnpm_compatibility.rs @@ -1,7 +1,11 @@ #![cfg(unix)] // running this on windows result in 'program not found' +pub mod _utils; +pub use _utils::*; + use assert_cmd::prelude::*; use command_extra::CommandExtra; use pacquet_testing_utils::{bin::pacquet_and_pnpm_with_temp_cwd, fs::get_all_files}; +use pipe_trait::Pipe; use pretty_assertions::assert_eq; use std::fs; @@ -73,3 +77,51 @@ fn same_file_structure() { drop(root); // cleanup } + +#[test] +fn same_index_file_contents() { + let (pacquet, pnpm, root, workspace) = pacquet_and_pnpm_with_temp_cwd(true); + + let store_dir = root.path().join("pacquet-store"); + let modules_dir = workspace.join("node_modules"); + let cleanup = || { + eprintln!("Cleaning up..."); + fs::remove_dir_all(&store_dir).expect("delete store dir"); + fs::remove_dir_all(&modules_dir).expect("delete node_modules"); + }; + + eprintln!("Creating package.json..."); + let manifest_path = workspace.join("package.json"); + let package_json_content = serde_json::json!({ + "dependencies": { + "is-odd": "3.0.1", + }, + "devDependencies": { + "fast-decode-uri-component": "1.0.1", + }, + }); + fs::write(manifest_path, package_json_content.to_string()).expect("write to package.json"); + + eprintln!("Installing with pacquet..."); + pacquet.with_arg("install").assert().success(); + let pacquet_index_file_contents = store_dir + .pipe_as_ref(index_file_contents) + .pipe(serde_json::to_value) + .expect("serialize pacquet index file contents"); + + cleanup(); + + eprintln!("Installing with pnpm..."); + pnpm.with_args(["install", "--ignore-scripts"]).assert().success(); + let pnpm_index_file_contents = store_dir + .pipe_as_ref(index_file_contents) + .pipe(serde_json::to_value) + .expect("serialize pnpm index file contents"); + + cleanup(); + + eprintln!("Produce the same store dir structure"); + assert_eq!(&pacquet_index_file_contents, &pnpm_index_file_contents); + + drop(root); // cleanup +} From 4d72247e55ff1d25d3557d78455cd61f874c6c67 Mon Sep 17 00:00:00 2001 From: khai96_ Date: Thu, 2 Nov 2023 11:41:52 +0700 Subject: [PATCH 102/102] docs: future plan --- crates/testing-utils/src/bin.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/testing-utils/src/bin.rs b/crates/testing-utils/src/bin.rs index 10f1ff374..bc236a6bb 100644 --- a/crates/testing-utils/src/bin.rs +++ b/crates/testing-utils/src/bin.rs @@ -17,6 +17,7 @@ fn create_default_npmrc(workspace: &Path) { fs::write(workspace.join(".npmrc"), DEFAULT_NPMRC).expect("write to .npmrc"); } +// TODO: convert this into a struct, add workspace and store_dir fields, make use of them pub fn pacquet_with_temp_cwd(create_npmrc: bool) -> (Command, TempDir, PathBuf) { let root = tempdir().expect("create temporary directory"); let workspace = root.path().join("workspace"); @@ -30,6 +31,7 @@ pub fn pacquet_with_temp_cwd(create_npmrc: bool) -> (Command, TempDir, PathBuf) (command, root, workspace) } +// TODO: convert this into a struct, add workspace and store_dir fields, make use of them pub fn pacquet_and_pnpm_with_temp_cwd(create_npmrc: bool) -> (Command, Command, TempDir, PathBuf) { let root = tempdir().expect("create temporary directory"); let workspace = root.path().join("workspace");