diff --git a/Cargo.lock b/Cargo.lock index ab26f4fa..54b2e4a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -198,16 +198,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "assert-json-diff" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "assert_cmd" version = "2.2.2" @@ -304,49 +294,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "aws-config" -version = "1.8.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "hex", - "http 1.4.2", - "sha1 0.10.6", - "time", - "tokio", - "tracing", - "url", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "1.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - [[package]] name = "aws-lc-rs" version = "1.17.1" @@ -371,433 +318,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "aws-runtime" -version = "1.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "bytes-utils", - "fastrand", - "http 0.2.12", - "http 1.4.2", - "http-body 0.4.6", - "http-body 1.0.1", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-s3" -version = "1.137.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd7213994e2ff9382ff100403b78c30d1b74cdfcd8fa9d0d1dc3a94a5c4874" -dependencies = [ - "arc-swap", - "aws-credential-types", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "fastrand", - "hex", - "hmac 0.13.0", - "http 0.2.12", - "http 1.4.2", - "http-body 1.0.1", - "lru 0.16.4", - "percent-encoding", - "regex-lite", - "sha2 0.11.0", - "tracing", - "url", -] - -[[package]] -name = "aws-sdk-sso" -version = "1.102.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" -dependencies = [ - "arc-swap", - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.2", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.104.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" -dependencies = [ - "arc-swap", - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.2", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "1.107.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" -dependencies = [ - "arc-swap", - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "fastrand", - "http 0.2.12", - "http 1.4.2", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "1.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" -dependencies = [ - "aws-credential-types", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "crypto-bigint", - "form_urlencoded", - "hex", - "hmac 0.13.0", - "http 0.2.12", - "http 1.4.2", - "p256", - "percent-encoding", - "sha2 0.11.0", - "subtle", - "time", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.64.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e8e65f4f81fcccdeb6c3eca2af17ac21d421a1786a26a394aecf421d616d3a" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc-fast", - "hex", - "http 1.4.2", - "http-body 1.0.1", - "http-body-util", - "md-5 0.11.0", - "pin-project-lite", - "sha1 0.11.0", - "sha2 0.11.0", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.60.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.63.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "futures-util", - "http 1.4.2", - "http-body 1.0.1", - "http-body-util", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http-client" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" -dependencies = [ - "aws-smithy-async", - "aws-smithy-protocol-test", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "h2", - "http 1.4.2", - "http-body 1.0.1", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "indexmap 2.14.0", - "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "tokio", - "tokio-rustls", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.62.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-observability" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" -dependencies = [ - "aws-smithy-runtime-api", -] - -[[package]] -name = "aws-smithy-protocol-test" -version = "0.63.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b227aa94af99a8e5ee52551cc7e3ee30a217019ef99207b6f0b7a1527685941" -dependencies = [ - "assert-json-diff", - "aws-smithy-runtime-api", - "base64-simd", - "cbor-diag", - "ciborium", - "http 0.2.12", - "pretty_assertions", - "regex-lite", - "roxmltree", - "serde_json", - "thiserror 2.0.18", -] - -[[package]] -name = "aws-smithy-query" -version = "0.60.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-observability", - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.4.2", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api-macros", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.4.2", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-runtime-api-macros" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.118", -] - -[[package]] -name = "aws-smithy-schema" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "http 1.4.2", -] - -[[package]] -name = "aws-smithy-types" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.4.2", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", - "rustc_version", - "tracing", -] - [[package]] name = "axum" version = "0.8.9" @@ -808,8 +328,8 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "hyper", "hyper-util", @@ -839,8 +359,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", @@ -850,6 +370,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -868,16 +399,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - [[package]] name = "base64ct" version = "1.8.3" @@ -1112,16 +633,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "bytesize" version = "2.4.0" @@ -1165,28 +676,9 @@ dependencies = [ name = "castaway" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - -[[package]] -name = "cbor-diag" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" dependencies = [ - "bs58", - "chrono", - "data-encoding", - "half", - "nom", - "num-bigint", - "num-rational", - "num-traits", - "separator", - "url", - "uuid", + "rustversion", ] [[package]] @@ -1389,6 +881,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1765,12 +1277,6 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "difflib" version = "0.4.0" @@ -1891,6 +1397,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "dmp" version = "0.2.3" @@ -2481,6 +1996,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "group" version = "0.13.0" @@ -2542,7 +2069,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http", "indexmap 2.14.0", "slab", "tokio", @@ -2656,7 +2183,7 @@ dependencies = [ "base64 0.22.1", "bytes", "headers-core", - "http 1.4.2", + "http", "httpdate", "mime", "sha1 0.10.6", @@ -2668,7 +2195,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.4.2", + "http", ] [[package]] @@ -2767,17 +2294,6 @@ dependencies = [ "markup5ever", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.4.2" @@ -2788,17 +2304,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -2806,7 +2311,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.2", + "http", ] [[package]] @@ -2817,8 +2322,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -2860,8 +2365,8 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -2877,7 +2382,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.2", + "http", "hyper", "hyper-util", "log", @@ -2912,8 +2417,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "hyper", "ipnet", "libc", @@ -3238,10 +2743,14 @@ checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" dependencies = [ "defmt", "jiff-static", + "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", + "windows-link 0.2.1", ] [[package]] @@ -3255,6 +2764,21 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -3397,10 +2921,6 @@ dependencies = [ "anyhow", "assert_cmd", "async-trait", - "aws-config", - "aws-sdk-s3", - "aws-smithy-http-client", - "aws-smithy-runtime-api", "blake3", "bytes", "bytesize", @@ -3416,9 +2936,17 @@ dependencies = [ "interprocess", "kache-core", "libc", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-retry", + "opendal-service-fs", + "opendal-service-s3", "predicates", "ratatui", "regex", + "reqsign-aws-v4", + "reqsign-command-execute-tokio", + "reqsign-core", "reqwest 0.13.4", "rusqlite", "rustls", @@ -3516,8 +3044,8 @@ dependencies = [ "bytes", "either", "futures", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "hyper", "hyper-rustls", @@ -3549,7 +3077,7 @@ checksum = "f126d2db7a8b532ec1d839ece2a71e2485dc3bbca6cc3c3f929becaa810e719e" dependencies = [ "derive_more", "form_urlencoded", - "http 1.4.2", + "http", "jiff", "k8s-openapi", "serde", @@ -3568,7 +3096,7 @@ dependencies = [ "base64 0.22.1", "chrono", "hex", - "http 1.4.2", + "http", "jsonwebtoken 9.3.1", "oauth2", "openidconnect", @@ -3698,15 +3226,6 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" -[[package]] -name = "lru" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" -dependencies = [ - "hashbrown 0.16.1", -] - [[package]] name = "lru" version = "0.18.0" @@ -3803,6 +3322,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "mea" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2640d335e7273dacdcf51044026139b2e269c3bb0dfc3f8cb3496b85e3f6a42c" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.8.2" @@ -4028,17 +3556,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -4077,7 +3594,7 @@ dependencies = [ "base64 0.22.1", "chrono", "getrandom 0.2.17", - "http 1.4.2", + "http", "rand 0.8.6", "reqwest 0.12.28", "serde", @@ -4119,7 +3636,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.4.2", + "http", "humantime", "itertools 0.14.0", "parking_lot", @@ -4145,6 +3662,92 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opendal-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8564bd76b75d2aea59178cb5b6e9f770be1da73b1bca062720ec151cc56215a" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "futures", + "http", + "jiff", + "log", + "md-5 0.11.0", + "mea", + "percent-encoding", + "quick-xml", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7fd001b204df76be5d2b3f7a81c48b5cf25b28e2cfb3b8a819bb89cdc0ea3a" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.4", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2df70875ab7fd6f80720d4787c49c70883cef0d81bfae947ecba88b8d1cd62e" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-service-fs" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f26a923b72b96d1a3dbbe961fcad4d534bf586c3f48c583e9f35649b1b6175f" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-s3" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "750d9cc8588c19b27c4c2c5d06d7eb6f2bff62549c48652f1f9a7603cd872377" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc-fast", + "http", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + [[package]] name = "openidconnect" version = "4.0.1" @@ -4156,7 +3759,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "hmac 0.12.1", - "http 1.4.2", + "http", "itertools 0.10.5", "log", "oauth2", @@ -4207,10 +3810,14 @@ dependencies = [ ] [[package]] -name = "outref" -version = "0.5.2" +name = "ordered-multimap" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] [[package]] name = "p256" @@ -4677,16 +4284,6 @@ dependencies = [ "termtree", ] -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - [[package]] name = "primeorder" version = "0.13.6" @@ -4794,6 +4391,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.6.24" @@ -4986,7 +4593,7 @@ dependencies = [ "hashbrown 0.17.1", "itertools 0.14.0", "kasuari", - "lru 0.18.0", + "lru", "palette", "serde", "strum", @@ -5160,12 +4767,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "regex-lite" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" - [[package]] name = "regex-syntax" version = "0.8.11" @@ -5181,6 +4782,71 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqsign-aws-v4" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e9e1168fab3883ec6afed1c2e20c25b2a09f366cdb662ac3e0878ae0332d63e" +dependencies = [ + "anyhow", + "bytes", + "form_urlencoded", + "hex", + "http", + "log", + "percent-encoding", + "quick-xml", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-command-execute-tokio" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf28fcf0aef694448cb235d393fbec7b5543d25050eacc0ccc2118596618c52" +dependencies = [ + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-core" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514a1e0b4aa288652a3fdbda4f0a610f379cdf5374e55a37c9edd03d57ed856b" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "futures", + "hex", + "hmac 0.13.0", + "http", + "jiff", + "log", + "percent-encoding", + "sha1 0.11.0", + "sha2 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -5190,8 +4856,8 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "hyper", "hyper-rustls", @@ -5229,8 +4895,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "hyper", "hyper-rustls", @@ -5376,15 +5042,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" -[[package]] -name = "roxmltree" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" -dependencies = [ - "xmlparser", -] - [[package]] name = "rsa" version = "0.9.10" @@ -5492,6 +5149,16 @@ dependencies = [ "sqlite-wasm-rs", ] +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rust-stemmers" version = "1.2.0" @@ -5777,12 +5444,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "separator" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" - [[package]] name = "serde" version = "1.0.228" @@ -5829,7 +5490,6 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ - "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6373,7 +6033,7 @@ dependencies = [ "half", "headers", "hex", - "http 1.4.2", + "http", "humantime", "ipnet", "jsonwebtoken 10.4.0", @@ -6481,7 +6141,7 @@ dependencies = [ "flatbuffers", "geo", "hex", - "http 1.4.2", + "http", "papaya", "rand 0.9.4", "regex", @@ -6814,6 +6474,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -6912,7 +6581,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.2", + "http", "httparse", "js-sys", "thiserror 2.0.18", @@ -6998,8 +6667,8 @@ dependencies = [ "base64 0.22.1", "bytes", "h2", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "hyper", "hyper-timeout", @@ -7056,8 +6725,8 @@ dependencies = [ "bitflags 2.13.0", "bytes", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http", + "http-body", "mime", "pin-project-lite", "tower", @@ -7166,7 +6835,7 @@ checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "data-encoding", - "http 1.4.2", + "http", "httparse", "log", "rand 0.9.4", @@ -7343,12 +7012,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "vsimd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" - [[package]] name = "vtparse" version = "0.6.2" @@ -7995,24 +7658,12 @@ dependencies = [ "rustix", ] -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - [[package]] name = "xxhash-rust" version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 8fafd780..de17481f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "kache" version = "0.11.0" edition = "2024" -description = "Zero-copy, content-addressed build cache for Rust, C/C++ and more. No copies, no wasted disk — just hardlinks locally and S3 for sharing." +description = "Zero-copy, content-addressed build cache for Rust, C/C++ and more, with S3 and shared-filesystem remotes." license = "Apache-2.0" repository = "https://github.com/kunobi-ninja/kache" homepage = "https://kunobi.ninja/docs/kache" @@ -20,35 +20,22 @@ resolver = "3" [dependencies] blake3 = "1" -# Already transitive via aws-sdk-s3; declared for RemoteBackend's body type. bytes = "1" -# TLS provider: `ring`, NOT aws-lc-rs. We avoid `aws-lc-sys` entirely -# because its cmake/NASM C build hangs when cross-compiling to -# `aarch64-pc-windows-msvc` under cargo-xwin; `ring` ships prebuilt asm -# for that target and cross-compiles cleanly. -# -# `aws-lc-sys` enters ONLY as the rustls crypto provider — `sigv4a` uses -# pure-Rust `p256`/`crypto-bigint`, not aws-lc. To force `ring`, every -# rustls consumer must opt out of aws-lc AND avoid re-enabling it via -# additive features: -# - aws-sdk-s3: drop `default-https-client` (it hard-wires -# `aws-smithy-http-client/rustls-aws-lc` on aws-smithy-runtime). -# We inject our own ring-backed Smithy client in `src/remote.rs`. -# - aws-config: `default-features = false` for the same reason — its -# default pulls `default-https-client`. We inject the client there too. -# - aws-smithy-http-client: `rustls-ring` (modern rustls 0.23 + ring). -# NOT `legacy-rustls-ring`, which pins vulnerable rustls 0.21 -# (RUSTSEC-2026-0098/0099/0104). -# - reqwest: `rustls-no-provider` + a process-default ring CryptoProvider -# installed at startup (see `src/main.rs`). -aws-sdk-s3 = { version = "1", default-features = false, features = ["sigv4a", "http-1x", "rt-tokio"] } -aws-config = { version = "1", default-features = false, features = ["rt-tokio", "credentials-process", "sso"] } -aws-smithy-http-client = { version = "1", features = ["rustls-ring"] } -# Already in-tree via aws-sdk-s3; direct dep only to name the concrete HTTP -# response type in SdkError so a raw GET 404 is detectable (#485 Phase 0). -aws-smithy-runtime-api = "1" -# Direct dep so we can install ring as the process-wide rustls crypto -# provider (reqwest uses `rustls-no-provider`, so one must be installed). +# OpenDAL 0.58 publishes its modules as separate crates. Depending on only the +# core, S3, filesystem, retry, and reqwest transport modules keeps Kache's +# supported backend set explicit and avoids compiling unused services. +opendal = { package = "opendal-core", version = "=0.58.0", default-features = false, features = ["executors-tokio"] } +opendal-http-transport-reqwest = { version = "=0.58.0", default-features = false, features = ["rustls-no-provider"] } +opendal-layer-retry = "=0.58.0" +opendal-service-fs = "=0.58.0" +opendal-service-s3 = "=0.58.0" +# OpenDAL exposes a custom credential chain but not a profile-name shortcut. +# These published reqsign APIs preserve Kache's explicit profile selection +# (including SSO and credential_process) without a git-only patch. +reqsign-aws-v4 = { version = "=3.0.2", default-features = false } +reqsign-command-execute-tokio = { version = "=3.0.2", default-features = false } +reqsign-core = { version = "=3.1.0", default-features = false } +# Direct dep so OpenDAL/reqwest can use ring without reintroducing aws-lc. rustls = { version = "0.23", default-features = false, features = ["ring"] } kache-core = { version = "0.11.0", path = "crates/kache-core", default-features = false, features = ["planning"] } async-trait = "0.1" @@ -107,10 +94,6 @@ windows-sys = { version = "0.61", features = [ assert_cmd = "2" predicates = "3" serde_json = "1" -# Wire-level mock S3 server (WireMockServer) for testing the remote/S3 code -# paths without network — see src/remote.rs and src/remote_layout.rs tests. -# `test-util` is dev-only; the release build never enables it. -aws-smithy-http-client = { version = "1", features = ["test-util", "wire-mock"] } bytes = "1" # `{ archive-suffix }` resolves from each target's pkg-fmt (".tar.gz" for @@ -140,7 +123,7 @@ extended-description = """\ kache is a zero-copy, content-addressed build cache for Rust, C/C++ and more. \ It avoids file \ copies and wasted disk by hardlinking artifacts locally and sharing them \ -via S3-compatible object storage.""" +via S3-compatible object storage or a shared filesystem.""" section = "utils" priority = "optional" # CRITICAL: empty depends. kache is built as a static musl binary, so it has diff --git a/README.md b/README.md index b40beea6..301ac10a 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,11 @@ [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) [![MSRV](https://img.shields.io/badge/MSRV-1.95-blue.svg)](Cargo.toml) -Zero-copy, content-addressed build cache for Rust and C/C++ object compiles. No copies, no wasted disk — reflinks where the filesystem supports them, hardlinks or copies otherwise, plus S3 for Rust artifact sharing. +Zero-copy, content-addressed build cache for Rust and C/C++ object compiles. No copies, no wasted disk — reflinks where the filesystem supports them, hardlinks or copies otherwise, plus S3 or shared-filesystem remotes for Rust artifact sharing. -A drop-in `RUSTC_WRAPPER` for Rust and a `cc` / `c++` compiler wrapper for C/C++ object compiles. Cache keys are blake3 hashes of normalized compiler inputs; cache hits restore zero-copy — a reflink (copy-on-write clone) where the filesystem supports it (APFS, btrfs, XFS-with-reflink), and a hardlink or copy otherwise — and identical blobs are stored once and shared. Optional S3 sync (AWS, Ceph, MinIO, R2) shares Rust artifacts across machines. +A drop-in `RUSTC_WRAPPER` for Rust and a `cc` / `c++` compiler wrapper for C/C++ object compiles. Cache keys are blake3 hashes of normalized compiler inputs; cache hits restore zero-copy — a reflink (copy-on-write clone) where the filesystem supports it (APFS, btrfs, XFS-with-reflink), and a hardlink or copy otherwise — and identical blobs are stored once and shared. Optional S3 (AWS, Ceph, MinIO, R2) or shared-filesystem sync shares Rust artifacts across machines. -Local Rust caching, local C/C++ object caching, and direct S3 sync are working today. C/C++ artifacts are local-only for now; unsupported compiler shapes pass through to the real compiler. +Local Rust caching, local C/C++ object caching, and direct S3/shared-filesystem sync are working today. C/C++ artifacts are local-only for now; unsupported compiler shapes pass through to the real compiler. **PREVIEW:** a remote planner that prefetches from workspace manifests, dependency history, and build intent — warming the right artifacts before rustc asks for them. The daemon already calls a planner when `KACHE_PLANNER_ENDPOINT` is set; the hosted service is still preview. @@ -167,7 +167,7 @@ set "CC=kache clang-cl" For **gcc/clang** the key is portable across machines and worktrees (paths are normalized via `-ffile-prefix-map`). Set `KACHE_BASE_DIR` to a user-declared root that gets stripped from the key, covering paths the derived source/build roots miss — e.g. objdir-built units whose `__FILE__` points above the derived root — for cross-checkout hits the automatic roots can't reach. If path normalization ever miscaches, `KACHE_CC_PATH_NORMALIZE=0` is the escape hatch: keys become path-literal (no cross-machine sharing, zero normalization risk). For **clang-cl** the key is already **machine-local**: clang-cl ignores `-ffile-prefix-map`, so kache keeps literal paths in the key — correct local hits, but no cross-machine sharing yet. clang-cl **debug** compiles (`/Z7`, `/Zi`, `-Z7`, or a `-g` form) are cached too: clang-cl embeds debug info straight into the `.obj` (there's no separate compile-time PDB), so kache folds the CodeView path inputs that object embeds — source path, output name, and compilation dir — into the same machine-local key. -**Not cached yet** (these pass through): link / whole-program steps, multi-source and multi-arch invocations, response-file (`@file`) invocations, precompiled headers, modules, coverage, and split-DWARF; for clang-cl also `-bigobj` and `-showIncludes`. C/C++ caching is also **local-only** for now — the Rust path's S3 sharing doesn't extend to `cc` artifacts yet. +**Not cached yet** (these pass through): link / whole-program steps, multi-source and multi-arch invocations, response-file (`@file`) invocations, precompiled headers, modules, coverage, and split-DWARF; for clang-cl also `-bigobj` and `-showIncludes`. C/C++ caching is also **local-only** for now — the Rust path's remote sharing doesn't extend to `cc` artifacts yet. C/C++ caching is live but still conservative by design: when in doubt, kache misses rather than serve a wrong artifact. See [C/C++ caching](docs/getting-started/c-cpp.mdx) for setup, status, and limitations. Scope and remaining work are tracked in [#49](https://github.com/kunobi-ninja/kache/issues/49). @@ -270,7 +270,7 @@ See [`scenarios/README.md`](scenarios/README.md) for the scenario format. | `kache list [] [--sort name\|size\|hits\|age]` | List cached entries, or show details for a specific crate | | `kache why-miss ` | Explain why a specific crate missed the cache | | `kache report [--format text\|json\|markdown\|github\|perfetto\|chrome-trace] [--since ] [--top ] [--output ]` | Generate a detailed hit/dup/miss build report or Perfetto/Chrome trace (`--top` defaults to 10) | -| `kache sync [--manifest-path ] [--pull] [--push] [--all] [--workspace] [--dry-run]` | Synchronize local cache with S3 remote (pull + push); `--manifest-path` points the Cargo.lock pull filter at a non-cwd manifest; `--workspace` scopes the pull to workspace members only (one LIST per member) | +| `kache sync [--manifest-path ] [--pull] [--push] [--all] [--workspace] [--dry-run]` | Synchronize the local cache with its configured remote (pull + push); `--manifest-path` controls the push-side workspace filter; `--workspace` scopes the pull to workspace members only (one LIST per member) | | `kache save-manifest [--manifest-key ] [--namespace ]` | Save a build manifest for future prefetch warming; `--manifest-key` overrides the default host-target-triple key | | `kache gc [--max-age ]` | Garbage collect — LRU eviction or age-based cleanup | | `kache purge [--crate-name ]` | Wipe entire cache or entries for a specific crate | @@ -291,14 +291,14 @@ Durations use days, hours, or bare hours: `7d`, `24h`, `1h`, `48`. ## Remote cache and configuration -`kache sync` can pull from and push to S3-compatible storage directly, without the daemon. Pulls are filtered by the current workspace's `Cargo.lock` by default. See [Sync](docs/remote-cache/sync.mdx) for the full command behavior and S3 layout. +`kache sync` can pull from and push to S3-compatible storage or a shared filesystem directly, without the daemon. Pulls are filtered by the current workspace's `Cargo.lock` by default. See [Sync](docs/remote-cache/sync.mdx) for the full command behavior and remote layout. Configuration is available through `kache config`, environment variables, or config files. Environment variables win over config files, and project-local `.kache.toml` files are supported. See [Configuration](docs/getting-started/configuration.mdx) for the full reference. ## Architecture - **Wrapper**: `RUSTC_WRAPPER` intercepts rustc calls, computes blake3 cache keys, restores hits zero-copy (reflink where supported, else hardlink or copy) -- **Daemon**: Background process handles async S3 uploads, remote checks, and prefetch. Auto-restarts when binary is updated +- **Daemon**: Background process handles async remote uploads, checks, and prefetch. Auto-restarts when binary is updated - **Store**: content-addressed blobs under `{cache_dir}/store/blobs//`, indexed by a SQLite DB; cache hits reflink or hardlink those blobs into `target/` - **Cache keys**: Deterministic blake3 hash of rustc version, crate name, source, dependencies, and normalized flags — portable across machines diff --git a/deny.toml b/deny.toml index 280dd33f..35db053a 100644 --- a/deny.toml +++ b/deny.toml @@ -110,7 +110,7 @@ license-files = [{ path = "LICENSE", hash = 0xbb1191d7 }] unknown-registry = "deny" unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] -# First-party git dependencies (kache-service HA + auth building blocks). +# First-party dependencies. allow-git = [ "https://github.com/kunobi-ninja/kunobi-auth.git", "https://github.com/kunobi-ninja/kunobi-ha.git", diff --git a/docs/commands/reference.mdx b/docs/commands/reference.mdx index 7a938ad1..02d70ebc 100644 --- a/docs/commands/reference.mdx +++ b/docs/commands/reference.mdx @@ -21,7 +21,7 @@ When invoked without arguments, kache prints `--help` and exits — it does not | `kache gc [--max-age ]` | Evict entries by LRU or age | | `kache purge [--crate-name ]` | Wipe entire cache or entries for one crate | | `kache clean [-n \| --dry-run] [-y \| --yes]` | Find and remove `target/` directories (interactive; `-n` previews, `-y` removes all non-interactively) | -| `kache sync [flags]` | Sync local cache with S3 remote | +| `kache sync [flags]` | Sync local cache with a configured remote | | `kache save-manifest [--manifest-key ] [--namespace ]` | Save a build manifest for future prefetch warming | | `kache doctor [--fix [--purge-sccache]] [--verify] [--checksums] [--repair]` | Diagnose and fix setup issues; verify cache integrity | | `kache config` | Open the TUI configuration editor | @@ -256,15 +256,16 @@ On macOS, the scan skips TCC-protected locations (`/System`, `/Library`, `/priva ## `kache sync` ```sh -kache sync [--pull] [--push] [--all] [--dry-run] [--manifest-path ] +kache sync [--pull] [--push] [--all] [--workspace] [--dry-run] [--manifest-path ] ``` -Synchronizes the local cache with the configured S3 remote. See [Sync](/docs/remote-cache/sync) for a full walkthrough. +Synchronizes the local cache with the configured S3 or filesystem remote. See +[Sync](/docs/remote-cache/sync) for a full walkthrough. ```sh kache sync # pull missing + push new artifacts kache sync --pull # download only, filtered to current workspace -kache sync --pull --all # download everything in the bucket +kache sync --pull --all # download everything in the remote kache sync --push # upload only kache sync --dry-run # preview transfers, make no changes ``` diff --git a/docs/daemon/lifecycle.mdx b/docs/daemon/lifecycle.mdx index e7b4534c..cf291939 100644 --- a/docs/daemon/lifecycle.mdx +++ b/docs/daemon/lifecycle.mdx @@ -86,7 +86,7 @@ A few behaviors worth knowing about: - **Single instance**: a second `kache daemon run` exits cleanly (code 0) if another process holds the run lock or the socket is already live, so launchd/systemd `KeepAlive` won't loop. Stale socket files are removed on startup when nothing is listening. - **Background log file**: auto-started daemons write stderr to `.log` next to the daemon socket (Linux `~/.cache/kache/daemon.log`, macOS `~/Library/Caches/kache/daemon.log`). It is reset to a `--- log rotated ---` marker once it grows past 2 MiB. `kache daemon log` tails this file. -- **Graceful drain**: on shutdown the daemon closes its upload queue and waits up to 30 s for in-flight S3 uploads to finish before aborting workers. +- **Graceful drain**: on shutdown the daemon closes its upload queue and waits up to 30 s for in-flight remote uploads to finish before aborting workers. ## Offline behavior diff --git a/docs/daemon/overview.mdx b/docs/daemon/overview.mdx index 681a2f0f..2174be3d 100644 --- a/docs/daemon/overview.mdx +++ b/docs/daemon/overview.mdx @@ -9,11 +9,11 @@ The daemon is a background process that handles everything that shouldn't block ## What the daemon handles -**Upload queue.** After a cache miss and successful compilation, the wrapper hands the daemon a reference to the freshly stored entry and returns immediately. The daemon reads it from the local store and uploads to S3 in the background. No build time is spent waiting for uploads. +**Upload queue.** After a cache miss and successful compilation, the wrapper hands the daemon a reference to the freshly stored entry and returns immediately. The daemon reads it from the local store and uploads to the configured remote in the background. No build time is spent waiting for uploads. -**Remote checks.** Before compiling a crate that missed locally, the wrapper asks the daemon to check S3 for the artifact. The daemon downloads it if found, then signals the wrapper to restore from the local store. +**Remote checks.** Before compiling a crate that missed locally, the wrapper asks the daemon to check the remote for the artifact. The daemon downloads it if found, then signals the wrapper to restore from the local store. -**Prefetch.** At the start of a new build session, the wrapper detects that a build is beginning (via a timestamped marker file), runs `cargo metadata` to get the full dependency graph, and sends the daemon a prefetch hint with all crate names in topological order. The daemon prefetches them from S3 before cargo even asks for them, turning remote hits into local hits for the rest of the build. +**Prefetch.** At the start of a new build session, the wrapper detects that a build is beginning (via a timestamped marker file), runs `cargo metadata` to get the full dependency graph, and sends the daemon a prefetch hint with all crate names in topological order. The daemon prefetches them from the remote before cargo even asks for them, turning remote hits into local hits for the rest of the build. The prefetch hint fires once per build session (with a 5-minute cooldown) to avoid redundant work across sequential cargo commands in CI — `cargo check`, `cargo clippy`, `cargo test` may all run within the same session window. @@ -29,7 +29,7 @@ If you're using kache locally with no remote configured, you don't need the daem The daemon becomes necessary as soon as you configure a remote cache. Without it: -- New artifacts won't be uploaded to S3 +- New artifacts won't be uploaded to the remote - The cache won't be checked before a local miss triggers compilation - Prefetch won't run diff --git a/docs/getting-started/c-cpp.mdx b/docs/getting-started/c-cpp.mdx index 6f51a96b..b12ace89 100644 --- a/docs/getting-started/c-cpp.mdx +++ b/docs/getting-started/c-cpp.mdx @@ -78,7 +78,7 @@ These shapes compile normally but are not cached yet: - clang-cl `-bigobj` and `-showIncludes` - flags kache has not classified -C/C++ artifacts are local-only today. S3 sync and remote sharing apply to Rust artifacts, not `cc` objects. +C/C++ artifacts are local-only today. Remote sync and sharing apply to Rust artifacts, not `cc` objects. ## Diagnosing Passthroughs diff --git a/docs/getting-started/configuration.mdx b/docs/getting-started/configuration.mdx index e2816c13..478c5a92 100644 --- a/docs/getting-started/configuration.mdx +++ b/docs/getting-started/configuration.mdx @@ -27,15 +27,38 @@ Config file priority: 2. The nearest project-local `.kache.toml`, walking up from the current directory 3. User config at `~/.config/kache/config.toml`, respecting `XDG_CONFIG_HOME` -A minimal config that enables a remote cache looks like this: - -```toml title="~/.config/kache/config.toml" -[cache.remote] -type = "s3" -bucket = "my-build-cache" -endpoint = "https://s3.example.com" # omit for AWS S3 -profile = "my-aws-profile" # omit to use the default credential chain -``` +kache currently compiles two remote types: `s3` and `filesystem`. + + + + ```toml title="~/.config/kache/config.toml" + [cache.remote] + type = "s3" + bucket = "my-build-cache" + endpoint = "https://s3.example.com" # omit for AWS S3 + profile = "my-aws-profile" # omit to use the default credential chain + ``` + + + ```toml title="~/.config/kache/config.toml" + [cache.remote] + type = "filesystem" + path = "/mnt/shared-kache" + prefix = "artifacts" + ``` + + `atomic_write_dir` is optional and defaults to + `/mnt/shared-kache/.kache-tmp`. See [Filesystem setup](/docs/remote-cache/filesystem-setup) + before overriding it. + + + +Other OpenDAL services are not included in the kache binary. Existing S3 +configuration names remain compatible: a legacy `[cache.remote]` table without +`type = "s3"` and an environment-only setup rooted in `KACHE_S3_BUCKET` still +select S3; the other `KACHE_S3_*` overrides remain available. See the +[S3 migration boundaries](/docs/remote-cache/s3-setup#opendal-migration-boundaries) +for AWS SDK-specific cases that need attention. ## All settings @@ -47,10 +70,13 @@ profile = "my-aws-profile" # omit to use the default credential chain | `KACHE_CONFIG` | — | — | Explicit config file path; overrides the project-local `.kache.toml` / XDG resolution | | `KACHE_BASE_DIR` | — | — | Path prefix stripped from cache keys (collapsed to ``) for checkout / container-mount paths the automatic sentinels don't catch — the analog of ccache's `CCACHE_BASEDIR` (see [Cache key](/docs/how-it-works/cache-key)) | | — (file-only) | `paths.base_dirs` | `[]` | Extra absolute path prefixes for container mounts, Snap, Flatpak, AppImage, or custom roots; each maps to a distinct deterministic sentinel (see [Extra path prefixes](#extra-path-prefixes)) | +| — (file-only) | `cache.remote.type` | — | Remote type: `s3` or `filesystem`. A legacy S3 table or `KACHE_S3_BUCKET` still selects S3 when this is omitted | +| — (file-only) | `cache.remote.path` | — | Shared root directory for a `filesystem` remote | +| — (file-only) | `cache.remote.atomic_write_dir` | `/.kache-tmp` | Staging directory for filesystem-remote atomic writes; must be on the same filesystem as `path` | | `KACHE_S3_BUCKET` | `cache.remote.bucket` | — | S3 bucket name | | `KACHE_S3_ENDPOINT` | `cache.remote.endpoint` | — | S3 endpoint URL (required for Ceph, MinIO, R2) | | `KACHE_S3_REGION` | `cache.remote.region` | `us-east-1` | AWS region | -| `KACHE_S3_PREFIX` | `cache.remote.prefix` | `artifacts` | Key prefix inside the bucket | +| `KACHE_S3_PREFIX` | `cache.remote.prefix` | `artifacts` | Object or path prefix. The environment override is retained for S3; filesystem remotes configure it in the file | | `KACHE_S3_PROFILE` | `cache.remote.profile` | — | AWS credentials profile | | `KACHE_S3_ACCESS_KEY` | — | — | Explicit S3 access key | | `KACHE_S3_SECRET_KEY` | — | — | Explicit S3 secret key | @@ -59,13 +85,13 @@ profile = "my-aws-profile" # omit to use the default credential chain | `KACHE_VERIFY_RESTORES` | — | `off` | Re-hash restored blobs before serving hits: `off`, `sampled` (~1/16 hits), or `always`. `1`/`true` map to `always` | | — | `cache.exclude` | `[]` | Source-path glob patterns that bypass kache and compile normally without lookup, store, or upload | | `KACHE_COMPRESSION_LEVEL` | `cache.compression_level` | `3` | Zstd compression level (1–22) | -| `KACHE_S3_CONCURRENCY` | `cache.s3_concurrency` | `16` | Max concurrent S3 operations | +| `KACHE_S3_CONCURRENCY` | `cache.s3_concurrency` | `16` | Max concurrent remote operations; the legacy S3 name applies to both remote types | | `KACHE_S3_POOL_IDLE_SECS` | `cache.s3_pool_idle_secs` | `300` | How long an idle S3 connection is kept in the HTTP pool. Higher values reuse warm TLS sessions across build phases; lower this if you sit behind a load balancer that drops idle connections aggressively | | `KACHE_DAEMON_IDLE_TIMEOUT` | `cache.daemon_idle_timeout_secs` | `600` | Idle daemon shutdown timeout in seconds (`0` disables auto-shutdown) | | `KACHE_KEY_SALT` | `cache.key_salt` | — | Opaque string folded into every cache key. Change it to force a cold cache on a toolchain change kache cannot otherwise see (see [Cache-key salt](#cache-key-salt)) | | `KACHE_CC_EXTRA_ALLOWLIST_FLAGS` | `cc.extra_allowlist_flags` | `[]` | C/C++ flags to opt into caching that kache's built-in allow-list doesn't yet model (see [Extra cc allowlist flags](#extra-cc-allowlist-flags)). Env value is whitespace-separated | | `KACHE_DISABLED` | — | `false` | Disable caching entirely (pass-through to rustc). Env-only, no config key: `1` or `true` (case-insensitive) disables; any other value — including `0`, `false`, or unset — leaves caching on | -| `KACHE_LOCAL_ONLY` | `cache.local_only` | `false` | Strict local-only mode: ignore **all** remote and planner config/env (no S3 bucket, no planner endpoint, no egress) for a guaranteed-hermetic build. Local caching stays fully on — unlike `KACHE_DISABLED`. `1`/`true` enables; env wins over the file (an explicit `0` overrides `local_only = true`) | +| `KACHE_LOCAL_ONLY` | `cache.local_only` | `false` | Strict local-only mode: ignore **all** remote and planner config/env (no configured remote, no planner endpoint, no egress) for a guaranteed-hermetic build. Local caching stays fully on — unlike `KACHE_DISABLED`. `1`/`true` enables; env wins over the file (an explicit `0` overrides `local_only = true`) | | `KACHE_REMOTE_READONLY` | `cache.remote_readonly` | `false` | Read-only remote consumer mode: when enabled, kache performs remote cache reads/restores as normal, but suppresses all remote uploads (including the sync push phase and manifest saving). `1`/`true` enables; env wins over the file (an explicit `0` overrides `remote_readonly = true`) | | `KACHE_WINDOWS_HARDLINK` | `cache.windows_hardlink` | `false` | Windows only: restore cache hits on non-CoW volumes (NTFS) via hardlink instead of copy, deduplicating the working tree against the store. Opt in **only** if your build never deletes or rewrites a restored output in place — a mutation would corrupt the shared store blob. ReFS (Dev Drive) volumes always block-clone regardless of this flag | | `KACHE_STORAGE_LAYOUT_ADVICE` | `cache.storage_layout_advice` | `true` | Surface an advisory (deduplicated, at most once per 5-minute window) when a cache hit is restored by copy because the storage layout prevents zero-copy dedup: no copy-on-write on the volume, cache and build tree on different volumes, or an inconclusive capability probe. Set to `0`/`false` when the layout is intentional (e.g. an NTFS-only machine that cannot host a ReFS Dev Drive); genuine clone faults are still reported | @@ -114,7 +140,7 @@ ignore_env = true key_salt = "v3-toolchain-abc" ``` -With this, kache ignores the `KACHE_*` overrides for **file-backed settings** (cache dir, max size, key salt, fallback, S3/planner settings, the toggles, etc.) and takes the file value (or built-in default) instead. When an ignored override is actually set, kache logs a warning naming it, so the suppression is never silent. +With this, kache ignores the `KACHE_*` overrides for **file-backed settings** (cache dir, max size, key salt, fallback, remote/planner settings, the toggles, etc.) and takes the file value (or built-in default) instead. When an ignored override is actually set, kache logs a warning naming it, so the suppression is never silent. `ignore_env` is **file-only** by design — an env var can't re-enable env overrides, or the lockdown would be trivially undone by the same stray export it defends against. It deliberately does **not** cover bootstrap/operational vars that have no file representation (`KACHE_CONFIG`, `KACHE_DISABLED`, `KACHE_LOG` / `KACHE_LOG_FILE` / `KACHE_PROGRESS`, `KACHE_NAMESPACE`, `KACHE_BASE_DIR`) or S3 credentials (`KACHE_S3_ACCESS_KEY` / `KACHE_S3_SECRET_KEY`), which are secrets rather than config. @@ -271,13 +297,15 @@ Avoid host-dependent flags like `-march=native`. The string is identical on ever To confirm it took effect, run with `KACHE_LOG=kache=debug`: the cc flag-classify summary gains a `user-allowed` count, and the flag no longer appears in any `unsupported flag(s): … — passthrough` line. At `kache=trace` each accepted flag logs as `user-allowed (config)` and each one folded into the key logs as `cc_extra_flag=`. -## Credential resolution order +## S3 credential resolution order Remote S3 credentials are resolved in this order: 1. `KACHE_S3_ACCESS_KEY` + `KACHE_S3_SECRET_KEY` -2. AWS profile from `cache.remote.profile` or `KACHE_S3_PROFILE` -3. AWS default chain, such as `AWS_ACCESS_KEY_ID`, `~/.aws/credentials [default]`, or IAM roles +2. Standard `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` +3. Static credentials in the selected profile (`KACHE_S3_PROFILE`, `cache.remote.profile`, `AWS_PROFILE`, then `default`) +4. Legacy inline SSO, then `credential_process` for the selected profile +5. Environment-based web identity, ECS/task credentials, then EC2 instance credentials For Ceph, MinIO, or R2, set `cache.remote.endpoint` or `KACHE_S3_ENDPOINT`. @@ -384,6 +412,7 @@ Don't bind-mount the host's cache directory into the container. Sharing it gains The same rule applies to network filesystems (NFS, SMB/CIFS, 9p) and any setup where more than one machine touches the same directory: keep `KACHE_CACHE_DIR` on a fast, local disk owned by a single machine. To reuse - build artifacts *across* machines, use a [remote cache](/docs/remote-cache/s3-setup) - — that's the supported path for cross-machine sharing. + build artifacts *across* machines, configure an [S3 remote](/docs/remote-cache/s3-setup) + or a separate [filesystem remote](/docs/remote-cache/filesystem-setup). Only + the filesystem remote's `path` belongs on the shared mount. diff --git a/docs/how-it-works/architecture.mdx b/docs/how-it-works/architecture.mdx index b4a63937..3ea31cf8 100644 --- a/docs/how-it-works/architecture.mdx +++ b/docs/how-it-works/architecture.mdx @@ -65,9 +65,9 @@ On filesystems without reflink, kache falls back to a hardlink for immutable art ## The daemon -The daemon is a long-running background process that handles everything async: uploading new artifacts to S3, checking the remote before a build starts, and prefetching artifacts for upcoming crates. +The daemon is a long-running background process that handles everything async: uploading new artifacts to the configured remote, checking it before a build starts, and prefetching artifacts for upcoming crates. -The wrapper communicates with the daemon over a Unix socket (`daemon.sock` inside the cache dir — see [The local store](#the-local-store) for the platform path; a named pipe on Windows) using lightweight RPC calls. Local-only calls — queuing an upload, a cached remote-check answer — return in well under a millisecond; a remote-check that has to ask S3 takes as long as the network round-trip. If the daemon is down, the wrapper continues without it — local caching works normally, remote features degrade gracefully. +The wrapper communicates with the daemon over a Unix socket (`daemon.sock` inside the cache dir — see [The local store](#the-local-store) for the platform path; a named pipe on Windows) using lightweight RPC calls. Local-only calls — queuing an upload, a cached remote-check answer — return in well under a millisecond; a remote check that reaches the configured backend takes as long as that round-trip. If the daemon is down, the wrapper continues without it — local caching works normally, remote features degrade gracefully. The daemon runs as a separate binary invocation (`kache daemon run`) and can be managed as a system service via launchd (macOS) or systemd (Linux). See [Daemon lifecycle](/docs/daemon/lifecycle) for details. @@ -79,6 +79,6 @@ By default, kache skips only user-facing executables — `bin` crates and `--tes ## The remote planner -By default, prefetch is driven entirely by the client running `cargo metadata` and asking the daemon to pull every crate in the resolved graph from S3. This is good but not optimal — the metadata pass is workspace-local and can't reason about which artifacts are *most* likely to hit on the current toolchain / target / feature set. +By default, prefetch is driven entirely by the client running `cargo metadata` and asking the daemon to pull every crate in the resolved graph from the configured remote. This is good but not optimal — the metadata pass is workspace-local and can't reason about which artifacts are *most* likely to hit on the current toolchain / target / feature set. -A separate planner service (see [Remote service](/docs/remote-service)) accepts manifests from clients (`kache save-manifest`), stores them in an embedded database, and replies to prefetch queries with a ranked candidate list. The client path is already wired: when `KACHE_PLANNER_ENDPOINT` is set, the daemon asks the planner before each build. When the planner has no useful data it returns a fallback disposition and the client behaves exactly as it does without it. The planner is purely additive — local caching, S3 sync, and metadata-driven prefetch keep working with no planner configured. (The hosted planner service itself is still in preview.) +A separate planner service (see [Remote service](/docs/remote-service)) accepts manifests from clients (`kache save-manifest`), stores them in an embedded database, and replies to prefetch queries with a ranked candidate list. The client path is already wired: when `KACHE_PLANNER_ENDPOINT` is set, the daemon asks the planner before each build. When the planner has no useful data it returns a fallback disposition and the client behaves exactly as it does without it. The planner is purely additive — local caching, remote sync, and metadata-driven prefetch keep working with no planner configured. (The hosted planner service itself is still in preview.) diff --git a/docs/index.mdx b/docs/index.mdx index ac23cd4a..39ab3f1e 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -1,13 +1,13 @@ --- title: kache -description: Zero-copy, content-addressed build cache for Rust and C/C++ object compiles. No copies, no wasted disk — zero-copy restores locally and S3 for Rust artifact sharing. +description: Zero-copy, content-addressed build cache for Rust and C/C++ object compiles. No copies, no wasted disk — local restores plus S3 or filesystem sharing. --- # Introduction -**kache** is a drop-in `RUSTC_WRAPPER` for Rust and a `cc` / `c++` compiler wrapper for C/C++ object compiles. Cache keys are blake3 hashes of normalized compiler inputs; cache hits restore zero-copy (reflink, with a hardlink/copy fallback on filesystems without CoW), and identical blobs are stored once and shared. Optional S3 sync (AWS, Ceph, MinIO, R2) shares Rust artifacts across machines. +**kache** is a drop-in `RUSTC_WRAPPER` for Rust and a `cc` / `c++` compiler wrapper for C/C++ object compiles. Cache keys are blake3 hashes of normalized compiler inputs; cache hits restore zero-copy (reflink, with a hardlink/copy fallback on filesystems without CoW), and identical blobs are stored once and shared. An optional S3-compatible or shared-filesystem remote shares Rust artifacts across machines. -Local Rust caching, local C/C++ object caching, and direct S3 sync are working today. C/C++ artifacts are local-only for now; unsupported compiler shapes pass through to the real compiler. +Local Rust caching, local C/C++ object caching, and direct remote sync are working today. C/C++ artifacts are local-only for now; unsupported compiler shapes pass through to the real compiler. **PREVIEW:** a remote planner that prefetches from workspace manifests, dependency history, and build intent — warming the right artifacts before rustc asks for them. The daemon already calls a planner when `KACHE_PLANNER_ENDPOINT` is set; the hosted service is still preview. See [Remote service](/docs/remote-service). @@ -45,7 +45,7 @@ kache doctor ## Architecture in one screen - **Wrapper** — `RUSTC_WRAPPER` intercepts rustc calls; `CC="kache cc"` and `CXX="kache c++"` intercept supported C/C++ object compiles. Hits restore zero-copy (reflink, falling back to hardlink/copy). -- **Daemon** — background process handles async S3 uploads, remote checks, and prefetch. Auto-restarts on binary update. +- **Daemon** — background process handles async remote uploads, checks, and prefetch. Auto-restarts on binary update. - **Store** — SQLite index at `{cache_dir}/index.db`, plus content-addressed blobs under `{cache_dir}/store/blobs/`; hits restore those blobs into `target/` zero-copy (reflink, falling back to hardlink/copy). - **Cache keys** — deterministic blake3 hash of rustc version, crate name, source, dependencies, and normalized flags — portable across machines. @@ -55,7 +55,8 @@ kache doctor - + + diff --git a/docs/monitor.mdx b/docs/monitor.mdx index 3b5029b8..5e295f69 100644 --- a/docs/monitor.mdx +++ b/docs/monitor.mdx @@ -99,7 +99,7 @@ Press `s` to cycle the sort order (size → hits → age → name). Press `f` to ### 4 — Transfer -A log of recent S3 uploads and downloads, with transfer rates and artifact sizes. Useful for verifying that the daemon is uploading new artifacts and that remote hits are flowing in correctly. +A log of recent remote uploads and downloads, with transfer rates and artifact sizes. Useful for verifying that the daemon is uploading new artifacts and that remote hits are flowing in correctly. ### 5 — Passthrough @@ -121,7 +121,7 @@ r refresh project list (Projects tab) ## Quick triage **High miss rate when the daemon is offline.** -The remote cache isn't being checked. Start the daemon (`kache daemon start`) and verify it can reach the S3 bucket. Check `kache daemon log` for errors. +The remote cache isn't being checked. Start the daemon (`kache daemon start`) and verify it can reach the configured remote. Check `kache daemon log` for errors. **Store usage is near the maximum.** Run `kache gc` to evict the least-recently-used entries, or `kache gc --max-age 7d` to remove anything older than a week. You can also raise `KACHE_MAX_SIZE` if disk space allows. diff --git a/docs/remote-cache/ci.mdx b/docs/remote-cache/ci.mdx index 14b24852..a74ff3cf 100644 --- a/docs/remote-cache/ci.mdx +++ b/docs/remote-cache/ci.mdx @@ -106,7 +106,7 @@ When no daemon is running in an ephemeral runner, use explicit sync steps: The `if: always()` on the push step stores newly built artifacts even when the build fails partway through. Partial cache population still speeds up future runs. -Both sync subcommands require a configured S3 remote. Without one they error with `No remote configured`. By default `kache sync --pull` is filtered to the crates in the current workspace's `Cargo.lock`, so an ephemeral runner only fetches what this build needs. Use `kache sync --pull --all` to pull every artifact in the bucket regardless of the local lockfile. +Both sync subcommands require a configured remote. Without one they error with `No remote configured`. By default `kache sync --pull` is filtered to the crates in the current workspace's `Cargo.lock`, so an ephemeral runner only fetches what this build needs. Use `kache sync --pull --all` to pull every artifact in the remote regardless of the local lockfile. `kache sync` reports failed transfers as `(N failed)` on its progress line but still exits `0`, so CI cannot detect partial transfer failures from the exit code alone. If that matters, scan the step's stderr for `failed`. diff --git a/docs/remote-cache/filesystem-setup.mdx b/docs/remote-cache/filesystem-setup.mdx new file mode 100644 index 00000000..26533305 --- /dev/null +++ b/docs/remote-cache/filesystem-setup.mdx @@ -0,0 +1,79 @@ +--- +title: Filesystem setup +description: Configure a shared filesystem as the kache remote cache. +--- + +# Filesystem setup + +A filesystem remote stores the same versioned cache objects as S3 in a shared +directory. It is useful when every builder can mount the same NFS, SMB, or other +shared filesystem and no object-storage service is needed. + +## Minimal configuration + +```toml title="~/.config/kache/config.toml" +[cache.remote] +type = "filesystem" +path = "/mnt/shared-kache" +prefix = "artifacts" +``` + +Use an absolute `path` so the wrapper, daemon, and `kache sync` resolve the same +directory regardless of their working directory. Every machine that reads or +writes the remote must mount that directory and have suitable permissions. + +Treat every writer to the shared directory as trusted. A filesystem remote is +not a security boundary: a peer that can replace cache paths or create symlinks +can influence what other machines read or overwrite. Use S3 with access controls +when writers are not mutually trusted. + + + kache currently compiles the `s3` and `filesystem` remote types. Other + OpenDAL services are not included in the binary. + + +## Atomic writes + +Writes are staged in a temporary directory and atomically renamed into their +final location, so readers never observe a partially written pack or manifest. +By default, the staging directory is `/.kache-tmp`. + +Override it only when the default is unsuitable: + +```toml title="~/.config/kache/config.toml" +[cache.remote] +type = "filesystem" +path = "/mnt/shared-kache" +prefix = "artifacts" +atomic_write_dir = "/mnt/shared-kache/.staging" +``` + + + `atomic_write_dir` and `path` must be on the same filesystem. The final + rename cannot be atomic across filesystems and will fail with a cross-device + error. In particular, do not use the machine's `/tmp` unless it is on the + same filesystem as the shared cache. + + +## Directory layout + +With the minimal configuration, the shared directory contains: + +```text +/mnt/shared-kache/ +├── .kache-tmp/ +└── artifacts/ + ├── v3/ + │ ├── manifests/{crate_name}/{cache_key}.json + │ └── packs/{crate_name}/{cache_key}.tar.zst + └── _manifests/ +``` + +`prefix` keeps kache's objects in their own subtree. The logical layout and +`kache sync` behavior are the same for S3 and filesystem remotes. Filesystem +prefixes cannot contain `:` so the same config cannot become a Windows drive +path or alternate data stream. + +Keep `KACHE_CACHE_DIR` on fast local storage for each machine. Only +`cache.remote.path` should point at the shared mount; the local store is not +safe to share between machines. diff --git a/docs/remote-cache/meta.json b/docs/remote-cache/meta.json index a42aa9d3..b39e6f36 100644 --- a/docs/remote-cache/meta.json +++ b/docs/remote-cache/meta.json @@ -1,4 +1,4 @@ { "title": "Remote cache", - "pages": ["s3-setup", "sync", "ci"] + "pages": ["s3-setup", "filesystem-setup", "sync", "ci"] } diff --git a/docs/remote-cache/s3-setup.mdx b/docs/remote-cache/s3-setup.mdx index 4941af29..81db41db 100644 --- a/docs/remote-cache/s3-setup.mdx +++ b/docs/remote-cache/s3-setup.mdx @@ -5,7 +5,13 @@ description: Configure an S3-compatible remote cache and connect kache to it. # S3 setup -kache supports any S3-compatible storage: AWS S3, Cloudflare R2, Ceph, MinIO, and others. The configuration is the same regardless of provider — you only need to adjust the endpoint and credentials. +kache supports AWS S3 and common S3-compatible storage such as Cloudflare R2, +Ceph, and MinIO. The configuration is the same across providers; adjust the +endpoint and credentials. + +For a shared mounted directory instead, see [Filesystem setup](/docs/remote-cache/filesystem-setup). +The kache binary currently compiles only these `s3` and `filesystem` remote +types. ## Minimal configuration @@ -66,10 +72,16 @@ With just a bucket name and no endpoint, kache uses AWS S3 with the default cred When kache needs S3 credentials, it checks these sources in order: 1. `KACHE_S3_ACCESS_KEY` + `KACHE_S3_SECRET_KEY` (explicit env var override) -2. AWS profile from `KACHE_S3_PROFILE` env var or `profile` config field -3. Standard AWS chain: `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` env vars, then `~/.aws/credentials [default]`, then IAM instance/task roles +2. Standard `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` environment credentials +3. The selected shared profile (`KACHE_S3_PROFILE`, `profile`, `AWS_PROFILE`, then `default`) +4. Legacy inline SSO for that profile +5. `credential_process` for that profile +6. Environment-based web identity (`AWS_ROLE_ARN` + `AWS_WEB_IDENTITY_TOKEN_FILE`) +7. ECS/task credentials, then EC2 instance credentials -For CI, option 1 (explicit env vars) or option 3 (IAM roles) are the most common. For local setups with multiple AWS accounts, option 2 (named profile) keeps credentials organized. +For CI, explicit environment credentials or IAM roles are the most common. For +local setups with multiple AWS accounts, a named profile keeps credentials +organized. Both halves of the explicit pair are required. If only `KACHE_S3_ACCESS_KEY` or only `KACHE_S3_SECRET_KEY` is set, kache logs a warning and falls back to the AWS chain rather than erroring — so a missing half surfaces as confusing "wrong credentials" behavior, not a clear failure. @@ -88,6 +100,43 @@ Every remote config field has a matching `KACHE_S3_*` env var, which takes prece | `KACHE_S3_PROFILE` | `profile` | | `KACHE_S3_ACCESS_KEY` / `KACHE_S3_SECRET_KEY` | (explicit credentials) | +Existing Kache S3 config keys and defaults remain accepted. A legacy +`[cache.remote]` table without a `type` field and an environment-only setup +rooted in `KACHE_S3_BUCKET` still select S3; the other `KACHE_S3_*` overrides +remain available. `type = "s3"` is the recommended explicit form for new +config files. The AWS SDK-specific migration boundaries below still apply. + +## OpenDAL migration boundaries + +The config-file and `KACHE_S3_*` interfaces are retained. Static shared +profiles, legacy inline SSO profiles, `credential_process`, environment-based +web identity, ECS/task credentials, and EC2 instance credentials remain +available. Web identity uses `cache.remote.region` when choosing its STS +endpoint. + + + OpenDAL currently signs with SigV4 only. SigV4a / Multi-Region Access Points, + assume-role profiles that rely on `role_arn` with `source_profile` or + `credential_source`, and modern profiles that refer to a separate + `[sso-session ...]` section are not yet compatible. Profile-based web + identity (`role_arn` + `web_identity_token_file` in the profile) is also not + loaded; use its standard environment variables instead. Resolve unsupported + profile credentials before launching kache (standard AWS credential env vars + work), or use `credential_process`. AWS China, ISO, FIPS, and dual-stack + endpoints should also set `endpoint` explicitly. + + +OpenDAL normalizes object paths. To prevent a silent switch to different keys, +`prefix` must now be a canonical relative path: no leading or trailing slash, +backslashes, surrounding whitespace, `//`, `.` or `..` segments. If an older +setup used a non-canonical prefix, migrate or copy those objects before +upgrading. + +`AWS_ENDPOINT_URL_S3` remains supported when `endpoint` / `KACHE_S3_ENDPOINT` +is unset. An `endpoint_url` stored only inside an AWS shared profile is not +loaded by OpenDAL; copy it to `cache.remote.endpoint` or +`KACHE_S3_ENDPOINT` during migration. + ## S3 bucket layout Each cached entry is two objects — a packed tarball plus a small JSON manifest used for existence checks and listing: diff --git a/docs/remote-cache/sync.mdx b/docs/remote-cache/sync.mdx index 35f2c17c..4c7a2765 100644 --- a/docs/remote-cache/sync.mdx +++ b/docs/remote-cache/sync.mdx @@ -1,11 +1,14 @@ --- title: Sync -description: Manually synchronize the local cache with S3 using kache sync. +description: Manually synchronize the local cache with a configured remote using kache sync. --- # Sync -`kache sync` transfers artifacts between the local store and S3. It runs directly against S3 without requiring the daemon, which makes it useful for CI steps that want explicit control over cache population timing. +`kache sync` transfers artifacts between the local store and the configured S3 +or filesystem remote. It talks directly to the remote without requiring the +daemon, which makes it useful for CI steps that want explicit control over +cache population timing. ## Basic usage @@ -22,31 +25,42 @@ kache sync --dry-run # preview what would transfer, make no changes The pull and push directions filter by two different mechanisms: -- **Pull** filters downloads to every crate listed in the current directory's `Cargo.lock` (all direct and transitive dependencies). This avoids pulling the entire bucket on every sync — in a shared cache used by multiple projects, you only download what your project needs. +- **Pull** filters downloads to every crate listed in the current directory's `Cargo.lock` (all direct and transitive dependencies). This avoids pulling the entire remote on every sync — in a shared cache used by multiple projects, you only download what your project needs. - **Push** filters uploads to workspace member packages, derived by running `cargo metadata --no-deps` on the current `Cargo.toml` (or the `--manifest-path` you pass). Non-workspace local artifacts are silently skipped; a push run outside any workspace (no `Cargo.toml`) uploads everything. ```sh # in your project directory (Cargo.lock must exist) kache sync --pull -# pull everything in the bucket regardless of Cargo.lock +# pull everything in the remote regardless of Cargo.lock kache sync --pull --all ``` -If kache can't find a `Cargo.lock`, it falls back to pulling everything. The same full-bucket fallback applies if `Cargo.lock` is present but yields no package names. +If kache can't find a `Cargo.lock`, it falls back to pulling everything. The same full-remote fallback applies if `Cargo.lock` is present but yields no package names. ### `--workspace`: scope the pull to workspace members -By default the pull issues one `ListObjectsV2` per crate in the `Cargo.lock` dependency set — often hundreds to thousands of LIST calls. `--workspace` instead scopes discovery to the **workspace members** (the same `cargo metadata --no-deps` set used by the push filter), issuing one LIST per member (typically tens). It conflicts with `--all`. - -This only narrows the up-front batch pull — dependency artifacts are still resolved on demand during the build (the rustc wrapper fetches a local miss from S3 via the daemon, and the daemon prefetches by build intent), so deps are not recompiled. It's most useful when the dependency artifacts are already present locally — e.g. a prebuilt `cargo chef` deps image whose compiled deps sit in `target/` — where they're local hits that never need an S3 round-trip. +By default the pull issues one remote LIST per crate in the `Cargo.lock` +dependency set — often hundreds to thousands of calls. For S3 each call is a +`ListObjectsV2`; a filesystem remote lists the corresponding manifest +directory. `--workspace` instead scopes discovery to the **workspace members** +(the same `cargo metadata --no-deps` set used by the push filter), issuing one +LIST per member (typically tens). It conflicts with `--all`. + +This only narrows the up-front batch pull — dependency artifacts are still +resolved on demand during the build (the rustc wrapper fetches a local miss +from the configured remote via the daemon, and the daemon prefetches by build +intent), so deps are not recompiled. It's most useful when the dependency +artifacts are already present locally — e.g. a prebuilt `cargo chef` deps image +whose compiled deps sit in `target/` — where they're local hits that never need +a remote round-trip. ```sh # discovery issues one LIST per workspace member instead of one per dependency kache sync --pull --workspace ``` -Unlike the default pull, `--workspace` does **not** fall back to a full-bucket scan: if the workspace set can't be resolved (`cargo metadata` failed, or you're not in a Cargo workspace), the command errors rather than silently listing the entire bucket. +Unlike the default pull, `--workspace` does **not** fall back to a full-remote scan: if the workspace set can't be resolved (`cargo metadata` failed, or you're not in a Cargo workspace), the command errors rather than silently listing the entire remote. `--manifest-path` controls the **push-side** workspace filter only (which local crates get uploaded). The pull filter always reads `Cargo.lock` from the current working directory and ignores `--manifest-path`, so to pull for a different project, run `kache sync` from that project's directory. @@ -64,13 +78,13 @@ Workspace filtering shells out to `cargo metadata`. If `cargo` is missing or met In CI, the typical pattern is: ```sh -# before the build: warm the local cache from S3 +# before the build: warm the local cache from the remote kache sync --pull # run the build (kache serves local hits, misses compile normally) cargo build --release -# after the build: push newly compiled artifacts to S3 +# after the build: push newly compiled artifacts to the remote kache sync --push ``` @@ -81,10 +95,13 @@ When the daemon is running and remote is configured, it handles uploads automati `kache sync` is safe to run alongside the daemon and alongside other concurrent syncs. Before downloading, it re-checks each entry and skips any that already landed on disk; imports use `INSERT OR REPLACE`; and pushes skip entries that vanished (via GC or purge) mid-run. -`--push` still performs a full, unfiltered LIST of all S3 keys to determine what is already uploaded — workspace filtering applies to the upload set, not to this listing. On a large shared bucket that LIST is a non-trivial cost. +`--push` still performs a full, unfiltered LIST of all remote keys to determine +what is already uploaded — workspace filtering applies to the upload set, not +to this listing. On a large S3 bucket that LIST is a non-trivial request cost; +on a filesystem remote it scans the corresponding manifest tree. -## S3 layout +## Remote layout Each cached entry is stored as two objects under a versioned `v3` layout — a small manifest used for listing and existence checks, and the packed artifact itself: @@ -93,7 +110,11 @@ Each cached entry is stored as two objects under a versioned `v3` layout — a s {prefix}/v3/packs/{crate_name}/{cache_key}.tar.zst # packed artifacts ``` -The default prefix is `artifacts`, so `serde` packs live under `artifacts/v3/packs/serde/...`. Organizing by crate name keeps filtered listing efficient: a workspace-filtered pull scans only the per-crate manifest prefix `{prefix}/v3/manifests/{crate_name}/`. +The default prefix is `artifacts`, so `serde` packs live under +`artifacts/v3/packs/serde/...`. On S3 these are object keys; on a filesystem +remote they are files below `{path}`. Organizing by crate name keeps filtered +listing efficient: a workspace-filtered pull scans only the per-crate manifest +prefix `{prefix}/v3/manifests/{crate_name}/`. ## Dry run @@ -116,7 +137,9 @@ Sizes are not shown — listing does not fetch object sizes. ## Concurrency -`kache sync` uses up to `s3_concurrency` (default 16) concurrent S3 operations. You can lower this if your S3 provider throttles requests: +`kache sync` uses up to `s3_concurrency` (a legacy setting name, default 16) +concurrent remote operations. You can lower it if your S3 provider throttles +requests or your shared filesystem performs better with less parallelism: ```sh KACHE_S3_CONCURRENCY=4 kache sync --push diff --git a/docs/remote-service.mdx b/docs/remote-service.mdx index 2469098c..73fc465e 100644 --- a/docs/remote-service.mdx +++ b/docs/remote-service.mdx @@ -9,15 +9,15 @@ description: Server-side kache — a remote planner that prefetches artifacts fr **SOON.** Server-side kache is the next milestone. The deployment model, auth integration, and HA behavior are still hardening — treat the planner service and Helm chart as a preview today. -The remote planner service warms the *right* artifacts before rustc asks for them. Where local kache reacts to cargo invoking rustc, the planner anticipates: it draws on build manifests (uploaded by `kache save-manifest`), dependency history, and the client's build intent, then advises clients which artifacts to prefetch from S3 at the start of a session. +The remote planner service warms the *right* artifacts before rustc asks for them. Where local kache reacts to cargo invoking rustc, the planner anticipates: it draws on build manifests (uploaded by `kache save-manifest`), dependency history, and the client's build intent, then advises clients which artifacts to prefetch from the configured remote at the start of a session. The service lives in [`crates/kache-service`](https://github.com/kunobi-ninja/kache/tree/main/crates/kache-service). It persists planner state in an embedded SurrealDB database and serves three HTTP routes: `POST /v1/prefetch-plan` and `POST /v2/prefetch-plan` (both aliased to the same handler — clients default to `/v2`), plus `GET /healthz` and `GET /readyz` probes. It safely returns a `use_fallback` disposition when the database has no matching candidates — so clients always have a working code path even before the planner has data for them. ## What it does -- **Manifest upload.** Clients call `kache save-manifest` at the end of a build. This uploads a build manifest (and, with `--namespace`, sharded indexes keyed by `Cargo.lock`) to the configured S3 remote — each entry records cache key, crate name, compile time, and artifact size. The planner service consumes that data out of band (it is seeded from a snapshot today; see `KACHE_PLANNER_SEED_STATE_FILE`) — `save-manifest` does not POST to the service. +- **Manifest upload.** Clients call `kache save-manifest` at the end of a build. This uploads a build manifest (and, with `--namespace`, sharded indexes keyed by `Cargo.lock`) to the configured remote — each entry records cache key, crate name, compile time, and artifact size. The planner service consumes that data out of band (it is seeded from a snapshot today; see `KACHE_PLANNER_SEED_STATE_FILE`) — `save-manifest` does not POST to the service. - **Prefetch hints.** At the start of a new build session, the client `POST`s a build intent (crate names, namespace, `Cargo.lock` deps) to the planner. The planner replies with a plan: a `disposition` (`Execute` or `UseFallback`) and, when executing, ranked `candidates` (cache key + crate name) likely to be needed. -- **Use-fallback safety.** If the planner has no service-side state, resolves no candidates, or hits an internal planning error, it returns a `UseFallback` plan rather than an HTTP error. Clients then fall back to filtering S3 prefetch by `Cargo.lock` (the same path the daemon uses today without a planner). +- **Use-fallback safety.** If the planner has no service-side state, resolves no candidates, or hits an internal planning error, it returns a `UseFallback` plan rather than an HTTP error. Clients then fall back to filtering remote prefetch by `Cargo.lock` (the same path the daemon uses today without a planner). ## Build and run @@ -90,4 +90,4 @@ When combining HA with PVC-backed planner state, use storage that can be mounted ## What you get without the service -Local caching, S3 sync, and `cargo metadata`-driven prefetch all work without the planner. The planner is purely additive: it raises the hit rate on the *first* build of a new branch or runner where `cargo metadata` alone would underprefetch. If you don't run the service, clients silently behave as they do today. +Local caching, remote sync, and `cargo metadata`-driven prefetch all work without the planner. The planner is purely additive: it raises the hit rate on the *first* build of a new branch or runner where `cargo metadata` alone would underprefetch. If you don't run the service, clients silently behave as they do today. diff --git a/src/cli.rs b/src/cli.rs index e705ee7d..b9374cdb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -407,12 +407,7 @@ pub(crate) fn render_stats( // Remote if let Some(ref remote) = config.remote { - let prefix = if remote.prefix.is_empty() { - String::new() - } else { - format!("/{}", remote.prefix) - }; - lines.push(format!("Remote: s3://{}{prefix}", remote.bucket)); + lines.push(format!("Remote: {}", remote.describe())); } else if config.local_only { lines.push("Remote: local-only mode (remote + planner ignored)".to_string()); } else { @@ -1921,7 +1916,7 @@ fn active_sccache_migration_line(line: &str) -> bool { /// Check environment for sccache and configuration issues. /// When `fix` is true, also run the sccache→kache migration after diagnostics. -/// The daemon is only needed when remote work happens: async S3 uploads, remote +/// The daemon is only needed when remote work happens: async uploads, remote /// checks, or planner prefetch. When neither a remote cache nor a planner is /// configured (including strict local-only mode, which suppresses both), the /// daemon is optional and `kache doctor` should not flag its absence as a problem. @@ -1946,7 +1941,7 @@ pub fn doctor( let config = crate::config::Config::load().ok(); let sccache_is_fallback = fallback_is_sccache(config.as_ref()); - // The daemon only matters when remote work is configured (S3 remote or a + // The daemon only matters when remote work is configured (cache remote or a // planner endpoint). When neither is set — including strict local-only mode, // which suppresses both — the daemon is optional (see README), so its checks // are shown for diagnostics but never counted as issues. See #443. @@ -2128,7 +2123,7 @@ pub fn doctor( checks.push(Check { label: "Remote", pass: true, - detail: format!("s3://{}", remote.bucket), + detail: remote.describe(), fix: None, }); } else if let Some(ref cfg) = config @@ -2631,10 +2626,10 @@ fn migrate(purge_sccache: bool) -> Result<()> { Ok(()) } -/// Synchronize local cache with S3 remote: pull missing artifacts, push new ones. +/// Synchronize the local cache with its remote: pull missing artifacts, push new ones. /// -/// Works directly against S3 (no daemon required). Safe to run alongside the daemon — -/// downloads use atomic extraction, imports use INSERT OR REPLACE, and S3 PUTs are idempotent. +/// Works directly against the remote (no daemon required). Safe to run alongside the daemon — +/// downloads use atomic extraction, imports use INSERT OR REPLACE, and uploads are idempotent. pub fn sync( config: &Config, manifest_path: Option<&str>, @@ -2644,10 +2639,9 @@ pub fn sync( pull_all: bool, pull_workspace: bool, ) -> Result<()> { - let remote = config - .remote - .as_ref() - .ok_or_else(|| anyhow::anyhow!("No remote configured. Run `kache config` to set up S3."))?; + let remote = config.remote.as_ref().ok_or_else(|| { + anyhow::anyhow!("No remote configured. Run `kache config` to set one up.") + })?; let store = Store::open(config)?; let workspace_crates = workspace_filter(manifest_path); @@ -2695,7 +2689,7 @@ async fn sync_inner( ) -> Result<()> { let backend = crate::remote_backend::create_backend(remote, config.s3_pool_idle_secs) .await - .context("connecting to the remote — check credentials and endpoint")?; + .context("connecting to the remote — check its configuration and access")?; sync_with_client( backend.as_ref(), config, @@ -2731,7 +2725,7 @@ async fn sync_with_client( ) -> Result<()> { let planner = crate::remote_plan::RemotePlanner::new(config); - // For pull: scope the S3 key listing to crate prefixes when possible (one + // For pull: scope the remote key listing to crate prefixes when possible (one // LIST per crate). `--workspace` narrows that to workspace members only; // otherwise it's the Cargo.lock dep set; `--all` (or no filter) lists the // whole bucket. @@ -2739,65 +2733,68 @@ async fn sync_with_client( if pull_workspace { // `--workspace` must resolve to a non-empty workspace set. If cargo // metadata failed or this isn't a Cargo workspace, refuse to fall - // back to a full-bucket scan — that's the exact opposite of what the + // back to a full-remote scan — that's the exact opposite of what the // flag asks for (and `lock_crates` is None here, so the dep path // can't catch it either). let crates = workspace_crates.filter(|c| !c.is_empty()).ok_or_else(|| { anyhow::anyhow!( "--workspace: no workspace members resolved (cargo metadata \ failed or this is not a Cargo workspace); refusing to fall \ - back to a full-bucket S3 scan" + back to a full remote scan" ) })?; - eprint!("Listing S3 keys for {} workspace crates...", crates.len()); + eprint!( + "Listing remote keys for {} workspace crates...", + crates.len() + ); let keys = planner .plan(crate::remote_plan::RemoteWorkload::KeyDiscovery) .layout(backend, remote) .list_keys_for_crates(crates) .await - .context("listing S3 keys for workspace crates")?; + .context("listing remote keys for workspace crates")?; eprintln!(" {} keys", keys.len()); keys } else if !pull_all && let Some(crates) = lock_crates && !crates.is_empty() { - eprint!("Listing S3 keys for {} crates...", crates.len()); + eprint!("Listing remote keys for {} crates...", crates.len()); let keys = planner .plan(crate::remote_plan::RemoteWorkload::KeyDiscovery) .layout(backend, remote) .list_keys_for_crates(crates) .await - .context("listing S3 keys for dependency crates")?; + .context("listing remote keys for dependency crates")?; eprintln!(" {} keys", keys.len()); keys } else { - eprint!("Listing S3 keys..."); + eprint!("Listing remote keys..."); let keys = planner .plan(crate::remote_plan::RemoteWorkload::KeyDiscovery) .layout(backend, remote) .list_keys() .await - .context("listing S3 keys")?; + .context("listing remote keys")?; eprintln!(" {} keys", keys.len()); keys } } else { - // push-only mode: still need to list S3 keys to know what's already uploaded - eprint!("Listing S3 keys..."); + // Push-only mode still lists remote keys to find what's already uploaded. + eprint!("Listing remote keys..."); let keys = planner .plan(crate::remote_plan::RemoteWorkload::KeyDiscovery) .layout(backend, remote) .list_keys() .await - .context("listing S3 keys")?; + .context("listing remote keys")?; eprintln!(" {} keys", keys.len()); keys }; let local_entries = store.list_entries("name")?; - // to_pull: S3 keys not present on disk locally — (cache_key, crate_name). + // to_pull: remote keys not present on disk locally — (cache_key, crate_name). let to_pull: Vec<(String, String)> = if !push_only { s3_keys .iter() @@ -2811,7 +2808,7 @@ async fn sync_with_client( Vec::new() }; - // to_push: local entries on disk but not in S3, filtered by workspace. + // to_push: local entries on disk but not in the remote, filtered by workspace. // Includes (cache_key, crate_name) for crate-prefixed uploads. let to_push: Vec<(String, String)> = if !pull_only && !config.remote_readonly { local_entries @@ -4289,31 +4286,94 @@ mod tests { assert!(filter.is_empty()); } - // ── upload_shards against a mock S3 ────────────────────────────────────── - use aws_smithy_http_client::test_util::wire::{ReplayedEvent, WireMockServer}; + // ── backend-neutral remote tests ────────────────────────────────────────── - async fn mock_s3( - events: Vec, - ) -> ( - WireMockServer, - Arc, - ) { - let server = WireMockServer::start(events).await; - let conf = aws_sdk_s3::config::Builder::new() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .region(aws_sdk_s3::config::Region::new("us-east-1")) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - "AK", "SK", None, None, "test", - )) - .endpoint_url(server.endpoint_url()) - .http_client(server.http_client()) - .force_path_style(true) - .build(); - let backend = crate::remote_backend::S3Backend::new( - aws_sdk_s3::Client::from_conf(conf), - "bucket".to_string(), - ); - (server, Arc::new(backend)) + #[derive(Default)] + struct BackendCalls { + gets: Vec, + puts: Vec, + lists: Vec, + } + + struct TestBackend { + inner: crate::remote_backend::OpenDalBackend, + calls: std::sync::Mutex, + fail_put: bool, + } + + impl TestBackend { + fn memory() -> Arc { + Arc::new(Self { + inner: crate::remote_backend::memory_backend(), + calls: std::sync::Mutex::new(BackendCalls::default()), + fail_put: false, + }) + } + + fn failing_put() -> Arc { + Arc::new(Self { + inner: crate::remote_backend::memory_backend(), + calls: std::sync::Mutex::new(BackendCalls::default()), + fail_put: true, + }) + } + + async fn seed(&self, key: &str, body: impl Into>) { + crate::remote_backend::RemoteBackend::put(&self.inner, key, body.into(), None) + .await + .expect("seed remote object"); + } + + fn get_calls(&self) -> Vec { + self.calls.lock().unwrap().gets.clone() + } + + fn put_calls(&self) -> Vec { + self.calls.lock().unwrap().puts.clone() + } + + fn list_calls(&self) -> Vec { + self.calls.lock().unwrap().lists.clone() + } + } + + fn as_remote_backend( + backend: &Arc, + ) -> Arc { + backend.clone() + } + + #[async_trait::async_trait] + impl crate::remote_backend::RemoteBackend for TestBackend { + async fn head(&self, key: &str) -> Result { + crate::remote_backend::RemoteBackend::head(&self.inner, key).await + } + + async fn get( + &self, + key: &str, + max_bytes: Option, + ) -> Result> { + self.calls.lock().unwrap().gets.push(key.to_string()); + crate::remote_backend::RemoteBackend::get(&self.inner, key, max_bytes).await + } + + async fn put(&self, key: &str, body: Vec, content_type: Option<&str>) -> Result<()> { + self.calls.lock().unwrap().puts.push(key.to_string()); + if self.fail_put { + anyhow::bail!("injected PUT failure for {key}"); + } + crate::remote_backend::RemoteBackend::put(&self.inner, key, body, content_type).await + } + + async fn list(&self, prefix: &str) -> Result> { + self.calls.lock().unwrap().lists.push(prefix.to_string()); + crate::remote_backend::RemoteBackend::list(&self.inner, prefix).await + } + + fn describe(&self, key: &str) -> String { + crate::remote_backend::RemoteBackend::describe(&self.inner, key) + } } #[tokio::test] @@ -4350,17 +4410,19 @@ mod tests { let shard_set = crate::shards::compute_shards("ns", &deps); let expected = shard_set.shards.len(); - // One OK per upload (over-provision is fine; each request consumes one). - let events = std::iter::repeat_with(ReplayedEvent::ok) - .take(expected + 2) - .collect(); - let (server, client) = mock_s3(events).await; + let backend = TestBackend::memory(); + let client = as_remote_backend(&backend); let uploaded = upload_shards(&client, "prefix", "ns", &lock, &entries) .await .expect("upload_shards should succeed"); assert_eq!(uploaded, expected); - server.shutdown(); + let puts = backend.put_calls(); + assert_eq!(puts.len(), expected); + assert!( + puts.iter() + .all(|key| key.starts_with("prefix/_manifests/v3/ns/shards/")) + ); } #[tokio::test] @@ -4375,12 +4437,13 @@ mod tests { ) .unwrap(); - let (server, client) = mock_s3(vec![]).await; + let backend = TestBackend::memory(); + let client = as_remote_backend(&backend); let uploaded = upload_shards(&client, "prefix", "ns", &lock, &[]) .await .expect("should succeed with nothing to upload"); assert_eq!(uploaded, 0); - server.shutdown(); + assert!(backend.put_calls().is_empty()); } #[tokio::test] @@ -4389,20 +4452,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let lock = dir.path().join("Cargo.lock"); std::fs::write(&lock, "not valid toml [[[[").unwrap(); - let conf = aws_sdk_s3::config::Builder::new() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .region(aws_sdk_s3::config::Region::new("us-east-1")) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - "AK", "SK", None, None, "test", - )) - .endpoint_url("http://127.0.0.1:1") - .force_path_style(true) - .build(); - let client: Arc = - Arc::new(crate::remote_backend::S3Backend::new( - aws_sdk_s3::Client::from_conf(conf), - "bucket".to_string(), - )); + let backend = TestBackend::memory(); + let client = as_remote_backend(&backend); let err = upload_shards(&client, "prefix", "ns", &lock, &[]) .await @@ -4411,6 +4462,7 @@ mod tests { err.to_string().contains("TOML") || err.to_string().contains("parse"), "got {err}" ); + assert!(backend.put_calls().is_empty()); } fn save_manifest_config( @@ -4998,18 +5050,12 @@ mod tests { } #[test] - fn save_manifest_with_no_events_returns_ok_before_touching_s3() { + fn save_manifest_with_no_events_returns_ok_before_touching_remote() { // A remote is configured, but the event log is empty, so save_manifest - // returns Ok early ("No build events found") without creating an S3 + // returns Ok early ("No build events found") without creating a remote // client or making any network call. let dir = tempfile::tempdir().unwrap(); - let remote = crate::config::RemoteConfig { - bucket: "b".to_string(), - endpoint: Some("http://127.0.0.1:1".to_string()), - region: "us-east-1".to_string(), - prefix: "p".to_string(), - profile: None, - }; + let remote = crate::config::RemoteConfig::test_s3("b", "p"); let config = save_manifest_config(dir.path().to_path_buf(), Some(remote)); // No event log written -> read_events yields empty -> early Ok. save_manifest(&config, Some("mykey"), None).expect("empty events -> Ok"); @@ -5145,19 +5191,14 @@ mod tests { } fn test_remote_cfg() -> crate::config::RemoteConfig { - crate::config::RemoteConfig { - bucket: "bucket".to_string(), - endpoint: None, - region: "us-east-1".to_string(), - prefix: "prefix".to_string(), - profile: None, - } + crate::config::RemoteConfig::test_s3("bucket", "prefix") } #[tokio::test] async fn upload_manifest_and_shards_uploads_manifest_only_without_namespace() { - // No namespace -> exactly one PutObject (the monolithic manifest). - let (server, client) = mock_s3(vec![ReplayedEvent::ok()]).await; + // No namespace -> exactly one object (the monolithic manifest). + let backend = TestBackend::memory(); + let client = as_remote_backend(&backend); let remote = test_remote_cfg(); let entries = vec![crate::remote::ManifestEntry { cache_key: "k".to_string(), @@ -5175,14 +5216,14 @@ mod tests { ) .await .expect("manifest-only upload should succeed"); - assert_eq!(server.events().len(), 3, "one request round-trip"); - server.shutdown(); + assert_eq!(backend.put_calls(), vec!["prefix/_manifests/mykey.json"]); } #[tokio::test] async fn upload_manifest_and_shards_skips_shards_when_lock_missing() { - // Namespace given but Cargo.lock absent -> still only the manifest PUT. - let (server, client) = mock_s3(vec![ReplayedEvent::ok()]).await; + // Namespace given but Cargo.lock absent -> still only the manifest. + let backend = TestBackend::memory(); + let client = as_remote_backend(&backend); let remote = test_remote_cfg(); let entries = vec![crate::remote::ManifestEntry { cache_key: "k".to_string(), @@ -5200,8 +5241,7 @@ mod tests { ) .await .expect("upload should succeed, shards skipped"); - assert_eq!(server.events().len(), 3, "manifest only; no shard PUTs"); - server.shutdown(); + assert_eq!(backend.put_calls(), vec!["prefix/_manifests/mykey.json"]); } #[tokio::test] @@ -5224,52 +5264,36 @@ mod tests { let deps = crate::shards::parse_cargo_lock(&lock).unwrap(); let expected_shards = crate::shards::compute_shards("ns", &deps).shards.len(); - // manifest PUT + one PUT per shard (over-provision OK). - let events = std::iter::repeat_with(ReplayedEvent::ok) - .take(expected_shards + 2) - .collect(); - let (server, client) = mock_s3(events).await; + let backend = TestBackend::memory(); + let client = as_remote_backend(&backend); let remote = test_remote_cfg(); upload_manifest_and_shards(&client, &remote, "mykey", Some("ns"), &lock, entries) .await .expect("upload with shards should succeed"); - server.shutdown(); - } - - /// A `ListObjectsV2` response listing the given manifest object keys. - fn list_bucket_xml(keys: &[&str]) -> String { - let contents: String = keys - .iter() - .map(|k| { - format!( - "{k}2025-01-01T00:00:00.000Z\ - \"x\"10STANDARD" - ) - }) - .collect(); - format!( - "\ - \ - bucketprefix/v3/manifests/\ - {}1000false\ - {contents}", - keys.len() - ) + let puts = backend.put_calls(); + assert_eq!(puts.len(), expected_shards + 1); + assert!(puts.contains(&"prefix/_manifests/mykey.json".to_string())); + assert_eq!( + puts.iter() + .filter(|key| key.starts_with("prefix/_manifests/v3/ns/shards/")) + .count(), + expected_shards + ); } #[tokio::test] async fn sync_with_client_dry_run_empty_remote_reports_nothing() { - // Empty remote + empty local store: the list returns no keys, so the - // diff is empty and sync reports "Nothing to sync" (one list call). + // Empty remote + empty local store: the diff is empty and sync reports + // "Nothing to sync" after one list call. let dir = tempfile::tempdir().unwrap(); let config = save_manifest_config(dir.path().to_path_buf(), Some(test_remote_cfg())); let store = Store::open(&config).unwrap(); let remote = test_remote_cfg(); - let (server, client) = mock_s3(vec![ReplayedEvent::with_body(list_bucket_xml(&[]))]).await; + let backend = TestBackend::memory(); sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5283,16 +5307,13 @@ mod tests { ) .await .expect("dry-run sync over empty remote should succeed"); - server.shutdown(); + assert_eq!(backend.list_calls(), vec!["prefix/v3/manifests/"]); } #[tokio::test] async fn sync_with_client_workspace_pull_scopes_listing_to_workspace_members() { - // `--workspace` must scope the pull listing to workspace members (one - // LIST per member) and ignore the Cargo.lock dep set. workspace = {wsfoo} - // → 1 LIST; lock = {dep_a, dep_b} → would be 2 LISTs. We provide exactly - // ONE list response, so the call only succeeds if it took the workspace - // path — the lock path would exhaust the mock on its second LIST. + // `--workspace` must scope the pull listing to workspace members and + // ignore the Cargo.lock dep set. let dir = tempfile::tempdir().unwrap(); let config = save_manifest_config(dir.path().to_path_buf(), Some(test_remote_cfg())); let store = Store::open(&config).unwrap(); @@ -5304,10 +5325,9 @@ mod tests { .into_iter() .collect(); - // One LIST response (workspace path = exactly one LIST, for `wsfoo`). - let (server, client) = mock_s3(vec![ReplayedEvent::with_body(list_bucket_xml(&[]))]).await; + let backend = TestBackend::memory(); sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5321,16 +5341,13 @@ mod tests { ) .await .expect("workspace-scoped pull should list only the workspace member(s)"); - server.shutdown(); + assert_eq!(backend.list_calls(), vec!["prefix/v3/manifests/wsfoo/"]); } #[tokio::test] async fn sync_with_client_workspace_pull_errors_when_no_workspace_resolved() { // `--workspace` with an unresolved (None) or empty workspace set must - // error, NOT silently fall back to a full-bucket scan. We hand the mock - // a valid full-bucket LIST response: if the guard were missing, the - // fallback `list_keys()` would consume it and return Ok — so an Err here - // proves the guard fired before any S3 round-trip. + // error, NOT silently fall back to a full-remote scan. let dir = tempfile::tempdir().unwrap(); let config = save_manifest_config(dir.path().to_path_buf(), Some(test_remote_cfg())); let store = Store::open(&config).unwrap(); @@ -5340,10 +5357,9 @@ mod tests { let cases: [Option<&std::collections::HashSet>; 2] = [None, Some(&empty)]; for workspace_crates in cases { - let (server, client) = - mock_s3(vec![ReplayedEvent::with_body(list_bucket_xml(&[]))]).await; + let backend = TestBackend::memory(); let err = sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5361,15 +5377,18 @@ mod tests { err.to_string().contains("no workspace members resolved"), "unexpected error: {err}" ); - server.shutdown(); + assert!( + backend.list_calls().is_empty(), + "guard must run before listing the remote" + ); } } #[tokio::test] async fn sync_with_client_push_uploads_local_only_entry() { // A populated local store + an empty remote: push-only sync uploads the - // local entry end-to-end (real pack creation + manifest PUT) through the - // mock. Exercises the push loop and upload_entry, not just planning. + // local entry end-to-end (real pack creation + manifest) through the + // backend. Exercises the push loop and upload_entry, not just planning. let dir = tempfile::tempdir().unwrap(); let config = save_manifest_config(dir.path().to_path_buf(), Some(test_remote_cfg())); let store = Store::open(&config).unwrap(); @@ -5394,13 +5413,10 @@ mod tests { .unwrap(); let remote = test_remote_cfg(); - // list (empty remote) + PUTs for the pack and manifest (over-provisioned). - let mut events = vec![ReplayedEvent::with_body(list_bucket_xml(&[]))]; - events.extend(std::iter::repeat_with(ReplayedEvent::ok).take(4)); - let (server, client) = mock_s3(events).await; + let backend = TestBackend::memory(); sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5414,12 +5430,15 @@ mod tests { ) .await .expect("push sync should succeed"); - server.shutdown(); + let puts = backend.put_calls(); + assert_eq!(puts.len(), 2); + assert!(puts.contains(&"prefix/v3/packs/foo/pushkey123.tar.zst".to_string())); + assert!(puts.contains(&"prefix/v3/manifests/foo/pushkey123.json".to_string())); } #[tokio::test] async fn sync_with_client_push_throttles_with_low_concurrency() { - // Two local entries with s3_concurrency=1 force the push loop's + // Two local entries with concurrency=1 force the push loop's // max-concurrency wait branch (the second upload waits for the first). let dir = tempfile::tempdir().unwrap(); let mut config = save_manifest_config(dir.path().to_path_buf(), Some(test_remote_cfg())); @@ -5447,13 +5466,10 @@ mod tests { } let remote = test_remote_cfg(); - // list (empty) + PUTs for two entries' pack+manifest (over-provisioned). - let mut events = vec![ReplayedEvent::with_body(list_bucket_xml(&[]))]; - events.extend(std::iter::repeat_with(ReplayedEvent::ok).take(8)); - let (server, client) = mock_s3(events).await; + let backend = TestBackend::memory(); sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5467,7 +5483,7 @@ mod tests { ) .await .expect("throttled push sync should succeed"); - server.shutdown(); + assert_eq!(backend.put_calls().len(), 4); } #[tokio::test] @@ -5479,10 +5495,15 @@ mod tests { let store = Store::open(&config).unwrap(); let remote = test_remote_cfg(); - let xml = list_bucket_xml(&["prefix/v3/manifests/serde/abc123def456.json"]); - let (server, client) = mock_s3(vec![ReplayedEvent::with_body(xml)]).await; + let backend = TestBackend::memory(); + backend + .seed( + "prefix/v3/manifests/serde/abc123def456.json", + b"{}".to_vec(), + ) + .await; sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5496,7 +5517,10 @@ mod tests { ) .await .expect("dry-run sync planning a pull should succeed"); - server.shutdown(); + assert!( + backend.get_calls().is_empty(), + "dry-run must not download the pack" + ); } #[tokio::test] @@ -5510,16 +5534,22 @@ mod tests { let store = Store::open(&config).unwrap(); let remote = test_remote_cfg(); - let xml = list_bucket_xml(&["prefix/v3/manifests/serde/abc123def456.json"]); - let mut events = vec![ReplayedEvent::with_body(xml)]; - // The pack GET returns non-zstd bytes -> download_entry fails. - events.extend( - std::iter::repeat_with(|| ReplayedEvent::with_body(b"not a valid pack")).take(4), - ); - let (server, client) = mock_s3(events).await; + let backend = TestBackend::memory(); + backend + .seed( + "prefix/v3/manifests/serde/abc123def456.json", + b"{}".to_vec(), + ) + .await; + backend + .seed( + "prefix/v3/packs/serde/abc123def456.tar.zst", + b"not a valid pack".to_vec(), + ) + .await; sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5533,16 +5563,17 @@ mod tests { ) .await .expect("pull sync should complete Ok even when a download fails"); - server.shutdown(); + assert_eq!( + backend.get_calls(), + vec!["prefix/v3/packs/serde/abc123def456.tar.zst"] + ); } #[tokio::test] async fn sync_with_client_push_reports_failed_uploads() { - // A local-only entry is scheduled for push, but the upload PUTs get a - // non-retryable 403, so upload_entry errors and the loop records a - // failure. Covers the push error arm + the "fail_count > 0" upload - // summary branch (cli.rs ~2651 + 2673-2682). Per-item errors don't - // abort the sync, so it still returns Ok. + // A local-only entry is scheduled for push, but the backend rejects + // uploads, so upload_entry errors and the loop records a failure. + // Per-item errors do not abort the sync, so it still returns Ok. let dir = tempfile::tempdir().unwrap(); let config = save_manifest_config(dir.path().to_path_buf(), Some(test_remote_cfg())); let store = Store::open(&config).unwrap(); @@ -5566,13 +5597,10 @@ mod tests { ) .unwrap(); - // list (empty) then 403s for every upload PUT (over-provisioned). - let mut events = vec![ReplayedEvent::with_body(list_bucket_xml(&[]))]; - events.extend(std::iter::repeat_with(|| ReplayedEvent::status(403)).take(6)); - let (server, client) = mock_s3(events).await; + let backend = TestBackend::failing_put(); sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5586,12 +5614,15 @@ mod tests { ) .await .expect("push sync should complete Ok even when an upload fails"); - server.shutdown(); + assert_eq!( + backend.put_calls(), + vec!["prefix/v3/packs/foo/pushfail1aaaa.tar.zst"] + ); } #[tokio::test] async fn sync_with_client_pull_throttles_with_low_concurrency() { - // Two remote-only keys with s3_concurrency=1 force the pull loop's + // Two remote-only keys with concurrency=1 force the pull loop's // max-concurrency wait branch (the second download waits for the first // to drain a slot). Packs are garbage so each download fails fast, but // the throttle path is still exercised; the sync completes Ok. @@ -5601,17 +5632,24 @@ mod tests { let store = Store::open(&config).unwrap(); let remote = test_remote_cfg(); - let xml = list_bucket_xml(&[ - "prefix/v3/manifests/aaa/key1111111111aa.json", - "prefix/v3/manifests/bbb/key2222222222bb.json", - ]); - let mut events = vec![ReplayedEvent::with_body(xml)]; - // Each download attempt makes several GETs; over-provision garbage bodies. - events.extend(std::iter::repeat_with(|| ReplayedEvent::with_body(b"not a pack")).take(8)); - let (server, client) = mock_s3(events).await; + let backend = TestBackend::memory(); + for (crate_name, key) in [("aaa", "key1111111111aa"), ("bbb", "key2222222222bb")] { + backend + .seed( + &format!("prefix/v3/manifests/{crate_name}/{key}.json"), + b"{}".to_vec(), + ) + .await; + backend + .seed( + &format!("prefix/v3/packs/{crate_name}/{key}.tar.zst"), + b"not a pack".to_vec(), + ) + .await; + } sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5625,7 +5663,7 @@ mod tests { ) .await .expect("throttled pull sync should complete Ok"); - server.shutdown(); + assert_eq!(backend.get_calls().len(), 2); } /// Build a valid v3 entry pack (tar.zst) for `key`/`crate_name` from a @@ -5673,15 +5711,19 @@ mod tests { let key = "abc123def456aaaa"; let pack = build_entry_pack(key, "serde"); - let xml = list_bucket_xml(&["prefix/v3/manifests/serde/abc123def456aaaa.json"]); - let events = vec![ - ReplayedEvent::with_body(xml), - ReplayedEvent::with_body(&pack), - ]; - let (server, client) = mock_s3(events).await; + let backend = TestBackend::memory(); + backend + .seed( + "prefix/v3/manifests/serde/abc123def456aaaa.json", + b"{}".to_vec(), + ) + .await; + backend + .seed("prefix/v3/packs/serde/abc123def456aaaa.tar.zst", pack) + .await; sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5701,7 +5743,10 @@ mod tests { config.store_dir().join(key).join("meta.json").exists(), "pulled entry should be materialized in the local store" ); - server.shutdown(); + assert_eq!( + backend.get_calls(), + vec!["prefix/v3/packs/serde/abc123def456aaaa.tar.zst"] + ); } #[test] @@ -5927,13 +5972,12 @@ mod tests { .unwrap(); let remote = test_remote_cfg(); - // Since remote_readonly is true, the plan lists the remote keys but will NOT push. - // We only mock the list_keys call. If any PUT is attempted, the mock S3 would fail (since we only supply 1 event for List). - let xml = list_bucket_xml(&[]); - let (server, client) = mock_s3(vec![ReplayedEvent::with_body(xml)]).await; + // Since remote_readonly is true, the plan lists the remote keys but + // does not push. + let backend = TestBackend::memory(); sync_with_client( - client.as_ref(), + backend.as_ref(), &config, &store, &remote, @@ -5947,7 +5991,7 @@ mod tests { ) .await .expect("push sync should succeed (by skipping pushes)"); - server.shutdown(); + assert!(backend.put_calls().is_empty()); } #[tokio::test] @@ -5972,7 +6016,8 @@ mod tests { use std::io::Write; writeln!(file, "{event}").unwrap(); - // Calling save_manifest should return Ok immediately without triggering any S3 client creation or calls. + // Calling save_manifest should return Ok immediately without creating + // a remote client or making any calls. save_manifest(&config, Some("mykey"), None) .expect("save_manifest should succeed by doing nothing"); } diff --git a/src/config.rs b/src/config.rs index bb76d4ac..d099c694 100644 --- a/src/config.rs +++ b/src/config.rs @@ -180,17 +180,81 @@ pub struct PlannerConfig { pub token: Option, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct RemoteConfig { + /// Key prefix for all remote artifacts (default: "artifacts"). + pub prefix: String, + pub backend: RemoteBackendConfig, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RemoteBackendConfig { + S3(S3RemoteConfig), + Filesystem(FilesystemRemoteConfig), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct S3RemoteConfig { pub bucket: String, pub endpoint: Option, pub region: String, - /// S3 key prefix for all artifacts (default: "artifacts"). - pub prefix: String, /// AWS profile name for credential lookup (e.g. "ceph"). pub profile: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FilesystemRemoteConfig { + pub root: PathBuf, + /// Staging directory used for atomic write-then-rename completion. + pub atomic_write_dir: PathBuf, +} + +impl RemoteConfig { + /// Human-readable root used by status, doctor, and logs. + pub fn describe(&self) -> String { + let base = match &self.backend { + RemoteBackendConfig::S3(s3) => format!("s3://{}", s3.bucket), + RemoteBackendConfig::Filesystem(fs) => format!("file://{}", fs.root.display()), + }; + if self.prefix.is_empty() { + base + } else { + format!("{base}/{}", self.prefix) + } + } + + #[cfg(test)] + pub(crate) fn test_s3(bucket: &str, prefix: &str) -> Self { + Self { + prefix: prefix.to_string(), + backend: RemoteBackendConfig::S3(S3RemoteConfig { + bucket: bucket.to_string(), + endpoint: None, + region: "us-east-1".to_string(), + profile: None, + }), + } + } +} + +pub(crate) fn validate_remote_prefix(prefix: &str) -> Result<()> { + let canonical = !prefix.is_empty() + && prefix == prefix.trim() + && !prefix.starts_with('/') + && !prefix.ends_with('/') + && !prefix.contains('\\') + && prefix + .split('/') + .all(|segment| !segment.is_empty() && segment != "." && segment != ".."); + if !canonical { + anyhow::bail!( + "remote prefix must be a canonical relative path without surrounding whitespace, \ + leading/trailing slashes, backslashes, empty segments, '.' or '..': {prefix:?}" + ); + } + Ok(()) +} + #[derive(Debug, Deserialize, Serialize, Default, Clone)] pub(crate) struct FileConfig { pub(crate) cache: Option, @@ -270,6 +334,10 @@ pub(crate) struct RemoteFileConfig { pub(crate) prefix: Option, #[serde(skip_serializing_if = "Option::is_none")] pub(crate) profile: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) atomic_write_dir: Option, } #[derive(Debug, Deserialize, Serialize, Default, Clone)] @@ -726,7 +794,7 @@ impl Config { let remote = if local_only { None } else { - Self::load_remote_config(&file_config) + Self::load_remote_config(&file_config)? }; Ok(Config { @@ -800,74 +868,159 @@ impl Config { toml::from_str(&content).context("parsing kache config file") } - fn load_remote_config(file_config: &Result) -> Option { + fn load_remote_config(file_config: &Result) -> Result> { let ignore_env = Self::ignore_env_enabled(file_config); + let file_remote = file_config + .as_ref() + .ok() + .and_then(|c| c.cache.as_ref()) + .and_then(|c| c.remote.as_ref()); + + let configured_type = file_remote + .and_then(|r| r._type.as_deref()) + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(str::to_ascii_lowercase); + + let file_has_s3_fields = file_remote.is_some_and(|r| { + [&r.bucket, &r.endpoint, &r.region, &r.profile] + .into_iter() + .any(|v| v.as_deref().is_some_and(|v| !v.trim().is_empty())) + }); + let file_has_filesystem_fields = file_remote.is_some_and(|r| { + [&r.path, &r.atomic_write_dir] + .into_iter() + .any(|v| v.as_deref().is_some_and(|v| !v.trim().is_empty())) + }); + + let use_filesystem = match configured_type.as_deref() { + Some("filesystem" | "fs") => { + if file_has_s3_fields { + anyhow::bail!( + "[cache.remote] type = \"filesystem\" cannot include S3 bucket, endpoint, region, or profile" + ); + } + true + } + Some("s3") => { + if file_has_filesystem_fields { + anyhow::bail!( + "[cache.remote] type = \"s3\" cannot include path or atomic_write_dir" + ); + } + false + } + Some(other) => { + anyhow::bail!( + "unsupported [cache.remote] type {other:?}; supported types are \"s3\" and \"filesystem\"" + ); + } + None if file_has_s3_fields && file_has_filesystem_fields => { + anyhow::bail!( + "[cache.remote] mixes S3 and filesystem fields; set type = \"s3\" or type = \"filesystem\"" + ); + } + None => file_has_filesystem_fields, + }; + + if use_filesystem { + let path = file_remote + .and_then(|r| r.path.as_deref()) + .map(str::trim) + .filter(|v| !v.is_empty()) + .with_context( + || "[cache.remote] type = \"filesystem\" requires a non-empty path", + )?; + let root = shellexpand(path); + if !root.is_absolute() { + anyhow::bail!( + "[cache.remote] filesystem path must be absolute: {}", + root.display() + ); + } + + let atomic_write_dir = file_remote + .and_then(|r| r.atomic_write_dir.as_deref()) + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(shellexpand) + .unwrap_or_else(|| root.join(".kache-tmp")); + if !atomic_write_dir.is_absolute() { + anyhow::bail!( + "[cache.remote] atomic_write_dir must be absolute: {}", + atomic_write_dir.display() + ); + } + + let prefix = file_remote + .and_then(|r| r.prefix.clone()) + .unwrap_or_else(|| "artifacts".to_string()); + validate_remote_prefix(&prefix)?; + if prefix.contains(':') { + anyhow::bail!( + "[cache.remote] filesystem prefix cannot contain ':' because it can escape \ + the configured root or address an alternate data stream on Windows: {prefix:?}" + ); + } + + return Ok(Some(RemoteConfig { + prefix, + backend: RemoteBackendConfig::Filesystem(FilesystemRemoteConfig { + root, + atomic_write_dir, + }), + })); + } + let bucket = env_or_ignored("KACHE_S3_BUCKET", ignore_env) .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) .or_else(|| { - file_config - .as_ref() - .ok() - .and_then(|c| c.cache.as_ref()) - .and_then(|c| c.remote.as_ref()) - .and_then(|r| r.bucket.clone()) - })?; + file_remote + .and_then(|r| r.bucket.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + }); + + let Some(bucket) = bucket else { + if configured_type.as_deref() == Some("s3") { + anyhow::bail!("[cache.remote] type = \"s3\" requires a non-empty bucket"); + } + return Ok(None); + }; let endpoint = env_or_ignored("KACHE_S3_ENDPOINT", ignore_env) .ok() - .or_else(|| { - file_config - .as_ref() - .ok() - .and_then(|c| c.cache.as_ref()) - .and_then(|c| c.remote.as_ref()) - .and_then(|r| r.endpoint.clone()) - }); + .or_else(|| file_remote.and_then(|r| r.endpoint.clone())); let region = env_or_ignored("KACHE_S3_REGION", ignore_env) .ok() - .or_else(|| { - file_config - .as_ref() - .ok() - .and_then(|c| c.cache.as_ref()) - .and_then(|c| c.remote.as_ref()) - .and_then(|r| r.region.clone()) - }) + .or_else(|| file_remote.and_then(|r| r.region.clone())) .unwrap_or_else(|| "us-east-1".to_string()); let prefix = env_or_ignored("KACHE_S3_PREFIX", ignore_env) .ok() - .or_else(|| { - file_config - .as_ref() - .ok() - .and_then(|c| c.cache.as_ref()) - .and_then(|c| c.remote.as_ref()) - .and_then(|r| r.prefix.clone()) - }) + .or_else(|| file_remote.and_then(|r| r.prefix.clone())) .unwrap_or_else(|| "artifacts".to_string()); + validate_remote_prefix(&prefix)?; let profile = env_or_ignored("KACHE_S3_PROFILE", ignore_env) .ok() - .or_else(|| { - file_config - .as_ref() - .ok() - .and_then(|c| c.cache.as_ref()) - .and_then(|c| c.remote.as_ref()) - .and_then(|r| r.profile.clone()) - }) + .or_else(|| file_remote.and_then(|r| r.profile.clone())) .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()); - Some(RemoteConfig { - bucket, - endpoint, - region, + Ok(Some(RemoteConfig { prefix, - profile, - }) + backend: RemoteBackendConfig::S3(S3RemoteConfig { + bucket, + endpoint, + region, + profile, + }), + })) } /// Whether strict local-only mode is active (#221). Env wins over the @@ -1210,7 +1363,7 @@ pub(crate) fn config_file_path() -> PathBuf { config_base.join("kache").join("config.toml") } -fn shellexpand(s: &str) -> PathBuf { +pub(crate) fn shellexpand(s: &str) -> PathBuf { if let Some(home) = dirs::home_dir() { if s == "~" { return home; @@ -1433,6 +1586,24 @@ mod tests { assert!(dir.to_string_lossy().contains("kache")); } + #[test] + fn remote_prefix_is_backend_neutral() { + assert!(validate_remote_prefix("artifacts/team").is_ok()); + for invalid in [ + r"artifacts\team", + r"..\escape", + "/artifacts", + "artifacts/", + "artifacts//team", + "artifacts/../team", + ] { + assert!( + validate_remote_prefix(invalid).is_err(), + "{invalid:?} must be rejected" + ); + } + } + #[test] fn test_shellexpand() { let expanded = shellexpand("~/foo"); @@ -1618,6 +1789,8 @@ mod tests { region: Some("eu-west-1".to_string()), prefix: Some("my-prefix".to_string()), profile: None, + path: None, + atomic_write_dir: None, }), }), }; @@ -2338,10 +2511,9 @@ exclude = ["src/generated/**", "vendor/problem/**"] remote: Some(RemoteFileConfig { _type: Some("s3".to_string()), bucket: Some("mybucket".to_string()), - endpoint: None, region: Some("eu-west-1".to_string()), - prefix: None, profile: Some("ceph".to_string()), + ..Default::default() }), ..Default::default() }), @@ -2391,17 +2563,23 @@ exclude = ["src/generated/**", "vendor/problem/**"] region: Some("eu-west-2".to_string()), prefix: Some("myprefix".to_string()), profile: Some(" ceph ".to_string()), + ..Default::default() }), ..Default::default() }), }; - let remote = Config::load_remote_config(&Ok(file)).expect("remote from file"); - assert_eq!(remote.bucket, "filebucket"); - assert_eq!(remote.endpoint.as_deref(), Some("https://s3.example.com")); - assert_eq!(remote.region, "eu-west-2"); + let remote = Config::load_remote_config(&Ok(file)) + .expect("valid remote config") + .expect("remote from file"); assert_eq!(remote.prefix, "myprefix"); - assert_eq!(remote.profile.as_deref(), Some("ceph")); // trimmed + let RemoteBackendConfig::S3(s3) = remote.backend else { + panic!("expected S3 remote"); + }; + assert_eq!(s3.bucket, "filebucket"); + assert_eq!(s3.endpoint.as_deref(), Some("https://s3.example.com")); + assert_eq!(s3.region, "eu-west-2"); + assert_eq!(s3.profile.as_deref(), Some("ceph")); // trimmed // No bucket anywhere -> None. let empty = FileConfig { @@ -2413,7 +2591,161 @@ exclude = ["src/generated/**", "vendor/problem/**"] ..Default::default() }), }; - assert!(Config::load_remote_config(&Ok(empty)).is_none()); + assert!( + Config::load_remote_config(&Ok(empty)) + .expect("empty config is valid") + .is_none() + ); + } + + #[test] + fn filesystem_remote_loads_without_a_bucket_and_defaults_atomic_dir() { + let _guard = config_path_lock(); + let root = tempfile::tempdir().unwrap(); + let file = FileConfig { + cc: None, + paths: None, + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("filesystem".to_string()), + path: Some(root.path().to_string_lossy().into_owned()), + prefix: Some("shared".to_string()), + ..Default::default() + }), + ..Default::default() + }), + }; + + let remote = Config::load_remote_config(&Ok(file)) + .expect("valid filesystem config") + .expect("filesystem remote"); + assert_eq!(remote.prefix, "shared"); + let RemoteBackendConfig::Filesystem(fs) = remote.backend else { + panic!("expected filesystem remote"); + }; + assert_eq!(fs.root, root.path()); + assert_eq!(fs.atomic_write_dir, root.path().join(".kache-tmp")); + } + + #[test] + fn filesystem_remote_ignores_legacy_s3_environment_overrides() { + let _guard = config_path_lock(); + let _bucket = set_env_var_for_test("KACHE_S3_BUCKET", "ambient-bucket"); + let _prefix = set_env_var_for_test("KACHE_S3_PREFIX", "ambient-prefix"); + let root = tempfile::tempdir().unwrap(); + let file = FileConfig { + cc: None, + paths: None, + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("filesystem".to_string()), + path: Some(root.path().to_string_lossy().into_owned()), + prefix: Some("file-prefix".to_string()), + ..Default::default() + }), + ..Default::default() + }), + }; + + let remote = Config::load_remote_config(&Ok(file)) + .expect("valid filesystem config") + .expect("filesystem remote"); + assert_eq!(remote.prefix, "file-prefix"); + assert!(matches!( + remote.backend, + RemoteBackendConfig::Filesystem(FilesystemRemoteConfig { root: loaded, .. }) + if loaded == root.path() + )); + } + + #[test] + fn filesystem_remote_rejects_a_windows_drive_prefix() { + let root = tempfile::tempdir().unwrap(); + let file = FileConfig { + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("filesystem".to_string()), + path: Some(root.path().to_string_lossy().into_owned()), + prefix: Some("C:/escape".to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let error = Config::load_remote_config(&Ok(file)) + .expect_err("filesystem drive prefix must be rejected") + .to_string(); + assert!(error.contains("cannot contain ':'"), "{error}"); + } + + #[test] + fn legacy_remote_without_type_still_infers_s3() { + let _guard = config_path_lock(); + let file = FileConfig { + cc: None, + paths: None, + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + bucket: Some("legacy".to_string()), + ..Default::default() + }), + ..Default::default() + }), + }; + + let remote = Config::load_remote_config(&Ok(file)) + .expect("legacy config is valid") + .expect("legacy S3 remote"); + assert!(matches!( + remote.backend, + RemoteBackendConfig::S3(S3RemoteConfig { bucket, .. }) if bucket == "legacy" + )); + } + + #[test] + fn explicit_s3_rejects_an_empty_bucket() { + let _guard = config_path_lock(); + let _bucket = set_env_var_for_test("KACHE_S3_BUCKET", ""); + let file = FileConfig { + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("s3".to_string()), + bucket: Some(" ".to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let error = Config::load_remote_config(&Ok(file)) + .expect_err("empty S3 bucket must be rejected") + .to_string(); + assert!(error.contains("non-empty bucket"), "{error}"); + } + + #[test] + fn filesystem_remote_rejects_mixed_s3_fields() { + let file = FileConfig { + cc: None, + paths: None, + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("filesystem".to_string()), + path: Some("/tmp/kache-remote".to_string()), + bucket: Some("wrong-backend".to_string()), + ..Default::default() + }), + ..Default::default() + }), + }; + + let error = Config::load_remote_config(&Ok(file)) + .expect_err("mixed backend fields must be rejected") + .to_string(); + assert!(error.contains("cannot include S3"), "{error}"); } #[test] diff --git a/src/config_tui.rs b/src/config_tui.rs index fed0a298..39ec76c9 100644 --- a/src/config_tui.rs +++ b/src/config_tui.rs @@ -13,6 +13,7 @@ use std::time::Duration; use crate::config::{ CacheFileConfig, CcFileConfig, Config, EnvOverrides, FileConfig, PathsFileConfig, PlannerFileConfig, RemoteFileConfig, default_cache_dir, parse_size, resolve_config_path, + shellexpand, validate_remote_prefix, }; // ── Field definitions ───────────────────────────────────────────────────── @@ -129,6 +130,29 @@ struct EditorState { fn build_fields(file_config: &FileConfig, env: &EnvOverrides) -> Vec { let cache = file_config.cache.as_ref(); let remote = cache.and_then(|c| c.remote.as_ref()); + let remote_type = remote + .and_then(|r| r._type.clone()) + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + remote.and_then(|r| { + if r.path.is_some() || r.atomic_write_dir.is_some() { + Some("filesystem".to_string()) + } else if r.bucket.is_some() + || r.endpoint.is_some() + || r.region.is_some() + || r.prefix.is_some() + || r.profile.is_some() + { + // Configs predating the `type` field were always S3. + Some("s3".to_string()) + } else { + None + } + }) + }) + .unwrap_or_default(); + let filesystem_remote = + remote_type.eq_ignore_ascii_case("filesystem") || remote_type.eq_ignore_ascii_case("fs"); let default_dir = default_cache_dir().to_string_lossy().to_string(); // Leak default_dir into a &'static str so FormField can hold it. @@ -241,61 +265,124 @@ fn build_fields(file_config: &FileConfig, env: &EnvOverrides) -> Vec validation_error: None, env_locked: false, }, - // [Remote (S3)] + // [Remote] + FormField { + key: "remote_type", + label: "Backend type", + kind: FieldKind::Text, + value: remote_type, + env_var: "", + env_value: None, + default_hint: "(s3 or filesystem)", + validation_error: None, + env_locked: false, + }, FormField { key: "s3_bucket", label: "Bucket", kind: FieldKind::Text, value: remote.and_then(|r| r.bucket.clone()).unwrap_or_default(), - env_var: "KACHE_S3_BUCKET", - env_value: env_val("KACHE_S3_BUCKET"), + env_var: if env.s3_bucket { "KACHE_S3_BUCKET" } else { "" }, + env_value: if env.s3_bucket { + env_val("KACHE_S3_BUCKET") + } else { + None + }, default_hint: "(none)", validation_error: None, - env_locked: env.s3_bucket, + env_locked: !filesystem_remote && env.s3_bucket, }, FormField { key: "s3_endpoint", label: "Endpoint", kind: FieldKind::Text, value: remote.and_then(|r| r.endpoint.clone()).unwrap_or_default(), - env_var: "KACHE_S3_ENDPOINT", - env_value: env_val("KACHE_S3_ENDPOINT"), + env_var: if env.s3_endpoint { + "KACHE_S3_ENDPOINT" + } else { + "" + }, + env_value: if env.s3_endpoint { + env_val("KACHE_S3_ENDPOINT") + } else { + None + }, default_hint: "(none)", validation_error: None, - env_locked: env.s3_endpoint, + env_locked: !filesystem_remote && env.s3_endpoint, }, FormField { key: "s3_region", label: "Region", kind: FieldKind::Text, value: remote.and_then(|r| r.region.clone()).unwrap_or_default(), - env_var: "KACHE_S3_REGION", - env_value: env_val("KACHE_S3_REGION"), + env_var: if env.s3_region { "KACHE_S3_REGION" } else { "" }, + env_value: if env.s3_region { + env_val("KACHE_S3_REGION") + } else { + None + }, default_hint: "(default: us-east-1)", validation_error: None, - env_locked: env.s3_region, - }, - FormField { - key: "s3_prefix", - label: "S3 Prefix", - kind: FieldKind::Text, - value: remote.and_then(|r| r.prefix.clone()).unwrap_or_default(), - env_var: "KACHE_S3_PREFIX", - env_value: env_val("KACHE_S3_PREFIX"), - default_hint: "(default: artifacts)", - validation_error: None, - env_locked: env.s3_prefix, + env_locked: !filesystem_remote && env.s3_region, }, FormField { key: "s3_profile", label: "AWS Profile", kind: FieldKind::Text, value: remote.and_then(|r| r.profile.clone()).unwrap_or_default(), - env_var: "KACHE_S3_PROFILE", - env_value: env_val("KACHE_S3_PROFILE"), + env_var: if env.s3_profile { + "KACHE_S3_PROFILE" + } else { + "" + }, + env_value: if env.s3_profile { + env_val("KACHE_S3_PROFILE") + } else { + None + }, default_hint: "(none)", validation_error: None, - env_locked: env.s3_profile, + env_locked: !filesystem_remote && env.s3_profile, + }, + FormField { + key: "fs_path", + label: "Filesystem path", + kind: FieldKind::Text, + value: remote.and_then(|r| r.path.clone()).unwrap_or_default(), + env_var: "", + env_value: None, + default_hint: "(required for filesystem)", + validation_error: None, + env_locked: false, + }, + FormField { + key: "fs_atomic_write_dir", + label: "Atomic write dir", + kind: FieldKind::Text, + value: remote + .and_then(|r| r.atomic_write_dir.clone()) + .unwrap_or_default(), + env_var: "", + env_value: None, + default_hint: "(optional; same filesystem)", + validation_error: None, + env_locked: false, + }, + FormField { + key: "remote_prefix", + label: "Prefix", + kind: FieldKind::Text, + value: remote.and_then(|r| r.prefix.clone()).unwrap_or_default(), + env_var: if env.s3_prefix { "KACHE_S3_PREFIX" } else { "" }, + env_value: if env.s3_prefix { + env_val("KACHE_S3_PREFIX") + } else { + None + }, + default_hint: "(default: artifacts)", + validation_error: None, + env_locked: !filesystem_remote && env.s3_prefix, }, // [Advanced] FormField { @@ -336,15 +423,15 @@ fn build_sections() -> Vec
{ }, Section { label: "Caching", - fields: 3..6, + fields: 3..8, }, Section { - label: "Remote (S3)", - fields: 6..11, + label: "Remote", + fields: 8..16, }, Section { label: "Advanced", - fields: 11..13, + fields: 16..18, }, ] } @@ -376,33 +463,166 @@ fn validate_field(field: &FormField) -> Option { fn validate_cross_field(fields: &[FormField]) -> Vec<(usize, String)> { let mut errors = Vec::new(); - let bucket_idx = fields.iter().position(|f| f.key == "s3_bucket"); - let endpoint_idx = fields.iter().position(|f| f.key == "s3_endpoint"); - let region_idx = fields.iter().position(|f| f.key == "s3_region"); - - let has_endpoint = endpoint_idx - .map(|i| !fields[i].value.is_empty()) - .unwrap_or(false); - let has_region = region_idx - .map(|i| !fields[i].value.is_empty()) - .unwrap_or(false); - let bucket_empty = bucket_idx - .map(|i| fields[i].value.is_empty()) - .unwrap_or(true); - - if (has_endpoint || has_region) - && bucket_empty - && let Some(idx) = bucket_idx + let index = |key: &str| fields.iter().position(|f| f.key == key); + let configured_value = |key: &str| { + fields.iter().find(|f| f.key == key).and_then(|f| { + let value = f.value.as_str(); + (!value.trim().is_empty()).then_some(value) + }) + }; + let value = |key: &str| { + fields.iter().find(|f| f.key == key).and_then(|f| { + let value = f.value.trim(); + if !value.is_empty() { + Some(value) + } else if f.env_locked { + f.env_value + .as_deref() + .map(str::trim) + .filter(|v| !v.is_empty()) + } else { + None + } + }) + }; + let is_set = |key: &str| value(key).is_some(); + let is_configured = |key: &str| configured_value(key).is_some(); + + let remote_type = value("remote_type").map(str::to_ascii_lowercase); + let has_s3_fields = ["s3_bucket", "s3_endpoint", "s3_region", "s3_profile"] + .iter() + .any(|key| is_set(key)); + let has_filesystem_fields = ["fs_path", "fs_atomic_write_dir"] + .iter() + .any(|key| is_set(key)); + let has_remote_fields = has_s3_fields || has_filesystem_fields || is_set("remote_prefix"); + let has_configured_remote_fields = [ + "s3_bucket", + "s3_endpoint", + "s3_region", + "s3_profile", + "fs_path", + "fs_atomic_write_dir", + "remote_prefix", + ] + .iter() + .any(|key| is_configured(key)); + + if let Some(prefix) = configured_value("remote_prefix") + && validate_remote_prefix(prefix).is_err() + && let Some(idx) = index("remote_prefix") { errors.push(( idx, - "Bucket required when endpoint/region is set".to_string(), + "Prefix must be a canonical relative path without leading/trailing slashes".to_string(), )); } + if remote_type.is_none() + && has_configured_remote_fields + && let Some(idx) = index("remote_type") + { + errors.push((idx, "Backend type required (s3 or filesystem)".to_string())); + } + + let effective_type = remote_type.as_deref().or({ + if has_filesystem_fields { + Some("filesystem") + } else if has_remote_fields { + Some("s3") + } else { + None + } + }); + + match effective_type { + Some("s3") => { + if !is_set("s3_bucket") + && let Some(idx) = index("s3_bucket") + { + errors.push((idx, "Bucket required for S3 backend".to_string())); + } + for key in ["fs_path", "fs_atomic_write_dir"] { + if is_set(key) + && let Some(idx) = index(key) + { + errors.push(( + idx, + "Filesystem field conflicts with S3 backend".to_string(), + )); + } + } + } + Some("filesystem" | "fs") => { + if !is_set("fs_path") + && let Some(idx) = index("fs_path") + { + errors.push(( + idx, + "Filesystem path required for filesystem backend".to_string(), + )); + } + for key in ["fs_path", "fs_atomic_write_dir"] { + if let Some(path) = configured_value(key) + && !shellexpand(path.trim()).is_absolute() + && let Some(idx) = index(key) + { + errors.push(( + idx, + "Filesystem paths must be absolute (or start with ~/)".to_string(), + )); + } + } + for key in ["s3_bucket", "s3_endpoint", "s3_region", "s3_profile"] { + // Legacy S3 environment variables do not apply to an explicitly + // selected filesystem backend. Only persisted form values conflict. + if is_configured(key) + && let Some(idx) = index(key) + { + errors.push(( + idx, + "S3 field conflicts with filesystem backend".to_string(), + )); + } + } + } + Some(_) => { + if let Some(idx) = index("remote_type") { + errors.push(( + idx, + "Backend type must be \"s3\" or \"filesystem\"".to_string(), + )); + } + } + None => {} + } + errors } +fn refresh_remote_env_scope(fields: &mut [FormField]) { + let filesystem = fields + .iter() + .find(|field| field.key == "remote_type") + .is_some_and(|field| { + matches!( + field.value.trim().to_ascii_lowercase().as_str(), + "filesystem" | "fs" + ) + }); + + for field in fields.iter_mut().filter(|field| { + matches!( + field.key, + "s3_bucket" | "s3_endpoint" | "s3_region" | "s3_profile" | "remote_prefix" + ) + }) { + // An active KACHE_S3_* override is stored in env_var/env_value, but it + // is irrelevant (and editable around) while filesystem is selected. + field.env_locked = !filesystem && !field.env_var.is_empty(); + } +} + fn has_validation_errors(fields: &[FormField]) -> bool { fields.iter().any(|f| f.validation_error.is_some()) } @@ -480,22 +700,32 @@ fn fields_to_file_config( let bucket = get("s3_bucket"); let endpoint = get("s3_endpoint"); let region = get("s3_region"); - let prefix = get("s3_prefix"); + let prefix = get("remote_prefix"); let profile = get("s3_profile"); - let has_remote = bucket.is_some() + let path = get("fs_path"); + let atomic_write_dir = get("fs_atomic_write_dir"); + let remote_type = get("remote_type") + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| !value.is_empty()); + let has_remote = remote_type.is_some() + || bucket.is_some() || endpoint.is_some() || region.is_some() || prefix.is_some() - || profile.is_some(); + || profile.is_some() + || path.is_some() + || atomic_write_dir.is_some(); let remote = if has_remote { Some(RemoteFileConfig { - _type: Some("s3".to_string()), + _type: remote_type, bucket, endpoint, region, prefix, profile, + path, + atomic_write_dir, }) } else { None @@ -824,7 +1054,11 @@ fn handle_editing(state: &mut EditorState, code: KeyCode) -> Action { } KeyCode::Enter => { // Commit the edit + let edited_remote_type = state.fields[state.cursor].key == "remote_type"; state.fields[state.cursor].value = state.edit_buffer.clone(); + if edited_remote_type { + refresh_remote_env_scope(&mut state.fields); + } state.dirty = true; state.mode = Mode::Navigate; run_all_validation(&mut state.fields); @@ -1098,6 +1332,13 @@ fn draw_footer(f: &mut ratatui::Frame, path_area: Rect, keys_area: Rect, state: mod tests { use super::*; + fn absolute_test_path() -> String { + std::env::temp_dir() + .join("kache") + .to_string_lossy() + .into_owned() + } + fn empty_env() -> EnvOverrides { EnvOverrides { fallback: false, @@ -1122,7 +1363,13 @@ mod tests { fn test_build_fields_count() { let config = FileConfig::default(); let fields = build_fields(&config, &empty_env()); - assert_eq!(fields.len(), 15); + assert_eq!(fields.len(), 18); + + let sections = build_sections(); + assert_eq!(sections[0].fields, 0..3); + assert_eq!(sections[1].fields, 3..8); + assert_eq!(sections[2].fields, 8..16); + assert_eq!(sections[3].fields, 16..18); } #[test] @@ -1212,8 +1459,332 @@ mod tests { } let errors = validate_cross_field(&fields); - assert!(!errors.is_empty()); - assert!(errors[0].1.contains("Bucket required")); + assert!( + errors + .iter() + .any(|(_, message)| message.contains("Backend type required")) + ); + assert!( + errors + .iter() + .any(|(_, message)| message.contains("Bucket required")) + ); + } + + #[test] + fn test_cross_validation_accepts_env_locked_s3_bucket() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + fields + .iter_mut() + .find(|field| field.key == "remote_type") + .unwrap() + .value = "s3".to_string(); + let bucket = fields + .iter_mut() + .find(|field| field.key == "s3_bucket") + .unwrap(); + bucket.env_locked = true; + bucket.env_value = Some("ambient-bucket".to_string()); + + let errors = validate_cross_field(&fields); + assert!( + !errors + .iter() + .any(|(_, message)| message.contains("Bucket required")), + "{errors:?}" + ); + } + + #[test] + fn test_cross_validation_filesystem_requires_path_and_rejects_s3_fields() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + fields + .iter_mut() + .find(|field| field.key == "remote_type") + .unwrap() + .value = "filesystem".to_string(); + fields + .iter_mut() + .find(|field| field.key == "s3_bucket") + .unwrap() + .value = "wrong-backend".to_string(); + + let errors = validate_cross_field(&fields); + assert!( + errors + .iter() + .any(|(_, message)| message.contains("Filesystem path required")) + ); + assert!( + errors + .iter() + .any(|(_, message)| message.contains("conflicts with filesystem")) + ); + } + + #[test] + fn test_cross_validation_s3_rejects_filesystem_fields() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + for (key, value) in [ + ("remote_type", "s3"), + ("s3_bucket", "bucket"), + ("fs_path", "/var/cache/kache"), + ] { + fields + .iter_mut() + .find(|field| field.key == key) + .unwrap() + .value = value.to_string(); + } + + let errors = validate_cross_field(&fields); + assert!( + errors + .iter() + .any(|(_, message)| message.contains("conflicts with S3")) + ); + } + + #[test] + fn test_cross_validation_rejects_unknown_backend() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + fields + .iter_mut() + .find(|field| field.key == "remote_type") + .unwrap() + .value = "azure".to_string(); + + let errors = validate_cross_field(&fields); + assert!( + errors + .iter() + .any(|(_, message)| message.contains("must be \"s3\" or \"filesystem\"")) + ); + } + + #[test] + fn test_build_fields_infers_legacy_s3_type() { + let config = FileConfig { + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + bucket: Some("legacy-bucket".to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let fields = build_fields(&config, &empty_env()); + assert_eq!( + fields + .iter() + .find(|field| field.key == "remote_type") + .unwrap() + .value, + "s3" + ); + } + + #[test] + fn test_filesystem_prefix_ignores_legacy_s3_env_lock() { + let config = FileConfig { + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("filesystem".to_string()), + path: Some(absolute_test_path()), + prefix: Some("shared".to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let mut env = empty_env(); + env.s3_prefix = true; + + let fields = build_fields(&config, &env); + let prefix = fields + .iter() + .find(|field| field.key == "remote_prefix") + .unwrap(); + assert!(!prefix.env_locked); + assert_eq!(prefix.env_var, "KACHE_S3_PREFIX"); + } + + #[test] + fn test_filesystem_ignores_all_legacy_s3_env_locks() { + let config = FileConfig { + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("filesystem".to_string()), + path: Some(absolute_test_path()), + prefix: Some("shared".to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let mut env = empty_env(); + env.s3_bucket = true; + env.s3_endpoint = true; + env.s3_region = true; + env.s3_prefix = true; + env.s3_profile = true; + + let fields = build_fields(&config, &env); + for key in [ + "s3_bucket", + "s3_endpoint", + "s3_region", + "s3_profile", + "remote_prefix", + ] { + let field = fields.iter().find(|field| field.key == key).unwrap(); + assert!(!field.env_locked, "{key}"); + if key == "remote_prefix" { + assert_eq!(field.env_var, "KACHE_S3_PREFIX"); + } else { + assert!(field.env_var.starts_with("KACHE_S3_"), "{key}"); + } + } + assert!(validate_cross_field(&fields).is_empty()); + } + + #[test] + fn test_filesystem_validation_ignores_ambient_s3_values() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + fields + .iter_mut() + .find(|field| field.key == "remote_type") + .unwrap() + .value = "filesystem".to_string(); + fields + .iter_mut() + .find(|field| field.key == "fs_path") + .unwrap() + .value = absolute_test_path(); + for key in ["s3_bucket", "s3_endpoint", "s3_region", "s3_profile"] { + let field = fields.iter_mut().find(|field| field.key == key).unwrap(); + field.env_locked = true; + field.env_value = Some("ambient".to_string()); + } + + assert!(validate_cross_field(&fields).is_empty()); + } + + #[test] + fn test_env_only_legacy_s3_does_not_require_persisted_type() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + let bucket = fields + .iter_mut() + .find(|field| field.key == "s3_bucket") + .unwrap(); + bucket.env_var = "KACHE_S3_BUCKET"; + bucket.env_value = Some("ambient-bucket".to_string()); + bucket.env_locked = true; + + assert!(validate_cross_field(&fields).is_empty()); + } + + #[test] + fn test_backend_switch_recomputes_s3_env_locks() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + for key in ["s3_bucket", "remote_prefix"] { + let field = fields.iter_mut().find(|field| field.key == key).unwrap(); + field.env_var = if key == "s3_bucket" { + "KACHE_S3_BUCKET" + } else { + "KACHE_S3_PREFIX" + }; + field.env_value = Some("ambient".to_string()); + field.env_locked = true; + } + + fields + .iter_mut() + .find(|field| field.key == "remote_type") + .unwrap() + .value = "filesystem".to_string(); + refresh_remote_env_scope(&mut fields); + assert!( + fields + .iter() + .filter(|field| matches!(field.key, "s3_bucket" | "remote_prefix")) + .all(|field| !field.env_locked) + ); + + fields + .iter_mut() + .find(|field| field.key == "remote_type") + .unwrap() + .value = "s3".to_string(); + refresh_remote_env_scope(&mut fields); + assert!( + fields + .iter() + .filter(|field| matches!(field.key, "s3_bucket" | "remote_prefix")) + .all(|field| field.env_locked) + ); + } + + #[test] + fn test_filesystem_validation_rejects_relative_paths() { + let config = FileConfig::default(); + let mut fields = build_fields(&config, &empty_env()); + for (key, value) in [ + ("remote_type", "filesystem"), + ("fs_path", "relative/cache"), + ("fs_atomic_write_dir", "relative/staging"), + ] { + fields + .iter_mut() + .find(|field| field.key == key) + .unwrap() + .value = value.to_string(); + } + + let errors = validate_cross_field(&fields); + assert_eq!( + errors + .iter() + .filter(|(_, message)| message.contains("must be absolute")) + .count(), + 2 + ); + } + + #[test] + fn test_remote_prefix_validation_rejects_noncanonical_paths() { + let config = FileConfig::default(); + for prefix in [" artifacts", "/artifacts", "artifacts/", "a//b", "a/../b"] { + let mut fields = build_fields(&config, &empty_env()); + for (key, value) in [ + ("remote_type", "s3"), + ("s3_bucket", "bucket"), + ("remote_prefix", prefix), + ] { + fields + .iter_mut() + .find(|field| field.key == key) + .unwrap() + .value = value.to_string(); + } + assert!( + validate_cross_field(&fields) + .iter() + .any(|(_, message)| message.contains("canonical relative path")), + "{prefix:?}" + ); + } } #[test] @@ -1262,7 +1833,9 @@ mod tests { endpoint: Some("https://s3.example.com".to_string()), region: Some("eu-west-1".to_string()), prefix: Some("my-project".to_string()), - profile: None, + profile: Some("build-account".to_string()), + path: None, + atomic_write_dir: None, }), }), }; @@ -1322,9 +1895,54 @@ mod tests { assert_eq!(cache.event_log_keep_lines, Some(500)); let remote = cache.remote.as_ref().unwrap(); + assert_eq!(remote._type.as_deref(), Some("s3")); assert_eq!(remote.bucket.as_deref(), Some("test-bucket")); assert_eq!(remote.endpoint.as_deref(), Some("https://s3.example.com")); assert_eq!(remote.region.as_deref(), Some("eu-west-1")); + assert_eq!(remote.prefix.as_deref(), Some("my-project")); + assert_eq!(remote.profile.as_deref(), Some("build-account")); + assert!(remote.path.is_none()); + assert!(remote.atomic_write_dir.is_none()); + } + + #[test] + fn test_fields_to_file_config_filesystem_roundtrip() { + let original = FileConfig { + cache: Some(CacheFileConfig { + remote: Some(RemoteFileConfig { + _type: Some("filesystem".to_string()), + prefix: Some("shared/artifacts".to_string()), + path: Some("/var/cache/kache".to_string()), + atomic_write_dir: Some("/var/cache/kache-tmp".to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let fields = build_fields(&original, &empty_env()); + let reconstructed = fields_to_file_config( + &fields, None, None, None, None, None, None, None, None, None, None, None, None, None, + None, + ); + let remote = reconstructed + .cache + .as_ref() + .and_then(|cache| cache.remote.as_ref()) + .unwrap(); + + assert_eq!(remote._type.as_deref(), Some("filesystem")); + assert_eq!(remote.path.as_deref(), Some("/var/cache/kache")); + assert_eq!( + remote.atomic_write_dir.as_deref(), + Some("/var/cache/kache-tmp") + ); + assert_eq!(remote.prefix.as_deref(), Some("shared/artifacts")); + assert!(remote.bucket.is_none()); + assert!(remote.endpoint.is_none()); + assert!(remote.region.is_none()); + assert!(remote.profile.is_none()); } #[test] diff --git a/src/daemon.rs b/src/daemon.rs index aa0a67f2..7affe836 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -1929,7 +1929,7 @@ impl Daemon { // Fallback: direct upload (no queue available) let Ok(_permit) = self.s3_semaphore.acquire().await else { - return Response::err("S3 semaphore closed"); + return Response::err("remote semaphore closed"); }; self.do_upload(job).await } @@ -1980,7 +1980,7 @@ impl Daemon { tracing::debug!( crate_name = job.crate_name, key = key_short, - "skipping upload — already in S3" + "skipping upload — already in remote" ); return Response::ok(); } @@ -1988,8 +1988,8 @@ impl Daemon { tracing::debug!( crate_name = job.crate_name, key = key_short, - bucket = remote.bucket, - "starting S3 upload" + remote = %remote.describe(), + "starting remote upload" ); let entry_dir = PathBuf::from(&job.entry_dir); @@ -2088,14 +2088,14 @@ impl Daemon { crate_name = job.crate_name, key = key_short, elapsed_ms, - "upload to S3 failed: {e:#}" + "remote upload failed: {e:#}" ); Response::err(format!("upload failed: {e:#}")) } } } - /// Handle a remote check: check S3 for a cache key, download if found. Gated by S3 semaphore. + /// Handle a remote check: look for a cache key and download it if found. /// Waits for the manifest prefetch to finish first so batch downloads aren't bypassed. pub async fn handle_remote_check(&self, req: &RemoteCheckRequest) -> Response { let Some(remote) = &self.config.remote else { @@ -2167,7 +2167,7 @@ impl Daemon { Some(age) if age <= KEY_CACHE_AUTHORITATIVE_FOR ); if authoritative { - tracing::debug!("key cache: {} not found (skipping S3)", &req.key); + tracing::debug!("key cache: {} not found (skipping remote)", &req.key); return Response::found(false); } if self.remote_health.head_probe_is_degraded() { @@ -2212,7 +2212,7 @@ impl Daemon { if needs_head_probe { let semaphore_start = Instant::now(); let Ok(_permit) = self.s3_semaphore.acquire().await else { - return Response::err("S3 semaphore closed"); + return Response::err("remote semaphore closed"); }; semaphore_wait_ms += semaphore_start.elapsed().as_millis() as u64; let head_start = Instant::now(); @@ -2232,7 +2232,7 @@ impl Daemon { Err(e) => { // A non-404 error is transient (throttling / 5xx), not a // confirmed miss. Do one bounded retry before falling back, - // so a single S3 blip under load isn't recorded as an + // so a single remote blip under load isn't recorded as an // authoritative "not in remote" — which would force a full // recompile + re-upload exactly when the remote is // struggling. Still fail safe (found=false) if it persists. @@ -2252,8 +2252,9 @@ impl Daemon { return Response::found(false); } Err(e2) => { - let error = - format!("S3 exists check failed (after retry): {e2}; first: {e}"); + let error = format!( + "remote exists check failed (after retry): {e2}; first: {e}" + ); self.remote_health.note_head_probe_failure(&error); return Response::found(false); } @@ -2379,7 +2380,7 @@ impl Daemon { // Acquire semaphore for download let semaphore_start = Instant::now(); let Ok(_permit) = self.s3_semaphore.acquire().await else { - return Response::err("S3 semaphore closed"); + return Response::err("remote semaphore closed"); }; semaphore_wait_ms += semaphore_start.elapsed().as_millis() as u64; @@ -2489,12 +2490,12 @@ impl Daemon { .unwrap_or_default() .as_secs(), }); - Response::err(format!("S3 download failed: {e}")) + Response::err(format!("remote download failed: {e}")) } } } - /// Handle a batch remote check: check multiple keys against S3 concurrently. + /// Handle a batch remote check concurrently. pub async fn handle_batch_remote_check( self: &Arc, req: &BatchRemoteCheckRequest, @@ -2546,7 +2547,7 @@ impl Daemon { } drop(downloading_guard); - // If empty keys were sent, fetch all S3 keys missing locally + // If empty keys were sent, fetch all remote keys missing locally. if req.keys.is_empty() && let Ok(backend) = self.get_remote_backend().await && let Ok(s3_keys) = crate::remote_plan::RemotePlanner::new(&self.config) @@ -2592,11 +2593,7 @@ impl Daemon { // Spawn a single coordinator task with bounded concurrency let daemon = Arc::clone(self); - let bucket = remote.bucket.clone(); - let prefix = remote.prefix.clone(); - let remote_endpoint = remote.endpoint.clone(); - let remote_region = remote.region.clone(); - let remote_profile = remote.profile.clone(); + let remote_config = remote.clone(); let cancel_rx = self.prefetch_cancel.subscribe(); tokio::spawn(async move { let mut in_flight = futures::stream::FuturesUnordered::new(); @@ -2646,11 +2643,7 @@ impl Daemon { let sem = daemon.s3_semaphore.clone(); let d = daemon.clone(); - let b = bucket.clone(); - let p = prefix.clone(); - let endpoint = remote_endpoint.clone(); - let region = remote_region.clone(); - let profile = remote_profile.clone(); + let remote_cfg = remote_config.clone(); let download_plan = crate::remote_plan::RemotePlanner::new(&d.config) .plan(crate::remote_plan::RemoteWorkload::Prefetch); in_flight.push(tokio::spawn(async move { @@ -2678,13 +2671,6 @@ impl Daemon { }; let blobs_dir = d.config.store_dir().join("blobs"); let start = Instant::now(); - let remote_cfg = crate::config::RemoteConfig { - bucket: b, - endpoint, - region, - prefix: p, - profile, - }; let download_result = download_plan .layout(backend.as_ref(), &remote_cfg) .download_entry(&key, &crate_name, &entry_dir, &blobs_dir) @@ -3363,7 +3349,7 @@ async fn server_main(config: &Config, coord: DaemonCoordFile) -> Result<()> { } }); - // S3 key cache population task (if remote configured) + // Remote key-cache population task (if configured). let cache_handle = if config.remote.is_some() { let cache_daemon = daemon.clone(); Some(tokio::spawn(async move { @@ -3372,11 +3358,13 @@ async fn server_main(config: &Config, coord: DaemonCoordFile) -> Result<()> { for attempt in 1..=5 { match populate_key_cache(&cache_daemon).await { Ok(count) => { - tracing::info!("S3 key cache populated: {count} keys"); + tracing::info!("remote key cache populated: {count} keys"); break; } Err(e) => { - tracing::warn!("S3 key cache population attempt {attempt}/5 failed: {e}"); + tracing::warn!( + "remote key cache population attempt {attempt}/5 failed: {e}" + ); if attempt < 5 { tokio::time::sleep(delay).await; delay *= 2; @@ -3397,21 +3385,21 @@ async fn server_main(config: &Config, coord: DaemonCoordFile) -> Result<()> { Ok(count) => { if consecutive_refresh_failures > 0 { tracing::info!( - "S3 key cache refresh recovered after {consecutive_refresh_failures} failed attempt(s)" + "remote key cache refresh recovered after {consecutive_refresh_failures} failed attempt(s)" ); consecutive_refresh_failures = 0; } - tracing::debug!("S3 key cache refreshed: {count} keys"); + tracing::debug!("remote key cache refreshed: {count} keys"); } Err(e) => { consecutive_refresh_failures += 1; if should_warn_key_cache_refresh_failure(consecutive_refresh_failures) { tracing::warn!( - "S3 key cache refresh failed (attempt {consecutive_refresh_failures}): {e}" + "remote key cache refresh failed (attempt {consecutive_refresh_failures}): {e}" ); } else { tracing::debug!( - "S3 key cache refresh failed (attempt {consecutive_refresh_failures}): {e}" + "remote key cache refresh failed (attempt {consecutive_refresh_failures}): {e}" ); } } @@ -7672,17 +7660,12 @@ mod tests { // CLIENT side: send_remote_check connects to a live in-process server, // sends a RemoteCheck, and parses the response. The daemon's key cache // is fresh + authoritative and lacks the key, so it answers a definitive - // miss with no S3. Covers send_remote_check's Ok(resp)+resp.ok success + // miss without touching the remote. Covers send_remote_check's + // Ok(resp)+resp.ok success // arm (daemon.rs 3309-3314) through the real socket + handle_connection. let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); - config.remote = Some(crate::config::RemoteConfig { - bucket: "test".into(), - endpoint: Some("http://localhost:9000".into()), - region: "us-east-1".into(), - prefix: "artifacts".into(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3("test", "artifacts")); let socket_path = config.socket_path(); std::fs::create_dir_all(socket_path.parent().unwrap()).unwrap(); @@ -7809,59 +7792,82 @@ mod tests { assert!(resp.evicted.is_some(), "gc reports an evicted count"); } - // ── Daemon remote handlers driven against an injected mock S3 client ───── - // - // The daemon's `s3_client` OnceCell can be pre-populated, so a test injects - // a wire-mock-backed client and exercises the REMOTE handler paths (which - // otherwise need a live bucket) with zero production change. - use aws_smithy_http_client::test_util::wire::{ReplayedEvent, WireMockServer}; + // ── Daemon remote handlers driven against an injected backend ──────────── fn test_remote_config() -> crate::config::RemoteConfig { - crate::config::RemoteConfig { - bucket: "bucket".to_string(), - endpoint: None, - region: "us-east-1".to_string(), - prefix: "prefix".to_string(), - profile: None, - } + crate::config::RemoteConfig::test_s3("bucket", "prefix") + } + + fn test_remote_backend() -> Arc { + Arc::new(crate::remote_backend::memory_backend()) } - async fn mock_s3_client( - events: Vec, - ) -> ( - WireMockServer, - Arc, + fn test_manifest_object_key(cache_key: &str, crate_name: &str) -> String { + format!("prefix/v3/manifests/{crate_name}/{cache_key}.json") + } + + fn test_pack_object_key(cache_key: &str, crate_name: &str) -> String { + format!("prefix/v3/packs/{crate_name}/{cache_key}.tar.zst") + } + + fn test_build_manifest_object_key() -> String { + format!( + "prefix/_manifests/{}.json", + crate::cli::default_manifest_key() + ) + } + + async fn put_test_object( + backend: &Arc, + key: &str, + body: &[u8], ) { - let server = WireMockServer::start(events).await; - let conf = aws_sdk_s3::config::Builder::new() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .region(aws_sdk_s3::config::Region::new("us-east-1")) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - "AK", "SK", None, None, "test", - )) - .endpoint_url(server.endpoint_url()) - .http_client(server.http_client()) - .force_path_style(true) - .build(); - let backend = crate::remote_backend::S3Backend::new( - aws_sdk_s3::Client::from_conf(conf), - "bucket".to_string(), - ); - (server, Arc::new(backend)) + backend + .put(key, body.to_vec(), None) + .await + .expect("seed test remote object"); + } + + struct PutFailBackend; + + #[async_trait::async_trait] + impl crate::remote_backend::RemoteBackend for PutFailBackend { + async fn head(&self, _key: &str) -> Result { + Ok(false) + } + + async fn get( + &self, + _key: &str, + _max_bytes: Option, + ) -> Result> { + Ok(None) + } + + async fn put(&self, _key: &str, _body: Vec, _content_type: Option<&str>) -> Result<()> { + anyhow::bail!("injected PUT failure") + } + + async fn list(&self, _prefix: &str) -> Result> { + Ok(Vec::new()) + } + + fn describe(&self, key: &str) -> String { + format!("failure://test/{key}") + } } #[tokio::test] async fn test_socket_remote_check_miss_with_injected_mock_client() { - // Remote configured + injected mock that 404s the HEAD: handle_remote_check - // runs its head-probe path and reports found=false (covers get_s3_client, - // RemoteLayout.exists_entry, and the not-found branch). + // Remote configured + an empty in-memory backend: handle_remote_check + // runs its head-probe path and reports found=false. let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); let socket_path = config.socket_path(); std::fs::create_dir_all(socket_path.parent().unwrap()).unwrap(); - let (server, client) = mock_s3_client(vec![ReplayedEvent::status(404)]).await; + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -7881,26 +7887,19 @@ mod tests { assert!(resp.ok, "remote check should return a response: {resp:?}"); assert_eq!(resp.found, Some(false), "missing remote key -> found=false"); - server.shutdown(); } #[tokio::test] async fn test_socket_prefetch_empty_keys_lists_remote_then_no_op() { - // Empty prefetch keys + injected mock returning an empty bucket listing: - // handle_prefetch lists S3, finds nothing missing, and returns ok - // ("nothing to fetch"). Covers get_s3_client + list_keys + no-op path. + // Empty prefetch keys + an empty backend: handle_prefetch lists the + // remote, finds nothing missing, and returns ok ("nothing to fetch"). let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); let socket_path = config.socket_path(); std::fs::create_dir_all(socket_path.parent().unwrap()).unwrap(); - let empty_list = "\ - \ - bucketprefix/v3/manifests/\ - 01000false\ - "; - let (server, client) = mock_s3_client(vec![ReplayedEvent::with_body(empty_list)]).await; + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -7915,19 +7914,23 @@ mod tests { .await; assert!(resp.ok, "prefetch over empty remote should be ok: {resp:?}"); - server.shutdown(); } #[tokio::test] async fn test_do_upload_skips_when_entry_already_in_remote() { - // Injected mock HEADs 200 for the manifest -> do_upload sees the entry - // already exists remotely and returns ok without uploading. Covers - // get_s3_client + RemoteLayout.exists_entry + the already-exists skip. + // A seeded manifest makes do_upload see that the entry already exists, + // so it returns ok without uploading. let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); - let (server, client) = mock_s3_client(vec![ReplayedEvent::ok()]).await; // 200 HEAD + let client = test_remote_backend(); + put_test_object( + &client, + &test_manifest_object_key("abc123def456", "serde"), + b"{}", + ) + .await; let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -7947,7 +7950,6 @@ mod tests { resp.ok, "already-present upload should be a no-op ok: {resp:?}" ); - server.shutdown(); } #[tokio::test] @@ -7962,12 +7964,10 @@ mod tests { seed_store_entry(&config, "upkey123", "serde", dir.path()); let entry_dir = config.store_dir().join("upkey123"); - let mut events = vec![ReplayedEvent::status(404)]; // HEAD: not present - events.extend(std::iter::repeat_with(ReplayedEvent::ok).take(4)); // pack + manifest PUTs - let (server, client) = mock_s3_client(events).await; + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); assert!( - daemon.remote_backend.set(client).is_ok(), + daemon.remote_backend.set(client.clone()).is_ok(), "inject mock backend" ); @@ -7981,7 +7981,18 @@ mod tests { .await; assert!(resp.ok, "upload of a new entry should succeed: {resp:?}"); - server.shutdown(); + assert!( + client + .head(&test_pack_object_key("upkey123", "serde")) + .await + .unwrap() + ); + assert!( + client + .head(&test_manifest_object_key("upkey123", "serde")) + .await + .unwrap() + ); } #[tokio::test] @@ -7995,9 +8006,7 @@ mod tests { seed_store_entry(&config, "upfail1", "serde", dir.path()); let entry_dir = config.store_dir().join("upfail1"); - let mut events = vec![ReplayedEvent::status(404)]; // HEAD: not present - events.extend(std::iter::repeat_with(|| ReplayedEvent::status(403)).take(4)); // PUTs denied - let (server, client) = mock_s3_client(events).await; + let client: Arc = Arc::new(PutFailBackend); let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8021,7 +8030,6 @@ mod tests { .load(Ordering::Relaxed), 1 ); - server.shutdown(); } #[tokio::test] @@ -8029,13 +8037,13 @@ mod tests { // With a remote configured but no planner endpoint (resolve_prefetch_plan // -> Ok(None)) and no local/remote candidates, handle_build_started runs // the fallback planner, finds nothing to prefetch, and returns ok. - // Covers the fallback-planning branch (daemon.rs 2120-2132). namespace is - // None so no S3 shard query is issued; an injected mock guards the client. + // Covers the fallback-planning branch (daemon.rs 2120-2132). Namespace is + // None so no remote shard query is issued. let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); - let (server, client) = mock_s3_client(vec![ReplayedEvent::status(404)]).await; + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8056,24 +8064,18 @@ mod tests { resp.ok, "fallback with nothing to prefetch should be ok: {resp:?}" ); - server.shutdown(); } #[tokio::test] async fn test_batch_remote_check_remote_path_with_injected_mock() { - // Two checks against an injected mock that 404s every HEAD: the batch + // Two checks against an empty backend: the batch // handler fans out handle_remote_check and returns one found=false per // check. Covers handle_batch_remote_check's remote path + join_all. let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); - // One 404 HEAD per distinct key (over-provisioned; concurrent order-safe - // since all are 404). - let events = std::iter::repeat_with(|| ReplayedEvent::status(404)) - .take(6) - .collect(); - let (server, client) = mock_s3_client(events).await; + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8101,7 +8103,6 @@ mod tests { let results = resp.batch_results.expect("batch results present"); assert_eq!(results.len(), 2); assert!(results.iter().all(|r| r.found == Some(false))); - server.shutdown(); } #[tokio::test] @@ -8113,10 +8114,19 @@ mod tests { let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); - let mut events = vec![ReplayedEvent::ok()]; // HEAD 200 -> exists - // GET pack returns non-zstd bytes -> download_entry errors. - events.extend(std::iter::repeat_with(|| ReplayedEvent::with_body(b"not a pack")).take(4)); - let (server, client) = mock_s3_client(events).await; + let client = test_remote_backend(); + put_test_object( + &client, + &test_manifest_object_key("hit01hit02hit03", "serde"), + b"{}", + ) + .await; + put_test_object( + &client, + &test_pack_object_key("hit01hit02hit03", "serde"), + b"not a pack", + ) + .await; let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8137,7 +8147,6 @@ mod tests { "download failure should surface as an error: {resp:?}" ); assert!(resp.error.is_some()); - server.shutdown(); } /// The prefetch cap must always leave head-room in the permit pool for @@ -8169,8 +8178,9 @@ mod tests { let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); - // Only event: the GET answers 404 (no HEAD happens — key cache says positive). - let (server, client) = mock_s3_client(vec![ReplayedEvent::status(404)]).await; + // The empty backend answers a clean miss (no HEAD happens — key cache + // says positive). + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8206,7 +8216,6 @@ mod tests { .load(Ordering::Relaxed), 0 ); - server.shutdown(); } /// Build a valid v3 entry pack for `key` from a throwaway store. @@ -8252,8 +8261,9 @@ mod tests { // finds the extracted entry. let entry_dir = config.store_dir().join(key); - let (server, client) = - mock_s3_client(vec![ReplayedEvent::ok(), ReplayedEvent::with_body(&pack)]).await; + let client = test_remote_backend(); + put_test_object(&client, &test_manifest_object_key(key, "serde"), b"{}").await; + put_test_object(&client, &test_pack_object_key(key, "serde"), &pack).await; let daemon = Arc::new(Daemon::new(config.clone())); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8274,13 +8284,12 @@ mod tests { config.store_dir().join(key).join("meta.json").exists(), "entry should be imported into the local store" ); - server.shutdown(); } #[tokio::test] async fn test_handle_prefetch_explicit_key_downloads_in_background() { // handle_prefetch with an explicit key spawns the background download - // coordinator. With the injected mock serving a valid pack, the + // coordinator. With the in-memory backend serving a valid pack, the // coordinator downloads + imports the entry. Covers the prefetch // coordinator + per-key download task (the biggest daemon block). let dir = tempfile::tempdir().unwrap(); @@ -8291,11 +8300,8 @@ mod tests { let key = key.as_str(); let pack = build_entry_pack(key, "serde"); - // Over-provision pack responses for the coordinator's GET(s). - let events = std::iter::repeat_with(|| ReplayedEvent::with_body(&pack)) - .take(4) - .collect(); - let (server, client) = mock_s3_client(events).await; + let client = test_remote_backend(); + put_test_object(&client, &test_pack_object_key(key, "serde"), &pack).await; let daemon = Arc::new(Daemon::new(config.clone())); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8323,12 +8329,11 @@ mod tests { imported, "background prefetch coordinator should download + import the entry" ); - server.shutdown(); } #[tokio::test] async fn test_handle_prefetch_records_a_failed_download() { - // The injected mock serves garbage for the pack GET, so the coordinator's + // The in-memory backend serves garbage for the pack GET, so the coordinator's // download_entry fails and the per-key task takes its error branch: // downloads_failed++ and a failure TransferEvent, with no import. // Covers handle_prefetch's download-error path (daemon.rs 2006-2034). @@ -8338,10 +8343,13 @@ mod tests { let key = "abcdef0123456789".repeat(4); let key = key.as_str(); - let events = std::iter::repeat_with(|| ReplayedEvent::with_body(b"not a valid pack")) - .take(6) - .collect(); - let (server, client) = mock_s3_client(events).await; + let client = test_remote_backend(); + put_test_object( + &client, + &test_pack_object_key(key, "serde"), + b"not a valid pack", + ) + .await; let daemon = Arc::new(Daemon::new(config.clone())); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8375,22 +8383,6 @@ mod tests { assert!(failed, "a garbage pack must record a failed download"); // Nothing was imported. assert!(!config.store_dir().join(key).join("meta.json").exists()); - server.shutdown(); - } - - fn list_manifests_xml(keys: &[&str]) -> String { - let contents: String = keys - .iter() - .map(|k| format!("{k}10")) - .collect(); - format!( - "\ - \ - bucketprefix/v3/manifests/\ - {}1000false\ - {contents}", - keys.len() - ) } #[tokio::test] @@ -8402,11 +8394,9 @@ mod tests { let mut config = test_config(dir.path()); config.remote = Some(test_remote_config()); - let xml = list_manifests_xml(&[ - "prefix/v3/manifests/serde/key1aaaa.json", - "prefix/v3/manifests/tokio/key2bbbb.json", - ]); - let (server, client) = mock_s3_client(vec![ReplayedEvent::with_body(xml)]).await; + let client = test_remote_backend(); + put_test_object(&client, "prefix/v3/manifests/serde/key1aaaa.json", b"{}").await; + put_test_object(&client, "prefix/v3/manifests/tokio/key2bbbb.json", b"{}").await; let daemon = Daemon::new(config); assert!( daemon.remote_backend.set(client).is_ok(), @@ -8419,7 +8409,6 @@ mod tests { assert_eq!(count, 2); // The cache now answers positively for a listed key. assert_eq!(daemon.key_cache.check("key1aaaa").await, Some(true)); - server.shutdown(); } #[tokio::test] @@ -8445,12 +8434,12 @@ mod tests { }], }; let body = serde_json::to_vec(&manifest).unwrap(); - let (server, client) = mock_s3_client(vec![ReplayedEvent::with_body(body)]).await; + let client = test_remote_backend(); + put_test_object(&client, &test_build_manifest_object_key(), &body).await; let daemon = Arc::new(Daemon::new(config)); // Should complete without panicking and without queuing the cheap crate. monolithic_manifest_prefetch(&daemon, client.as_ref(), &remote).await; - server.shutdown(); } #[tokio::test] @@ -8463,11 +8452,10 @@ mod tests { let remote = test_remote_config(); config.remote = Some(remote.clone()); - let (server, client) = mock_s3_client(vec![ReplayedEvent::status(404)]).await; + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); monolithic_manifest_prefetch(&daemon, client.as_ref(), &remote).await; // must not panic - server.shutdown(); } #[tokio::test] @@ -8494,15 +8482,17 @@ mod tests { }], }; let body = serde_json::to_vec(&manifest).unwrap(); - // Manifest GET, then garbage for the background pack download (which may - // fail — dispatch is what we're covering, and handle_prefetch returns ok). - let mut events = vec![ReplayedEvent::with_body(body)]; - events.extend(std::iter::repeat_with(|| ReplayedEvent::with_body(b"nope")).take(6)); - let (server, client) = mock_s3_client(events).await; + let client = test_remote_backend(); + put_test_object(&client, &test_build_manifest_object_key(), &body).await; + // The background pack download may fail — dispatch is what this covers. + put_test_object(&client, &test_pack_object_key(&key, "expensive"), b"nope").await; let daemon = Arc::new(Daemon::new(config)); + assert!( + daemon.remote_backend.set(client.clone()).is_ok(), + "inject mock backend" + ); monolithic_manifest_prefetch(&daemon, client.as_ref(), &remote).await; // dispatches + returns - server.shutdown(); } #[tokio::test] @@ -8522,22 +8512,13 @@ mod tests { ) .unwrap(); - // One NoSuchKey 404 per shard GET (over-provisioned; concurrent/order-safe). - let not_found = || ReplayedEvent::HttpResponse { - status: 404, - body: bytes::Bytes::from_static( - b"NoSuchKey", - ), - }; - let events = std::iter::repeat_with(not_found).take(70).collect(); - let (server, client) = mock_s3_client(events).await; + let client = test_remote_backend(); let daemon = Arc::new(Daemon::new(config)); let count = shard_prefetch(&daemon, &client, "prefix", "ns", &lock) .await .expect("shard prefetch should succeed"); assert_eq!(count, 0, "no shards matched -> nothing queued"); - server.shutdown(); } // ── New protocol types serde tests ──────────────────────────── @@ -8743,13 +8724,7 @@ mod tests { async fn test_handle_remote_check_skips_head_when_probe_circuit_is_open() { let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); - config.remote = Some(crate::config::RemoteConfig { - bucket: "test".into(), - endpoint: Some("http://localhost:9000".into()), - region: "us-east-1".into(), - prefix: "artifacts".into(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3("test", "artifacts")); let daemon = Daemon::new(config); daemon.signal_warming_complete(); @@ -8778,18 +8753,11 @@ mod tests { async fn test_handle_remote_check_authoritative_key_cache_skips_s3() { // A freshly-populated key cache that doesn't contain the requested key is // authoritative (age <= KEY_CACHE_AUTHORITATIVE_FOR): the daemon answers - // a definitive miss without ever touching S3. Covers handle_remote_check's - // Some(false)+authoritative branch. No mock needed — it returns before - // get_s3_client(). + // a definitive miss without ever touching the remote. Covers + // handle_remote_check's Some(false)+authoritative branch. let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); - config.remote = Some(crate::config::RemoteConfig { - bucket: "test".into(), - endpoint: Some("http://localhost:9000".into()), - region: "us-east-1".into(), - prefix: "artifacts".into(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3("test", "artifacts")); let daemon = Daemon::new(config); daemon.signal_warming_complete(); @@ -8851,13 +8819,7 @@ mod tests { async fn test_handle_upload_with_queue_returns_immediately() { let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); - config.remote = Some(crate::config::RemoteConfig { - bucket: "test".into(), - endpoint: Some("http://localhost:9000".into()), - region: "us-east-1".into(), - prefix: "artifacts".into(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3("test", "artifacts")); let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::(); let daemon = Daemon::new(config); @@ -8880,13 +8842,7 @@ mod tests { async fn test_handle_upload_queue_closed() { let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); - config.remote = Some(crate::config::RemoteConfig { - bucket: "test".into(), - endpoint: Some("http://localhost:9000".into()), - region: "us-east-1".into(), - prefix: "artifacts".into(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3("test", "artifacts")); let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); let daemon = Daemon::new(config); @@ -8910,13 +8866,7 @@ mod tests { async fn test_handle_upload_dedup() { let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); - config.remote = Some(crate::config::RemoteConfig { - bucket: "test".into(), - endpoint: Some("http://localhost:9000".into()), - region: "us-east-1".into(), - prefix: "artifacts".into(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3("test", "artifacts")); let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::(); let daemon = Daemon::new(config); @@ -8964,13 +8914,7 @@ mod tests { async fn test_handle_upload_after_queue_close_rejects_without_direct_upload() { let dir = tempfile::tempdir().unwrap(); let mut config = test_config(dir.path()); - config.remote = Some(crate::config::RemoteConfig { - bucket: "test".into(), - endpoint: Some("http://localhost:9000".into()), - region: "us-east-1".into(), - prefix: "artifacts".into(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3("test", "artifacts")); let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::(); let daemon = Daemon::new(config); diff --git a/src/fallback_planner.rs b/src/fallback_planner.rs index e28d1dbf..ad1387e7 100644 --- a/src/fallback_planner.rs +++ b/src/fallback_planner.rs @@ -95,7 +95,7 @@ impl PlannerDataSource for LocalPlannerSource<'_> { let keys = self.daemon.key_cache_keys_for_crate(crate_name).await; if !keys.is_empty() { tracing::info!( - "fallback planner: resolved {} extra candidates from S3 key cache for crate '{}'", + "fallback planner: resolved {} extra candidates from remote key cache for crate '{}'", keys.len(), crate_name ); @@ -208,7 +208,7 @@ mod tests { #[tokio::test] async fn key_cache_keys_for_crate_is_empty_until_populated() { - // A fresh daemon's S3 key cache is unpopulated, so the planner source + // A fresh daemon's remote key cache is unpopulated, so the planner source // resolves no extra candidates (the `if !keys.is_empty()` false arm). let dir = tempfile::tempdir().unwrap(); let config = test_config(dir.path().join("cache"), None); diff --git a/src/main.rs b/src/main.rs index e4e4ab23..114ef73a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -55,7 +55,7 @@ pub const VERSION: &str = { } }; -/// kache: Content-addressed build cache for Rust, C/C++ and more with hardlinks and S3 remote storage. +/// kache: Content-addressed build cache for Rust, C/C++ and more, with S3 and filesystem remotes. /// /// When invoked as RUSTC_WRAPPER (arg[1] is a path to rustc), kache acts as a /// transparent build cache. Otherwise, it provides CLI commands for cache management. @@ -142,21 +142,21 @@ enum Commands { repair: bool, }, - /// Synchronize local cache with S3 remote (pull + push) + /// Synchronize the local cache with its configured remote (pull + push) Sync { /// Path to Cargo.toml (default: current directory) #[arg(long)] manifest_path: Option, - /// Only download from S3 (skip uploads) + /// Only download from the remote (skip uploads) #[arg(long)] pull: bool, - /// Only upload to S3 (skip downloads) + /// Only upload to the remote (skip downloads) #[arg(long)] push: bool, /// Show what would be synced without transferring #[arg(long)] dry_run: bool, - /// Pull all artifacts from S3 (ignore workspace filtering) + /// Pull all remote artifacts (ignore workspace filtering) #[arg(long)] all: bool, /// Scope the pull listing to workspace members (one LIST per member) @@ -164,16 +164,16 @@ enum Commands { /// /// This only narrows the up-front batch pull. Dependency artifacts are /// still resolved on demand during the build — the rustc wrapper fetches - /// any local miss from S3 via the daemon (a remote hit) and the daemon + /// any local miss from the remote via the daemon (a remote hit) and the daemon /// prefetches by build intent — so deps are not recompiled. Most useful /// when the dependency artifacts are already present locally (e.g. a /// prebuilt `cargo chef` deps image whose compiled deps sit in target/), - /// where they're local hits and never need an S3 round-trip; in a plain + /// where they're local hits and never need a remote round-trip; in a plain /// setup deps are fetched lazily during the build rather than pre-warmed. /// /// Errors out if the workspace set can't be resolved (cargo metadata /// failed or this isn't a Cargo workspace) rather than silently falling - /// back to a full-bucket scan. + /// back to a full remote scan. #[arg(long, conflicts_with = "all")] workspace: bool, }, diff --git a/src/remote.rs b/src/remote.rs index b93731e7..78fc1c07 100644 --- a/src/remote.rs +++ b/src/remote.rs @@ -1,12 +1,6 @@ use anyhow::{Context, Result}; -use aws_smithy_http_client::{ - Builder as SmithyHttpClientBuilder, - tls::{self, rustls_provider::CryptoMode}, -}; use serde::{Deserialize, Serialize}; -use std::time::Duration; -use crate::config::RemoteConfig; use crate::remote_backend::RemoteBackend; /// Result of a download operation with timing breakdown. @@ -52,67 +46,6 @@ pub struct UploadResult { pub network_ms: u64, } -pub async fn create_s3_client( - remote: &RemoteConfig, - pool_idle_secs: u64, -) -> Result { - // Build one ring-backed HTTPS client and share it across both the - // aws-config credential-resolution path and the S3 client. Injecting it - // is what lets us drop `default-https-client` (which would otherwise - // force the aws-lc-rs crypto provider, pulling `aws-lc-sys`). See the TLS - // note in Cargo.toml. - let http_client = SmithyHttpClientBuilder::new() - .tls_provider(tls::Provider::Rustls(CryptoMode::Ring)) - .pool_idle_timeout(Duration::from_secs(pool_idle_secs)) - .build_https(); - - let mut config_builder = aws_config::defaults(aws_config::BehaviorVersion::latest()) - .http_client(http_client.clone()) - .region(aws_config::Region::new(remote.region.clone())); - - if let Some(profile) = &remote.profile { - config_builder = config_builder.profile_name(profile); - } - - let has_access = std::env::var("KACHE_S3_ACCESS_KEY").ok(); - let has_secret = std::env::var("KACHE_S3_SECRET_KEY").ok(); - match (&has_access, &has_secret) { - (Some(access_key), Some(secret_key)) => { - config_builder = - config_builder.credentials_provider(aws_sdk_s3::config::Credentials::new( - access_key, - secret_key, - None, - None, - "kache-env", - )); - } - (Some(_), None) => { - tracing::warn!( - "KACHE_S3_ACCESS_KEY is set but KACHE_S3_SECRET_KEY is missing — ignoring partial credentials" - ); - } - (None, Some(_)) => { - tracing::warn!( - "KACHE_S3_SECRET_KEY is set but KACHE_S3_ACCESS_KEY is missing — ignoring partial credentials" - ); - } - (None, None) => {} - } - - let sdk_config = config_builder.load().await; - let mut s3_config = aws_sdk_s3::config::Builder::from(&sdk_config).force_path_style(true); - - s3_config = s3_config.http_client(http_client); - tracing::debug!(pool_idle_secs, "S3 HTTP client configured"); - - if let Some(endpoint) = &remote.endpoint { - s3_config = s3_config.endpoint_url(endpoint); - } - - Ok(aws_sdk_s3::Client::from_conf(s3_config.build())) -} - const MANIFEST_PREFIX: &str = "_manifests"; pub const MANIFEST_VERSION: &str = "v3"; @@ -234,38 +167,6 @@ pub async fn upload_shard( mod tests { use super::*; - #[tokio::test] - async fn create_s3_client_builds_with_profile_and_endpoint() { - // Config-only (no network): a RemoteConfig with both a named profile and - // a custom endpoint exercises the profile_name and endpoint_url branches - // of create_s3_client. Building the client must succeed offline. - let remote = RemoteConfig { - bucket: "b".to_string(), - endpoint: Some("http://localhost:9000".to_string()), - region: "us-east-1".to_string(), - prefix: "artifacts".to_string(), - profile: Some("my-profile".to_string()), - }; - let _client = create_s3_client(&remote, 30) - .await - .expect("client builds offline with profile + endpoint"); - } - - #[tokio::test] - async fn create_s3_client_builds_without_profile_or_endpoint() { - // The plain path (no profile, no custom endpoint) also builds. - let remote = RemoteConfig { - bucket: "b".to_string(), - endpoint: None, - region: "us-east-1".to_string(), - prefix: "artifacts".to_string(), - profile: None, - }; - let _client = create_s3_client(&remote, 30) - .await - .expect("client builds offline with defaults"); - } - #[test] fn test_manifest_serde_roundtrip() { let manifest = BuildManifest { @@ -347,36 +248,6 @@ mod tests { ); } - // ── Mock-S3 round-trips ───────────────────────────────────────────────── - // - // These drive the real aws-sdk-s3 client against an in-process wire mock - // (no network), exercising our request construction + response parsing for - // the manifest/shard object paths. - use aws_smithy_http_client::test_util::wire::{ReplayedEvent, WireMockServer}; - - /// Build an S3 client whose HTTP traffic is served by `server`, replaying - /// the canned `events` in request order. - async fn mock_client( - events: Vec, - ) -> (WireMockServer, crate::remote_backend::S3Backend) { - let server = WireMockServer::start(events).await; - let conf = aws_sdk_s3::config::Builder::new() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .region(aws_sdk_s3::config::Region::new("us-east-1")) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - "AK", "SK", None, None, "test", - )) - .endpoint_url(server.endpoint_url()) - .http_client(server.http_client()) - .force_path_style(true) - .build(); - let backend = crate::remote_backend::S3Backend::new( - aws_sdk_s3::Client::from_conf(conf), - "bucket".to_string(), - ); - (server, backend) - } - fn sample_manifest() -> BuildManifest { BuildManifest { version: 3, @@ -392,30 +263,34 @@ mod tests { } #[tokio::test] - async fn download_manifest_parses_a_served_json_object() { + async fn download_manifest_parses_a_stored_json_object() { + let backend = crate::remote_backend::memory_backend(); let body = serde_json::to_vec(&sample_manifest()).unwrap(); - let (server, client) = mock_client(vec![ReplayedEvent::with_body(&body)]).await; + backend + .put("prefix/_manifests/key.json", body, Some("application/json")) + .await + .unwrap(); - let got = download_manifest(&client, "prefix", "key") + let got = download_manifest(&backend, "prefix", "key") .await .expect("download should succeed"); assert_eq!(got.entries.len(), 1); assert_eq!(got.entries[0].crate_name, "serde"); - server.shutdown(); } #[tokio::test] - async fn upload_manifest_puts_without_error() { - // Exercises PutObject request construction + signing against the mock; - // a 200 OK is all the operation needs. (The wire mock records only - // connection/response events, not the request URI, so we assert on the - // operation result rather than the path.) - let (server, client) = mock_client(vec![ReplayedEvent::ok()]).await; - upload_manifest(&client, "prefix", "mykey", &sample_manifest()) + async fn upload_manifest_writes_the_expected_object() { + let backend = crate::remote_backend::memory_backend(); + upload_manifest(&backend, "prefix", "mykey", &sample_manifest()) .await .expect("upload should succeed"); - assert_eq!(server.events().len(), 3, "one request round-trip expected"); - server.shutdown(); + let stored = backend + .get("prefix/_manifests/mykey.json", None) + .await + .unwrap() + .expect("manifest object"); + let parsed: BuildManifest = serde_json::from_slice(&stored.body).unwrap(); + assert_eq!(parsed.manifest_key, "x86_64-unknown-linux-gnu"); } #[tokio::test] @@ -427,32 +302,29 @@ mod tests { crate_name: "tokio".to_string(), }], }; - let body = serde_json::to_vec(&shard).unwrap(); - let (server, client) = mock_client(vec![ReplayedEvent::with_body(&body)]).await; + let backend = crate::remote_backend::memory_backend(); + backend + .put( + &shard_object_key("prefix", "ns", "hash"), + serde_json::to_vec(&shard).unwrap(), + Some("application/json"), + ) + .await + .unwrap(); - let got = download_shard(&client, "prefix", "ns", "hash") + let got = download_shard(&backend, "prefix", "ns", "hash") .await .expect("download should succeed") .expect("shard should be present"); assert_eq!(got.entries, shard.entries); - server.shutdown(); } #[tokio::test] async fn download_shard_missing_returns_none() { - // A 404 with the S3 NoSuchKey error code maps to Ok(None), not an error. - let not_found = ReplayedEvent::HttpResponse { - status: 404, - body: bytes::Bytes::from_static( - b"NoSuchKey\ - The specified key does not exist.", - ), - }; - let (server, client) = mock_client(vec![not_found]).await; - let got = download_shard(&client, "prefix", "ns", "missing") + let backend = crate::remote_backend::memory_backend(); + let got = download_shard(&backend, "prefix", "ns", "missing") .await - .expect("a 404 NoSuchKey must not be an error"); + .expect("a missing object must not be an error"); assert!(got.is_none()); - server.shutdown(); } } diff --git a/src/remote_backend.rs b/src/remote_backend.rs index 21ca0c46..25b5105e 100644 --- a/src/remote_backend.rs +++ b/src/remote_backend.rs @@ -1,26 +1,48 @@ //! Transport abstraction for the remote cache. //! -//! The remote layout ([`crate::remote_layout`]) and the manifest/shard sync -//! ([`crate::remote`]) speak in opaque byte objects addressed by key. This -//! module is the seam between that key/bytes vocabulary and whatever actually -//! stores the bytes — today S3, next a plain shared folder (#414). -//! -//! Everything above the trait (pack framing, zstd, blake3 validation, key -//! naming) is transport-independent and stays shared. +//! The remote layout ([`crate::remote_layout`]) and manifest/shard sync +//! ([`crate::remote`]) speak in opaque byte objects addressed by key. OpenDAL +//! supplies the concrete S3 and shared-filesystem transports behind this seam. + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use async_trait::async_trait; -use aws_sdk_s3::error::ProvideErrorMetadata; -use aws_sdk_s3::operation::head_object::HeadObjectError; use bytes::Bytes; -use std::sync::Arc; - -use crate::config::RemoteConfig; +use futures::TryStreamExt; +#[cfg(test)] +use opendal::services::Memory; +use opendal::{ErrorKind, HttpTransporter, OperationContext, Operator}; +use opendal_http_transport_reqwest::ReqwestTransport; +use opendal_layer_retry::RetryLayer; +use opendal_service_fs::Fs; +use opendal_service_s3::S3; +use reqsign_aws_v4::{ + AssumeRoleWithWebIdentityCredentialProvider, Credential, DefaultCredentialProvider, + ECSCredentialProvider, EnvCredentialProvider, IMDSv2CredentialProvider, + ProcessCredentialProvider, ProfileCredentialProvider, SSOCredentialProvider, + StaticCredentialProvider, +}; +use reqsign_command_execute_tokio::TokioCommandExecute; +use reqsign_core::{ + CommandExecute, Context as SigningContext, Env, OsEnv, ProvideCredential, + ProvideCredentialChain, +}; + +use crate::config::{FilesystemRemoteConfig, RemoteBackendConfig, RemoteConfig, S3RemoteConfig}; + +/// Abort a LIST that cannot yield an entry or completion. Repeated entries are +/// detected separately because a malformed continuation response can keep +/// yielding the first page without ever stalling. +const LIST_PROGRESS_TIMEOUT: Duration = Duration::from_secs(60); /// A fetched object plus the timing split callers report as transfer telemetry. #[derive(Debug)] pub struct GetObject { - /// `Bytes` so a restore doesn't copy the whole pack a second time. + /// `Bytes` so a restore does not copy the whole pack a second time. pub body: Bytes, /// Time to response headers, ms. pub request_ms: u64, @@ -41,242 +63,418 @@ pub trait RemoteBackend: Send + Sync { /// Fetch `key`, or `None` when it is absent. /// /// `max_bytes` checks the object's advertised size before the body is - /// buffered when `Content-Length` is present, and enforces the cap against - /// collected bytes when `Content-Length` is missing or under-reported. - /// Note that when `Content-Length` is unavailable, the body is fully - /// buffered before post-collect rejection. + /// buffered when metadata is available, and always enforces the cap while + /// streaming the body. async fn get(&self, key: &str, max_bytes: Option) -> Result>; /// Store `body` at `key`. async fn put(&self, key: &str, body: Vec, content_type: Option<&str>) -> Result<()>; - /// Keys under `prefix`. + /// File keys under `prefix`. async fn list(&self, prefix: &str) -> Result>; - /// Where `key` lives, for logs and errors (e.g. `s3://bucket/key`). + /// Where `key` lives, for logs and errors. fn describe(&self, key: &str) -> String; } -/// S3 (and S3-compatible: MinIO, Ceph RGW, R2) transport. -pub struct S3Backend { - client: aws_sdk_s3::Client, - bucket: String, +/// OpenDAL-backed object transport. +pub struct OpenDalBackend { + operator: Operator, + root_description: String, + is_filesystem: bool, } -impl S3Backend { - pub fn new(client: aws_sdk_s3::Client, bucket: String) -> Self { - Self { client, bucket } - } - - pub(crate) async fn collect_get_object( - resp: aws_sdk_s3::operation::get_object::GetObjectOutput, - max_bytes: Option, - describe: &str, - request_ms: u64, - ) -> Result> { - // Refuse before collecting: the point of the cap is to not buffer the - // body at all if an advertised Content-Length is present. - if let Some(max) = max_bytes - && let Some(len) = resp.content_length() - && len as u64 > max - { - anyhow::bail!("{describe} too large: {len} bytes (max {max})"); - } - - let body_start = std::time::Instant::now(); - let body = resp - .body - .collect() - .await - .with_context(|| format!("reading body of {describe}"))?; - let body_ms = body_start.elapsed().as_millis() as u64; - - let bytes = body.into_bytes(); - if let Some(max) = max_bytes - && bytes.len() as u64 > max - { - anyhow::bail!("{describe} too large: {} bytes (max {max})", bytes.len()); +impl OpenDalBackend { + pub(crate) fn new(operator: Operator, root_description: String) -> Self { + Self { + operator, + root_description, + is_filesystem: false, } + } - Ok(Some(GetObject { - body: bytes, - request_ms, - body_ms, - })) + fn contextual_error(&self, operation: &str, key: &str, error: opendal::Error) -> anyhow::Error { + anyhow::Error::new(error).context(format!("{operation} {}", self.describe(key))) } -} -/// True when a GetObject failure means "no such object" rather than a -/// transport/service error. -/// -/// A structured code is authoritative: `NoSuchBucket` is also a 404, and -/// treating it as absence would turn a misconfigured bucket into a cache miss. -/// The raw status is consulted only when there is no code, because some S3 -/// clones answer a missing key with a bare 404 and no XML error body. -fn is_missing_get_object( - err: &aws_sdk_s3::error::SdkError< - aws_sdk_s3::operation::get_object::GetObjectError, - aws_smithy_runtime_api::client::orchestrator::HttpResponse, - >, -) -> bool { - match err.as_service_error() { - Some(se) if se.code().is_some() || se.is_no_such_key() => { - se.is_no_such_key() - || matches!( - se.code(), - Some("NotFound" | "NoSuchKey" | "404" | "NoSuchObject") - ) + fn validate_key(&self, operation: &str, key: &str, list_prefix: bool) -> Result<()> { + let original = key; + let key = if list_prefix && !key.is_empty() { + key.strip_suffix('/').unwrap_or(key) + } else { + key + }; + let valid_empty = list_prefix && original.is_empty(); + let canonical = valid_empty + || (!key.is_empty() + && !original.starts_with('/') + && !key.contains('\\') + // On Windows a colon can introduce a drive prefix or alternate + // data stream. Reject it for filesystem keys on every platform + // so a shared config stays portable and contained by its root. + && !(self.is_filesystem && key.contains(':')) + && key + .split('/') + .all(|segment| !segment.is_empty() && segment != "." && segment != "..")); + if !canonical { + anyhow::bail!( + "{operation} rejected non-canonical remote key {original:?} under {}", + self.root_description + ); } - _ => err - .raw_response() - .is_some_and(|r| r.status().as_u16() == 404), + Ok(()) } } -fn is_missing_head_object(err: &HeadObjectError) -> bool { - err.is_not_found() - || matches!( - err.code(), - Some("NotFound" | "NoSuchKey" | "404" | "NoSuchObject") - ) -} - -fn describe_head_object_error(err: &HeadObjectError, http_status: Option) -> String { - let mut details = Vec::new(); - - if let Some(status) = http_status { - details.push(format!("HTTP {status}")); - } - if let Some(code) = err.code() { - details.push(format!("code={code}")); - } - if let Some(message) = err.message() { - details.push(format!("message={message}")); - } - // Fallback: if no structured info, include Display output - if details.is_empty() || (http_status.is_none() && err.code().is_none()) { - details.push(err.to_string()); - } - - details.join(", ") +#[cfg(test)] +pub(crate) fn memory_backend() -> OpenDalBackend { + ensure_rustls_provider(); + let operator = Operator::new(Memory::default()).expect("memory operator"); + OpenDalBackend::new(operator, "memory://test".to_string()) } #[async_trait] -impl RemoteBackend for S3Backend { +impl RemoteBackend for OpenDalBackend { async fn head(&self, key: &str) -> Result { - match self - .client - .head_object() - .bucket(&self.bucket) - .key(key) - .send() - .await - { - Ok(_) => Ok(true), - Err(e) => { - let http_status = e.raw_response().map(|r| r.status().as_u16()); - let err = e.into_service_error(); - if is_missing_head_object(&err) { - Ok(false) - } else { - Err(anyhow::anyhow!( - "S3 head_object error for {}: {}", - self.describe(key), - describe_head_object_error(&err, http_status) - )) - } - } + self.validate_key("HEAD", key, false)?; + match self.operator.stat(key).await { + Ok(metadata) => Ok(metadata.is_file()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(false), + Err(error) => Err(self.contextual_error("HEAD", key, error)), } } async fn get(&self, key: &str, max_bytes: Option) -> Result> { - let request_start = std::time::Instant::now(); - let resp = match self - .client - .get_object() - .bucket(&self.bucket) - .key(key) - .send() + self.validate_key("GET", key, false)?; + let request_start = Instant::now(); + let reader = self + .operator + .reader(key) .await - { - Ok(resp) => resp, - Err(e) if is_missing_get_object(&e) => return Ok(None), - Err(e) => { - return Err(anyhow::Error::new(e)) - .with_context(|| format!("GET {}", self.describe(key))); + .map_err(|error| self.contextual_error("GET", key, error))?; + let mut stream = reader + .into_stream(..) + .await + .map_err(|error| self.contextual_error("GET", key, error))?; + + // Opening stream metadata starts the real read request for S3 without + // consuming its body. Filesystem readers do not expose open metadata, + // so fall back to stat there. + let advertised_length = match stream.metadata().await { + Ok(metadata) => Some(metadata.content_length()), + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) if error.kind() == ErrorKind::Unsupported => { + match self.operator.stat(key).await { + Ok(metadata) => Some(metadata.content_length()), + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(self.contextual_error("STAT", key, error)), + } } + Err(error) => return Err(self.contextual_error("GET", key, error)), }; let request_ms = request_start.elapsed().as_millis() as u64; - Self::collect_get_object(resp, max_bytes, &self.describe(key), request_ms).await - } + if let (Some(max), Some(length)) = (max_bytes, advertised_length) + && length > max + { + anyhow::bail!( + "{} too large: {length} bytes (max {max})", + self.describe(key) + ); + } - async fn put(&self, key: &str, body: Vec, content_type: Option<&str>) -> Result<()> { - let mut req = self - .client - .put_object() - .bucket(&self.bucket) - .key(key) - .body(body.into()); - - if let Some(ct) = content_type { - req = req.content_type(ct); + let body_start = Instant::now(); + let mut chunks = Vec::new(); + let mut length = 0_u64; + loop { + let chunk = match stream.try_next().await { + Ok(Some(chunk)) => chunk, + Ok(None) => break, + Err(error) if error.kind() == ErrorKind::NotFound && length == 0 => { + return Ok(None); + } + Err(error) => return Err(self.contextual_error("reading body of", key, error)), + }; + length = length + .checked_add(chunk.len() as u64) + .context("remote object length overflow")?; + if let Some(max) = max_bytes + && length > max + { + anyhow::bail!( + "{} too large: at least {length} bytes (max {max})", + self.describe(key) + ); + } + chunks.extend(chunk); } + let body_ms = body_start.elapsed().as_millis() as u64; + let body = chunks.into_iter().collect::().to_bytes(); - req.send() - .await - .with_context(|| format!("PUT {}", self.describe(key)))?; + Ok(Some(GetObject { + body, + request_ms, + body_ms, + })) + } - Ok(()) + async fn put(&self, key: &str, body: Vec, content_type: Option<&str>) -> Result<()> { + self.validate_key("PUT", key, false)?; + let request = self.operator.write_with(key, body); + let result = match content_type { + Some(content_type) => request.content_type(content_type).await, + None => request.await, + }; + result + .map(|_| ()) + .map_err(|error| self.contextual_error("PUT", key, error)) } async fn list(&self, prefix: &str) -> Result> { - let mut keys = Vec::new(); - let mut continuation_token: Option = None; + self.validate_key("LIST", prefix, true)?; + let mut lister = match self.operator.lister_with(prefix).recursive(true).await { + Ok(lister) => lister, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(self.contextual_error("LIST", prefix, error)), + }; + let mut entries = Vec::new(); + let mut seen = HashSet::new(); loop { - let mut req = self - .client - .list_objects_v2() - .bucket(&self.bucket) - .prefix(prefix); - - if let Some(token) = &continuation_token { - req = req.continuation_token(token); + let next = tokio::time::timeout(LIST_PROGRESS_TIMEOUT, lister.try_next()) + .await + .with_context(|| { + format!( + "LIST {} made no progress for {}s", + self.describe(prefix), + LIST_PROGRESS_TIMEOUT.as_secs() + ) + })?; + match next { + Ok(Some(entry)) => { + let path = entry.path().to_string(); + if !seen.insert(path.clone()) { + anyhow::bail!( + "LIST {} returned duplicate entry {path:?}; \ + the remote likely supplied an invalid continuation token", + self.describe(prefix) + ); + } + if entry.metadata().is_file() { + entries.push(path); + } + } + Ok(None) => break, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(self.contextual_error("LIST", prefix, error)), } + } + Ok(entries) + } - let resp = req - .send() - .await - .with_context(|| format!("listing s3://{}/{prefix}", self.bucket))?; + fn describe(&self, key: &str) -> String { + if key.is_empty() { + self.root_description.clone() + } else { + format!("{}/{}", self.root_description, key) + } + } +} - keys.extend( - resp.contents() - .iter() - .filter_map(|o| o.key()) - .map(String::from), - ); +fn with_retries(operator: Operator) -> Operator { + operator.layer( + RetryLayer::new() + .with_jitter() + .with_min_delay(Duration::from_millis(100)) + .with_max_delay(Duration::from_secs(10)) + .with_max_times(3), + ) +} - // A truncated response must carry a continuation token. Some - // non-AWS S3 implementations (MinIO/Ceph RGW/R2/proxies) have - // emitted is_truncated=true with a missing/empty token; treat that - // as end-of-pagination rather than looping forever re-listing page 1. - match resp.next_continuation_token() { - Some(token) if resp.is_truncated == Some(true) && !token.is_empty() => { - continuation_token = Some(token.to_string()); - } - _ => break, - } +fn ensure_rustls_provider() { + // Kache's reqwest client is compiled with rustls-no-provider. Install ring + // before constructing the S3 transport; the operation is process-wide and + // idempotent. + let _ = rustls::crypto::ring::default_provider().install_default(); +} + +/// Override only `AWS_PROFILE`, preserving every other process environment +/// value and the platform home-directory lookup. +#[derive(Debug, Clone)] +struct ProfileSelectingEnv { + inner: E, + profile: String, +} + +impl Env for ProfileSelectingEnv { + fn var(&self, key: &str) -> Option { + if key == "AWS_PROFILE" { + Some(self.profile.clone()) + } else { + self.inner.var(key) } + } - Ok(keys) + fn vars(&self) -> HashMap { + let mut vars = self.inner.vars(); + vars.insert("AWS_PROFILE".to_string(), self.profile.clone()); + vars } - fn describe(&self, key: &str) -> String { - format!("s3://{}/{}", self.bucket, key) + fn home_dir(&self) -> Option { + self.inner.home_dir() } } +/// OpenDAL exposes a custom credential chain but does not expose the selected +/// profile or command executor on its S3 builder. Wrap reqsign's default +/// provider so Kache can preserve both behaviors without mutating the process +/// environment. +#[derive(Debug)] +struct KacheCredentialProvider { + inner: DefaultCredentialProvider, + profile: Option, +} + +impl KacheCredentialProvider { + fn new(profile: Option, region: &str) -> Self { + // Keep the AWS SDK's broad precedence: environment credentials first, + // then all selected-profile providers, then workload identity/roles. + let chain = ProvideCredentialChain::new() + .push(EnvCredentialProvider::new()) + .push(ProfileCredentialProvider::default()) + .push(SSOCredentialProvider::default()) + .push(ProcessCredentialProvider::default()) + .push( + AssumeRoleWithWebIdentityCredentialProvider::new().with_region(region.to_string()), + ) + .push(ECSCredentialProvider::default()) + .push(IMDSv2CredentialProvider::default()); + Self { + inner: DefaultCredentialProvider::with_chain(chain), + profile, + } + } +} + +/// Reassemble reqsign's tokenized `credential_process` and run it through the +/// platform shell, matching the AWS SDK's support for quoted arguments and +/// executable paths containing spaces. +#[derive(Debug, Clone, Copy)] +struct KacheCommandExecute; + +impl CommandExecute for KacheCommandExecute { + async fn command_execute( + &self, + program: &str, + args: &[&str], + ) -> reqsign_core::Result { + let mut command = program.to_string(); + for arg in args { + command.push(' '); + command.push_str(arg); + } + #[cfg(windows)] + let (shell, shell_args) = ("cmd.exe", ["/C", command.as_str()]); + #[cfg(not(windows))] + let (shell, shell_args) = ("sh", ["-c", command.as_str()]); + TokioCommandExecute + .command_execute(shell, &shell_args) + .await + } +} + +impl ProvideCredential for KacheCredentialProvider { + type Credential = Credential; + + async fn provide_credential( + &self, + context: &SigningContext, + ) -> reqsign_core::Result> { + let context = context.clone().with_command_execute(KacheCommandExecute); + if let Some(profile) = &self.profile { + let context = context.with_env(ProfileSelectingEnv { + inner: OsEnv, + profile: profile.clone(), + }); + self.inner.provide_credential(&context).await + } else { + self.inner.provide_credential(&context).await + } + } +} + +fn create_s3_operator(config: &S3RemoteConfig, pool_idle_secs: u64) -> Result { + // reqwest is compiled with rustls-no-provider. Installing ring here keeps + // direct library/test callers safe; the operation is idempotent when + // another Kache HTTP client already installed it. + ensure_rustls_provider(); + let client = reqwest::Client::builder() + .pool_idle_timeout(Duration::from_secs(pool_idle_secs)) + .build() + .context("building S3 HTTP client")?; + let context = OperationContext::new() + .with_http_transport(HttpTransporter::new(ReqwestTransport::new(client))); + + let mut builder = S3::default() + .bucket(&config.bucket) + .region(&config.region) + // Keep transport integrity without requiring a provider to implement + // the newer full-object x-amz-checksum-* headers. Content-MD5 is + // supported by AWS S3 and common S3-compatible PutObject endpoints. + .checksum_algorithm("md5"); + let endpoint = config + .endpoint + .clone() + .or_else(|| std::env::var("AWS_ENDPOINT_URL_S3").ok()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + if let Some(endpoint) = endpoint { + builder = builder.endpoint(&endpoint); + } + + let mut credential_chain = ProvideCredentialChain::new().push(KacheCredentialProvider::new( + config.profile.clone(), + &config.region, + )); + let access_key = std::env::var("KACHE_S3_ACCESS_KEY").ok(); + let secret_key = std::env::var("KACHE_S3_SECRET_KEY").ok(); + match (access_key.as_deref(), secret_key.as_deref()) { + (Some(access_key), Some(secret_key)) => { + credential_chain = + credential_chain.push_front(StaticCredentialProvider::new(access_key, secret_key)); + } + (Some(_), None) => tracing::warn!( + "KACHE_S3_ACCESS_KEY is set but KACHE_S3_SECRET_KEY is missing — ignoring partial credentials" + ), + (None, Some(_)) => tracing::warn!( + "KACHE_S3_SECRET_KEY is set but KACHE_S3_ACCESS_KEY is missing — ignoring partial credentials" + ), + (None, None) => {} + } + builder = builder.credential_provider_chain(credential_chain); + + let operator = Operator::new(builder) + .context("building OpenDAL S3 operator")? + .with_context(context); + Ok(with_retries(operator)) +} + +fn create_filesystem_operator(config: &FilesystemRemoteConfig) -> Result { + ensure_rustls_provider(); + let root = config + .root + .to_str() + .context("filesystem remote path is not valid UTF-8")?; + let atomic_write_dir = config + .atomic_write_dir + .to_str() + .context("filesystem remote atomic_write_dir is not valid UTF-8")?; + let builder = Fs::default().root(root).atomic_write_dir(atomic_write_dir); + let operator = Operator::new(builder).context("building OpenDAL filesystem operator")?; + Ok(with_retries(operator)) +} + /// Build the backend named by `remote`. /// /// `Arc` rather than `Box`: the prefetch path fans shard downloads out across @@ -285,206 +483,358 @@ pub async fn create_backend( remote: &RemoteConfig, pool_idle_secs: u64, ) -> Result> { - let client = crate::remote::create_s3_client(remote, pool_idle_secs).await?; - Ok(Arc::new(S3Backend::new(client, remote.bucket.clone()))) + let backend = match &remote.backend { + RemoteBackendConfig::S3(config) => OpenDalBackend::new( + create_s3_operator(config, pool_idle_secs)?, + format!("s3://{}", config.bucket), + ), + RemoteBackendConfig::Filesystem(config) => { + let mut backend = OpenDalBackend::new( + create_filesystem_operator(config)?, + format!("file://{}", config.root.display()), + ); + backend.is_filesystem = true; + backend + } + }; + + Ok(Arc::new(backend)) } #[cfg(test)] mod tests { use super::*; - use aws_sdk_s3::error::ErrorMetadata; - use aws_smithy_http_client::test_util::wire::{ReplayedEvent, WireMockServer}; - - async fn mock_backend(events: Vec) -> (WireMockServer, S3Backend) { - let server = WireMockServer::start(events).await; - let conf = aws_sdk_s3::config::Builder::new() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .region(aws_sdk_s3::config::Region::new("us-east-1")) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - "AK", "SK", None, None, "test", - )) - .endpoint_url(server.endpoint_url()) - .http_client(server.http_client()) - .force_path_style(true) - .build(); - let backend = S3Backend::new(aws_sdk_s3::Client::from_conf(conf), "bucket".to_string()); - (server, backend) + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + async fn mock_http_server( + responses: Vec, + ) -> (String, tokio::sync::oneshot::Receiver>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (requests_tx, requests_rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let mut requests = Vec::new(); + for response in responses { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + loop { + let read = stream.read(&mut chunk).await.unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + requests.push(String::from_utf8_lossy(&request).into_owned()); + stream.write_all(response.as_bytes()).await.unwrap(); + stream.shutdown().await.unwrap(); + } + let _ = requests_tx.send(requests); + }); + (format!("http://{address}"), requests_rx) + } + + fn http_response(status: &str, body: &str) -> String { + format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Type: application/xml\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + } + + fn anonymous_s3_backend(endpoint: &str) -> OpenDalBackend { + ensure_rustls_provider(); + let client = reqwest::Client::builder().build().unwrap(); + let builder = S3::default() + .bucket("bucket") + .region("us-east-1") + .endpoint(endpoint) + .checksum_algorithm("md5") + .skip_signature(); + let context = OperationContext::new() + .with_http_transport(HttpTransporter::new(ReqwestTransport::new(client))); + let operator = Operator::new(builder).unwrap().with_context(context); + OpenDalBackend::new(operator, "s3://bucket".to_string()) } #[tokio::test] - async fn get_under_the_cap_is_allowed() { - let (_server, backend) = mock_backend(vec![ReplayedEvent::with_body("hello")]).await; + async fn object_round_trip_head_get_and_list() { + let backend = memory_backend(); + assert!(!backend.head("nested/key").await.unwrap()); + assert!(backend.get("nested/key", None).await.unwrap().is_none()); + backend + .put("nested/key", b"hello".to_vec(), Some("text/plain")) + .await + .unwrap(); + assert!(backend.head("nested/key").await.unwrap()); let fetched = backend - .get("k", Some(MAX_METADATA_BYTES_TEST)) + .get("nested/key", Some(5)) .await - .expect("under-cap GET succeeds") - .expect("object is present"); + .unwrap() + .expect("present"); assert_eq!(fetched.body, "hello"); + assert_eq!(backend.list("nested/").await.unwrap(), ["nested/key"]); } #[tokio::test] - async fn get_over_the_cap_is_refused() { - // The cap is checked against the advertised Content-Length before the - // body is buffered, so a 5-byte body still trips a 1-byte cap. - let (_server, backend) = mock_backend(vec![ReplayedEvent::with_body("hello")]).await; + async fn get_refuses_an_object_over_the_cap() { + let backend = memory_backend(); + backend.put("key", b"hello".to_vec(), None).await.unwrap(); - let err = backend - .get("k", Some(1)) + let error = backend + .get("key", Some(1)) .await - .expect_err("over-cap length must be rejected"); - let msg = err.to_string(); - assert!(msg.contains("too large"), "unexpected message: {msg}"); - assert!( - msg.contains("s3://bucket/k"), - "should name the object: {msg}" - ); + .expect_err("over-cap object must fail") + .to_string(); + assert!(error.contains("too large"), "{error}"); + assert!(error.contains("memory://test/key"), "{error}"); } #[tokio::test] - async fn get_without_content_length_over_the_cap_is_refused() { - use aws_sdk_s3::operation::get_object::GetObjectOutput; - use aws_sdk_s3::primitives::ByteStream; - - // Build a GetObjectOutput without content_length set (simulating missing Content-Length header, e.g. chunked transfer encoding) - let resp = GetObjectOutput::builder() - .body(ByteStream::from_static(b"hello")) - .build(); - - // Verify that Content-Length is indeed absent so the pre-check cannot trigger - assert!(resp.content_length().is_none()); + async fn filesystem_backend_uses_nested_paths_and_atomic_staging() { + let root = tempfile::tempdir().unwrap(); + let atomic_write_dir = root.path().join(".staging"); + let remote = RemoteConfig { + prefix: "artifacts".to_string(), + backend: RemoteBackendConfig::Filesystem(FilesystemRemoteConfig { + root: root.path().to_path_buf(), + atomic_write_dir: atomic_write_dir.clone(), + }), + }; + let backend = create_backend(&remote, 30).await.unwrap(); - let err = S3Backend::collect_get_object(resp, Some(1), "s3://bucket/k", 0) + assert!(backend.list("artifacts/").await.unwrap().is_empty()); + backend + .put( + "artifacts/v3/key", + b"shared".to_vec(), + Some("application/json"), + ) .await - .expect_err("over-cap body without Content-Length must be rejected post-collect"); - let msg = err.to_string(); - assert!(msg.contains("too large"), "unexpected message: {msg}"); - assert!( - msg.contains("s3://bucket/k"), - "should name the object: {msg}" + .unwrap(); + assert_eq!( + std::fs::read(root.path().join("artifacts/v3/key")).unwrap(), + b"shared" + ); + assert!(atomic_write_dir.is_dir()); + assert_eq!( + backend.list("artifacts/").await.unwrap(), + ["artifacts/v3/key"] + ); + backend + .put( + "artifacts/v3/key", + b"updated".to_vec(), + Some("application/json"), + ) + .await + .unwrap(); + assert_eq!( + backend + .get("artifacts/v3/key", None) + .await + .unwrap() + .unwrap() + .body, + "updated" ); } #[tokio::test] - async fn get_without_a_cap_is_unbounded() { - let (_server, backend) = mock_backend(vec![ReplayedEvent::with_body("hello")]).await; + async fn filesystem_backend_rejects_parent_traversal() { + let root = tempfile::tempdir().unwrap(); + let remote = RemoteConfig { + prefix: "artifacts".to_string(), + backend: RemoteBackendConfig::Filesystem(FilesystemRemoteConfig { + root: root.path().to_path_buf(), + atomic_write_dir: root.path().join(".staging"), + }), + }; + let backend = create_backend(&remote, 30).await.unwrap(); - let fetched = backend - .get("k", None) + backend + .put("../escape", b"nope".to_vec(), None) .await - .expect("uncapped GET succeeds") - .expect("object is present"); - assert_eq!(fetched.body, "hello"); + .expect_err("parent traversal must be rejected"); + backend + .put(r"..\escape", b"nope".to_vec(), None) + .await + .expect_err("Windows parent traversal must be rejected"); + backend + .put("/absolute", b"nope".to_vec(), None) + .await + .expect_err("absolute paths must be rejected"); + backend + .put("C:/escape", b"nope".to_vec(), None) + .await + .expect_err("Windows drive prefixes must be rejected"); } #[tokio::test] - async fn get_absent_object_is_none_not_an_error() { - let (_server, backend) = mock_backend(vec![ReplayedEvent::status(404)]).await; - - let fetched = backend.get("k", None).await.expect("404 is a clean miss"); - assert!(fetched.is_none()); + async fn s3_operator_builds_with_profile_and_custom_endpoint() { + let config = S3RemoteConfig { + bucket: "bucket".to_string(), + endpoint: Some("http://127.0.0.1:9000".to_string()), + region: "us-east-1".to_string(), + profile: Some("team".to_string()), + }; + create_s3_operator(&config, 30).expect("S3 operator builds without network I/O"); } #[tokio::test] - async fn get_at_exactly_the_cap_is_allowed() { - // The guard is strictly greater, so a body equal to the cap passes. - let (_server, backend) = mock_backend(vec![ReplayedEvent::with_body("hello")]).await; - - let fetched = backend - .get("k", Some(5)) - .await - .expect("at-cap GET succeeds") - .expect("object is present"); - assert_eq!(fetched.body, "hello"); - } + async fn s3_wire_uses_path_style_and_maps_bare_404_to_missing() { + let (endpoint, requests) = mock_http_server(vec![http_response("404 Not Found", "")]).await; + let backend = anonymous_s3_backend(&endpoint); - /// A 404 carrying a structured S3 error code. - fn s3_error(code: &str) -> ReplayedEvent { - ReplayedEvent::HttpResponse { - status: 404, - body: format!( - "\ - {code}test" - ) - .into(), - } + assert!( + backend + .get("nested/key", Some(1024)) + .await + .unwrap() + .is_none() + ); + let requests = requests.await.unwrap(); + assert_eq!( + requests[0].lines().next(), + Some("GET /bucket/nested/key HTTP/1.1") + ); } #[tokio::test] - async fn get_no_such_key_is_a_miss() { - let (_server, backend) = mock_backend(vec![s3_error("NoSuchKey")]).await; + async fn s3_wire_does_not_treat_no_such_bucket_as_a_cache_miss() { + let body = "NoSuchBucket\ + The bucket does not exist"; + let (endpoint, _requests) = + mock_http_server(vec![http_response("404 Not Found", body)]).await; + let backend = anonymous_s3_backend(&endpoint); - let fetched = backend.get("k", None).await.expect("NoSuchKey is a miss"); - assert!(fetched.is_none()); + backend + .get("key", None) + .await + .expect_err("a missing bucket is a configuration error"); } #[tokio::test] - async fn get_no_such_bucket_is_an_error_not_a_miss() { - // NoSuchBucket is also a 404. Reporting it as absence would turn a - // misconfigured bucket into an ordinary cache miss. - let (_server, backend) = mock_backend(vec![s3_error("NoSuchBucket")]).await; + async fn s3_wire_rejects_advertised_oversize_before_returning_body() { + let (endpoint, _requests) = mock_http_server(vec![http_response("200 OK", "hello")]).await; + let backend = anonymous_s3_backend(&endpoint); - backend - .get("k", None) + let error = backend + .get("key", Some(4)) .await - .expect_err("NoSuchBucket must not classify as absence"); + .expect_err("content-length above cap must fail") + .to_string(); + assert!(error.contains("too large"), "{error}"); } #[tokio::test] - async fn get_bare_404_without_a_code_is_a_miss() { - // Some S3 clones answer a missing key with a bare 404 and no XML body. - let (_server, backend) = mock_backend(vec![ReplayedEvent::status(404)]).await; - - let fetched = backend.get("k", None).await.expect("bare 404 is a miss"); - assert!(fetched.is_none()); + async fn s3_wire_follows_continuation_tokens() { + let first = "\ + \ + bucketartifacts/1\ + 1000true\ + artifacts/a1\ + 2026-07-24T00:00:00.000Z\ + next"; + let second = "\ + \ + bucketartifacts/1\ + 1000false\ + artifacts/b1\ + 2026-07-24T00:00:00.000Z\ + "; + let (endpoint, requests) = mock_http_server(vec![ + http_response("200 OK", first), + http_response("200 OK", second), + ]) + .await; + let backend = anonymous_s3_backend(&endpoint); + + assert_eq!( + backend.list("artifacts/").await.unwrap(), + ["artifacts/a", "artifacts/b"] + ); + let requests = requests.await.unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].contains("list-type=2"), "{requests:?}"); + assert!( + requests[1].contains("continuation-token=next"), + "{requests:?}" + ); } #[tokio::test] - async fn list_follows_the_continuation_token_across_pages() { - fn page(key: &str, next: Option<&str>) -> ReplayedEvent { - let (truncated, token) = match next { - Some(t) => ( - "true", - format!("{t}"), - ), - None => ("false", String::new()), - }; - ReplayedEvent::with_body(format!( - "\ - \ - bucket11\ - {truncated}{token}\ - {key}10" - )) - } + async fn s3_wire_put_includes_an_integrity_checksum() { + let (endpoint, requests) = mock_http_server(vec![http_response("200 OK", "")]).await; + let backend = anonymous_s3_backend(&endpoint); - let (_server, backend) = - mock_backend(vec![page("a.json", Some("tok")), page("b.json", None)]).await; + backend + .put("key", b"hello".to_vec(), Some("application/octet-stream")) + .await + .unwrap(); - let keys = backend.list("p/").await.expect("paginated LIST succeeds"); - assert_eq!(keys, vec!["a.json".to_string(), "b.json".to_string()]); + let requests = requests.await.unwrap(); + let request = &requests[0]; + assert!( + request.to_ascii_lowercase().contains("\r\ncontent-md5:"), + "{request}" + ); } - /// Mirrors `remote::MAX_METADATA_BYTES`; the cap under test is the - /// backend's `max_bytes` argument, not that specific constant. - const MAX_METADATA_BYTES_TEST: u64 = 64 * 1024 * 1024; - - #[test] - fn generic_head_object_not_found_codes_are_treated_as_misses() { - let err = HeadObjectError::generic(ErrorMetadata::builder().code("NotFound").build()); - assert!(is_missing_head_object(&err)); + #[tokio::test] + async fn s3_wire_rejects_a_truncated_page_without_a_continuation_token() { + let malformed = "\ + \ + bucketartifacts/1\ + 1000true\ + artifacts/a1\ + 2026-07-24T00:00:00.000Z\ + "; + let (endpoint, requests) = mock_http_server(vec![ + http_response("200 OK", malformed), + http_response("200 OK", malformed), + ]) + .await; + let backend = anonymous_s3_backend(&endpoint); + + let error = backend + .list("artifacts/") + .await + .expect_err("a repeated first page must not loop") + .to_string(); + assert!(error.contains("duplicate entry"), "{error}"); + assert_eq!(requests.await.unwrap().len(), 2); + } - let err = HeadObjectError::generic(ErrorMetadata::builder().code("NoSuchKey").build()); - assert!(is_missing_head_object(&err)); + #[cfg(unix)] + #[tokio::test] + async fn credential_process_executor_preserves_quoted_arguments() { + let output = KacheCommandExecute + .command_execute("printf", &["'%s'", "'hello world'"]) + .await + .unwrap(); + assert!(output.success()); + assert_eq!(output.stdout, b"hello world"); } #[test] - fn generic_head_object_non_miss_code_is_not_treated_as_missing() { - let err = HeadObjectError::generic( - ErrorMetadata::builder() - .code("SignatureDoesNotMatch") - .build(), - ); - assert!(!is_missing_head_object(&err)); + fn explicit_profile_overrides_only_the_profile_environment_value() { + let env = ProfileSelectingEnv { + inner: reqsign_core::StaticEnv { + home_dir: Some(PathBuf::from("/home/test")), + envs: HashMap::from([ + ("AWS_PROFILE".to_string(), "ambient".to_string()), + ("AWS_REGION".to_string(), "eu-west-1".to_string()), + ]), + }, + profile: "selected".to_string(), + }; + + assert_eq!(env.var("AWS_PROFILE").as_deref(), Some("selected")); + assert_eq!(env.var("AWS_REGION").as_deref(), Some("eu-west-1")); + assert_eq!(env.home_dir(), Some(PathBuf::from("/home/test"))); } } diff --git a/src/remote_layout.rs b/src/remote_layout.rs index 677ca092..3092e4d9 100644 --- a/src/remote_layout.rs +++ b/src/remote_layout.rs @@ -252,7 +252,7 @@ impl std::fmt::Display for EntryNotFound { impl std::error::Error for EntryNotFound {} // pub(crate) so other modules' tests can build a valid entry pack fixture to -// drive the S3 download-success paths against the mock (sync pull, daemon +// drive the remote download-success paths against the mock (sync pull, daemon // remote-check HIT, prefetch). Production callers are all within this module. pub(crate) fn create_entry_pack_zstd( entry_dir: &Path, @@ -486,10 +486,14 @@ fn copy_dir_all(src: &Path, dst: &Path) -> Result<()> { #[cfg(test)] mod tests { use super::{ - blob_path, copy_dir_all, create_entry_pack_zstd, extract_entry_pack, is_blob_hash, - is_rooted_path, is_safe_artifact_name, v3_manifest_key, v3_pack_key, + RemoteLayout, V3Manifest, blob_path, copy_dir_all, create_entry_pack_zstd, + extract_entry_pack, is_blob_hash, is_rooted_path, is_safe_artifact_name, v3_manifest_key, + v3_pack_key, }; - use crate::config::{Config, DEFAULT_DAEMON_IDLE_TIMEOUT_SECS, DEFAULT_S3_POOL_IDLE_SECS}; + use crate::config::{ + Config, DEFAULT_DAEMON_IDLE_TIMEOUT_SECS, DEFAULT_S3_POOL_IDLE_SECS, RemoteConfig, + }; + use crate::remote_backend::{GetObject, RemoteBackend, memory_backend}; use crate::store::{EntryMeta, Store}; use std::path::Path; @@ -818,13 +822,10 @@ mod tests { assert!(result.unwrap_err().to_string().contains("path traversal")); } - // ── Mock-S3 round-trips for the v3 RemoteLayout ───────────────────────── + // ── Backend-neutral round-trips for the v3 RemoteLayout ───────────────── // - // Drive RemoteLayout against an in-process wire mock (no network) to cover - // the head/get/put object paths and pack (de)serialization end to end. - use super::RemoteLayout; - use crate::config::RemoteConfig; - use aws_smithy_http_client::test_util::wire::{ReplayedEvent, WireMockServer}; + // Drive RemoteLayout through its byte-object seam so the layout behavior + // is exercised identically for S3 and filesystem transports. fn min_config(cache_dir: std::path::PathBuf) -> Config { Config { @@ -858,34 +859,41 @@ mod tests { } fn test_remote() -> RemoteConfig { - RemoteConfig { - bucket: "bucket".to_string(), - endpoint: None, - region: "us-east-1".to_string(), - prefix: "artifacts".to_string(), - profile: None, - } + RemoteConfig::test_s3("bucket", "artifacts") } - async fn mock_client( - events: Vec, - ) -> (WireMockServer, crate::remote_backend::S3Backend) { - let server = WireMockServer::start(events).await; - let conf = aws_sdk_s3::config::Builder::new() - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .region(aws_sdk_s3::config::Region::new("us-east-1")) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - "AK", "SK", None, None, "test", - )) - .endpoint_url(server.endpoint_url()) - .http_client(server.http_client()) - .force_path_style(true) - .build(); - let backend = crate::remote_backend::S3Backend::new( - aws_sdk_s3::Client::from_conf(conf), - "bucket".to_string(), - ); - (server, backend) + struct FailingBackend; + + #[async_trait::async_trait] + impl RemoteBackend for FailingBackend { + async fn head(&self, _key: &str) -> anyhow::Result { + anyhow::bail!("permission denied") + } + + async fn get( + &self, + _key: &str, + _max_bytes: Option, + ) -> anyhow::Result> { + unreachable!("unexpected get") + } + + async fn put( + &self, + _key: &str, + _body: Vec, + _content_type: Option<&str>, + ) -> anyhow::Result<()> { + unreachable!("unexpected put") + } + + async fn list(&self, _prefix: &str) -> anyhow::Result> { + unreachable!("unexpected list") + } + + fn describe(&self, key: &str) -> String { + format!("failing://test/{key}") + } } /// Build a one-file store entry and return (tmpdir, store, entry_dir). @@ -914,41 +922,48 @@ mod tests { } #[tokio::test] - async fn exists_entry_true_on_200_false_on_404() { + async fn exists_entry_reports_present_and_missing_objects() { let remote = test_remote(); + let backend = memory_backend(); + let layout = RemoteLayout::new(&backend, &remote); - let (server, client) = mock_client(vec![ReplayedEvent::ok()]).await; - let layout = RemoteLayout::new(&client, &remote); + backend + .put( + &v3_manifest_key(&remote.prefix, "key123", "foo"), + b"{}".to_vec(), + Some("application/json"), + ) + .await + .unwrap(); assert!(layout.exists_entry("key123", "foo").await.unwrap()); - server.shutdown(); - - let (server, client) = mock_client(vec![ReplayedEvent::status(404)]).await; - let layout = RemoteLayout::new(&client, &remote); - assert!(!layout.exists_entry("key123", "foo").await.unwrap()); - server.shutdown(); + assert!(!layout.exists_entry("missing", "foo").await.unwrap()); } #[tokio::test] async fn exists_entry_propagates_unexpected_errors() { - // A 403 is not a "missing" code, so it must surface as an error rather - // than a silent false (exercises describe_head_object_error). let remote = test_remote(); - let (server, client) = mock_client(vec![ReplayedEvent::status(403)]).await; - let layout = RemoteLayout::new(&client, &remote); - assert!(layout.exists_entry("key123", "foo").await.is_err()); - server.shutdown(); + let layout = RemoteLayout::new(&FailingBackend, &remote); + let error = layout + .exists_entry("key123", "foo") + .await + .expect_err("backend errors must not become cache misses"); + assert!(error.to_string().contains("permission denied"), "{error:#}"); } #[tokio::test] - async fn download_entry_extracts_a_served_pack() { + async fn download_entry_extracts_a_stored_pack() { let (_tmp, store, entry_dir) = populated_entry(); let meta: EntryMeta = serde_json::from_slice(&std::fs::read(entry_dir.join("meta.json")).unwrap()).unwrap(); let packed = create_entry_pack_zstd(&entry_dir, &store.blobs_dir(), &meta, 3).unwrap(); let remote = test_remote(); - let (server, client) = mock_client(vec![ReplayedEvent::with_body(&packed)]).await; - let layout = RemoteLayout::new(&client, &remote); + let backend = memory_backend(); + backend + .put(&v3_pack_key(&remote.prefix, "key123", "foo"), packed, None) + .await + .unwrap(); + let layout = RemoteLayout::new(&backend, &remote); let dest = _tmp.path().join("restored"); let result = layout @@ -960,77 +975,70 @@ mod tests { std::fs::read(dest.join("libfoo.rlib")).unwrap(), b"hello world" ); - server.shutdown(); } #[tokio::test] async fn upload_entry_puts_pack_and_manifest() { let (_tmp, store, entry_dir) = populated_entry(); let remote = test_remote(); - // upload_entry issues two PutObjects: the pack, then the manifest. - let (server, client) = mock_client(vec![ReplayedEvent::ok(), ReplayedEvent::ok()]).await; - let layout = RemoteLayout::new(&client, &remote); + let backend = memory_backend(); + let layout = RemoteLayout::new(&backend, &remote); let result = layout .upload_entry("key123", "foo", &entry_dir, &store.blobs_dir(), 3) .await .expect("upload_entry should succeed"); assert_eq!(result.format, "v3"); - server.shutdown(); - } - /// A ListBucketResult XML body listing the given object keys. - fn list_bucket_xml(keys: &[&str]) -> String { - let contents: String = keys - .iter() - .map(|k| format!("{k}10")) - .collect(); - format!( - "\ - \ - bucket{}1000\ - false{contents}", - keys.len() - ) + let pack_key = v3_pack_key(&remote.prefix, "key123", "foo"); + let manifest_key = v3_manifest_key(&remote.prefix, "key123", "foo"); + assert!(backend.head(&pack_key).await.unwrap()); + let manifest_body = backend + .get(&manifest_key, None) + .await + .unwrap() + .expect("manifest stored") + .body; + let manifest: V3Manifest = serde_json::from_slice(&manifest_body).unwrap(); + assert_eq!(manifest.pack_key, pack_key); + assert_eq!(manifest.cache_key, "key123"); + assert_eq!(manifest.crate_name, "foo"); + assert_eq!(manifest.file_count, 1); } #[tokio::test] async fn list_keys_maps_manifest_objects_to_crate_and_key() { let remote = test_remote(); - let xml = list_bucket_xml(&[ + let backend = memory_backend(); + for key in [ "artifacts/v3/manifests/serde/aaaa1111.json", "artifacts/v3/manifests/tokio/bbbb2222.json", // Non-manifest junk under the prefix is ignored (no .json suffix). "artifacts/v3/manifests/serde/notes.txt", - ]); - let (server, client) = mock_client(vec![ReplayedEvent::with_body(xml)]).await; - let layout = RemoteLayout::new(&client, &remote); + ] { + backend.put(key, vec![], None).await.unwrap(); + } + let layout = RemoteLayout::new(&backend, &remote); let keys = layout.list_keys().await.expect("list_keys should succeed"); assert_eq!(keys.get("aaaa1111").map(String::as_str), Some("serde")); assert_eq!(keys.get("bbbb2222").map(String::as_str), Some("tokio")); assert_eq!(keys.len(), 2, "non-.json objects must be ignored: {keys:?}"); - server.shutdown(); } #[tokio::test] async fn list_keys_for_crates_queries_each_crate_prefix() { let remote = test_remote(); - // One LIST per crate; HashSet iteration order is nondeterministic, so - // each response carries BOTH crates' manifests. The per-crate prefix the - // method queries (and strips) is what selects the right key — a key under - // a different crate's prefix is ignored. So the mapping is correct no - // matter which crate's LIST consumes which (identical) response. - let body = list_bucket_xml(&[ + let backend = memory_backend(); + for key in [ "artifacts/v3/manifests/serde/aaaa1111.json", "artifacts/v3/manifests/tokio/bbbb2222.json", - ]); - let (server, client) = mock_client(vec![ - ReplayedEvent::with_body(&body), - ReplayedEvent::with_body(&body), - ]) - .await; - let layout = RemoteLayout::new(&client, &remote); + // Listing only the requested crate prefixes must exclude this. + "artifacts/v3/manifests/other/cccc3333.json", + ] { + backend.put(key, vec![], None).await.unwrap(); + } + let layout = RemoteLayout::new(&backend, &remote); let mut crates = std::collections::HashSet::new(); crates.insert("serde".to_string()); @@ -1041,6 +1049,6 @@ mod tests { .expect("list_keys_for_crates should succeed"); assert_eq!(keys.get("aaaa1111").map(String::as_str), Some("serde")); assert_eq!(keys.get("bbbb2222").map(String::as_str), Some("tokio")); - server.shutdown(); + assert_eq!(keys.len(), 2); } } diff --git a/src/report.rs b/src/report.rs index 64a95caf..20d259dc 100644 --- a/src/report.rs +++ b/src/report.rs @@ -208,7 +208,7 @@ pub struct NetworkAnalysis { pub max_download_ms: u64, /// Throughput based on total wall-clock time (includes local restore work). pub throughput_mbps: f64, - /// Throughput based on network time only (S3 GET + body collection). + /// Throughput based on remote-read time only (GET + body collection). pub network_throughput_mbps: f64, /// Throughput based on response body time only. #[serde(default)] @@ -226,7 +226,7 @@ pub struct NetworkAnalysis { /// Time spent reading response bodies across GET requests. #[serde(default)] pub total_body_ms: u64, - /// Time spent waiting for S3 concurrency permits. + /// Time spent waiting for remote-operation concurrency permits. #[serde(default)] pub total_semaphore_wait_ms: u64, /// Time spent on HEAD/existence checks before downloads. @@ -554,7 +554,7 @@ pub fn generate_report_with_filter( avg_hit_ms }; - // Network + // Remote transfers let network = if transfers.is_empty() { None } else { @@ -1200,7 +1200,7 @@ fn build_network_analysis(transfers: &[TransferEvent], top: usize) -> NetworkAna 0.0 }; - // Network-only throughput (S3 GET + body collection, excludes decompress/disk) + // Remote-read throughput (request + body, excludes decompress/disk). let network_throughput_mbps = if total_network_ms > 0 { (total_download_bytes as f64 / (1024.0 * 1024.0)) / (total_network_ms as f64 / 1000.0) } else { @@ -1381,40 +1381,40 @@ fn generate_suggestions( ); } - // Network issues + // Remote-transfer issues if let Some(net) = network { let total_downloads = net.downloads_ok + net.downloads_failed; if total_downloads > 0 { let fail_rate = net.downloads_failed as f64 / total_downloads as f64 * 100.0; if fail_rate > 10.0 { suggestions.push(format!( - "{:.0}% of downloads failed — check network connectivity and S3 credentials", + "{:.0}% of downloads failed — check remote connectivity, paths, and credentials", fail_rate )); } } if net.downloads_ok > 0 && net.total_get_requests > net.downloads_ok as u32 * 3 { suggestions.push(format!( - "Downloads fan out to {:.1} GETs per cache hit — check remote layout granularity or prefer pack-first downloads on CI", + "Downloads fan out to {:.1} remote reads per cache hit — check remote layout granularity or prefer pack-first downloads on CI", net.total_get_requests as f64 / net.downloads_ok as f64 )); } if net.total_semaphore_wait_ms > 10_000 { suggestions.push(format!( - "Aggregate S3 semaphore wait totaled {} — tune concurrency only if the object store can absorb it", + "Aggregate remote semaphore wait totaled {} — tune concurrency only if the remote can absorb it", format_duration_ms(net.total_semaphore_wait_ms) )); } if net.total_request_ms > 30_000 && net.total_request_ms > net.total_body_ms { suggestions.push(format!( - "Aggregate request/header latency ({}) exceeds body transfer ({}) — check RGW/request path, connection reuse, or object fan-out", + "Aggregate remote open/setup latency ({}) exceeds read/transfer time ({}) — check the remote path, storage latency, or read fan-out", format_duration_ms(net.total_request_ms), format_duration_ms(net.total_body_ms) )); } if net.total_extract_ms > 30_000 && net.total_extract_ms > net.total_body_ms { suggestions.push(format!( - "Aggregate archive extract time ({}) exceeds body transfer ({}) — profile zstd/tar extraction and SQLite import separately", + "Aggregate archive extract time ({}) exceeds read/transfer time ({}) — profile zstd/tar extraction and SQLite import separately", format_duration_ms(net.total_extract_ms), format_duration_ms(net.total_body_ms) )); @@ -1423,11 +1423,11 @@ fn generate_suggestions( if root_filtered { suggestions.push( - "Network transfer data omitted because transfer events are not root-scoped yet" + "Remote transfer data omitted because transfer events are not root-scoped yet" .to_string(), ); } else if network.is_none() { - suggestions.push("No network transfer data available for this session".to_string()); + suggestions.push("No remote transfer data available for this session".to_string()); } suggestions @@ -1669,6 +1669,18 @@ fn trace_metadata_event(name: &str, tid: u32, value: &str) -> serde_json::Value }) } +/// Backend-neutral label for the phase names retained in the JSON telemetry. +fn human_download_phase(phase: &str) -> &str { + match phase { + "wait" => "queue wait", + "HEAD" => "existence check", + "request" => "open/setup", + "body" => "read/transfer", + "disk" => "local disk", + other => other, + } +} + pub fn format_markdown(report: &BuildReport) -> String { use crate::cli::format_duration_ms; @@ -1765,9 +1777,9 @@ pub fn format_markdown(report: &BuildReport) -> String { )); lines.push(String::new()); - // Network table + // Remote-transfer table if let Some(net) = &report.network { - lines.push("#### Network".to_string()); + lines.push("#### Remote transfer".to_string()); lines.push("| Metric | Value |".to_string()); lines.push("|---|---|".to_string()); lines.push(format!( @@ -1786,11 +1798,11 @@ pub fn format_markdown(report: &BuildReport) -> String { )); lines.push(format!("| P95 download time | {}ms |", net.p95_download_ms)); lines.push(format!( - "| Throughput (network) | {:.1} MB/s |", + "| Throughput (open + read) | {:.1} MB/s |", net.network_throughput_mbps )); lines.push(format!( - "| Throughput (body only) | {:.1} MB/s |", + "| Throughput (read only) | {:.1} MB/s |", net.body_throughput_mbps )); lines.push(format!( @@ -1800,7 +1812,7 @@ pub fn format_markdown(report: &BuildReport) -> String { if !net.dominant_download_phase.is_empty() && net.dominant_download_phase_ms > 0 { lines.push(format!( "| Dominant aggregate download phase | {} — {} ({:.1}%) |", - net.dominant_download_phase, + human_download_phase(&net.dominant_download_phase), format_duration_ms(net.dominant_download_phase_ms), net.dominant_download_phase_pct )); @@ -1821,7 +1833,7 @@ pub fn format_markdown(report: &BuildReport) -> String { || net.total_disk_io_ms > 0 { lines.push(format!( - "| Aggregate download phase time | wait {}ms, HEAD {}ms, request {}ms, body {}ms, decompress {}ms, extract {}ms, import {}ms, disk I/O {}ms |", + "| Aggregate download phase time | queue wait {}ms, existence check {}ms, open/setup {}ms, read/transfer {}ms, decompress {}ms, extract {}ms, import {}ms, local disk I/O {}ms |", net.total_semaphore_wait_ms, net.total_head_ms, net.total_request_ms, @@ -1856,7 +1868,8 @@ pub fn format_markdown(report: &BuildReport) -> String { if !net.slowest_downloads.is_empty() { lines.push("#### Slowest Downloads".to_string()); lines.push( - "| Crate | Size | Time | Key | Wait/HEAD | Req/Body | Extract/Import |".to_string(), + "| Crate | Size | Time | Key | Wait/Check | Open/Read | Extract/Import |" + .to_string(), ); lines.push("|---|---|---|---|---|---|---|".to_string()); for d in &net.slowest_downloads { @@ -2159,7 +2172,7 @@ pub fn format_github(report: &BuildReport) -> String { lines.push("".to_string()); } - // ── Network (collapsed) ── + // ── Remote transfer (collapsed) ── if let Some(net) = &report.network { let net_tp = if net.network_throughput_mbps > 0.0 { net.network_throughput_mbps @@ -2171,12 +2184,15 @@ pub fn format_github(report: &BuildReport) -> String { lines.push("
".to_string()); let dominant_summary = if !net.dominant_download_phase.is_empty() && net.dominant_download_phase_ms > 0 { - format!(", dominant aggregate {}", net.dominant_download_phase) + format!( + ", dominant aggregate {}", + human_download_phase(&net.dominant_download_phase) + ) } else { String::new() }; lines.push(format!( - "Network — {} downloaded, {:.0} MB/s body{}", + "Remote transfer — {} downloaded, {:.0} MB/s read{}", format_bytes(net.bytes_down), net.body_throughput_mbps, dominant_summary @@ -2197,7 +2213,7 @@ pub fn format_github(report: &BuildReport) -> String { )); if net.total_compression_ms > 0 || net.total_head_checks_ms > 0 { lines.push(format!( - "| Upload time split | compress {}ms + HEAD checks {}ms |", + "| Upload time split | compress {}ms + existence checks {}ms |", net.total_compression_ms, net.total_head_checks_ms, )); } @@ -2219,18 +2235,18 @@ pub fn format_github(report: &BuildReport) -> String { if net.total_get_requests > 0 { let req_per_download = net.total_get_requests as f64 / net.downloads_ok.max(1) as f64; lines.push(format!( - "| GET fan-out | {} GETs total · {:.1} per download |", + "| Read fan-out | {} reads total · {:.1} per download |", net.total_get_requests, req_per_download )); } lines.push(format!( - "| Throughput | {:.1} MB/s body · {:.1} MB/s request+body · {:.1} MB/s end-to-end |", + "| Throughput | {:.1} MB/s read · {:.1} MB/s open+read · {:.1} MB/s end-to-end |", net.body_throughput_mbps, net_tp, net.throughput_mbps )); if !net.dominant_download_phase.is_empty() && net.dominant_download_phase_ms > 0 { lines.push(format!( "| Dominant aggregate download phase | {} — {} ({:.1}%) |", - net.dominant_download_phase, + human_download_phase(&net.dominant_download_phase), format_duration_ms(net.dominant_download_phase_ms), net.dominant_download_phase_pct )); @@ -2251,7 +2267,7 @@ pub fn format_github(report: &BuildReport) -> String { || net.total_disk_io_ms > 0 { lines.push(format!( - "| Aggregate download phase time | wait {}ms · HEAD {}ms · request {}ms · body {}ms · decompress {}ms · extract {}ms · import {}ms · disk {}ms |", + "| Aggregate download phase time | queue wait {}ms · existence check {}ms · open/setup {}ms · read/transfer {}ms · decompress {}ms · extract {}ms · import {}ms · local disk {}ms |", net.total_semaphore_wait_ms, net.total_head_ms, net.total_request_ms, @@ -2286,7 +2302,7 @@ pub fn format_github(report: &BuildReport) -> String { lines.push("**Slowest downloads:**".to_string()); lines.push(String::new()); lines.push( - "| Crate | Fmt | Size | Time | GETs | Key | Wait/HEAD | Req/Body | Extract/Import |" + "| Crate | Fmt | Size | Time | Reads | Key | Wait/Check | Open/Read | Extract/Import |" .to_string(), ); lines.push( @@ -2523,9 +2539,9 @@ pub fn format_text(report: &BuildReport) -> String { } lines.push(String::new()); - // Network + // Remote transfers if let Some(net) = &report.network { - lines.push("Network:".to_string()); + lines.push("Remote transfer:".to_string()); lines.push(format!( " Downloaded: {} ({} ok, {} failed)", format_bytes(net.bytes_down), @@ -2543,13 +2559,13 @@ pub fn format_text(report: &BuildReport) -> String { net.avg_download_ms, net.p95_download_ms, net.max_download_ms )); lines.push(format!( - " Throughput: {:.1} MB/s body, {:.1} MB/s request+body, {:.1} MB/s incl. restore", + " Throughput: {:.1} MB/s read, {:.1} MB/s open+read, {:.1} MB/s incl. restore", net.body_throughput_mbps, net.network_throughput_mbps, net.throughput_mbps )); if !net.dominant_download_phase.is_empty() && net.dominant_download_phase_ms > 0 { lines.push(format!( " Dominant aggregate phase: {} — {} ({:.1}%)", - net.dominant_download_phase, + human_download_phase(&net.dominant_download_phase), format_duration_ms(net.dominant_download_phase_ms), net.dominant_download_phase_pct )); @@ -2570,7 +2586,7 @@ pub fn format_text(report: &BuildReport) -> String { || net.total_disk_io_ms > 0 { lines.push(format!( - " Aggregate phase time: wait {}ms, HEAD {}ms, request {}ms, body {}ms, decompress {}ms, extract {}ms, import {}ms, disk I/O {}ms", + " Aggregate phase time: queue wait {}ms, existence check {}ms, open/setup {}ms, read/transfer {}ms, decompress {}ms, extract {}ms, import {}ms, local disk I/O {}ms", net.total_semaphore_wait_ms, net.total_head_ms, net.total_request_ms, @@ -3173,7 +3189,7 @@ mod tests { report .suggestions .iter() - .any(|s| s.contains("Network transfer data omitted")) + .any(|s| s.contains("Remote transfer data omitted")) ); assert!(report.all_events.iter().all(|event| event.root == root)); assert!( @@ -3273,7 +3289,7 @@ mod tests { assert!(md.contains("### kache build report")); assert!(md.contains("#### Summary")); assert!(md.contains("#### Timing")); - assert!(md.contains("#### Network")); + assert!(md.contains("#### Remote transfer")); assert!(md.contains("#### Prefetch")); assert!(md.contains("#### Passthroughs & Skips")); assert!(md.contains("#### Top Compiled Cache-Key Misses")); @@ -3318,7 +3334,12 @@ mod tests { let report = generate_report(&config, 24, 10).unwrap(); assert!(report.network.is_none()); - assert!(report.suggestions.iter().any(|s| s.contains("No network"))); + assert!( + report + .suggestions + .iter() + .any(|s| s.contains("No remote transfer")) + ); } #[test] @@ -3486,16 +3507,16 @@ mod tests { report.suggestions ); assert!( - joined.contains("GETs per cache hit"), - "expected GET fan-out suggestion: {:?}", + joined.contains("remote reads per cache hit"), + "expected remote-read fan-out suggestion: {:?}", report.suggestions ); } #[test] fn test_suggestion_network_latency_thresholds() { - // A download with high semaphore-wait, request/header latency exceeding - // body transfer, and extract time exceeding body transfer triggers the + // A download with high semaphore wait, open/setup latency exceeding + // read/transfer time, and extract time exceeding read/transfer time triggers the // three latency-threshold suggestions (report.rs 999-1018) that the // fixed-ratio test_transfer helper can't reach. let dir = tempfile::tempdir().unwrap(); @@ -3565,8 +3586,8 @@ mod tests { report.suggestions ); assert!( - joined.contains("request/header latency"), - "expected request-latency suggestion: {:?}", + joined.contains("remote open/setup latency"), + "expected open-latency suggestion: {:?}", report.suggestions ); assert!( @@ -3598,13 +3619,13 @@ mod tests { assert!(gh.contains("Passthroughs & skips")); assert!(gh.contains("via fallback")); assert!(gh.contains("refused: unsupported rustc invocation")); - assert!(gh.contains("Network")); + assert!(gh.contains("Remote transfer")); assert!(gh.contains("Timing & Prefetch")); assert!(gh.contains("Download format")); - assert!(gh.contains("GET fan-out")); + assert!(gh.contains("Read fan-out")); assert!(gh.contains("v3 3")); - assert!(gh.contains("request")); - assert!(gh.contains("body")); + assert!(gh.contains("open")); + assert!(gh.contains("read")); } #[test] @@ -3617,7 +3638,7 @@ mod tests { assert!(text.contains("kache build report")); assert!(text.contains("hit rate")); assert!(text.contains("Timing:")); - assert!(text.contains("Network:")); + assert!(text.contains("Remote transfer:")); assert!(text.contains("Passthroughs/skips:")); } @@ -3755,7 +3776,7 @@ mod tests { format_text(&report), ] { let lower = rendered.to_lowercase(); - // Upload row (uploads_ok > 0) and its compress/HEAD split. + // Upload row (uploads_ok > 0) and its compression/existence split. assert!( lower.contains("upload"), "missing upload section: {rendered}" diff --git a/src/tui.rs b/src/tui.rs index e5bdd515..d9b93ea3 100644 --- a/src/tui.rs +++ b/src/tui.rs @@ -1226,7 +1226,7 @@ fn draw_projects_overview(frame: &mut Frame, state: &AppState, area: Rect) { let wrapper_status = crate::wrapper_config::wrapper_status_line(); let remote_status = if let Some(remote) = &state.config.remote { - format!("S3: {}", remote.bucket) + remote.describe() } else { "not configured".to_string() }; @@ -1564,7 +1564,7 @@ fn draw_transfer_activity(frame: &mut Frame, state: &AppState, area: Rect) { Style::default() }, ), - Span::raw(format!(" S3 slots: {s3_slots}")), + Span::raw(format!(" Remote slots: {s3_slots}")), ]), Line::from(vec![ Span::styled(" Speed: ", Style::default().fg(Color::Cyan)), @@ -2155,13 +2155,7 @@ mod tests { let mut state = test_state(); state.active_tab = Tab::Build; state.service_installed = true; - state.config.remote = Some(crate::config::RemoteConfig { - bucket: "b".into(), - endpoint: None, - region: "us-east-1".into(), - prefix: "p".into(), - profile: None, - }); + state.config.remote = Some(crate::config::RemoteConfig::test_s3("b", "p")); state.stats_loaded = true; let snap = &mut state.stats_snapshot; diff --git a/src/wrapper.rs b/src/wrapper.rs index a3a78ad8..a3bed436 100644 --- a/src/wrapper.rs +++ b/src/wrapper.rs @@ -3002,13 +3002,10 @@ mod tests { let mut config = test_config(temp.path().to_path_buf()); // Enable remote so prefetch actually triggers its path - config.remote = Some(crate::config::RemoteConfig { - bucket: "test-bucket".to_string(), - endpoint: Some("http://localhost".to_string()), - region: "us-east-1".to_string(), - prefix: "kache/".to_string(), - profile: None, - }); + config.remote = Some(crate::config::RemoteConfig::test_s3( + "test-bucket", + "kache/", + )); // Use dummy args let args = rustc_args(&["rustc", "foo.rs"]);