diff --git a/Cargo.lock b/Cargo.lock index bdbd43576308d..c5e77ceb74e64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "RustyXML" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" - [[package]] name = "addr2line" version = "0.25.1" @@ -524,30 +518,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" dependencies = [ - "event-listener 5.4.1", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" -dependencies = [ - "concurrent-queue", + "event-listener", "event-listener-strategy", "futures-core", "pin-project-lite", @@ -570,32 +541,13 @@ dependencies = [ "zstd-safe", ] -[[package]] -name = "async-io" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" -dependencies = [ - "async-lock", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.6.0", - "parking", - "polling", - "rustix 0.38.44", - "slab", - "tracing", - "windows-sys 0.59.0", -] - [[package]] name = "async-lock" version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 5.4.1", + "event-listener", "event-listener-strategy", "pin-project-lite", ] @@ -612,25 +564,6 @@ dependencies = [ "url", ] -[[package]] -name = "async-process" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" -dependencies = [ - "async-channel 2.3.1", - "async-io", - "async-lock", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener 5.4.1", - "futures-lite 2.6.0", - "rustix 0.38.44", - "tracing", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -642,24 +575,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "async-signal" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" -dependencies = [ - "async-io", - "async-lock", - "atomic-waker", - "cfg-if", - "futures-core", - "futures-io", - "rustix 0.38.44", - "signal-hook-registry", - "slab", - "windows-sys 0.59.0", -] - [[package]] name = "async-stream" version = "0.3.6" @@ -682,12 +597,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - [[package]] name = "async-trait" version = "0.1.89" @@ -763,7 +672,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand", "hex", "http 1.4.2", "sha1", @@ -826,7 +735,7 @@ dependencies = [ "aws-types", "bytes", "bytes-utils", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "http-body 0.4.6", @@ -854,7 +763,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -878,7 +787,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -906,7 +815,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand", "hex", "hmac", "http 0.2.12", @@ -937,7 +846,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -961,7 +870,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -985,7 +894,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -1010,7 +919,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -1206,7 +1115,7 @@ dependencies = [ "aws-smithy-schema", "aws-smithy-types", "bytes", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.4.2", "http-body 0.4.6", @@ -1410,109 +1319,74 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.21.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +checksum = "4e41cbd819986ba41904c207d8ffc4106f8f8352a548d773e9554906379bb2fb" dependencies = [ + "async-lock", "async-trait", - "base64 0.22.1", + "azure_core_macros", "bytes", - "dyn-clone", "futures", - "getrandom 0.2.16", "hmac", - "http-types", - "once_cell", - "paste", "pin-project", - "quick-xml 0.31.0", - "rand 0.8.5", - "reqwest 0.12.28", "rustc_version", "serde", "serde_json", "sha2", - "time", + "tokio", "tracing", - "url", - "uuid", + "typespec", + "typespec_client_core", ] [[package]] -name = "azure_identity" -version = "0.21.0" +name = "azure_core_macros" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddd80344317c40c04b603807b63a5cefa532f1b43522e72f480a988141f744" +checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" dependencies = [ - "async-lock", - "async-process", - "async-trait", - "azure_core", - "futures", - "oauth2", - "pin-project", - "serde", - "time", + "proc-macro2", + "quote", + "syn 2.0.119", "tracing", - "tz-rs", - "url", - "uuid", ] [[package]] -name = "azure_storage" -version = "0.21.0" +name = "azure_identity" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +checksum = "32edf96b356ca7c51d7590c4925cc36efc3947a5da4468e8e0b25c56ecbb3de5" dependencies = [ - "RustyXML", "async-lock", "async-trait", "azure_core", - "bytes", - "serde", - "serde_derive", - "time", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "azure_storage_blobs" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" -dependencies = [ - "RustyXML", - "azure_core", - "azure_storage", - "azure_svc_blobstorage", - "bytes", "futures", + "pin-project", "serde", - "serde_derive", "serde_json", "time", "tracing", "url", - "uuid", ] [[package]] -name = "azure_svc_blobstorage" -version = "0.21.0" +name = "azure_storage_blob" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" +checksum = "1756febbcca86c862ef718b983b505d08bd65a9bc984a915b0a16af4a4c3fe5b" dependencies = [ + "async-stream", + "async-trait", "azure_core", "bytes", "futures", - "log", - "once_cell", + "percent-encoding", + "pin-project", "serde", "serde_json", "time", + "tokio", ] [[package]] @@ -1521,7 +1395,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "fastrand 2.3.0", + "fastrand", "gloo-timers", "tokio", ] @@ -1711,19 +1585,6 @@ dependencies = [ "objc2", ] -[[package]] -name = "blocking" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" -dependencies = [ - "async-channel 2.3.1", - "async-task", - "futures-io", - "futures-lite 2.6.0", - "piper", -] - [[package]] name = "bnum" version = "0.12.1" @@ -2370,12 +2231,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "const_fn" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" - [[package]] name = "cookie" version = "0.18.1" @@ -3153,7 +3008,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -3501,12 +3356,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "event-listener" version = "5.4.1" @@ -3524,7 +3373,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener 5.4.1", + "event-listener", "pin-project-lite", ] @@ -3616,15 +3465,6 @@ dependencies = [ "serde", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -3919,34 +3759,6 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-lite" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" -dependencies = [ - "fastrand 2.3.0", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.33" @@ -4053,17 +3865,6 @@ dependencies = [ "unicode-width 0.2.2", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.16" @@ -4098,9 +3899,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -4504,26 +4307,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-types" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" -dependencies = [ - "anyhow", - "async-channel 1.9.0", - "base64 0.13.1", - "futures-lite 1.13.0", - "infer", - "pin-project-lite", - "rand 0.7.3", - "serde", - "serde_json", - "serde_qs", - "serde_urlencoded", - "url", -] - [[package]] name = "httparse" version = "1.10.1" @@ -4703,7 +4486,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.5", "system-configuration", "tokio", "tower-service", @@ -4941,12 +4724,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "infer" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" - [[package]] name = "inout" version = "0.1.4" @@ -4970,15 +4747,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "integer-encoding" version = "3.0.4" @@ -5032,7 +4800,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -5119,7 +4887,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -5804,6 +5572,12 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lsp-types" version = "0.94.1" @@ -5958,7 +5732,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "equivalent", - "event-listener 5.4.1", + "event-listener", "futures-util", "parking_lot", "portable-atomic", @@ -7788,8 +7562,7 @@ dependencies = [ "aws-types", "azure_core", "azure_identity", - "azure_storage", - "azure_storage_blobs", + "azure_storage_blob", "base64 0.22.1", "bytes", "deadpool-postgres", @@ -7814,12 +7587,11 @@ dependencies = [ "prost", "prost-build", "rand 0.9.4", - "reqwest 0.12.28", + "reqwest 0.13.4", "serde", "serde_json", "sha2", "tempfile", - "time", "timely", "tokio", "tokio-postgres", @@ -9373,40 +9145,12 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - [[package]] name = "numa_maps" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "919af26e03f635d91244e3e3e5a63f0377a78044b39e09a69fdad36514dc919f" -[[package]] -name = "oauth2" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" -dependencies = [ - "base64 0.13.1", - "chrono", - "getrandom 0.2.16", - "http 0.2.12", - "rand 0.8.5", - "serde", - "serde_json", - "serde_path_to_error", - "sha2", - "thiserror 1.0.69", - "url", -] - [[package]] name = "objc2" version = "0.6.4" @@ -9750,7 +9494,7 @@ dependencies = [ "bytes", "http 1.4.2", "opentelemetry", - "reqwest 0.13.3", + "reqwest 0.13.4", ] [[package]] @@ -9765,7 +9509,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.13.3", + "reqwest 0.13.4", "thiserror 2.0.18", "tokio", "tonic", @@ -10196,7 +9940,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cbb1126afed61dd6368748dae63b1ee7dc480191c6262a3b4ff1e29d86a6c5b" dependencies = [ - "fastrand 2.3.0", + "fastrand", "phf_shared 0.12.1", ] @@ -10206,7 +9950,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ - "fastrand 2.3.0", + "fastrand", "phf_shared 0.13.1", ] @@ -10290,17 +10034,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" -dependencies = [ - "atomic-waker", - "fastrand 2.3.0", - "futures-io", -] - [[package]] name = "pkcs1" version = "0.7.5" @@ -10373,21 +10106,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polling" -version = "3.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53a684391ad002dd6a596ceb6c74fd004fdce75f4be2e3f615068abbea5fd50" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi", - "pin-project-lite", - "rustix 1.1.4", - "tracing", - "windows-sys 0.59.0", -] - [[package]] name = "portable-atomic" version = "1.13.1" @@ -10738,7 +10456,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -10759,7 +10477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -10937,7 +10655,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", - "serde", ] [[package]] @@ -10960,6 +10677,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quickcheck" version = "1.0.3" @@ -10969,6 +10696,63 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.5", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.5", + "tracing", + "windows-sys 0.61.1", +] + [[package]] name = "quote" version = "1.0.47" @@ -10996,19 +10780,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -11042,16 +10813,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -11072,15 +10833,6 @@ dependencies = [ "rand_core 0.9.3", ] -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -11117,12 +10869,12 @@ dependencies = [ ] [[package]] -name = "rand_hc" -version = "0.2.0" +name = "rand_pcg" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core 0.5.1", + "rand_core 0.10.1", ] [[package]] @@ -11398,15 +11150,15 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.1", "web-sys", ] [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", @@ -11417,6 +11169,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.9.0", + "hyper-rustls", "hyper-tls 0.6.0", "hyper-util", "js-sys", @@ -11424,18 +11177,24 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", + "tokio-util", "tower 0.5.3", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -11724,7 +11483,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -11761,9 +11520,37 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.1", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -11992,7 +11779,7 @@ dependencies = [ "cfg_aliases", "httpdate", "native-tls", - "reqwest 0.13.3", + "reqwest 0.13.4", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -12215,17 +12002,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" -dependencies = [ - "percent-encoding", - "serde", - "thiserror 1.0.69", -] - [[package]] name = "serde_repr" version = "0.1.20" @@ -12860,11 +12636,11 @@ version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "fastrand 2.3.0", - "getrandom 0.3.3", + "fastrand", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -12883,7 +12659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -13025,10 +12801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "js-sys", - "libc", "num-conv", - "num_threads", "powerfmt", "quickcheck", "serde_core", @@ -13842,6 +13615,58 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "typespec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "753a2fe021e407d4fc9ee6f4f0a33403cc306d5c54c4e4ebe1b8cbde0ca052b9" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures", + "quick-xml 0.41.0", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0373af0f9d4f580b3a1a9d9639cedaabe015ed262b35bfbe13941bfb14fe1ea6" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "pin-project", + "rand 0.10.2", + "reqwest 0.13.4", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c608f4427943f8adb211abc95c87672b1b98847152783507d54e3246e502f60" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "typetag" version = "0.2.21" @@ -13866,15 +13691,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "tz-rs" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" -dependencies = [ - "const_fn", -] - [[package]] name = "ucd-trie" version = "0.1.6" @@ -14144,12 +13960,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waker-fn" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" - [[package]] name = "walkdir" version = "2.5.0" @@ -14170,12 +13980,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -14265,6 +14069,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmtimer" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index 2752a257f242e..f5eebfd2b5762 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -302,10 +302,14 @@ aws-types = "1.3.9" axum = { version = "0.8.9", features = ["ws"] } axum-extra = { version = "0.12.5", features = ["typed-header"] } axum-server = { version = "0.8.0", features = ["tls-rustls"] } -azure_core = "0.21.0" -azure_identity = "0.21.0" -azure_storage = "0.21.0" -azure_storage_blobs = "0.21.0" +# The GA (1.x) Azure SDK. It transports requests over reqwest 0.13, pulled in +# via `typespec_client_core`; see `reqwest_0_13` below. +# +# `hmac_rust` provides `azure_core::hmac::hmac_sha256`, which signs the Azurite +# requests our tests make. +azure_core = { version = "1.1.0", features = ["hmac_rust"] } +azure_identity = "1.0.0" +azure_storage_blob = "1.0.0" backtrace = "0.3.76" base64 = "0.22.1" bincode = "1.3.3" @@ -474,6 +478,12 @@ rdkafka-sys = { version = "4.3.0", features = ["cmake-build", "libz-static", "ss regex = "1.12.3" regex-syntax = "0.8.10" reqwest = { version = "0.12.28", features = ["blocking", "charset", "cookies", "default-tls", "http2", "json", "native-tls-vendored", "stream"] } +# The Azure SDK implements its `HttpClient` transport trait for reqwest 0.13 +# only, so persist's custom Azure transport client cannot be built from the +# workspace `reqwest` above. Naming the 0.13 major explicitly keeps the two +# uses from being confused at the call site. `rustls` puts Azure traffic on +# rustls + aws-lc-rs, matching the SDK's own default transport. +reqwest_0_13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["rustls", "stream"] } reqwest-middleware = { version = "0.4.2", features = ["json"] } reqwest-retry = "0.8.0" rlimit = "0.11.0" diff --git a/deny.toml b/deny.toml index 0b1f46fc16a98..8bbf96e730ea9 100644 --- a/deny.toml +++ b/deny.toml @@ -71,17 +71,9 @@ skip = [ { name = "memmap2", version = "0.5.4" }, - { name = "async-channel", version = "1.9.0" }, - { name = "event-listener", version = "2.5.3" }, - { name = "fastrand", version = "1.9.0" }, - { name = "futures-lite", version = "1.13.0" }, - { name = "getrandom", version = "0.1.16" }, { name = "getrandom", version = "0.2.10" }, - { name = "rand", version = "0.7.3" }, { name = "rand", version = "0.8.5" }, - { name = "rand_chacha", version = "0.2.2" }, { name = "rand_chacha", version = "0.3.0" }, - { name = "rand_core", version = "0.5.1" }, { name = "rand_core", version = "0.6.2" }, # Used by reqwest { name = "thiserror", version = "1.0.61" }, @@ -100,14 +92,14 @@ skip = [ { name = "nix", version = "0.26.4" }, # Held back by mysql_async `v0.36.1` { name = "socket2", version = "0.5.10" }, - # Used by azure_core + # Used by junit-report { name = "quick-xml", version = "0.31.0" }, # Used by reqsign (via iceberg); opendal pulls quick-xml 0.38 { name = "quick-xml", version = "0.37.5" }, # Used by opendal via iceberg { name = "quick-xml", version = "0.38.4" }, - # Used by sentry - { name = "reqwest", version = "0.13.3" }, + # Used by sentry and the Azure SDK; see `reqwest_0_13` in Cargo.toml. + { name = "reqwest", version = "0.13.4" }, # Conflicts between `bon` in apache-avro and `derive_builder` in iceberg { name = "darling_macro", version = "0.20.11" }, { name = "darling_core", version = "0.20.11" }, @@ -198,7 +190,6 @@ name = "strum-macros" [[bans.deny]] name = "log" wrappers = [ - "azure_svc_blobstorage", "apache-avro", "bindgen", "buildid", @@ -221,6 +212,8 @@ wrappers = [ "reqsign", "reqwest", "rustls", + # Pulled in by reqwest 0.13's rustls backend, which the Azure SDK uses. + "rustls-platform-verifier", "sqlparser", "tokio-postgres", "tokio-tungstenite", @@ -270,10 +263,6 @@ ignore = [ # unmaintained. Pulled in transitively via `tabled` -> `tabled_derive`. No # safe upgrade is available. "RUSTSEC-2026-0173", - # `http-types`'s `Authorization::value`/`WwwAuthenticate::value` can violate - # ASCII invariants. The crate is unmaintained and pulled in transitively via - # the `azure_*` crates. No safe upgrade is available. - "RUSTSEC-2026-0174", # `derivative` is unmaintained; consider using an alternative (unmaintained) "RUSTSEC-2024-0388", # The creator of the crate `paste` has stated in the [`README.md`](https://github.com/dtolnay/paste/blob/master/README.md) that this project is not longer maintained as well as archived the repository @@ -281,7 +270,7 @@ ignore = [ # Bincode is unmaintained, the team considers version 1.3.3 a complete version of bincode that is not in need of any updates. "RUSTSEC-2025-0141", # rand unsoundness when a custom logger accesses ThreadRng during reseeding. - # Affects transitive deps (rand 0.7.3 via http-types, rand 0.8.5 via fail/quickcheck/etc). + # Affects transitive deps (rand 0.8.5 via fail/quickcheck/etc). "RUSTSEC-2026-0097", # Allows private key leakage through timing information # rsa is required through iceberg-storage-opendal diff --git a/src/persist/Cargo.toml b/src/persist/Cargo.toml index f0e3f092985ad..3894531be05d1 100644 --- a/src/persist/Cargo.toml +++ b/src/persist/Cargo.toml @@ -30,10 +30,9 @@ aws-config.workspace = true aws-credential-types.workspace = true aws-sdk-s3.workspace = true aws-types.workspace = true -azure_identity.workspace = true -azure_storage.workspace = true -azure_storage_blobs.workspace = true azure_core.workspace = true +azure_identity.workspace = true +azure_storage_blob.workspace = true base64.workspace = true bytes.workspace = true deadpool-postgres.workspace = true @@ -56,10 +55,10 @@ proptest.workspace = true proptest-derive.workspace = true prost.workspace = true rand = { workspace = true, features = ["small_rng"] } -reqwest.workspace = true +# Transport client for the Azure SDK; see the workspace Cargo.toml. +reqwest_0_13.workspace = true serde.workspace = true serde_json = { workspace = true, optional = true } -time.workspace = true timely.workspace = true tokio.workspace = true tokio-postgres.workspace = true diff --git a/src/persist/src/azure.rs b/src/persist/src/azure.rs index f48658c789a6b..1e13454ff515c 100644 --- a/src/persist/src/azure.rs +++ b/src/persist/src/azure.rs @@ -9,314 +9,158 @@ //! An Azure Blob Storage implementation of [Blob] storage. -use anyhow::{Context, anyhow}; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::anyhow; use async_trait::async_trait; -use azure_core::auth::{AccessToken, TokenCredential}; +use azure_core::credentials::{AccessToken, TokenCredential, TokenRequestOptions}; use azure_core::error::ErrorKind; -use azure_core::{ExponentialRetryOptions, RetryOptions, StatusCode, TransportOptions}; +use azure_core::http::headers::HeaderName; +use azure_core::http::{ExponentialRetryOptions, RetryOptions, StatusCode, Transport}; use azure_identity::{ - TokenCredentialOptions, create_default_credential, federated_credentials_flow, + DeveloperToolsCredential, ManagedIdentityCredential, WorkloadIdentityCredential, +}; +use azure_storage_blob::models::{ + BlobClientDownloadResult, BlobClientGetPropertiesResultHeaders, + BlobContainerClientListBlobsOptions, }; -use azure_storage::{CloudLocation, EMULATOR_ACCOUNT, prelude::*}; -use azure_storage_blobs::blob::operations::GetBlobResponse; -use azure_storage_blobs::prelude::*; +use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; use bytes::Bytes; -use futures_util::future::BoxFuture; -use futures_util::stream::FuturesOrdered; -use futures_util::{FutureExt, StreamExt}; -use std::collections::BTreeMap; -use std::fmt::{Debug, Formatter}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::Duration; -use time::OffsetDateTime; -use tokio::sync::RwLock; -use tracing::{info, warn}; +use futures_util::{StreamExt, TryStreamExt}; +use tracing::info; use url::Url; use uuid::Uuid; use mz_ore::bytes::SegmentedBytes; use mz_ore::cast::CastFrom; use mz_ore::metrics::MetricsRegistry; -use mz_ore::task::AbortOnDropHandle; use crate::cfg::BlobKnobs; use crate::error::Error; use crate::location::{Blob, BlobMetadata, Determinate, ExternalError}; use crate::metrics::S3BlobMetrics; -/// Environment variables that configure AKS-style workload identity. The -/// names match the ones `azure_identity`'s credential chain reads. -const AZURE_TENANT_ID: &str = "AZURE_TENANT_ID"; -const AZURE_CLIENT_ID: &str = "AZURE_CLIENT_ID"; -const AZURE_FEDERATED_TOKEN: &str = "AZURE_FEDERATED_TOKEN"; -const AZURE_FEDERATED_TOKEN_FILE: &str = "AZURE_FEDERATED_TOKEN_FILE"; - -/// Time before an access token's expiry at which its refresh task fetches a -/// replacement, so requests keep being served from an unexpired token while -/// the refresh round trip to AAD is in flight. -const TOKEN_REFRESH_BUFFER: Duration = Duration::from_secs(5 * 60); - -/// Minimum time a refresh task waits between fetch attempts once a refresh -/// is due. This paces retries after failures, e.g. when AAD is transiently -/// unreachable, and prevents hot-looping if issued tokens are already within -/// [TOKEN_REFRESH_BUFFER] of expiry. -const TOKEN_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(10); - -/// Exchanges a client assertion (the projected service account token) for an -/// AAD access token with the given scopes. -type ExchangeFn = Arc< - dyn Fn(String, Vec) -> BoxFuture<'static, azure_core::Result> - + Send - + Sync, ->; - -/// A shared slot holding the current access token for one scope set. -type TokenSlot = Arc>; - -/// A [TokenCredential] for AKS-style workload identity that re-reads the -/// projected service account token file on every AAD access token refresh. -/// -/// `azure_identity`'s `WorkloadIdentityCredential` reads -/// `AZURE_FEDERATED_TOKEN_FILE` once at construction and holds the contents -/// for the life of the process. Kubernetes rotates the projected token, so -/// once the last cached AAD access token expires, every refresh presents an -/// expired client assertion and fails, permanently locking a long-running -/// process out of blob storage. Deferring the file read to refresh time picks -/// up rotations. -struct RefreshingWorkloadIdentityCredential { - federated_token_file: PathBuf, - exchange: ExchangeFn, - /// One token slot and refresh task per requested scope set. The task - /// keeps the slot fresh, so [TokenCredential::get_token] only blocks on - /// the first use of a scope set. - cache: RwLock, (TokenSlot, AbortOnDropHandle<()>)>>, - refresh_buffer: Duration, - retry_interval: Duration, -} +mod azurite; -impl Debug for RefreshingWorkloadIdentityCredential { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RefreshingWorkloadIdentityCredential") - .field("federated_token_file", &self.federated_token_file) - .finish_non_exhaustive() - } -} +/// A [TokenCredential] that tries each of its sources in order and returns the +/// first token one of them produces. +/// +/// `azure_identity` 1.x offers no chaining credential, and the sources cannot +/// be picked once at construction time instead: [ManagedIdentityCredential] +/// constructs successfully on any host, so an eager choice would shadow the +/// developer-tools fallback on a laptop. Sources are re-tried on every token +/// request rather than latching onto the first that worked, so an identity that +/// becomes available later is picked up. +#[derive(Debug)] +struct ChainedTokenCredential(Vec>); -impl RefreshingWorkloadIdentityCredential { - /// Returns a credential if the workload identity environment variables - /// are present, or `None` to indicate that a different credential type - /// must be used. - fn from_env() -> Option> { - // A token provided directly via AZURE_FEDERATED_TOKEN is static, so - // there is nothing to re-read. `azure_identity`'s credential chain - // prefers it over the token file, defer to it to preserve that - // precedence. - if std::env::var(AZURE_FEDERATED_TOKEN).is_ok() { - return None; +#[async_trait] +impl TokenCredential for ChainedTokenCredential { + async fn get_token( + &self, + scopes: &[&str], + options: Option>, + ) -> azure_core::Result { + let mut errors = Vec::new(); + for source in &self.0 { + match source.get_token(scopes, options.clone()).await { + Ok(token) => return Ok(token), + Err(err) => errors.push(format!("{source:?}: {err}")), + } } - let (Ok(tenant_id), Ok(client_id), Ok(token_file)) = ( - std::env::var(AZURE_TENANT_ID), - std::env::var(AZURE_CLIENT_ID), - std::env::var(AZURE_FEDERATED_TOKEN_FILE), - ) else { - return None; - }; - Some(Self::new(tenant_id, client_id, PathBuf::from(token_file))) - } - - fn new( - tenant_id: String, - client_id: String, - federated_token_file: PathBuf, - ) -> azure_core::Result { - let options = TokenCredentialOptions::default(); - let http_client = options.http_client(); - let authority_host = options.authority_host()?; - let exchange: ExchangeFn = Arc::new(move |assertion, scopes| { - let http_client = Arc::clone(&http_client); - let authority_host = authority_host.clone(); - let tenant_id = tenant_id.clone(); - let client_id = client_id.clone(); - async move { - let scopes: Vec<&str> = scopes.iter().map(String::as_str).collect(); - let res = federated_credentials_flow::perform( - http_client, - &client_id, - &assertion, - &scopes, - &tenant_id, - &authority_host, + Err(azure_core::Error::with_message_fn( + ErrorKind::Credential, + || { + format!( + "no Azure credential produced a token:\n{}", + errors.join("\n") ) - .await - .map_err(|err| { - azure_core::error::Error::full( - ErrorKind::Credential, - err, - "request token error", - ) - })?; - Ok(AccessToken::new( - res.access_token().clone(), - OffsetDateTime::now_utc() + Duration::from_secs(res.expires_in), - )) - } - .boxed() - }); - Ok(Self::with_exchange( - federated_token_file, - exchange, - TOKEN_REFRESH_BUFFER, - TOKEN_REFRESH_RETRY_INTERVAL, + }, )) } - - fn with_exchange( - federated_token_file: PathBuf, - exchange: ExchangeFn, - refresh_buffer: Duration, - retry_interval: Duration, - ) -> Self { - Self { - federated_token_file, - exchange, - cache: RwLock::new(BTreeMap::new()), - refresh_buffer, - retry_interval, - } - } -} - -/// Reads the projected service account token file and exchanges its contents -/// for an AAD access token. -async fn fetch_token( - federated_token_file: &Path, - exchange: &ExchangeFn, - scopes: Vec, -) -> azure_core::Result { - let assertion = tokio::fs::read_to_string(federated_token_file) - .await - .map_err(|err| { - azure_core::error::Error::full( - ErrorKind::Credential, - err, - format!( - "failed to read federated token from file {}", - federated_token_file.display() - ), - ) - })?; - // Kubernetes writes the projected token without surrounding whitespace, - // but a hand-provisioned file may have a trailing newline, which would - // corrupt the client assertion. - (exchange)(assertion.trim().to_string(), scopes).await } -/// Keeps `slot` holding an unexpired token by fetching a replacement within -/// `refresh_buffer` of the current token's expiry. A failed fetch leaves the -/// current token in place and is retried after `retry_interval`. -async fn refresh_task( - federated_token_file: PathBuf, - exchange: ExchangeFn, - slot: TokenSlot, - scopes: Vec, - refresh_buffer: Duration, - retry_interval: Duration, -) { - loop { - let refresh_at = slot.read().expect("lock poisoned").expires_on - refresh_buffer; - let wait = refresh_at - OffsetDateTime::now_utc(); - let wait = if wait.is_positive() { - wait.unsigned_abs() - } else { - Duration::ZERO - }; - tokio::time::sleep(wait.max(retry_interval)).await; - match fetch_token(&federated_token_file, &exchange, scopes.clone()).await { - Ok(token) => *slot.write().expect("lock poisoned") = token, - Err(err) => { - warn!("failed to refresh Azure workload identity token, will retry: {err}") - } - } +/// Returns the token credential to use when the blob URL carries no SAS token. +/// +/// The chain mirrors the credential types the SDK's own default chain covered +/// before it was removed in 1.x: workload identity (AKS), managed identity +/// (App Service and VM/IMDS), then the local developer tools (`az login`, +/// `azd auth login`). Client secrets read from the environment are not +/// included; 1.x dropped that credential type, and we never provisioned one. +fn token_credential() -> Arc { + let mut sources: Vec> = Vec::new(); + // Construction fails when the credential's environment is absent, e.g. + // workload identity outside of a pod with a projected token. Log and skip: + // a later source may still authenticate. + match WorkloadIdentityCredential::new(None) { + Ok(credential) => sources.push(credential), + Err(err) => info!("azure: workload identity credentials unavailable: {err}"), } -} - -#[async_trait] -impl TokenCredential for RefreshingWorkloadIdentityCredential { - async fn get_token(&self, scopes: &[&str]) -> azure_core::Result { - let scopes_key: Vec = scopes.iter().map(ToString::to_string).collect(); - - { - let cache = self.cache.read().await; - if let Some((slot, _refresh)) = cache.get(&scopes_key) { - return Ok(slot.read().expect("lock poisoned").clone()); - } - } - - let mut cache = self.cache.write().await; - if let Some((slot, _refresh)) = cache.get(&scopes_key) { - return Ok(slot.read().expect("lock poisoned").clone()); - } - - // First use of this scope set: fetch the initial token, then hand - // the slot to a task that keeps it fresh. A failed initial fetch is - // not cached, the next call retries it. - let token = fetch_token( - &self.federated_token_file, - &self.exchange, - scopes_key.clone(), - ) - .await?; - let slot = Arc::new(std::sync::RwLock::new(token.clone())); - let refresh = mz_ore::task::spawn( - || "azure-workload-identity-token-refresh", - refresh_task( - self.federated_token_file.clone(), - Arc::clone(&self.exchange), - Arc::clone(&slot), - scopes_key.clone(), - self.refresh_buffer, - self.retry_interval, - ), - ) - .abort_on_drop(); - cache.insert(scopes_key, (slot, refresh)); - Ok(token) + match ManagedIdentityCredential::new(None) { + Ok(credential) => sources.push(credential), + Err(err) => info!("azure: managed identity credentials unavailable: {err}"), } - - async fn clear_cache(&self) -> azure_core::Result<()> { - // Dropping the entries aborts their refresh tasks with them. - self.cache.write().await.clear(); - Ok(()) + match DeveloperToolsCredential::new(None) { + Ok(credential) => sources.push(credential), + Err(err) => info!("azure: developer tools credentials unavailable: {err}"), } + Arc::new(ChainedTokenCredential(sources)) } -/// Returns the token credential to use when the blob URL carries no SAS -/// token. +/// Builds the HTTP client the Azure SDK transports its requests over. /// -/// Prefers [RefreshingWorkloadIdentityCredential] when its environment -/// variables are present, because the workload identity credential in -/// `azure_identity`'s default chain never re-reads the rotated token file. -/// Otherwise falls back to the default chain, whose remaining credential -/// types (e.g. managed identity via IMDS) refresh correctly. -fn token_credential() -> Arc { - match RefreshingWorkloadIdentityCredential::from_env() { - Some(credential) => { - info!("azure: using refreshing workload identity credentials"); - Arc::new(credential.expect("Azure workload identity credentials")) - } - None => create_default_credential().expect("Azure default credentials"), - } +/// The SDK's own client hardcodes 20s connect and 60s read timeouts, so we +/// supply a client that honors [BlobKnobs] instead. +/// +/// NOTE: automatic decompression must stay off. `BlobClient::download` +/// reassembles a blob from range requests keyed by byte offset, which a +/// transparently decompressed body would invalidate. reqwest turns +/// decompression on by default for every codec whose feature is enabled, and +/// the SDK enables gzip and deflate, so we have to opt out explicitly. +fn http_client(knobs: &dyn BlobKnobs) -> reqwest_0_13::Client { + reqwest_0_13::ClientBuilder::new() + // The SDK defaults to rustls; pin it so an unrelated dependency + // enabling native-tls cannot silently move Azure traffic onto it. + .tls_backend_rustls() + .timeout(knobs.operation_attempt_timeout()) + .read_timeout(knobs.read_timeout()) + .connect_timeout(knobs.connect_timeout()) + // Azure's REST API does not redirect, and following one would leak the + // Authorization header to the redirect target. + .redirect(reqwest_0_13::redirect::Policy::none()) + .no_gzip() + .no_deflate() + .no_brotli() + .no_zstd() + .build() + .expect("valid config for azure HTTP client") } /// Configuration for opening an [AzureBlob]. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct AzureBlobConfig { + // The metrics struct here is a bit of a misnomer. We only need access + // to the LgBytes metrics, which has an Azure-specific field. For now, + // it saves considerable plumbing to reuse [S3BlobMetrics]. + // + // TODO: spin up an AzureBlobMetrics and do the plumbing. metrics: S3BlobMetrics, - client: ContainerClient, + // `BlobContainerClient` is neither `Clone` nor `Debug`, so it is shared + // behind an `Arc` and `Debug` is implemented by hand. + client: Arc, prefix: String, } +impl Debug for AzureBlobConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AzureBlobConfig") + .field("prefix", &self.prefix) + .finish_non_exhaustive() + } +} + impl AzureBlobConfig { const EXTERNAL_TESTS_AZURE_CONTAINER: &'static str = "MZ_PERSIST_EXTERNAL_STORAGE_TEST_AZURE_CONTAINER"; @@ -333,59 +177,52 @@ impl AzureBlobConfig { url: Url, knobs: Box, ) -> Result { - let transport = TransportOptions::new(Arc::new( - reqwest::ClientBuilder::new() - .timeout(knobs.operation_attempt_timeout()) - .read_timeout(knobs.read_timeout()) - .connect_timeout(knobs.connect_timeout()) - .build() - .expect("valid config for azure HTTP client"), - )); - let retry = RetryOptions::exponential( - ExponentialRetryOptions::default().max_total_elapsed(knobs.operation_timeout()), - ); + let mut options = BlobContainerClientOptions::default(); + options.client_options.transport = Some(Transport::new(Arc::new(http_client(&*knobs)))); + options.client_options.retry = RetryOptions::exponential(ExponentialRetryOptions { + max_total_elapsed: azure_core::time::Duration::try_from(knobs.operation_timeout()) + .map_err(|e| Error::from(format!("operation timeout out of range: {e}")))?, + ..Default::default() + }); - let client = if account == EMULATOR_ACCOUNT { + let (container_url, credential) = if account == azurite::ACCOUNT { info!("Connecting to Azure emulator"); - ClientBuilder::with_location( - CloudLocation::Emulator { - address: url.domain().expect("domain for Azure emulator").to_string(), - port: url.port().expect("port for Azure emulator"), - }, - StorageCredentials::emulator(), - ) + // Azurite rejects Entra ID tokens, so requests are signed with the + // Shared Key scheme by a policy instead of by a credential. + options + .client_options + .per_try_policies + .push(Arc::new(azurite::SharedKeyPolicy)); + options.version = azurite::API_VERSION.to_string(); + (azurite::container_url(&url, &container)?, None) } else { - let sas_credentials = match url.query() { - Some(query) => Some(StorageCredentials::sas_token(query)), - None => None, - }; - - let credentials = match sas_credentials { - Some(Ok(credentials)) => credentials, - Some(Err(err)) => { - warn!("Failed to parse SAS token: {err}"); - // TODO: should we fallback here? Or can we fully rely on query params - // to determine whether a SAS token was provided? - StorageCredentials::token_credential(token_credential()) + let endpoint = format!("https://{account}.blob.core.windows.net/{container}"); + match url.query() { + // A SAS token is self-authenticating: it travels in the query + // string and no credential is attached. + // + // NOTE: a SAS token provided this way is static and never + // refreshed, so callers must provision one that outlives the + // process. Token credentials refresh themselves. + Some(sas) => { + let url = Url::parse(&format!("{endpoint}?{sas}")) + .map_err(|e| Error::from(format!("bad Azure container URL: {e}")))?; + (url, None) } - None => StorageCredentials::token_credential(token_credential()), - }; - - ClientBuilder::new(account, credentials) - } - .transport(transport) - .retry(retry) - .blob_service_client() - .container_client(container); + None => { + let url = Url::parse(&endpoint) + .map_err(|e| Error::from(format!("bad Azure container URL: {e}")))?; + (url, Some(token_credential())) + } + } + }; - // NOTE: a SAS token provided via the URL query string is static and - // never refreshed, so callers must provision one that outlives the - // process. Token credentials (workload identity and managed identity) - // refresh themselves. + let client = BlobContainerClient::new(container_url, credential, Some(options)) + .map_err(|e| Error::from(format!("azure container client: {e}")))?; Ok(AzureBlobConfig { metrics, - client, + client: Arc::new(client), prefix, }) } @@ -435,7 +272,7 @@ impl AzureBlobConfig { let metrics = S3BlobMetrics::new(&MetricsRegistry::new()); let config = AzureBlobConfig::new( - EMULATOR_ACCOUNT.to_string(), + azurite::ACCOUNT.to_string(), container_name.clone(), prefix, metrics, @@ -448,21 +285,28 @@ impl AzureBlobConfig { } /// Implementation of [Blob] backed by Azure Blob Storage. -#[derive(Debug)] pub struct AzureBlob { metrics: S3BlobMetrics, - client: ContainerClient, + client: Arc, prefix: String, } +impl Debug for AzureBlob { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AzureBlob") + .field("prefix", &self.prefix) + .finish_non_exhaustive() + } +} + impl AzureBlob { /// Opens the given location for non-exclusive read-write access. pub async fn open(config: AzureBlobConfig) -> Result { - if config.client.service_client().account() == EMULATOR_ACCOUNT { + if azurite::is_emulator_url(config.client.url()) { // TODO: we could move this logic into the test harness. // it's currently here because it's surprisingly annoying to // create the container out-of-band - if let Err(error) = config.client.create().await { + if let Err(error) = config.client.create(None).await { info!( ?error, "failed to create emulator container; this is expected on repeat runs" @@ -484,70 +328,54 @@ impl AzureBlob { } } +/// The blob's total size according to a download's initial response, or `None` +/// if the response did not report one. +/// +/// `download` fetches a blob as a sequence of range requests, so its +/// `content_length` covers only the first range. `Content-Range` carries the +/// total after the slash (`bytes 0-1023/4096`). A blob served in a single +/// unranged response has no `Content-Range`, and its `content_length` is then +/// the whole blob. +fn total_len(response: &BlobClientDownloadResult) -> Option { + const CONTENT_RANGE: HeaderName = HeaderName::from_static("content-range"); + match response.headers.get_optional_str(&CONTENT_RANGE) { + Some(content_range) => content_range + .rsplit_once('/') + .and_then(|(_, total)| total.parse().ok()), + None => response.properties.content_length, + } +} + #[async_trait] impl Blob for AzureBlob { async fn get(&self, key: &str) -> Result, ExternalError> { let path = self.get_path(key); - let blob = self.client.blob_client(path); - - /// Fetch the body of a single [`GetBlobResponse`]. - async fn fetch_chunk( - response: GetBlobResponse, - metrics: S3BlobMetrics, - ) -> Result, ExternalError> { - let content_length = response.blob.properties.content_length; - - let mut parts: Vec = Vec::new(); - let mut total_len: u64 = 0; - let mut body = response.data; - while let Some(value) = body.next().await { - let value = value - .map_err(|e| ExternalError::from(e.context("azure blob get body error")))?; - total_len += u64::cast_from(value.len()); - parts.push(value); - } + let blob = self.client.blob_client(&path); - // Report if the content-length header didn't match the number of - // bytes we read from the network. - if content_length != total_len { - metrics.get_invalid_resp.inc(); + let response = match blob.download(None).await { + Ok(response) => response, + Err(e) => { + if e.http_status() == Some(StatusCode::NotFound) { + return Ok(None); + } + return Err(ExternalError::from(e.with_context("azure blob get error"))); } + }; - Ok(parts) - } - - let mut requests = FuturesOrdered::new(); - // TODO: the default chunk size is 1MB. We have not tried tuning it, - // but making this configurable / running some benchmarks could be - // valuable. - let mut stream = blob.get().into_stream(); - - while let Some(value) = stream.next().await { - // Return early if any of the individual fetch requests return an error. - let response = match value { - Ok(v) => v, - Err(e) => { - if let Some(e) = e.as_http_error() { - if e.status() == StatusCode::NotFound { - return Ok(None); - } - } - - return Err(ExternalError::from(e.context("azure blob get error"))); - } - }; + let expected_len = total_len(&response); + let mut body = response.body; - // Drive all of the fetch requests concurrently. - let metrics = self.metrics.clone(); - requests.push_back(fetch_chunk(response, metrics)); + let mut segments = SegmentedBytes::new(); + while let Some(value) = body.next().await { + let value = value + .map_err(|e| ExternalError::from(e.with_context("azure blob get body error")))?; + segments.push(value); } - // Await on all of our chunks. - let mut segments = SegmentedBytes::with_capacity(requests.len()); - while let Some(body) = requests.next().await { - for part in body.context("azure blob get body err")? { - segments.push(part); - } + // Report if the length the service told us to expect didn't match the + // number of bytes we read from the network. + if expected_len.is_some_and(|len| len != u64::cast_from(segments.len())) { + self.metrics.get_invalid_resp.inc(); } Ok(Some(segments)) @@ -561,27 +389,29 @@ impl Blob for AzureBlob { let blob_key_prefix = self.get_path(key_prefix); let strippable_root_prefix = format!("{}/", self.prefix); - let mut stream = self + let mut pager = self .client - .list_blobs() - .prefix(blob_key_prefix.clone()) - .into_stream(); - - while let Some(response) = stream.next().await { - let response = - response.map_err(|e| ExternalError::from(e.context("azure blob list error")))?; - - for blob in response.blobs.items { - let azure_storage_blobs::container::operations::list_blobs::BlobItem::Blob(blob) = - blob - else { - continue; - }; - - if let Some(key) = blob.name.strip_prefix(&strippable_root_prefix) { - let size_in_bytes = blob.properties.content_length; - f(BlobMetadata { key, size_in_bytes }); - } + .list_blobs(Some(BlobContainerClientListBlobsOptions { + prefix: Some(blob_key_prefix), + ..Default::default() + })) + .map_err(|e| ExternalError::from(e.with_context("azure blob list error")))?; + + while let Some(blob) = pager + .try_next() + .await + .map_err(|e| ExternalError::from(e.with_context("azure blob list error")))? + { + let Some(name) = blob.name.as_deref() else { + continue; + }; + if let Some(key) = name.strip_prefix(&strippable_root_prefix) { + let size_in_bytes = blob + .properties + .as_ref() + .and_then(|properties| properties.content_length) + .unwrap_or(0); + f(BlobMetadata { key, size_in_bytes }); } } @@ -590,56 +420,57 @@ impl Blob for AzureBlob { async fn set(&self, key: &str, value: Bytes) -> Result<(), ExternalError> { let path = self.get_path(key); - let blob = self.client.blob_client(path); + let blob = self.client.blob_client(&path); - blob.put_block_blob(value) + // `.into()` selects `From`; the inherent `from(Vec)` would + // shadow it and copy. + blob.upload(value.into(), None) .await - .map_err(|e| ExternalError::from(e.context("azure blob put error")))?; + .map_err(|e| ExternalError::from(e.with_context("azure blob put error")))?; Ok(()) } async fn delete(&self, key: &str) -> Result, ExternalError> { let path = self.get_path(key); - let blob = self.client.blob_client(path); - - match blob.get_properties().await { - Ok(props) => { - let size = usize::cast_from(props.blob.properties.content_length); - blob.delete() - .await - .map_err(|e| ExternalError::from(e.context("azure blob delete error")))?; - Ok(Some(size)) - } + let blob = self.client.blob_client(&path); + + let properties = match blob.get_properties(None).await { + Ok(properties) => properties, Err(e) => { - if let Some(e) = e.as_http_error() { - if e.status() == StatusCode::NotFound { - return Ok(None); - } + if e.http_status() == Some(StatusCode::NotFound) { + return Ok(None); } - - Err(ExternalError::from(e.context("azure blob error"))) + return Err(ExternalError::from(e.with_context("azure blob error"))); } - } + }; + + let size = usize::cast_from( + properties + .content_length() + .map_err(|e| ExternalError::from(e.with_context("azure blob error")))? + .unwrap_or(0), + ); + blob.delete(None) + .await + .map_err(|e| ExternalError::from(e.with_context("azure blob delete error")))?; + Ok(Some(size)) } async fn restore(&self, key: &str) -> Result<(), ExternalError> { let path = self.get_path(key); let blob = self.client.blob_client(&path); - match blob.get_properties().await { + match blob.get_properties(None).await { Ok(_) => Ok(()), Err(e) => { - if let Some(e) = e.as_http_error() { - if e.status() == StatusCode::NotFound { - return Err(Determinate::new(anyhow!( - "azure blob error: unable to restore non-existent key {key}" - )) - .into()); - } + if e.http_status() == Some(StatusCode::NotFound) { + return Err(Determinate::new(anyhow!( + "azure blob error: unable to restore non-existent key {key}" + )) + .into()); } - - Err(ExternalError::from(e.context("azure blob error"))) + Err(ExternalError::from(e.with_context("azure blob error"))) } } } @@ -647,145 +478,10 @@ impl Blob for AzureBlob { #[cfg(test)] mod tests { - use azure_core::auth::Secret; - use std::sync::Mutex; - use tracing::info; - use crate::location::tests::blob_impl_test; use super::*; - /// A [MockExchange] wrapped for sharing with the credential's exchange - /// closure. - struct MockExchange { - /// Client assertions passed to each exchange call. - assertions: Vec, - /// Whether the next exchange calls fail. - fail: bool, - } - - fn mock_exchange(state: &Arc>) -> ExchangeFn { - let state = Arc::clone(state); - Arc::new(move |assertion, _scopes| { - let state = Arc::clone(&state); - async move { - let mut state = state.lock().unwrap(); - state.assertions.push(assertion); - if state.fail { - return Err(azure_core::error::Error::message( - ErrorKind::Credential, - "mock exchange failure", - )); - } - Ok(AccessToken::new( - Secret::new(format!("aad-{}", state.assertions.len())), - OffsetDateTime::now_utc() + Duration::from_secs(3600), - )) - } - .boxed() - }) - } - - /// Tests that the token file is re-read (and trimmed) on every fetch, - /// that fetched tokens are served from the slot without further - /// exchanges, and that a failed initial fetch is not cached. - #[mz_ore::test(tokio::test)] - async fn refreshing_workload_identity_credential() { - let token_file = tempfile::NamedTempFile::new().expect("create temp token file"); - std::fs::write(token_file.path(), "token-a\n").expect("write token file"); - - let state = Arc::new(Mutex::new(MockExchange { - assertions: Vec::new(), - fail: false, - })); - let credential = RefreshingWorkloadIdentityCredential::with_exchange( - token_file.path().to_path_buf(), - mock_exchange(&state), - TOKEN_REFRESH_BUFFER, - TOKEN_REFRESH_RETRY_INTERVAL, - ); - let scopes = &["https://storage.azure.com/"]; - - let token = credential.get_token(scopes).await.expect("token"); - assert_eq!(token.token.secret(), "aad-1"); - let token = credential.get_token(scopes).await.expect("token"); - assert_eq!(token.token.secret(), "aad-1"); - assert_eq!(state.lock().unwrap().assertions, vec!["token-a"]); - - // A failed initial fetch surfaces the error without caching it, and - // the rotated token file is re-read on the next fetch. - std::fs::write(token_file.path(), "token-b").expect("write token file"); - credential.clear_cache().await.expect("clear cache"); - state.lock().unwrap().fail = true; - assert!(credential.get_token(scopes).await.is_err()); - state.lock().unwrap().fail = false; - let token = credential.get_token(scopes).await.expect("token"); - assert_eq!(token.token.secret(), "aad-3"); - assert_eq!( - state.lock().unwrap().assertions, - vec!["token-a", "token-b", "token-b"] - ); - } - - /// Tests that the background task refreshes the slot with fresh token - /// file contents and keeps the last good token through failed refreshes. - #[mz_ore::test(tokio::test)] - async fn workload_identity_credential_background_refresh() { - let token_file = tempfile::NamedTempFile::new().expect("create temp token file"); - std::fs::write(token_file.path(), "token-a").expect("write token file"); - - let state = Arc::new(Mutex::new(MockExchange { - assertions: Vec::new(), - fail: false, - })); - // A refresh buffer longer than the issued validity makes every token - // immediately due, so refreshes run continuously at the (shortened) - // retry interval. - let credential = RefreshingWorkloadIdentityCredential::with_exchange( - token_file.path().to_path_buf(), - mock_exchange(&state), - Duration::from_secs(7200), - Duration::from_millis(10), - ); - let scopes = &["https://storage.azure.com/"]; - - let token = credential.get_token(scopes).await.expect("token"); - assert_eq!(token.token.secret(), "aad-1"); - - // The background task picks up the rotated token file without any - // caller blocking on the refresh. - std::fs::write(token_file.path(), "token-b").expect("write token file"); - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let token = credential.get_token(scopes).await.expect("token"); - if token.token.secret() != "aad-1" { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("token refreshed within timeout"); - assert_eq!( - state.lock().unwrap().assertions.last().map(String::as_str), - Some("token-b") - ); - - // Failed refreshes keep the last good token in the slot and retry. - state.lock().unwrap().fail = true; - let held = credential.get_token(scopes).await.expect("token"); - let calls_when_failing = state.lock().unwrap().assertions.len(); - tokio::time::timeout(Duration::from_secs(30), async { - while state.lock().unwrap().assertions.len() <= calls_when_failing + 2 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("retries within timeout"); - let token = credential.get_token(scopes).await.expect("token"); - assert_eq!(token.token.secret(), held.token.secret()); - } - #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `TLS_method` on OS `linux` #[mz_ore::test(tokio::test(flavor = "multi_thread"))] async fn azure_blob() -> Result<(), ExternalError> { @@ -805,7 +501,7 @@ mod tests { async move { let config = AzureBlobConfig { metrics: config.metrics.clone(), - client: config.client.clone(), + client: Arc::clone(&config.client), prefix: config.prefix.clone(), }; AzureBlob::open(config).await diff --git a/src/persist/src/azure/azurite.rs b/src/persist/src/azure/azurite.rs new file mode 100644 index 0000000000000..dc23dd5ebc106 --- /dev/null +++ b/src/persist/src/azure/azurite.rs @@ -0,0 +1,191 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Support for Azurite, the Azure Storage emulator that our tests run against. +//! +//! Azurite does not accept Entra ID tokens, only the Azure Storage "Shared Key" +//! scheme. The Azure SDK dropped its built-in emulator support in 1.x, so +//! [SharedKeyPolicy] signs each outgoing request instead. +//! +//! Spec: +//! +//! The emulator key below is a publicly documented constant, and this module is +//! only reachable for the equally well-known [ACCOUNT] name, so compiling it +//! into release binaries grants nothing that the account name does not. + +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +use async_trait::async_trait; +use azure_core::credentials::Secret; +use azure_core::hmac::hmac_sha256; +use azure_core::http::headers::HeaderName; +use azure_core::http::policies::{Policy, PolicyResult}; +use azure_core::http::{Context, Method, Request}; +use url::Url; + +use crate::error::Error; + +/// The well-known account name Azurite serves. +pub const ACCOUNT: &str = "devstoreaccount1"; + +/// The well-known Azurite account key, published in the emulator's docs. +const KEY: &str = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + +/// The storage service API version to ask Azurite for. +/// +/// Azurite rejects any `x-ms-version` newer than the one it implements, and the +/// SDK's default is newer than the version our pinned emulator (see +/// `test/azurite/Dockerfile`) knows about. This is the newest version that +/// emulator accepts; raise it when the emulator is upgraded. +pub const API_VERSION: &str = "2025-01-05"; + +/// Builds the container URL to address Azurite at, given the URL persist was +/// configured with. +/// +/// Azurite is run with `--disableProductStyleUrl`, so it reads the account name +/// from the first path segment rather than from the host: the URL has the form +/// `http://{host}:{port}/devstoreaccount1/{container}`. The host may carry the +/// account name as a subdomain as well, which Azurite ignores. +pub fn container_url(url: &Url, container: &str) -> Result { + let host = url + .host_str() + .ok_or_else(|| Error::from(format!("Azurite URL missing host: {url}")))?; + let port = url + .port() + .ok_or_else(|| Error::from(format!("Azurite URL missing port: {url}")))?; + Url::parse(&format!("http://{host}:{port}/{ACCOUNT}/{container}")) + .map_err(|e| Error::from(format!("invalid Azurite URL: {e}"))) +} + +/// Whether `url` addresses Azurite, i.e. whether it was built by +/// [container_url]. +pub fn is_emulator_url(url: &Url) -> bool { + url.path_segments() + .is_some_and(|mut segments| segments.next() == Some(ACCOUNT)) +} + +/// Signs each request with the Azure Storage Shared Key scheme. +/// +/// Install as a per-try policy: the signature covers `x-ms-date`, so it has to +/// be recomputed for every attempt rather than once per operation. +pub struct SharedKeyPolicy; + +impl Debug for SharedKeyPolicy { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str("SharedKeyPolicy") + } +} + +#[async_trait] +impl Policy for SharedKeyPolicy { + async fn send( + &self, + ctx: &Context, + request: &mut Request, + next: &[Arc], + ) -> PolicyResult { + // `insert_header` stores values with a `'static` bound, so it takes an + // owned `String`. `x-ms-version` is already set by the generated + // clients and is signed along with everything else. + let now = azure_core::time::OffsetDateTime::now_utc(); + request.insert_header("x-ms-date", azure_core::time::to_rfc7231(&now)); + + let signature = hmac_sha256(&string_to_sign(request), &Secret::new(KEY)) + .expect("valid base64 Azurite key"); + request.insert_header("authorization", format!("SharedKey {ACCOUNT}:{signature}")); + + next[0].send(ctx, request, &next[1..]).await + } +} + +/// Builds the canonical string to sign for `request`, per the Shared Key spec. +fn string_to_sign(request: &Request) -> String { + let method = match request.method() { + Method::Delete => "DELETE", + Method::Get => "GET", + Method::Head => "HEAD", + Method::Patch => "PATCH", + Method::Post => "POST", + Method::Put => "PUT", + // `Method` is `#[non_exhaustive]`. No blob operation we issue uses + // another verb, so failing to sign here would be a bug we want to see. + other => panic!("unsigned HTTP method {other:?}"), + }; + + let headers = request.headers(); + let header = |name: &'static str| { + headers + .get_optional_str(&HeaderName::from_static(name)) + .unwrap_or("") + }; + + // A zero content length is signed as the empty string. Requests without a + // body have no `content-length` header at this point: the transport policy, + // which runs after this one, is what adds `content-length: 0`. + let content_length = match header("content-length") { + "0" => "", + other => other, + }; + + // `x-ms-*` headers, lowercased, sorted by name, one `name:value` per line. + let mut x_ms_headers: Vec<(String, &str)> = headers + .iter() + .filter_map(|(name, value)| { + let name = name.as_str().to_ascii_lowercase(); + name.starts_with("x-ms-") + .then(|| (name, value.as_str().trim())) + }) + .collect(); + x_ms_headers.sort(); + let canonicalized_headers = x_ms_headers + .iter() + .map(|(name, value)| format!("{name}:{value}")) + .collect::>() + .join("\n"); + + // The canonicalized resource is `/{account}` followed by the resource path, + // then the query parameters lowercased, sorted by name, one `\nname:value` + // per parameter. + // + // NOTE: the account name appears twice. The emulator addresses the account + // through the URL path (see [container_url]), so the resource path already + // begins with it, and Azurite still prepends the account it resolved the + // request to. Signing this any other way fails with `AuthorizationFailure`. + let url = request.url(); + let mut canonicalized_resource = format!("/{ACCOUNT}{}", url.path()); + let mut query_pairs: Vec<(String, String)> = url + .query_pairs() + .map(|(name, value)| (name.to_ascii_lowercase(), value.into_owned())) + .collect(); + query_pairs.sort(); + for (name, value) in query_pairs { + canonicalized_resource.push_str(&format!("\n{name}:{value}")); + } + + [ + method, + header("content-encoding"), + header("content-language"), + content_length, + header("content-md5"), + header("content-type"), + // `Date` is empty because we sign `x-ms-date` instead. + "", + header("if-modified-since"), + header("if-match"), + header("if-none-match"), + header("if-unmodified-since"), + header("range"), + &canonicalized_headers, + &canonicalized_resource, + ] + .join("\n") +} diff --git a/src/persist/src/location.rs b/src/persist/src/location.rs index 98a3f79c13c65..72e825fc59148 100644 --- a/src/persist/src/location.rs +++ b/src/persist/src/location.rs @@ -16,7 +16,7 @@ use std::time::Instant; use anyhow::anyhow; use async_trait::async_trait; -use azure_core::StatusCode; +use azure_core::http::StatusCode; use bytes::Bytes; use futures_util::Stream; use mz_ore::bytes::SegmentedBytes; @@ -347,16 +347,9 @@ impl From for ExternalError { impl From for ExternalError { fn from(value: azure_core::Error) -> Self { - let definitely_determinate = if let Some(http) = value.as_http_error() { - match http.status() { - // There are many other status codes that _ought_ to be determinate, according to - // the HTTP spec, but this includes only codes that we've observed in practice for now. - StatusCode::TooManyRequests => true, - _ => false, - } - } else { - false - }; + // There are many other status codes that _ought_ to be determinate, according to + // the HTTP spec, but this includes only codes that we've observed in practice for now. + let definitely_determinate = value.http_status() == Some(StatusCode::TooManyRequests); if definitely_determinate { ExternalError::Determinate(Determinate { inner: anyhow!(value),