From 7ac1f0d16066f6f6d1fcbaa790a2269236427f17 Mon Sep 17 00:00:00 2001 From: Kynan Rilee Date: Thu, 6 Aug 2026 11:40:51 -0400 Subject: [PATCH] update iceberg-rust dep to v0.10.1 --- Cargo.lock | 944 +++++++++++++----- Cargo.toml | 23 +- deny.toml | 46 +- src/persist-types/src/parquet.rs | 2 +- src/persist/src/indexed/columnar/parquet.rs | 2 +- .../src/s3_oneshot_sink/parquet.rs | 2 +- src/storage-types/Cargo.toml | 1 + src/storage-types/src/connections.rs | 66 +- 8 files changed, 807 insertions(+), 279 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92a168f4646aa..38cf246cb67ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.6", + "generic-array", +] + [[package]] name = "aes" version = "0.8.4" @@ -34,6 +44,20 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.7.8" @@ -208,7 +232,8 @@ checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", - "digest", + "crc32fast", + "digest 0.10.7", "log", "miniz_oxide", "num-bigint", @@ -218,6 +243,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "snap", "strum", "strum_macros", "thiserror 2.0.18", @@ -279,16 +305,34 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 57.1.0", + "arrow-array 57.2.0", + "arrow-buffer 57.3.0", + "arrow-cast 57.2.0", + "arrow-data 57.2.0", + "arrow-ord 57.2.0", + "arrow-row 57.1.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", + "arrow-string 57.1.0", +] + +[[package]] +name = "arrow" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +dependencies = [ + "arrow-arith 58.4.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-cast 58.4.0", + "arrow-data 58.4.0", + "arrow-ord 58.4.0", + "arrow-row 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", + "arrow-string 58.4.0", ] [[package]] @@ -297,10 +341,24 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.2.0", + "arrow-buffer 57.3.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "chrono", "num-traits", ] @@ -312,9 +370,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" dependencies = [ "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 57.3.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", "chrono", "half", "hashbrown 0.16.1", @@ -323,6 +381,24 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "57.3.0" @@ -335,18 +411,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow-array 57.2.0", + "arrow-buffer 57.3.0", + "arrow-data 57.2.0", + "arrow-ord 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", "atoi", "base64 0.22.1", "chrono", @@ -357,14 +445,48 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-ord 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + [[package]] name = "arrow-data" version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 57.3.0", + "arrow-schema 57.2.0", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer 58.4.0", + "arrow-schema 58.4.0", "half", "num-integer", "num-traits", @@ -372,15 +494,15 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.2.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "flatbuffers", ] @@ -390,11 +512,24 @@ version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.2.0", + "arrow-buffer 57.3.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", ] [[package]] @@ -403,10 +538,23 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.2.0", + "arrow-buffer 57.3.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "half", ] @@ -419,6 +567,12 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" + [[package]] name = "arrow-select" version = "57.2.0" @@ -426,10 +580,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.2.0", + "arrow-buffer 57.3.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "num-traits", +] + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash 0.8.12", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "num-traits", ] @@ -439,11 +607,28 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.2.0", + "arrow-buffer 57.3.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "memchr", "num-traits", "regex", @@ -772,7 +957,7 @@ dependencies = [ "fastrand 2.3.0", "hex", "http 1.4.2", - "sha1", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -914,14 +1099,14 @@ dependencies = [ "bytes", "fastrand 2.3.0", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.4.2", "http-body 1.0.1", "lru", "percent-encoding", "regex-lite", - "sha2", + "sha2 0.10.9", "tracing", "url", ] @@ -1037,11 +1222,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.4.2", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -1071,10 +1256,10 @@ dependencies = [ "http 1.4.2", "http-body 1.0.1", "http-body-util", - "md-5", + "md-5 0.10.6", "pin-project-lite", - "sha1", - "sha2", + "sha1 0.10.6", + "sha2 0.10.9", "tracing", ] @@ -1341,7 +1526,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -1426,7 +1611,7 @@ dependencies = [ "dyn-clone", "futures", "getrandom 0.2.16", - "hmac", + "hmac 0.12.1", "http-types", "once_cell", "paste", @@ -1437,7 +1622,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "time", "tracing", "url", @@ -1565,6 +1750,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1693,6 +1884,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -2106,7 +2306,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "inout", ] @@ -2180,6 +2380,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -2342,6 +2548,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2483,7 +2695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" dependencies = [ "crc", - "digest", + "digest 0.10.7", "rustversion", "spin 0.10.1", ] @@ -2651,9 +2863,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2701,6 +2923,24 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "custom-labels" version = "0.4.6" @@ -2978,7 +3218,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -3121,12 +3361,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "6.0.0" @@ -3177,7 +3429,7 @@ name = "duckdb" version = "1.4.3" source = "git+https://github.com/MaterializeInc/duckdb-rs.git?rev=752c7efe2582#752c7efe25820590f590fb0a112443b9ddd396e7" dependencies = [ - "arrow", + "arrow 57.1.0", "cast", "fallible-iterator 0.3.0", "fallible-streaming-iterator", @@ -3221,7 +3473,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ "der", - "digest", + "digest 0.10.7", "elliptic-curve", "rfc6979", "signature", @@ -3254,7 +3506,7 @@ checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", @@ -3677,7 +3929,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "socket2 0.6.4", "thiserror 2.0.18", "tokio", @@ -4086,6 +4338,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gimli" version = "0.32.3" @@ -4245,6 +4507,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.9.1" @@ -4288,7 +4556,7 @@ dependencies = [ "http 1.4.2", "httpdate", "mime", - "sha1", + "sha1 0.10.6", ] [[package]] @@ -4409,16 +4677,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] -name = "home" -version = "0.5.5" +name = "hmac" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "windows-sys 0.48.0", + "digest 0.11.3", ] [[package]] @@ -4525,6 +4793,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -4710,21 +4987,22 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.9.0" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=dedd9231ee88ee979b648e14792878b40e74c20a#dedd9231ee88ee979b648e14792878b40e74c20a" +version = "0.10.1" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=a51ff2c8dce700a1c158c372f8ce94906b5631b1#a51ff2c8dce700a1c158c372f8ce94906b5631b1" dependencies = [ + "aes-gcm", "anyhow", "apache-avro", "array-init", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 58.4.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-cast 58.4.0", + "arrow-ord 58.4.0", + "arrow-row 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", + "arrow-string 58.4.0", "as-any", "async-trait", "backon", @@ -4744,7 +5022,7 @@ dependencies = [ "once_cell", "ordered-float 4.6.0", "parquet", - "rand 0.8.5", + "rand 0.9.4", "reqwest 0.12.28", "roaring", "serde", @@ -4755,17 +5033,19 @@ dependencies = [ "serde_with", "strum", "tokio", + "tracing", "typed-builder 0.20.1", "typetag", "url", "uuid", + "zeroize", "zstd", ] [[package]] name = "iceberg-catalog-rest" -version = "0.9.0" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=dedd9231ee88ee979b648e14792878b40e74c20a#dedd9231ee88ee979b648e14792878b40e74c20a" +version = "0.10.1" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=a51ff2c8dce700a1c158c372f8ce94906b5631b1#a51ff2c8dce700a1c158c372f8ce94906b5631b1" dependencies = [ "async-trait", "chrono", @@ -4777,24 +5057,24 @@ dependencies = [ "serde_derive", "serde_json", "tokio", - "tracing", "typed-builder 0.20.1", "uuid", ] [[package]] name = "iceberg-storage-opendal" -version = "0.9.0" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=dedd9231ee88ee979b648e14792878b40e74c20a#dedd9231ee88ee979b648e14792878b40e74c20a" +version = "0.10.1" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=a51ff2c8dce700a1c158c372f8ce94906b5631b1#a51ff2c8dce700a1c158c372f8ce94906b5631b1" dependencies = [ "anyhow", "async-trait", "bytes", "cfg-if", + "futures", "iceberg", "opendal", - "reqsign", - "reqwest 0.12.28", + "reqsign-aws-v4", + "reqsign-core", "serde", "typetag", "url", @@ -5097,10 +5377,12 @@ checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", "windows-sys 0.61.1", ] @@ -5235,21 +5517,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "jsonwebtoken" -version = "9.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" -dependencies = [ - "base64 0.22.1", - "js-sys", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - [[package]] name = "jsonwebtoken" version = "10.3.0" @@ -5496,7 +5763,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha1", + "sha1 0.10.6", ] [[package]] @@ -5820,6 +6087,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + [[package]] name = "maplit" version = "1.0.2" @@ -5861,7 +6137,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "mea" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0" +dependencies = [ + "slab", ] [[package]] @@ -6022,8 +6317,8 @@ dependencies = [ "saturating", "serde", "serde_json", - "sha1", - "sha2", + "sha1 0.10.6", + "sha2 0.10.9", "thiserror 2.0.18", "uuid", "zstd", @@ -6073,7 +6368,7 @@ name = "mz-adapter" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "async-stream", "async-trait", "base64 0.22.1", @@ -6152,7 +6447,7 @@ dependencies = [ "serde", "serde_json", "serde_plain", - "sha2", + "sha2 0.10.9", "smallvec", "static_assertions", "thiserror 2.0.18", @@ -6206,7 +6501,7 @@ name = "mz-arrow-util" version = "0.1.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "chrono", "dec", "half", @@ -6253,7 +6548,7 @@ dependencies = [ name = "mz-authenticator" version = "0.1.0" dependencies = [ - "jsonwebtoken 10.3.0", + "jsonwebtoken", "mz-adapter", "mz-adapter-types", "mz-auth", @@ -6366,7 +6661,7 @@ dependencies = [ "hyper 1.9.0", "hyper-openssl", "hyper-util", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "launchdarkly-server-sdk", "mz-alloc", "mz-alloc-default", @@ -6479,7 +6774,7 @@ dependencies = [ "serde", "serde_json", "serde_plain", - "sha2", + "sha2 0.10.9", "static_assertions", "thiserror 2.0.18", "timely", @@ -6520,7 +6815,7 @@ name = "mz-catalog-protos" version = "0.0.0" dependencies = [ "anyhow", - "md-5", + "md-5 0.10.6", "mz-audit-log", "mz-catalog-protos", "mz-compute-types", @@ -6603,7 +6898,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tracing", "uuid", ] @@ -6950,7 +7245,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "strsim", "supports-color 3.0.2", "tempfile", @@ -7075,7 +7370,7 @@ dependencies = [ "insta", "ipnet", "itertools 0.14.0", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "maplit", "mime", "mz-adapter", @@ -7197,10 +7492,10 @@ dependencies = [ "enum-iterator", "fallible-iterator 0.2.0", "hex", - "hmac", + "hmac 0.12.1", "insta", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "murmur2", "mz-build-tools", "mz-expr", @@ -7290,7 +7585,7 @@ dependencies = [ "clap", "derivative", "futures", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "lru", "mz-auth", "mz-ore", @@ -7311,7 +7606,7 @@ dependencies = [ name = "mz-frontegg-client" version = "0.0.0" dependencies = [ - "jsonwebtoken 10.3.0", + "jsonwebtoken", "mz-frontegg-auth", "mz-ore", "reqwest 0.12.28", @@ -7334,7 +7629,7 @@ dependencies = [ "chrono", "clap", "hyper 1.9.0", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "mz-frontegg-auth", "mz-ore", "openssl", @@ -7447,7 +7742,7 @@ dependencies = [ "aws-sdk-kms", "base64 0.22.1", "clap", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "mz-aws-util", "mz-ore", "pem", @@ -7564,7 +7859,7 @@ dependencies = [ "hex", "hex-literal", "reqwest 0.12.28", - "sha2", + "sha2 0.10.9", "tar", "walkdir", ] @@ -7576,7 +7871,7 @@ dependencies = [ "anyhow", "axum", "base64 0.22.1", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "mz-ore", "openssl", "reqwest 0.12.28", @@ -7622,7 +7917,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tokio", "tracing", ] @@ -7647,7 +7942,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", - "sha1", + "sha1 0.10.6", "sysinfo", "tokio", "tracing", @@ -7713,7 +8008,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 2.0.18", "tokio", "tokio-postgres", @@ -7751,7 +8046,7 @@ dependencies = [ "itertools 0.14.0", "lgalloc", "libc", - "lz4_flex", + "lz4_flex 0.12.1", "mz-ore", "mz-ore-proc", "native-tls", @@ -7813,7 +8108,7 @@ name = "mz-persist" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "async-stream", "async-trait", "aws-config", @@ -7831,7 +8126,7 @@ dependencies = [ "fail", "futures-util", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "mz-aws-util", "mz-build-tools", "mz-dyncfg", @@ -7854,7 +8149,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tempfile", "timely", "tokio", @@ -7872,7 +8167,7 @@ version = "26.37.0-dev.0" dependencies = [ "anyhow", "arrayvec 0.7.8", - "arrow", + "arrow 58.4.0", "async-stream", "async-trait", "bytes", @@ -7931,7 +8226,7 @@ name = "mz-persist-types" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "bytes", "chrono", "hex", @@ -8033,7 +8328,7 @@ dependencies = [ "enum-kinds", "futures", "itertools 0.14.0", - "jsonwebtoken 10.3.0", + "jsonwebtoken", "mz-adapter", "mz-adapter-types", "mz-auth", @@ -8212,7 +8507,7 @@ name = "mz-repr" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "bincode", "bitflags 1.3.2", "bytemuck", @@ -8605,7 +8900,7 @@ dependencies = [ "itertools 0.14.0", "junit-report", "maplit", - "md-5", + "md-5 0.10.6", "mz-adapter-types", "mz-authenticator", "mz-build-info", @@ -8674,7 +8969,7 @@ name = "mz-storage" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "arrow-ipc", "async-stream", "async-trait", @@ -8745,7 +9040,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "sha2", + "sha2 0.10.9", "sqlparser", "tempfile", "thiserror 2.0.18", @@ -8843,7 +9138,7 @@ name = "mz-storage-operators" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "arrow-ipc", "async-compression", "async-stream", @@ -8898,7 +9193,7 @@ name = "mz-storage-types" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "async-trait", "aws-config", "aws-credential-types", @@ -8959,6 +9254,7 @@ dependencies = [ "rand 0.9.4", "rdkafka", "regex", + "reqsign-core", "reqwest 0.12.28", "serde", "serde_json", @@ -8992,7 +9288,7 @@ name = "mz-testdrive" version = "26.37.0-dev.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "async-compression", "async-trait", "aws-credential-types", @@ -9013,7 +9309,7 @@ dependencies = [ "itertools 0.14.0", "junit-report", "maplit", - "md-5", + "md-5 0.10.6", "mysql_async", "mz-adapter", "mz-avro", @@ -9087,7 +9383,7 @@ dependencies = [ "futures-util", "itertools 0.14.0", "lgalloc", - "lz4_flex", + "lz4_flex 0.12.1", "mz-ore", "num-traits", "proptest", @@ -9496,7 +9792,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -9526,6 +9822,12 @@ version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebcec7c9c2a95cacc7cd0ecb89d8a8454eca13906f6deb55258ffff0adeb9405" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open" version = "5.3.5" @@ -9539,31 +9841,106 @@ dependencies = [ [[package]] name = "opendal" -version = "0.55.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +dependencies = [ + "opendal-core", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-gcs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" dependencies = [ "anyhow", - "backon", "base64 0.22.1", "bytes", - "crc32c", "futures", - "getrandom 0.2.16", "http 1.4.2", "http-body 1.0.1", "jiff", "log", - "md-5", + "md-5 0.11.0", + "mea", "percent-encoding", - "quick-xml 0.38.4", - "reqsign", - "reqwest 0.12.28", + "quick-xml 0.39.4", + "reqsign-core", + "reqwest 0.13.3", "serde", "serde_json", "tokio", "url", "uuid", + "web-time", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.39.4", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-s3" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "http 1.4.2", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.39.4", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", ] [[package]] @@ -9825,7 +10202,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -9837,7 +10214,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -9851,7 +10228,7 @@ dependencies = [ "elliptic-curve", "primeorder", "rand_core 0.6.4", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -9906,18 +10283,17 @@ dependencies = [ [[package]] name = "parquet" -version = "57.2.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a2926a30477c0b95fea6c28c3072712b139337a242c2cc64817bdc20a8854" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "base64 0.22.1", "brotli", "bytes", @@ -9925,12 +10301,13 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.1", - "lz4_flex", + "hashbrown 0.17.1", + "lz4_flex 0.13.1", "num-bigint", "num-integer", "num-traits", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -9964,8 +10341,8 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", - "hmac", + "digest 0.10.7", + "hmac 0.12.1", ] [[package]] @@ -9998,7 +10375,7 @@ name = "persistcli" version = "0.0.0" dependencies = [ "anyhow", - "arrow", + "arrow 58.4.0", "async-trait", "axum", "bytes", @@ -10068,7 +10445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -10256,7 +10633,7 @@ dependencies = [ "der", "pbkdf2", "scrypt", - "sha2", + "sha2 0.10.9", "spki", ] @@ -10321,6 +10698,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -10369,11 +10758,11 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator 0.2.0", - "hmac", - "md-5", + "hmac 0.12.1", + "md-5 0.10.6", "memchr", "rand 0.9.4", - "sha2", + "sha2 0.10.9", "stringprep", ] @@ -10875,9 +11264,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ "memchr", "serde", @@ -10885,9 +11274,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -11259,33 +11648,91 @@ dependencies = [ ] [[package]] -name = "reqsign" -version = "0.16.5" +name = "reqsign-aws-core" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "e4af084e1f3cbf3e67e0c972765399bce54ecec804cceba46b39a8331f3c1bff" dependencies = [ - "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", + "bytes", "form_urlencoded", - "getrandom 0.2.16", "hex", - "hmac", - "home", "http 1.4.2", - "jsonwebtoken 9.3.1", "log", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest 0.12.28", - "rsa", + "quick-xml 0.41.0", + "reqsign-core", "rust-ini", "serde", "serde_json", - "sha1", - "sha2", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac5b3b7cefa28933792b439186459f77f19f9b6edbeab41b8b187150361a206" +dependencies = [ + "bytes", + "http 1.4.2", + "log", + "quick-xml 0.41.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-core" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07dd510b1e1b9b241883e483358147fb2ed2d497a7b39b065ba61eb93deceb0" +dependencies = [ + "anyhow", + "base64 0.23.1", + "bytes", + "futures", + "hex", + "hmac 0.13.0", + "http 1.4.2", + "jiff", + "log", + "percent-encoding", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", + "windows-sys 0.61.1", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "663d9d55abd0df0830ef0ae43708297cc1371cf4e8ca91f3ac813c309cca8c98" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4080a227f82a09f68540ecd028622065d7ac4c0bcb8727a25bdcfc0526235792" +dependencies = [ + "form_urlencoded", + "http 1.4.2", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", "tokio", ] @@ -11331,7 +11778,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.1", "web-sys", ] @@ -11365,12 +11812,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-util", "tower 0.5.3", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -11437,7 +11886,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -11549,15 +11998,15 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", "pkcs8", "rand_core 0.6.4", - "sha2", + "sha2 0.10.9", "signature", "spki", "subtle", @@ -11829,7 +12278,7 @@ checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ "pbkdf2", "salsa20", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -12259,7 +12708,18 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -12276,10 +12736,21 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", "sha2-asm", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2-asm" version = "0.6.4" @@ -12331,7 +12802,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -12403,12 +12874,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -12507,7 +12975,7 @@ checksum = "eb9242b9ef4108a78e8cd1a2c98e193ef372437f8c22be363075233321dd4a15" dependencies = [ "base64ct", "pem-rfc7468", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -12522,7 +12990,7 @@ dependencies = [ "rand_core 0.6.4", "rsa", "sec1", - "sha2", + "sha2 0.10.9", "signature", "ssh-cipher", "ssh-encoding", @@ -13695,7 +14163,7 @@ dependencies = [ "httparse", "log", "rand 0.9.4", - "sha1", + "sha1 0.10.6", "thiserror 2.0.18", "utf-8", ] @@ -13902,6 +14370,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.6", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -14202,6 +14680,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmtimer" version = "0.4.3" @@ -14362,15 +14853,6 @@ dependencies = [ "windows-link 0.1.1", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.0", -] - [[package]] name = "windows-sys" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index e72662ef8b42b..8a03a961a4235 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -279,8 +279,8 @@ annotate-snippets = "0.12.15" anyhow = "1.0.102" array-concat = "0.5.5" arrayvec = "0.7.6" -arrow = { version = "57", default-features = false } -arrow-ipc = "57" +arrow = { version = "58", default-features = false } +arrow-ipc = "58" askama = { version = "0.12.1", default-features = false, features = ["config", "serde-json"] } assert_cmd = "2.2.2" async-compression = { version = "0.4.27", features = ["bzip2-sys", "gzip", "tokio", "xz", "zstd"] } @@ -385,9 +385,9 @@ hyper-0-14 = { package = "hyper", version = "0.14", features = ["client", "tcp"] hyper-openssl = "0.10.2" hyper-util = "0.1.20" tower-service = "0.3.3" -iceberg = "0.9.0" -iceberg-catalog-rest = "0.9.0" -iceberg-storage-opendal = { version = "0.9.0", default-features = false, features = ["opendal-s3", "opendal-gcs"] } +iceberg = "0.10.1" +iceberg-catalog-rest = "0.10.1" +iceberg-storage-opendal = { version = "0.10.1", default-features = false, features = ["opendal-s3", "opendal-gcs"] } imbl = { version = "7.0.0", features = ["serde"] } include_dir = "0.7.4" indexmap = { version = "2.10.0", default-features = false, features = ["std"] } @@ -435,7 +435,7 @@ opentelemetry_sdk = { version = "0.31.0", features = ["experimental_trace_batch_ ordered-float = { version = "5.1.0", features = ["serde"] } os_info = "3.11.0" owo-colors = "4.3.0" -parquet = { version = "57", default-features = false, features = ["arrow", "async", "brotli", "flate2", "flate2-zlib-rs", "lz4", "snap", "zstd"] } +parquet = { version = "58", default-features = false, features = ["arrow", "async", "brotli", "flate2", "flate2-zlib-rs", "lz4", "snap", "zstd"] } paste = "1.0.15" pem = "3.0.6" phf = { version = "0.13.1", features = ["uncased"] } @@ -474,6 +474,9 @@ rdkafka = { version = "0.29.0", features = ["cmake-build", "libz-static", "ssl-v rdkafka-sys = { version = "4.3.0", features = ["cmake-build", "libz-static", "ssl-vendored", "zstd"] } regex = "1.12.3" regex-syntax = "0.8.10" +# Must stay compatible with the version `iceberg-storage-opendal` uses, otherwise +# the `ProvideCredential` we implement is a different trait than the one it wants. +reqsign-core = "3.0.0" reqwest = { version = "0.12.28", features = ["blocking", "charset", "cookies", "default-tls", "http2", "json", "native-tls-vendored", "stream"] } reqwest-middleware = { version = "0.4.2", features = ["json"] } reqwest-retry = "0.8.0" @@ -663,10 +666,10 @@ tiberius = { git = "https://github.com/MaterializeInc/tiberius", rev="64ca594cc2 async-compression = { git = "https://github.com/MaterializeInc/async-compression.git", rev = "fe7411eb6104a02a89e2c3a76ab326dd6594214d" } # Custom iceberg features for mz -# All changes should go to the `mz_v0.9.0` branch. -iceberg = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "dedd9231ee88ee979b648e14792878b40e74c20a" } -iceberg-catalog-rest = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "dedd9231ee88ee979b648e14792878b40e74c20a" } -iceberg-storage-opendal = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "dedd9231ee88ee979b648e14792878b40e74c20a" } +# All changes should go to the `mz_v0.10.x` branch. +iceberg = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "a51ff2c8dce700a1c158c372f8ce94906b5631b1" } +iceberg-catalog-rest = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "a51ff2c8dce700a1c158c372f8ce94906b5631b1" } +iceberg-storage-opendal = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "a51ff2c8dce700a1c158c372f8ce94906b5631b1" } # Custom duckdb crate to support mz needs # All changes should go to the `mz_changes` branch. diff --git a/deny.toml b/deny.toml index 9224be13166cd..b564f205c329f 100644 --- a/deny.toml +++ b/deny.toml @@ -104,10 +104,8 @@ skip = [ { name = "socket2", version = "0.5.10" }, # Used by azure_core { name = "quick-xml", version = "0.31.0" }, - # Used by reqsign (via iceberg); opendal pulls quick-xml 0.38 - { name = "quick-xml", version = "0.37.5" }, - # Used by opendal via iceberg - { name = "quick-xml", version = "0.38.4" }, + # Used by aws-sdk-*; opendal (via iceberg) pulls quick-xml 0.41 + { name = "quick-xml", version = "0.39.4" }, # Used by sentry { name = "reqwest", version = "0.13.3" }, # Conflicts between `bon` in apache-avro and `derive_builder` in iceberg @@ -126,7 +124,21 @@ skip = [ { name = "phf_codegen", version = "0.12.1" }, # duckdb { name = "fallible-iterator", version = "0.3.0" }, + # duckdb 1.4 pins arrow 57. The workspace is on arrow 58, which iceberg requires. + { name = "arrow", version = "57.1.0" }, + { name = "arrow-arith", version = "57.1.0" }, + { name = "arrow-array", version = "57.2.0" }, + { name = "arrow-buffer", version = "57.3.0" }, + { name = "arrow-cast", version = "57.2.0" }, + { name = "arrow-data", version = "57.2.0" }, + { name = "arrow-ord", version = "57.2.0" }, + { name = "arrow-row", version = "57.1.0" }, + { name = "arrow-schema", version = "57.2.0" }, + { name = "arrow-select", version = "57.2.0" }, + { name = "arrow-string", version = "57.1.0" }, + { name = "lz4_flex", version = "0.12.1" }, # arrow + { name = "hashbrown", version = "0.15.3" }, { name = "hashbrown", version = "0.16.1" }, # Used by dynfmt; iceberg/typetag pulls in v0.4. { name = "erased-serde", version = "0.3.26" }, @@ -135,10 +147,17 @@ skip = [ { name = "core-foundation", version = "0.10.1" }, { name = "security-framework", version = "3.7.0" }, { name = "openssl-probe", version = "0.2.1" }, - # reqsign (via iceberg-storage-opendal / opendal) pins older deps - # than the workspace. - { name = "jsonwebtoken", version = "9.3.1" }, - { name = "quick-xml", version = "0.37.5" }, + # reqsign 3 (via iceberg-storage-opendal / opendal) is on the RustCrypto 0.11/0.12 + # line while the rest of the workspace is still on 0.10. + { name = "base64", version = "0.22.1" }, + { name = "block-buffer", version = "0.10.4" }, + { name = "const-oid", version = "0.9.6" }, + { name = "crypto-common", version = "0.1.6" }, + { name = "digest", version = "0.10.7" }, + { name = "hmac", version = "0.12.1" }, + { name = "md-5", version = "0.10.6" }, + { name = "sha1", version = "0.10.6" }, + { name = "sha2", version = "0.10.9" }, # aws-lc-rs (via jsonwebtoken 10) and ring pull different `untrusted`. { name = "untrusted", version = "0.7.1" }, # Held back by lazy_static 1.4.0 (used by num-bigint-dig). @@ -154,7 +173,6 @@ skip = [ # Pulled by launchdarkly-server-sdk 3.x via launchdarkly-sdk-transport / # eventsource-client (proxy/timeout/rustls stack and the SDK's RNG path). # NB: tower 0.4.13 is already skipped above (mz-deploy). - { name = "rustls-native-certs", version = "0.7.3" }, { name = "rand", version = "0.10.1" }, { name = "rand_core", version = "0.10.1" }, { name = "getrandom", version = "0.4.2" }, @@ -217,14 +235,20 @@ wrappers = [ "launchdarkly-server-sdk-evaluation", "launchdarkly-sdk-transport", "native-tls", - "opendal", + "opendal-core", + "opendal-layer-retry", + "opendal-service-gcs", + "opendal-service-s3", "os_info", "postgres", "pprof", "prost-build", # TODO(guswynn): switch to tracing in rdkafka "rdkafka", - "reqsign", + "reqsign-aws-core", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-google", "reqwest", "rustls", "sqlparser", diff --git a/src/persist-types/src/parquet.rs b/src/persist-types/src/parquet.rs index de264ff500b82..adbf302af0a99 100644 --- a/src/persist-types/src/parquet.rs +++ b/src/persist-types/src/parquet.rs @@ -176,7 +176,7 @@ pub fn encode_arrays( .set_compression(config.compression.into()) .set_writer_version(WriterVersion::PARQUET_2_0) .set_data_page_size_limit(1024 * 1024) - .set_max_row_group_size(usize::MAX) + .set_max_row_group_row_count(None) .build(); let mut writer = ArrowWriter::try_new(w, Arc::clone(&schema), Some(props))?; diff --git a/src/persist/src/indexed/columnar/parquet.rs b/src/persist/src/indexed/columnar/parquet.rs index 4727c459cd724..45957a3d8f247 100644 --- a/src/persist/src/indexed/columnar/parquet.rs +++ b/src/persist/src/indexed/columnar/parquet.rs @@ -118,7 +118,7 @@ pub fn encode_parquet_kvtd( .set_compression(cfg.compression.into()) .set_writer_version(WriterVersion::PARQUET_2_0) .set_data_page_size_limit(1024 * 1024) - .set_max_row_group_size(usize::MAX) + .set_max_row_group_row_count(None) .set_key_value_metadata(Some(vec![metadata])) .build(); diff --git a/src/storage-operators/src/s3_oneshot_sink/parquet.rs b/src/storage-operators/src/s3_oneshot_sink/parquet.rs index 8ee191c6c7bfa..4b0bcdd459df2 100644 --- a/src/storage-operators/src/s3_oneshot_sink/parquet.rs +++ b/src/storage-operators/src/s3_oneshot_sink/parquet.rs @@ -287,7 +287,7 @@ impl ParquetFile { let props = WriterProperties::builder() // This refers to the number of rows per row-group, which we don't want the writer // to enforce since we will flush based on the byte-size of the active row group - .set_max_row_group_size(usize::MAX) + .set_max_row_group_row_count(None) // Max compatibility .set_writer_version(WriterVersion::PARQUET_1_0) .set_compression(Compression::SNAPPY) diff --git a/src/storage-types/Cargo.toml b/src/storage-types/Cargo.toml index 952e1b4b8dfb4..145367778aad0 100644 --- a/src/storage-types/Cargo.toml +++ b/src/storage-types/Cargo.toml @@ -65,6 +65,7 @@ proptest-derive = { workspace = true, optional = true } prost.workspace = true rdkafka.workspace = true regex.workspace = true +reqsign-core.workspace = true reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index 7f76c8ff2b5f4..ff292e5a97b54 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -34,7 +34,7 @@ use iceberg_catalog_rest::{ REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RequestAuthenticator, RestCatalogBuilder, }; use iceberg_storage_opendal::{ - AwsCredential, AwsCredentialLoad, CustomAwsCredentialLoader, OpenDalStorageFactory, + AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, }; use itertools::Itertools; use mz_ccsr::tls::{Certificate, Identity}; @@ -99,6 +99,7 @@ const REST_CATALOG_PROP_CREDENTIAL: &str = "credential"; /// We use this instead of OpenDAL's built-in assume role support because Materialize /// has a runtime-defined credential chain (ambient → jump role → user role with external ID) /// that can't be expressed via OpenDAL's static configuration properties. +#[derive(Debug)] struct AwsSdkCredentialLoader { /// The underlying AWS SDK credentials provider. For assume role auth, this provider /// already handles the full chain: ambient creds -> jump role -> user role. @@ -111,37 +112,56 @@ impl AwsSdkCredentialLoader { } } -#[async_trait] -impl AwsCredentialLoad for AwsSdkCredentialLoader { - async fn load_credential( +impl ProvideCredential for AwsSdkCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( &self, - _client: reqwest::Client, - ) -> anyhow::Result> { - let creds = self - .provider - .provide_credentials() - .await - .map_err(|e| { - warn!( - error = %e.display_with_causes(), - "failed to load AWS credentials for Iceberg FileIO from SDK provider" - ); - e - }) - .context( + _ctx: &reqsign_core::Context, + ) -> reqsign_core::Result> { + let creds = self.provider.provide_credentials().await.map_err(|e| { + warn!( + error = %e.display_with_causes(), + "failed to load AWS credentials for Iceberg FileIO from SDK provider" + ); + reqsign_core::Error::credential_invalid( "failed to load AWS credentials from SDK provider for Iceberg FileIO \ (credential source may be temporarily unavailable)", - )?; + ) + .with_source(e) + })?; + + // `expires_in` is what tells reqsign to ask us for fresh credentials, so a failed + // conversion has to be an error. Reporting `None` would pin the signer to credentials + // that stop working once the underlying role session expires. + let expires_in = match creds.expiry() { + Some(expiry) => Some(system_time_to_timestamp(expiry)?), + None => None, + }; Ok(Some(AwsCredential { access_key_id: creds.access_key_id().to_string(), secret_access_key: creds.secret_access_key().to_string(), session_token: creds.session_token().map(|s| s.to_string()), - expires_in: creds.expiry().map(|t| t.into()), + expires_in, })) } } +/// Converts a `SystemTime` into reqsign's timestamp type, which has no `From` impl for it. +fn system_time_to_timestamp(t: SystemTime) -> reqsign_core::Result { + let millis = t + .duration_since(SystemTime::UNIX_EPOCH) + .map_err(|e| { + reqsign_core::Error::unexpected("timestamp precedes the Unix epoch").with_source(e) + })? + .as_millis(); + let millis = i64::try_from(millis).map_err(|e| { + reqsign_core::Error::unexpected("timestamp too far in the future").with_source(e) + })?; + reqsign_core::time::Timestamp::from_millisecond(millis) +} + /// Signs each outgoing REST-catalog request with AWS SigV4. /// /// Holds a [`SharedCredentialsProvider`] (not static `Credentials`) so each @@ -865,15 +885,14 @@ impl IcebergCatalogConnection { // N.B. We're using the AWS credentials from the catalog connection for the storage layer // even though the sink comes with its own (unused) AWS credentials for storage. let customized_credential_load = if matches!(aws_auth, AwsAuth::AssumeRole(_)) { - Some(CustomAwsCredentialLoader::new(Arc::new( - AwsSdkCredentialLoader::new(credentials_provider), + Some(CustomAwsCredentialLoader::new(AwsSdkCredentialLoader::new( + credentials_provider, ))) } else { None }; let storage_factory = Arc::new(OpenDalStorageFactory::S3 { - configured_scheme: "s3".to_string(), customized_credential_load, }); @@ -927,7 +946,6 @@ impl IcebergCatalogConnection { } ( OpenDalStorageFactory::S3 { - configured_scheme: "s3".to_string(), // When used with MinIO, Polaris returns a config with: // s3.access-key-id, s3.secret-access-key, s3.endpoint, ... // `iceberg-rust` forwards these props to `opendal`.