diff --git a/.arg b/.arg index bd67345cda..9cb9876cde 100644 --- a/.arg +++ b/.arg @@ -1,2 +1,2 @@ # File used by earthly -PLAYWRIGHT_SNAPSHOTS_COMMIT=6d7a9e13a28a4a95a92ef5ca3acba864ed399ee5 +PLAYWRIGHT_SNAPSHOTS_COMMIT=983dd831d58229a12a0150dbb65c0ff04394671c diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ec4d00fcce..89bf0c62c0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -21,7 +21,9 @@ "vadimcn.vscode-lldb", "mike-co.import-sorter", "lihui.vs-color-picker", - "ms-playwright.playwright" + "ms-playwright.playwright", + "svelte.svelte-vscode", + "bradlc.vscode-tailwindcss" ], "settings": { "editor.formatOnSave": true, diff --git a/.github/workflows/benchmark-command.yml b/.github/workflows/benchmark-command.yml index 8ddffdca84..ee21f83b16 100644 --- a/.github/workflows/benchmark-command.yml +++ b/.github/workflows/benchmark-command.yml @@ -41,6 +41,9 @@ jobs: - name: Run Benchmarks run: earthly --verbose -P +benchmark + - name: Fetch previous results + run: bash ./scripts/clone-gh-pages.bash + - name: Publish results run: bash ./scripts/bench-publish.bash diff --git a/.github/workflows/flink-benchmark.yml b/.github/workflows/flink-benchmark.yml new file mode 100644 index 0000000000..e8f0bae8d3 --- /dev/null +++ b/.github/workflows/flink-benchmark.yml @@ -0,0 +1,43 @@ +name: Benchmark Flink + +on: + schedule: + # Run every month on the first day + - cron: '0 0 1 * *' + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + CARGO_NET_RETRY: 10 + RUST_BACKTRACE: short + RUSTUP_MAX_RETRIES: 10 + CI_MACHINE_TYPE: "skylake-2x" + FORCE_COLOR: 1 + +jobs: + benchmark: + permissions: + contents: read + issues: write + pull-requests: write + runs-on: [ self-hosted, skylake40, benchmark-machine ] + steps: + - uses: actions/checkout@v3 + + - name: Install earthly + uses: earthly/actions-setup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: 0.8.14 + + - name: Earthly version + run: earthly --version + + - name: Fetch previous results + run: bash ./scripts/clone-gh-pages.bash + + - name: Refresh Flink Benchmark + run: bash ./benchmark/flink/refresh-flink-benchmark.sh + + - name: Publish Flink Results + run: bash ./benchmark/flink/publish-flink.sh \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7b75c76283..4fb21a8ff6 100644 --- a/.gitignore +++ b/.gitignore @@ -59,10 +59,14 @@ gh-pages # Playwright playwright-artifacts/ playwright-snapshots/ -/test-results/ -/playwright-report/ -/blob-report/ -/playwright/.cache/ +test-results/ +playwright-report/ +test-results-ct/ +playwright-report-ct/ +test-results-e2e/ +playwright-report-e2e/ +blob-report/ +playwright/.cache/ # Local cargo configuration. /.cargo/config.toml diff --git a/.vscode/settings.json b/.vscode/settings.json index 5d4d06c750..d1f1ad93c1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -25,5 +25,7 @@ "files.trimTrailingWhitespace": true, "prettier.requireConfig": true, "rust-analyzer.showUnlinkedFileNotification": false, - "java.compile.nullAnalysis.mode": "disabled" -} \ No newline at end of file + "java.compile.nullAnalysis.mode": "disabled", + "svelte.ask-to-enable-ts-plugin": false, + "svelte.enable-ts-plugin": true +} diff --git a/CHANGELOG.md b/CHANGELOG.md index abb5bd9c93..ce77d653ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Service probing is removed: API endpoints and corresponding + demo are deleted, and the database table `service_probe` is dropped + ([#2002](https://github.com/feldera/feldera/pull/2002)) + +## [0.20.0] - 2024-07-09 + - [SQL] Added `MATERIALIZED` views ([#1959](https://github.com/feldera/feldera/pull/1959)) - [WebConsole] Drop Data Services functionality (#1945) - Pipeline manager metrics endpoint now includes metrics of all running pipelines ([#1969](https://github.com/feldera/feldera/pull/1969)). +- Added ability to trace pipeline steps with Jaegertracing + ([#1999](https://github.com/feldera/feldera/pull/1999)) ## [0.19.0] - 2024-06-25 diff --git a/Cargo.lock b/Cargo.lock index 6a3e26654e..a67f6a0d4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "actix-macros", "actix-rt", "actix_derive", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes", "crossbeam-channel", "futures-core", @@ -33,7 +33,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes", "futures-core", "futures-sink", @@ -69,7 +69,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes", "derive_more", "futures-core", @@ -94,8 +94,8 @@ dependencies = [ "actix-utils", "ahash 0.8.11", "base64 0.22.1", - "bitflags 2.5.0", - "brotli 6.0.0", + "bitflags 2.6.0", + "brotli", "bytes", "bytestring", "derive_more", @@ -118,7 +118,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "zstd 0.13.0", + "zstd 0.13.2", ] [[package]] @@ -153,7 +153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -316,7 +316,7 @@ dependencies = [ "actix-router", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -354,7 +354,7 @@ checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -600,71 +600,35 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +checksum = "6127ea5e585a12ec9f742232442828ebaf264dfa5eefdd71282376c599562b77" dependencies = [ - "arrow-arith 51.0.0", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-csv 51.0.0", - "arrow-data 51.0.0", - "arrow-ipc 51.0.0", - "arrow-json 51.0.0", - "arrow-ord 51.0.0", - "arrow-row 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "arrow-string 51.0.0", -] - -[[package]] -name = "arrow" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" -dependencies = [ - "arrow-arith 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-csv 52.0.0", - "arrow-data 52.0.0", - "arrow-ipc 52.0.0", - "arrow-json 52.0.0", - "arrow-ord 52.0.0", - "arrow-row 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", - "arrow-string 52.0.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] name = "arrow-arith" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +checksum = "7add7f39210b7d726e2a8efc0083e7bf06e8f2d15bdb4896b564dce4410fbf5d" dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono 0.4.38", - "half", - "num", -] - -[[package]] -name = "arrow-arith" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" -dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono 0.4.38", "half", "num", @@ -672,54 +636,26 @@ dependencies = [ [[package]] name = "arrow-array" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +checksum = "81c16ec702d3898c2f5cfdc148443c6cd7dbe5bac28399859eb0a3d38f072827" dependencies = [ "ahash 0.8.11", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono 0.4.38", - "chrono-tz 0.8.6", + "chrono-tz", "half", "hashbrown 0.14.5", "num", ] -[[package]] -name = "arrow-array" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" -dependencies = [ - "ahash 0.8.11", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "chrono 0.4.38", - "chrono-tz 0.9.0", - "half", - "hashbrown 0.14.5", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" -dependencies = [ - "bytes", - "half", - "num", -] - [[package]] name = "arrow-buffer" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" +checksum = "cae6970bab043c4fbc10aee1660ceb5b306d0c42c8cc5f6ae564efcd9759b663" dependencies = [ "bytes", "half", @@ -728,35 +664,15 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "atoi 2.0.0", - "base64 0.22.1", - "chrono 0.4.38", - "half", - "lexical-core", - "num", - "ryu", -] - -[[package]] -name = "arrow-cast" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" +checksum = "1c7ef44f26ef4f8edc392a048324ed5d757ad09135eff6d5509e6450d39e0398" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi 2.0.0", "base64 0.22.1", "chrono 0.4.38", @@ -769,34 +685,15 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono 0.4.38", - "csv", - "csv-core", - "lazy_static", - "lexical-core", - "regex", -] - -[[package]] -name = "arrow-csv" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" +checksum = "5f843490bd258c5182b66e888161bb6f198f49f3792f7c7f98198b924ae0f564" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono 0.4.38", "csv", "csv-core", @@ -807,68 +704,42 @@ dependencies = [ [[package]] name = "arrow-data" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +checksum = "a769666ffac256dd301006faca1ca553d0ae7cffcf4cd07095f73f95eb226514" dependencies = [ - "arrow-buffer 51.0.0", - "arrow-schema 51.0.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] -[[package]] -name = "arrow-data" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" -dependencies = [ - "arrow-buffer 52.0.0", - "arrow-schema 52.0.0", - "half", - "num", -] - -[[package]] -name = "arrow-ipc" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "flatbuffers 23.5.26", -] - [[package]] name = "arrow-ipc" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" +checksum = "dbf9c3fb57390a1af0b7bb3b5558c1ee1f63905f3eccf49ae7676a8d1e6e5a72" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "flatbuffers 24.3.25", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", "lz4_flex", ] [[package]] name = "arrow-json" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +checksum = "654e7f3724176b66ddfacba31af397c48e106fbe4d281c8144e7d237df5acfd7" dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono 0.4.38", "half", "indexmap 2.2.6", @@ -878,160 +749,70 @@ dependencies = [ "serde_json", ] -[[package]] -name = "arrow-json" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" -dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "chrono 0.4.38", - "half", - "indexmap 2.2.6", - "lexical-core", - "num", - "serde", - "serde_json", -] - -[[package]] -name = "arrow-ord" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "half", - "num", -] - [[package]] name = "arrow-ord" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" +checksum = "e8008370e624e8e3c68174faaf793540287106cfda8ad1da862fdc53d8e096b4" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "half", "num", ] [[package]] name = "arrow-row" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" -dependencies = [ - "ahash 0.8.11", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "half", - "hashbrown 0.14.5", -] - -[[package]] -name = "arrow-row" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" +checksum = "ca5e3a6b7fda8d9fe03f3b18a2d946354ea7f3c8e4076dbdb502ad50d9d44824" dependencies = [ "ahash 0.8.11", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", "hashbrown 0.14.5", ] [[package]] name = "arrow-schema" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" -dependencies = [ - "serde", -] - -[[package]] -name = "arrow-schema" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" +checksum = "dab1c12b40e29d9f3b699e0203c2a73ba558444c05e388a4377208f8f9c97eee" dependencies = [ "serde", ] [[package]] name = "arrow-select" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +checksum = "e80159088ffe8c48965cb9b1a7c968b2729f29f37363df7eca177fc3281fe7c3" dependencies = [ "ahash 0.8.11", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "num", -] - -[[package]] -name = "arrow-select" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" -dependencies = [ - "ahash 0.8.11", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] [[package]] name = "arrow-string" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +checksum = "0fd04a6ea7de183648edbcb7a6dd925bbd04c210895f6384c780e27a9b54afcd" dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "memchr", - "num", - "regex", - "regex-syntax", -] - -[[package]] -name = "arrow-string" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" -dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num", "regex", @@ -1101,8 +882,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd 0.13.0", - "zstd-safe 7.0.0", + "zstd 0.13.2", + "zstd-safe 7.2.0", ] [[package]] @@ -1286,7 +1067,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1297,13 +1078,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1330,15 +1111,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" -dependencies = [ - "bytemuck", -] - [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -1427,7 +1199,7 @@ dependencies = [ "fastrand 1.9.0", "hex", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "ring 0.16.20", "time", "tokio", @@ -1438,27 +1210,27 @@ dependencies = [ [[package]] name = "aws-config" -version = "1.5.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac9889352d632214df943e26740c46a0f3da6e329fbd28164fe7ae1b061da7b" +checksum = "caf6cfe2881cb1fcbba9ae946fb9a6480d3b7a714ca84c74925014a89ef3387a" dependencies = [ "aws-credential-types 1.2.0", "aws-runtime", - "aws-sdk-sso 1.29.0", + "aws-sdk-sso 1.34.0", "aws-sdk-ssooidc", - "aws-sdk-sts 1.29.0", + "aws-sdk-sts 1.34.0", "aws-smithy-async 1.2.1", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", - "aws-types 1.3.2", + "aws-types 1.3.3", "bytes", "fastrand 2.1.0", "hex", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "ring 0.17.8", "time", "tokio", @@ -1555,18 +1327,18 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4a5e448145999d7de17bf44a886900ecb834953408dae8aaf90465ce91c1dd" +checksum = "87c5f920ffd1e0526ec9e70e50bf444db50b204395a0fa7016bbf9e31ea1698f" dependencies = [ "aws-credential-types 1.2.0", - "aws-sigv4 1.2.2", + "aws-sigv4 1.2.3", "aws-smithy-async 1.2.1", "aws-smithy-eventstream", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", - "aws-types 1.3.2", + "aws-types 1.3.3", "bytes", "fastrand 2.1.0", "http 0.2.12", @@ -1604,19 +1376,19 @@ dependencies = [ [[package]] name = "aws-sdk-dynamodb" -version = "1.32.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56b456ed83e68bf5a1493fb2c3898841676fab73540dd7970dbeb8ce74aae123" +checksum = "68e3e4257dc0abce20aabccd0017b1aca0f09e563b92456af37378e85821168b" dependencies = [ "aws-credential-types 1.2.0", "aws-runtime", "aws-smithy-async 1.2.1", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", - "aws-types 1.3.2", + "aws-types 1.3.3", "bytes", "fastrand 2.1.0", "http 0.2.12", @@ -1627,24 +1399,24 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.34.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "724119d8fd2d2638b9979673f3b5c2979fa388c9ca27815e3cb5ad6234fac3f5" +checksum = "8367c403fdf27690684b926a46ed9524099a69dd5dfcef62028bf4096b5b809f" dependencies = [ "ahash 0.8.11", "aws-credential-types 1.2.0", "aws-runtime", - "aws-sigv4 1.2.2", + "aws-sigv4 1.2.3", "aws-smithy-async 1.2.1", "aws-smithy-checksums", "aws-smithy-eventstream", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", "aws-smithy-xml 0.60.8", - "aws-types 1.3.2", + "aws-types 1.3.3", "bytes", "fastrand 2.1.0", "hex", @@ -1687,19 +1459,19 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.29.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da75cf91cbb46686a27436d639a720a3a198b148efa76dc2467b7e5374a67fc0" +checksum = "cdcfae7bf8b8f14cade7579ffa8956fcee91dc23633671096b4b5de7d16f682a" dependencies = [ "aws-credential-types 1.2.0", "aws-runtime", "aws-smithy-async 1.2.1", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", - "aws-types 1.3.2", + "aws-types 1.3.3", "bytes", "http 0.2.12", "once_cell", @@ -1709,19 +1481,19 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.30.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2ec8a6687299685ed0a4a3137c129cdb132b5235bc3aa3443f6cffe468b9ff" +checksum = "33b30def8f02ba81276d5dbc22e7bf3bed20d62d1b175eef82680d6bdc7a6f4c" dependencies = [ "aws-credential-types 1.2.0", "aws-runtime", "aws-smithy-async 1.2.1", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", - "aws-types 1.3.2", + "aws-types 1.3.3", "bytes", "http 0.2.12", "once_cell", @@ -1757,21 +1529,21 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.29.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f1031e094b1411b59b49b19e4118f069e1fe13a9c5b8888e933daaf7ffdd6" +checksum = "0804f840ad31537d5d1a4ec48d59de5e674ad05f1db7d3def2c9acadaf1f7e60" dependencies = [ "aws-credential-types 1.2.0", "aws-runtime", "aws-smithy-async 1.2.1", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-json 0.60.7", "aws-smithy-query 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", "aws-smithy-xml 0.60.8", - "aws-types 1.3.2", + "aws-types 1.3.3", "http 0.2.12", "once_cell", "regex-lite", @@ -1813,13 +1585,13 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31eed8d45759b2c5fe7fd304dd70739060e9e0de509209036eabea14d0720cce" +checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be" dependencies = [ "aws-credential-types 1.2.0", "aws-smithy-eventstream", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", "bytes", @@ -1865,11 +1637,11 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.10" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5b30ea96823b8b25fb6471643a516e1bd475fd5575304e6240aea179f213216" +checksum = "48c4134cf3adaeacff34d588dbe814200357b0c466d730cf1c0d8054384a2de4" dependencies = [ - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-types 1.2.0", "bytes", "crc32c", @@ -1898,7 +1670,7 @@ dependencies = [ "fastrand 1.9.0", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-rustls 0.23.2", "lazy_static", "pin-project-lite", @@ -1931,7 +1703,7 @@ dependencies = [ "futures-core", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "once_cell", "percent-encoding", "pin-project-lite", @@ -1943,9 +1715,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.8" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a7de001a1b9a25601016d8057ea16e31a45fdca3751304c8edf4ad72e706c08" +checksum = "d9cd0ae3d97daa0a2bf377a4d8e8e1362cae590c4a1aad0d40058ebca18eb91e" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -2018,12 +1790,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.5.5" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d3965f6417a92a6d1009c5958a67042f57e46342afb37ca58f9ad26744ec73" +checksum = "ce87155eba55e11768b8c1afa607f3e864ae82f03caf63258b37455b0ad02537" dependencies = [ "aws-smithy-async 1.2.1", - "aws-smithy-http 0.60.8", + "aws-smithy-http 0.60.9", "aws-smithy-runtime-api", "aws-smithy-types 1.2.0", "bytes", @@ -2032,7 +1804,8 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "http-body 1.0.0", - "hyper 0.14.29", + "httparse", + "hyper 0.14.30", "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", @@ -2044,9 +1817,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b570ea39eb95bd32543f6e4032bce172cb6209b9bc8c83c770d08169e875afc" +checksum = "30819352ed0a04ecf6a2f3477e344d2d1ba33d43e0f09ad9047c12e0d923616f" dependencies = [ "aws-smithy-async 1.2.1", "aws-smithy-types 1.2.0", @@ -2134,9 +1907,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2009a9733865d0ebf428a314440bbe357cc10d0c16d86a8e15d32e9b47c1e80e" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ "aws-credential-types 1.2.0", "aws-smithy-async 1.2.1", @@ -2230,7 +2003,7 @@ version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -2243,7 +2016,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.68", + "syn 2.0.70", "which", ] @@ -2294,9 +2067,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bitvec" @@ -2374,21 +2147,10 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", "syn_derive", ] -[[package]] -name = "brotli" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 2.5.1", -] - [[package]] name = "brotli" version = "6.0.0" @@ -2397,17 +2159,7 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 4.0.1", -] - -[[package]] -name = "brotli-decompressor" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -2477,7 +2229,7 @@ checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2532,6 +2284,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cache-padded" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" + [[package]] name = "cached" version = "0.38.0" @@ -2609,9 +2367,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.100" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c891175c3fb232128f48de6590095e59198bbeb8620c310be349bfc3afd12c7b" +checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" dependencies = [ "jobserver", "libc", @@ -2661,7 +2419,7 @@ dependencies = [ "rkyv", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2676,18 +2434,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", -] - -[[package]] -name = "chrono-tz" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" -dependencies = [ - "chrono 0.4.38", - "chrono-tz-build 0.2.1", - "phf", + "windows-targets 0.52.6", ] [[package]] @@ -2697,19 +2444,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" dependencies = [ "chrono 0.4.38", - "chrono-tz-build 0.3.0", - "phf", -] - -[[package]] -name = "chrono-tz-build" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" -dependencies = [ - "parse-zoneinfo", + "chrono-tz-build", "phf", - "phf_codegen", ] [[package]] @@ -2800,19 +2536,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ "clap_builder", - "clap_derive 4.5.5", + "clap_derive 4.5.8", ] [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" dependencies = [ "anstream", "anstyle", @@ -2836,14 +2572,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3065,7 +2801,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.7", + "clap 4.5.9", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -3213,6 +2949,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote 1.0.36", + "syn 1.0.109", +] + [[package]] name = "cty" version = "0.2.2" @@ -3241,12 +2987,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "darling_core 0.20.9", - "darling_macro 0.20.9", + "darling_core 0.20.10", + "darling_macro 0.20.10", ] [[package]] @@ -3279,16 +3025,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.86", "quote 1.0.36", "strsim 0.11.1", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3315,13 +3061,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.20.9", + "darling_core 0.20.10", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3370,10 +3116,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" dependencies = [ "ahash 0.8.11", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-ipc 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", "async-compression", "async-trait", "bytes", @@ -3403,7 +3149,7 @@ dependencies = [ "num_cpus", "object_store 0.10.1", "parking_lot 0.12.3", - "parquet 52.0.0", + "parquet", "paste", "pin-project-lite", "rand 0.8.5", @@ -3414,7 +3160,7 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.13.0", + "zstd 0.13.2", ] [[package]] @@ -3424,10 +3170,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" dependencies = [ "ahash 0.8.11", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", "chrono 0.4.38", "half", "hashbrown 0.14.5", @@ -3435,7 +3181,7 @@ dependencies = [ "libc", "num_cpus", "object_store 0.10.1", - "parquet 52.0.0", + "parquet", "sqlparser", ] @@ -3454,7 +3200,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" dependencies = [ - "arrow 52.0.0", + "arrow", "chrono 0.4.38", "dashmap", "datafusion-common", @@ -3476,9 +3222,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" dependencies = [ "ahash 0.8.11", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", "chrono 0.4.38", "datafusion-common", "paste", @@ -3494,7 +3240,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" dependencies = [ - "arrow 52.0.0", + "arrow", "base64 0.22.1", "blake2", "blake3", @@ -3522,8 +3268,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" dependencies = [ "ahash 0.8.11", - "arrow 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3539,11 +3285,11 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" dependencies = [ - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3559,7 +3305,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" dependencies = [ - "arrow 52.0.0", + "arrow", "async-trait", "chrono 0.4.38", "datafusion-common", @@ -3579,12 +3325,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" dependencies = [ "ahash 0.8.11", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", - "arrow-string 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", "base64 0.22.1", "chrono 0.4.38", "datafusion-common", @@ -3609,7 +3355,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" dependencies = [ - "arrow 52.0.0", + "arrow", "datafusion-common", "datafusion-expr", "rand 0.8.5", @@ -3622,11 +3368,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" dependencies = [ "ahash 0.8.11", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "async-trait", "chrono 0.4.38", "datafusion-common", @@ -3655,7 +3401,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db8eb8d706c73f01c0e2630c64ffc61c33831b064a813ec08a3e094dc3190c0f" dependencies = [ - "arrow 52.0.0", + "arrow", "chrono 0.4.38", "datafusion", "datafusion-common", @@ -3671,7 +3417,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde7115772d326eeb78a1c77c974e7753b1538e1747675cb6b428058b654b31c" dependencies = [ - "arrow 52.0.0", + "arrow", "chrono 0.4.38", "datafusion-common", "object_store 0.9.1", @@ -3684,9 +3430,9 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" dependencies = [ - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-schema", "datafusion-common", "datafusion-expr", "log", @@ -3703,7 +3449,7 @@ dependencies = [ "arc-swap", "binrw", "chrono 0.4.31", - "clap 4.5.7", + "clap 4.5.9", "crc32c", "criterion", "crossbeam", @@ -3723,8 +3469,10 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "libc", - "metrics 0.23.0", + "log", + "metrics 0.22.3", "mimalloc-rust-sys", + "minitrace", "nix 0.27.1", "num", "num-derive 0.4.2", @@ -3760,7 +3508,6 @@ dependencies = [ "thiserror", "time", "tokio", - "tracing", "typedmap", "uuid", "xxhash-rust", @@ -3778,18 +3525,18 @@ dependencies = [ "actix-web", "anyhow", "apache-avro", - "arrow 52.0.0", + "arrow", "async-stream", "async-trait", "awc", "aws-sdk-s3", - "aws-types 1.3.2", + "aws-types 1.3.3", "bstr", "bytes", "bytestring", "chrono 0.4.31", "circular-queue", - "clap 4.5.7", + "clap 4.5.9", "colored", "crossbeam", "csv", @@ -3804,17 +3551,20 @@ dependencies = [ "futures-util", "jemalloc_pprof", "lazy_static", + "log", "metrics 0.23.0", "metrics-exporter-prometheus", "metrics-util 0.17.0", "mime", + "minitrace", + "minitrace-jaeger", "mockall", "num-bigint", "num-derive 0.3.3", "num-traits", "once_cell", - "ordered-float 4.2.0", - "parquet 52.0.0", + "ordered-float 4.2.1", + "parquet", "pipeline_types", "pretty_assertions", "prometheus", @@ -3841,7 +3591,6 @@ dependencies = [ "tempfile", "test_bin", "tokio", - "tracing", "url", "utoipa", "uuid", @@ -3927,12 +3676,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1ddfe35af3696786ab5f23cd995df33a66f6cff272ac1f85e09c1a6316acd4c" dependencies = [ - "arrow-arith 52.0.0", - "arrow-array 52.0.0", - "arrow-json 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-arith", + "arrow-array", + "arrow-json", + "arrow-ord", + "arrow-schema", + "arrow-select", "bytes", "chrono 0.4.38", "delta_kernel_derive", @@ -3941,7 +3690,7 @@ dependencies = [ "indexmap 2.2.6", "itertools 0.13.0", "lazy_static", - "parquet 52.0.0", + "parquet", "roaring", "rustc_version", "serde", @@ -3962,7 +3711,7 @@ checksum = "e4d2127a34b12919a6bce08225f0ca6fde8a19342a32675370edfc8795e7c38a" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3973,7 +3722,7 @@ checksum = "b287fdc19e7adbfc4f94bd2c329909f5f0e6d7b7e26d83ea0a7bbc85a0790204" dependencies = [ "deltalake-aws", "deltalake-azure", - "deltalake-core 0.18.1", + "deltalake-core", "deltalake-gcp", ] @@ -3984,14 +3733,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ec47767908a5d4abb52537e96355a2a886651c07c9828097c74c559cce2f47d" dependencies = [ "async-trait", - "aws-config 1.5.1", + "aws-config 1.5.4", "aws-credential-types 1.2.0", "aws-sdk-dynamodb", - "aws-sdk-sts 1.29.0", + "aws-sdk-sts 1.34.0", "aws-smithy-runtime-api", "backoff", "bytes", - "deltalake-core 0.18.1", + "deltalake-core", "futures", "lazy_static", "maplit", @@ -4006,75 +3755,21 @@ dependencies = [ [[package]] name = "deltalake-azure" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e074599ebb06706867093e06f29ca4eb77a310a8ca24c83066063d4062acbbfe" +checksum = "299f7bad095ba316d7c009bba4c423761393b3499b9ed1cca2b027ab6b201e53" dependencies = [ "async-trait", "bytes", - "deltalake-core 0.17.3", + "deltalake-core", "futures", "lazy_static", - "object_store 0.9.1", - "regex", - "thiserror", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "deltalake-core" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8dc1bcd91be689ee7f6ce363798dc2e9b6954be9d597f884de11ba27b33add" -dependencies = [ - "arrow 51.0.0", - "arrow-arith 51.0.0", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-ipc 51.0.0", - "arrow-json 51.0.0", - "arrow-ord 51.0.0", - "arrow-row 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "async-trait", - "bytes", - "cfg-if", - "chrono 0.4.38", - "dashmap", - "either", - "errno", - "fix-hidden-lifetime-bug", - "futures", - "hashbrown 0.14.5", - "indexmap 2.2.6", - "itertools 0.12.1", - "lazy_static", - "libc", - "maplit", - "num-bigint", - "num-traits", - "num_cpus", - "object_store 0.9.1", - "once_cell", - "parking_lot 0.12.3", - "parquet 51.0.0", - "percent-encoding", - "pin-project-lite", - "rand 0.8.5", + "object_store 0.10.1", "regex", - "roaring", - "serde", - "serde_json", "thiserror", "tokio", "tracing", "url", - "uuid", - "z85", ] [[package]] @@ -4083,17 +3778,17 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0068bd92347795c1c5e3b4ef7c5489de289ebe78cc9ab6667c0fbab539da3d7c" dependencies = [ - "arrow 52.0.0", - "arrow-arith 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-ipc 52.0.0", - "arrow-json 52.0.0", - "arrow-ord 52.0.0", - "arrow-row 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", "async-trait", "bytes", "cfg-if", @@ -4124,7 +3819,7 @@ dependencies = [ "object_store 0.10.1", "once_cell", "parking_lot 0.12.3", - "parquet 52.0.0", + "parquet", "percent-encoding", "pin-project-lite", "rand 0.8.5", @@ -4143,16 +3838,16 @@ dependencies = [ [[package]] name = "deltalake-gcp" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c067c1b226d80bfa5468e481708293367dda7664d039d01c75c3f9efb2cd398a" +checksum = "dd9e13fc0f1c03e0f26b07910560a26b3c52a6d4e00f6933081d65b0984942e1" dependencies = [ "async-trait", "bytes", - "deltalake-core 0.17.3", + "deltalake-core", "futures", "lazy_static", - "object_store 0.9.1", + "object_store 0.10.1", "regex", "thiserror", "tokio", @@ -4190,7 +3885,7 @@ dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", "rustc_version", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4344,9 +4039,9 @@ dependencies = [ [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "elliptic-curve" @@ -4399,7 +4094,7 @@ dependencies = [ "num-traits", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4592,16 +4287,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" -[[package]] -name = "flatbuffers" -version = "23.5.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" -dependencies = [ - "bitflags 1.3.2", - "rustc_version", -] - [[package]] name = "flatbuffers" version = "24.3.25" @@ -4783,7 +4468,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -5115,11 +4800,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5225,9 +4910,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -5249,9 +4934,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -5275,7 +4960,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "log", "rustls 0.20.9", "rustls-native-certs 0.6.3", @@ -5291,7 +4976,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "log", "rustls 0.21.12", "rustls-native-certs 0.6.3", @@ -5307,11 +4992,11 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "log", - "rustls 0.23.10", - "rustls-native-certs 0.7.0", + "rustls 0.23.11", + "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -5325,7 +5010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.29", + "hyper 0.14.30", "native-tls", "tokio", "tokio-native-tls", @@ -5339,7 +5024,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "native-tls", "tokio", @@ -5349,16 +5034,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.3.1", + "hyper 1.4.1", "pin-project-lite", "socket2 0.5.7", "tokio", @@ -5809,7 +5494,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -5824,7 +5509,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "libc", ] @@ -5898,9 +5583,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" dependencies = [ "value-bag", ] @@ -6008,7 +5693,7 @@ checksum = "bf0af7a0d7ced10c0151f870e5e3f3f8bc9ffc5992d32873566ca1f9169ae776" dependencies = [ "base64 0.22.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-rustls 0.27.2", "hyper-util", "indexmap 2.2.6", @@ -6034,7 +5719,7 @@ dependencies = [ "indexmap 2.2.6", "metrics 0.22.3", "num_cpus", - "ordered-float 4.2.0", + "ordered-float 4.2.1", "quanta", "radix_trie", "sketches-ddsketch", @@ -6053,7 +5738,7 @@ dependencies = [ "indexmap 2.2.6", "metrics 0.23.0", "num_cpus", - "ordered-float 4.2.0", + "ordered-float 4.2.1", "quanta", "radix_trie", "sketches-ddsketch", @@ -6077,9 +5762,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", @@ -6091,6 +5776,44 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "minitrace" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197d538cd69839d49a593c8c72df44291b0ea3296ecc0c85529002c53c8fbc6f" +dependencies = [ + "minitrace-macro", + "minstant", + "once_cell", + "parking_lot 0.12.3", + "pin-project", + "rand 0.8.5", + "rtrb", +] + +[[package]] +name = "minitrace-jaeger" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e63612364589c56423b9d3d08083adfadc44570158b49bc76a10562214785d2" +dependencies = [ + "log", + "minitrace", + "thrift_codec", +] + +[[package]] +name = "minitrace-macro" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14efd4b574325fcb981bce1ac700b9ccf071ec2eb94f7a6a6b583a84f228ba47" +dependencies = [ + "proc-macro-error", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -6100,6 +5823,16 @@ dependencies = [ "adler", ] +[[package]] +name = "minstant" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fb9b5c752f145ac5046bccc3c4f62892e3c950c1d1eab80c5949cd68a2078db" +dependencies = [ + "ctor", + "web-time", +] + [[package]] name = "mio" version = "0.8.11" @@ -6142,7 +5875,7 @@ dependencies = [ "cfg-if", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -6199,7 +5932,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "libc", ] @@ -6239,9 +5972,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -6281,7 +6014,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -6374,9 +6107,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" dependencies = [ "memchr", ] @@ -6388,22 +6121,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", - "base64 0.21.7", "bytes", "chrono 0.4.38", "futures", "humantime", - "hyper 0.14.29", "itertools 0.12.1", "parking_lot 0.12.3", "percent-encoding", - "quick-xml 0.31.0", - "rand 0.8.5", - "reqwest 0.11.27", - "ring 0.17.8", - "rustls-pemfile 2.1.2", - "serde", - "serde_json", "snafu", "tokio", "tracing", @@ -6423,7 +6147,7 @@ dependencies = [ "chrono 0.4.38", "futures", "humantime", - "hyper 1.3.1", + "hyper 1.4.1", "itertools 0.12.1", "md-5", "parking_lot 0.12.3", @@ -6432,6 +6156,7 @@ dependencies = [ "rand 0.8.5", "reqwest 0.12.5", "ring 0.17.8", + "rustls-pemfile 2.1.2", "serde", "serde_json", "snafu", @@ -6449,9 +6174,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" @@ -6459,7 +6184,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -6476,7 +6201,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -6579,9 +6304,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.0" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" dependencies = [ "num-traits", "rand 0.8.5", @@ -6688,60 +6413,25 @@ dependencies = [ "libc", "redox_syscall 0.5.2", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] name = "parquet" -version = "51.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" +checksum = "0f22ba0d95db56dde8685e3fadcb915cdaadda31ab8abbe3ff7f0ad1ef333267" dependencies = [ "ahash 0.8.11", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-ipc 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", "base64 0.22.1", - "brotli 3.5.0", - "bytes", - "chrono 0.4.38", - "flate2", - "futures", - "half", - "hashbrown 0.14.5", - "lz4_flex", - "num", - "num-bigint", - "object_store 0.9.1", - "paste", - "seq-macro", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd 0.13.0", -] - -[[package]] -name = "parquet" -version = "52.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" -dependencies = [ - "ahash 0.8.11", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-ipc 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", - "base64 0.22.1", - "brotli 6.0.0", + "brotli", "bytes", "chrono 0.4.38", "flate2", @@ -6759,7 +6449,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.0", + "zstd 0.13.2", "zstd-sys", ] @@ -6954,7 +6644,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -6971,7 +6661,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline-manager" -version = "0.19.0" +version = "0.20.0" dependencies = [ "actix-cors", "actix-files", @@ -6988,7 +6678,7 @@ dependencies = [ "cached 0.43.0", "change-detection", "chrono 0.4.38", - "clap 4.5.7", + "clap 4.5.9", "colored", "deadpool-postgres", "dirs 5.0.1", @@ -7006,7 +6696,6 @@ dependencies = [ "proptest", "proptest-derive 0.3.0", "rand 0.8.5", - "rdkafka", "refinery", "reqwest 0.11.27", "serde", @@ -7253,7 +6942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2 1.0.86", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -7350,7 +7039,7 @@ checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.5.0", + "bitflags 2.6.0", "lazy_static", "num-traits", "rand 0.8.5", @@ -7436,7 +7125,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -7561,7 +7250,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.10", + "rustls 0.23.11", "thiserror", "tokio", "tracing", @@ -7577,7 +7266,7 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash", - "rustls 0.23.10", + "rustls 0.23.11", "slab", "thiserror", "tinyvec", @@ -7728,7 +7417,7 @@ version = "11.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -7825,7 +7514,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -7881,7 +7570,7 @@ dependencies = [ "quote 1.0.36", "refinery-core", "regex", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -7948,8 +7637,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", - "hyper-rustls 0.24.2", + "hyper 0.14.30", "hyper-tls 0.5.0", "ipnet", "js-sys", @@ -7959,8 +7647,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -7969,13 +7655,10 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.24.1", - "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", "winreg 0.50.0", ] @@ -7995,7 +7678,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-rustls 0.27.2", "hyper-tls 0.6.0", "hyper-util", @@ -8008,8 +7691,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.10", - "rustls-native-certs 0.7.0", + "rustls 0.23.11", + "rustls-native-certs 0.7.1", "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", @@ -8048,9 +7731,9 @@ dependencies = [ [[package]] name = "rgb" -version = "0.8.37" +version = "0.8.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" +checksum = "1aee83dc281d5a3200d37b299acd13b81066ea126a7f16f0eae70fc9aed241d9" dependencies = [ "bytemuck", ] @@ -8129,9 +7812,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7699249cc2c7d71939f30868f47e9d7add0bdc030d90ee10bfd16887ff8bb1c8" +checksum = "8f4b84ba6e838ceb47b41de5194a60244fac43d9fe03b71dbe8c5a201081d6d1" dependencies = [ "bytemuck", "byteorder", @@ -8179,11 +7862,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rtrb" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e704dd104faf2326a320140f70f0b736d607c1caa1b1748a6c568a79819109" +dependencies = [ + "cache-padded", +] + [[package]] name = "rust-embed" -version = "8.4.0" +version = "8.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19549741604902eb99a7ed0ee177a0663ee1eda51a29f71401f166e47e77806a" +checksum = "fa66af4a4fdd5e7ebc276f115e895611a34739a9c1c01028383d612d550953c0" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -8192,23 +7884,23 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.4.0" +version = "8.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9f96e283ec64401f30d3df8ee2aaeb2561f34c824381efa24a35f79bf40ee4" +checksum = "6125dbc8867951125eec87294137f4e9c2c96566e61bf72c45095a7c77761478" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", "rust-embed-utils", "shellexpand", - "syn 2.0.68", + "syn 2.0.70", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "8.4.0" +version = "8.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c74a686185620830701348de757fd36bef4aa9680fd23c49fc539ddcc1af32" +checksum = "2e5347777e9aacb56039b0e1f28785929a8a3b709e87482e7442c72e7c12529d" dependencies = [ "sha2", "walkdir", @@ -8290,7 +7982,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys 0.4.14", @@ -8323,16 +8015,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.10" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] @@ -8351,9 +8043,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.2", @@ -8399,9 +8091,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "aws-lc-rs", "ring 0.17.8", @@ -8521,7 +8213,7 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -8552,9 +8244,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] @@ -8565,10 +8257,10 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff56acef131ef74bacc5e86c5038b524d61dee59d65c9e3e5e0f35b9de98cf99" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "bytemuck", "chrono 0.4.38", "half", @@ -8577,20 +8269,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -8631,9 +8323,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.8.1" +version = "3.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +checksum = "e73139bc5ec2d45e6c5fd85be5a46949c1c39a4c18e56915f5eb4c12f975e377" dependencies = [ "base64 0.22.1", "chrono 0.4.38", @@ -8649,14 +8341,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.8.1" +version = "3.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +checksum = "b80d3d6b56b64335c0180e5ffde23b3c5e08c14c585b51a15bd0e95393f46703" dependencies = [ - "darling 0.20.9", + "darling 0.20.10", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8694,7 +8386,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -8980,7 +8672,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9092,9 +8784,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "static-files" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64712ea1e3e140010e1d9605872ba205afa2ab5bd38191cc6ebd248ae1f6a06b" +checksum = "4e8590e848e1c53be9258210bcd4a8f4118e08988f03a4e2d63b62e4ad9f7ced" dependencies = [ "change-detection", "mime_guess", @@ -9161,7 +8853,7 @@ dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", "rustversion", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9174,7 +8866,7 @@ dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", "rustversion", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9230,9 +8922,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", @@ -9248,7 +8940,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9265,9 +8957,9 @@ checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" [[package]] name = "sysinfo" -version = "0.30.12" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae" +checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" dependencies = [ "cfg-if", "core-foundation-sys", @@ -9428,7 +9120,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9452,6 +9144,16 @@ dependencies = [ "ordered-float 2.10.1", ] +[[package]] +name = "thrift_codec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d957f535b242b91aa9f47bde08080f9a6fef276477e55b0079979d002759d5" +dependencies = [ + "byteorder", + "trackable", +] + [[package]] name = "tikv-jemalloc-ctl" version = "0.5.4" @@ -9535,9 +9237,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -9575,7 +9277,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9653,7 +9355,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "tokio", ] @@ -9708,7 +9410,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit 0.22.15", ] [[package]] @@ -9744,9 +9446,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" dependencies = [ "indexmap 2.2.6", "serde", @@ -9832,7 +9534,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -9869,6 +9571,25 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "trackable" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15bd114abb99ef8cee977e517c8f37aee63f184f2d08e3e6ceca092373369ae" +dependencies = [ + "trackable_derive", +] + +[[package]] +name = "trackable_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb235c5847e2f82cfe0f07eb971d1e5f6804b18dac2ae16349cc604380f82f" +dependencies = [ + "quote 1.0.36", + "syn 1.0.109", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -9902,7 +9623,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -10062,7 +9783,7 @@ dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", "regex", - "syn 2.0.68", + "syn 2.0.70", "uuid", ] @@ -10084,11 +9805,10 @@ dependencies = [ [[package]] name = "uuid" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea73390fe27785838dcbf75b91b1d84799e28f1ce71e6f372a5dc2200c80de5" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ - "atomic", "getrandom 0.2.15", "serde", ] @@ -10131,7 +9851,7 @@ checksum = "b3fd98999db9227cf28e59d83e1f120f42bc233d4b152e8fab9bc87d5bb1e0f8" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -10213,7 +9933,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-shared", ] @@ -10247,7 +9967,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10281,6 +10001,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki" version = "0.22.4" @@ -10367,7 +10097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -10376,7 +10106,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -10394,7 +10124,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -10414,18 +10144,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -10436,9 +10166,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -10448,9 +10178,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -10460,15 +10190,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -10478,9 +10208,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -10490,9 +10220,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -10502,9 +10232,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -10514,9 +10244,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -10569,7 +10299,7 @@ dependencies = [ "futures", "futures-timer", "http-types", - "hyper 0.14.29", + "hyper 0.14.30", "log", "once_cell", "regex", @@ -10606,9 +10336,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" +checksum = "63658493314859b4dfdf3fb8c1defd61587839def09582db50b8a4e93afca6bb" [[package]] name = "xz2" @@ -10633,22 +10363,22 @@ checksum = "2a599daf1b507819c1121f0bf87fa37eb19daac6aff3aefefd4e6e2e0f2020fc" [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -10668,7 +10398,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.36", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -10711,11 +10441,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 7.0.0", + "zstd-safe 7.2.0", ] [[package]] @@ -10740,18 +10470,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.11+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" dependencies = [ "cc", "pkg-config", diff --git a/Earthfile b/Earthfile index 865cb1aa4f..cf161d713a 100644 --- a/Earthfile +++ b/Earthfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install --yes sudo WORKDIR /dbsp ENV RUSTUP_HOME=$HOME/.rustup ENV CARGO_HOME=$HOME/.cargo -# Adds python and rust binaries to thep path +# Adds python and rust binaries to the path ENV PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH ENV RUST_VERSION=1.78.0 ENV RUST_BUILD_MODE='' # set to --release for release builds @@ -35,6 +35,10 @@ install-deps: RUN npm install --global yarn RUN npm install --global openapi-typescript-codegen RUN apt install unzip -y + ## Install Bun.js + RUN curl -fsSL https://bun.sh/install | bash + ENV PATH="$HOME/.bun/bin:$PATH" + RUN apt install python3-requests -y RUN arch=`dpkg --print-architecture`; \ curl -LO https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-$arch.zip \ @@ -90,6 +94,8 @@ clippy: FROM +rust-sources ENV WEBUI_BUILD_DIR=/dbsp/web-console/out COPY ( +build-webui/out ) ./web-console/out + ENV WEBCONSOLE_BUILD_DIR=/dbsp/web-console-sveltekit/build + COPY ( +build-webui/build ) ./web-console-sveltekit/build DO rust+CARGO --args="clippy -- -D warnings" ENV RUSTDOCFLAGS="-D warnings" DO rust+CARGO --args="doc --no-deps" @@ -121,9 +127,12 @@ build-webui-deps: FROM +install-deps COPY web-console/package.json ./web-console/package.json COPY web-console/yarn.lock ./web-console/yarn.lock - RUN cd web-console && yarn install + COPY web-console-sveltekit/package.json ./web-console-sveltekit/ + COPY web-console-sveltekit/bun.lockb ./web-console-sveltekit/ + RUN cd web-console-sveltekit && $HOME/.bun/bin/bun install + build-webui: FROM +build-webui-deps COPY --dir web-console/public web-console/public @@ -143,6 +152,21 @@ build-webui: RUN cd web-console && yarn build SAVE ARTIFACT ./web-console/out + COPY --dir web-console-sveltekit/static web-console-sveltekit/static + COPY --dir web-console-sveltekit/src web-console-sveltekit/src + COPY web-console-sveltekit/.prettierignore web-console-sveltekit/ + COPY web-console-sveltekit/.prettierrc web-console-sveltekit/ + COPY web-console-sveltekit/eslint.config.js web-console-sveltekit/ + COPY web-console-sveltekit/postcss.config.js web-console-sveltekit/ + COPY web-console-sveltekit/svelte.config.js ./web-console-sveltekit/ + COPY web-console-sveltekit/tailwind.config.ts ./web-console-sveltekit/ + COPY web-console-sveltekit/tsconfig.json ./web-console-sveltekit/ + COPY web-console-sveltekit/vite.config.ts ./web-console-sveltekit/ + + # RUN cd web-console-sveltekit && $HOME/.bun/bin/bun run check + RUN cd web-console-sveltekit && $HOME/.bun/bin/bun run build + SAVE ARTIFACT ./web-console-sveltekit/build + build-dbsp: FROM +rust-sources DO rust+CARGO --args="build --package dbsp --benches" @@ -173,6 +197,8 @@ build-manager: # For some reason if this ENV before the FROM line it gets invalidated ENV WEBUI_BUILD_DIR=/dbsp/web-console/out COPY ( +build-webui/out ) ./web-console/out + ENV WEBCONSOLE_BUILD_DIR=/dbsp/web-console-sveltekit/build + COPY ( +build-webui/build ) ./web-console-sveltekit/build DO rust+CARGO --args="build --package pipeline-manager --features pg-embed" --output="debug/pipeline-manager" IF [ -f ./target/debug/pipeline-manager ] @@ -329,6 +355,7 @@ build-pipeline-manager-container: # Install cargo and rust for this non-root user RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal ENV PATH="$PATH:/home/feldera/.cargo/bin" + RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler --compilation-profile=unoptimized --dbsp-override-path=/home/feldera/database-stream-processor --precompile ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/database-stream-processor", "--compilation-profile=unoptimized"] @@ -372,13 +399,13 @@ test-docker-compose: test-docker-compose-stable: FROM earthly/dind:alpine COPY deploy/docker-compose.yml . - ENV FELDERA_VERSION=0.19.0 + ENV FELDERA_VERSION=0.20.0 RUN apk --no-cache add curl WITH DOCKER --pull postgres \ --pull docker.redpanda.com/vectorized/redpanda:v23.2.3 \ - --pull ghcr.io/feldera/pipeline-manager:0.19.0 \ + --pull ghcr.io/feldera/pipeline-manager:0.20.0 \ --load ghcr.io/feldera/pipeline-manager:latest=+build-pipeline-manager-container \ - --pull ghcr.io/feldera/demo-container:0.19.0 + --pull ghcr.io/feldera/demo-container:0.20.0 RUN COMPOSE_HTTP_TIMEOUT=120 SECOPS_DEMO_ARGS="--prepare-args 200000" RUST_LOG=debug,tokio_postgres=info docker-compose -f docker-compose.yml --profile demo up --force-recreate --exit-code-from demo && \ # This should run the latest version of the code and in the process, trigger a migration. COMPOSE_HTTP_TIMEOUT=120 SECOPS_DEMO_ARGS="--prepare-args 200000" FELDERA_VERSION=latest RUST_LOG=debug,tokio_postgres=info docker-compose -f docker-compose.yml up -d db pipeline-manager redpanda && \ @@ -512,7 +539,7 @@ ui-playwright-container: # Install zip to prepare test artifacts for export RUN apt-get install -y zip -ui-playwright-tests: +ui-playwright-tests-e2e: FROM +ui-playwright-container ENV FELDERA_VERSION=latest @@ -521,15 +548,31 @@ ui-playwright-tests: --compose ../docker-compose.yml \ --service pipeline-manager # We zip artifacts regardless of test success or error, and then we complete the command preserving test's exit_code - RUN if yarn playwright test; then exit_code=0; else exit_code=$?; fi \ + RUN sleep 10 && if yarn playwright test -c playwright-e2e.config.ts; then exit_code=0; else exit_code=$?; fi \ && cd /dbsp \ - && zip -r playwright-report.zip playwright-report \ - && zip -r test-results.zip test-results \ + && zip -r playwright-report-e2e.zip playwright-report-e2e \ + && zip -r test-results-e2e.zip test-results-e2e \ && exit $exit_code END FINALLY - SAVE ARTIFACT --if-exists /dbsp/playwright-report.zip AS LOCAL ./playwright-artifacts/ - SAVE ARTIFACT --if-exists /dbsp/test-results.zip AS LOCAL ./playwright-artifacts/ + SAVE ARTIFACT --if-exists /dbsp/playwright-report-e2e.zip AS LOCAL ./playwright-artifacts/ + SAVE ARTIFACT --if-exists /dbsp/test-results-e2e.zip AS LOCAL ./playwright-artifacts/ + END + +ui-playwright-tests-ct: + FROM +ui-playwright-container + ENV FELDERA_VERSION=latest + + TRY + # We zip artifacts regardless of test success or error, and then we complete the command preserving test's exit_code + RUN if yarn playwright test -c playwright-ct.config.ts; then exit_code=0; else exit_code=$?; fi \ + && cd /dbsp \ + && zip -r playwright-report-ct.zip playwright-report-ct \ + && zip -r test-results-ct.zip test-results-ct \ + && exit $exit_code + FINALLY + SAVE ARTIFACT --if-exists /dbsp/playwright-report-ct.zip AS LOCAL ./playwright-artifacts/ + SAVE ARTIFACT --if-exists /dbsp/test-results-ct.zip AS LOCAL ./playwright-artifacts/ END benchmark: @@ -564,6 +607,28 @@ benchmark: SAVE ARTIFACT crates/dbsp/galen_results.csv AS LOCAL . #SAVE ARTIFACT crates/dbsp/ldbc_results.csv AS LOCAL . +flink-benchmark: + FROM +rust-sources + + # Install docker compose - earthly can do this automatically, but it installs an older version + ENV DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} + RUN mkdir -p $DOCKER_CONFIG/cli-plugins + RUN curl -SL https://github.com/docker/compose/releases/download/v2.24.0-birthday.10/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose + RUN chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose + RUN mkdir -p benchmark/flink + COPY benchmark/flink/* benchmark/flink + COPY benchmark/run-nexmark.sh benchmark + RUN apt-get install maven + RUN benchmark/flink/setup-flink.sh + + RUN echo "when,runner,mode,language,name,num_cores,num_events,elapsed" >> flink_results.csv + WITH DOCKER + RUN docker compose -f benchmark/flink/docker-compose.yml -p nexmark up --build --force-recreate --renew-anon-volumes -d && \ + docker exec -i nexmark-jobmanager-1 run.sh -q q0 2>&1 | tee flink_log.txt && \ + ./benchmark/run-nexmark.sh --runner flink --parse < flink_log.txt >> flink_results.csv + END + SAVE ARTIFACT flink_results.csv AS LOCAL . + all-tests: BUILD +formatting-check BUILD +machete @@ -573,7 +638,8 @@ all-tests: BUILD +openapi-checker BUILD +test-sql BUILD +integration-tests - # BUILD +ui-playwright-tests + BUILD +ui-playwright-tests-ct + # BUILD +ui-playwright-tests-e2e BUILD +test-docker-compose # BUILD +test-docker-compose-stable BUILD +test-debezium-mysql diff --git a/benchmark/flink/publish-flink.sh b/benchmark/flink/publish-flink.sh new file mode 100644 index 0000000000..8f08fb0a67 --- /dev/null +++ b/benchmark/flink/publish-flink.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Push results to gh-pages repo +cd "$(dirname "$0")" + +cd ../../gh-pages +if [ "$CI" = true ] ; then + git config user.email "no-reply@dbsp.systems" + git config user.name "dbsp-ci" +fi +git add . +git commit -a -m "Refreshed Flink benchmark results." +git push origin main +cd .. +rm -rf gh-pages +git clean -f diff --git a/benchmark/flink/refresh-flink-benchmark.sh b/benchmark/flink/refresh-flink-benchmark.sh new file mode 100755 index 0000000000..99243e1e08 --- /dev/null +++ b/benchmark/flink/refresh-flink-benchmark.sh @@ -0,0 +1,11 @@ +#!/bin/sh -x + +cd "$(dirname "$0")" + +# Run Flink benchmark +flink_dir=../../gh-pages/flink/ +mkdir $flink_dir +cd ../.. +earthly --verbose -P +flink-benchmark +cd benchmark/flink +mv ../../flink_results.csv $flink_dir \ No newline at end of file diff --git a/benchmark/flink/setup-flink.sh b/benchmark/flink/setup-flink.sh new file mode 100755 index 0000000000..3a053de27e --- /dev/null +++ b/benchmark/flink/setup-flink.sh @@ -0,0 +1,6 @@ +#!/bin/sh -x +cd "$(dirname "$0")" +rm -rf nexmark nexmark-flink +git clone https://github.com/blp/nexmark.git +(cd nexmark/nexmark-flink && ./build.sh) +tar xzf nexmark/nexmark-flink/nexmark-flink.tgz \ No newline at end of file diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000000..33a5860123 Binary files /dev/null and b/bun.lockb differ diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml index 6efabd3017..5378a2a29a 100644 --- a/crates/adapters/Cargo.toml +++ b/crates/adapters/Cargo.toml @@ -44,7 +44,7 @@ aws-types = "1.1.7" actix = "0.13.1" actix-web = { version = "4.4.0", default-features = false, features = ["cookies", "macros", "compress-gzip", "compress-brotli"] } mime = "0.3.16" -tracing = { version = "0.1.40" } +log = "0.4.20" # Once chrono is released with `849932` chrono version needs to be updated in size-of crate: size-of = { git = "https://github.com/gz/size-of.git", rev = "3ec40db", features = ["time-std", "ordered-float"], optional = true } futures = { version = "0.3.28" } @@ -83,6 +83,8 @@ metrics = "0.23" metrics-util = "0.17" metrics-exporter-prometheus = "0.15.1" ordered-float = { version = "4.2.0", features = ["serde"] } +minitrace = { version = "0.6", features = ["enable"] } +minitrace-jaeger = { version = "0.6" } [target.'cfg(target_os = "linux")'.dependencies] jemalloc_pprof = "0.1.0" diff --git a/crates/adapters/src/controller/mod.rs b/crates/adapters/src/controller/mod.rs index dffff4c1c1..5168b39e72 100644 --- a/crates/adapters/src/controller/mod.rs +++ b/crates/adapters/src/controller/mod.rs @@ -51,6 +51,7 @@ use crossbeam::{ }; use dbsp::circuit::{CircuitConfig, Layout}; use dbsp::profile::GraphProfile; +use log::{debug, error, info, trace}; use metrics::set_global_recorder; use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; use metrics_util::{ @@ -73,7 +74,6 @@ use std::{ thread::{spawn, JoinHandle}, time::{Duration, Instant}, }; -use tracing::{debug, error, info, trace}; use uuid::Uuid; mod error; diff --git a/crates/adapters/src/controller/stats.rs b/crates/adapters/src/controller/stats.rs index f5ccc10332..6397b07dfc 100644 --- a/crates/adapters/src/controller/stats.rs +++ b/crates/adapters/src/controller/stats.rs @@ -37,6 +37,7 @@ use crate::{ }; use anyhow::Error as AnyError; use crossbeam::sync::{ShardedLock, ShardedLockReadGuard, Unparker}; +use log::error; use metrics::{KeyName, SharedString as MetricString, Unit as MetricUnit}; use metrics_util::{ debugging::{DebugValue, Snapshot}, @@ -57,7 +58,6 @@ use std::{ Mutex, }, }; -use tracing::error; #[derive(Default, Serialize)] pub struct GlobalControllerMetrics { diff --git a/crates/adapters/src/format/avro/output.rs b/crates/adapters/src/format/avro/output.rs index 15e410dab2..5482792534 100644 --- a/crates/adapters/src/format/avro/output.rs +++ b/crates/adapters/src/format/avro/output.rs @@ -5,6 +5,7 @@ use actix_web::HttpRequest; use anyhow::{anyhow, bail, Result as AnyResult}; use apache_avro::{to_avro_datum, Schema as AvroSchema}; use erased_serde::Serialize as ErasedSerialize; +use log::{debug, error}; use pipeline_types::format::avro::AvroEncoderConfig; use pipeline_types::program_schema::Relation; use schema_registry_converter::avro_common::get_supplied_schema; @@ -15,7 +16,6 @@ use serde_yaml::Value as YamlValue; use std::borrow::Cow; use std::str::FromStr; use std::time::Duration; -use tracing::{debug, error}; // TODOs: // - This connector currently only supports raw Avro format, i.e., deletes cannot be represented. @@ -312,11 +312,11 @@ mod test { use crate::{Encoder, SerBatch}; use dbsp::utils::Tup2; use dbsp::{DBData, OrdZSet}; + use log::trace; use pipeline_types::serde_with_context::{SerializeWithContext, SqlSerdeConfig}; use proptest::proptest; use serde::Deserialize; use std::sync::Arc; - use tracing::trace; fn test_avro(avro_schema: &str, batches: Vec>>) where diff --git a/crates/adapters/src/format/json/input.rs b/crates/adapters/src/format/json/input.rs index f37b17dcdd..ef2742986c 100644 --- a/crates/adapters/src/format/json/input.rs +++ b/crates/adapters/src/format/json/input.rs @@ -452,13 +452,13 @@ mod test { transport::InputConsumer, FormatConfig, ParseError, }; + use log::trace; use pipeline_types::{ deserialize_table_record, format::json::{JsonFlavor, JsonParserConfig, JsonUpdateFormat}, serde_with_context::{DeserializeWithContext, SqlSerdeConfig}, }; use std::{borrow::Cow, fmt::Debug}; - use tracing::trace; #[derive(PartialEq, Debug, Eq)] struct TestStruct { diff --git a/crates/adapters/src/format/json/output.rs b/crates/adapters/src/format/json/output.rs index 63ab2bd0ee..2e7313f86a 100644 --- a/crates/adapters/src/format/json/output.rs +++ b/crates/adapters/src/format/json/output.rs @@ -386,12 +386,12 @@ mod test { test::{generate_test_batches_with_weights, MockOutputConsumer, TestStruct}, }; use dbsp::{utils::Tup2, OrdZSet}; + use log::trace; use pipeline_types::format::json::JsonUpdateFormat; use pipeline_types::program_schema::Relation; use proptest::prelude::*; use serde::Deserialize; use std::{cell::RefCell, fmt::Debug, rc::Rc, sync::Arc}; - use tracing::trace; trait OutputUpdate: Debug + for<'de> Deserialize<'de> + Eq + Ord { type Val; diff --git a/crates/adapters/src/format/parquet/test.rs b/crates/adapters/src/format/parquet/test.rs index b26c9336e8..c0a3c9eaea 100644 --- a/crates/adapters/src/format/parquet/test.rs +++ b/crates/adapters/src/format/parquet/test.rs @@ -173,6 +173,6 @@ fn debug_parquet_buffer(buffer: Vec) { .expect("Row iterator creation should succeed"); for maybe_record in row_iter { let record = maybe_record.expect("Record should be read successfully"); - tracing::info!("record = {:?}", record.to_string()); + log::info!("record = {:?}", record.to_string()); } } diff --git a/crates/adapters/src/integrated/delta_table/input.rs b/crates/adapters/src/integrated/delta_table/input.rs index 3697bdc4b9..a0f00255d5 100644 --- a/crates/adapters/src/integrated/delta_table/input.rs +++ b/crates/adapters/src/integrated/delta_table/input.rs @@ -27,6 +27,7 @@ use deltalake::table::PeekCommit; use deltalake::{datafusion, DeltaTable, DeltaTableBuilder, Path}; use env_logger::builder; use futures_util::StreamExt; +use log::{debug, error, info, trace}; use pipeline_types::config::{InputEndpointConfig, TransportConfigVariant}; use pipeline_types::format::json::JsonFlavor; use pipeline_types::program_schema::Relation; @@ -42,7 +43,6 @@ use tokio::select; use tokio::sync::mpsc; use tokio::sync::watch::{channel, Receiver, Sender}; use tokio::time::sleep; -use tracing::{debug, error, info, trace}; use url::Url; use utoipa::ToSchema; diff --git a/crates/adapters/src/integrated/delta_table/output.rs b/crates/adapters/src/integrated/delta_table/output.rs index f6e7544025..0298ec0bbf 100644 --- a/crates/adapters/src/integrated/delta_table/output.rs +++ b/crates/adapters/src/integrated/delta_table/output.rs @@ -20,6 +20,7 @@ use deltalake::operations::transaction::{CommitBuilder, TableReference}; use deltalake::operations::writer::{DeltaWriter, WriterConfig}; use deltalake::protocol::{DeltaOperation, SaveMode}; use deltalake::DeltaTable; +use log::{debug, error, info, trace}; use pipeline_types::transport::delta_table::DeltaTableWriteMode; use pipeline_types::{program_schema::Relation, transport::delta_table::DeltaTableWriterConfig}; use serde_arrow::schema::SerdeArrowSchema; @@ -27,7 +28,6 @@ use serde_arrow::ArrayBuilder; use std::cmp::min; use std::sync::{Arc, Weak}; use tokio::sync::mpsc::{channel, Receiver, Sender}; -use tracing::{debug, error, info, trace}; struct DeltaTableWriterInner { endpoint_id: EndpointId, diff --git a/crates/adapters/src/integrated/delta_table/test.rs b/crates/adapters/src/integrated/delta_table/test.rs index a810faef40..1cd0d3eee1 100644 --- a/crates/adapters/src/integrated/delta_table/test.rs +++ b/crates/adapters/src/integrated/delta_table/test.rs @@ -25,6 +25,7 @@ use deltalake::operations::create::CreateBuilder; use deltalake::protocol::SaveMode; use deltalake::{DeltaOps, DeltaTable, DeltaTableBuilder}; use env_logger::Env; +use log::{debug, info, trace}; use parquet::file::reader::Length; use pipeline_types::config::PipelineConfig; use pipeline_types::program_schema::{Field, Relation}; @@ -51,7 +52,6 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tempfile::{tempdir, NamedTempFile, TempDir}; use tokio::time::sleep; -use tracing::{debug, info, trace}; use uuid::Uuid; fn delta_output_serde_config() -> SqlSerdeConfig { diff --git a/crates/adapters/src/server/error.rs b/crates/adapters/src/server/error.rs index ed8cbd9d07..ea604bc11a 100644 --- a/crates/adapters/src/server/error.rs +++ b/crates/adapters/src/server/error.rs @@ -46,6 +46,7 @@ use actix_web::{ }; use anyhow::Error as AnyError; use dbsp::{operator::sample::MAX_QUANTILES, DetailedError}; +use log::{error, log, warn, Level}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as JsonValue}; use std::{ @@ -54,7 +55,6 @@ use std::{ fmt::{Display, Error as FmtError, Formatter}, sync::Arc, }; -use tracing::{error, warn, Level}; use utoipa::ToSchema; pub const MAX_REPORTED_PARSE_ERRORS: usize = 1_000; @@ -83,18 +83,6 @@ where } } -macro_rules! dyn_event { - ($lvl:ident, $($arg:tt)+) => { - match $lvl { - ::tracing::Level::TRACE => ::tracing::trace!($($arg)+), - ::tracing::Level::DEBUG => ::tracing::debug!($($arg)+), - ::tracing::Level::INFO => ::tracing::info!($($arg)+), - ::tracing::Level::WARN => ::tracing::warn!($($arg)+), - ::tracing::Level::ERROR => ::tracing::error!($($arg)+), - } - }; -} - impl ErrorResponse { pub fn from_anyerror(error: &AnyError) -> Self { let message = error.to_string(); @@ -115,9 +103,9 @@ impl ErrorResponse { E: DetailedError, { let result = Self::from_error_nolog(error); - let level = error.log_level(); - dyn_event!( - level, + + log!( + error.log_level(), "[HTTP error response] {}: {}", result.error_code, result.message @@ -294,10 +282,10 @@ impl DetailedError for PipelineError { fn log_level(&self) -> Level { match self { - Self::Initializing => Level::INFO, - Self::Terminating => Level::INFO, + Self::Initializing => Level::Info, + Self::Terminating => Level::Info, Self::ControllerError { error } => error.log_level(), - _ => Level::ERROR, + _ => Level::Error, } } } diff --git a/crates/adapters/src/server/mod.rs b/crates/adapters/src/server/mod.rs index 3a107e3bd7..86504de92a 100644 --- a/crates/adapters/src/server/mod.rs +++ b/crates/adapters/src/server/mod.rs @@ -22,6 +22,8 @@ use colored::Colorize; use dbsp::circuit::CircuitConfig; use dbsp::operator::sample::MAX_QUANTILES; use env_logger::Env; +use log::{debug, error, info, warn}; +use minitrace::collector::Config; use pipeline_types::{format::json::JsonFlavor, transport::http::EgressMode}; use pipeline_types::{query::OutputQuery, transport::http::SERVER_PORT_FILE}; use serde::Deserialize; @@ -44,7 +46,6 @@ use tokio::{ oneshot, }, }; -use tracing::{debug, error, info, warn}; use uuid::Uuid; pub mod error; @@ -382,6 +383,27 @@ where }); let _ = loginit_sender.send(()); + if config.global.tracing { + use std::net::{SocketAddr, ToSocketAddrs}; + let socket_addrs: Vec = config + .global + .tracing_endpoint_jaeger + .to_socket_addrs() + .expect("Valid `tracing_endpoint_jaeger` value (e.g., localhost:6831)") + .collect(); + let reporter = minitrace_jaeger::JaegerReporter::new( + *socket_addrs + .first() + .expect("Valid `tracing_endpoint_jaeger` value (e.g., localhost:6831)"), + config + .name + .clone() + .unwrap_or("unknown pipeline".to_string()), + ) + .unwrap(); + minitrace::set_reporter(reporter, Config::default()); + } + *state.metadata.write().unwrap() = match &args.metadata_file { None => String::new(), Some(metadata_file) => { @@ -980,8 +1002,8 @@ mod test_with_kafka { .unwrap() .current(); - //let _ = tracing::set_logger(&TEST_LOGGER); - //tracing::set_max_level(LevelFilter::Debug); + //let _ = log::set_logger(&TEST_LOGGER); + //log::set_max_level(LevelFilter::Debug); // Create topics. let kafka_resources = KafkaResources::create_topics(&[ diff --git a/crates/adapters/src/test/http.rs b/crates/adapters/src/test/http.rs index 491d3cf99e..75724e466b 100644 --- a/crates/adapters/src/test/http.rs +++ b/crates/adapters/src/test/http.rs @@ -7,7 +7,7 @@ use awc::{error::PayloadError, ClientRequest}; use csv::ReaderBuilder as CsvReaderBuilder; use csv::WriterBuilder as CsvWriterBuilder; use futures::{Stream, StreamExt}; -use tracing::trace; +use log::trace; pub struct TestHttpSender; pub struct TestHttpReceiver; diff --git a/crates/adapters/src/test/kafka.rs b/crates/adapters/src/test/kafka.rs index db9cad2a78..a8bb48938a 100644 --- a/crates/adapters/src/test/kafka.rs +++ b/crates/adapters/src/test/kafka.rs @@ -7,6 +7,7 @@ use anyhow::{anyhow, bail, Result as AnyResult}; use csv::WriterBuilder as CsvWriterBuilder; use futures::executor::block_on; use lazy_static::lazy_static; +use log::{error, info}; use pipeline_types::program_schema::Relation; use pipeline_types::transport::kafka::default_redpanda_server; use rdkafka::message::BorrowedMessage; @@ -28,7 +29,6 @@ use std::{ thread::{sleep, JoinHandle}, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use tracing::{error, info}; static MAX_TOPIC_PROBE_TIMEOUT: Duration = Duration::from_millis(20_000); diff --git a/crates/adapters/src/test/mod.rs b/crates/adapters/src/test/mod.rs index a2b8902162..7274d6ee60 100644 --- a/crates/adapters/src/test/mod.rs +++ b/crates/adapters/src/test/mod.rs @@ -6,6 +6,7 @@ use crate::{ }; use anyhow::Result as AnyResult; use dbsp::{DBData, OrdZSet, Runtime}; +use log::{Log, Metadata, Record}; use pipeline_types::serde_with_context::{ DeserializeWithContext, SerializeWithContext, SqlSerdeConfig, }; @@ -42,8 +43,24 @@ pub use mock_input_consumer::MockInputConsumer; pub use mock_output_consumer::MockOutputConsumer; use pipeline_types::program_schema::{Field, Relation}; +pub struct TestLogger; + +pub static TEST_LOGGER: TestLogger = TestLogger; + pub static DEFAULT_TIMEOUT_MS: u128 = 600_000; +impl Log for TestLogger { + fn enabled(&self, _metadata: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + println!("{} - {}", record.level(), record.args()); + } + + fn flush(&self) {} +} + /// Wait for `predicate` to become `true`. /// /// Returns the number of milliseconds elapsed or `Err(())` on timeout. diff --git a/crates/adapters/src/transport/http/input.rs b/crates/adapters/src/transport/http/input.rs index 2a3b708b45..b5dc7056a7 100644 --- a/crates/adapters/src/transport/http/input.rs +++ b/crates/adapters/src/transport/http/input.rs @@ -9,6 +9,7 @@ use actix_web::{web::Payload, HttpResponse}; use anyhow::{anyhow, Error as AnyError, Result as AnyResult}; use circular_queue::CircularQueue; use futures_util::StreamExt; +use log::debug; use num_traits::FromPrimitive; use serde::Deserialize; use std::{ @@ -19,7 +20,6 @@ use std::{ time::Duration, }; use tokio::{sync::watch, time::timeout}; -use tracing::debug; #[derive(Clone, Debug, Deserialize)] pub(crate) enum HttpIngressMode { diff --git a/crates/adapters/src/transport/http/output.rs b/crates/adapters/src/transport/http/output.rs index b3f4935673..d46c25f5d5 100644 --- a/crates/adapters/src/transport/http/output.rs +++ b/crates/adapters/src/transport/http/output.rs @@ -3,6 +3,8 @@ use actix_web::{http::header::ContentType, web::Bytes, HttpResponse}; use anyhow::{anyhow, bail, Result as AnyResult}; use async_stream::stream; use crossbeam::sync::ShardedLock; +use log::debug; +use log::error; use serde::{ser::SerializeStruct, Serializer}; use serde_json::value::RawValue; use std::{ @@ -16,8 +18,6 @@ use tokio::{ sync::{mpsc, oneshot}, time::timeout, }; -use tracing::debug; -use tracing::error; // TODO: make this configurable via endpoint config. const MAX_BUFFERS: usize = 100; diff --git a/crates/adapters/src/transport/kafka/ft/input.rs b/crates/adapters/src/transport/kafka/ft/input.rs index 86699e4187..bf29854aac 100644 --- a/crates/adapters/src/transport/kafka/ft/input.rs +++ b/crates/adapters/src/transport/kafka/ft/input.rs @@ -5,6 +5,7 @@ use crate::{InputConsumer, TransportInputEndpoint}; use anyhow::{anyhow, bail, Context, Error as AnyError, Result as AnyResult}; use crossbeam::sync::{Parker, Unparker}; use futures::executor::block_on; +use log::{debug, error, info, warn}; use pipeline_types::transport::kafka::KafkaInputConfig; use rdkafka::admin::{AdminClient, AdminOptions, NewTopic}; use rdkafka::config::{FromClientConfig, FromClientConfigAndContext}; @@ -31,7 +32,6 @@ use std::{ sync::{Arc, Mutex}, thread::{Builder, JoinHandle}, }; -use tracing::{debug, error, info, warn}; use super::CommonConfig; diff --git a/crates/adapters/src/transport/kafka/ft/mod.rs b/crates/adapters/src/transport/kafka/ft/mod.rs index 2429ad7135..475316fcc0 100644 --- a/crates/adapters/src/transport/kafka/ft/mod.rs +++ b/crates/adapters/src/transport/kafka/ft/mod.rs @@ -16,6 +16,7 @@ mod output; use crate::transport::kafka::refine_kafka_error; use anyhow::{anyhow, bail, Context, Error as AnyError, Result as AnyResult}; +use log::{debug, error, info, warn}; use pipeline_types::transport::kafka::{default_redpanda_server, KafkaLogLevel}; use rdkafka::{ client::Client as KafkaClient, @@ -37,7 +38,6 @@ use std::{ sync::Arc, time::Duration, }; -use tracing::{debug, error, info, warn}; use uuid::Uuid; pub use input::Endpoint as KafkaFtInputEndpoint; diff --git a/crates/adapters/src/transport/kafka/ft/output.rs b/crates/adapters/src/transport/kafka/ft/output.rs index 8e0b8520b8..14d778f5e6 100644 --- a/crates/adapters/src/transport/kafka/ft/output.rs +++ b/crates/adapters/src/transport/kafka/ft/output.rs @@ -4,6 +4,7 @@ use crate::{ AsyncErrorCallback, OutputEndpoint, }; use anyhow::{anyhow, bail, Context, Error as AnyError, Result as AnyResult}; +use log::{debug, info, warn}; use pipeline_types::transport::kafka::KafkaOutputConfig; use rdkafka::message::OwnedHeaders; use rdkafka::{ @@ -20,7 +21,6 @@ use std::{ sync::{Condvar, Mutex, RwLock}, time::Duration, }; -use tracing::{debug, info, warn}; use super::{count_partitions_in_topic, CommonConfig, Ctp, DataConsumerContext}; diff --git a/crates/adapters/src/transport/kafka/ft/test.rs b/crates/adapters/src/transport/kafka/ft/test.rs index 08fa3f03a0..595b828e7d 100644 --- a/crates/adapters/src/transport/kafka/ft/test.rs +++ b/crates/adapters/src/transport/kafka/ft/test.rs @@ -11,6 +11,7 @@ use crate::{ use anyhow::Error as AnyError; use crossbeam::sync::{Parker, Unparker}; use env_logger::Env; +use log::info; use proptest::prelude::*; use rdkafka::mocking::MockCluster; use std::{ @@ -22,7 +23,6 @@ use std::{ thread::sleep, time::{Duration, Instant}, }; -use tracing::info; use uuid::Uuid; /// Wait to receive all records in `data` in the same order. diff --git a/crates/adapters/src/transport/kafka/mod.rs b/crates/adapters/src/transport/kafka/mod.rs index ea30e5407c..90dd63bb42 100644 --- a/crates/adapters/src/transport/kafka/mod.rs +++ b/crates/adapters/src/transport/kafka/mod.rs @@ -104,7 +104,7 @@ impl DeferredLogging { *self.0.lock().unwrap() = Some(Vec::new()); let r = f(); for (level, fac, message) in self.0.lock().unwrap().take().unwrap().drain(..) { - tracing::info!("{level:?} {fac} {message}"); + log::info!("{level:?} {fac} {message}"); } r } @@ -151,19 +151,19 @@ impl DeferredLogging { | RDKafkaLogLevel::Alert | RDKafkaLogLevel::Critical | RDKafkaLogLevel::Error => { - tracing::error!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) + log::error!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) } RDKafkaLogLevel::Warning => { - tracing::warn!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) + log::warn!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) } RDKafkaLogLevel::Notice => { - tracing::info!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) + log::info!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) } RDKafkaLogLevel::Info => { - tracing::info!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) + log::info!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) } RDKafkaLogLevel::Debug => { - tracing::debug!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) + log::debug!(target: "librdkafka", "librdkafka: {} {}", fac, log_message) } } } diff --git a/crates/adapters/src/transport/kafka/nonft/input.rs b/crates/adapters/src/transport/kafka/nonft/input.rs index a14082d392..d49f8563a8 100644 --- a/crates/adapters/src/transport/kafka/nonft/input.rs +++ b/crates/adapters/src/transport/kafka/nonft/input.rs @@ -9,6 +9,7 @@ use crate::{ }; use anyhow::{anyhow, bail, Error as AnyError, Result as AnyResult}; use crossbeam::queue::ArrayQueue; +use log::debug; use num_traits::FromPrimitive; use pipeline_types::{secret_ref::MaybeSecretRef, transport::kafka::KafkaInputConfig}; use rdkafka::config::RDKafkaLogLevel; @@ -31,7 +32,6 @@ use std::{ thread::spawn, time::{Duration, Instant}, }; -use tracing::debug; const POLL_TIMEOUT: Duration = Duration::from_millis(100); diff --git a/crates/adapters/src/transport/kafka/nonft/output.rs b/crates/adapters/src/transport/kafka/nonft/output.rs index 291382444d..b839f48ef2 100644 --- a/crates/adapters/src/transport/kafka/nonft/output.rs +++ b/crates/adapters/src/transport/kafka/nonft/output.rs @@ -3,6 +3,7 @@ use crate::transport::secret_resolver::MaybeSecret; use crate::{AsyncErrorCallback, OutputEndpoint}; use anyhow::{anyhow, bail, Error as AnyError, Result as AnyResult}; use crossbeam::sync::{Parker, Unparker}; +use log::debug; use pipeline_types::secret_ref::MaybeSecretRef; use pipeline_types::transport::kafka::KafkaOutputConfig; use rdkafka::message::OwnedHeaders; @@ -14,7 +15,6 @@ use rdkafka::{ ClientConfig, ClientContext, }; use std::{sync::RwLock, time::Duration}; -use tracing::debug; const OUTPUT_POLLING_INTERVAL: Duration = Duration::from_millis(100); diff --git a/crates/adapters/src/transport/kafka/nonft/test.rs b/crates/adapters/src/transport/kafka/nonft/test.rs index 70d1a8236f..99f8e0cc6d 100644 --- a/crates/adapters/src/transport/kafka/nonft/test.rs +++ b/crates/adapters/src/transport/kafka/nonft/test.rs @@ -7,6 +7,7 @@ use crate::{ Controller, PipelineConfig, }; use env_logger::Env; +use log::info; use parquet::data_type::AsBytes; use proptest::prelude::*; use rdkafka::message::{BorrowedMessage, Header, Headers}; @@ -20,7 +21,6 @@ use std::{ thread::sleep, time::Duration, }; -use tracing::info; /// Wait to receive all records in `data` in the same order. fn wait_for_output_ordered(zset: &MockDeZSet, data: &[Vec]) { diff --git a/crates/adapters/src/transport/s3/mod.rs b/crates/adapters/src/transport/s3/mod.rs index be117750b0..b877249af4 100644 --- a/crates/adapters/src/transport/s3/mod.rs +++ b/crates/adapters/src/transport/s3/mod.rs @@ -1,11 +1,11 @@ use std::sync::Arc; use aws_sdk_s3::operation::{get_object::GetObjectOutput, list_objects_v2::ListObjectsV2Error}; +use log::error; use tokio::sync::{ mpsc, watch::{channel, Receiver, Sender}, }; -use tracing::error; use crate::{InputConsumer, InputReader, PipelineState, TransportInputEndpoint}; use dbsp::circuit::tokio::TOKIO; diff --git a/crates/adapters/src/transport/secret_resolver.rs b/crates/adapters/src/transport/secret_resolver.rs index b9eb570681..261ed19efb 100644 --- a/crates/adapters/src/transport/secret_resolver.rs +++ b/crates/adapters/src/transport/secret_resolver.rs @@ -6,9 +6,9 @@ use std::fs; use std::path::Path; use anyhow::{anyhow, Result as AnyResult}; +use log::debug; use pipeline_types::secret_ref::MaybeSecretRef; use regex::Regex; -use tracing::debug; /// Enumeration which holds a simple string or a resolved secret's string. /// diff --git a/crates/dbsp/Cargo.toml b/crates/dbsp/Cargo.toml index edbdd521e0..3a1fa5f9d4 100644 --- a/crates/dbsp/Cargo.toml +++ b/crates/dbsp/Cargo.toml @@ -63,8 +63,8 @@ thiserror = "1.0" uuid = { version = "1.6.1", features = ["v7"] } clap = { version = "4.4.14", features = ["derive", "env", "wrap_help"] } fdlimit = { version = "0.3.0" } -metrics = { version = "0.23.0" } -tracing = { version = "0.1.40" } +metrics = { version = "0.22.0" } +log = { version = "0.4", features = [] } rlimit = "0.10.1" serde = { version = "1.0", features = ["derive"] } ptr_meta = "0.2.0" @@ -75,6 +75,7 @@ static_assertions = "1.1.0" lazy_static = "1.4.0" dashmap = "5" zip = "0.6.2" +minitrace = "0.6" [dependencies.time] version = "0.3.20" diff --git a/crates/dbsp/src/circuit/checkpointer.rs b/crates/dbsp/src/circuit/checkpointer.rs index ad6e96e94c..52a444311d 100644 --- a/crates/dbsp/src/circuit/checkpointer.rs +++ b/crates/dbsp/src/circuit/checkpointer.rs @@ -126,15 +126,15 @@ impl Checkpointer { ) || path.is_dir() { if path.is_dir() { - tracing::debug!("Removing unused directory '{}'", path.display()); + log::debug!("Removing unused directory '{}'", path.display()); fs::remove_dir_all(path)?; } else { match fs::remove_file(path) { Ok(_) => { - tracing::debug!("Removed unused file '{}'", path.display()); + log::debug!("Removed unused file '{}'", path.display()); } Err(e) => { - tracing::warn!("Unable to remove old-checkpoint file {}: {} (the pipeline will try to delete the file again on a restart)", path.display(), e); + log::warn!("Unable to remove old-checkpoint file {}: {} (the pipeline will try to delete the file again on a restart)", path.display(), e); } } } @@ -250,10 +250,10 @@ impl Checkpointer { ); match fs::remove_file(file) { Ok(_) => { - tracing::debug!("Removed file {}", file.as_ref().display()); + log::debug!("Removed file {}", file.as_ref().display()); } Err(e) => { - tracing::warn!("Unable to remove old-checkpoint file {}: {} (the pipeline will try to delete the file again on a restart)", file.as_ref().display(), e); + log::warn!("Unable to remove old-checkpoint file {}: {} (the pipeline will try to delete the file again on a restart)", file.as_ref().display(), e); } } } @@ -266,7 +266,7 @@ impl Checkpointer { assert_ne!(cpm.uuid, Uuid::nil()); let checkpoint_dir = self.storage_path.join(cpm.uuid.to_string()); if !checkpoint_dir.exists() { - tracing::warn!( + log::warn!( "Tried to remove a checkpoint directory '{}' that doesn't exist.", checkpoint_dir.display() ); diff --git a/crates/dbsp/src/circuit/circuit_builder.rs b/crates/dbsp/src/circuit/circuit_builder.rs index ea8f907474..5d94e9ee1e 100644 --- a/crates/dbsp/src/circuit/circuit_builder.rs +++ b/crates/dbsp/src/circuit/circuit_builder.rs @@ -4517,7 +4517,6 @@ impl CircuitHandle { pub fn step(&self) -> Result<(), SchedulerError> { // TODO: Add a runtime check to prevent re-entering this method from an // operator. - self.executor.run(&self.circuit) } diff --git a/crates/dbsp/src/circuit/dbsp_handle.rs b/crates/dbsp/src/circuit/dbsp_handle.rs index 42b7ea30b7..64cf49a6e8 100644 --- a/crates/dbsp/src/circuit/dbsp_handle.rs +++ b/crates/dbsp/src/circuit/dbsp_handle.rs @@ -9,7 +9,11 @@ use crossbeam::channel::{bounded, Receiver, Select, Sender, TryRecvError}; use hashbrown::HashMap; use itertools::Either; use metrics::counter; +use minitrace::collector::SpanContext; +use minitrace::local::LocalSpan; +use minitrace::Span; pub use pipeline_types::config::{StorageCacheConfig, StorageConfig}; +use std::sync::Arc; use std::{ collections::HashSet, error::Error as StdError, @@ -328,6 +332,7 @@ impl Runtime { let runtime = Self::run(&config, move || { let worker_index = Runtime::worker_index() - worker_ofs; + let worker_index_str: &'static str = worker_index.to_string().leak(); // Drop all but one channels. This makes sure that if one of the worker panics // or exits, its channel will become disconnected. @@ -352,14 +357,13 @@ impl Runtime { } }; - // TODO: uncomment this when we have support for background compaction. - // let mut moregc = true; - while !Runtime::kill_in_progress() { // Wait for command. match command_receiver.try_recv() { - Ok(Command::Step) => { - //moregc = true; + Ok(Command::Step(span)) => { + let _guard = span.set_local_parent(); + let _worker_span = LocalSpan::enter_with_local_parent("worker-step") + .with_property(|| ("worker", worker_index_str)); let status = circuit.step().map(|_| Response::Unit); // Send response. if status_sender.send(status).is_err() { @@ -413,12 +417,7 @@ impl Runtime { // Nothing to do: do some housekeeping and relinquish the CPU if there's none // left. Err(TryRecvError::Empty) => { - // GC - /*if moregc { - moregc = circuit.gc(); - } else {*/ Runtime::parker().with(|parker| parker.park()); - //} } Err(_) => { break; @@ -471,7 +470,7 @@ impl Runtime { #[derive(Clone)] enum Command { - Step, + Step(Arc), EnableProfiler, DumpProfile, RetrieveProfile, @@ -605,7 +604,12 @@ impl DBSPHandle { pub fn step(&mut self) -> Result<(), DBSPError> { counter!("feldera.dbsp.step").increment(1); self.step_id += 1; - self.broadcast_command(Command::Step, |_, _| {}) + let span = Arc::new( + Span::root("step", SpanContext::random()) + .with_properties(|| [("step", format!("{}", self.step_id))]), + ); + let _guard = span.set_local_parent(); + self.broadcast_command(Command::Step(span), |_, _| {}) } /// Used by the checkpointer to initiate a commit on the circuit. diff --git a/crates/dbsp/src/circuit/runtime.rs b/crates/dbsp/src/circuit/runtime.rs index 4b2d66da11..96160ca036 100644 --- a/crates/dbsp/src/circuit/runtime.rs +++ b/crates/dbsp/src/circuit/runtime.rs @@ -848,7 +848,7 @@ impl RuntimeHandle { match st { StorageLocation::Temporary(path) => { if std::thread::panicking() || did_runtime_panic { - tracing::info!("Preserved runtime storage at: {:?} due to panic", path); + log::info!("Preserved runtime storage at: {:?} due to panic", path); } else { let _ = std::fs::remove_dir_all(path); } diff --git a/crates/dbsp/src/error.rs b/crates/dbsp/src/error.rs index 9e563cf447..55abfa1b26 100644 --- a/crates/dbsp/src/error.rs +++ b/crates/dbsp/src/error.rs @@ -1,5 +1,6 @@ use crate::{storage::backend::StorageError, RuntimeError, SchedulerError}; use anyhow::Error as AnyError; +use log::Level; use serde::{ser::SerializeStruct, Serialize, Serializer}; use std::{ borrow::Cow, @@ -7,12 +8,11 @@ use std::{ fmt::{Display, Error as FmtError, Formatter}, io::Error as IOError, }; -use tracing::Level; pub trait DetailedError: StdError + Serialize { fn error_code(&self) -> Cow<'static, str>; fn log_level(&self) -> Level { - Level::ERROR + Level::Error } } diff --git a/crates/dbsp/src/operator/asof_join.rs b/crates/dbsp/src/operator/asof_join.rs new file mode 100644 index 0000000000..9c50bceaeb --- /dev/null +++ b/crates/dbsp/src/operator/asof_join.rs @@ -0,0 +1,89 @@ +use crate::{ + dynamic::{DowncastTrait, DynData, DynUnit, Erase}, + DBData, OrdIndexedZSet, OrdZSet, RootCircuit, Stream, +}; + +use super::dynamic::asof_join::AsofJoinFactories; + +impl Stream> +where + K1: DBData, + V1: DBData, +{ + /// Asof-join operator. + /// + /// An asof-join operator combines records from two tables based on a common key + /// (similar to an equi-join), as well as a timestamp. It assumes that both tables + /// contain a timestamp column (`ts`). It matches each value `v` in `self` with + /// the value in `other` that has the same key and the largest timestamp not + /// exceeding `v.ts`. If there are multiple values with the same timestamp, the + /// operator picks the largest one based on the ordering (according to `Ord`) on + /// type `V2`. If there is no value `v2`, such that `v2.ts <= v.ts` in `other`, + /// then the value `None` is used, i.e., this operator behaves as a left join. + /// + /// The operator assumes that values in both collections are sorted by timestamp, + /// i.e., `impl Ord for V1` must satisfy `ts_func1(v) < ts_func1(u) ==> v < u`. + /// Similarly for `V2`: `ts_func2(v) < ts_func2(u) ==> v < u`. + /// + /// # Arguments + /// + /// * `self` - the left-hand side of the join. + /// * `other` - the right-hand side of the join. + /// * `join` - join function that maps a key, a value from `self`, and an optional + /// value from `other` to an output value. + /// * `ts_func1` - extracts the value of the timestamp column from a record in `self`. + /// * `ts_func2` - extracts the value of the timestamp column from a record in `other`. + pub fn asof_join( + &self, + other: &Stream>, + join: F, + ts_func1: TSF1, + ts_func2: TSF2, + ) -> Stream> + where + TS: DBData, + V2: DBData, + V: DBData, + F: Fn(&K1, &V1, Option<&V2>) -> V + Clone + 'static, + TSF1: Fn(&V1) -> TS + Clone + 'static, + TSF2: Fn(&V2) -> TS + 'static, + { + let join_factories = AsofJoinFactories::new::(); + + let ts_func1_clone = ts_func1.clone(); + + let dyn_ts_func1 = Box::new(move |v: &DynData, ts: &mut DynData| unsafe { + *ts.downcast_mut() = ts_func1_clone(v.downcast()) + }); + + let ts_func1_clone = ts_func1.clone(); + + let tscmp_func = Box::new(move |v1: &DynData, v2: &DynData| unsafe { + ts_func1_clone(v1.downcast()).cmp(&ts_func2(v2.downcast())) + }); + let valts_cmp_func = Box::new(move |v1: &DynData, ts: &DynData| unsafe { + // println!("cmp {v1:?}, {ts:?}"); + ts_func1(v1.downcast()).cmp(ts.downcast()) + }); + let join_func = Box::new( + move |key: &DynData, + v1: &DynData, + v2: Option<&DynData>, + cb: &mut dyn FnMut(&mut DynData, &mut DynUnit)| unsafe { + let mut v = join(key.downcast(), v1.downcast(), v2.map(|v| v.downcast())); + cb(v.erase_mut(), ().erase_mut()); + }, + ); + + self.inner() + .dyn_asof_join( + &join_factories, + &other.inner(), + dyn_ts_func1, + tscmp_func, + valts_cmp_func, + join_func, + ) + .typed() + } +} diff --git a/crates/dbsp/src/operator/dynamic/aggregate/mod.rs b/crates/dbsp/src/operator/dynamic/aggregate/mod.rs index 6603f8246f..d19afe1825 100644 --- a/crates/dbsp/src/operator/dynamic/aggregate/mod.rs +++ b/crates/dbsp/src/operator/dynamic/aggregate/mod.rs @@ -1,6 +1,7 @@ //! Aggregation operators. use dyn_clone::{clone_box, DynClone}; +use minitrace::trace; use std::{ any::TypeId, borrow::Cow, @@ -577,6 +578,7 @@ where O: IndexedZSet, Acc: DataTrait + ?Sized, { + #[trace] fn eval(&mut self, i: &Z) -> O { let mut builder = O::Builder::with_capacity(&self.factories, (), i.len()); let mut agg = self.option_output_factory.default_box(); @@ -714,6 +716,7 @@ where /// aggregation logic in the `Aggregator` trait. The second iteration /// is only needed inside nested scopes and can in the future be /// optimized to terminate early. + #[trace] fn eval_key( &mut self, key: &Z::Key, @@ -841,6 +844,7 @@ where Acc: DataTrait + ?Sized, Out: DataTrait + ?Sized, { + #[trace] fn eval(&mut self, delta: &Z, input_trace: &IT) -> Box>> { // println!( // "{}: AggregateIncremental::eval @{:?}\ndelta:{delta}", diff --git a/crates/dbsp/src/operator/dynamic/asof_join.rs b/crates/dbsp/src/operator/dynamic/asof_join.rs new file mode 100644 index 0000000000..cb36eb3bc3 --- /dev/null +++ b/crates/dbsp/src/operator/dynamic/asof_join.rs @@ -0,0 +1,1038 @@ +use std::{borrow::Cow, cmp::Ordering, marker::PhantomData, panic::Location}; + +use crate::{ + algebra::{IndexedZSet, IndexedZSetReader, OrdIndexedZSet, OrdZSet, ZCursor, ZTrace}, + circuit::{ + metadata::OperatorLocation, + operator_traits::{Operator, QuaternaryOperator}, + }, + dynamic::{ + ClonableTrait, DataTrait, DowncastTrait, DynPair, DynUnit, DynVec, DynWeightedPairs, Erase, + Factory, LeanVec, WeightTrait, WithFactory, + }, + trace::{ + cursor::{CursorEmpty, CursorPair}, + BatchFactories, BatchReaderFactories, Cursor, + }, + Circuit, DBData, DynZWeight, RootCircuit, Scope, Stream, ZWeight, +}; + +pub struct AsofJoinFactories +where + TS: DataTrait + ?Sized, + I1: IndexedZSetReader, + I2: IndexedZSetReader, + O: IndexedZSet, +{ + pub timestamp_factory: &'static dyn Factory, + pub timestamps_factory: &'static dyn Factory>, + pub left_factories: I1::Factories, + pub right_factories: I2::Factories, + pub output_factories: O::Factories, +} + +impl AsofJoinFactories +where + TS: DataTrait + ?Sized, + I1: IndexedZSetReader, + I2: IndexedZSetReader, + O: IndexedZSet, +{ + pub fn new() -> Self + where + TSType: DBData + Erase, + KType: DBData + Erase, + V1Type: DBData + Erase, + V2Type: DBData + Erase, + OKType: DBData + Erase, + OVType: DBData + Erase, + { + Self { + timestamp_factory: WithFactory::::FACTORY, + timestamps_factory: WithFactory::>::FACTORY, + left_factories: BatchReaderFactories::new::(), + right_factories: BatchReaderFactories::new::(), + output_factories: BatchReaderFactories::new::(), + } + } +} + +impl Clone for AsofJoinFactories +where + TS: DataTrait + ?Sized, + I1: IndexedZSetReader, + I2: IndexedZSetReader, + O: IndexedZSet, +{ + fn clone(&self) -> Self { + Self { + timestamp_factory: self.timestamp_factory, + timestamps_factory: self.timestamps_factory, + left_factories: self.left_factories.clone(), + right_factories: self.right_factories.clone(), + output_factories: self.output_factories.clone(), + } + } +} + +impl Stream +where + I1: IndexedZSet + Send, +{ + /// See [`Stream::asof_join`]. + #[track_caller] + pub fn dyn_asof_join( + &self, + factories: &AsofJoinFactories>, + other: &Stream, + ts_func1: Box, + tscmp_func: Box Ordering>, + valts_cmp_func: Box Ordering>, + join_func: Box>, + ) -> Stream> + where + TS: DataTrait + ?Sized, + I2: IndexedZSet, + V: DataTrait + ?Sized, + { + self.dyn_asof_join_generic( + factories, + other, + ts_func1, + tscmp_func, + valts_cmp_func, + join_func, + ) + } + + #[track_caller] + pub fn dyn_asof_join_index( + &self, + factories: &AsofJoinFactories>, + other: &Stream, + ts_func1: Box, + tscmp_func: Box Ordering>, + valts_cmp_func: Box Ordering>, + join_func: Box>, + ) -> Stream> + where + TS: DataTrait + ?Sized, + I2: IndexedZSet, + K: DataTrait + ?Sized, + V: DataTrait + ?Sized, + { + self.dyn_asof_join_generic( + factories, + other, + ts_func1, + tscmp_func, + valts_cmp_func, + join_func, + ) + } + + /// Like [`Self::dyn_asof_join_index`], but can return any indexed Z-set type. + #[track_caller] + pub fn dyn_asof_join_generic( + &self, + factories: &AsofJoinFactories, + other: &Stream, + ts_func1: Box, + tscmp_func: Box Ordering>, + valts_cmp_func: Box Ordering>, + join_func: Box>, + ) -> Stream + where + TS: DataTrait + ?Sized, + I2: IndexedZSet, + Z: IndexedZSet, + { + self.circuit().region("asof_join", || { + let left = self.dyn_shard(&factories.left_factories); + let right = other.dyn_shard(&factories.right_factories); + + let left_trace = left + .dyn_integrate_trace(&factories.left_factories) + .delay_trace(); + let right_trace = right + .dyn_integrate_trace(&factories.right_factories) + .delay_trace(); + + self.circuit().add_quaternary_operator( + AsofJoin::new( + factories.clone(), + ts_func1, + tscmp_func, + valts_cmp_func, + join_func, + Location::caller(), + ), + &left, + &left_trace, + &right, + &right_trace, + ) + }) + } +} + +pub type AsofJoinFunc = + dyn Fn(&K, &V1, Option<&V2>, &mut dyn FnMut(&mut OK, &mut OV)); + +pub struct AsofJoin +where + TS: DataTrait + ?Sized, + I1: IndexedZSet, + T1: ZTrace, + I2: IndexedZSet, + T2: ZTrace, + Z: IndexedZSet, +{ + factories: AsofJoinFactories, + ts_func1: Box, + tscmp_func: Box Ordering>, + valts_cmp_func: Box Ordering>, + join_func: Box>, + location: &'static Location<'static>, + phantom: PhantomData<(I1, T1, I2, T2, Z)>, +} + +impl AsofJoin +where + TS: DataTrait + ?Sized, + I1: IndexedZSet, + T1: ZTrace, + I2: IndexedZSet, + T2: ZTrace, + Z: IndexedZSet, +{ + pub fn new( + factories: AsofJoinFactories, + ts_func1: Box, + tscmp_func: Box Ordering>, + valts_cmp_func: Box Ordering>, + join_func: Box>, + location: &'static Location<'static>, + ) -> Self { + Self { + factories, + ts_func1, + tscmp_func, + valts_cmp_func, + join_func, + location, + phantom: PhantomData, + } + } + + /// Seek to a key; return `None` if exact match was not found. + fn try_seek<'a, C, K, V, T, R>(cursor: &'a mut C, key: &K) -> Option<&'a mut C> + where + K: DataTrait + ?Sized, + V: DataTrait + ?Sized, + R: WeightTrait + ?Sized, + C: Cursor, + { + cursor.seek_key(key); + if cursor.get_key() == Some(key) { + Some(cursor) + } else { + None + } + } + + /// Compute all timestamps affected by the changes. We will + /// update the value of the asof-join for these timestamps. + fn compute_affected_times( + &mut self, + delta1: &mut DC1, + delta2: &mut DC2, + delayed_cursor1: &mut Option<&mut ZC1>, + cursor2: &mut C2, + affected_times: &mut DynVec, + ) where + DC1: ZCursor, + DC2: ZCursor, + ZC1: ZCursor, + C2: ZCursor, + { + affected_times.clear(); + + // Update all timestamps in `delta1`. + while delta1.val_valid() { + affected_times.push_with(&mut |ts| (self.ts_func1)(delta1.val(), ts)); + delta1.step_val(); + } + + debug_assert!(affected_times.is_sorted_by(&|ts1, ts2| ts1.cmp(ts2))); + + // For `delta2`, we want to determine all timestamps in `delayed_cursor1` + // that could potentially be affected by changes in `delta2` (we don't care + // about timestamps in `cursor1` that are not in `delayed_cursor1`, since + // they are already accounted for in `delta1`). + // + // For each value in `delta2`, this includes timestamps in `delayed_cursor1` + // between the `delta2.val` (inclusive) and the next bigger value in `cursor2` + // (exclusive). + if let Some(delayed_cursor1) = delayed_cursor1 { + while delta2.val_valid() { + // Find the first value in `delayed_cursor1` following the current + // timestamp in `delta2`. + delayed_cursor1 + .seek_val_with(&|v| (self.tscmp_func)(v, delta2.val()) != Ordering::Less); + + // Find the next timestamp in `cursor2` with non-zero weight. + cursor2.seek_val_with(&|v| v > delta2.val()); + while cursor2.val_valid() && **cursor2.weight() == 0 { + cursor2.step_val() + } + + // Enumerate all timestamps in `delayed_cursor1` preceding the current + // position of `cursor2`. + while delayed_cursor1.val_valid() + && (!cursor2.val_valid() + || (self.tscmp_func)(delayed_cursor1.val(), cursor2.val()) + == Ordering::Less) + { + affected_times.push_with(&mut |ts| (self.ts_func1)(delayed_cursor1.val(), ts)); + delayed_cursor1.step_val(); + } + + if !cursor2.val_valid() { + break; + } + if !delayed_cursor1.val_valid() { + break; + } + + delta2.seek_val(cursor2.val()); + } + + affected_times.sort(); + } + + affected_times.dedup(); + } + + /// Compute asof-join for the given timestamp. + /// + /// Assumes that, if `ts` is present in `cursor1`, then `cursor1` can reverse-seek + /// to it (i.e., it hasn't passed the timestamp yet). Assumes the same for `cursor2`. + /// + /// By setting `multiplier` to +1 or -1, we get this function to produce + /// insertions and retractions respectively. + fn eval_val( + &mut self, + ts: &TS, + cursor1: &mut Option<&mut C1>, + cursor2: &mut C2, + multiplier: ZWeight, + output_tuples: &mut DynWeightedPairs, DynZWeight>, + ) where + C1: ZCursor, + C2: ZCursor, + { + let Some(cursor1) = cursor1 else { + return; + }; + + cursor1.seek_val_with_reverse(&|v| (self.valts_cmp_func)(v, ts) != Ordering::Greater); + + // if !cursor1.val_valid() { + // println!("timestamp not found"); + // } else if (self.valts_cmp_func)(cursor1.val(), ts) != Ordering::Equal { + // println!("missed timestamp, current value: {:?}", cursor1.val()); + // } + + // Iterate over all values with the same timestamp in `cursor1`. + while cursor1.val_valid() && (self.valts_cmp_func)(cursor1.val(), ts) == Ordering::Equal { + // println!("Processing value {:?}", cursor1.val()); + + cursor2 + .seek_val_with_reverse(&|v| (self.tscmp_func)(cursor1.val(), v) != Ordering::Less); + + // Ignore zero weights. + while cursor2.val_valid() && **cursor2.weight() == 0 { + cursor2.step_val_reverse(); + } + + // The weight of the result is the product of input weights. + // If there is no matching RHS value, then asof-join behaves like + // the left join: we pass `None` to the join function with weight 1. + let w1 = **cursor1.weight(); + let w2 = if cursor2.val_valid() { + **cursor2.weight() + } else { + 1 + }; + + let w = w1 * w2 * multiplier; + + (self.join_func)( + cursor1.key(), + cursor1.val(), + cursor2.get_val(), + &mut |k, v| { + // println!(" output: k={k:?}, w={w}"); + output_tuples.push_with(&mut move |tup| { + let (kv, neww) = tup.split_mut(); + let (newk, newv) = kv.split_mut(); + k.move_to(newk); + v.move_to(newv); + *unsafe { neww.downcast_mut() } = w; + }); + }, + ); + + cursor1.step_val_reverse(); + } + } + + /// Evaluate operator for the current key. + #[allow(clippy::too_many_arguments)] + fn eval_key( + &mut self, + delta1: &mut DC1, + delta2: &mut DC2, + delayed_cursor1: &mut ZC1, + delayed_cursor2: &mut ZC2, + cursor1: &mut C1, + cursor2: &mut C2, + affected_times: &mut DynVec, + output_tuples: &mut DynWeightedPairs, DynZWeight>, + ) where + DC1: ZCursor, + DC2: ZCursor, + ZC1: ZCursor, + ZC2: ZCursor, + C1: ZCursor, + C2: ZCursor, + { + let key = if delta1.key_valid() { + delta1.key() + } else { + delta2.key() + }; + + // println!("key: {key:?}"); + + // Make sure that all cursors point to the same key or are None. + let mut delayed_cursor1 = Self::try_seek(delayed_cursor1, key); + let mut delayed_cursor2 = Self::try_seek(delayed_cursor2, key); + let mut cursor1 = Self::try_seek(cursor1, key); + let mut cursor2 = Self::try_seek(cursor2, key); + + let mut empty_cursor = CursorEmpty::new(WithFactory::::FACTORY); + + if let Some(cursor2) = &mut cursor2 { + self.compute_affected_times( + delta1, + delta2, + &mut delayed_cursor1, + *cursor2, + affected_times, + ); + } else { + self.compute_affected_times( + delta1, + delta2, + &mut delayed_cursor1, + &mut empty_cursor, + affected_times, + ); + } + + // println!("Affected times: {affected_times:?}"); + + // We iterate in reverse when computing asof join, because we + // cannot use forward iteration to get cursor2 to stop at the last + // value <=cursor1. + if let Some(c) = cursor1.as_mut() { + c.fast_forward_vals() + } + if let Some(c) = cursor2.as_mut() { + c.fast_forward_vals() + } + if let Some(c) = delayed_cursor1.as_mut() { + c.fast_forward_vals() + } + if let Some(c) = delayed_cursor2.as_mut() { + c.fast_forward_vals() + } + + // if let Some(delayed_cursor1) = &mut delayed_cursor1 { + // println!("Enumerating previous values:"); + // while delayed_cursor1.val_valid() { + // println!(" {:?}", delayed_cursor1.val()); + // delayed_cursor1.step_val_reverse(); + // } + // delayed_cursor1.fast_forward_vals(); + // } + + for i in (0..affected_times.len()).rev() { + let ts = unsafe { affected_times.index_unchecked(i) }; + // println!("Processing timestamp {ts:?}"); + + // Retract old values. + if let Some(delayed_cursor2) = &mut delayed_cursor2 { + self.eval_val( + ts, + &mut delayed_cursor1, + *delayed_cursor2, + -1, + output_tuples, + ); + } else { + self.eval_val( + ts, + &mut delayed_cursor1, + &mut empty_cursor, + -1, + output_tuples, + ); + } + + // Insert new values. + if let Some(cursor2) = &mut cursor2 { + self.eval_val(ts, &mut cursor1, *cursor2, 1, output_tuples); + } else { + self.eval_val(ts, &mut cursor1, &mut empty_cursor, 1, output_tuples); + } + } + } +} + +impl Operator for AsofJoin +where + TS: DataTrait + ?Sized, + I1: IndexedZSet, + T1: ZTrace, + I2: IndexedZSet, + T2: ZTrace, + Z: IndexedZSet, +{ + fn name(&self) -> Cow<'static, str> { + Cow::Borrowed("AsofJoin") + } + + fn location(&self) -> OperatorLocation { + Some(self.location) + } + + /*fn metadata(&self, meta: &mut OperatorMeta) { + // Find the percentage of consolidated outputs + let mut output_redundancy = ((self.stats.output_tuples as f64 + - self.stats.produced_tuples as f64) + / self.stats.output_tuples as f64) + * 100.0; + if output_redundancy.is_nan() { + output_redundancy = 0.0; + } else if output_redundancy.is_infinite() { + output_redundancy = 100.0; + } + + meta.extend(metadata! { + NUM_ENTRIES_LABEL => total_size, + "batch sizes" => batch_sizes, + USED_BYTES_LABEL => MetaItem::bytes(bytes.used_bytes()), + "allocations" => bytes.distinct_allocations(), + SHARED_BYTES_LABEL => MetaItem::bytes(bytes.shared_bytes()), + "left inputs" => self.stats.lhs_tuples, + "right inputs" => self.stats.rhs_tuples, + "computed outputs" => self.stats.output_tuples, + "produced outputs" => self.stats.produced_tuples, + "output redundancy" => MetaItem::Percent(output_redundancy), + }); + }*/ + + fn fixedpoint(&self, _scope: Scope) -> bool { + true + } +} + +impl QuaternaryOperator + for AsofJoin +where + TS: DataTrait + ?Sized, + I1: IndexedZSet, + T1: ZTrace + Clone, + I2: IndexedZSet, + T2: ZTrace + Clone, + Z: IndexedZSet, +{ + fn eval<'a>( + &mut self, + delta1: Cow<'a, I1>, + delayed_trace1: Cow<'a, T1>, + delta2: Cow<'a, I2>, + delayed_trace2: Cow<'a, T2>, + ) -> Z { + let mut delta1_cursor = delta1.cursor(); + let mut delta2_cursor = delta2.cursor(); + + let mut delayed_trace1_cursor = delayed_trace1.cursor(); + let mut delayed_trace2_cursor = delayed_trace2.cursor(); + + let mut trace1_cursor = CursorPair::new(&mut delta1_cursor, &mut delayed_trace1_cursor); + let mut trace2_cursor = CursorPair::new(&mut delta2_cursor, &mut delayed_trace2_cursor); + + let mut delta1_cursor = delta1.cursor(); + let mut delta2_cursor = delta2.cursor(); + + let mut delayed_trace1_cursor = delayed_trace1.cursor(); + let mut delayed_trace2_cursor = delayed_trace2.cursor(); + + let mut output_tuples = self + .factories + .output_factories + .weighted_items_factory() + .default_box(); + + // Timestamps that need to be recomputed for each key, created here for allocation + // reuse across keys. + let mut affected_times = self.factories.timestamps_factory.default_box(); + + // Iterate over keys in delta1 and delta2. + while delta1_cursor.key_valid() && delta2_cursor.key_valid() { + match delta1_cursor.key().cmp(delta2_cursor.key()) { + Ordering::Less => { + self.eval_key( + &mut delta1_cursor, + &mut CursorEmpty::new(WithFactory::::FACTORY), + &mut delayed_trace1_cursor, + &mut delayed_trace2_cursor, + &mut trace1_cursor, + &mut trace2_cursor, + affected_times.as_mut(), + output_tuples.as_mut(), + ); + delta1_cursor.step_key(); + } + Ordering::Equal => { + self.eval_key( + &mut delta1_cursor, + &mut delta2_cursor, + &mut delayed_trace1_cursor, + &mut delayed_trace2_cursor, + &mut trace1_cursor, + &mut trace2_cursor, + affected_times.as_mut(), + output_tuples.as_mut(), + ); + delta1_cursor.step_key(); + delta2_cursor.step_key(); + } + Ordering::Greater => { + self.eval_key( + &mut CursorEmpty::new(WithFactory::::FACTORY), + &mut delta2_cursor, + &mut delayed_trace1_cursor, + &mut delayed_trace2_cursor, + &mut trace1_cursor, + &mut trace2_cursor, + affected_times.as_mut(), + output_tuples.as_mut(), + ); + delta2_cursor.step_key(); + } + } + } + + while delta1_cursor.key_valid() { + self.eval_key( + &mut delta1_cursor, + &mut CursorEmpty::new(WithFactory::::FACTORY), + &mut delayed_trace1_cursor, + &mut delayed_trace2_cursor, + &mut trace1_cursor, + &mut trace2_cursor, + affected_times.as_mut(), + output_tuples.as_mut(), + ); + delta1_cursor.step_key(); + } + + while delta2_cursor.key_valid() { + self.eval_key( + &mut CursorEmpty::new(WithFactory::::FACTORY), + &mut delta2_cursor, + &mut delayed_trace1_cursor, + &mut delayed_trace2_cursor, + &mut trace1_cursor, + &mut trace2_cursor, + affected_times.as_mut(), + output_tuples.as_mut(), + ); + delta2_cursor.step_key(); + } + + Z::dyn_from_tuples(&self.factories.output_factories, (), &mut output_tuples) + } +} + +#[cfg(test)] +mod test { + use crate::{ + algebra::F32, + circuit::CircuitConfig, + utils::{Tup2, Tup3, Tup4}, + zset, DBData, DBSPHandle, OrdIndexedZSet, OrdZSet, OutputHandle, Runtime, ZSetHandle, + ZWeight, + }; + use proptest::{collection::vec, prelude::*}; + + type Time = u64; + type CCNum = u64; + type Amt = F32; + type Transaction = Tup3; + type User = Tup3; + type Output = Tup4>; + + fn test_circuit() -> ( + DBSPHandle, + ( + ZSetHandle, + ZSetHandle, + OutputHandle>, + ), + ) { + Runtime::init_circuit(CircuitConfig::with_workers(2), |circuit| { + let (transactions, transactions_handle) = circuit.add_input_zset::(); + let (users, users_handle) = circuit.add_input_zset::(); + + let transactions = + transactions.map_index(|transaction| (transaction.1, transaction.clone())); + let users = users.map_index(|user| (user.1, user.clone())); + + let join = |_key: &CCNum, transaction: &Transaction, user: Option<&User>| { + Tup4( + transaction.0, + transaction.1, + transaction.2, + user.map(|u| u.2.clone()), + ) + }; + let ts_func1 = |transaction: &Transaction| transaction.0; + let ts_func2 = |user: &User| user.0; + + let result = transactions.asof_join(&users, join, ts_func1, ts_func2); + + let expected_result = transactions + .shard() + .integrate() + .apply2(&users.shard().integrate(), move |t, u| { + asof_join_reference(t, u, join, ts_func1, ts_func2) + }); + + result + .integrate() + .apply2(&expected_result, |actual, expected| { + assert_eq!(actual, expected) + }); + + let output_handle = result.output(); + + Ok((transactions_handle, users_handle, output_handle)) + }) + .unwrap() + } + + #[test] + fn asof_join_test() { + let (mut dbsp, (transactions, users, result)) = test_circuit(); + + // Step 1. Add some transactions without users. + transactions.append(&mut vec![ + // CCNum = 1 + Tup2(Tup3(100, 1, F32::new(10.0)), 1), + Tup2(Tup3(200, 1, F32::new(10.0)), 1), + Tup2(Tup3(300, 1, F32::new(10.0)), 1), + // CCNum = 2 + Tup2(Tup3(100, 2, F32::new(20.0)), 1), + // CCNum = 3 + Tup2(Tup3(100, 3, F32::new(30.0)), 1), + ]); + dbsp.step().unwrap(); + + assert_eq!( + result.consolidate(), + zset! { + Tup4(100, 1, F32::new(10.0), None) => 1, + Tup4(200, 1, F32::new(10.0), None) => 1, + Tup4(300, 1, F32::new(10.0), None) => 1, + Tup4(100, 2, F32::new(20.0), None) => 1, + Tup4(100, 3, F32::new(30.0), None) => 1, + } + ); + + // Step 2. Add matching users. + users.append(&mut vec![ + // CCNum = 1 + Tup2(Tup3(50, 1, "A50".to_string()), 1), + // CCNum = 2 + Tup2(Tup3(100, 2, "B100".to_string()), 1), + // CCNum = 3 + Tup2(Tup3(110, 3, "C110".to_string()), 1), + ]); + dbsp.step().unwrap(); + + assert_eq!( + result.consolidate(), + zset! { + Tup4(100, 1, F32::new(10.0), None) => -1, + Tup4(200, 1, F32::new(10.0), None) => -1, + Tup4(300, 1, F32::new(10.0), None) => -1, + Tup4(100, 2, F32::new(20.0), None) => -1, + Tup4(100, 1, F32::new(10.0), Some("A50".to_string())) => 1, + Tup4(200, 1, F32::new(10.0), Some("A50".to_string())) => 1, + Tup4(300, 1, F32::new(10.0), Some("A50".to_string())) => 1, + Tup4(100, 2, F32::new(20.0), Some("B100".to_string())) => 1, + } + ); + + // Step 3. + // - Add a more recent version of user 1. + // - Add a "future" version of user 2. + // - Add an older version of user 3. + users.append(&mut vec![ + // CCNum = 1 + Tup2(Tup3(60, 1, "A60".to_string()), 1), + Tup2(Tup3(120, 2, "B120".to_string()), 1), + Tup2(Tup3(50, 3, "C50".to_string()), 1), + ]); + + transactions.append(&mut vec![Tup2(Tup3(200, 3, F32::new(30.0)), 1)]); + + dbsp.step().unwrap(); + + assert_eq!( + result.consolidate(), + zset! { + Tup4(100, 1, F32::new(10.0), Some("A50".to_string())) => -1, + Tup4(100, 1, F32::new(10.0), Some("A60".to_string())) => 1, + Tup4(200, 1, F32::new(10.0), Some("A50".to_string())) => -1, + Tup4(200, 1, F32::new(10.0), Some("A60".to_string())) => 1, + Tup4(300, 1, F32::new(10.0), Some("A50".to_string())) => -1, + Tup4(300, 1, F32::new(10.0), Some("A60".to_string())) => 1, + Tup4(100, 3, F32::new(30.0), None) => -1, + Tup4(100, 3, F32::new(30.0), Some("C50".to_string())) => 1, + Tup4(200, 3, F32::new(30.0), Some("C110".to_string())) => 1, + } + ); + + // Step 4. Add users with old timestamps. + users.append(&mut vec![ + // CCNum = 1 + Tup2(Tup3(10, 1, "A10".to_string()), 1), + Tup2(Tup3(10, 2, "B10".to_string()), 1), + Tup2(Tup3(10, 3, "C10".to_string()), 1), + Tup2(Tup3(110, 3, "C105".to_string()), 1), + ]); + + dbsp.step().unwrap(); + + assert_eq!(result.consolidate(), zset! {}); + + // Step 5. Add multiple transactions per timestamp. + transactions.append(&mut vec![ + // CCNum = 1 + Tup2(Tup3(100, 1, F32::new(100.0)), 1), + Tup2(Tup3(200, 1, F32::new(100.0)), 1), + Tup2(Tup3(300, 1, F32::new(100.0)), 1), + // CCNum = 2 + Tup2(Tup3(100, 2, F32::new(200.0)), 1), + // CCNum = 3 + Tup2(Tup3(100, 3, F32::new(300.0)), 1), + ]); + dbsp.step().unwrap(); + + assert_eq!( + result.consolidate(), + zset! { + Tup4(100, 1, F32::new(100.0), Some("A60".to_string())) => 1, + Tup4(200, 1, F32::new(100.0), Some("A60".to_string())) => 1, + Tup4(300, 1, F32::new(100.0), Some("A60".to_string())) => 1, + Tup4(100, 2, F32::new(200.0), Some("B100".to_string())) => 1, + Tup4(100, 3, F32::new(300.0), Some("C50".to_string())) => 1, + } + ); + + // Step 6. Delete users. + users.append(&mut vec![ + Tup2(Tup3(10, 1, "A10".to_string()), -1), + Tup2(Tup3(10, 2, "B10".to_string()), -1), + Tup2(Tup3(10, 3, "C10".to_string()), -1), + ]); + + dbsp.step().unwrap(); + + assert_eq!(result.consolidate(), zset! {}); + + // Step 7. Delete more users. + users.append(&mut vec![ + // CCNum = 1 + // Tup2(Tup3(50, 1, "A50".to_string()), 1), + Tup2(Tup3(60, 1, "A60".to_string()), -1), + // CCNum = 2 + // Tup2(Tup3(100, 2, "B100".to_string()), 1), + Tup2(Tup3(120, 2, "B120".to_string()), -1), + // CCNum = 3 + // Tup2(Tup3(50, 3, "C50".to_string()), 1), + // Tup2(Tup3(110, 3, "C105".to_string()), 1), + Tup2(Tup3(110, 3, "C110".to_string()), -1), + ]); + + dbsp.step().unwrap(); + + assert_eq!( + result.consolidate(), + zset! { + Tup4(100, 1, F32::new(100.0), Some("A60".to_string())) => -1, + Tup4(200, 1, F32::new(100.0), Some("A60".to_string())) => -1, + Tup4(300, 1, F32::new(100.0), Some("A60".to_string())) => -1, + Tup4(100, 1, F32::new(10.0), Some("A60".to_string())) => -1, + Tup4(200, 1, F32::new(10.0), Some("A60".to_string())) => -1, + Tup4(300, 1, F32::new(10.0), Some("A60".to_string())) => -1, + Tup4(100, 1, F32::new(100.0), Some("A50".to_string())) => 1, + Tup4(200, 1, F32::new(100.0), Some("A50".to_string())) => 1, + Tup4(300, 1, F32::new(100.0), Some("A50".to_string())) => 1, + Tup4(100, 1, F32::new(10.0), Some("A50".to_string())) => 1, + Tup4(200, 1, F32::new(10.0), Some("A50".to_string())) => 1, + Tup4(300, 1, F32::new(10.0), Some("A50".to_string())) => 1, + Tup4(200, 3, F32::new(30.0), Some("C110".to_string())) => -1, + Tup4(200, 3, F32::new(30.0), Some("C105".to_string())) => 1, + } + ); + } + + #[test] + fn asof_join_regressions() { + let (mut dbsp, (transactions, users, _result)) = test_circuit(); + + users.append(&mut vec![ + Tup2(Tup3(37, 0, "L".to_string()), 1), + Tup2(Tup3(0, 0, "A".to_string()), 1), + ]); + dbsp.step().unwrap(); + + transactions.append(&mut vec![Tup2(Tup3(37, 0, F32::new(0.0)), 1)]); + dbsp.step().unwrap(); + + users.append(&mut vec![Tup2(Tup3(37, 0, "L".to_string()), -1)]); + dbsp.step().unwrap(); + + users.append(&mut vec![Tup2(Tup3(0, 0, "A".to_string()), -1)]); + dbsp.step().unwrap(); + } + + /// Reference implementaton of asof-join for testing. + fn asof_join_reference( + left: &OrdIndexedZSet, + right: &OrdIndexedZSet, + join: F, + ts_func1: TSF1, + ts_func2: TSF2, + ) -> OrdZSet + where + TS: DBData, + K: DBData, + V1: DBData, + V2: DBData, + OV: DBData, + F: Fn(&K, &V1, Option<&V2>) -> OV + Clone + 'static, + TSF1: Fn(&V1) -> TS + Clone + 'static, + TSF2: Fn(&V2) -> TS + 'static, + { + let left = left.iter().collect::>(); + let right = right.iter().collect::>(); + + let mut result = Vec::new(); + + for (k, v1, w1) in left.iter() { + let (ov, ow) = right + .iter() + .rev() + .find_map(|(k2, v2, w2)| { + if k2 == k && ts_func2(v2) <= ts_func1(v1) { + Some((join(k, v1, Some(v2)), w1 * w2)) + } else { + None + } + }) + .unwrap_or_else(|| (join(k, v1, None), *w1)); + result.push(Tup2(ov, ow)); + } + + OrdZSet::from_keys((), result) + } + + prop_compose! { + fn transaction() + (time in 0..100u64, + cc_num in 0..10u64, + amt in 0..100i32, + w in 1..=2 as ZWeight) + -> Tup2 { + Tup2(Tup3(time, cc_num, F32::new(amt as f32)), w) + } + } + + prop_compose! { + fn user() + (time in 0..100u64, + cc_num in 0..5u64, + name in "[A-Z][a-z]{5}", + w in 1..=2 as ZWeight) + -> Tup2 { + Tup2(Tup3(time, cc_num, name), w) + } + } + + prop_compose! { + fn input() + (transactions in vec(transaction(), 0..20), + users in vec(user(), 0..10)) + -> (Vec>, Vec>) { + (transactions, users) + } + } + + prop_compose! { + fn inputs(steps: usize) + (inputs in vec(input(), 0..=steps)) + -> Vec<(Vec>, Vec>)> { + inputs + } + } + + proptest! { + #[test] + fn asof_join_proptest(inputs in inputs(50)) { + let (mut dbsp, (htransactions, husers, _hresult)) = test_circuit(); + + let mut deletions = inputs.clone(); + for (ts, us) in deletions.iter_mut() { + for Tup2(_t, w) in ts.iter_mut(){ + *w = -*w; + } + for Tup2(_u, w) in us.iter_mut() { + *w =-*w; + } + } + + for (mut transactions, mut users) in inputs { + htransactions.append(&mut transactions); + husers.append(&mut users); + + dbsp.step().unwrap(); + } + + for (mut transactions, mut users) in deletions { + htransactions.append(&mut transactions); + husers.append(&mut users); + + dbsp.step().unwrap(); + } + } + } +} diff --git a/crates/dbsp/src/operator/dynamic/communication/gather.rs b/crates/dbsp/src/operator/dynamic/communication/gather.rs index 9022bf6916..f73bf4daac 100644 --- a/crates/dbsp/src/operator/dynamic/communication/gather.rs +++ b/crates/dbsp/src/operator/dynamic/communication/gather.rs @@ -11,6 +11,7 @@ use crate::{ use arc_swap::ArcSwap; use crossbeam::atomic::AtomicConsume; use crossbeam_utils::CachePadded; +use minitrace::trace; use std::{ borrow::Cow, marker::PhantomData, @@ -259,11 +260,13 @@ impl SinkOperator for GatherProducer where T: Clone + Send + 'static, { + #[trace] fn eval(&mut self, input: &T) { // Safety: `worker` is guaranteed to be a valid & unique worker index unsafe { self.gather.push(self.worker, input.clone()) } } + #[trace] fn eval_owned(&mut self, input: T) { // Safety: `worker` is guaranteed to be a valid & unique worker index unsafe { self.gather.push(self.worker, input) } @@ -322,6 +325,7 @@ impl SourceOperator for GatherConsumer where T: Batch