From 18577c2c2dde24b3c85ae3dfcd6225c01150622e Mon Sep 17 00:00:00 2001 From: Qiping Luo Date: Thu, 8 Aug 2024 15:54:14 +0800 Subject: [PATCH] [rust] implement pushconsumer (#767) * [WIP] push consumer * add push consumer * use enum BackOffRetryPolicy * refactor to optimize struct of Client * add testcases * implement standard push consumer * fix typo * format code * add license * remove extra clones * flush queues when shutting down actors * use TryFrom trait to parse retry policy * optimize message parsing error * return option instead of result * fix code style * remove lifetime * save time on repeating Box * chore: update README doc, reflecting we are supporting push-consumer in Rust Signed-off-by: Zhanhui Li * refactor pushconsumer * fix typo * fix format * use correct tokio_util version * log error on process assignments * fix tokio-util version * implement Debug trait for AckEntryItem * increase converage rate * optimize code structure * chore: test macos on macos-12 Signed-off-by: Zhanhui Li * fix: upgrade MSRV to 1.70 as dependent crate home requires it Signed-off-by: Zhanhui Li * fix: remove Cargo.lock.min Signed-off-by: Zhanhui Li * fix: fix MSRV check Signed-off-by: Zhanhui Li * fix: specify working-directory Signed-off-by: Zhanhui Li * chore: make cargo-clippy pass Signed-off-by: Zhanhui Li --------- Signed-off-by: Zhanhui Li Co-authored-by: Zhanhui Li --- .github/workflows/rust_build.yml | 38 +- README-CN.md | 4 +- README.md | 4 +- rust/.cargo/Cargo.lock.min | 2226 ------------------------------ rust/Cargo.toml | 3 +- rust/examples/push_consumer.rs | 53 + rust/src/client.rs | 759 +++++----- rust/src/conf.rs | 355 ++++- rust/src/error.rs | 3 + rust/src/lib.rs | 3 + rust/src/model/common.rs | 75 +- rust/src/model/message.rs | 13 +- rust/src/model/transaction.rs | 2 +- rust/src/producer.rs | 74 +- rust/src/push_consumer.rs | 1956 ++++++++++++++++++++++++++ rust/src/session.rs | 234 ++-- rust/src/simple_consumer.rs | 20 +- rust/src/util.rs | 118 +- 18 files changed, 3132 insertions(+), 2808 deletions(-) delete mode 100644 rust/.cargo/Cargo.lock.min create mode 100644 rust/examples/push_consumer.rs create mode 100644 rust/src/push_consumer.rs diff --git a/.github/workflows/rust_build.yml b/.github/workflows/rust_build.yml index 7113df759..90e48cedc 100644 --- a/.github/workflows/rust_build.yml +++ b/.github/workflows/rust_build.yml @@ -3,8 +3,11 @@ on: workflow_call: jobs: fmt: + name: code style check runs-on: ubuntu-latest - name: code style check + defaults: + run: + working-directory: ./rust steps: - uses: actions/checkout@v3 with: @@ -15,11 +18,13 @@ jobs: toolchain: stable components: rustfmt - name: Code format check - working-directory: ./rust run: cargo fmt --check clippy: name: clippy runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust steps: - uses: actions/checkout@v3 with: @@ -31,11 +36,13 @@ jobs: components: clippy # Run clippy - name: clippy check - working-directory: ./rust run: cargo clippy --all-features -- -D warnings doc: + name: doc check runs-on: ubuntu-latest - name: doc check + defaults: + run: + working-directory: ./rust steps: - uses: actions/checkout@v3 with: @@ -45,27 +52,26 @@ jobs: with: toolchain: stable - name: Build doc - working-directory: ./rust run: cargo doc --no-deps --all-features env: RUSTDOCFLAGS: --cfg docsrs msrv: runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust strategy: matrix: - msrv: [1.61] - name: msrv ${{ matrix.msrv }} check + msrv: [1.70.0] + name: MSRV ${{ matrix.msrv }} check steps: - uses: actions/checkout@v3 with: submodules: true - - name: Install ${{ matrix.msrv }} - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.msrv }} - - name: Check MSRV ${{ matrix.msrv }} - working-directory: ./rust - run: cp .cargo/Cargo.lock.min Cargo.lock && cargo +${{ matrix.msrv }} fetch && cargo +${{ matrix.msrv }} check --locked --frozen + - name: Install cargo-msrv + run: | + cargo install cargo-msrv + cargo msrv --min ${{ matrix.msrv }} build: name: "${{ matrix.os }}" runs-on: ${{ matrix.os }} @@ -76,8 +82,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-11, windows-2022] - msrv: [1.61] + os: [ubuntu-20.04, macos-12, windows-2022] + msrv: [1.70] steps: - uses: actions/checkout@v2 with: diff --git a/README-CN.md b/README-CN.md index 51d92e496..08dfc6d74 100644 --- a/README-CN.md +++ b/README-CN.md @@ -24,8 +24,8 @@ | Producer with timed/delay messages | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | | Producer with transactional messages | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | | Simple consumer | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | -| Push consumer with concurrent message listener | ✅ | ✅ | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | -| Push consumer with FIFO message listener | ✅ | ✅ | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | +| Push consumer with concurrent message listener | ✅ | ✅ | 🚧 | 🚧 | ✅ | 🚧 | 🚧 | 🚧 | +| Push consumer with FIFO message listener | ✅ | ✅ | 🚧 | 🚧 | ✅ | 🚧 | 🚧 | 🚧 | ## 先决条件和构建 diff --git a/README.md b/README.md index 5ab623285..a9edcc848 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ Provide cloud-native and robust solutions for Java, C++, C#, Golang, Rust and al | Producer with timed/delay messages | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | | Producer with transactional messages | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | | Simple consumer | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | -| Push consumer with concurrent message listener | ✅ | ✅ | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | -| Push consumer with FIFO message listener | ✅ | ✅ | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | +| Push consumer with concurrent message listener | ✅ | ✅ | 🚧 | 🚧 | ✅ | 🚧 | 🚧 | 🚧 | +| Push consumer with FIFO message listener | ✅ | ✅ | 🚧 | 🚧 | ✅ | 🚧 | 🚧 | 🚧 | ## Prerequisite and Build diff --git a/rust/.cargo/Cargo.lock.min b/rust/.cargo/Cargo.lock.min deleted file mode 100644 index 4935f17a4..000000000 --- a/rust/.cargo/Cargo.lock.min +++ /dev/null @@ -1,2226 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "anyhow" -version = "1.0.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" - -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "async-trait" -version = "0.1.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "awaitility" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "640d02c7cde0d8b9e09b12cc08c38744fe26b8df6ed641c62a1d0d3fd691e3b5" - -[[package]] -name = "axum" -version = "0.6.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113713495a32dd0ab52baf5c10044725aa3aec00b31beda84218e469029b72a3" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bumpalo" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" - -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "ctor" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "dashmap" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" -dependencies = [ - "cfg-if", - "hashbrown", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "downcast" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" - -[[package]] -name = "either" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" - -[[package]] -name = "errno" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "float-cmp" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" -dependencies = [ - "num-traits", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "fragile" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" - -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-executor" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - -[[package]] -name = "futures-task" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" - -[[package]] -name = "futures-util" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", -] - -[[package]] -name = "h2" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util 0.7.7", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - -[[package]] -name = "http" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" - -[[package]] -name = "hyper" -version = "0.14.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" - -[[package]] -name = "js-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.141" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" - -[[package]] -name = "linux-raw-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2" - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "mac_address" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e3235c8382b7653c6408ed1b08dd379bdb9fdf990fb0bbae3db2cc0ae963" -dependencies = [ - "nix", - "winapi", -] - -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - -[[package]] -name = "matchit" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "minitrace" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317e28b8c337ada2fd437611c241ce053d5b7f5480b79e945597996b87b1de96" -dependencies = [ - "futures", - "minitrace-macro", - "minstant", - "once_cell", - "parking_lot", - "pin-project", -] - -[[package]] -name = "minitrace-macro" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77814d165883613a1846517efdc50b88fabd9c210b7ff4d3745b38b99d539652" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "minstant" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5dcfca9a0725105ac948b84cfeb69c3942814c696326743797215413f854b9" -dependencies = [ - "ctor", - "libc", - "wasi 0.7.0", -] - -[[package]] -name = "mio" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", -] - -[[package]] -name = "mockall" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" -dependencies = [ - "cfg-if", - "downcast", - "fragile", - "lazy_static", - "mockall_derive", - "predicates", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "mockall_double" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71c7bb287375187c775cf82e2dcf1bef3388aaf58f0789a77f9c7ab28466f6" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "nix" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" -dependencies = [ - "bitflags", - "cc", - "cfg-if", - "libc", - "memoffset", -] - -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - -[[package]] -name = "num-traits" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" -dependencies = [ - "hermit-abi 0.2.6", - "libc", -] - -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - -[[package]] -name = "once_cell" -version = "1.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "opentelemetry" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4b8347cc26099d3aeee044065ecc3ae11469796b4d65d065a23a584ed92a6f" -dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", -] - -[[package]] -name = "opentelemetry-otlp" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8af72d59a4484654ea8eb183fea5ae4eb6a41d7ac3e3bae5f4d2a282a3a7d3ca" -dependencies = [ - "async-trait", - "futures", - "futures-util", - "http", - "opentelemetry", - "opentelemetry-proto", - "prost 0.11.9", - "thiserror", - "tokio", - "tonic 0.8.3", -] - -[[package]] -name = "opentelemetry-proto" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "045f8eea8c0fa19f7d48e7bc3128a39c2e5c533d5c61298c548dfefc1064474c" -dependencies = [ - "futures", - "futures-util", - "opentelemetry", - "prost 0.11.9", - "tonic 0.8.3", -] - -[[package]] -name = "opentelemetry_api" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed41783a5bf567688eb38372f2b7a8530f5a607a4b49d38dd7573236c23ca7e2" -dependencies = [ - "fnv", - "futures-channel", - "futures-util", - "indexmap", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b3a2a91fdbfdd4d212c0dcc2ab540de2c2bcbbd90be17de7a7daf8822d010c1" -dependencies = [ - "async-trait", - "crossbeam-channel", - "dashmap", - "fnv", - "futures-channel", - "futures-executor", - "futures-util", - "once_cell", - "opentelemetry_api", - "percent-encoding", - "rand", - "thiserror", - "tokio", - "tokio-stream", -] - -[[package]] -name = "os_info" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006e42d5b888366f1880eda20371fedde764ed2213dc8496f49622fa0c99cd5e" -dependencies = [ - "log", - "serde", - "winapi", -] - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "windows-sys 0.45.0", -] - -[[package]] -name = "percent-encoding" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" - -[[package]] -name = "petgraph" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" -dependencies = [ - "fixedbitset", - "indexmap", -] - -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "predicates" -version = "2.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" -dependencies = [ - "difflib", - "float-cmp", - "itertools", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f883590242d3c6fc5bf50299011695fa6590c2c70eac95ee1bdb9a733ad1a2" - -[[package]] -name = "predicates-tree" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ff541861505aabf6ea722d2131ee980b8276e10a1297b94e896dd8b621850d" -dependencies = [ - "predicates-core", - "termtree", -] - -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "1.0.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive 0.9.0", -] - -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive 0.11.9", -] - -[[package]] -name = "prost-build" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" -dependencies = [ - "bytes", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost 0.11.9", - "prost-types", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost 0.11.9", -] - -[[package]] -name = "quote" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", -] - -[[package]] -name = "regex" -version = "1.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi", -] - -[[package]] -name = "rocketmq" -version = "0.1.2" -dependencies = [ - "anyhow", - "async-trait", - "awaitility", - "byteorder", - "futures", - "hex", - "hostname", - "lazy_static", - "mac_address", - "minitrace", - "mockall", - "mockall_double", - "once_cell", - "opentelemetry", - "opentelemetry-otlp", - "os_info", - "parking_lot", - "prost 0.11.9", - "prost-types", - "regex", - "ring", - "siphasher", - "slog", - "slog-async", - "slog-json", - "slog-term", - "thiserror", - "time", - "tokio", - "tokio-rustls", - "tokio-stream", - "tonic 0.9.2", - "tonic-build", - "version_check", - "which", - "wiremock-grpc", -] - -[[package]] -name = "rustix" -version = "0.37.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustls" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07180898a28ed6a7f7ba2311594308f595e3dd2e3c3812fa0a80a47b45f17e5d" -dependencies = [ - "log", - "ring", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" -dependencies = [ - "base64 0.21.0", -] - -[[package]] -name = "rustls-webpki" -version = "0.100.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" - -[[package]] -name = "ryu" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" - -[[package]] -name = "schannel" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" -dependencies = [ - "windows-sys 0.42.0", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "security-framework" -version = "2.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.160" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" - -[[package]] -name = "serde_json" -version = "1.0.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "siphasher" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" - -[[package]] -name = "slab" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] - -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" - -[[package]] -name = "slog-async" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" -dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", -] - -[[package]] -name = "slog-json" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" -dependencies = [ - "serde", - "serde_json", - "slog", - "time", -] - -[[package]] -name = "slog-term" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" -dependencies = [ - "atty", - "slog", - "term", - "thread_local", - "time", -] - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - -[[package]] -name = "tempfile" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" -dependencies = [ - "cfg-if", - "fastrand", - "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", -] - -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - -[[package]] -name = "termtree" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" - -[[package]] -name = "thiserror" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "thread_local" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" -dependencies = [ - "cfg-if", - "once_cell", -] - -[[package]] -name = "time" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c" -dependencies = [ - "itoa", - "libc", - "num_threads", - "time-macros", -] - -[[package]] -name = "time-macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" - -[[package]] -name = "tokio" -version = "1.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" -dependencies = [ - "autocfg", - "bytes", - "libc", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.45.0", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-macros" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "tokio-rustls" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" -dependencies = [ - "async-stream", - "async-trait", - "base64 0.13.1", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.9.0", - "prost-derive 0.9.0", - "tokio", - "tokio-stream", - "tokio-util 0.6.10", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.1", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.11.9", - "prost-derive 0.11.9", - "tokio", - "tokio-stream", - "tokio-util 0.7.7", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.21.0", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.11.9", - "rustls-native-certs", - "rustls-pemfile", - "tokio", - "tokio-rustls", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util 0.7.7", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" -dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "tracing-core" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" -dependencies = [ - "once_cell", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - -[[package]] -name = "unicode-ident" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - -[[package]] -name = "urlencoding" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 1.0.109", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" - -[[package]] -name = "web-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" - -[[package]] -name = "wiremock-grpc" -version = "0.0.3-alpha2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "319deda70ff78f1ee33c271bda813160ae13ef33bda15ebe66b5a377d1911ec9" -dependencies = [ - "http-body", - "log", - "prost 0.9.0", - "rand", - "tokio", - "tonic 0.6.2", -] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index bc888279c..a5f4dcb36 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -18,7 +18,7 @@ name = "rocketmq" version = "5.0.0" edition = "2021" -rust-version = "1.61" +rust-version = "1.70" authors = [ "SSpirits ", "Zhanhui Li ", @@ -66,6 +66,7 @@ mockall_double = "0.3.0" siphasher = "0.3.10" ring = "0.16.20" +tokio-util = { version = "=0.7.10", features = ["rt"] } [build-dependencies] tonic-build = "0.9.0" diff --git a/rust/examples/push_consumer.rs b/rust/examples/push_consumer.rs new file mode 100644 index 000000000..69a6a3ac5 --- /dev/null +++ b/rust/examples/push_consumer.rs @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +use rocketmq::{ + conf::{ClientOption, PushConsumerOption}, + model::common::{ConsumeResult, FilterExpression, FilterType}, + MessageListener, PushConsumer, +}; +use std::time::Duration; +use tokio::time; + +#[tokio::main] +async fn main() { + let mut client_option = ClientOption::default(); + client_option.set_access_url("localhost:8081"); + client_option.set_enable_tls(false); + + let mut option = PushConsumerOption::default(); + option.set_consumer_group("test"); + option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + + let callback: MessageListener = Box::new(|message| { + println!("Receive message: {:?}", message); + ConsumeResult::SUCCESS + }); + + let mut push_consumer = PushConsumer::new(client_option, option, callback).unwrap(); + let start_result = push_consumer.start().await; + if start_result.is_err() { + eprintln!( + "push consumer start failed: {:?}", + start_result.unwrap_err() + ); + return; + } + + time::sleep(Duration::from_secs(60)).await; + + let _ = push_consumer.shutdown().await; +} diff --git a/rust/src/client.rs b/rust/src/client.rs index b47a7a020..b2c770924 100644 --- a/rust/src/client.rs +++ b/rust/src/client.rs @@ -26,10 +26,10 @@ use parking_lot::Mutex; use prost_types::Duration; use slog::{debug, error, info, o, warn, Logger}; use tokio::select; -use tokio::sync::{mpsc, oneshot, RwLock}; +use tokio::sync::{mpsc, oneshot}; use tokio::time::Instant; -use crate::conf::{ClientOption, SettingsAware}; +use crate::conf::ClientOption; use crate::error::{ClientError, ErrorKind}; use crate::model::common::{ClientType, Endpoints, Route, RouteStatus, SendReceipt}; use crate::model::message::AckMessageEntry; @@ -39,24 +39,34 @@ use crate::pb::{ AckMessageRequest, AckMessageResultEntry, ChangeInvisibleDurationRequest, Code, FilterExpression, HeartbeatRequest, HeartbeatResponse, Message, MessageQueue, NotifyClientTerminationRequest, QueryRouteRequest, ReceiveMessageRequest, Resource, - SendMessageRequest, Status, TelemetryCommand, + SendMessageRequest, TelemetryCommand, }; +use crate::session::RPCClient; #[double] -use crate::session::SessionManager; -use crate::session::{RPCClient, Session}; +use crate::session::Session; +use crate::util::{handle_response_status, select_message_queue}; -pub(crate) struct Client { +#[derive(Debug)] +pub(crate) struct Client { logger: Logger, option: ClientOption, session_manager: Arc, - route_table: Arc>>, + route_manager: TopicRouteManager, id: String, access_endpoints: Endpoints, - settings: Arc>, + settings: TelemetryCommand, telemetry_command_tx: Option>, shutdown_tx: Option>, } +#[derive(Debug, Clone)] +pub(crate) struct TopicRouteManager { + route_table: Arc>>, + logger: Logger, + access_endpoints: Endpoints, + namespace: String, +} + static CLIENT_ID_SEQUENCE: Lazy = Lazy::new(|| AtomicUsize::new(0)); const OPERATION_CLIENT_NEW: &str = "client.new"; @@ -69,38 +79,27 @@ const OPERATION_SEND_MESSAGE: &str = "client.send_message"; const OPERATION_RECEIVE_MESSAGE: &str = "client.receive_message"; const OPERATION_ACK_MESSAGE: &str = "client.ack_message"; -impl Debug for Client -where - S: SettingsAware + 'static, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Client") - .field("id", &self.id) - .field("access_endpoints", &self.access_endpoints) - .field("option", &self.option) - .finish() - } -} - #[automock] -impl Client -where - S: SettingsAware + 'static + Send + Sync, -{ +impl Client { pub(crate) fn new( logger: &Logger, option: ClientOption, - settings: Arc>, + settings: TelemetryCommand, ) -> Result { let id = Self::generate_client_id(); let endpoints = Endpoints::from_url(option.access_url()) .map_err(|e| e.with_operation(OPERATION_CLIENT_NEW))?; let session_manager = SessionManager::new(logger, id.clone(), &option); + let route_manager = TopicRouteManager::new( + logger.clone(), + option.get_namespace().to_string(), + endpoints.clone(), + ); Ok(Client { logger: logger.new(o!("component" => "client")), option, session_manager: Arc::new(session_manager), - route_table: Arc::new(Mutex::new(HashMap::new())), + route_manager, id, access_endpoints: endpoints, settings, @@ -117,6 +116,10 @@ where self.shutdown_tx.is_some() } + pub(crate) fn get_route_manager(&self) -> TopicRouteManager { + self.route_manager.clone() + } + pub(crate) async fn start( &mut self, telemetry_command_tx: mpsc::Sender, @@ -133,15 +136,14 @@ where self.telemetry_command_tx = Some(telemetry_command_tx); - let rpc_client = self + let mut rpc_client = self .get_session() .await .map_err(|error| error.with_operation(OPERATION_CLIENT_START))?; - let route_table = Arc::clone(&self.route_table); - let endpoints = self.access_endpoints.clone(); + let route_manager = self.route_manager.clone(); - let settings = Arc::clone(&self.settings); + let settings = self.settings.clone(); tokio::spawn(async move { rpc_client.is_started(); let seconds_30 = std::time::Duration::from_secs(30); @@ -193,12 +195,8 @@ where continue; } for mut session in sessions.unwrap() { - let command; - { - command = settings.read().await.build_telemetry_command(); - } let peer = session.peer().to_string(); - let result = session.update_settings(command).await; + let result = session.update_settings(settings.clone()).await; if result.is_err() { error!(logger, "sync settings failed: failed to call rpc: {}", result.unwrap_err()); continue; @@ -208,29 +206,9 @@ where }, _ = sync_route_timer.tick() => { - let topics: Vec; - { - topics = route_table.lock().keys().cloned().collect(); - } - debug!(logger, "update topic route of topics {:?}", topics); - for topic in topics { - let result = Self::topic_route_inner( - logger.clone(), - rpc_client.shadow_session(), - Arc::clone(&route_table), - namespace.clone(), - endpoints.clone(), - topic.as_str(), - ) - .await; - if result.is_err() { - warn!( - logger, - "sync route of topic = {:?} failed, reason = {:?}", - topic, - result.err() - ); - } + let result = route_manager.sync_route_data(&mut rpc_client).await; + if result.is_err() { + error!(logger, "sync route failed: {}", result.unwrap_err()); } }, _ = &mut shutdown_rx => { @@ -296,17 +274,13 @@ where ) } - async fn build_telemetry_command(&self) -> TelemetryCommand { - self.settings.read().await.build_telemetry_command() - } - pub(crate) async fn get_session(&self) -> Result { self.check_started(OPERATION_GET_SESSION)?; let session = self .session_manager .get_or_create_session( &self.access_endpoints, - self.build_telemetry_command().await, + self.settings.clone(), self.telemetry_command_tx.clone().unwrap(), ) .await?; @@ -321,7 +295,7 @@ where .session_manager .get_or_create_session( endpoints, - self.build_telemetry_command().await, + self.settings.clone(), self.telemetry_command_tx.clone().unwrap(), ) .await?; @@ -329,14 +303,7 @@ where } pub(crate) fn topic_route_from_cache(&self, topic: &str) -> Option> { - self.route_table.lock().get(topic).and_then(|route_status| { - if let RouteStatus::Found(route) = route_status { - // debug!(self.logger, "get route for topic={} from cache", topic); - Some(Arc::clone(route)) - } else { - None - } - }) + self.route_manager.find_topic_route(topic) } pub(crate) async fn topic_route( @@ -349,139 +316,10 @@ where return Ok(route); } } - let rpc_client = self.get_session().await?; - let logger = self.logger.clone(); - let route_table = Arc::clone(&self.route_table); - Self::topic_route_inner( - logger, - rpc_client, - route_table, - self.option.namespace.clone(), - self.access_endpoints.clone(), - topic, - ) - .await - } - - async fn query_topic_route( - mut rpc_client: T, - namespace: String, - access_endpoints: Endpoints, - topic: &str, - ) -> Result { - let request = QueryRouteRequest { - topic: Some(Resource { - name: topic.to_owned(), - resource_namespace: namespace, - }), - endpoints: Some(access_endpoints.into_inner()), - }; - - let response = rpc_client.query_route(request).await?; - handle_response_status(response.status, OPERATION_QUERY_ROUTE)?; - - let route = Route { - index: AtomicUsize::new(0), - queue: response.message_queues, - }; - Ok(route) - } - - async fn topic_route_inner( - logger: Logger, - rpc_client: T, - route_table: Arc>>, - namespace: String, - endpoints: Endpoints, - topic: &str, - ) -> Result, ClientError> { - debug!(logger, "query route for topic={}", topic); - let rx = match route_table - .lock() - .entry(topic.to_owned()) - .or_insert_with(|| RouteStatus::Querying(None)) - { - RouteStatus::Found(_route) => None, - RouteStatus::Querying(ref mut option) => { - match option { - Some(vec) => { - // add self to waiting list - let (tx, rx) = oneshot::channel(); - vec.push(tx); - Some(rx) - } - None => { - // there is no ongoing request, so we need to send a new request - let _ = option.insert(Vec::new()); - None - } - } - } - }; - - // wait for inflight request - if let Some(rx) = rx { - return match rx.await { - Ok(route) => route, - Err(_e) => Err(ClientError::new( - ErrorKind::ChannelReceive, - "wait for inflight query topic route request failed", - OPERATION_QUERY_ROUTE, - )), - }; - } - - let result = Self::query_topic_route(rpc_client, namespace, endpoints, topic).await; - - // send result to all waiters - if let Ok(route) = result { - debug!( - logger, - "query route for topic={} success: route={:?}", topic, route - ); - let route = Arc::new(route); - let mut route_table_lock = route_table.lock(); - - // if message queues in previous and new route are the same, just keep the previous. - if let Some(RouteStatus::Found(prev)) = route_table_lock.get(topic) { - if prev.queue == route.queue { - return Ok(Arc::clone(prev)); - } - } - - let prev = - route_table_lock.insert(topic.to_owned(), RouteStatus::Found(Arc::clone(&route))); - info!(logger, "update route for topic={}", topic); - - if let Some(RouteStatus::Querying(Some(mut v))) = prev { - for item in v.drain(..) { - let _ = item.send(Ok(Arc::clone(&route))); - } - }; - Ok(route) - } else { - let err = result.unwrap_err(); - warn!( - logger, - "query route for topic={} failed: error={}", topic, err - ); - let mut route_table_lock = route_table.lock(); - // keep the existing route if error occurs. - if let Some(RouteStatus::Found(prev)) = route_table_lock.get(topic) { - return Ok(Arc::clone(prev)); - } - let prev = route_table_lock.remove(topic); - if let Some(RouteStatus::Querying(Some(mut v))) = prev { - for item in v.drain(..) { - let _ = item.send(Err(ClientError::new( - ErrorKind::Server, - "query topic route failed", - OPERATION_QUERY_ROUTE, - ))); - } - }; - Err(err) - } + let mut rpc_client = self.get_session().await?; + self.route_manager + .topic_route_inner(&mut rpc_client, topic) + .await } async fn heart_beat_inner( @@ -677,24 +515,255 @@ where } } -pub fn handle_response_status( - status: Option, - operation: &'static str, -) -> Result<(), ClientError> { - let status = status.ok_or(ClientError::new( - ErrorKind::Server, - "server do not return status, this may be a bug", - operation, - ))?; - - if status.code != Code::Ok as i32 { - return Err( - ClientError::new(ErrorKind::Server, "server return an error", operation) - .with_context("code", format!("{}", status.code)) - .with_context("message", status.message), - ); +impl TopicRouteManager { + pub(crate) fn new(logger: Logger, namespace: String, access_endpoints: Endpoints) -> Self { + Self { + logger, + namespace, + access_endpoints, + route_table: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub(crate) async fn sync_route_data( + &self, + rpc_client: &mut T, + ) -> Result<(), ClientError> { + let topics: Vec; + { + topics = self.route_table.lock().keys().cloned().collect(); + } + debug!(self.logger, "sync topic route of topics {:?}", topics); + for topic in topics { + self.topic_route_inner(rpc_client, &topic).await?; + } + Ok(()) + } + + pub(crate) async fn sync_topic_routes( + &self, + rpc_client: &mut T, + topics: Vec, + ) -> Result<(), ClientError> { + debug!(self.logger, "sync topic route of topics {:?}.", topics); + for topic in topics { + self.topic_route_inner(rpc_client, topic.as_str()).await?; + } + Ok(()) + } + + pub(crate) async fn topic_route_inner( + &self, + rpc_client: &mut T, + topic: &str, + ) -> Result, ClientError> { + debug!(self.logger, "query route for topic={}", topic); + let rx = match self + .route_table + .lock() + .entry(topic.to_owned()) + .or_insert_with(|| RouteStatus::Querying(None)) + { + RouteStatus::Found(_route) => None, + RouteStatus::Querying(ref mut option) => { + match option { + Some(vec) => { + // add self to waiting list + let (tx, rx) = oneshot::channel(); + vec.push(tx); + Some(rx) + } + None => { + // there is no ongoing request, so we need to send a new request + let _ = option.insert(Vec::new()); + None + } + } + } + }; + + // wait for inflight request + if let Some(rx) = rx { + return match rx.await { + Ok(route) => route, + Err(_e) => Err(ClientError::new( + ErrorKind::ChannelReceive, + "wait for inflight query topic route request failed", + OPERATION_QUERY_ROUTE, + )), + }; + } + + let result = Self::query_topic_route( + rpc_client, + self.namespace.clone(), + self.access_endpoints.clone(), + topic, + ) + .await; + + // send result to all waiters + if let Ok(route) = result { + debug!( + self.logger, + "query route for topic={} success: route={:?}", topic, route + ); + let route = Arc::new(route); + let mut route_table_lock = self.route_table.lock(); + + // if message queues in previous and new route are the same, just keep the previous. + if let Some(RouteStatus::Found(prev)) = route_table_lock.get(topic) { + if prev.queue == route.queue { + return Ok(Arc::clone(prev)); + } + } + + let prev = + route_table_lock.insert(topic.to_owned(), RouteStatus::Found(Arc::clone(&route))); + info!(self.logger, "update route for topic={}", topic); + + if let Some(RouteStatus::Querying(Some(mut v))) = prev { + for item in v.drain(..) { + let _ = item.send(Ok(Arc::clone(&route))); + } + }; + Ok(route) + } else { + let err = result.unwrap_err(); + warn!( + self.logger, + "query route for topic={} failed: error={}", topic, err + ); + let mut route_table_lock = self.route_table.lock(); + // keep the existing route if error occurs. + if let Some(RouteStatus::Found(prev)) = route_table_lock.get(topic) { + return Ok(Arc::clone(prev)); + } + let prev = route_table_lock.remove(topic); + if let Some(RouteStatus::Querying(Some(mut v))) = prev { + for item in v.drain(..) { + let _ = item.send(Err(ClientError::new( + ErrorKind::Server, + "query topic route failed", + OPERATION_QUERY_ROUTE, + ))); + } + }; + Err(err) + } + } + + async fn query_topic_route( + rpc_client: &mut T, + namespace: String, + access_endpoints: Endpoints, + topic: &str, + ) -> Result { + let request = QueryRouteRequest { + topic: Some(Resource { + name: topic.to_owned(), + resource_namespace: namespace, + }), + endpoints: Some(access_endpoints.into_inner()), + }; + + let response = rpc_client.query_route(request).await?; + handle_response_status(response.status, OPERATION_QUERY_ROUTE)?; + + let route = Route { + index: AtomicUsize::new(0), + queue: response.message_queues, + }; + Ok(route) + } + + pub(crate) fn find_topic_route(&self, topic: &str) -> Option> { + self.route_table.lock().get(topic).and_then(|route_status| { + if let RouteStatus::Found(route) = route_status { + // debug!(self.logger, "get route for topic={} from cache", topic); + Some(Arc::clone(route)) + } else { + None + } + }) + } + + pub(crate) fn pick_endpoints(&self, topic: &str) -> Option { + self.route_table.lock().get(topic).and_then(|route_status| { + if let RouteStatus::Found(route) = route_status { + if let Some(broker) = select_message_queue(Arc::clone(route)).broker { + if let Some(endpoints) = broker.endpoints { + return Some(Endpoints::from_pb_endpoints(endpoints)); + } + } + } + None + }) + } +} + +#[derive(Debug)] +pub(crate) struct SessionManager { + logger: Logger, + client_id: String, + option: ClientOption, + session_map: tokio::sync::Mutex>, +} + +#[automock] +impl SessionManager { + pub(crate) fn new(logger: &Logger, client_id: String, option: &ClientOption) -> Self { + let logger = logger.new(o!("component" => "session")); + let session_map = tokio::sync::Mutex::new(HashMap::new()); + SessionManager { + logger, + client_id, + option: option.clone(), + session_map, + } + } + + pub(crate) async fn get_or_create_session( + &self, + endpoints: &Endpoints, + settings: TelemetryCommand, + telemetry_command_tx: mpsc::Sender, + ) -> Result { + let mut session_map = self.session_map.lock().await; + let endpoint_url = endpoints.endpoint_url().to_string(); + return if session_map.contains_key(&endpoint_url) { + Ok(session_map.get(&endpoint_url).unwrap().shadow_session()) + } else { + let mut session = Session::new( + &self.logger, + endpoints, + self.client_id.clone(), + &self.option, + ) + .await?; + session.start(settings, telemetry_command_tx).await?; + let shadow_session = session.shadow_session(); + session_map.insert(endpoint_url.clone(), session); + Ok(shadow_session) + }; + } + + pub(crate) async fn get_all_sessions(&self) -> Result, ClientError> { + let session_map = self.session_map.lock().await; + let mut sessions = Vec::new(); + for (_, session) in session_map.iter() { + sessions.push(session.shadow_session()); + } + Ok(sessions) + } + + pub(crate) async fn shutdown(&self) { + let mut session_map = self.session_map.lock().await; + for (_, session) in session_map.iter_mut() { + session.shutdown(); + } + session_map.clear(); } - Ok(()) } #[cfg(test)] @@ -717,48 +786,60 @@ pub(crate) mod tests { ReceiveMessageResponse, Resource, SendMessageResponse, Status, TelemetryCommand, }; use crate::session; + use crate::session::MockSession; use super::*; // The lock is used to prevent the mocking static function at same time during parallel testing. pub(crate) static MTX: Lazy> = Lazy::new(|| Mutex::new(())); - #[derive(Default)] - struct MockSettings {} - - impl SettingsAware for MockSettings { - fn build_telemetry_command(&self) -> TelemetryCommand { - TelemetryCommand::default() + fn new_route_manager_for_test() -> TopicRouteManager { + TopicRouteManager { + logger: terminal_logger(), + route_table: Arc::new(Mutex::new(HashMap::new())), + access_endpoints: Endpoints::from_url("http://localhost:8081").unwrap(), + namespace: "".to_string(), } } - fn new_client_for_test() -> Client { + fn new_session_manager() -> SessionManager { + SessionManager::new( + &terminal_logger(), + Client::generate_client_id(), + &ClientOption { + group: Some("group".to_string()), + ..Default::default() + }, + ) + } + + fn new_client_for_test() -> Client { Client { logger: terminal_logger(), option: ClientOption { group: Some("group".to_string()), ..Default::default() }, - session_manager: Arc::new(SessionManager::default()), - route_table: Arc::new(Mutex::new(HashMap::new())), - id: Client::::generate_client_id(), + session_manager: Arc::new(new_session_manager()), + route_manager: new_route_manager_for_test(), + id: Client::generate_client_id(), access_endpoints: Endpoints::from_url("http://localhost:8081").unwrap(), - settings: Arc::new(RwLock::new(MockSettings::default())), + settings: TelemetryCommand::default(), telemetry_command_tx: None, shutdown_tx: None, } } - fn new_client_with_session_manager(session_manager: SessionManager) -> Client { + fn new_client_with_session_manager(session_manager: SessionManager) -> Client { let (tx, _) = mpsc::channel(16); Client { logger: terminal_logger(), option: ClientOption::default(), session_manager: Arc::new(session_manager), - route_table: Arc::new(Mutex::new(HashMap::new())), - id: Client::::generate_client_id(), + route_manager: new_route_manager_for_test(), + id: Client::generate_client_id(), access_endpoints: Endpoints::from_url("http://localhost:8081").unwrap(), - settings: Arc::new(RwLock::new(MockSettings::default())), + settings: TelemetryCommand::default(), telemetry_command_tx: Some(tx), shutdown_tx: None, } @@ -773,27 +854,26 @@ pub(crate) mod tests { #[test] fn client_new() -> Result<(), ClientError> { - let ctx = crate::session::MockSessionManager::new_context(); - ctx.expect() - .return_once(|_, _, _| SessionManager::default()); Client::new( &terminal_logger(), ClientOption::default(), - Arc::new(RwLock::new(MockSettings::default())), + TelemetryCommand::default(), )?; Ok(()) } #[tokio::test(flavor = "multi_thread")] async fn client_start() -> Result<(), ClientError> { - let mut session_manager = SessionManager::default(); - session_manager - .expect_get_all_sessions() - .returning(|| Ok(vec![])); - session_manager - .expect_get_or_create_session() - .returning(|_, _, _| Ok(Session::mock())); - + let context = MockSession::new_context(); + context.expect().returning(|_, _, _, _| { + let mut session = MockSession::default(); + session.expect_start().returning(|_, _| Ok(())); + session + .expect_shadow_session() + .returning(|| MockSession::default()); + Ok(session) + }); + let session_manager = new_session_manager(); let mut client = new_client_with_session_manager(session_manager); let (tx, _) = mpsc::channel(16); client.start(tx).await?; @@ -806,11 +886,16 @@ pub(crate) mod tests { #[tokio::test] async fn client_get_session() { - let mut session_manager = SessionManager::default(); - session_manager - .expect_get_or_create_session() - .returning(|_, _, _| Ok(Session::mock())); - + let context = MockSession::new_context(); + context.expect().returning(|_, _, _, _| { + let mut session = MockSession::default(); + session.expect_start().returning(|_, _| Ok(())); + session + .expect_shadow_session() + .returning(|| MockSession::default()); + Ok(session) + }); + let session_manager = new_session_manager(); let mut client = new_client_with_session_manager(session_manager); let (tx, _rx) = mpsc::channel(16); let _ = client.start(tx).await; @@ -822,51 +907,6 @@ pub(crate) mod tests { assert!(result.is_ok()); } - #[test] - fn test_handle_response_status() { - let result = handle_response_status(None, "test"); - assert!(result.is_err(), "should return error when status is None"); - let result = result.unwrap_err(); - assert_eq!(result.kind, ErrorKind::Server); - assert_eq!( - result.message, - "server do not return status, this may be a bug" - ); - assert_eq!(result.operation, "test"); - - let result = handle_response_status( - Some(Status { - code: Code::BadRequest as i32, - message: "test failed".to_string(), - }), - "test failed", - ); - assert!( - result.is_err(), - "should return error when status is BadRequest" - ); - let result = result.unwrap_err(); - assert_eq!(result.kind, ErrorKind::Server); - assert_eq!(result.message, "server return an error"); - assert_eq!(result.operation, "test failed"); - assert_eq!( - result.context, - vec![ - ("code", format!("{}", Code::BadRequest as i32)), - ("message", "test failed".to_string()), - ] - ); - - let result = handle_response_status( - Some(Status { - code: Code::Ok as i32, - message: "test success".to_string(), - }), - "test success", - ); - assert!(result.is_ok(), "should not return error when status is Ok"); - } - pub(crate) fn new_topic_route_response() -> Result { Ok(QueryRouteResponse { status: Some(Status { @@ -889,7 +929,7 @@ pub(crate) mod tests { #[tokio::test] async fn client_query_route_from_cache() { let client = new_client_for_test(); - client.route_table.lock().insert( + client.route_manager.route_table.lock().insert( "DefaultCluster".to_string(), RouteStatus::Found(Arc::new(Route { index: AtomicUsize::new(0), @@ -908,16 +948,10 @@ pub(crate) mod tests { mock.expect_query_route() .return_once(|_| Box::pin(futures::future::ready(new_topic_route_response()))); - let logger = client.logger.clone(); - let result = Client::::topic_route_inner( - logger, - mock, - Arc::clone(&client.route_table), - client.option.namespace.clone(), - client.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result = client + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result.is_ok()); let route = result.unwrap(); @@ -944,30 +978,20 @@ pub(crate) mod tests { Box::pin(futures::future::ready(new_topic_route_response())) }); - let result = Client::::topic_route_inner( - client_clone.logger.clone(), - mock, - Arc::clone(&client_clone.route_table), - client_clone.option.namespace.clone(), - client_clone.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result = client_clone + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result.is_ok()); }); let handle = tokio::spawn(async move { sleep(Duration::from_millis(100)); let mut mock = session::MockRPCClient::new(); - let result = Client::::topic_route_inner( - client.logger.clone(), - mock, - Arc::clone(&client.route_table), - client.option.namespace.clone(), - client.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result = client + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result.is_ok()); }); @@ -991,30 +1015,20 @@ pub(crate) mod tests { )))) }); - let result = Client::::topic_route_inner( - client_clone.logger.clone(), - mock, - Arc::clone(&client_clone.route_table), - client_clone.option.namespace.clone(), - client_clone.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result = client_clone + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result.is_err()); }); let handle = tokio::spawn(async move { sleep(Duration::from_millis(100)); let mut mock = session::MockRPCClient::new(); - let result = Client::::topic_route_inner( - client.logger.clone(), - mock, - Arc::clone(&client.route_table), - client.option.namespace.clone(), - client.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result = client + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result.is_err()); }); @@ -1033,7 +1047,7 @@ pub(crate) mod tests { } else { vec![] }; - client.route_table.lock().insert( + client.route_manager.route_table.lock().insert( "DefaultCluster".to_string(), RouteStatus::Found(Arc::new(Route { index: AtomicUsize::new(0), @@ -1051,15 +1065,10 @@ pub(crate) mod tests { )))) }); - let result = Client::::topic_route_inner( - client.logger.clone(), - mock, - Arc::clone(&client.route_table), - client.option.namespace.clone(), - client.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result = client + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result.is_ok()); } @@ -1071,15 +1080,10 @@ pub(crate) mod tests { mock.expect_query_route() .return_once(|_| Box::pin(futures::future::ready(new_topic_route_response()))); - let result = Client::::topic_route_inner( - client.logger.clone(), - mock, - Arc::clone(&client.route_table), - client.option.namespace.clone(), - client.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result = client + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result.is_ok()); let route = result.unwrap(); @@ -1097,15 +1101,10 @@ pub(crate) mod tests { mock.expect_query_route() .return_once(|_| Box::pin(futures::future::ready(new_topic_route_response()))); - let result2 = Client::::topic_route_inner( - client.logger.clone(), - mock, - Arc::clone(&client.route_table), - client.option.namespace.clone(), - client.access_endpoints.clone(), - "DefaultCluster", - ) - .await; + let result2 = client + .route_manager + .topic_route_inner(&mut mock, "DefaultCluster") + .await; assert!(result2.is_ok()); let route2 = result2.unwrap(); @@ -1123,13 +1122,9 @@ pub(crate) mod tests { mock.expect_heartbeat() .return_once(|_| Box::pin(futures::future::ready(response))); - let send_result = Client::::heart_beat_inner( - mock, - &Some("group".to_string()), - "", - &ClientType::Producer, - ) - .await; + let send_result = + Client::heart_beat_inner(mock, &Some("group".to_string()), "", &ClientType::Producer) + .await; assert!(send_result.is_ok()); } diff --git a/rust/src/conf.rs b/rust/src/conf.rs index 8b270612b..79d92f605 100644 --- a/rust/src/conf.rs +++ b/rust/src/conf.rs @@ -17,15 +17,15 @@ //! Configuration of RocketMQ rust client. +use std::collections::HashMap; use std::time::Duration; -use crate::model::common::ClientType; -use crate::pb::TelemetryCommand; +use crate::model::common::{ClientType, FilterExpression}; +use crate::pb::{self, Resource}; #[allow(unused_imports)] use crate::producer::Producer; #[allow(unused_imports)] use crate::simple_consumer::SimpleConsumer; -use crate::util::{build_producer_settings, build_simple_consumer_settings}; /// [`ClientOption`] is the configuration of internal client, which manages the connection and request with RocketMQ proxy. #[derive(Debug, Clone)] @@ -113,6 +113,14 @@ impl ClientOption { pub fn set_secret_key(&mut self, secret_key: impl Into) { self.secret_key = Some(secret_key.into()); } + + pub fn get_namespace(&self) -> &str { + &self.namespace + } + /// Set the namespace + pub fn set_namespace(&mut self, namespace: impl Into) { + self.namespace = namespace.into(); + } } /// Log format for output. @@ -278,24 +286,243 @@ impl SimpleConsumerOption { } } -pub trait SettingsAware { - fn build_telemetry_command(&self) -> TelemetryCommand; +#[derive(Debug, Clone)] +pub struct PushConsumerOption { + logging_format: LoggingFormat, + consumer_group: String, + namespace: String, + timeout: Duration, + long_polling_timeout: Duration, + subscription_expressions: HashMap, + fifo: bool, + batch_size: i32, + consumer_worker_count_each_queue: usize, } -impl SettingsAware for ProducerOption { - fn build_telemetry_command(&self) -> TelemetryCommand { - build_producer_settings(self) +impl Default for PushConsumerOption { + fn default() -> Self { + Self { + logging_format: LoggingFormat::Terminal, + consumer_group: "".to_string(), + namespace: "".to_string(), + timeout: Duration::from_secs(3), + long_polling_timeout: Duration::from_secs(40), + subscription_expressions: HashMap::new(), + fifo: false, + batch_size: 32, + consumer_worker_count_each_queue: 4, + } } } -impl SettingsAware for SimpleConsumerOption { - fn build_telemetry_command(&self) -> TelemetryCommand { - build_simple_consumer_settings(self) +impl PushConsumerOption { + pub fn timeout(&self) -> &Duration { + &self.timeout + } + + pub fn consumer_group(&self) -> &str { + &self.consumer_group + } + + pub fn get_consumer_group_resource(&self) -> Resource { + Resource { + name: self.consumer_group.clone(), + resource_namespace: self.namespace.clone(), + } + } + + pub fn set_consumer_group(&mut self, consumer_group: impl Into) { + self.consumer_group = consumer_group.into(); + } + + pub fn namespace(&self) -> &str { + &self.namespace + } + + pub fn long_polling_timeout(&self) -> &Duration { + &self.long_polling_timeout + } + + pub fn subscription_expressions(&self) -> &HashMap { + &self.subscription_expressions + } + + pub fn subscribe(&mut self, topic: impl Into, filter_expression: FilterExpression) { + self.subscription_expressions + .insert(topic.into(), filter_expression); + } + + pub fn get_filter_expression(&self, topic: &str) -> Option<&FilterExpression> { + self.subscription_expressions.get(topic) + } + + pub fn fifo(&self) -> bool { + self.fifo + } + + pub(crate) fn set_fifo(&mut self, fifo: bool) { + self.fifo = fifo; + } + + pub fn logging_format(&self) -> &LoggingFormat { + &self.logging_format + } + + pub fn set_logging_format(&mut self, logging_format: LoggingFormat) { + self.logging_format = logging_format; + } + + pub fn batch_size(&self) -> i32 { + self.batch_size + } + + pub fn consumer_worker_count_each_queue(&self) -> usize { + self.consumer_worker_count_each_queue + } + + pub fn set_consumer_worker_count_each_queue(&mut self, count: usize) { + self.consumer_worker_count_each_queue = count; + } +} + +#[derive(Debug, Clone)] +pub enum BackOffRetryPolicy { + Exponential(ExponentialBackOffRetryPolicy), + Customized(CustomizedBackOffRetryPolicy), +} + +const INVALID_COMMAND: &str = "invalid command"; + +impl TryFrom for BackOffRetryPolicy { + type Error = &'static str; + + fn try_from(value: pb::telemetry_command::Command) -> Result { + if let pb::telemetry_command::Command::Settings(settings) = value { + let pubsub = settings.pub_sub.ok_or(INVALID_COMMAND)?; + if let pb::settings::PubSub::Subscription(_) = pubsub { + let retry_policy = settings.backoff_policy.ok_or(INVALID_COMMAND)?; + let strategy = retry_policy.strategy.ok_or(INVALID_COMMAND)?; + return match strategy { + pb::retry_policy::Strategy::ExponentialBackoff(strategy) => { + Ok(BackOffRetryPolicy::Exponential( + ExponentialBackOffRetryPolicy::new(strategy, retry_policy.max_attempts), + )) + } + pb::retry_policy::Strategy::CustomizedBackoff(strategy) => { + Ok(BackOffRetryPolicy::Customized( + CustomizedBackOffRetryPolicy::new(strategy, retry_policy.max_attempts), + )) + } + }; + } + } + Err(INVALID_COMMAND) + } +} + +impl BackOffRetryPolicy { + pub fn get_next_attempt_delay(&self, attempts: i32) -> Duration { + match self { + BackOffRetryPolicy::Exponential(policy) => policy.get_next_attempt_delay(attempts), + BackOffRetryPolicy::Customized(policy) => policy.get_next_attempt_delay(attempts), + } + } + + pub fn get_max_attempts(&self) -> i32 { + match self { + BackOffRetryPolicy::Exponential(policy) => policy.max_attempts(), + BackOffRetryPolicy::Customized(policy) => policy.max_attempts(), + } + } +} + +#[derive(Clone, Debug)] +pub struct ExponentialBackOffRetryPolicy { + initial: Duration, + max: Duration, + multiplier: f32, + max_attempts: i32, +} + +impl ExponentialBackOffRetryPolicy { + pub fn new( + strategy: pb::ExponentialBackoff, + max_attempts: i32, + ) -> ExponentialBackOffRetryPolicy { + let initial = strategy.initial.map_or(Duration::ZERO, |d| { + Duration::new(d.seconds as u64, d.nanos as u32) + }); + let max = strategy.max.map_or(Duration::ZERO, |d| { + Duration::new(d.seconds as u64, d.nanos as u32) + }); + let multiplier = strategy.multiplier; + ExponentialBackOffRetryPolicy { + initial, + max, + multiplier, + max_attempts, + } + } + + pub(crate) fn get_next_attempt_delay(&self, attempts: i32) -> Duration { + let delay_nanos = (self.initial.as_nanos() * self.multiplier.powi(attempts - 1) as u128) + .min(self.max.as_nanos()); + Duration::from_nanos(delay_nanos as u64) + } + + pub(crate) fn max_attempts(&self) -> i32 { + self.max_attempts + } +} + +impl Default for ExponentialBackOffRetryPolicy { + fn default() -> Self { + Self { + initial: Duration::ZERO, + max: Duration::ZERO, + multiplier: 1.0, + max_attempts: 1, + } + } +} + +#[derive(Clone, Debug)] +pub struct CustomizedBackOffRetryPolicy { + next_list: Vec, + max_attempts: i32, +} + +impl CustomizedBackOffRetryPolicy { + pub fn new(strategy: pb::CustomizedBackoff, max_attempts: i32) -> CustomizedBackOffRetryPolicy { + CustomizedBackOffRetryPolicy { + next_list: strategy + .next + .iter() + .map(|d| Duration::new(d.seconds as u64, d.nanos as u32)) + .collect(), + max_attempts, + } + } + + pub(crate) fn get_next_attempt_delay(&self, attempts: i32) -> Duration { + let mut index = attempts.min(self.next_list.len() as i32) - 1; + if index < 0 { + index = 0; + } + self.next_list[index as usize] + } + + pub(crate) fn max_attempts(&self) -> i32 { + self.max_attempts } } #[cfg(test)] mod tests { + use std::str::FromStr; + + use crate::model::common::FilterType; + use super::*; #[test] @@ -321,4 +548,110 @@ mod tests { assert_eq!(option.logging_format(), &LoggingFormat::Terminal); assert!(option.prefetch_route()); } + + #[test] + fn conf_push_consumer_option() { + let mut option = PushConsumerOption::default(); + option.subscribe("topic", FilterExpression::new(FilterType::Tag, "*")); + assert!(option.subscription_expressions().contains_key("topic")); + } + + #[test] + fn test_exponential_backoff() { + let policy = ExponentialBackOffRetryPolicy::new( + pb::ExponentialBackoff { + initial: Some(prost_types::Duration::from_str("1s").unwrap()), + max: Some(prost_types::Duration::from_str("60s").unwrap()), + multiplier: 2.0, + }, + 1, + ); + assert_eq!(Duration::from_secs(1), policy.get_next_attempt_delay(1)); + assert_eq!(Duration::from_secs(2), policy.get_next_attempt_delay(2)); + assert_eq!(Duration::from_secs(4), policy.get_next_attempt_delay(3)); + assert_eq!(Duration::from_secs(8), policy.get_next_attempt_delay(4)); + assert_eq!(Duration::from_secs(16), policy.get_next_attempt_delay(5)); + assert_eq!(Duration::from_secs(32), policy.get_next_attempt_delay(6)); + assert_eq!(Duration::from_secs(60), policy.get_next_attempt_delay(7)); + assert_eq!(Duration::from_secs(60), policy.get_next_attempt_delay(8)); + } + + #[test] + fn test_customized_backoff() { + let next: Vec = vec![ + prost_types::Duration::from_str("1s").unwrap(), + prost_types::Duration::from_str("10s").unwrap(), + prost_types::Duration::from_str("20s").unwrap(), + prost_types::Duration::from_str("30s").unwrap(), + ]; + let policy = CustomizedBackOffRetryPolicy::new(pb::CustomizedBackoff { next }, 1); + assert_eq!(Duration::from_secs(1), policy.get_next_attempt_delay(1)); + assert_eq!(Duration::from_secs(10), policy.get_next_attempt_delay(2)); + assert_eq!(Duration::from_secs(20), policy.get_next_attempt_delay(3)); + assert_eq!(Duration::from_secs(30), policy.get_next_attempt_delay(4)); + assert_eq!(Duration::from_secs(30), policy.get_next_attempt_delay(5)); + } + + use pb::settings::PubSub; + use pb::CustomizedBackoff; + use pb::ExponentialBackoff; + use pb::RetryPolicy; + use pb::Settings; + use pb::Subscription; + + #[test] + fn test_parse_back_policy() -> Result<(), String> { + let command = pb::telemetry_command::Command::VerifyMessageCommand( + pb::VerifyMessageCommand::default(), + ); + let result = BackOffRetryPolicy::try_from(command); + assert!(result.is_err()); + + let command2 = pb::telemetry_command::Command::Settings(pb::Settings::default()); + let result2 = BackOffRetryPolicy::try_from(command2); + assert!(result2.is_err()); + + let exponential_backoff_settings = pb::Settings { + pub_sub: Some(PubSub::Subscription(Subscription::default())), + client_type: Some(ClientType::PushConsumer as i32), + backoff_policy: Some(RetryPolicy { + max_attempts: 0, + strategy: Some(pb::retry_policy::Strategy::ExponentialBackoff( + ExponentialBackoff { + initial: Some(prost_types::Duration::from_str("1s").unwrap()), + max: Some(prost_types::Duration::from_str("60s").unwrap()), + multiplier: 2.0, + }, + )), + }), + ..Settings::default() + }; + let command3 = pb::telemetry_command::Command::Settings(exponential_backoff_settings); + let result3 = BackOffRetryPolicy::try_from(command3); + if let Ok(BackOffRetryPolicy::Exponential(_)) = result3 { + } else { + return Err("get_backoff_policy failed, expected ExponentialBackoff.".to_string()); + } + + let customized_backoff_settings = Settings { + pub_sub: Some(PubSub::Subscription(Subscription::default())), + client_type: Some(ClientType::PushConsumer as i32), + backoff_policy: Some(RetryPolicy { + max_attempts: 0, + strategy: Some(pb::retry_policy::Strategy::CustomizedBackoff( + CustomizedBackoff { + next: vec![prost_types::Duration::from_str("1s").unwrap()], + }, + )), + }), + ..Settings::default() + }; + let command4 = pb::telemetry_command::Command::Settings(customized_backoff_settings); + let result4 = BackOffRetryPolicy::try_from(command4); + if let Ok(BackOffRetryPolicy::Customized(_)) = result4 { + } else { + return Err("get_backoff_policy failed, expected CustomizedBackoff.".to_string()); + } + Ok(()) + } } diff --git a/rust/src/error.rs b/rust/src/error.rs index 5210842ed..55afd9153 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -36,6 +36,9 @@ pub enum ErrorKind { #[error("Message type not match with topic accept message type")] MessageTypeNotMatch, + #[error("Message queue is invalid")] + InvalidMessageQueue, + #[error("Server error")] Server, diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 875b3594f..dc7478a0b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -121,6 +121,8 @@ // Export structs that are part of crate API. pub use producer::Producer; +pub use push_consumer::MessageListener; +pub use push_consumer::PushConsumer; pub use simple_consumer::SimpleConsumer; #[allow(dead_code)] @@ -141,4 +143,5 @@ pub mod model; mod util; mod producer; +mod push_consumer; mod simple_consumer; diff --git a/rust/src/model/common.rs b/rust/src/model/common.rs index 5dee554cb..8c256f834 100644 --- a/rust/src/model/common.rs +++ b/rust/src/model/common.rs @@ -17,6 +17,7 @@ //! Common data model of RocketMQ rust client. +use std::hash::Hash; use std::net::IpAddr; use std::sync::atomic::AtomicUsize; use std::sync::Arc; @@ -24,8 +25,8 @@ use std::sync::Arc; use tokio::sync::oneshot; use crate::error::{ClientError, ErrorKind}; -use crate::pb; -use crate::pb::{Address, AddressScheme, MessageQueue}; +use crate::pb::{self, Broker, Resource}; +use crate::pb::{Address, AddressScheme}; #[derive(Debug, Clone)] pub(crate) enum ClientType { @@ -40,7 +41,7 @@ pub(crate) enum ClientType { #[derive(Debug)] pub(crate) struct Route { pub(crate) index: AtomicUsize, - pub queue: Vec, + pub queue: Vec, } type InflightRequest = Option, ClientError>>>>; @@ -191,7 +192,7 @@ impl Endpoints { /// Filter type for message filtering. /// /// RocketMQ allows to filter messages by tag or SQL. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] #[repr(i32)] pub enum FilterType { /// Filter by tag @@ -201,6 +202,7 @@ pub enum FilterType { } /// Filter expression for message filtering. +#[derive(Clone, Debug)] pub struct FilterExpression { filter_type: FilterType, expression: String, @@ -257,6 +259,71 @@ impl SendReceipt { } } +#[derive(Debug)] +pub enum ConsumeResult { + SUCCESS, + FAILURE, +} + +impl Eq for Resource {} + +impl Hash for Resource { + fn hash(&self, state: &mut H) { + self.name.hash(state); + self.resource_namespace.hash(state); + } +} + +impl Eq for Broker {} + +impl Hash for Broker { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct MessageQueue { + pub(crate) id: i32, + pub(crate) topic: Resource, + pub(crate) broker: Broker, + pub(crate) accept_message_types: Vec, + pub(crate) permission: i32, +} + +impl MessageQueue { + pub(crate) fn from_pb_message_queue( + message_queue: pb::MessageQueue, + ) -> Result { + if let Some(broker) = message_queue.broker { + if let Some(topic) = message_queue.topic { + return Ok(Self { + id: message_queue.id, + topic, + broker, + accept_message_types: message_queue.accept_message_types, + permission: message_queue.permission, + }); + } + } + Err(ClientError::new( + ErrorKind::InvalidMessageQueue, + "message queue is not valid.", + "", + )) + } + + pub(crate) fn to_pb_message_queue(&self) -> pb::MessageQueue { + pb::MessageQueue { + id: self.id, + topic: Some(self.topic.clone()), + broker: Some(self.broker.clone()), + accept_message_types: self.accept_message_types.clone(), + permission: self.permission, + } + } +} + #[cfg(test)] mod tests { use crate::error::ErrorKind; diff --git a/rust/src/model/message.rs b/rust/src/model/message.rs index 1defcd89d..035861b94 100644 --- a/rust/src/model/message.rs +++ b/rust/src/model/message.rs @@ -359,10 +359,10 @@ impl AckMessageEntry for MessageView { } impl MessageView { - pub(crate) fn from_pb_message(message: pb::Message, endpoints: Endpoints) -> Self { - let system_properties = message.system_properties.unwrap(); - let topic = message.topic.unwrap(); - MessageView { + pub(crate) fn from_pb_message(message: pb::Message, endpoints: Endpoints) -> Option { + let system_properties = message.system_properties?; + let topic = message.topic?; + Some(MessageView { message_id: system_properties.message_id, receipt_handle: system_properties.receipt_handle, namespace: topic.resource_namespace, @@ -377,7 +377,7 @@ impl MessageView { born_timestamp: system_properties.born_timestamp.map_or(0, |t| t.seconds), delivery_attempt: system_properties.delivery_attempt.unwrap_or(0), endpoints, - } + }) } /// Get message id @@ -535,7 +535,8 @@ mod tests { }), }, Endpoints::from_url("localhost:8081").unwrap(), - ); + ) + .unwrap(); assert_eq!(message_view.message_id(), "message_id"); assert_eq!(message_view.topic(), "test"); diff --git a/rust/src/model/transaction.rs b/rust/src/model/transaction.rs index 0a4c6871c..6d5e90805 100644 --- a/rust/src/model/transaction.rs +++ b/rust/src/model/transaction.rs @@ -21,12 +21,12 @@ use std::fmt::{Debug, Formatter}; use async_trait::async_trait; -use crate::client::handle_response_status; use crate::error::ClientError; use crate::model::common::SendReceipt; use crate::model::message::MessageView; use crate::pb::{EndTransactionRequest, Resource, TransactionSource}; use crate::session::RPCClient; +use crate::util::handle_response_status; /// An entity to describe an independent transaction. /// diff --git a/rust/src/producer.rs b/rust/src/producer.rs index 26ba496ab..94cd295df 100644 --- a/rust/src/producer.rs +++ b/rust/src/producer.rs @@ -20,13 +20,12 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use mockall_double::double; +use parking_lot::RwLock; use prost_types::Timestamp; use slog::{error, info, warn, Logger}; use tokio::select; -use tokio::sync::RwLock; use tokio::sync::{mpsc, oneshot}; -use crate::client::handle_response_status; #[double] use crate::client::Client; use crate::conf::{ClientOption, ProducerOption}; @@ -41,8 +40,8 @@ use crate::pb::telemetry_command::Command::{RecoverOrphanedTransactionCommand, S use crate::pb::{Encoding, EndTransactionRequest, Resource, SystemProperties, TransactionSource}; use crate::session::RPCClient; use crate::util::{ - build_endpoints_by_message_queue, select_message_queue, select_message_queue_by_message_group, - HOST_NAME, + build_endpoints_by_message_queue, build_producer_settings, handle_response_status, + select_message_queue, select_message_queue_by_message_group, HOST_NAME, }; use crate::{log, pb}; @@ -55,7 +54,7 @@ use crate::{log, pb}; pub struct Producer { option: Arc>, logger: Logger, - client: Client, + client: Client, transaction_checker: Option>, shutdown_tx: Option>, } @@ -86,10 +85,9 @@ impl Producer { ..client_option }; let logger = log::logger(option.logging_format()); - let option = Arc::new(RwLock::new(option)); - let client = Client::new(&logger, client_option, Arc::clone(&option))?; + let client = Client::new(&logger, client_option, build_producer_settings(&option))?; Ok(Producer { - option, + option: Arc::new(RwLock::new(option)), logger, client, transaction_checker: None, @@ -115,10 +113,9 @@ impl Producer { ..client_option }; let logger = log::logger(option.logging_format()); - let option = Arc::new(RwLock::new(option)); - let client = Client::::new(&logger, client_option, Arc::clone(&option))?; + let client = Client::new(&logger, client_option, build_producer_settings(&option))?; Ok(Producer { - option, + option: Arc::new(RwLock::new(option)), logger, client, transaction_checker: Some(transaction_checker), @@ -126,8 +123,17 @@ impl Producer { }) } - async fn get_resource_namespace(&self) -> String { - self.option.read().await.namespace().to_string() + fn get_resource_namespace(&self) -> String { + self.option.read().namespace().to_string() + } + + fn get_topics(&self) -> Option> { + let binding = self.option.read(); + let topics = binding.topics(); + if let Some(topics) = topics { + return Some(topics.iter().map(|topic| topic.to_string()).collect()); + } + None } /// Start the producer @@ -136,13 +142,10 @@ impl Producer { let telemetry_command_tx: mpsc::Sender = telemetry_command_tx; self.client.start(telemetry_command_tx).await?; - { - let option_guard = self.option.read().await; - let topics = option_guard.topics(); - if let Some(topics) = topics { - for topic in topics { - self.client.topic_route(topic, true).await?; - } + let topics = self.get_topics(); + if let Some(topics) = topics { + for topic in topics { + self.client.topic_route(topic.as_str(), true).await?; } } let transaction_checker = self.transaction_checker.take(); @@ -172,7 +175,7 @@ impl Producer { }; } Settings(command) => { - let option = &mut producer_option.write().await; + let option = &mut producer_option.write(); Self::handle_settings_command(command, option); info!(logger, "handle setting command success."); } @@ -231,7 +234,11 @@ impl Producer { if let Some(transaction_checker) = transaction_checker { let resolution = transaction_checker( transaction_id.clone(), - MessageView::from_pb_message(message, endpoints), + MessageView::from_pb_message(message, endpoints).ok_or(ClientError::new( + ErrorKind::InvalidMessage, + "error parsing from pb", + Self::OPERATION_END_TRANSACTION, + ))?, ); let response = rpc_client .end_transaction(EndTransactionRequest { @@ -326,7 +333,7 @@ impl Producer { let pb_message = pb::Message { topic: Some(Resource { name: message.take_topic(), - resource_namespace: self.get_resource_namespace().await, + resource_namespace: self.get_resource_namespace(), }), user_properties: message.take_properties(), system_properties: Some(SystemProperties { @@ -421,7 +428,7 @@ impl Producer { } async fn validate_message_type(&self) -> bool { - self.option.read().await.validate_message_type() + self.option.read().validate_message_type() } pub fn has_transaction_checker(&self) -> bool { @@ -446,7 +453,7 @@ impl Producer { Ok(TransactionImpl::new( Box::new(rpc_client), Resource { - resource_namespace: self.get_resource_namespace().await, + resource_namespace: self.get_resource_namespace(), name: topic, }, receipt, @@ -465,14 +472,15 @@ impl Producer { mod tests { use std::sync::Arc; - use crate::client::MockClient; use crate::error::ErrorKind; use crate::log::terminal_logger; use crate::model::common::Route; use crate::model::message::{MessageBuilder, MessageImpl, MessageType}; use crate::model::transaction::TransactionResolution; use crate::pb::{Broker, Code, EndTransactionResponse, MessageQueue, Status}; - use crate::session::{self, Session}; + use crate::session::MockRPCClient; + #[double] + use crate::session::Session; use super::*; @@ -500,7 +508,7 @@ mod tests { async fn producer_start() -> Result<(), ClientError> { let _m = crate::client::tests::MTX.lock(); - let ctx = Client::::new_context(); + let ctx = Client::new_context(); ctx.expect().return_once(|_, _, _| { let mut client = Client::default(); client.expect_topic_route().returning(|_, _| { @@ -515,7 +523,7 @@ mod tests { .return_const("fake_id".to_string()); client .expect_get_session() - .return_once(|| Ok(Session::mock())); + .return_once(|| Ok(Session::default())); client .expect_get_endpoints() .return_once(|| Endpoints::from_url("foobar.com:8080").unwrap()); @@ -533,7 +541,7 @@ mod tests { async fn transaction_producer_start() -> Result<(), ClientError> { let _m = crate::client::tests::MTX.lock(); - let ctx = Client::::new_context(); + let ctx = Client::new_context(); ctx.expect().return_once(|_, _, _| { let mut client = Client::default(); client.expect_topic_route().returning(|_, _| { @@ -548,7 +556,7 @@ mod tests { .return_const("fake_id".to_string()); client .expect_get_session() - .return_once(|| Ok(Session::mock())); + .return_once(|| Ok(Session::default())); client .expect_get_endpoints() .return_once(|| Endpoints::from_url("foobar.com:8080").unwrap()); @@ -750,7 +758,7 @@ mod tests { producer .client .expect_get_session() - .return_once(|| Ok(Session::mock())); + .return_once(|| Ok(Session::default())); let _ = producer .send_transaction_message( @@ -770,7 +778,7 @@ mod tests { message: "".to_string(), }), }); - let mut mock = session::MockRPCClient::new(); + let mut mock = MockRPCClient::new(); mock.expect_end_transaction() .return_once(|_| Box::pin(futures::future::ready(response))); diff --git a/rust/src/push_consumer.rs b/rust/src/push_consumer.rs new file mode 100644 index 000000000..3ccac5cda --- /dev/null +++ b/rust/src/push_consumer.rs @@ -0,0 +1,1956 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use mockall::automock; +use mockall_double::double; +use parking_lot::{Mutex, RwLock}; +use slog::Logger; +use slog::{debug, error, info, warn}; +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; +use std::time::Duration; +use tokio::select; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +#[double] +use crate::client::Client; +use crate::conf::{BackOffRetryPolicy, ClientOption, PushConsumerOption}; +use crate::error::{ClientError, ErrorKind}; +use crate::model::common::{ClientType, ConsumeResult, MessageQueue}; +use crate::model::message::{AckMessageEntry, MessageView}; +use crate::pb::receive_message_response::Content; +use crate::pb::{ + AckMessageRequest, Assignment, ChangeInvisibleDurationRequest, + ForwardMessageToDeadLetterQueueRequest, QueryAssignmentRequest, ReceiveMessageRequest, + Resource, +}; +use crate::session::RPCClient; +#[double] +use crate::session::Session; +use crate::util::{ + build_endpoints_by_message_queue, build_push_consumer_settings, handle_response_status, +}; +use crate::{log, pb}; + +const OPERATION_NEW_PUSH_CONSUMER: &str = "push_consumer.new"; +const OPERATION_RECEIVE_MESSAGE: &str = "push_consumer.receive_message"; +const OPERATION_ACK_MESSAGE: &str = "push_consumer.ack_message"; +const OPERATION_START_PUSH_CONSUMER: &str = "push_consumer.start"; +const OPERATION_CHANGE_INVISIBLE_DURATION: &str = "push_consumer.change_invisible_duration"; +const OPERATION_FORWARD_TO_DEADLETTER_QUEUE: &str = "push_consumer.forward_to_deadletter_queue"; + +pub type MessageListener = Box ConsumeResult + Send + Sync>; + +pub struct PushConsumer { + logger: Logger, + client: Client, + message_listener: Arc, + option: Arc>, + shutdown_token: Option, + task_tracker: Option, +} + +impl PushConsumer { + pub fn new( + client_option: ClientOption, + option: PushConsumerOption, + message_listener: MessageListener, + ) -> Result { + if option.consumer_group().is_empty() { + return Err(ClientError::new( + ErrorKind::Config, + "consumer group is required.", + OPERATION_NEW_PUSH_CONSUMER, + )); + } + if option.subscription_expressions().is_empty() { + return Err(ClientError::new( + ErrorKind::Config, + "subscription expressions is required.", + OPERATION_NEW_PUSH_CONSUMER, + )); + } + let client_option = ClientOption { + client_type: ClientType::PushConsumer, + group: Some(option.consumer_group().to_string()), + ..client_option + }; + let logger = log::logger(option.logging_format()); + let client = Client::new( + &logger, + client_option, + build_push_consumer_settings(&option), + )?; + Ok(Self { + logger, + client, + message_listener: Arc::new(message_listener), + option: Arc::new(RwLock::new(option)), + shutdown_token: None, + task_tracker: None, + }) + } + + pub async fn start(&mut self) -> Result<(), ClientError> { + let (telemetry_command_tx, mut telemetry_command_rx) = mpsc::channel(16); + self.client.start(telemetry_command_tx).await?; + let option = Arc::clone(&self.option); + let mut rpc_client = self.client.get_session().await?; + let route_manager = self.client.get_route_manager(); + let topics; + { + topics = option + .read() + .subscription_expressions() + .keys() + .cloned() + .collect(); + } + route_manager + .sync_topic_routes(&mut rpc_client, topics) + .await?; + let logger = self.logger.clone(); + let mut actor_table: HashMap = HashMap::new(); + + let message_listener = Arc::clone(&self.message_listener); + let retry_policy: Arc>> = Arc::new(Mutex::new(None)); + + let shutdown_token = CancellationToken::new(); + self.shutdown_token = Some(shutdown_token.clone()); + let task_tracker = TaskTracker::new(); + self.task_tracker = Some(task_tracker.clone()); + + task_tracker.spawn(async move { + let mut scan_assignments_timer = + tokio::time::interval(std::time::Duration::from_secs(30)); + loop { + select! { + command = telemetry_command_rx.recv() => { + if let Some(command) = command { + let remote_backoff_policy = BackOffRetryPolicy::try_from(command); + if let Ok(remote_backoff_policy) = remote_backoff_policy { + retry_policy.lock().replace(remote_backoff_policy); + } + } + } + _ = scan_assignments_timer.tick() => { + let option_retry_policy; + { + option_retry_policy = retry_policy.lock().clone(); + } + if option_retry_policy.is_none() { + warn!(logger, "retry policy is not set. skip scanning."); + continue; + } + let retry_policy_inner = option_retry_policy.unwrap(); + let consumer_option; + { + consumer_option = option.read().clone(); + } + let subscription_table = consumer_option.subscription_expressions(); + // query endpoints from topic route + for topic in subscription_table.keys() { + if let Some(endpoints) = route_manager.pick_endpoints(topic.as_str()) { + let request = QueryAssignmentRequest { + topic: Some(Resource { + name: topic.to_string(), + resource_namespace: consumer_option.namespace().to_string(), + }), + group: Some(Resource { + name: consumer_option.consumer_group().to_string(), + resource_namespace: consumer_option.namespace().to_string(), + }), + endpoints: Some(endpoints.into_inner()), + }; + let result = rpc_client.query_assignment(request).await; + if let Ok(response) = result { + if handle_response_status(response.status, OPERATION_START_PUSH_CONSUMER).is_ok() { + let result = Self::process_assignments(logger.clone(), + &rpc_client, + &consumer_option, + Arc::clone(&message_listener), + &mut actor_table, + response.assignments, + retry_policy_inner.clone(), + ).await; + if result.is_err() { + error!(logger, "process assignments failed: {:?}", result.unwrap_err()); + } + } else { + error!(logger, "query assignment failed, no status in response."); + } + } else { + error!(logger, "query assignment failed: {:?}", result.unwrap_err()); + } + } + } + } + _ = shutdown_token.cancelled() => { + let entries = actor_table.drain(); + info!(logger, "shutdown {:?} actors", entries.len()); + for (_, actor) in entries { + let _ = actor.shutdown().await; + } + break; + } + } + } + }); + Ok(()) + } + + async fn process_assignments( + logger: Logger, + rpc_client: &Session, + option: &PushConsumerOption, + message_listener: Arc, + actor_table: &mut HashMap, + mut assignments: Vec, + retry_policy: BackOffRetryPolicy, + ) -> Result<(), ClientError> { + let message_queues: Vec = assignments + .iter_mut() + .filter_map(|assignment| { + if let Some(message_queue) = assignment.message_queue.take() { + return MessageQueue::from_pb_message_queue(message_queue).ok(); + } + None + }) + .collect(); + // remove existing actors from map, the remaining ones will be shutdown. + let mut actors: Vec = Vec::with_capacity(actor_table.len()); + message_queues.iter().for_each(|message_queue| { + let entry = actor_table.remove(message_queue); + if let Some(actor) = entry { + actors.push(actor); + } + }); + // the remaining actors will be shutdown. + let shutdown_entries = actor_table.drain(); + for (_, actor) in shutdown_entries { + let _ = actor.shutdown().await; + } + + for mut actor in actors { + let option = option.clone(); + actor.set_option(option); + actor.set_retry_policy(retry_policy.clone()); + actor_table.insert(actor.message_queue.clone(), actor); + } + + // start new actors + for message_queue in message_queues { + if actor_table.contains_key(&message_queue) { + continue; + } + let option = option.clone(); + let mut actor = MessageQueueActor::new( + logger.clone(), + rpc_client.shadow_session(), + message_queue.clone(), + option, + Arc::clone(&message_listener), + retry_policy.clone(), + ); + let result = actor.start().await; + if result.is_ok() { + actor_table.insert(message_queue, actor); + } + } + Ok(()) + } + + pub async fn shutdown(mut self) -> Result<(), ClientError> { + if let Some(shutdown_token) = self.shutdown_token.take() { + shutdown_token.cancel(); + } + if let Some(task_tracker) = self.task_tracker.take() { + task_tracker.close(); + task_tracker.wait().await; + } + self.client.shutdown().await?; + Ok(()) + } + + async fn receive_messages( + rpc_client: &mut T, + logger: &Logger, + message_queue: &MessageQueue, + option: &PushConsumerOption, + ) -> Result, ClientError> { + let filter_expression = option + .get_filter_expression(&message_queue.topic.name) + .ok_or(ClientError::new( + ErrorKind::Unknown, + "no filter expression presents", + OPERATION_RECEIVE_MESSAGE, + ))?; + let request = ReceiveMessageRequest { + group: Some(option.get_consumer_group_resource()), + message_queue: Some(message_queue.to_pb_message_queue()), + filter_expression: Some(pb::FilterExpression { + expression: filter_expression.expression().to_string(), + r#type: filter_expression.filter_type() as i32, + }), + batch_size: option.batch_size(), + auto_renew: true, + invisible_duration: None, + long_polling_timeout: Some( + prost_types::Duration::try_from(*option.long_polling_timeout()).unwrap(), + ), + }; + let responses = rpc_client.receive_message(request).await?; + let mut messages: Vec = Vec::with_capacity(option.batch_size() as usize); + let endpoints = build_endpoints_by_message_queue( + &message_queue.to_pb_message_queue(), + OPERATION_RECEIVE_MESSAGE, + )?; + for response in responses { + if response.content.is_some() { + let content = response.content.unwrap(); + match content { + Content::Status(status) => { + warn!(logger, "unhandled status message {:?}", status); + } + Content::DeliveryTimestamp(_) => { + warn!(logger, "unhandled delivery timestamp message"); + } + Content::Message(message) => { + messages.push( + MessageView::from_pb_message(message, endpoints.clone()).ok_or( + ClientError::new( + ErrorKind::InvalidMessage, + "error parsing from pb", + OPERATION_RECEIVE_MESSAGE, + ), + )?, + ); + } + } + } + } + Ok(messages) + } +} + +#[derive(Debug)] +enum AckEntryItem { + Ack(AckEntry), + Nack(NackEntry), + Dlq(DlqEntry), +} + +impl AckEntryItem { + fn inc_attempt(&mut self) { + match self { + Self::Ack(ack_entry) => { + ack_entry.attempt += 1; + } + Self::Nack(nack_entry) => { + nack_entry.attempt += 1; + } + Self::Dlq(entry) => { + entry.attempt += 1; + } + } + } +} + +#[derive(Debug)] +struct AckEntry { + message: MessageView, + attempt: usize, +} + +impl AckEntry { + fn new(message: MessageView) -> Self { + Self { + message, + attempt: 0, + } + } +} + +#[derive(Debug)] +struct NackEntry { + message: MessageView, + attempt: usize, + invisible_duration: Duration, +} + +impl NackEntry { + fn new(message: MessageView, invisible_duration: Duration) -> Self { + Self { + message, + invisible_duration, + attempt: 0, + } + } +} + +#[derive(Debug)] +struct DlqEntry { + message: MessageView, + attempt: usize, + delivery_attempt: i32, + max_delivery_attempt: i32, +} + +impl DlqEntry { + fn new(message: MessageView, delivery_attempt: i32, max_delivery_attempt: i32) -> Self { + Self { + message, + delivery_attempt, + max_delivery_attempt, + attempt: 0, + } + } +} + +struct MessageQueueActor { + logger: Logger, + rpc_client: Session, + message_queue: MessageQueue, + option: PushConsumerOption, + message_listener: Arc, + retry_policy: BackOffRetryPolicy, + shutdown_token: Option, + task_tracker: Option, +} + +#[automock] +impl MessageQueueActor { + pub(crate) fn new( + logger: Logger, + rpc_client: Session, + message_queue: MessageQueue, + option: PushConsumerOption, + message_listener: Arc, + retry_policy: BackOffRetryPolicy, + ) -> Self { + Self { + logger, + rpc_client, + message_queue, + option, + message_listener: Arc::clone(&message_listener), + retry_policy, + shutdown_token: None, + task_tracker: None, + } + } + + // The following two methods won't affect the running actor in fact. + pub(crate) fn set_option(&mut self, option: PushConsumerOption) { + self.option = option; + } + + pub(crate) fn set_retry_policy(&mut self, retry_policy: BackOffRetryPolicy) { + self.retry_policy = retry_policy; + } + + pub(crate) async fn start(&mut self) -> Result<(), ClientError> { + debug!( + self.logger, + "start a new queue actor {:?}", self.message_queue + ); + let shutdown_token = CancellationToken::new(); + self.shutdown_token = Some(shutdown_token.clone()); + let task_tracker = TaskTracker::new(); + self.task_tracker = Some(task_tracker.clone()); + + let mut consumer_worker_count = self.option.consumer_worker_count_each_queue(); + if self.option.fifo() { + consumer_worker_count = 1; + } + info!( + self.logger, + "start consumer worker count: {}", consumer_worker_count + ); + + for _ in 0..consumer_worker_count { + // create consumer worker + let mut consumer_worker: ConsumerWorker; + if self.option.fifo() { + consumer_worker = ConsumerWorker::Fifo(FifoConsumerWorker::new( + self.logger.clone(), + self.rpc_client.shadow_session(), + Arc::clone(&self.message_listener), + )); + } else { + consumer_worker = ConsumerWorker::Standard(StandardConsumerWorker::new( + self.logger.clone(), + self.rpc_client.shadow_session(), + Arc::clone(&self.message_listener), + )); + } + let mut ticker = tokio::time::interval(Duration::from_millis(1)); + let message_queue = self.message_queue.clone(); + let option = self.option.clone(); + let retry_policy = self.retry_policy.clone(); + let mut ack_processor = AckEntryProcessor::new( + self.logger.clone(), + self.rpc_client.shadow_session(), + self.option.get_consumer_group_resource(), + self.message_queue.topic.to_owned(), + ); + ack_processor.start().await?; + let shutdown_token = shutdown_token.clone(); + let logger = self.logger.clone(); + task_tracker.spawn(async move { + loop { + select! { + _ = ticker.tick() => { + let result = consumer_worker.receive_messages(&message_queue, &option, &mut ack_processor, &retry_policy).await; + if result.is_err() { + error!(logger, "receive messages error: {:?}", result.err()); + } + } + _ = shutdown_token.cancelled() => { + ack_processor.shutdown().await; + break; + } + } + } + }); + } + Ok(()) + } + + pub(crate) async fn shutdown(mut self) -> Result<(), ClientError> { + debug!(self.logger, "shutdown queue actor {:?}", self.message_queue); + if let Some(shutdown_token) = self.shutdown_token.take() { + shutdown_token.cancel(); + } + if let Some(task_tracker) = self.task_tracker.take() { + task_tracker.close(); + task_tracker.wait().await; + } + Ok(()) + } +} + +enum ConsumerWorker { + Standard(StandardConsumerWorker), + Fifo(FifoConsumerWorker), +} + +impl ConsumerWorker { + async fn receive_messages( + &mut self, + message_queue: &MessageQueue, + option: &PushConsumerOption, + ack_processor: &mut AckEntryProcessor, + retry_policy: &BackOffRetryPolicy, + ) -> Result<(), ClientError> { + match self { + ConsumerWorker::Standard(consumer_worker) => { + consumer_worker + .receive_messages(message_queue, option, ack_processor, retry_policy) + .await + } + ConsumerWorker::Fifo(consumer_worker) => { + consumer_worker + .receive_messages(message_queue, option, ack_processor, retry_policy) + .await + } + } + } +} + +struct StandardConsumerWorker { + logger: Logger, + rpc_client: Session, + message_listener: Arc, +} + +impl StandardConsumerWorker { + fn new(logger: Logger, rpc_client: Session, message_listener: Arc) -> Self { + Self { + logger, + rpc_client, + message_listener, + } + } + + async fn receive_messages( + &mut self, + message_queue: &MessageQueue, + option: &PushConsumerOption, + ack_processor: &mut AckEntryProcessor, + retry_policy: &BackOffRetryPolicy, + ) -> Result<(), ClientError> { + let messages = PushConsumer::receive_messages( + &mut self.rpc_client, + &self.logger, + message_queue, + option, + ) + .await?; + for message in messages { + let consume_result = (self.message_listener)(&message); + match consume_result { + ConsumeResult::SUCCESS => { + ack_processor.ack_message(message).await; + } + ConsumeResult::FAILURE => { + let delay = retry_policy.get_next_attempt_delay(message.delivery_attempt()); + ack_processor + .change_invisible_duration(message, delay) + .await; + } + } + } + + Ok(()) + } +} + +struct FifoConsumerWorker { + logger: Logger, + rpc_client: Session, + message_listener: Arc, +} + +impl FifoConsumerWorker { + fn new(logger: Logger, rpc_client: Session, message_listener: Arc) -> Self { + Self { + logger, + rpc_client, + message_listener, + } + } + + async fn receive_messages( + &mut self, + message_queue: &MessageQueue, + option: &PushConsumerOption, + ack_processor: &mut AckEntryProcessor, + retry_policy: &BackOffRetryPolicy, + ) -> Result<(), ClientError> { + let messages = PushConsumer::receive_messages( + &mut self.rpc_client, + &self.logger, + message_queue, + option, + ) + .await?; + for message in messages { + let mut delivery_attempt = message.delivery_attempt(); + let max_delivery_attempts = retry_policy.get_max_attempts(); + loop { + let consume_result = (self.message_listener)(&message); + match consume_result { + ConsumeResult::SUCCESS => { + ack_processor.ack_message(message).await; + break; + } + ConsumeResult::FAILURE => { + delivery_attempt += 1; + if delivery_attempt > max_delivery_attempts { + ack_processor + .forward_to_deadletter_queue( + message, + delivery_attempt, + max_delivery_attempts, + ) + .await; + break; + } else { + tokio::time::sleep( + retry_policy.get_next_attempt_delay(delivery_attempt), + ) + .await; + } + } + } + } + } + + Ok(()) + } +} + +struct AckEntryProcessor { + logger: Logger, + rpc_client: Session, + consumer_group: Resource, + topic: Resource, + ack_entry_sender: Option>, + shutdown_token: Option, + task_tracker: Option, +} + +impl AckEntryProcessor { + fn new(logger: Logger, rpc_client: Session, consumer_group: Resource, topic: Resource) -> Self { + Self { + logger, + rpc_client, + consumer_group, + topic, + ack_entry_sender: None, + shutdown_token: None, + task_tracker: None, + } + } + + fn shadow_self(&self) -> Self { + Self { + logger: self.logger.clone(), + rpc_client: self.rpc_client.shadow_session(), + consumer_group: self.consumer_group.clone(), + topic: self.topic.clone(), + ack_entry_sender: None, + shutdown_token: None, + task_tracker: None, + } + } + + async fn ack_message(&mut self, message: MessageView) { + let result = self.ack_message_inner(&message).await; + if result.is_err() { + if let Some(ack_entry_sender) = self.ack_entry_sender.as_ref() { + let send_result = ack_entry_sender + .send(AckEntryItem::Ack(AckEntry::new(message))) + .await; + if send_result.is_err() { + error!( + self.logger, + "put ack entry to queue error {:?}", + send_result.err() + ); + } + } else { + error!( + self.logger, + "The ack entry sender is not set. Drop the ack message." + ); + } + } + } + + async fn change_invisible_duration( + &mut self, + message: MessageView, + invisible_duration: Duration, + ) { + let result = self + .change_invisible_duration_inner(&message, invisible_duration) + .await; + if result.is_err() { + if let Some(ack_entry_sender) = self.ack_entry_sender.as_ref() { + let send_result = ack_entry_sender + .send(AckEntryItem::Nack(NackEntry::new( + message, + invisible_duration, + ))) + .await; + if send_result.is_err() { + error!( + self.logger, + "put nack entry to queue error {:?}", + send_result.err() + ); + } + } else { + error!( + self.logger, + "Drop the nack message due to ack entry sender is not set." + ) + } + } + } + + async fn forward_to_deadletter_queue( + &mut self, + message: MessageView, + delivery_attempt: i32, + max_delivery_attempts: i32, + ) { + let result = self + .forward_to_deadletter_queue_inner(&message, delivery_attempt, max_delivery_attempts) + .await; + if result.is_err() { + if let Some(ack_entry_sender) = self.ack_entry_sender.as_ref() { + let send_result = ack_entry_sender + .send(AckEntryItem::Dlq(DlqEntry::new( + message, + delivery_attempt, + max_delivery_attempts, + ))) + .await; + if send_result.is_err() { + error!( + self.logger, + "put dlq entry to queue error {:?}", + send_result.err() + ); + } + } else { + error!( + self.logger, + "Drop the dlq message due to ack edntry sender is not set." + ); + } + } + } + + async fn forward_to_deadletter_queue_inner( + &mut self, + message: &MessageView, + delivery_attempt: i32, + max_delivery_attempts: i32, + ) -> Result<(), ClientError> { + let request = ForwardMessageToDeadLetterQueueRequest { + group: Some(self.consumer_group.clone()), + topic: Some(self.topic.clone()), + receipt_handle: message.receipt_handle(), + message_id: message.message_id().to_string(), + delivery_attempt, + max_delivery_attempts, + }; + let response = self.rpc_client.forward_to_deadletter_queue(request).await?; + handle_response_status(response.status, OPERATION_FORWARD_TO_DEADLETTER_QUEUE) + } + + async fn change_invisible_duration_inner( + &mut self, + ack_entry: &MessageView, + invisible_duration: Duration, + ) -> Result<(), ClientError> { + let request = ChangeInvisibleDurationRequest { + group: Some(self.consumer_group.clone()), + topic: Some(self.topic.clone()), + receipt_handle: ack_entry.receipt_handle().to_string(), + message_id: ack_entry.message_id().to_string(), + invisible_duration: Some(prost_types::Duration::try_from(invisible_duration).map_err( + |_| { + ClientError::new( + ErrorKind::InvalidMessage, + "invalid invisbile duration", + OPERATION_CHANGE_INVISIBLE_DURATION, + ) + }, + )?), + }; + let response = self.rpc_client.change_invisible_duration(request).await?; + handle_response_status(response.status, OPERATION_CHANGE_INVISIBLE_DURATION) + } + + async fn start(&mut self) -> Result<(), ClientError> { + let (ack_entry_sender, mut ack_entry_receiver) = mpsc::channel(1024); + self.ack_entry_sender = Some(ack_entry_sender); + let mut ack_entry_queue: VecDeque = VecDeque::new(); + let mut ack_ticker = tokio::time::interval(Duration::from_millis(100)); + let mut processor = self.shadow_self(); + let shutdown_token = CancellationToken::new(); + self.shutdown_token = Some(shutdown_token.clone()); + let task_tracker = TaskTracker::new(); + self.task_tracker = Some(task_tracker.clone()); + task_tracker.spawn(async move { + loop { + select! { + _ = ack_ticker.tick() => { + let result = processor.process_ack_entry_queue(&mut ack_entry_queue).await; + if result.is_err() { + error!(processor.logger, "process ack entry queue failed: {:?}", result); + } + } + Some(ack_entry) = ack_entry_receiver.recv() => { + ack_entry_queue.push_back(ack_entry); + debug!(processor.logger, "ack entry queue size: {}", ack_entry_queue.len()); + } + _ = shutdown_token.cancelled() => { + info!(processor.logger, "need to process remaining {} entries on shutdown.", ack_entry_queue.len()); + processor.flush_ack_entry_queue(&mut ack_entry_queue).await; + break; + } + } + } + }); + Ok(()) + } + + async fn process_ack_entry_queue( + &mut self, + ack_entry_queue: &mut VecDeque, + ) -> Result<(), ClientError> { + if let Some(ack_entry_item) = ack_entry_queue.front_mut() { + let result = match ack_entry_item { + AckEntryItem::Ack(ack_entry) => self.ack_message_inner(&ack_entry.message).await, + AckEntryItem::Nack(nack_entry) => { + self.change_invisible_duration_inner( + &nack_entry.message, + nack_entry.invisible_duration, + ) + .await + } + AckEntryItem::Dlq(entry) => { + self.forward_to_deadletter_queue_inner( + &entry.message, + entry.delivery_attempt, + entry.max_delivery_attempt, + ) + .await + } + }; + if result.is_ok() { + ack_entry_queue.pop_front(); + } else { + error!( + self.logger, + "ack message failed: {:?}, will deliver later.", result + ); + ack_entry_item.inc_attempt(); + } + } + Ok(()) + } + + async fn flush_ack_entry_queue(&mut self, ack_entry_queue: &mut VecDeque) { + for ack_entry_item in ack_entry_queue { + match ack_entry_item { + AckEntryItem::Ack(entry) => { + let _ = self.ack_message_inner(&entry.message).await; + } + AckEntryItem::Nack(entry) => { + let _ = self + .change_invisible_duration_inner(&entry.message, entry.invisible_duration) + .await; + } + AckEntryItem::Dlq(entry) => { + let _ = self + .forward_to_deadletter_queue_inner( + &entry.message, + entry.delivery_attempt, + entry.max_delivery_attempt, + ) + .await; + } + } + } + } + + async fn ack_message_inner(&mut self, ack_entry: &MessageView) -> Result<(), ClientError> { + let request = AckMessageRequest { + group: Some(self.consumer_group.clone()), + topic: Some(self.topic.clone()), + entries: vec![pb::AckMessageEntry { + message_id: ack_entry.message_id().to_string(), + receipt_handle: ack_entry.receipt_handle().to_string(), + }], + }; + let response = self.rpc_client.ack_message(request).await?; + handle_response_status(response.status, OPERATION_ACK_MESSAGE) + } + + async fn shutdown(mut self) { + if let Some(shutdown_token) = self.shutdown_token.take() { + shutdown_token.cancel(); + } + if let Some(task_tracker) = self.task_tracker.take() { + info!(self.logger, "waiting for task to complete"); + task_tracker.close(); + task_tracker.wait().await; + info!(self.logger, "task completed"); + } + info!( + self.logger, + "The ack entry processor shuts down completely." + ); + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::{str::FromStr, vec}; + + use log::terminal_logger; + use pb::{ + AckMessageResponse, Address, Broker, ChangeInvisibleDurationResponse, Code, + ForwardMessageToDeadLetterQueueResponse, QueryRouteResponse, ReceiveMessageResponse, + Status, SystemProperties, + }; + + use crate::client::TopicRouteManager; + use crate::model::common::{Endpoints, FilterType}; + use crate::{ + conf::{CustomizedBackOffRetryPolicy, ExponentialBackOffRetryPolicy}, + model::common::FilterExpression, + }; + + use super::*; + + #[test] + fn test_new_push_consumer() { + let _m = crate::client::tests::MTX.lock(); + let result = PushConsumer::new( + ClientOption::default(), + PushConsumerOption::default(), + Box::new(|_| ConsumeResult::SUCCESS), + ); + assert!(result.is_err()); + + let mut option = PushConsumerOption::default(); + option.set_consumer_group("test"); + let result2 = PushConsumer::new( + ClientOption::default(), + option, + Box::new(|_| ConsumeResult::SUCCESS), + ); + assert!(result2.is_err()); + + let mut option2 = PushConsumerOption::default(); + option2.set_consumer_group("test"); + option2.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + let context = Client::new_context(); + context.expect().returning(|_, _, _| Ok(Client::default())); + let result3 = PushConsumer::new( + ClientOption::default(), + option2, + Box::new(|_| ConsumeResult::SUCCESS), + ); + assert!(result3.is_ok()); + } + + #[tokio::test] + async fn test_push_consumer_start_shutdown() -> Result<(), ClientError> { + let _m = crate::client::tests::MTX.lock(); + let context = Client::new_context(); + let mut client = Client::default(); + client.expect_start().return_once(|_| Ok(())); + client.expect_shutdown().return_once(|| Ok(())); + client.expect_get_session().returning(|| { + let mut session = Session::default(); + session.expect_query_route().returning(|_| { + Ok(QueryRouteResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "Success".to_string(), + }), + message_queues: vec![pb::MessageQueue { + topic: Some(Resource { + name: "DefaultCluster".to_string(), + resource_namespace: "default".to_string(), + }), + id: 0, + permission: 0, + broker: None, + accept_message_types: vec![], + }], + }) + }); + Ok(session) + }); + client.expect_get_route_manager().returning(|| { + TopicRouteManager::new( + terminal_logger(), + "".to_string(), + Endpoints::from_url("http://localhost:8081").unwrap(), + ) + }); + context.expect().return_once(|_, _, _| Ok(client)); + let mut push_consumer_option = PushConsumerOption::default(); + push_consumer_option.set_consumer_group("test_group"); + push_consumer_option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + let mut push_consumer = PushConsumer::new( + ClientOption::default(), + push_consumer_option, + Box::new(|_| ConsumeResult::SUCCESS), + )?; + push_consumer.start().await?; + push_consumer.shutdown().await?; + + Ok(()) + } + + #[tokio::test] + async fn test_process_assignments_invalid_assignment() -> Result<(), ClientError> { + let rpc_client = Session::default(); + let logger = terminal_logger(); + let option = &PushConsumerOption::default(); + let message_listener: Arc = Arc::new(Box::new(|_| ConsumeResult::SUCCESS)); + let mut actor_table: HashMap = HashMap::new(); + let retry_policy = + BackOffRetryPolicy::Exponential(ExponentialBackOffRetryPolicy::default()); + let assignments = vec![Assignment { + message_queue: Some(pb::MessageQueue { + topic: None, + id: 0, + permission: 0, + broker: None, + accept_message_types: vec![], + }), + }]; + let context = MockMessageQueueActor::new_context(); + context.expect().returning(|_, _, _, _, _, _| { + let mut actor = MockMessageQueueActor::default(); + actor.expect_start().returning(|| Ok(())); + return actor; + }); + PushConsumer::process_assignments( + logger, + &rpc_client, + option, + message_listener, + &mut actor_table, + assignments, + retry_policy, + ) + .await?; + assert!(actor_table.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn test_process_assignments() -> Result<(), ClientError> { + let logger = terminal_logger(); + let option = &PushConsumerOption::default(); + let message_listener: Arc = Arc::new(Box::new(|_| ConsumeResult::SUCCESS)); + + let mut rpc_client = Session::default(); + + rpc_client.expect_shadow_session().returning(|| { + let mut mock = Session::default(); + mock.expect_shadow_session().returning(|| { + let mut mock = Session::default(); + mock.expect_shadow_session() + .returning(|| Session::default()); + mock + }); + mock + }); + let mut actor_table: HashMap = HashMap::new(); + let retry_policy = + BackOffRetryPolicy::Exponential(ExponentialBackOffRetryPolicy::default()); + let assignments = vec![Assignment { + message_queue: Some(pb::MessageQueue { + topic: Some(Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }), + id: 0, + permission: 0, + broker: Some(Broker { + name: "".to_string(), + id: 0, + endpoints: Some(pb::Endpoints { + scheme: 0, + addresses: vec![Address { + host: "localhost".to_string(), + port: 8081, + }], + }), + }), + accept_message_types: vec![], + }), + }]; + PushConsumer::process_assignments( + logger, + &rpc_client, + option, + message_listener, + &mut actor_table, + assignments, + retry_policy, + ) + .await?; + assert_eq!(1, actor_table.len()); + Ok(()) + } + + #[tokio::test] + async fn test_process_assignments_two_queues() -> Result<(), ClientError> { + let logger = terminal_logger(); + let option = &PushConsumerOption::default(); + let message_listener: Arc = Arc::new(Box::new(|_| ConsumeResult::SUCCESS)); + let mut actor_table: HashMap = HashMap::new(); + let retry_policy = + BackOffRetryPolicy::Exponential(ExponentialBackOffRetryPolicy::default()); + let assignments = vec![ + Assignment { + message_queue: Some(pb::MessageQueue { + topic: Some(Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }), + id: 0, + permission: 0, + broker: Some(Broker { + name: "".to_string(), + id: 0, + endpoints: Some(pb::Endpoints { + scheme: 0, + addresses: vec![Address { + host: "localhost".to_string(), + port: 8081, + }], + }), + }), + accept_message_types: vec![], + }), + }, + Assignment { + message_queue: Some(pb::MessageQueue { + topic: Some(Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }), + id: 1, + permission: 0, + broker: Some(Broker { + name: "".to_string(), + id: 0, + endpoints: Some(pb::Endpoints { + scheme: 0, + addresses: vec![Address { + host: "localhost".to_string(), + port: 8081, + }], + }), + }), + accept_message_types: vec![], + }), + }, + ]; + let mut session = Session::default(); + session.expect_shadow_session().returning(|| { + let mut mock = Session::default(); + mock.expect_shadow_session().returning(|| { + let mut mock = Session::default(); + mock.expect_shadow_session() + .returning(|| Session::default()); + mock + }); + mock + }); + PushConsumer::process_assignments( + logger.clone(), + &session, + option, + message_listener, + &mut actor_table, + assignments.clone(), + retry_policy.clone(), + ) + .await?; + let assignments2 = vec![Assignment { + message_queue: Some(pb::MessageQueue { + topic: Some(Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }), + id: 0, + permission: 0, + broker: Some(Broker { + name: "".to_string(), + id: 0, + endpoints: Some(pb::Endpoints { + scheme: 0, + addresses: vec![Address { + host: "localhost".to_string(), + port: 8081, + }], + }), + }), + accept_message_types: vec![], + }), + }]; + PushConsumer::process_assignments( + logger, + &session, + option, + Arc::new(Box::new(|_| ConsumeResult::SUCCESS)), + &mut actor_table, + assignments2, + retry_policy, + ) + .await?; + assert_eq!(1, actor_table.len()); + assert!(actor_table.contains_key(&MessageQueue { + id: 0, + permission: 0, + topic: Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }, + broker: Broker { + name: "".to_string(), + id: 0, + endpoints: Some(pb::Endpoints { + scheme: 0, + addresses: vec![Address { + host: "localhost".to_string(), + port: 8081, + }], + }), + }, + accept_message_types: vec![], + })); + Ok(()) + } + + fn new_message_queue() -> MessageQueue { + MessageQueue { + id: 0, + topic: Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }, + broker: Broker { + name: "broker-0".to_string(), + id: 0, + endpoints: Some(pb::Endpoints { + scheme: 0, + addresses: vec![Address { + host: "localhost".to_string(), + port: 8081, + }], + }), + }, + accept_message_types: vec![], + permission: 0, + } + } + + fn new_message() -> pb::Message { + pb::Message { + topic: Some(Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }), + user_properties: HashMap::new(), + system_properties: Some(SystemProperties { + receipt_handle: Some("receipt".to_string()), + ..SystemProperties::default() + }), + body: vec![], + } + } + + #[tokio::test] + async fn test_actor_standard_start() { + let message_queue = new_message_queue(); + let mut option = PushConsumerOption::default(); + option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + option.set_consumer_worker_count_each_queue(3); + let retry_policy = BackOffRetryPolicy::Customized(CustomizedBackOffRetryPolicy::new( + pb::CustomizedBackoff { + next: vec![prost_types::Duration::from_str("1s").unwrap()], + }, + 1, + )); + let mut session = Session::default(); + session.expect_shadow_session().times(6).returning(|| { + let mut mock = Session::default(); + mock.expect_shadow_session() + .returning(|| Session::default()); + mock + }); + let mut actor = MessageQueueActor::new( + terminal_logger(), + session, + message_queue, + option, + Arc::new(Box::new(|_| ConsumeResult::SUCCESS)), + retry_policy, + ); + let result = actor.start().await; + assert!(result.is_ok()); + let result2 = actor.shutdown().await; + assert!(result2.is_ok()); + } + + #[tokio::test] + async fn test_actor_fifo_start() { + let message_queue = new_message_queue(); + let mut option = PushConsumerOption::default(); + option.set_fifo(true); + option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + option.set_consumer_worker_count_each_queue(3); + let retry_policy = BackOffRetryPolicy::Customized(CustomizedBackOffRetryPolicy::new( + pb::CustomizedBackoff { + next: vec![prost_types::Duration::from_str("1s").unwrap()], + }, + 1, + )); + let mut session = Session::default(); + session.expect_shadow_session().times(2).returning(|| { + let mut mock = Session::default(); + mock.expect_shadow_session() + .returning(|| Session::default()); + mock + }); + let mut actor = MessageQueueActor::new( + terminal_logger(), + session, + message_queue, + option, + Arc::new(Box::new(|_| ConsumeResult::SUCCESS)), + retry_policy, + ); + let result = actor.start().await; + assert!(result.is_ok()); + let result2 = actor.shutdown().await; + assert!(result2.is_ok()); + } + + #[tokio::test] + async fn test_ack_messages_success() { + let mut rpc_client = Session::default(); + rpc_client.expect_shadow_session().returning(|| { + let mut mock = Session::default(); + mock.expect_ack_message().never(); + mock + }); + rpc_client.expect_ack_message().times(1).returning(|_| { + Ok(AckMessageResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + entries: vec![], + }) + }); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), rpc_client, consumer_group, topic); + let result = ack_processor.start().await; + assert!(result.is_ok()); + + ack_processor + .ack_message( + MessageView::from_pb_message( + new_message(), + Endpoints::from_url("http://127.0.0.1:8080").unwrap(), + ) + .unwrap(), + ) + .await; + + tokio::time::sleep(Duration::from_millis(500)).await; + ack_processor.shutdown().await; + } + + #[tokio::test] + async fn test_ack_messages_failure_once() { + let mut rpc_client = Session::default(); + let mut mock = Session::default(); + mock.expect_ack_message().times(1).returning(|_| { + Ok(AckMessageResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + entries: vec![], + }) + }); + rpc_client.expect_shadow_session().return_once(|| mock); + rpc_client.expect_ack_message().times(1).returning(|_| { + Ok(AckMessageResponse { + status: Some(Status { + code: Code::BadRequest as i32, + message: "".to_string(), + }), + entries: vec![], + }) + }); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), rpc_client, consumer_group, topic); + let result = ack_processor.start().await; + assert!(result.is_ok()); + + ack_processor + .ack_message( + MessageView::from_pb_message( + new_message(), + Endpoints::from_url("http://127.0.0.1:8080").unwrap(), + ) + .unwrap(), + ) + .await; + + tokio::time::sleep(Duration::from_millis(500)).await; + ack_processor.shutdown().await; + } + + #[tokio::test] + async fn test_nack_messages_success() { + let mut rpc_client = Session::default(); + rpc_client.expect_shadow_session().returning(|| { + let mut mock = Session::default(); + mock.expect_change_invisible_duration().never(); + mock + }); + rpc_client + .expect_change_invisible_duration() + .times(1) + .returning(|_| { + Ok(ChangeInvisibleDurationResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + receipt_handle: "".to_string(), + }) + }); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), rpc_client, consumer_group, topic); + let result = ack_processor.start().await; + assert!(result.is_ok()); + + ack_processor + .change_invisible_duration( + MessageView::from_pb_message( + new_message(), + Endpoints::from_url("http://127.0.0.1:8080").unwrap(), + ) + .unwrap(), + Duration::from_secs(3), + ) + .await; + + tokio::time::sleep(Duration::from_millis(500)).await; + ack_processor.shutdown().await; + } + + #[tokio::test] + async fn test_nack_messages_failure_once() { + let mut rpc_client = Session::default(); + let mut mock = Session::default(); + mock.expect_change_invisible_duration() + .times(1) + .returning(|_| { + Ok(ChangeInvisibleDurationResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + receipt_handle: "".to_string(), + }) + }); + rpc_client.expect_shadow_session().return_once(|| mock); + rpc_client + .expect_change_invisible_duration() + .times(1) + .returning(|_| { + Ok(ChangeInvisibleDurationResponse { + status: Some(Status { + code: Code::BadRequest as i32, + message: "".to_string(), + }), + receipt_handle: "".to_string(), + }) + }); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), rpc_client, consumer_group, topic); + let result = ack_processor.start().await; + assert!(result.is_ok()); + + ack_processor + .change_invisible_duration( + MessageView::from_pb_message( + new_message(), + Endpoints::from_url("http://127.0.0.1:8080").unwrap(), + ) + .unwrap(), + Duration::from_secs(3), + ) + .await; + + tokio::time::sleep(Duration::from_millis(500)).await; + ack_processor.shutdown().await; + } + + #[tokio::test] + async fn test_send_dlq_messages_success() { + let mut rpc_client = Session::default(); + rpc_client.expect_shadow_session().returning(|| { + let mut mock = Session::default(); + mock.expect_forward_to_deadletter_queue().never(); + mock + }); + rpc_client + .expect_forward_to_deadletter_queue() + .times(1) + .returning(|_| { + Ok(ForwardMessageToDeadLetterQueueResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + }) + }); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), rpc_client, consumer_group, topic); + let result = ack_processor.start().await; + assert!(result.is_ok()); + + ack_processor + .forward_to_deadletter_queue( + MessageView::from_pb_message( + new_message(), + Endpoints::from_url("http://127.0.0.1:8080").unwrap(), + ) + .unwrap(), + 1, + 2, + ) + .await; + + tokio::time::sleep(Duration::from_millis(500)).await; + ack_processor.shutdown().await; + } + + #[tokio::test] + async fn test_send_dlq_messages_failure_once() { + let mut rpc_client = Session::default(); + let mut mock = Session::default(); + mock.expect_forward_to_deadletter_queue() + .times(1) + .returning(|_| { + Ok(ForwardMessageToDeadLetterQueueResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + }) + }); + rpc_client.expect_shadow_session().return_once(|| mock); + rpc_client + .expect_forward_to_deadletter_queue() + .times(1) + .returning(|_| { + Ok(ForwardMessageToDeadLetterQueueResponse { + status: Some(Status { + code: Code::BadRequest as i32, + message: "".to_string(), + }), + }) + }); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), rpc_client, consumer_group, topic); + let result = ack_processor.start().await; + assert!(result.is_ok()); + + ack_processor + .forward_to_deadletter_queue( + MessageView::from_pb_message( + new_message(), + Endpoints::from_url("http://127.0.0.1:8080").unwrap(), + ) + .unwrap(), + 1, + 2, + ) + .await; + + tokio::time::sleep(Duration::from_millis(500)).await; + ack_processor.shutdown().await; + } + + #[tokio::test] + async fn test_standard_consume_messages_success() { + let mut rpc_client = Session::default(); + rpc_client.expect_receive_message().returning(|_| { + Ok(vec![ReceiveMessageResponse { + content: Some(Content::Message(new_message())), + }]) + }); + let consumer_worker = StandardConsumerWorker::new( + terminal_logger(), + rpc_client, + Arc::new(Box::new(|_| ConsumeResult::SUCCESS)), + ); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let retry_policy = &BackOffRetryPolicy::Exponential(ExponentialBackOffRetryPolicy::new( + pb::ExponentialBackoff { + initial: Some(prost_types::Duration::from_str("1s").unwrap()), + max: Some(prost_types::Duration::from_str("60s").unwrap()), + multiplier: 2.0, + }, + 3, + )); + let mut option = PushConsumerOption::default(); + option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + let mut session = Session::default(); + session + .expect_shadow_session() + .returning(|| Session::default()); + session.expect_ack_message().times(1).returning(|_| { + Ok(AckMessageResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + entries: vec![], + }) + }); + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), session, consumer_group, topic); + let is_start_ok = ack_processor.start().await; + assert!(is_start_ok.is_ok()); + let result = ConsumerWorker::Standard(consumer_worker) + .receive_messages( + &new_message_queue(), + &option, + &mut ack_processor, + retry_policy, + ) + .await; + assert!(result.is_ok(), "{:?}", result); + } + + #[tokio::test] + async fn test_standard_consume_messages_failure() { + let mut rpc_client = Session::default(); + rpc_client.expect_receive_message().returning(|_| { + Ok(vec![ReceiveMessageResponse { + content: Some(Content::Message(new_message())), + }]) + }); + let consumer_worker = StandardConsumerWorker::new( + terminal_logger(), + rpc_client, + Arc::new(Box::new(|_| ConsumeResult::FAILURE)), + ); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let retry_policy = &BackOffRetryPolicy::Exponential(ExponentialBackOffRetryPolicy::new( + pb::ExponentialBackoff { + initial: Some(prost_types::Duration::from_str("1s").unwrap()), + max: Some(prost_types::Duration::from_str("60s").unwrap()), + multiplier: 2.0, + }, + 3, + )); + let mut option = PushConsumerOption::default(); + option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + let mut session = Session::default(); + session + .expect_shadow_session() + .returning(|| Session::default()); + session.expect_ack_message().never(); + session + .expect_change_invisible_duration() + .times(1) + .returning(|_| { + Ok(ChangeInvisibleDurationResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + receipt_handle: "".to_string(), + }) + }); + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), session, consumer_group, topic); + let is_start_ok = ack_processor.start().await; + assert!(is_start_ok.is_ok()); + let result = ConsumerWorker::Standard(consumer_worker) + .receive_messages( + &new_message_queue(), + &option, + &mut ack_processor, + retry_policy, + ) + .await; + assert!(result.is_ok(), "{:?}", result); + } + + #[tokio::test] + async fn test_fifo_consume_message_success() { + let mut rpc_client = Session::default(); + rpc_client.expect_receive_message().returning(|_| { + Ok(vec![ReceiveMessageResponse { + content: Some(Content::Message(new_message())), + }]) + }); + let consumer_worker = FifoConsumerWorker::new( + terminal_logger(), + rpc_client, + Arc::new(Box::new(|_| ConsumeResult::SUCCESS)), + ); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let retry_policy = &BackOffRetryPolicy::Customized(CustomizedBackOffRetryPolicy::new( + pb::CustomizedBackoff { + next: vec![prost_types::Duration::from_str("0.01s").unwrap()], + }, + 1, + )); + let mut option = PushConsumerOption::default(); + option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + let mut session = Session::default(); + session + .expect_shadow_session() + .returning(|| Session::default()); + session.expect_ack_message().times(1).returning(|_| { + Ok(AckMessageResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + entries: vec![], + }) + }); + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), session, consumer_group, topic); + let result = ConsumerWorker::Fifo(consumer_worker) + .receive_messages( + &new_message_queue(), + &option, + &mut ack_processor, + retry_policy, + ) + .await; + assert!(result.is_ok(), "{:?}", result); + } + + #[tokio::test] + async fn test_fifo_consume_message_failure() { + let mut rpc_client = Session::default(); + rpc_client.expect_receive_message().returning(|_| { + Ok(vec![ReceiveMessageResponse { + content: Some(Content::Message(new_message())), + }]) + }); + let receive_call_count = Arc::new(AtomicUsize::new(0)); + let outer_call_counter = Arc::clone(&receive_call_count); + let consumer_worker = FifoConsumerWorker::new( + terminal_logger(), + rpc_client, + Arc::new(Box::new(move |_| { + receive_call_count.fetch_add(1, Ordering::Relaxed); + ConsumeResult::FAILURE + })), + ); + let consumer_group = Resource { + name: "test_group".to_string(), + resource_namespace: "".to_string(), + }; + let topic = Resource { + name: "test_topic".to_string(), + resource_namespace: "".to_string(), + }; + let retry_policy = &BackOffRetryPolicy::Customized(CustomizedBackOffRetryPolicy::new( + pb::CustomizedBackoff { + next: vec![prost_types::Duration::from_str("0.01s").unwrap()], + }, + 1, + )); + let mut option = PushConsumerOption::default(); + option.subscribe("test_topic", FilterExpression::new(FilterType::Tag, "*")); + let mut session = Session::default(); + session + .expect_shadow_session() + .returning(|| Session::default()); + session.expect_ack_message().never(); + session + .expect_forward_to_deadletter_queue() + .times(1) + .returning(|_| { + Ok(ForwardMessageToDeadLetterQueueResponse { + status: Some(Status { + code: Code::Ok as i32, + message: "".to_string(), + }), + }) + }); + let mut ack_processor = + AckEntryProcessor::new(terminal_logger(), session, consumer_group, topic); + let result = ConsumerWorker::Fifo(consumer_worker) + .receive_messages( + &new_message_queue(), + &option, + &mut ack_processor, + retry_policy, + ) + .await; + assert!(result.is_ok(), "{:?}", result); + assert_eq!(2, outer_call_counter.load(Ordering::Relaxed)); + } +} diff --git a/rust/src/session.rs b/rust/src/session.rs index cd84f9270..d16a70d8b 100644 --- a/rust/src/session.rs +++ b/rust/src/session.rs @@ -14,15 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -use std::collections::HashMap; use async_trait::async_trait; -use mockall::automock; +use mockall::{automock, mock}; use ring::hmac; use slog::{debug, error, info, o, Logger}; use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; -use tokio::sync::{mpsc, oneshot, Mutex}; +use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; use tokio_stream::StreamExt; use tonic::metadata::{AsciiMetadataValue, MetadataMap}; @@ -35,9 +34,11 @@ use crate::pb::telemetry_command::Command; use crate::pb::{ AckMessageRequest, AckMessageResponse, ChangeInvisibleDurationRequest, ChangeInvisibleDurationResponse, EndTransactionRequest, EndTransactionResponse, + ForwardMessageToDeadLetterQueueRequest, ForwardMessageToDeadLetterQueueResponse, HeartbeatRequest, HeartbeatResponse, NotifyClientTerminationRequest, - NotifyClientTerminationResponse, QueryRouteRequest, QueryRouteResponse, ReceiveMessageRequest, - ReceiveMessageResponse, SendMessageRequest, SendMessageResponse, TelemetryCommand, + NotifyClientTerminationResponse, QueryAssignmentRequest, QueryAssignmentResponse, + QueryRouteRequest, QueryRouteResponse, ReceiveMessageRequest, ReceiveMessageResponse, + SendMessageRequest, SendMessageResponse, TelemetryCommand, }; use crate::util::{PROTOCOL_VERSION, SDK_LANGUAGE, SDK_VERSION}; use crate::{error::ClientError, pb::messaging_service_client::MessagingServiceClient}; @@ -53,6 +54,8 @@ const OPERATION_ACK_MESSAGE: &str = "rpc.ack_message"; const OPERATION_CHANGE_INVISIBLE_DURATION: &str = "rpc.change_invisible_duration"; const OPERATION_END_TRANSACTION: &str = "rpc.end_transaction"; const OPERATION_NOTIFY_CLIENT_TERMINATION: &str = "rpc.notify_client_termination"; +const OPERATION_QUERY_ASSIGNMENT: &str = "rpc.query_assignment"; +const OPERATION_FORWARD_TO_DEADLETTER_QUEUE: &str = "rpc.forward_to_deadletter_queue"; #[async_trait] #[automock] @@ -89,6 +92,14 @@ pub(crate) trait RPCClient { &mut self, request: NotifyClientTerminationRequest, ) -> Result; + async fn query_assignment( + &mut self, + request: QueryAssignmentRequest, + ) -> Result; + async fn forward_to_deadletter_queue( + &mut self, + request: ForwardMessageToDeadLetterQueueRequest, + ) -> Result; } #[derive(Debug)] @@ -103,6 +114,11 @@ pub(crate) struct Session { } impl Session { + const OPERATION_CREATE: &'static str = "session.create_session"; + + const HTTP_SCHEMA: &'static str = "http"; + const HTTPS_SCHEMA: &'static str = "https"; + pub(crate) fn shadow_session(&self) -> Self { Session { logger: self.logger.clone(), @@ -114,31 +130,8 @@ impl Session { shutdown_tx: None, } } -} - -impl Session { - const OPERATION_CREATE: &'static str = "session.create_session"; - const HTTP_SCHEMA: &'static str = "http"; - const HTTPS_SCHEMA: &'static str = "https"; - - #[cfg(test)] - pub(crate) fn mock() -> Self { - use crate::log::terminal_logger; - Session { - logger: terminal_logger(), - client_id: "fake_id".to_string(), - option: ClientOption::default(), - endpoints: Endpoints::from_url("http://localhost:8081").unwrap(), - stub: MessagingServiceClient::new( - Channel::from_static("http://localhost:8081").connect_lazy(), - ), - telemetry_tx: None, - shutdown_tx: None, - } - } - - async fn new( + pub(crate) async fn new( logger: &Logger, endpoints: &Endpoints, client_id: String, @@ -362,7 +355,6 @@ impl Session { self.shutdown_tx.is_some() } - #[allow(dead_code)] pub(crate) async fn update_settings( &mut self, settings: TelemetryCommand, @@ -381,7 +373,6 @@ impl Session { } } -#[automock] #[async_trait] impl RPCClient for Session { async fn query_route( @@ -539,70 +530,112 @@ impl RPCClient for Session { })?; Ok(response.into_inner()) } -} - -#[derive(Debug)] -pub(crate) struct SessionManager { - logger: Logger, - client_id: String, - option: ClientOption, - session_map: Mutex>, -} -#[automock] -impl SessionManager { - pub(crate) fn new(logger: &Logger, client_id: String, option: &ClientOption) -> Self { - let logger = logger.new(o!("component" => "session")); - let session_map = Mutex::new(HashMap::new()); - SessionManager { - logger, - client_id, - option: option.clone(), - session_map, - } + async fn query_assignment( + &mut self, + request: QueryAssignmentRequest, + ) -> Result { + let request = self.sign(request); + let response = self.stub.query_assignment(request).await.map_err(|e| { + ClientError::new( + ErrorKind::ClientInternal, + "send rpc query_assignment failed", + OPERATION_QUERY_ASSIGNMENT, + ) + .set_source(e) + })?; + Ok(response.into_inner()) } - pub(crate) async fn get_or_create_session( - &self, - endpoints: &Endpoints, - settings: TelemetryCommand, - telemetry_command_tx: mpsc::Sender, - ) -> Result { - let mut session_map = self.session_map.lock().await; - let endpoint_url = endpoints.endpoint_url().to_string(); - return if session_map.contains_key(&endpoint_url) { - Ok(session_map.get(&endpoint_url).unwrap().shadow_session()) - } else { - let mut session = Session::new( - &self.logger, - endpoints, - self.client_id.clone(), - &self.option, - ) - .await?; - session.start(settings, telemetry_command_tx).await?; - let shadow_session = session.shadow_session(); - session_map.insert(endpoint_url.clone(), session); - Ok(shadow_session) - }; + async fn forward_to_deadletter_queue( + &mut self, + request: ForwardMessageToDeadLetterQueueRequest, + ) -> Result { + let request = self.sign(request); + let response = self + .stub + .forward_message_to_dead_letter_queue(request) + .await + .map_err(|e| { + ClientError::new( + ErrorKind::ClientInternal, + "send rpc forward_to_deadletter_queue failed", + OPERATION_FORWARD_TO_DEADLETTER_QUEUE, + ) + .set_source(e) + })?; + Ok(response.into_inner()) } +} - pub(crate) async fn get_all_sessions(&self) -> Result, ClientError> { - let session_map = self.session_map.lock().await; - let mut sessions = Vec::new(); - for (_, session) in session_map.iter() { - sessions.push(session.shadow_session()); - } - Ok(sessions) +mock! { + #[derive(Debug)] + pub(crate) Session { + pub(crate) fn shadow_session(&self) -> Self; + pub(crate) async fn new( + logger: &Logger, + endpoints: &Endpoints, + client_id: String, + option: &ClientOption, + ) -> Result; + pub(crate) fn peer(&self) -> &str; + pub(crate) fn is_started(&self) -> bool; + pub(crate) async fn update_settings( + &mut self, + settings: TelemetryCommand, + ) -> Result<(), ClientError>; + pub(crate) async fn start( + &mut self, + settings: TelemetryCommand, + telemetry_command_tx: mpsc::Sender, + ) -> Result<(), ClientError>; + pub(crate) fn shutdown(&mut self); } - pub(crate) async fn shutdown(&self) { - let mut session_map = self.session_map.lock().await; - for (_, session) in session_map.iter_mut() { - session.shutdown(); - } - session_map.clear(); + #[async_trait] + impl RPCClient for Session { + async fn query_route( + &mut self, + request: QueryRouteRequest, + ) -> Result; + async fn heartbeat( + &mut self, + request: HeartbeatRequest, + ) -> Result; + async fn send_message( + &mut self, + request: SendMessageRequest, + ) -> Result; + async fn receive_message( + &mut self, + request: ReceiveMessageRequest, + ) -> Result, ClientError>; + async fn ack_message( + &mut self, + request: AckMessageRequest, + ) -> Result; + async fn change_invisible_duration( + &mut self, + request: ChangeInvisibleDurationRequest, + ) -> Result; + async fn end_transaction( + &mut self, + request: EndTransactionRequest, + ) -> Result; + async fn notify_shutdown( + &mut self, + request: NotifyClientTerminationRequest, + ) -> Result; + async fn query_assignment( + &mut self, + request: QueryAssignmentRequest, + ) -> Result; + async fn forward_to_deadletter_queue( + &mut self, + request: ForwardMessageToDeadLetterQueueRequest, + ) -> Result; } + } #[cfg(test)] @@ -682,31 +715,4 @@ mod tests { assert!(result.is_ok()); assert!(session.is_started()); } - - #[tokio::test] - async fn session_manager_new() { - let mut server = RocketMQMockServer::start_default().await; - server.setup( - MockBuilder::when() - .path("/apache.rocketmq.v2.MessagingService/Telemetry") - .then() - .return_status(Code::Ok), - ); - - let logger = terminal_logger(); - let mut client_option = ClientOption::default(); - client_option.set_enable_tls(false); - let session_manager = - SessionManager::new(&logger, "test_client".to_string(), &client_option); - let (tx, _) = mpsc::channel(16); - let session = session_manager - .get_or_create_session( - &Endpoints::from_url(&format!("localhost:{}", server.address().port())).unwrap(), - build_producer_settings(&ProducerOption::default()), - tx, - ) - .await - .unwrap(); - debug!(logger, "session: {:?}", session); - } } diff --git a/rust/src/simple_consumer.rs b/rust/src/simple_consumer.rs index 8d6550002..cc15949d2 100644 --- a/rust/src/simple_consumer.rs +++ b/rust/src/simple_consumer.rs @@ -15,13 +15,12 @@ * limitations under the License. */ -use std::sync::Arc; use std::time::Duration; use mockall_double::double; use slog::{info, warn, Logger}; use tokio::select; -use tokio::sync::{mpsc, oneshot, RwLock}; +use tokio::sync::{mpsc, oneshot}; #[double] use crate::client::Client; @@ -29,7 +28,9 @@ use crate::conf::{ClientOption, SimpleConsumerOption}; use crate::error::{ClientError, ErrorKind}; use crate::model::common::{ClientType, FilterExpression}; use crate::model::message::{AckMessageEntry, MessageView}; -use crate::util::{build_endpoints_by_message_queue, select_message_queue}; +use crate::util::{ + build_endpoints_by_message_queue, build_simple_consumer_settings, select_message_queue, +}; use crate::{log, pb}; /// [`SimpleConsumer`] is a lightweight consumer to consume messages from RocketMQ proxy. @@ -45,7 +46,7 @@ use crate::{log, pb}; pub struct SimpleConsumer { option: SimpleConsumerOption, logger: Logger, - client: Client, + client: Client, shutdown_tx: Option>, } @@ -74,8 +75,11 @@ impl SimpleConsumer { ..client_option }; let logger = log::logger(option.logging_format()); - let settings = Arc::new(RwLock::new(option.clone())); - let client = Client::::new(&logger, client_option, settings)?; + let client = Client::new( + &logger, + client_option, + build_simple_consumer_settings(&option), + )?; Ok(SimpleConsumer { option, logger, @@ -180,7 +184,7 @@ impl SimpleConsumer { .await?; Ok(messages .into_iter() - .map(|message| MessageView::from_pb_message(message, endpoints.clone())) + .filter_map(|message| MessageView::from_pb_message(message, endpoints.clone())) .collect()) } @@ -229,7 +233,7 @@ mod tests { async fn simple_consumer_start() -> Result<(), ClientError> { let _m = crate::client::tests::MTX.lock(); - let ctx = Client::::new_context(); + let ctx = Client::new_context(); ctx.expect().return_once(|_, _, _| { let mut client = Client::default(); client.expect_topic_route().returning(|_, _| { diff --git a/rust/src/util.rs b/rust/src/util.rs index 1e5cf11c5..a71027998 100644 --- a/rust/src/util.rs +++ b/rust/src/util.rs @@ -21,13 +21,14 @@ use std::sync::Arc; use once_cell::sync::Lazy; use siphasher::sip::SipHasher24; -use crate::conf::{ProducerOption, SimpleConsumerOption}; +use crate::conf::{ProducerOption, PushConsumerOption, SimpleConsumerOption}; use crate::error::{ClientError, ErrorKind}; use crate::model::common::{ClientType, Endpoints, Route}; use crate::pb::settings::PubSub; use crate::pb::telemetry_command::Command; use crate::pb::{ - Language, MessageQueue, Publishing, Resource, Settings, Subscription, TelemetryCommand, Ua, + Code, FilterExpression, Language, MessageQueue, Publishing, Resource, Settings, Status, + Subscription, SubscriptionEntry, TelemetryCommand, Ua, }; pub(crate) static SDK_LANGUAGE: Language = Language::Rust; @@ -154,6 +155,74 @@ pub(crate) fn build_simple_consumer_settings(option: &SimpleConsumerOption) -> T } } +pub(crate) fn build_push_consumer_settings(option: &PushConsumerOption) -> TelemetryCommand { + let subscriptions: Vec = option + .subscription_expressions() + .iter() + .map(|(topic, filter_expression)| SubscriptionEntry { + topic: Some(Resource { + name: topic.to_string(), + resource_namespace: option.namespace().to_string(), + }), + expression: Some(FilterExpression { + expression: filter_expression.expression().to_string(), + r#type: filter_expression.filter_type() as i32, + }), + }) + .collect(); + let platform = os_info::get(); + TelemetryCommand { + command: Some(Command::Settings(Settings { + client_type: Some(ClientType::PushConsumer as i32), + request_timeout: Some(prost_types::Duration { + seconds: option.timeout().as_secs() as i64, + nanos: option.timeout().subsec_nanos() as i32, + }), + pub_sub: Some(PubSub::Subscription(Subscription { + group: Some(Resource { + name: option.consumer_group().to_string(), + resource_namespace: option.namespace().to_string(), + }), + subscriptions, + fifo: Some(option.fifo()), + receive_batch_size: None, + long_polling_timeout: Some(prost_types::Duration { + seconds: option.long_polling_timeout().as_secs() as i64, + nanos: option.long_polling_timeout().subsec_nanos() as i32, + }), + })), + user_agent: Some(Ua { + language: SDK_LANGUAGE as i32, + version: SDK_VERSION.to_string(), + platform: format!("{} {}", platform.os_type(), platform.version()), + hostname: HOST_NAME.clone(), + }), + ..Settings::default() + })), + ..TelemetryCommand::default() + } +} + +pub fn handle_response_status( + status: Option, + operation: &'static str, +) -> Result<(), ClientError> { + let status = status.ok_or(ClientError::new( + ErrorKind::Server, + "server do not return status, this may be a bug", + operation, + ))?; + + if status.code != Code::Ok as i32 { + return Err( + ClientError::new(ErrorKind::Server, "server return an error", operation) + .with_context("code", format!("{}", status.code)) + .with_context("message", status.message), + ); + } + Ok(()) +} + #[cfg(test)] mod tests { use std::sync::atomic::AtomicUsize; @@ -274,4 +343,49 @@ mod tests { fn util_build_simple_consumer_settings() { build_simple_consumer_settings(&SimpleConsumerOption::default()); } + + #[test] + fn test_handle_response_status() { + let result = handle_response_status(None, "test"); + assert!(result.is_err(), "should return error when status is None"); + let result = result.unwrap_err(); + assert_eq!(result.kind, ErrorKind::Server); + assert_eq!( + result.message, + "server do not return status, this may be a bug" + ); + assert_eq!(result.operation, "test"); + + let result = handle_response_status( + Some(Status { + code: Code::BadRequest as i32, + message: "test failed".to_string(), + }), + "test failed", + ); + assert!( + result.is_err(), + "should return error when status is BadRequest" + ); + let result = result.unwrap_err(); + assert_eq!(result.kind, ErrorKind::Server); + assert_eq!(result.message, "server return an error"); + assert_eq!(result.operation, "test failed"); + assert_eq!( + result.context, + vec![ + ("code", format!("{}", Code::BadRequest as i32)), + ("message", "test failed".to_string()), + ] + ); + + let result = handle_response_status( + Some(Status { + code: Code::Ok as i32, + message: "test success".to_string(), + }), + "test success", + ); + assert!(result.is_ok(), "should not return error when status is Ok"); + } }