diff --git a/.dockerignore b/.dockerignore index b4da54bad..792ad6870 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,12 @@ +.config +.github /assets /local_data +/certs +/scripts +target +Dockerfile +docker-compose.yml +.dockerignore .git .gitignore diff --git a/Cargo.lock b/Cargo.lock index d90c7188e..cc3268ad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,6 +185,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-broadcast" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-channel" version = "2.3.1" @@ -1061,7 +1073,7 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cli" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "clap", @@ -2228,10 +2240,11 @@ dependencies = [ [[package]] name = "iggy" -version = "0.5.1" +version = "0.6.0" dependencies = [ "aes-gcm", "anyhow", + "async-broadcast 0.7.1", "async-dropper", "async-trait", "base64 0.22.1", @@ -2242,9 +2255,12 @@ dependencies = [ "comfy-table", "convert_case 0.6.0", "crc32fast", + "derive_more", "dirs", "fast-async-mutex", "flume", + "futures", + "futures-util", "humantime", "keyring", "lazy_static", @@ -2276,6 +2292,7 @@ dependencies = [ "anyhow", "bytes", "clap", + "futures-util", "iggy", "rand", "serde", @@ -4216,7 +4233,7 @@ dependencies = [ [[package]] name = "server" -version = "0.3.6" +version = "0.4.0" dependencies = [ "anyhow", "async-stream", @@ -5612,7 +5629,7 @@ version = "3.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "675d170b632a6ad49804c8cf2105d7c31eddd3312555cffd4b740e08e97c25e6" dependencies = [ - "async-broadcast", + "async-broadcast 0.5.1", "async-executor", "async-fs", "async-io 1.13.0", diff --git a/bench/src/benchmarks/benchmark.rs b/bench/src/benchmarks/benchmark.rs index fd4e0ca0a..0fc634184 100644 --- a/bench/src/benchmarks/benchmark.rs +++ b/bench/src/benchmarks/benchmark.rs @@ -10,7 +10,7 @@ use crate::{ use async_trait::async_trait; use futures::Future; use iggy::client::{StreamClient, TopicClient}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::compression::compression_algorithm::CompressionAlgorithm; use iggy::error::IggyError; use iggy::utils::expiry::IggyExpiry; @@ -64,13 +64,7 @@ pub trait Benchmarkable { let topic_id: u32 = 1; let partitions_count: u32 = self.args().number_of_partitions(); let client = self.client_factory().create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; let streams = client.get_streams().await?; for i in 1..=number_of_streams { @@ -106,13 +100,7 @@ pub trait Benchmarkable { let start_stream_id = self.args().start_stream_id(); let number_of_streams = self.args().number_of_streams(); let client = self.client_factory().create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; let streams = client.get_streams().await?; for i in 1..=number_of_streams { diff --git a/bench/src/benchmarks/consumer_group_benchmark.rs b/bench/src/benchmarks/consumer_group_benchmark.rs index 51e3e8093..3a4871a84 100644 --- a/bench/src/benchmarks/consumer_group_benchmark.rs +++ b/bench/src/benchmarks/consumer_group_benchmark.rs @@ -5,9 +5,7 @@ use crate::{ }; use async_trait::async_trait; use iggy::{ - client::ConsumerGroupClient, - clients::client::{IggyClient, IggyClientBackgroundConfig}, - error::IggyError, + client::ConsumerGroupClient, clients::client::IggyClient, error::IggyError, utils::byte_size::IggyByteSize, }; use integration::test_server::{login_root, ClientFactory}; @@ -33,13 +31,7 @@ impl ConsumerGroupBenchmark { let start_stream_id = self.args().start_stream_id(); let topic_id: u32 = 1; let client = self.client_factory().create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; for i in 1..=consumer_groups_count { let consumer_group_id = CONSUMER_GROUP_BASE_ID + i; diff --git a/bench/src/consumer.rs b/bench/src/consumer.rs index f246bee71..6e80dcbc5 100644 --- a/bench/src/consumer.rs +++ b/bench/src/consumer.rs @@ -1,7 +1,7 @@ use crate::args::simple::BenchmarkKind; use crate::benchmark_result::{BenchmarkResult, LatencyPercentiles}; use iggy::client::{ConsumerGroupClient, MessageClient}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::consumer::Consumer as IggyConsumer; use iggy::error::IggyError; use iggy::messages::poll_messages::PollingStrategy; @@ -48,13 +48,7 @@ impl Consumer { let default_partition_id: u32 = 1; let total_messages = (self.messages_per_batch * self.message_batches) as u64; let client = self.client_factory.create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; let stream_id = self.stream_id.try_into().unwrap(); let topic_id = topic_id.try_into().unwrap(); diff --git a/bench/src/producer.rs b/bench/src/producer.rs index 06429933d..65fa01c03 100644 --- a/bench/src/producer.rs +++ b/bench/src/producer.rs @@ -1,7 +1,7 @@ use crate::args::simple::BenchmarkKind; use crate::benchmark_result::{BenchmarkResult, LatencyPercentiles}; use iggy::client::MessageClient; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::error::IggyError; use iggy::messages::send_messages::{Message, Partitioning}; use iggy::utils::duration::IggyDuration; @@ -52,13 +52,7 @@ impl Producer { let default_partition_id: u32 = 1; let total_messages = (self.messages_per_batch * self.message_batches) as u64; let client = self.client_factory.create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; info!( "Producer #{} → preparing the test messages...", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8612a7f2a..53fef5b61 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cli" -version = "0.3.0" +version = "0.4.0" edition = "2021" authors = ["bartosz.ciesla@gmail.com"] repository = "https://github.com/iggy-rs/iggy" diff --git a/cli/src/main.rs b/cli/src/main.rs index 24be53151..42e9595c3 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -61,7 +61,7 @@ use iggy::cli::{ }; use iggy::cli_command::{CliCommand, PRINT_TARGET}; use iggy::client_provider::{self, ClientProviderConfig}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::utils::crypto::{Aes256GcmEncryptor, Encryptor}; use iggy::utils::personal_access_token_expiry::PersonalAccessTokenExpiry; use std::sync::Arc; @@ -312,9 +312,9 @@ async fn main() -> Result<(), IggyCmdError> { // Create credentials based on command line arguments and command let mut credentials = IggyCredentials::new(&cli_options, &iggy_args, command.login_required())?; - let encryptor: Option> = match iggy_args.encryption_key.is_empty() { + let encryptor: Option> = match iggy_args.encryption_key.is_empty() { true => None, - false => Some(Box::new( + false => Some(Arc::new( Aes256GcmEncryptor::from_base64_key(&iggy_args.encryption_key).unwrap(), )), }; @@ -323,13 +323,7 @@ async fn main() -> Result<(), IggyCmdError> { let client = client_provider::get_raw_client(client_provider_config, command.connection_required()) .await?; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - encryptor, - ); + let client = IggyClient::create(client, None, encryptor); credentials.set_iggy_client(&client); credentials.login_user().await?; diff --git a/configs/server.json b/configs/server.json index c07a5bb6d..20aeb1415 100644 --- a/configs/server.json +++ b/configs/server.json @@ -126,6 +126,9 @@ "path": "compatibility" } }, + "state": { + "enforce_fsync": false + }, "runtime": { "path": "runtime" }, diff --git a/configs/server.toml b/configs/server.toml index 717f05741..f908a282e 100644 --- a/configs/server.toml +++ b/configs/server.toml @@ -286,6 +286,12 @@ path = "compatibility" ## Specifies the directory where database files are stored, relative to `system.path`. #path = "database" +[system.state] +# Determines whether to enforce file synchronization on state updates (boolean). +# `true` ensures immediate writing of data to disk for durability. +# `false` allows the OS to manage write operations, which can improve performance. +enforce_fsync = false + # Runtime configuration. [system.runtime] # Path for storing runtime data. diff --git a/examples/Cargo.toml b/examples/Cargo.toml index a4b4b0316..0d0bbfd28 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -35,10 +35,27 @@ path = "src/message-headers/consumer/main.rs" name = "message-headers-producer" path = "src/message-headers/producer/main.rs" +[[example]] +name = "multi-tenant-consumer" +path = "src/multi-tenant/consumer/main.rs" + +[[example]] +name = "multi-tenant-producer" +path = "src/multi-tenant/producer/main.rs" + +[[example]] +name = "new-sdk-consumer" +path = "src/new-sdk/consumer/main.rs" + +[[example]] +name = "new-sdk-producer" +path = "src/new-sdk/producer/main.rs" + [dependencies] anyhow = "1.0.86" bytes = "1.6.0" clap = { version = "4.5.4", features = ["derive"] } +futures-util = "0.3.30" iggy = { path = "../sdk" } rand = "0.8.5" serde = { version = "1.0.203", features = ["derive", "rc"] } diff --git a/examples/src/basic/consumer/main.rs b/examples/src/basic/consumer/main.rs index 8b6310c41..4a76a58d3 100644 --- a/examples/src/basic/consumer/main.rs +++ b/examples/src/basic/consumer/main.rs @@ -19,7 +19,6 @@ async fn main() -> Result<(), Box> { let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); let client = client_provider::get_raw_connected_client(client_provider_config).await?; let client = client.as_ref(); - system::login_root(client).await; system::init_by_consumer(&args, client).await; system::consume_messages(&args, client, &handle_message).await } diff --git a/examples/src/basic/producer/main.rs b/examples/src/basic/producer/main.rs index 2600979c7..33a461db4 100644 --- a/examples/src/basic/producer/main.rs +++ b/examples/src/basic/producer/main.rs @@ -21,17 +21,23 @@ async fn main() -> Result<(), Box> { let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); let client = client_provider::get_raw_connected_client(client_provider_config).await?; let client = client.as_ref(); - system::login_root(client).await; system::init_by_producer(&args, client).await?; produce_messages(&args, client).await } async fn produce_messages(args: &Args, client: &dyn Client) -> Result<(), Box> { + let interval = args.get_interval(); + info!( - "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {} ms.", - args.stream_id, args.topic_id, args.partition_id, args.interval + "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {}.", + args.stream_id, + args.topic_id, + args.partition_id, + interval.map_or("none".to_string(), |i| i.as_human_time_string()) ); - let mut interval = tokio::time::interval(std::time::Duration::from_millis(args.interval)); + let stream_id = args.stream_id.clone().try_into()?; + let topic_id = args.topic_id.clone().try_into()?; + let mut interval = interval.map(|interval| tokio::time::interval(interval.get_duration())); let mut current_id = 0u64; let mut sent_batches = 0; let partitioning = Partitioning::partition_id(args.partition_id); @@ -41,6 +47,10 @@ async fn produce_messages(args: &Args, client: &dyn Client) -> Result<(), Box Result<(), Box Result<(), Box> { } async fn consume_messages(client: &dyn Client) -> Result<(), Box> { - let interval = Duration::from_millis(500); + let interval = IggyDuration::from_str("500ms")?; info!( - "Messages will be consumed from stream: {}, topic: {}, partition: {} with interval {} ms.", + "Messages will be consumed from stream: {}, topic: {}, partition: {} with interval {}.", STREAM_ID, TOPIC_ID, PARTITION_ID, - interval.as_millis() + interval.as_human_time_string() ); let mut offset = 0; @@ -69,7 +70,7 @@ async fn consume_messages(client: &dyn Client) -> Result<(), Box> { if polled_messages.messages.is_empty() { info!("No messages found."); - sleep(interval).await; + sleep(interval.get_duration()).await; continue; } @@ -78,7 +79,7 @@ async fn consume_messages(client: &dyn Client) -> Result<(), Box> { handle_message(&message)?; } consumed_batches += 1; - sleep(interval).await; + sleep(interval.get_duration()).await; } } diff --git a/examples/src/getting-started/producer/main.rs b/examples/src/getting-started/producer/main.rs index df3e96228..efbac9447 100644 --- a/examples/src/getting-started/producer/main.rs +++ b/examples/src/getting-started/producer/main.rs @@ -4,13 +4,12 @@ use iggy::clients::client::IggyClient; use iggy::compression::compression_algorithm::CompressionAlgorithm; use iggy::messages::send_messages::{Message, Partitioning}; use iggy::users::defaults::*; +use iggy::utils::duration::IggyDuration; use iggy::utils::expiry::IggyExpiry; use iggy::utils::topic_size::MaxTopicSize; use std::env; use std::error::Error; use std::str::FromStr; -use std::time::Duration; -use tokio::time::sleep; use tracing::{info, warn}; const STREAM_ID: u32 = 1; @@ -64,13 +63,14 @@ async fn init_system(client: &IggyClient) { } async fn produce_messages(client: &dyn Client) -> Result<(), Box> { - let interval = Duration::from_millis(500); + let duration = IggyDuration::from_str("500ms")?; + let mut interval = tokio::time::interval(duration.get_duration()); info!( - "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {} ms.", + "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {}.", STREAM_ID, TOPIC_ID, PARTITION_ID, - interval.as_millis() + duration.as_human_time_string() ); let mut current_id = 0; @@ -83,6 +83,7 @@ async fn produce_messages(client: &dyn Client) -> Result<(), Box> { return Ok(()); } + interval.tick().await; let mut messages = Vec::new(); for _ in 0..messages_per_batch { current_id += 1; @@ -100,7 +101,6 @@ async fn produce_messages(client: &dyn Client) -> Result<(), Box> { .await?; sent_batches += 1; info!("Sent {messages_per_batch} message(s)."); - sleep(interval).await; } } diff --git a/examples/src/message-envelope/consumer/main.rs b/examples/src/message-envelope/consumer/main.rs index c673aad84..71ed227df 100644 --- a/examples/src/message-envelope/consumer/main.rs +++ b/examples/src/message-envelope/consumer/main.rs @@ -2,9 +2,7 @@ use anyhow::Result; use clap::Parser; use iggy::client_provider; use iggy::client_provider::ClientProviderConfig; -use iggy::clients::client::{ - IggyClient, IggyClientBackgroundConfig, PollMessagesConfig, StoreOffsetKind, -}; +use iggy::clients::client::IggyClient; use iggy::models::messages::PolledMessage; use iggy_examples::shared::args::Args; use iggy_examples::shared::messages::*; @@ -23,17 +21,7 @@ async fn main() -> Result<(), Box> { ); let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); let client = client_provider::get_raw_connected_client(client_provider_config).await?; - let client = IggyClient::builder() - .with_background_config(IggyClientBackgroundConfig { - poll_messages: PollMessagesConfig { - interval: args.interval, - store_offset_kind: StoreOffsetKind::WhenMessagesAreProcessed, - }, - ..Default::default() - }) - .with_client(client) - .build()?; - system::login_root(&client).await; + let client = IggyClient::new(client); system::init_by_consumer(&args, &client).await; system::consume_messages(&args, &client, &handle_message).await } diff --git a/examples/src/message-envelope/producer/main.rs b/examples/src/message-envelope/producer/main.rs index d7e365fb9..701f73130 100644 --- a/examples/src/message-envelope/producer/main.rs +++ b/examples/src/message-envelope/producer/main.rs @@ -24,17 +24,22 @@ async fn main() -> Result<(), Box> { let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); let client = client_provider::get_raw_connected_client(client_provider_config).await?; let client = IggyClient::builder().with_client(client).build()?; - system::login_root(&client).await; system::init_by_producer(&args, &client).await?; produce_messages(&args, &client).await } async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box> { + let interval = args.get_interval(); info!( - "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {} ms.", - args.stream_id, args.topic_id, args.partition_id, args.interval + "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {}.", + args.stream_id, + args.topic_id, + args.partition_id, + interval.map_or("none".to_string(), |i| i.as_human_time_string()) ); - let mut interval = tokio::time::interval(std::time::Duration::from_millis(args.interval)); + let stream_id = args.stream_id.clone().try_into()?; + let topic_id = args.topic_id.clone().try_into()?; + let mut interval = interval.map(|interval| tokio::time::interval(interval.get_duration())); let mut message_generator = MessagesGenerator::new(); let mut sent_batches = 0; let partitioning = Partitioning::partition_id(args.partition_id); @@ -44,6 +49,10 @@ async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box Result<(), Box Result<(), Box> { ); let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); let client = client_provider::get_raw_connected_client(client_provider_config).await?; - let client = IggyClient::builder() - .with_background_config(IggyClientBackgroundConfig { - poll_messages: PollMessagesConfig { - interval: args.interval, - store_offset_kind: StoreOffsetKind::WhenMessagesAreProcessed, - }, - ..Default::default() - }) - .with_client(client) - .build()?; - system::login_root(&client).await; + let client = IggyClient::new(client); system::init_by_consumer(&args, &client).await; system::consume_messages(&args, &client, &handle_message).await } diff --git a/examples/src/message-headers/producer/main.rs b/examples/src/message-headers/producer/main.rs index 7073fa918..2854cd272 100644 --- a/examples/src/message-headers/producer/main.rs +++ b/examples/src/message-headers/producer/main.rs @@ -27,17 +27,22 @@ async fn main() -> Result<(), Box> { let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); let client = client_provider::get_raw_connected_client(client_provider_config).await?; let client = IggyClient::builder().with_client(client).build()?; - system::login_root(&client).await; system::init_by_producer(&args, &client).await?; produce_messages(&args, &client).await } async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box> { + let interval = args.get_interval(); info!( - "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {} ms.", - args.stream_id, args.topic_id, args.partition_id, args.interval + "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {}.", + args.stream_id, + args.topic_id, + args.partition_id, + interval.map_or("none".to_string(), |i| i.as_human_time_string()) ); - let mut interval = tokio::time::interval(std::time::Duration::from_millis(args.interval)); + let stream_id = args.stream_id.clone().try_into()?; + let topic_id = args.topic_id.clone().try_into()?; + let mut interval = interval.map(|interval| tokio::time::interval(interval.get_duration())); let mut message_generator = MessagesGenerator::new(); let mut sent_batches = 0; let partitioning = Partitioning::partition_id(args.partition_id); @@ -47,6 +52,10 @@ async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box Result<(), Box anyhow::Result<(), Box> { + let args = Args::parse(); + tracing_subscriber::fmt::init(); + print_info("Multi-tenant consumer has started"); + let address = args.tcp_server_address; + + print_info("Creating root client to manage streams and users"); + let root_client = create_client(&address, DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD).await?; + + print_info("Creating users with permissions for each tenant"); + create_user(TENANT1_STREAM, TOPICS, TENANT1_USER, &root_client).await?; + create_user(TENANT2_STREAM, TOPICS, TENANT2_USER, &root_client).await?; + create_user(TENANT3_STREAM, TOPICS, TENANT3_USER, &root_client).await?; + + print_info("Disconnecting root client"); + root_client.disconnect().await?; + + print_info("Creating clients for each tenant"); + let tenant1_client = create_client(&address, TENANT1_USER, PASSWORD).await?; + let tenant2_client = create_client(&address, TENANT2_USER, PASSWORD).await?; + let tenant3_client = create_client(&address, TENANT3_USER, PASSWORD).await?; + + print_info("Ensuring access to topics for each tenant"); + ensure_topics_access( + &tenant1_client, + TOPICS, + TENANT1_STREAM, + &[TENANT2_STREAM, TENANT3_STREAM], + ) + .await?; + ensure_topics_access( + &tenant2_client, + TOPICS, + TENANT2_STREAM, + &[TENANT1_STREAM, TENANT3_STREAM], + ) + .await?; + ensure_topics_access( + &tenant3_client, + TOPICS, + TENANT3_STREAM, + &[TENANT1_STREAM, TENANT2_STREAM], + ) + .await?; + + print_info("Creating consumer for each tenant"); + let consumers1 = create_consumers( + "tenant_1", + &tenant1_client, + TENANT1_STREAM, + TOPICS, + args.messages_per_batch, + &args.interval, + ) + .await?; + let consumers2 = create_consumers( + "tenant_2", + &tenant2_client, + TENANT2_STREAM, + TOPICS, + args.messages_per_batch, + &args.interval, + ) + .await?; + let consumers3 = create_consumers( + "tenant_3", + &tenant3_client, + TENANT3_STREAM, + TOPICS, + args.messages_per_batch, + &args.interval, + ) + .await?; + + print_info("Starting consumers for each tenant"); + let consumer1_tasks = start_consumers(consumers1); + let consumer2_tasks = start_consumers(consumers2); + let consumer3_tasks = start_consumers(consumers3); + + let mut tasks = Vec::new(); + tasks.extend(consumer1_tasks); + tasks.extend(consumer2_tasks); + tasks.extend(consumer3_tasks); + join_all(tasks).await; + + print_info("Disconnecting clients"); + + Ok(()) +} + +async fn create_user( + stream_name: &str, + topics: &[&str], + username: &str, + client: &IggyClient, +) -> Result<(), IggyError> { + let stream = client.get_stream(&stream_name.try_into()?).await?; + let mut topic_permissions = HashMap::new(); + for topic in topics { + let topic_id = Identifier::named(topic)?; + let topic = client + .get_topic(&stream_name.try_into()?, &topic_id) + .await?; + topic_permissions.insert( + topic.id, + TopicPermissions { + read_topic: true, + poll_messages: true, + ..Default::default() + }, + ); + } + + let mut streams_permissions = HashMap::new(); + streams_permissions.insert( + stream.id, + StreamPermissions { + read_stream: true, + topics: Some(topic_permissions), + ..Default::default() + }, + ); + let permissions = Permissions { + streams: Some(streams_permissions), + ..Default::default() + }; + let user = client + .create_user(username, PASSWORD, UserStatus::Active, Some(permissions)) + .await?; + info!( + "Created user: {username} with ID: {}, with permissions for topics: {:?} in stream: {stream_name}", + user.id, topics + ); + Ok(()) +} + +fn start_consumers(consumers: Vec) -> Vec> { + let mut tasks = Vec::new(); + for mut consumer in consumers { + let task = tokio::spawn(async move { + let tenant = consumer.tenant; + while let Some(message) = consumer.consumer.next().await { + if let Ok(message) = message { + let current_offset = message.current_offset; + let partition_id = message.partition_id; + let offset = message.message.offset; + let payload = std::str::from_utf8(&message.message.payload); + if payload.is_err() { + let error = payload.unwrap_err(); + error!("Error while decoding the message payload at offset: {offset}, partition ID: {partition_id}, perhaps it's encrypted? {error}"); + continue; + } + + let payload = payload.unwrap(); + info!("Tenant: {tenant} consumer received: {payload} from partition: {partition_id}, at offset: {offset}, current offset: {current_offset}"); + } else if let Err(error) = message { + error!("Error while handling message: {error}, by: {tenant} consumer."); + continue; + } + } + }); + tasks.push(task); + } + tasks +} + +async fn create_consumers( + tenant: &str, + client: &IggyClient, + stream: &str, + topics: &[&str], + batch_size: u32, + interval: &str, +) -> Result, IggyError> { + let mut consumers = Vec::new(); + for topic in topics { + let mut consumer = client + .consumer_group(CONSUMER_GROUP, stream, topic)? + .batch_size(batch_size) + .poll_interval(IggyDuration::from_str(interval).expect("Invalid duration")) + .polling_strategy(PollingStrategy::next()) + .auto_join_consumer_group() + .auto_commit(AutoCommit::After(AutoCommitAfter::PollingMessages)) + .build(); + consumer.init().await?; + consumers.push(TenantConsumer { + tenant: tenant.to_owned(), + consumer, + }); + } + Ok(consumers) +} + +async fn ensure_topics_access( + client: &IggyClient, + topics: &[&str], + available_stream: &str, + unavailable_streams: &[&str], +) -> Result<(), IggyError> { + for topic in topics { + let topic_id = Identifier::named(topic)?; + client + .get_topic(&available_stream.try_into()?, &topic_id) + .await + .unwrap_or_else(|_| { + panic!("No access to topic: {topic} in stream: {available_stream}") + }); + info!("Ensured access to topic: {topic} in stream: {available_stream}"); + for stream in unavailable_streams { + if client + .get_topic(&Identifier::named(stream)?, &topic_id) + .await + .is_err() + { + info!("Ensured no access to topic: {topic} in stream: {stream}"); + } else { + panic!("Access to topic: {topic} in stream: {stream} should not be allowed"); + } + } + } + Ok(()) +} + +async fn create_client( + address: &str, + username: &str, + password: &str, +) -> Result { + let connection_string = format!("iggy://{username}:{password}@{address}"); + let client = IggyClient::builder_from_connection_string(&connection_string)?.build()?; + client.connect().await?; + Ok(client) +} + +fn print_info(message: &str) { + info!("\n\n--- {message} ---\n"); +} diff --git a/examples/src/multi-tenant/producer/main.rs b/examples/src/multi-tenant/producer/main.rs new file mode 100644 index 000000000..b7fd2fecb --- /dev/null +++ b/examples/src/multi-tenant/producer/main.rs @@ -0,0 +1,258 @@ +use clap::Parser; +use futures_util::future::join_all; +use iggy::client::{Client, StreamClient, UserClient}; +use iggy::clients::builder::IggyClientBuilder; +use iggy::clients::client::IggyClient; +use iggy::clients::producer::IggyProducer; +use iggy::error::IggyError; +use iggy::identifier::Identifier; +use iggy::messages::send_messages::{Message, Partitioning}; +use iggy::models::permissions::{Permissions, StreamPermissions}; +use iggy::models::user_status::UserStatus; +use iggy::users::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME}; +use iggy::utils::duration::IggyDuration; +use iggy_examples::shared::args::Args; +use std::collections::HashMap; +use std::error::Error; +use std::str::FromStr; +use tokio::task::JoinHandle; +use tracing::{error, info}; + +const TENANT1_STREAM: &str = "tenant_1"; +const TENANT2_STREAM: &str = "tenant_2"; +const TENANT3_STREAM: &str = "tenant_3"; +const TENANT1_USER: &str = "tenant_1_producer"; +const TENANT2_USER: &str = "tenant_2_producer"; +const TENANT3_USER: &str = "tenant_3_producer"; +const PASSWORD: &str = "secret"; +const TOPICS: &[&str] = &["events", "logs", "notifications"]; +const PRODUCERS_COUNT: usize = 3; +const PARTITIONS_COUNT: u32 = 3; + +#[tokio::main] +async fn main() -> anyhow::Result<(), Box> { + let args = Args::parse(); + tracing_subscriber::fmt::init(); + print_info("Multi-tenant producer has started"); + let address = args.tcp_server_address; + + print_info("Creating root client to manage streams and users"); + let root_client = create_client(&address, DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD).await?; + + print_info("Creating streams and users with permissions for each tenant"); + create_stream_and_user(TENANT1_STREAM, TENANT1_USER, &root_client).await?; + create_stream_and_user(TENANT2_STREAM, TENANT2_USER, &root_client).await?; + create_stream_and_user(TENANT3_STREAM, TENANT3_USER, &root_client).await?; + + print_info("Disconnecting root client"); + root_client.disconnect().await?; + + print_info("Creating clients for each tenant"); + let tenant1_client = create_client(&address, TENANT1_USER, PASSWORD).await?; + let tenant2_client = create_client(&address, TENANT2_USER, PASSWORD).await?; + let tenant3_client = create_client(&address, TENANT3_USER, PASSWORD).await?; + + print_info("Ensuring access to streams for each tenant"); + ensure_stream_access( + &tenant1_client, + TENANT1_STREAM, + &[TENANT2_STREAM, TENANT3_STREAM], + ) + .await?; + ensure_stream_access( + &tenant2_client, + TENANT2_STREAM, + &[TENANT1_STREAM, TENANT3_STREAM], + ) + .await?; + ensure_stream_access( + &tenant3_client, + TENANT3_STREAM, + &[TENANT1_STREAM, TENANT2_STREAM], + ) + .await?; + + print_info("Creating {PRODUCERS_COUNT} producers for each tenant"); + let producers1 = create_producers( + &tenant1_client, + TENANT1_STREAM, + TOPICS, + args.messages_per_batch, + &args.interval, + ) + .await?; + let producers2 = create_producers( + &tenant2_client, + TENANT2_STREAM, + TOPICS, + args.messages_per_batch, + &args.interval, + ) + .await?; + let producers3 = create_producers( + &tenant3_client, + TENANT3_STREAM, + TOPICS, + args.messages_per_batch, + &args.interval, + ) + .await?; + + print_info("Starting producers for each tenant"); + let producers1_tasks = start_producers(producers1, args.message_batches_limit); + let producers2_tasks = start_producers(producers2, args.message_batches_limit); + let producers3_tasks = start_producers(producers3, args.message_batches_limit); + + let mut tasks = Vec::new(); + tasks.extend(producers1_tasks); + tasks.extend(producers2_tasks); + tasks.extend(producers3_tasks); + + join_all(tasks).await; + + print_info("Disconnecting clients"); + + Ok(()) +} + +fn start_producers(producers: Vec, batches_count: u64) -> Vec> { + let mut tasks = Vec::new(); + let mut producer_id = 1; + let topics_count = TOPICS.len() as u64; + for producer in producers { + if producer_id > topics_count { + producer_id = 1; + } + + let task = tokio::spawn(async move { + let mut counter = 1; + while counter <= topics_count * batches_count { + let message = match producer + .topic() + .get_string_value() + .expect("Invalid topic") + .as_str() + { + "events" => "event", + "logs" => "log", + "notifications" => "notification", + _ => panic!("Invalid topic"), + }; + let payload = format!("{message}-{producer_id}-{counter}"); + let message = Message::from_str(&payload).expect("Invalid message"); + if let Err(error) = producer.send(vec![message]).await { + error!( + "Failed to send: '{payload}' to: {} -> {} with error: {error}", + producer.stream(), + producer.topic(), + error = error + ); + continue; + } + + counter += 1; + info!( + "Sent: '{payload}' to: {} -> {}", + producer.stream(), + producer.topic() + ); + } + }); + producer_id += 1; + tasks.push(task); + } + tasks +} + +async fn create_producers( + client: &IggyClient, + stream: &str, + topics: &[&str], + batch_size: u32, + interval: &str, +) -> Result, IggyError> { + let mut producers = Vec::new(); + for topic in topics { + for _ in 0..PRODUCERS_COUNT { + let mut producer = client + .producer(stream, topic)? + .batch_size(batch_size) + .send_interval(IggyDuration::from_str(interval).expect("Invalid duration")) + .partitioning(Partitioning::balanced()) + .create_topic_if_not_exists(PARTITIONS_COUNT, None) + .build(); + producer.init().await?; + producers.push(producer); + } + } + Ok(producers) +} + +async fn ensure_stream_access( + client: &IggyClient, + available_stream: &str, + unavailable_streams: &[&str], +) -> Result<(), IggyError> { + client + .get_stream(&available_stream.try_into()?) + .await + .unwrap_or_else(|_| panic!("No access to stream: {available_stream}")); + info!("Ensured access to stream: {available_stream}"); + for stream in unavailable_streams { + if client + .get_stream(&Identifier::named(stream)?) + .await + .is_err() + { + info!("Ensured no access to stream: {stream}"); + } else { + panic!("Access to stream: {stream} should not be allowed"); + } + } + Ok(()) +} + +async fn create_client( + address: &str, + username: &str, + password: &str, +) -> Result { + let connection_string = format!("iggy://{username}:{password}@{address}"); + let client = IggyClientBuilder::from_connection_string(&connection_string)?.build()?; + client.connect().await?; + Ok(client) +} + +async fn create_stream_and_user( + stream_name: &str, + username: &str, + client: &IggyClient, +) -> Result<(), IggyError> { + let stream = client.create_stream(stream_name, None).await?; + info!("Created stream: {stream_name} with ID: {}", stream.id); + let mut streams_permissions = HashMap::new(); + streams_permissions.insert( + stream.id, + StreamPermissions { + read_stream: true, + manage_topics: true, + ..Default::default() + }, + ); + let permissions = Permissions { + streams: Some(streams_permissions), + ..Default::default() + }; + let user = client + .create_user(username, PASSWORD, UserStatus::Active, Some(permissions)) + .await?; + info!( + "Created user: {username} with ID: {}, with permissions for stream: {stream_name}", + user.id + ); + Ok(()) +} + +fn print_info(message: &str) { + info!("\n\n--- {message} ---\n"); +} diff --git a/examples/src/new-sdk/consumer/main.rs b/examples/src/new-sdk/consumer/main.rs new file mode 100644 index 000000000..d36eb2546 --- /dev/null +++ b/examples/src/new-sdk/consumer/main.rs @@ -0,0 +1,110 @@ +use clap::Parser; +use futures_util::StreamExt; +use iggy::client_provider; +use iggy::client_provider::ClientProviderConfig; +use iggy::clients::client::IggyClient; +use iggy::clients::consumer::{AutoCommit, AutoCommitAfter, IggyConsumer}; +use iggy::consumer::ConsumerKind; +use iggy::messages::poll_messages::PollingStrategy; +use iggy::models::messages::PolledMessage; +use iggy::utils::duration::IggyDuration; +use iggy_examples::shared::args::Args; +use iggy_examples::shared::messages::{ + Envelope, OrderConfirmed, OrderCreated, OrderRejected, ORDER_CONFIRMED_TYPE, + ORDER_CREATED_TYPE, ORDER_REJECTED_TYPE, +}; +use std::error::Error; +use std::str::FromStr; +use std::sync::Arc; +use tracing::{error, info, warn}; + +#[tokio::main] +async fn main() -> anyhow::Result<(), Box> { + let args = Args::parse(); + tracing_subscriber::fmt::init(); + info!( + "New SDK consumer has started, selected transport: {}", + args.transport + ); + let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); + let client = client_provider::get_raw_connected_client(client_provider_config).await?; + let client = IggyClient::new(client); + + let name = "new-sdk-consumer"; + let mut consumer = match ConsumerKind::from_code(args.consumer_kind)? { + ConsumerKind::Consumer => { + client.consumer(name, &args.stream_id, &args.topic_id, args.partition_id)? + } + ConsumerKind::ConsumerGroup => { + client.consumer_group(name, &args.stream_id, &args.topic_id)? + } + } + .auto_commit(AutoCommit::After(AutoCommitAfter::PollingMessages)) + .create_consumer_group_if_not_exists() + .auto_join_consumer_group() + .polling_strategy(PollingStrategy::next()) + .poll_interval(IggyDuration::from_str(&args.interval)?) + .batch_size(args.messages_per_batch) + .build(); + + consumer.init().await?; + consume_messages(&args, &mut consumer).await?; + + Ok(()) +} + +pub async fn consume_messages( + args: &Args, + consumer: &mut IggyConsumer, +) -> Result<(), Box> { + let interval = args.get_interval(); + let mut consumed_batches = 0; + + info!("Messages will be polled by consumer: {} from stream: {}, topic: {}, partition: {} with interval {}.", + args.consumer_id, args.stream_id, args.topic_id, args.partition_id, interval.map_or("none".to_string(), |i| i.as_human_time_string())); + + while let Some(message) = consumer.next().await { + if args.message_batches_limit > 0 && consumed_batches == args.message_batches_limit { + info!("Consumed {consumed_batches} batches of messages, exiting."); + return Ok(()); + } + + if let Ok(message) = message { + handle_message(&message.message)?; + consumed_batches += 1; + } else if let Err(error) = message { + error!("Error while handling message: {error}"); + continue; + } + } + Ok(()) +} + +fn handle_message(message: &PolledMessage) -> anyhow::Result<(), Box> { + // The payload can be of any type as it is a raw byte array. In this case it's a JSON string. + let json = std::str::from_utf8(&message.payload)?; + // The message envelope can be used to send the different types of messages to the same topic. + let envelope = serde_json::from_str::(json)?; + info!( + "Handling message type: {} at offset: {}...", + envelope.message_type, message.offset + ); + match envelope.message_type.as_str() { + ORDER_CREATED_TYPE => { + let order_created = serde_json::from_str::(&envelope.payload)?; + info!("{:#?}", order_created); + } + ORDER_CONFIRMED_TYPE => { + let order_confirmed = serde_json::from_str::(&envelope.payload)?; + info!("{:#?}", order_confirmed); + } + ORDER_REJECTED_TYPE => { + let order_rejected = serde_json::from_str::(&envelope.payload)?; + info!("{:#?}", order_rejected); + } + _ => { + warn!("Received unknown message type: {}", envelope.message_type); + } + } + Ok(()) +} diff --git a/examples/src/new-sdk/producer/main.rs b/examples/src/new-sdk/producer/main.rs new file mode 100644 index 000000000..82468c182 --- /dev/null +++ b/examples/src/new-sdk/producer/main.rs @@ -0,0 +1,73 @@ +use clap::Parser; +use iggy::client_provider; +use iggy::client_provider::ClientProviderConfig; +use iggy::clients::client::IggyClient; +use iggy::clients::producer::IggyProducer; +use iggy::messages::send_messages::{Message, Partitioning}; +use iggy::utils::duration::IggyDuration; +use iggy_examples::shared::args::Args; +use iggy_examples::shared::messages_generator::MessagesGenerator; +use std::error::Error; +use std::str::FromStr; +use std::sync::Arc; +use tracing::info; + +#[tokio::main] +async fn main() -> anyhow::Result<(), Box> { + let args = Args::parse(); + tracing_subscriber::fmt::init(); + info!( + "New SDK producer has started, selected transport: {}", + args.transport + ); + let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); + let client = client_provider::get_raw_connected_client(client_provider_config).await?; + let client = IggyClient::builder().with_client(client).build()?; + let mut producer = client + .producer(&args.stream_id, &args.topic_id)? + .batch_size(args.messages_per_batch) + .send_interval(IggyDuration::from_str(&args.interval)?) + .partitioning(Partitioning::balanced()) + .create_topic_if_not_exists(3, None) + .build(); + producer.init().await?; + produce_messages(&args, &producer).await?; + Ok(()) +} + +async fn produce_messages( + args: &Args, + producer: &IggyProducer, +) -> anyhow::Result<(), Box> { + let interval = args.get_interval(); + info!( + "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {}.", + args.stream_id, + args.topic_id, + args.partition_id, + interval.map_or("none".to_string(), |i| i.as_human_time_string()) + ); + let mut message_generator = MessagesGenerator::new(); + let mut sent_batches = 0; + + loop { + if args.message_batches_limit > 0 && sent_batches == args.message_batches_limit { + info!("Sent {sent_batches} batches of messages, exiting."); + return Ok(()); + } + + let mut messages = Vec::new(); + for _ in 0..args.messages_per_batch { + let serializable_message = message_generator.generate(); + let json_envelope = serializable_message.to_json_envelope(); + let message = Message::from_str(&json_envelope)?; + messages.push(message); + } + producer.send(messages).await?; + sent_batches += 1; + info!( + "Sent batch {sent_batches} of {} messages.", + args.messages_per_batch + ); + } +} diff --git a/examples/src/shared/args.rs b/examples/src/shared/args.rs index 626cb11dc..9fef274f6 100644 --- a/examples/src/shared/args.rs +++ b/examples/src/shared/args.rs @@ -1,25 +1,28 @@ use clap::Parser; +use iggy::users::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME}; +use iggy::utils::duration::IggyDuration; +use std::str::FromStr; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] pub struct Args { - #[arg(long, default_value = "0")] + #[arg(long, default_value = "10")] pub message_batches_limit: u64, - #[arg(long, default_value = "iggy")] + #[arg(long, default_value = DEFAULT_ROOT_USERNAME)] pub username: String, - #[arg(long, default_value = "iggy")] + #[arg(long, default_value = DEFAULT_ROOT_PASSWORD)] pub password: String, - #[arg(long, default_value = "1000")] - pub interval: u64, + #[arg(long, default_value = "1ms")] + pub interval: String, - #[arg(long, default_value = "9999")] - pub stream_id: u32, + #[arg(long, default_value = "example-stream")] + pub stream_id: String, - #[arg(long, default_value = "1")] - pub topic_id: u32, + #[arg(long, default_value = "example-topic")] + pub topic_id: String, #[arg(long, default_value = "1")] pub partition_id: u32, @@ -39,6 +42,12 @@ pub struct Args { #[arg(long, default_value = "1")] pub messages_per_batch: u32, + #[arg(long, default_value = "0")] + pub offset: u64, + + #[arg(long, default_value = "false")] + pub auto_commit: bool, + #[arg(long, default_value = "tcp")] pub transport: String, @@ -51,11 +60,17 @@ pub struct Args { #[arg(long, default_value = "3")] pub http_retries: u32, - #[arg(long, default_value = "3")] - pub tcp_reconnection_retries: u32, + #[arg(long, default_value = "true")] + pub tcp_reconnection_enabled: bool, + + #[arg(long)] + pub tcp_reconnection_max_retries: Option, - #[arg(long, default_value = "1000")] - pub tcp_reconnection_interval: u64, + #[arg(long, default_value = "1s")] + pub tcp_reconnection_interval: String, + + #[arg(long, default_value = "5s")] + pub tcp_reconnection_re_establish_after: String, #[arg(long, default_value = "127.0.0.1:8090")] pub tcp_server_address: String, @@ -75,11 +90,17 @@ pub struct Args { #[arg(long, default_value = "localhost")] pub quic_server_name: String, - #[arg(long, default_value = "3")] - pub quic_reconnection_retries: u32, + #[arg(long, default_value = "true")] + pub quic_reconnection_enabled: bool, + + #[arg(long)] + pub quic_reconnection_max_retries: Option, + + #[arg(long, default_value = "1s")] + pub quic_reconnection_interval: String, - #[arg(long, default_value = "1000")] - pub quic_reconnection_interval: u64, + #[arg(long, default_value = "5s")] + pub quic_reconnection_re_establish_after: String, #[arg(long, default_value = "10000")] pub quic_max_concurrent_bidi_streams: u64, @@ -116,16 +137,22 @@ impl Args { encryption_key: self.encryption_key.clone(), http_api_url: self.http_api_url.clone(), http_retries: self.http_retries, + username: self.username.clone(), + password: self.password.clone(), tcp_server_address: self.tcp_server_address.clone(), - tcp_reconnection_retries: self.tcp_reconnection_retries, - tcp_reconnection_interval: self.tcp_reconnection_interval, + tcp_reconnection_enabled: self.tcp_reconnection_enabled, + tcp_reconnection_max_retries: self.tcp_reconnection_max_retries, + tcp_reconnection_interval: self.tcp_reconnection_interval.clone(), + tcp_reconnection_re_establish_after: self.tcp_reconnection_re_establish_after.clone(), tcp_tls_enabled: self.tcp_tls_enabled, tcp_tls_domain: self.tcp_tls_domain.clone(), quic_client_address: self.quic_client_address.clone(), quic_server_address: self.quic_server_address.clone(), quic_server_name: self.quic_server_name.clone(), - quic_reconnection_retries: self.quic_reconnection_retries, - quic_reconnection_interval: self.quic_reconnection_interval, + quic_reconnection_enabled: self.quic_reconnection_enabled, + quic_reconnection_max_retries: self.quic_reconnection_max_retries, + quic_reconnection_re_establish_after: self.quic_reconnection_re_establish_after.clone(), + quic_reconnection_interval: self.quic_reconnection_interval.clone(), quic_max_concurrent_bidi_streams: self.quic_max_concurrent_bidi_streams, quic_datagram_send_buffer_size: self.quic_datagram_send_buffer_size, quic_initial_mtu: self.quic_initial_mtu, @@ -137,4 +164,11 @@ impl Args { quic_validate_certificate: self.quic_validate_certificate, } } + + pub fn get_interval(&self) -> Option { + match self.interval.to_lowercase().as_str() { + "" | "0" | "none" => None, + x => Some(IggyDuration::from_str(x).expect("Invalid interval format")), + } + } } diff --git a/examples/src/shared/system.rs b/examples/src/shared/system.rs index e29c379bf..eaa3386e1 100644 --- a/examples/src/shared/system.rs +++ b/examples/src/shared/system.rs @@ -6,42 +6,35 @@ use iggy::error::IggyError; use iggy::identifier::Identifier; use iggy::messages::poll_messages::PollingStrategy; use iggy::models::messages::PolledMessage; -use iggy::users::defaults::*; use iggy::utils::expiry::IggyExpiry; use iggy::utils::topic_size::MaxTopicSize; use tracing::info; type MessageHandler = dyn Fn(&PolledMessage) -> Result<(), Box>; -pub async fn login_root(client: &dyn Client) { - client - .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) - .await - .unwrap(); -} - pub async fn init_by_consumer(args: &Args, client: &dyn Client) { - let (stream_id, topic_id, partition_id) = (args.stream_id, args.topic_id, args.partition_id); + let (stream_id, topic_id, partition_id) = ( + args.stream_id.clone(), + args.topic_id.clone(), + args.partition_id, + ); let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + let stream_id = stream_id.try_into().unwrap(); + let topic_id = topic_id.try_into().unwrap(); loop { - info!("Validating if stream: {} exists..", stream_id); - let stream = client.get_stream(&args.stream_id.try_into().unwrap()).await; + interval.tick().await; + info!("Validating if stream: {stream_id} exists.."); + let stream = client.get_stream(&stream_id).await; if stream.is_ok() { - info!("Stream: {} was found.", stream_id); + info!("Stream: {stream_id} was found."); break; } - interval.tick().await; } loop { + interval.tick().await; info!("Validating if topic: {} exists..", topic_id); - let topic = client - .get_topic( - &stream_id.try_into().unwrap(), - &topic_id.try_into().unwrap(), - ) - .await; + let topic = client.get_topic(&stream_id, &topic_id).await; if topic.is_err() { - interval.tick().await; continue; } @@ -59,21 +52,23 @@ pub async fn init_by_consumer(args: &Args, client: &dyn Client) { } pub async fn init_by_producer(args: &Args, client: &dyn Client) -> Result<(), IggyError> { - let stream = client.get_stream(&args.stream_id.try_into()?).await; + let stream_id = args.stream_id.clone().try_into()?; + let topic_name = args.topic_id.clone(); + let stream = client.get_stream(&stream_id).await; if stream.is_ok() { return Ok(()); } info!("Stream does not exist, creating..."); - client.create_stream("sample", Some(args.stream_id)).await?; + client.create_stream(&args.stream_id, None).await?; client .create_topic( - &args.stream_id.try_into()?, - "orders", + &stream_id, + &topic_name, args.partitions_count, CompressionAlgorithm::from_code(args.compression_algorithm)?, None, - Some(args.topic_id), + None, IggyExpiry::NeverExpire, MaxTopicSize::ServerDefault, ) @@ -86,25 +81,33 @@ pub async fn consume_messages( client: &dyn Client, handle_message: &MessageHandler, ) -> Result<(), Box> { - info!("Messages will be polled by consumer: {} from stream: {}, topic: {}, partition: {} with interval {} ms.", - args.consumer_id, args.stream_id, args.topic_id, args.partition_id, args.interval); + let interval = args.get_interval(); + info!("Messages will be polled by consumer: {} from stream: {}, topic: {}, partition: {} with interval {}.", + args.consumer_id, args.stream_id, args.topic_id, args.partition_id, interval.map_or("none".to_string(), |i| i.as_human_time_string())); - let mut interval = tokio::time::interval(std::time::Duration::from_millis(args.interval)); + let stream_id = args.stream_id.clone().try_into()?; + let topic_id = args.topic_id.clone().try_into()?; + let mut interval = interval.map(|interval| tokio::time::interval(interval.get_duration())); let mut consumed_batches = 0; let consumer = Consumer { kind: ConsumerKind::from_code(args.consumer_kind)?, id: Identifier::numeric(args.consumer_id).unwrap(), }; + loop { if args.message_batches_limit > 0 && consumed_batches == args.message_batches_limit { info!("Consumed {consumed_batches} batches of messages, exiting."); return Ok(()); } + if let Some(interval) = &mut interval { + interval.tick().await; + } + let polled_messages = client .poll_messages( - &args.stream_id.try_into()?, - &args.topic_id.try_into()?, + &stream_id, + &topic_id, Some(args.partition_id), &consumer, &PollingStrategy::next(), @@ -114,13 +117,11 @@ pub async fn consume_messages( .await?; if polled_messages.messages.is_empty() { info!("No messages found."); - interval.tick().await; continue; } consumed_batches += 1; for message in polled_messages.messages { handle_message(&message)?; } - interval.tick().await; } } diff --git a/integration/tests/cli/common/mod.rs b/integration/tests/cli/common/mod.rs index 2593cd8bf..91b80e900 100644 --- a/integration/tests/cli/common/mod.rs +++ b/integration/tests/cli/common/mod.rs @@ -6,7 +6,7 @@ use assert_cmd::assert::{Assert, OutputAssertExt}; use assert_cmd::prelude::CommandCargoExt; use async_trait::async_trait; use iggy::client::{Client, SystemClient, UserClient}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::tcp::client::TcpClient; use iggy::tcp::config::TcpClientConfig; use iggy::users::defaults::*; @@ -90,13 +90,7 @@ impl IggyCmdTest { ..TcpClientConfig::default() }; let client = Box::new(TcpClient::create(Arc::new(tcp_client_config)).unwrap()); - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); Self { server, client } } diff --git a/integration/tests/cli/general/test_help_command.rs b/integration/tests/cli/general/test_help_command.rs index dba3027e8..6d15e8137 100644 --- a/integration/tests/cli/general/test_help_command.rs +++ b/integration/tests/cli/general/test_help_command.rs @@ -43,6 +43,16 @@ Options: {CLAP_INDENT} [default: ] + --credentials-username + Optional username for initial login +{CLAP_INDENT} + [default: DEFAULT_ROOT_USERNAME] + + --credentials-password + Optional password for initial login +{CLAP_INDENT} + [default: DEFAULT_ROOT_PASSWORD] + --http-api-url The optional API URL for the HTTP transport {CLAP_INDENT} @@ -58,15 +68,15 @@ Options: {CLAP_INDENT} [default: 127.0.0.1:8090] - --tcp-reconnection-retries - The optional number of reconnect retries for the TCP transport + --tcp-reconnection-max-retries + The optional number of max reconnect retries for the TCP transport {CLAP_INDENT} [default: 3] --tcp-reconnection-interval The optional reconnect interval for the TCP transport {CLAP_INDENT} - [default: 1000] + [default: "1s"] --tcp-tls-enabled Flag to enable TLS for the TCP transport @@ -91,15 +101,15 @@ Options: {CLAP_INDENT} [default: localhost] - --quic-reconnection-retries - The optional number of reconnect retries for the QUIC transport + --quic-reconnection-max-retries + The optional number of max reconnect retries for the QUIC transport {CLAP_INDENT} [default: 3] --quic-reconnection-interval The optional reconnect interval for the QUIC transport {CLAP_INDENT} - [default: 1000] + [default: "1s"] --quic-max-concurrent-bidi-streams The optional maximum number of concurrent bidirectional streams for QUIC diff --git a/integration/tests/data_integrity/verify_after_server_restart.rs b/integration/tests/data_integrity/verify_after_server_restart.rs index 244d98ece..5179a2150 100644 --- a/integration/tests/data_integrity/verify_after_server_restart.rs +++ b/integration/tests/data_integrity/verify_after_server_restart.rs @@ -1,7 +1,6 @@ use crate::bench::run_bench_and_wait_for_finish; use iggy::client::SystemClient; use iggy::clients::client::IggyClient; -use iggy::clients::client::IggyClientBackgroundConfig; use iggy::utils::byte_size::IggyByteSize; use integration::{ tcp_client::TcpClientFactory, @@ -40,13 +39,7 @@ async fn should_fill_data_and_verify_after_restart() { // 4. Connect and login to newly started server let client = TcpClientFactory { server_addr }.create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; // 5. Save stats from the first server @@ -78,8 +71,6 @@ async fn should_fill_data_and_verify_after_restart() { } .create_client() .await, - IggyClientBackgroundConfig::default(), - None, None, None, ); diff --git a/integration/tests/examples/mod.rs b/integration/tests/examples/mod.rs index 7df2199d5..cd29ad148 100644 --- a/integration/tests/examples/mod.rs +++ b/integration/tests/examples/mod.rs @@ -5,7 +5,7 @@ mod test_message_headers; use assert_cmd::Command; use iggy::client::{Client, StreamClient, SystemClient, TopicClient, UserClient}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::compression::compression_algorithm::CompressionAlgorithm; use iggy::tcp::client::TcpClient; use iggy::tcp::config::TcpClientConfig; @@ -90,13 +90,7 @@ impl<'a> IggyExampleTest<'a> { ..TcpClientConfig::default() }; let client = Box::new(TcpClient::create(Arc::new(tcp_client_config)).unwrap()); - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); Self { server, diff --git a/integration/tests/examples/test_basic.rs b/integration/tests/examples/test_basic.rs index 067701866..b32cbb8b0 100644 --- a/integration/tests/examples/test_basic.rs +++ b/integration/tests/examples/test_basic.rs @@ -42,17 +42,17 @@ async fn should_successfully_execute() { .execute_test(TestBasic { expected_producer_output: vec![ "Basic producer has started, selected transport: tcp", - "Received an invalid response with status: 1009 (stream_id_not_found).", + "Received an invalid response with status: 1010 (stream_name_not_found).", "Stream does not exist, creating...", - "Messages will be sent to stream: 9999, topic: 1, partition: 1 with interval 1000 ms.", + "Messages will be sent to stream: example-stream, topic: example-topic, partition: 1 with interval 1ms.", ], expected_consumer_output: vec![ "Basic consumer has started, selected transport: tcp", - "Validating if stream: 9999 exists..", - "Stream: 9999 was found.", - "Validating if topic: 1 exists..", - "Topic: 1 was found.", - "Messages will be polled by consumer: 1 from stream: 9999, topic: 1, partition: 1 with interval 1000 ms." + "Validating if stream: example-stream exists..", + "Stream: example-stream was found.", + "Validating if topic: example-topic exists..", + "Topic: example-topic was found.", + "Messages will be polled by consumer: 1 from stream: example-stream, topic: example-topic, partition: 1 with interval 1ms." ], }) .await; diff --git a/integration/tests/examples/test_getting_started.rs b/integration/tests/examples/test_getting_started.rs index 28b30c5d1..156155234 100644 --- a/integration/tests/examples/test_getting_started.rs +++ b/integration/tests/examples/test_getting_started.rs @@ -3,7 +3,7 @@ use crate::examples::{IggyExampleTest, IggyExampleTestCase}; use serial_test::parallel; static EXPECTED_CONSUMER_OUTPUT: [&str; 6] = [ - "Messages will be consumed from stream: 1, topic: 1, partition: 1 with interval 500 ms.", + "Messages will be consumed from stream: 1, topic: 1, partition: 1 with interval 500ms.", "Handling message at offset: 0, payload: message-1...", "Handling message at offset: 1, payload: message-2...", "Handling message at offset: 2, payload: message-3...", @@ -40,7 +40,7 @@ async fn should_succeed_with_no_existing_stream_or_topic() { expected_producer_output: vec![ "Stream was created.", "Topic was created.", - "Messages will be sent to stream: 1, topic: 1, partition: 1 with interval 500 ms.", + "Messages will be sent to stream: 1, topic: 1, partition: 1 with interval 500ms.", "Sent 10 message(s).", ], expected_consumer_output: EXPECTED_CONSUMER_OUTPUT.to_vec(), diff --git a/integration/tests/examples/test_message_envelope.rs b/integration/tests/examples/test_message_envelope.rs index 7783edb1a..ddc23c80e 100644 --- a/integration/tests/examples/test_message_envelope.rs +++ b/integration/tests/examples/test_message_envelope.rs @@ -40,17 +40,17 @@ async fn should_successfully_execute() { .execute_test(TestMessageEnvelope { expected_producer_output: vec![ "Message envelope producer has started, selected transport: tcp", - "Received an invalid response with status: 1009 (stream_id_not_found).", + "Received an invalid response with status: 1010 (stream_name_not_found).", "Stream does not exist, creating...", - "Messages will be sent to stream: 9999, topic: 1, partition: 1 with interval 1000 ms.", + "Messages will be sent to stream: example-stream, topic: example-topic, partition: 1 with interval 1ms.", ], expected_consumer_output: vec![ "Message envelope consumer has started, selected transport: tcp", - "Validating if stream: 9999 exists..", - "Stream: 9999 was found.", - "Validating if topic: 1 exists..", - "Topic: 1 was found.", - "Messages will be polled by consumer: 1 from stream: 9999, topic: 1, partition: 1 with interval 1000 ms.", + "Validating if stream: example-stream exists..", + "Stream: example-stream was found.", + "Validating if topic: example-topic exists..", + "Topic: example-topic was found.", + "Messages will be polled by consumer: 1 from stream: example-stream, topic: example-topic, partition: 1 with interval 1ms." ], }) .await; diff --git a/integration/tests/examples/test_message_headers.rs b/integration/tests/examples/test_message_headers.rs index 581f2ff1f..287d74a09 100644 --- a/integration/tests/examples/test_message_headers.rs +++ b/integration/tests/examples/test_message_headers.rs @@ -2,12 +2,12 @@ use super::{parse_sent_message, verify_stdout_contains_expected_logs}; use crate::examples::{IggyExampleTest, IggyExampleTestCase}; use serial_test::parallel; -struct TestMessageMeaders<'a> { +struct TestMessageHeaders<'a> { expected_producer_output: Vec<&'a str>, expected_consumer_output: Vec<&'a str>, } -impl<'a> IggyExampleTestCase for TestMessageMeaders<'a> { +impl<'a> IggyExampleTestCase for TestMessageHeaders<'a> { fn verify_log_output(&self, producer_stdout: &str, consumer_stdout: &str) { verify_stdout_contains_expected_logs( producer_stdout, @@ -38,20 +38,20 @@ async fn should_successfully_execute() { iggy_example_test.setup(false).await; iggy_example_test - .execute_test(TestMessageMeaders { + .execute_test(TestMessageHeaders { expected_producer_output: vec![ "Message headers producer has started, selected transport: tcp", - "Received an invalid response with status: 1009 (stream_id_not_found).", + "Received an invalid response with status: 1010 (stream_name_not_found).", "Stream does not exist, creating...", - "Messages will be sent to stream: 9999, topic: 1, partition: 1 with interval 1000 ms." + "Messages will be sent to stream: example-stream, topic: example-topic, partition: 1 with interval 1ms.", ], expected_consumer_output: vec![ "Message headers consumer has started, selected transport: tcp", - "Validating if stream: 9999 exists..", - "Stream: 9999 was found.", - "Validating if topic: 1 exists..", - "Topic: 1 was found.", - "Messages will be polled by consumer: 1 from stream: 9999, topic: 1, partition: 1 with interval 1000 ms.", + "Validating if stream: example-stream exists..", + "Stream: example-stream was found.", + "Validating if topic: example-topic exists..", + "Topic: example-topic was found.", + "Messages will be polled by consumer: 1 from stream: example-stream, topic: example-topic, partition: 1 with interval 1ms." ] }) .await; diff --git a/integration/tests/server/scenarios/create_message_payload.rs b/integration/tests/server/scenarios/create_message_payload.rs index f592df125..bce6c92cd 100644 --- a/integration/tests/server/scenarios/create_message_payload.rs +++ b/integration/tests/server/scenarios/create_message_payload.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use iggy::client::{MessageClient, StreamClient, TopicClient}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::consumer::Consumer; use iggy::messages::poll_messages::PollingStrategy; use iggy::messages::send_messages::{Message, Partitioning}; @@ -21,13 +21,7 @@ const PARTITION_ID: u32 = 1; pub async fn run(client_factory: &dyn ClientFactory) { let client = client_factory.create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; init_system(&client).await; diff --git a/integration/tests/server/scenarios/message_size_scenario.rs b/integration/tests/server/scenarios/message_size_scenario.rs index 2b76dad9c..0b885ad4e 100644 --- a/integration/tests/server/scenarios/message_size_scenario.rs +++ b/integration/tests/server/scenarios/message_size_scenario.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use iggy::client::{MessageClient, StreamClient, TopicClient}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::consumer::Consumer; use iggy::error::IggyError; use iggy::error::IggyError::InvalidResponse; @@ -29,13 +29,7 @@ enum MessageToSend { pub async fn run(client_factory: &dyn ClientFactory) { let client = client_factory.create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); login_root(&client).await; init_system(&client).await; diff --git a/integration/tests/server/scenarios/mod.rs b/integration/tests/server/scenarios/mod.rs index 76a4c30c0..70a94af72 100644 --- a/integration/tests/server/scenarios/mod.rs +++ b/integration/tests/server/scenarios/mod.rs @@ -1,5 +1,5 @@ use iggy::client::{ConsumerGroupClient, StreamClient}; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::consumer::ConsumerKind; use iggy::identifier::Identifier; use iggy::models::consumer_group::ConsumerGroupDetails; @@ -32,13 +32,7 @@ const MESSAGES_COUNT: u32 = 1000; async fn create_client(client_factory: &dyn ClientFactory) -> IggyClient { let client = client_factory.create_client().await; - IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ) + IggyClient::create(client, None, None) } async fn get_consumer_group(client: &IggyClient) -> ConsumerGroupDetails { diff --git a/integration/tests/server/scenarios/system_scenario.rs b/integration/tests/server/scenarios/system_scenario.rs index 18ae0fc81..1526ce2ed 100644 --- a/integration/tests/server/scenarios/system_scenario.rs +++ b/integration/tests/server/scenarios/system_scenario.rs @@ -10,7 +10,7 @@ use iggy::client::{ ConsumerGroupClient, ConsumerOffsetClient, MessageClient, PartitionClient, StreamClient, SystemClient, TopicClient, UserClient, }; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::compression::compression_algorithm::CompressionAlgorithm; use iggy::consumer::Consumer; use iggy::error::IggyError; @@ -26,13 +26,7 @@ use integration::test_server::{assert_clean_system, ClientFactory}; pub async fn run(client_factory: &dyn ClientFactory) { let client = client_factory.create_client().await; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - None, - ); + let client = IggyClient::create(client, None, None); let consumer = Consumer { kind: CONSUMER_KIND, @@ -53,11 +47,14 @@ pub async fn run(client_factory: &dyn ClientFactory) { assert!(streams.is_empty()); // 3. Create the stream - client + let stream = client .create_stream(STREAM_NAME, Some(STREAM_ID)) .await .unwrap(); + assert_eq!(stream.id, STREAM_ID); + assert_eq!(stream.name, STREAM_NAME); + // 4. Get streams and validate that created stream exists let streams = client.get_streams().await.unwrap(); assert_eq!(streams.len(), 1); @@ -99,7 +96,7 @@ pub async fn run(client_factory: &dyn ClientFactory) { assert!(create_stream_result.is_err()); // 9. Create the topic - client + let topic = client .create_topic( &Identifier::numeric(STREAM_ID).unwrap(), TOPIC_NAME, @@ -113,6 +110,9 @@ pub async fn run(client_factory: &dyn ClientFactory) { .await .unwrap(); + assert_eq!(topic.id, TOPIC_ID); + assert_eq!(topic.name, TOPIC_NAME); + // 10. Get topics and validate that created topic exists let topics = client .get_topics(&Identifier::numeric(STREAM_ID).unwrap()) @@ -396,7 +396,7 @@ pub async fn run(client_factory: &dyn ClientFactory) { assert!(consumer_groups.is_empty()); // 28. Create the consumer group - client + let consumer_group = client .create_consumer_group( &Identifier::numeric(STREAM_ID).unwrap(), &Identifier::numeric(TOPIC_ID).unwrap(), @@ -406,6 +406,9 @@ pub async fn run(client_factory: &dyn ClientFactory) { .await .unwrap(); + assert_eq!(consumer_group.id, CONSUMER_GROUP_ID); + assert_eq!(consumer_group.name, CONSUMER_GROUP_NAME); + // 29. Get the consumer groups and validate that there is one group let consumer_groups = client .get_consumer_groups( diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index 6f29a903c..2d7e29513 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iggy" -version = "0.5.1" +version = "0.6.0" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2021" license = "MIT" @@ -14,6 +14,7 @@ readme = "../README.md" [dependencies] aes-gcm = "0.10.3" anyhow = "1.0.86" +async-broadcast = { version = "0.7.1" } async-dropper = { version = "0.3.1", features = ["tokio", "simple"] } async-trait = "0.1.80" base64 = "0.22.1" @@ -27,9 +28,12 @@ chrono = { version = "0.4.38" } clap = { version = "4.5.4", features = ["derive"] } comfy-table = { version = "7.1.1", optional = true } crc32fast = "1.4.2" +derive_more = "0.99.18" dirs = "5.0.1" fast-async-mutex = { version = "0.6.7", optional = true } flume = "0.11.0" +futures = "0.3.30" +futures-util = "0.3.30" humantime = "2.1.0" keyring = { version = "2.3.3", optional = true } lazy_static = "1.4.0" diff --git a/sdk/src/args.rs b/sdk/src/args.rs index 5ca7bd229..228d7b845 100644 --- a/sdk/src/args.rs +++ b/sdk/src/args.rs @@ -1,3 +1,4 @@ +use crate::users::defaults::{DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME}; use clap::Parser; use serde::{Deserialize, Serialize}; @@ -23,6 +24,20 @@ pub struct ArgsOptional { #[serde(skip_serializing_if = "Option::is_none")] pub encryption_key: Option, + /// Optional username for initial login + /// + /// [default: DEFAULT_ROOT_USERNAME] + #[arg(long)] + #[serde(skip_serializing_if = "Option::is_none")] + pub credentials_username: Option, + + /// Optional password for initial login + /// + /// [default: DEFAULT_ROOT_PASSWORD] + #[arg(long)] + #[serde(skip_serializing_if = "Option::is_none")] + pub credentials_password: Option, + /// The optional API URL for the HTTP transport /// /// [default: http://localhost:3000] @@ -44,19 +59,19 @@ pub struct ArgsOptional { #[serde(skip_serializing_if = "Option::is_none")] pub tcp_server_address: Option, - /// The optional number of reconnect retries for the TCP transport + /// The optional number of max reconnect retries for the TCP transport /// /// [default: 3] #[arg(long)] #[serde(skip_serializing_if = "Option::is_none")] - pub tcp_reconnection_retries: Option, + pub tcp_reconnection_max_retries: Option, /// The optional reconnect interval for the TCP transport /// - /// [default: 1000] + /// [default: "1s"] #[arg(long)] #[serde(skip_serializing_if = "Option::is_none")] - pub tcp_reconnection_interval: Option, + pub tcp_reconnection_interval: Option, /// Flag to enable TLS for the TCP transport #[arg(long, default_missing_value(Some("true")), num_args(0..1))] @@ -91,19 +106,19 @@ pub struct ArgsOptional { #[serde(skip_serializing_if = "Option::is_none")] pub quic_server_name: Option, - /// The optional number of reconnect retries for the QUIC transport + /// The optional number of max reconnect retries for the QUIC transport /// /// [default: 3] #[arg(long)] #[serde(skip_serializing_if = "Option::is_none")] - pub quic_reconnection_retries: Option, + pub quic_reconnection_max_retries: Option, /// The optional reconnect interval for the QUIC transport /// - /// [default: 1000] + /// [default: "1s"] #[arg(long)] #[serde(skip_serializing_if = "Option::is_none")] - pub quic_reconnection_interval: Option, + pub quic_reconnection_interval: Option, /// The optional maximum number of concurrent bidirectional streams for QUIC /// @@ -182,14 +197,26 @@ pub struct Args { /// The optional number of retries for the HTTP transport pub http_retries: u32, + // The optional username for initial login + pub username: String, + + // The optional password for initial login + pub password: String, + /// The optional client address for the TCP transport pub tcp_server_address: String, - /// The optional number of reconnect retries for the TCP transport - pub tcp_reconnection_retries: u32, + /// The optional number of maximum reconnect retries for the TCP transport + pub tcp_reconnection_enabled: bool, + + /// The optional number of maximum reconnect retries for the TCP transport + pub tcp_reconnection_max_retries: Option, /// The optional reconnect interval for the TCP transport - pub tcp_reconnection_interval: u64, + pub tcp_reconnection_interval: String, + + /// The optional re-establish after last connection interval for TCP + pub tcp_reconnection_re_establish_after: String, /// Flag to enable TLS for the TCP transport pub tcp_tls_enabled: bool, @@ -206,11 +233,17 @@ pub struct Args { /// The optional server name for the QUIC transport pub quic_server_name: String, - /// The optional number of reconnect retries for the QUIC transport - pub quic_reconnection_retries: u32, + /// The optional number of maximum reconnect retries for the QUIC transport + pub quic_reconnection_enabled: bool, + + /// The optional number of maximum reconnect retries for the QUIC transport + pub quic_reconnection_max_retries: Option, /// The optional reconnect interval for the QUIC transport - pub quic_reconnection_interval: u64, + pub quic_reconnection_interval: String, + + /// The optional re-establish after last connection interval for QUIC + pub quic_reconnection_re_establish_after: String, /// The optional maximum number of concurrent bidirectional streams for QUIC pub quic_max_concurrent_bidi_streams: u64, @@ -271,16 +304,22 @@ impl Default for Args { encryption_key: "".to_string(), http_api_url: "http://localhost:3000".to_string(), http_retries: 3, + username: DEFAULT_ROOT_USERNAME.to_string(), + password: DEFAULT_ROOT_PASSWORD.to_string(), tcp_server_address: "127.0.0.1:8090".to_string(), - tcp_reconnection_retries: 3, - tcp_reconnection_interval: 1000, + tcp_reconnection_enabled: true, + tcp_reconnection_max_retries: None, + tcp_reconnection_interval: "1s".to_string(), + tcp_reconnection_re_establish_after: "5s".to_string(), tcp_tls_enabled: false, tcp_tls_domain: "localhost".to_string(), quic_client_address: "127.0.0.1:0".to_string(), quic_server_address: "127.0.0.1:8080".to_string(), quic_server_name: "localhost".to_string(), - quic_reconnection_retries: 3, - quic_reconnection_interval: 1000, + quic_reconnection_enabled: true, + quic_reconnection_max_retries: None, + quic_reconnection_interval: "1s".to_string(), + quic_reconnection_re_establish_after: "5s".to_string(), quic_max_concurrent_bidi_streams: 10000, quic_datagram_send_buffer_size: 100000, quic_initial_mtu: 1200, @@ -305,6 +344,12 @@ impl From> for Args { if let Some(encryption_key) = optional_args.encryption_key { args.encryption_key = encryption_key; } + if let Some(username) = optional_args.credentials_username { + args.username = username; + } + if let Some(password) = optional_args.credentials_password { + args.password = password; + } if let Some(http_api_url) = optional_args.http_api_url { args.http_api_url = http_api_url; } @@ -314,8 +359,8 @@ impl From> for Args { if let Some(tcp_server_address) = optional_args.tcp_server_address { args.tcp_server_address = tcp_server_address; } - if let Some(tcp_reconnection_retries) = optional_args.tcp_reconnection_retries { - args.tcp_reconnection_retries = tcp_reconnection_retries; + if let Some(tcp_reconnection_retries) = optional_args.tcp_reconnection_max_retries { + args.tcp_reconnection_max_retries = Some(tcp_reconnection_retries); } if let Some(tcp_reconnection_interval) = optional_args.tcp_reconnection_interval { args.tcp_reconnection_interval = tcp_reconnection_interval; @@ -335,8 +380,8 @@ impl From> for Args { if let Some(quic_server_name) = optional_args.quic_server_name { args.quic_server_name = quic_server_name; } - if let Some(quic_reconnection_retries) = optional_args.quic_reconnection_retries { - args.quic_reconnection_retries = quic_reconnection_retries; + if let Some(quic_reconnection_retries) = optional_args.quic_reconnection_max_retries { + args.quic_reconnection_max_retries = Some(quic_reconnection_retries); } if let Some(quic_reconnection_interval) = optional_args.quic_reconnection_interval { args.quic_reconnection_interval = quic_reconnection_interval; diff --git a/sdk/src/binary/mod.rs b/sdk/src/binary/mod.rs index b6c508e81..2f7f37ad6 100644 --- a/sdk/src/binary/mod.rs +++ b/sdk/src/binary/mod.rs @@ -1,7 +1,9 @@ use crate::command::Command; +use crate::diagnostic::DiagnosticEvent; use crate::error::IggyError; use async_trait::async_trait; use bytes::Bytes; +use derive_more::Display; #[allow(deprecated)] pub mod binary_client; @@ -26,13 +28,22 @@ pub mod topics; pub mod users; /// The state of the client. -#[derive(Debug, Copy, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq, Display)] pub enum ClientState { /// The client is disconnected. + #[display(fmt = "disconnected")] Disconnected, + /// The client is connecting. + #[display(fmt = "connecting")] + Connecting, /// The client is connected. + #[display(fmt = "connected")] Connected, + /// The client is authenticating. + #[display(fmt = "authenticating")] + Authenticating, /// The client is connected and authenticated. + #[display(fmt = "authenticated")] Authenticated, } @@ -42,14 +53,18 @@ pub trait BinaryTransport { async fn get_state(&self) -> ClientState; /// Sets the state of the client. async fn set_state(&self, state: ClientState); + async fn publish_event(&self, event: DiagnosticEvent); /// Sends a command and returns the response. async fn send_with_response(&self, command: &T) -> Result; async fn send_raw_with_response(&self, code: u32, payload: Bytes) -> Result; } async fn fail_if_not_authenticated(transport: &T) -> Result<(), IggyError> { - if transport.get_state().await != ClientState::Authenticated { - return Err(IggyError::Unauthenticated); + match transport.get_state().await { + ClientState::Disconnected | ClientState::Connecting | ClientState::Authenticating => { + Err(IggyError::Disconnected) + } + ClientState::Connected => Err(IggyError::Unauthenticated), + ClientState::Authenticated => Ok(()), } - Ok(()) } diff --git a/sdk/src/binary/streams.rs b/sdk/src/binary/streams.rs index f5d35e364..7b1db6a08 100644 --- a/sdk/src/binary/streams.rs +++ b/sdk/src/binary/streams.rs @@ -29,14 +29,19 @@ impl StreamClient for B { mapper::map_streams(response) } - async fn create_stream(&self, name: &str, stream_id: Option) -> Result<(), IggyError> { + async fn create_stream( + &self, + name: &str, + stream_id: Option, + ) -> Result { fail_if_not_authenticated(self).await?; - self.send_with_response(&CreateStream { - name: name.to_string(), - stream_id, - }) - .await?; - Ok(()) + let response = self + .send_with_response(&CreateStream { + name: name.to_string(), + stream_id, + }) + .await?; + mapper::map_stream(response) } async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError> { diff --git a/sdk/src/binary/topics.rs b/sdk/src/binary/topics.rs index 483d51386..55d014888 100644 --- a/sdk/src/binary/topics.rs +++ b/sdk/src/binary/topics.rs @@ -51,20 +51,21 @@ impl TopicClient for B { topic_id: Option, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - ) -> Result<(), IggyError> { + ) -> Result { fail_if_not_authenticated(self).await?; - self.send_with_response(&CreateTopic { - stream_id: stream_id.clone(), - name: name.to_string(), - partitions_count, - compression_algorithm, - replication_factor, - topic_id, - message_expiry, - max_topic_size, - }) - .await?; - Ok(()) + let response = self + .send_with_response(&CreateTopic { + stream_id: stream_id.clone(), + name: name.to_string(), + partitions_count, + compression_algorithm, + replication_factor, + topic_id, + message_expiry, + max_topic_size, + }) + .await?; + mapper::map_topic(response) } async fn update_topic( diff --git a/sdk/src/binary/users.rs b/sdk/src/binary/users.rs index 1ba890a95..af071873f 100644 --- a/sdk/src/binary/users.rs +++ b/sdk/src/binary/users.rs @@ -1,6 +1,7 @@ use crate::binary::binary_client::BinaryClient; use crate::binary::{fail_if_not_authenticated, mapper, ClientState}; use crate::client::UserClient; +use crate::diagnostic::DiagnosticEvent; use crate::error::IggyError; use crate::identifier::Identifier; use crate::models::identity_info::IdentityInfo; @@ -41,16 +42,17 @@ impl UserClient for B { password: &str, status: UserStatus, permissions: Option, - ) -> Result<(), IggyError> { + ) -> Result { fail_if_not_authenticated(self).await?; - self.send_with_response(&CreateUser { - username: username.to_string(), - password: password.to_string(), - status, - permissions, - }) - .await?; - Ok(()) + let response = self + .send_with_response(&CreateUser { + username: username.to_string(), + password: password.to_string(), + status, + permissions, + }) + .await?; + mapper::map_user(response) } async fn delete_user(&self, user_id: &Identifier) -> Result<(), IggyError> { @@ -118,6 +120,7 @@ impl UserClient for B { }) .await?; self.set_state(ClientState::Authenticated).await; + self.publish_event(DiagnosticEvent::SignedIn).await; mapper::map_identity_info(response) } @@ -125,6 +128,7 @@ impl UserClient for B { fail_if_not_authenticated(self).await?; self.send_with_response(&LogoutUser {}).await?; self.set_state(ClientState::Connected).await; + self.publish_event(DiagnosticEvent::SignedOut).await; Ok(()) } } diff --git a/sdk/src/client.rs b/sdk/src/client.rs index ed1f03966..319f775d4 100644 --- a/sdk/src/client.rs +++ b/sdk/src/client.rs @@ -1,5 +1,6 @@ use crate::compression::compression_algorithm::CompressionAlgorithm; use crate::consumer::Consumer; +use crate::diagnostic::DiagnosticEvent; use crate::error::IggyError; use crate::identifier::Identifier; use crate::messages::poll_messages::PollingStrategy; @@ -16,11 +17,36 @@ use crate::models::stream::{Stream, StreamDetails}; use crate::models::topic::{Topic, TopicDetails}; use crate::models::user_info::{UserInfo, UserInfoDetails}; use crate::models::user_status::UserStatus; +use crate::tcp::config::{TcpClientConfig, TcpClientReconnectionConfig}; +use crate::utils::duration::IggyDuration; use crate::utils::expiry::IggyExpiry; use crate::utils::personal_access_token_expiry::PersonalAccessTokenExpiry; use crate::utils::topic_size::MaxTopicSize; +use async_broadcast::Receiver; use async_trait::async_trait; use std::fmt::Debug; +use std::str::FromStr; + +const CONNECTION_STRING_PREFIX: &str = "iggy://"; + +#[derive(Debug)] +pub(crate) struct ConnectionString { + server_address: String, + auto_login: AutoLogin, + options: ConnectionStringOptions, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum AutoLogin { + Disabled, + Enabled(Credentials), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Credentials { + UsernamePassword(String, String), + PersonalAccessToken(String), +} /// The client trait which is the main interface to the Iggy server. /// It consists of multiple modules, each of which is responsible for a specific set of commands. @@ -46,6 +72,9 @@ pub trait Client: /// Disconnect from the server. If the client is not connected, it will do nothing. async fn disconnect(&self) -> Result<(), IggyError>; + + /// Subscribe to diagnostic events. + async fn subscribe_events(&self) -> Receiver; } /// This trait defines the methods to interact with the system module. @@ -91,7 +120,7 @@ pub trait UserClient { password: &str, status: UserStatus, permissions: Option, - ) -> Result<(), IggyError>; + ) -> Result; /// Delete a user by unique ID or username. /// /// Authentication is required, and the permission to manage the users. @@ -162,7 +191,11 @@ pub trait StreamClient { /// Create a new stream. /// /// Authentication is required, and the permission to manage the streams. - async fn create_stream(&self, name: &str, stream_id: Option) -> Result<(), IggyError>; + async fn create_stream( + &self, + name: &str, + stream_id: Option, + ) -> Result; /// Update a stream by unique ID or name. /// /// Authentication is required, and the permission to manage the streams. @@ -207,7 +240,7 @@ pub trait TopicClient { topic_id: Option, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - ) -> Result<(), IggyError>; + ) -> Result; /// Update a topic by unique ID or name. /// /// Authentication is required, and the permission to manage the topics. @@ -379,3 +412,249 @@ pub trait ConsumerGroupClient { group_id: &Identifier, ) -> Result<(), IggyError>; } + +impl FromStr for ConnectionString { + type Err = IggyError; + fn from_str(s: &str) -> Result { + ConnectionString::new(s) + } +} + +impl ConnectionString { + pub fn new(connection_string: &str) -> Result { + if connection_string.is_empty() { + return Err(IggyError::InvalidConnectionString); + } + + if !connection_string.starts_with(CONNECTION_STRING_PREFIX) { + return Err(IggyError::InvalidConnectionString); + } + + let connection_string = connection_string.replace(CONNECTION_STRING_PREFIX, ""); + let parts = connection_string.split("@").collect::>(); + + if parts.len() != 2 { + return Err(IggyError::InvalidConnectionString); + } + + let credentials = parts[0].split(":").collect::>(); + if credentials.len() != 2 { + return Err(IggyError::InvalidConnectionString); + } + + let username = credentials[0]; + let password = credentials[1]; + if username.is_empty() || password.is_empty() { + return Err(IggyError::InvalidConnectionString); + } + + let server_and_options = parts[1].split("?").collect::>(); + if server_and_options.len() > 2 { + return Err(IggyError::InvalidConnectionString); + } + + let server_address = server_and_options[0]; + if server_address.is_empty() { + return Err(IggyError::InvalidConnectionString); + } + + if !server_address.contains(":") { + return Err(IggyError::InvalidConnectionString); + } + + let port = server_address.split(":").collect::>()[1]; + if port.is_empty() { + return Err(IggyError::InvalidConnectionString); + } + + if port.parse::().is_err() { + return Err(IggyError::InvalidConnectionString); + } + + let connection_string_options; + if let Some(options) = server_and_options.get(1) { + connection_string_options = ConnectionString::parse_options(options)?; + } else { + connection_string_options = ConnectionStringOptions::default(); + } + + Ok(ConnectionString { + server_address: server_address.to_owned(), + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + username.to_owned(), + password.to_owned(), + )), + options: connection_string_options, + }) + } + + fn parse_options(options: &str) -> Result { + let options = options.split("&").collect::>(); + let mut tls_enabled = false; + let mut tls_domain = "localhost".to_string(); + let mut reconnection_retries = "unlimited".to_owned(); + let mut reconnection_interval = "1s".to_owned(); + let mut re_establish_after = "5s".to_owned(); + + for option in options { + let option_parts = option.split("=").collect::>(); + if option_parts.len() != 2 { + return Err(IggyError::InvalidConnectionString); + } + match option_parts[0] { + "tls" => { + tls_enabled = option_parts[1] == "true"; + } + "tls_domain" => { + tls_domain = option_parts[1].to_string(); + } + "reconnection_retries" => { + reconnection_retries = option_parts[1].to_string(); + } + "reconnection_interval" => { + reconnection_interval = option_parts[1].to_string(); + } + "re_establish_after" => { + re_establish_after = option_parts[1].to_string(); + } + _ => { + return Err(IggyError::InvalidConnectionString); + } + } + } + Ok(ConnectionStringOptions { + tls_enabled, + tls_domain, + reconnection: TcpClientReconnectionConfig { + enabled: true, + max_retries: match reconnection_retries.as_str() { + "unlimited" => None, + _ => Some(reconnection_retries.parse().unwrap()), + }, + interval: IggyDuration::from_str(reconnection_interval.as_str()) + .map_err(|_| IggyError::InvalidConnectionString)?, + re_establish_after: IggyDuration::from_str(re_establish_after.as_str()) + .map_err(|_| IggyError::InvalidConnectionString)?, + }, + }) + } +} + +#[derive(Debug, Default)] +struct ConnectionStringOptions { + tls_enabled: bool, + tls_domain: String, + reconnection: TcpClientReconnectionConfig, +} + +impl From for TcpClientConfig { + fn from(connection_string: ConnectionString) -> Self { + TcpClientConfig { + server_address: connection_string.server_address, + auto_login: connection_string.auto_login, + tls_enabled: connection_string.options.tls_enabled, + tls_domain: connection_string.options.tls_domain, + reconnection: connection_string.options.reconnection, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn connection_string_without_username_should_fail() { + let server_address = "localhost:1234"; + let value = format!("{CONNECTION_STRING_PREFIX}:secret@{server_address}"); + let connection_string = ConnectionString::new(&value); + assert!(connection_string.is_err()); + } + + #[test] + fn connection_string_without_password_should_fail() { + let server_address = "localhost:1234"; + let value = format!("{CONNECTION_STRING_PREFIX}user1@{server_address}"); + let connection_string = ConnectionString::new(&value); + assert!(connection_string.is_err()); + } + + #[test] + fn connection_string_without_server_address_should_fail() { + let value = format!("{CONNECTION_STRING_PREFIX}user:secret"); + let connection_string = ConnectionString::new(&value); + assert!(connection_string.is_err()); + } + + #[test] + fn connection_string_without_port_should_fail() { + let value = format!("{CONNECTION_STRING_PREFIX}user:secret@localhost"); + let connection_string = ConnectionString::new(&value); + assert!(connection_string.is_err()); + } + + #[test] + fn connection_string_without_options_should_be_parsed_correctly() { + let username = "user1"; + let password = "secret"; + let server_address = "localhost:1234"; + let value = format!("{CONNECTION_STRING_PREFIX}{username}:{password}@{server_address}"); + let connection_string = ConnectionString::new(&value); + assert!(connection_string.is_ok()); + let connection_string = connection_string.unwrap(); + assert_eq!(connection_string.server_address, server_address); + assert_eq!( + connection_string.auto_login, + AutoLogin::Enabled(Credentials::UsernamePassword( + username.to_string(), + password.to_string() + )) + ); + assert!(!connection_string.options.tls_enabled); + assert!(connection_string.options.tls_domain.is_empty()); + assert!(connection_string.options.reconnection.enabled); + assert!(connection_string.options.reconnection.max_retries.is_none()); + assert_eq!( + connection_string.options.reconnection.interval, + IggyDuration::from_str("1s").unwrap() + ); + } + + #[test] + fn connection_string_with_options_should_be_parsed_correctly() { + let username = "user1"; + let password = "secret"; + let server_address = "localhost:1234"; + let tls_domain = "test.com"; + let reconnection_retries = 5; + let reconnection_interval = "5s"; + let re_establish_after = "10s"; + let value = format!("{CONNECTION_STRING_PREFIX}{username}:{password}@{server_address}?tls=true&tls_domain={tls_domain}&reconnection_retries={reconnection_retries}&reconnection_interval={reconnection_interval}&re_establish_after={re_establish_after}"); + let connection_string = ConnectionString::new(&value); + assert!(connection_string.is_ok()); + let connection_string = connection_string.unwrap(); + assert_eq!(connection_string.server_address, server_address); + assert_eq!( + connection_string.auto_login, + AutoLogin::Enabled(Credentials::UsernamePassword( + username.to_string(), + password.to_string() + )) + ); + assert!(connection_string.options.tls_enabled); + assert_eq!(connection_string.options.tls_domain, tls_domain); + assert!(connection_string.options.reconnection.enabled); + assert_eq!( + connection_string.options.reconnection.max_retries, + Some(reconnection_retries) + ); + assert_eq!( + connection_string.options.reconnection.interval, + IggyDuration::from_str(reconnection_interval).unwrap() + ); + assert_eq!( + connection_string.options.reconnection.re_establish_after, + IggyDuration::from_str(re_establish_after).unwrap() + ); + } +} diff --git a/sdk/src/client_provider.rs b/sdk/src/client_provider.rs index b73a4144a..d07d28f39 100644 --- a/sdk/src/client_provider.rs +++ b/sdk/src/client_provider.rs @@ -1,13 +1,15 @@ -use crate::client::Client; +use crate::client::{AutoLogin, Client, Credentials}; use crate::client_error::ClientError; #[allow(deprecated)] use crate::clients::client::IggyClient; use crate::http::client::HttpClient; use crate::http::config::HttpClientConfig; use crate::quic::client::QuicClient; -use crate::quic::config::QuicClientConfig; +use crate::quic::config::{QuicClientConfig, QuicClientReconnectionConfig}; use crate::tcp::client::TcpClient; -use crate::tcp::config::TcpClientConfig; +use crate::tcp::config::{TcpClientConfig, TcpClientReconnectionConfig}; +use crate::utils::duration::IggyDuration; +use std::str::FromStr; use std::sync::Arc; const QUIC_TRANSPORT: &str = "quic"; @@ -59,8 +61,19 @@ impl ClientProviderConfig { client_address: args.quic_client_address, server_address: args.quic_server_address, server_name: args.quic_server_name, - reconnection_retries: args.quic_reconnection_retries, - reconnection_interval: args.quic_reconnection_interval, + reconnection: QuicClientReconnectionConfig { + enabled: args.quic_reconnection_enabled, + max_retries: args.quic_reconnection_max_retries, + interval: IggyDuration::from_str(&args.quic_reconnection_interval).unwrap(), + re_establish_after: IggyDuration::from_str( + &args.quic_reconnection_re_establish_after, + ) + .unwrap(), + }, + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + args.username, + args.password, + )), response_buffer_size: args.quic_response_buffer_size, max_concurrent_bidi_streams: args.quic_max_concurrent_bidi_streams, datagram_send_buffer_size: args.quic_datagram_send_buffer_size, @@ -81,10 +94,21 @@ impl ClientProviderConfig { TCP_TRANSPORT => { config.tcp = Some(Arc::new(TcpClientConfig { server_address: args.tcp_server_address, - reconnection_retries: args.tcp_reconnection_retries, - reconnection_interval: args.tcp_reconnection_interval, tls_enabled: args.tcp_tls_enabled, tls_domain: args.tcp_tls_domain, + reconnection: TcpClientReconnectionConfig { + enabled: args.tcp_reconnection_enabled, + max_retries: args.tcp_reconnection_max_retries, + interval: IggyDuration::from_str(&args.tcp_reconnection_interval).unwrap(), + re_establish_after: IggyDuration::from_str( + &args.tcp_reconnection_re_establish_after, + ) + .unwrap(), + }, + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + args.username, + args.password, + )), })); } _ => return Err(ClientError::InvalidTransport(config.transport.clone())), diff --git a/sdk/src/clients/builder.rs b/sdk/src/clients/builder.rs index 1b8596981..88878707a 100644 --- a/sdk/src/clients/builder.rs +++ b/sdk/src/clients/builder.rs @@ -1,15 +1,15 @@ use crate::client::Client; -use crate::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use crate::clients::client::IggyClient; use crate::error::IggyError; use crate::http::client::HttpClient; use crate::http::config::HttpClientConfigBuilder; -use crate::message_handler::MessageHandler; use crate::partitioner::Partitioner; use crate::quic::client::QuicClient; use crate::quic::config::QuicClientConfigBuilder; use crate::tcp::client::TcpClient; use crate::tcp::config::TcpClientConfigBuilder; use crate::utils::crypto::Encryptor; +use crate::utils::duration::IggyDuration; use std::sync::Arc; use tracing::error; @@ -17,10 +17,8 @@ use tracing::error; #[derive(Debug, Default)] pub struct IggyClientBuilder { client: Option>, - background_config: Option, - partitioner: Option>, - encryptor: Option>, - message_handler: Option>, + partitioner: Option>, + encryptor: Option>, } impl IggyClientBuilder { @@ -30,6 +28,14 @@ impl IggyClientBuilder { IggyClientBuilder::default() } + pub fn from_connection_string(connection_string: &str) -> Result { + let mut builder = Self::new(); + builder.client = Some(Box::new(TcpClient::from_connection_string( + connection_string, + )?)); + Ok(builder) + } + /// Apply the provided client implementation for the specific transport. Setting client clears the client config. pub fn with_client(mut self, client: Box) -> Self { self.client = Some(client); @@ -37,29 +43,17 @@ impl IggyClientBuilder { } /// Use the custom partitioner implementation. - pub fn with_partitioner(mut self, partitioner: Box) -> Self { + pub fn with_partitioner(mut self, partitioner: Arc) -> Self { self.partitioner = Some(partitioner); self } - /// Apply the provided background configuration. - pub fn with_background_config(mut self, background_config: IggyClientBackgroundConfig) -> Self { - self.background_config = Some(background_config); - self - } - /// Use the custom encryptor implementation. - pub fn with_encryptor(mut self, encryptor: Box) -> Self { + pub fn with_encryptor(mut self, encryptor: Arc) -> Self { self.encryptor = Some(encryptor); self } - /// Use the custom message handler implementation. This handler will be used only for `start_polling_messages` method, if neither `subscribe_to_polled_messages` (which returns the receiver for the messages channel) is called nor `on_message` closure is provided. - pub fn with_message_handler(mut self, message_handler: Box) -> Self { - self.message_handler = Some(message_handler); - self - } - /// This method provides fluent API for the TCP client configuration. /// It returns the `TcpClientBuilder` instance, which allows to configure the TCP client with custom settings or using defaults. /// This should be called after the non-protocol specific methods, such as `with_partitioner`, `with_encryptor` or `with_message_handler`. @@ -100,13 +94,7 @@ impl IggyClientBuilder { return Err(IggyError::InvalidConfiguration); }; - Ok(IggyClient::create( - client, - self.background_config.unwrap_or_default(), - self.message_handler, - self.partitioner, - self.encryptor, - )) + Ok(IggyClient::create(client, self.partitioner, self.encryptor)) } } @@ -123,14 +111,16 @@ impl TcpClientBuilder { self } - /// Sets the number of retries when connecting to the server. - pub fn with_reconnection_retries(mut self, reconnection_retries: u32) -> Self { - self.config = self.config.with_reconnection_retries(reconnection_retries); + /// Sets the number of max retries when connecting to the server. + pub fn with_reconnection_max_retries(mut self, reconnection_retries: Option) -> Self { + self.config = self + .config + .with_reconnection_max_retries(reconnection_retries); self } /// Sets the interval between retries when connecting to the server. - pub fn with_reconnection_interval(mut self, reconnection_interval: u64) -> Self { + pub fn with_reconnection_interval(mut self, reconnection_interval: IggyDuration) -> Self { self.config = self .config .with_reconnection_interval(reconnection_interval); @@ -171,13 +161,15 @@ impl QuicClientBuilder { } /// Sets the number of retries when connecting to the server. - pub fn with_reconnection_retries(mut self, reconnection_retries: u32) -> Self { - self.config = self.config.with_reconnection_retries(reconnection_retries); + pub fn with_reconnection_max_retries(mut self, reconnection_retries: Option) -> Self { + self.config = self + .config + .with_reconnection_max_retries(reconnection_retries); self } /// Sets the interval between retries when connecting to the server. - pub fn with_reconnection_interval(mut self, reconnection_interval: u64) -> Self { + pub fn with_reconnection_interval(mut self, reconnection_interval: IggyDuration) -> Self { self.config = self .config .with_reconnection_interval(reconnection_interval); diff --git a/sdk/src/clients/client.rs b/sdk/src/clients/client.rs index ebe8eb156..52484e264 100644 --- a/sdk/src/clients/client.rs +++ b/sdk/src/clients/client.rs @@ -7,13 +7,12 @@ use crate::error::IggyError; use crate::identifier::Identifier; use crate::locking::IggySharedMut; use crate::locking::IggySharedMutFn; -use crate::message_handler::MessageHandler; -use crate::messages::send_messages::{Message, Partitioning, PartitioningKind, SendMessages}; +use crate::messages::send_messages::{Message, Partitioning}; use crate::models::client_info::{ClientInfo, ClientInfoDetails}; use crate::models::consumer_group::{ConsumerGroup, ConsumerGroupDetails}; use crate::models::consumer_offset_info::ConsumerOffsetInfo; use crate::models::identity_info::IdentityInfo; -use crate::models::messages::{PolledMessage, PolledMessages}; +use crate::models::messages::PolledMessages; use crate::models::personal_access_token::{PersonalAccessTokenInfo, RawPersonalAccessToken}; use crate::models::stats::Stats; use crate::models::stream::{Stream, StreamDetails}; @@ -22,112 +21,35 @@ use crate::models::user_info::{UserInfo, UserInfoDetails}; use crate::partitioner::Partitioner; use crate::tcp::client::TcpClient; use crate::utils::crypto::Encryptor; +use async_broadcast::Receiver; use async_dropper::AsyncDrop; use async_trait::async_trait; -use flume::{Receiver, Sender}; -use std::collections::VecDeque; +use bytes::Bytes; use std::fmt::Debug; use std::sync::Arc; -use std::time::Duration; -use tokio::sync::Mutex; -use tokio::task::JoinHandle; -use tokio::time::sleep; -use tracing::{error, info, warn}; +use tracing::info; use crate::clients::builder::IggyClientBuilder; +use crate::clients::consumer::IggyConsumerBuilder; +use crate::clients::producer::IggyProducerBuilder; use crate::compression::compression_algorithm::CompressionAlgorithm; -use crate::messages::poll_messages::{PollingKind, PollingStrategy}; +use crate::diagnostic::DiagnosticEvent; +use crate::messages::poll_messages::PollingStrategy; use crate::models::permissions::Permissions; use crate::models::user_status::UserStatus; use crate::utils::expiry::IggyExpiry; use crate::utils::personal_access_token_expiry::PersonalAccessTokenExpiry; use crate::utils::topic_size::MaxTopicSize; -// The default interval between sending the messages as batches in the background. -pub const DEFAULT_SEND_MESSAGES_INTERVAL_MS: u64 = 100; - -// The default interval between polling the messages in the background. -pub const DEFAULT_POLL_MESSAGES_INTERVAL_MS: u64 = 100; - /// The main client struct which implements all the `Client` traits and wraps the underlying low-level client for the specific transport. /// -/// It also provides additional functionality (outside the shared trait) like sending messages in background, partitioning, client-side encryption or message handling via channels. +/// It also provides the additional builders for the standalone consumer, consumer group, and producer. #[derive(Debug)] #[allow(dead_code)] pub struct IggyClient { client: IggySharedMut>, - config: Option, - send_messages_batch: Option>>, - partitioner: Option>, - encryptor: Option>, - message_handler: Option>>, - message_channel_sender: Option>>, -} - -#[derive(Debug)] -struct SendMessagesBatch { - pub commands: VecDeque, -} - -/// The optional configuration for the `IggyClient` instance, consisting of the optional configuration for sending and polling the messages in the background. -#[derive(Debug, Default)] -pub struct IggyClientBackgroundConfig { - /// The configuration for sending the messages in the background. - pub send_messages: SendMessagesConfig, - /// The configuration for polling the messages in the background. - pub poll_messages: PollMessagesConfig, -} - -/// The configuration for sending the messages in the background. It allows to configure the interval between sending the messages as batches in the background and the maximum number of messages in the batch. -#[derive(Debug)] -pub struct SendMessagesConfig { - /// Whether the sending messages as batches in the background is enabled. Interval must be greater than 0. - pub enabled: bool, - /// The interval in milliseconds between sending the messages as batches in the background. - pub interval: u64, - /// The maximum number of messages in the batch. - pub max_messages: u32, -} - -/// The configuration for polling the messages in the background. It allows to configure the interval between polling the messages and the offset storing strategy. -#[derive(Debug, Copy, Clone)] -pub struct PollMessagesConfig { - /// The interval in milliseconds between polling the messages. - pub interval: u64, - /// The offset storing strategy. - pub store_offset_kind: StoreOffsetKind, -} - -/// The consumer offset storing strategy on the server. -#[derive(Debug, PartialEq, Copy, Clone)] -pub enum StoreOffsetKind { - /// The offset is never stored on the server. - Never, - /// The offset is stored on the server when the messages are received. - WhenMessagesAreReceived, - /// The offset is stored on the server when the messages are processed. - WhenMessagesAreProcessed, - /// The offset is stored on the server after processing each message. - AfterProcessingEachMessage, -} - -impl Default for SendMessagesConfig { - fn default() -> Self { - SendMessagesConfig { - enabled: false, - interval: DEFAULT_SEND_MESSAGES_INTERVAL_MS, - max_messages: 1000, - } - } -} - -impl Default for PollMessagesConfig { - fn default() -> Self { - PollMessagesConfig { - interval: DEFAULT_POLL_MESSAGES_INTERVAL_MS, - store_offset_kind: StoreOffsetKind::WhenMessagesAreProcessed, - } - } + partitioner: Option>, + encryptor: Option>, } impl Default for IggyClient { @@ -142,27 +64,33 @@ impl IggyClient { IggyClientBuilder::new() } + /// Creates a new `IggyClientBuilder`. + pub fn builder_from_connection_string( + connection_string: &str, + ) -> Result { + IggyClientBuilder::from_connection_string(connection_string) + } + /// Creates a new `IggyClient` with the provided client implementation for the specific transport. pub fn new(client: Box) -> Self { + let client = IggySharedMut::new(client); IggyClient { - client: IggySharedMut::new(client), - config: None, - send_messages_batch: None, + client, partitioner: None, encryptor: None, - message_handler: None, - message_channel_sender: None, } } - /// Creates a new `IggyClient` with the provided client implementation for the specific transport and the optional configuration for sending and polling the messages in the background. - /// Additionally, it allows to provide the custom implementations for the message handler, partitioner and encryptor. + pub fn from_connection_string(connection_string: &str) -> Result { + let client = Box::new(TcpClient::from_connection_string(connection_string)?); + Ok(IggyClient::new(client)) + } + + /// Creates a new `IggyClient` with the provided client implementation for the specific transport and the optional implementations for the `partitioner` and `encryptor`. pub fn create( client: Box, - config: IggyClientBackgroundConfig, - message_handler: Option>, - partitioner: Option>, - encryptor: Option>, + partitioner: Option>, + encryptor: Option>, ) -> Self { if partitioner.is_some() { info!("Partitioner is enabled."); @@ -172,302 +100,68 @@ impl IggyClient { } let client = IggySharedMut::new(client); - let send_messages_batch = Arc::new(Mutex::new(SendMessagesBatch { - commands: VecDeque::new(), - })); - if config.send_messages.enabled && config.send_messages.interval > 0 { - info!("Messages will be sent in background."); - Self::send_messages_in_background( - config.send_messages.interval, - config.send_messages.max_messages, - client.clone(), - send_messages_batch.clone(), - ); - } - IggyClient { client, - config: Some(config), - send_messages_batch: Some(send_messages_batch), - message_handler: message_handler.map(Arc::new), - message_channel_sender: None, partitioner, encryptor, } } - /// Returns the channel receiver for the messages which are polled in the background. This will only work if the `start_polling_messages` method is called. - pub fn subscribe_to_polled_messages(&mut self) -> Receiver { - let (sender, receiver) = flume::unbounded(); - self.message_channel_sender = Some(Arc::new(sender)); - receiver + /// Returns the underlying client implementation for the specific transport. + pub fn client(&self) -> IggySharedMut> { + self.client.clone() } - /// Starts polling the messages in the background. It returns the `JoinHandle` which can be used to await for the completion of the task. - #[allow(clippy::too_many_arguments)] - pub fn start_polling_messages( + /// Returns the builder for the standalone consumer. + pub fn consumer( &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: Option, - consumer: &Consumer, - strategy: &PollingStrategy, - count: u32, - on_message: Option, - config_override: Option, - ) -> JoinHandle<()> - where - F: Fn(PolledMessage) + Send + Sync + 'static, - { - let client = self.client.clone(); - let mut interval = Duration::from_millis(DEFAULT_POLL_MESSAGES_INTERVAL_MS); - let message_handler = self.message_handler.clone(); - let message_channel_sender = self.message_channel_sender.clone(); - let mut store_offset_after_processing_each_message = false; - let mut store_offset_when_messages_are_processed = false; - let mut auto_commit = false; - let mut strategy = *strategy; - let consumer = consumer.clone(); - let stream_id = stream_id.clone(); - let topic_id = topic_id.clone(); - - let config = match config_override { - Some(config) => Some(config), - None => self.config.as_ref().map(|config| config.poll_messages), - }; - if let Some(config) = config { - if config.interval > 0 { - interval = Duration::from_millis(config.interval); - } - match config.store_offset_kind { - StoreOffsetKind::Never => { - auto_commit = false; - } - StoreOffsetKind::WhenMessagesAreReceived => { - auto_commit = true; - } - StoreOffsetKind::WhenMessagesAreProcessed => { - auto_commit = false; - store_offset_when_messages_are_processed = true; - } - StoreOffsetKind::AfterProcessingEachMessage => { - auto_commit = false; - store_offset_after_processing_each_message = true; - } - } - } - - tokio::spawn(async move { - loop { - sleep(interval).await; - let client = client.read().await; - let polled_messages = client - .poll_messages( - &stream_id, - &topic_id, - partition_id, - &consumer, - &strategy, - count, - auto_commit, - ) - .await; - if let Err(error) = polled_messages { - error!("There was an error while polling messages: {:?}", error); - continue; - } - - let messages = polled_messages.unwrap().messages; - if messages.is_empty() { - continue; - } - - let mut current_offset = 0; - for message in messages { - current_offset = message.offset; - // Send a message to the subscribed channel (if created), otherwise to the provided closure or message handler. - if let Some(sender) = &message_channel_sender { - if sender.send_async(message).await.is_err() { - error!("Error when sending a message to the channel."); - } - } else if let Some(on_message) = &on_message { - on_message(message); - } else if let Some(message_handler) = &message_handler { - message_handler.handle(message); - } else { - warn!("Received a message with ID: {} at offset: {} which won't be processed. Consider providing the custom `MessageHandler` trait implementation or `on_message` closure.", message.id, message.offset); - } - if store_offset_after_processing_each_message { - if let Err(error) = client - .store_consumer_offset( - &consumer, - &stream_id, - &topic_id, - partition_id, - current_offset, - ) - .await - { - error!("There was an error while storing offset: {:?}", error); - } - } - } - - if store_offset_when_messages_are_processed { - if let Err(error) = client - .store_consumer_offset( - &consumer, - &stream_id, - &topic_id, - partition_id, - current_offset, - ) - .await - { - error!("There was an error while storing offset: {:?}", error); - } - } - - if strategy.kind == PollingKind::Offset { - strategy.value = current_offset + 1; - } - } - }) - } - - /// Sends the provided messages in the background using the custom partitioner implementation. - pub async fn send_messages_using_partitioner( + name: &str, + stream: &str, + topic: &str, + partition: u32, + ) -> Result { + Ok(IggyConsumerBuilder::new( + self.client.clone(), + name.to_owned(), + Consumer::new(name.try_into()?), + stream.try_into()?, + topic.try_into()?, + Some(partition), + self.encryptor.clone(), + None, + )) + } + + /// Returns the builder for the consumer group. + pub fn consumer_group( &self, - stream_id: &Identifier, - topic_id: &Identifier, - partitioning: &Partitioning, - messages: &mut [Message], - partitioner: &dyn Partitioner, - ) -> Result<(), IggyError> { - let partition_id = - partitioner.calculate_partition_id(stream_id, topic_id, partitioning, messages)?; - let partitioning = Partitioning::partition_id(partition_id); - self.send_messages(stream_id, topic_id, &partitioning, messages) - .await - } - - fn send_messages_in_background( - interval: u64, - max_messages: u32, - client: IggySharedMut>, - send_messages_batch: Arc>, - ) { - tokio::spawn(async move { - let max_messages = max_messages as usize; - let interval = Duration::from_millis(interval); - loop { - sleep(interval).await; - let mut send_messages_batch = send_messages_batch.lock().await; - if send_messages_batch.commands.is_empty() { - continue; - } - - let mut initialized = false; - let mut stream_id = Identifier::default(); - let mut topic_id = Identifier::default(); - let mut key = Partitioning::partition_id(1); - let mut batch_messages = true; - - for send_messages in &send_messages_batch.commands { - if !initialized { - if send_messages.partitioning.kind != PartitioningKind::PartitionId { - batch_messages = false; - break; - } - - stream_id = Identifier::from_identifier(&send_messages.stream_id); - topic_id = Identifier::from_identifier(&send_messages.topic_id); - key.value.clone_from(&send_messages.partitioning.value); - initialized = true; - } - - // Batching the messages is only possible for the same stream, topic and partition. - if send_messages.stream_id != stream_id - || send_messages.topic_id != topic_id - || send_messages.partitioning.kind != PartitioningKind::PartitionId - || send_messages.partitioning.value != key.value - { - batch_messages = false; - break; - } - } - - if !batch_messages { - for send_messages in &mut send_messages_batch.commands { - if let Err(error) = client - .read() - .await - .send_messages( - &send_messages.stream_id, - &send_messages.topic_id, - &send_messages.partitioning, - &mut send_messages.messages, - ) - .await - { - error!("There was an error when sending the messages: {:?}", error); - } - } - send_messages_batch.commands.clear(); - continue; - } - - let mut batches = VecDeque::new(); - let mut messages = Vec::new(); - while let Some(send_messages) = send_messages_batch.commands.pop_front() { - messages.extend(send_messages.messages); - if messages.len() >= max_messages { - batches.push_back(messages); - messages = Vec::new(); - } - } - - if !messages.is_empty() { - batches.push_back(messages); - } - - while let Some(messages) = batches.pop_front() { - let mut send_messages = SendMessages { - stream_id: Identifier::from_identifier(&stream_id), - topic_id: Identifier::from_identifier(&topic_id), - partitioning: Partitioning { - kind: PartitioningKind::PartitionId, - length: 4, - value: key.value.clone(), - }, - messages, - }; - - if let Err(error) = client - .read() - .await - .send_messages( - &send_messages.stream_id, - &send_messages.topic_id, - &send_messages.partitioning, - &mut send_messages.messages, - ) - .await - { - error!( - "There was an error when sending the messages batch: {:?}", - error - ); - - if !send_messages.messages.is_empty() { - batches.push_back(send_messages.messages); - } - } - } - - send_messages_batch.commands.clear(); - } - }); + name: &str, + stream: &str, + topic: &str, + ) -> Result { + Ok(IggyConsumerBuilder::new( + self.client.clone(), + name.to_owned(), + Consumer::group(name.try_into()?), + stream.try_into()?, + topic.try_into()?, + None, + self.encryptor.clone(), + None, + )) + } + + /// Returns the builder for the producer. + pub fn producer(&self, stream: &str, topic: &str) -> Result { + Ok(IggyProducerBuilder::new( + self.client.clone(), + stream.try_into()?, + stream.to_owned(), + topic.try_into()?, + topic.to_owned(), + self.encryptor.clone(), + None, + )) } } @@ -480,6 +174,10 @@ impl Client for IggyClient { async fn disconnect(&self) -> Result<(), IggyError> { self.client.read().await.disconnect().await } + + async fn subscribe_events(&self) -> Receiver { + self.client.read().await.subscribe_events().await + } } #[async_trait] @@ -498,7 +196,7 @@ impl UserClient for IggyClient { password: &str, status: UserStatus, permissions: Option, - ) -> Result<(), IggyError> { + ) -> Result { self.client .read() .await @@ -632,7 +330,11 @@ impl StreamClient for IggyClient { self.client.read().await.get_streams().await } - async fn create_stream(&self, name: &str, stream_id: Option) -> Result<(), IggyError> { + async fn create_stream( + &self, + name: &str, + stream_id: Option, + ) -> Result { self.client .read() .await @@ -685,7 +387,7 @@ impl TopicClient for IggyClient { topic_id: Option, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - ) -> Result<(), IggyError> { + ) -> Result { self.client .read() .await @@ -797,7 +499,8 @@ impl MessageClient for IggyClient { return Err(IggyError::InvalidMessagesCount); } - self.client + let mut polled_messages = self + .client .read() .await .poll_messages( @@ -809,7 +512,17 @@ impl MessageClient for IggyClient { count, auto_commit, ) - .await + .await?; + + if let Some(ref encryptor) = self.encryptor { + for message in &mut polled_messages.messages { + let payload = encryptor.decrypt(&message.payload)?; + message.payload = Bytes::from(payload); + message.length = message.payload.len() as u32; + } + } + + Ok(polled_messages) } async fn send_messages( @@ -823,6 +536,13 @@ impl MessageClient for IggyClient { return Err(IggyError::InvalidMessagesCount); } + if let Some(encryptor) = &self.encryptor { + for message in &mut *messages { + message.payload = Bytes::from(encryptor.encrypt(&message.payload)?); + message.length = message.payload.len() as u32; + } + } + self.client .read() .await diff --git a/sdk/src/clients/consumer.rs b/sdk/src/clients/consumer.rs new file mode 100644 index 000000000..3f6caa92c --- /dev/null +++ b/sdk/src/clients/consumer.rs @@ -0,0 +1,816 @@ +use crate::client::Client; +use crate::consumer::{Consumer, ConsumerKind}; +use crate::diagnostic::DiagnosticEvent; +use crate::error::IggyError; +use crate::identifier::{IdKind, Identifier}; +use crate::locking::{IggySharedMut, IggySharedMutFn}; +use crate::messages::poll_messages::{PollingKind, PollingStrategy}; +use crate::models::messages::{PolledMessage, PolledMessages}; +use crate::utils::crypto::Encryptor; +use crate::utils::duration::{IggyDuration, SEC_IN_MICRO}; +use crate::utils::timestamp::IggyTimestamp; +use bytes::Bytes; +use futures::Stream; +use futures_util::{FutureExt, StreamExt}; +use std::collections::VecDeque; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64}; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; +use tokio::time::sleep; +use tracing::{error, info, trace, warn}; + +const ORDERING: std::sync::atomic::Ordering = std::sync::atomic::Ordering::SeqCst; +type PollMessagesFuture = Pin>>>; + +/// The auto-commit configuration for storing the offset on the server. +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum AutoCommit { + /// The auto-commit is disabled and the offset must be stored manually by the consumer. + Disabled, + /// The auto-commit is enabled and the offset is stored on the server depending on the mode. + After(AutoCommitAfter), + /// The auto-commit is enabled and the offset is stored on the server after a certain interval. + Interval(IggyDuration), + /// The auto-commit is enabled and the offset is stored on the server after a certain interval or depending on the mode. + IntervalOrAfter(IggyDuration, AutoCommitAfter), +} + +/// The auto-commit mode for storing the offset on the server. +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum AutoCommitAfter { + /// The offset is stored on the server when the messages are received. + PollingMessages, + /// The offset is stored on the server after all the messages are consumed. + ConsumingAllMessages, + /// The offset is stored on the server after consuming each message. + ConsumingEachMessage, + /// The offset is stored on the server after consuming every Nth message. + EveryNthMessage(u32), +} + +unsafe impl Send for IggyConsumer {} +unsafe impl Sync for IggyConsumer {} + +pub struct IggyConsumer { + initialized: bool, + can_poll: Arc, + client: IggySharedMut>, + consumer_name: String, + consumer: Arc, + is_consumer_group: bool, + joined_consumer_group: Arc, + stream_id: Arc, + topic_id: Arc, + partition_id: Option, + polling_strategy: PollingStrategy, + poll_interval_micros: u64, + batch_size: u32, + auto_commit: AutoCommit, + auto_commit_after_polling: bool, + auto_join_consumer_group: bool, + create_consumer_group_if_not_exists: bool, + next_offset: Arc, + last_stored_offset: Arc, + poll_future: Option, + buffered_messages: VecDeque, + encryptor: Option>, + store_offset_sender: Option>, + store_offset_receiver: Option>, + store_offset_after_each_message: bool, + store_offset_after_all_messages: bool, + store_after_every_nth_message: u64, + last_polled_at: Arc, + current_partition_id: Arc, + current_offset: Arc, + polled_messages_count: Arc, + retry_interval: IggyDuration, +} + +impl IggyConsumer { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + client: IggySharedMut>, + consumer_name: String, + consumer: Consumer, + stream_id: Identifier, + topic_id: Identifier, + partition_id: Option, + polling_interval: Option, + polling_strategy: PollingStrategy, + batch_size: u32, + auto_commit: AutoCommit, + auto_join_consumer_group: bool, + create_consumer_group_if_not_exists: bool, + encryptor: Option>, + retry_interval: IggyDuration, + ) -> Self { + let (store_offset_sender, store_offset_receiver) = if matches!( + auto_commit, + AutoCommit::After( + AutoCommitAfter::ConsumingEachMessage | AutoCommitAfter::EveryNthMessage(_) + ) | AutoCommit::IntervalOrAfter(_, AutoCommitAfter::ConsumingEachMessage) + ) { + let (sender, receiver) = flume::unbounded(); + (Some(sender), Some(receiver)) + } else { + (None, None) + }; + + Self { + initialized: false, + is_consumer_group: consumer.kind == ConsumerKind::ConsumerGroup, + joined_consumer_group: Arc::new(AtomicBool::new(false)), + can_poll: Arc::new(AtomicBool::new(true)), + client, + consumer_name, + consumer: Arc::new(consumer), + stream_id: Arc::new(stream_id), + topic_id: Arc::new(topic_id), + partition_id, + polling_strategy, + poll_interval_micros: polling_interval.map_or(0, |interval| interval.as_micros()), + next_offset: Arc::new(AtomicU64::new(0)), + last_stored_offset: Arc::new(AtomicU64::new(0)), + poll_future: None, + batch_size, + auto_commit, + auto_commit_after_polling: matches!( + auto_commit, + AutoCommit::After(AutoCommitAfter::PollingMessages) + | AutoCommit::IntervalOrAfter(_, AutoCommitAfter::PollingMessages) + ), + auto_join_consumer_group, + create_consumer_group_if_not_exists, + buffered_messages: VecDeque::new(), + encryptor, + store_offset_sender, + store_offset_receiver, + store_offset_after_each_message: matches!( + auto_commit, + AutoCommit::After(AutoCommitAfter::ConsumingEachMessage) + | AutoCommit::IntervalOrAfter(_, AutoCommitAfter::ConsumingEachMessage) + ), + store_offset_after_all_messages: matches!( + auto_commit, + AutoCommit::After(AutoCommitAfter::ConsumingAllMessages) + | AutoCommit::IntervalOrAfter(_, AutoCommitAfter::ConsumingAllMessages) + ), + store_after_every_nth_message: match auto_commit { + AutoCommit::After(AutoCommitAfter::EveryNthMessage(n)) + | AutoCommit::IntervalOrAfter(_, AutoCommitAfter::EveryNthMessage(n)) => n as u64, + _ => 0, + }, + last_polled_at: Arc::new(AtomicU64::new(0)), + current_partition_id: Arc::new(AtomicU32::new(0)), + current_offset: Arc::new(AtomicU64::new(0)), + polled_messages_count: Arc::new(AtomicU64::new(0)), + retry_interval, + } + } + + pub fn stream(&self) -> &Identifier { + &self.stream_id + } + + pub fn topic(&self) -> &Identifier { + &self.topic_id + } + + /// Initializes the consumer by subscribing to diagnostic events, initializing the consumer group if needed, storing the offsets in the background etc. + pub async fn init(&mut self) -> Result<(), IggyError> { + if self.initialized { + return Ok(()); + } + + self.subscribe_events().await; + self.init_consumer_group().await?; + + match self.auto_commit { + AutoCommit::Interval(interval) => self.store_offsets_in_background(interval), + AutoCommit::IntervalOrAfter(interval, _) => self.store_offsets_in_background(interval), + _ => {} + } + + if let Some(store_offset_receiver) = self.store_offset_receiver.clone() { + let client = self.client.clone(); + let consumer = self.consumer.clone(); + let stream_id = self.stream_id.clone(); + let topic_id = self.topic_id.clone(); + let partition_id = self.partition_id; + let last_stored_offset = self.last_stored_offset.clone(); + tokio::spawn(async move { + while let Ok(offset) = store_offset_receiver.recv_async().await { + let last_offset = last_stored_offset.load(ORDERING); + if offset <= last_offset { + continue; + } + + let client = client.read().await; + if let Err(error) = client + .store_consumer_offset( + &consumer, + &stream_id, + &topic_id, + partition_id, + offset, + ) + .await + { + error!("Failed to store offset: {offset}, error: {error}"); + continue; + } + trace!("Stored offset: {offset}"); + last_stored_offset.store(offset, ORDERING); + } + }); + } + + self.initialized = true; + Ok(()) + } + + fn store_offset(&self, offset: u64) { + if let Some(sender) = self.store_offset_sender.as_ref() { + if let Err(error) = sender.send(offset) { + error!("Failed to send offset to store: {error}"); + } + } + } + + fn store_offsets_in_background(&self, interval: IggyDuration) { + let client = self.client.clone(); + let consumer = self.consumer.clone(); + let stream_id = self.stream_id.clone(); + let topic_id = self.topic_id.clone(); + let partition_id = self.partition_id; + let next_offset_to_poll = self.next_offset.clone(); + let last_stored_offset = self.last_stored_offset.clone(); + tokio::spawn(async move { + loop { + sleep(interval.get_duration()).await; + let next_offset = next_offset_to_poll.load(ORDERING); + let last_offset = last_stored_offset.load(ORDERING); + if last_offset == 0 && next_offset == 0 { + continue; + } + + let offset = next_offset - 1; + if offset <= last_offset { + continue; + } + + let client = client.read().await; + if let Err(error) = client + .store_consumer_offset(&consumer, &stream_id, &topic_id, partition_id, offset) + .await + { + error!("Failed to store offset: {offset} in the background, error: {error}"); + continue; + } + trace!("Stored offset: {offset} in the background"); + last_stored_offset.store(offset, ORDERING); + } + }); + } + + async fn init_consumer_group(&self) -> Result<(), IggyError> { + if !self.is_consumer_group { + return Ok(()); + } + + if !self.auto_join_consumer_group { + warn!("Auto join consumer group is disabled"); + return Ok(()); + } + + Self::initialize_consumer_group( + self.client.clone(), + self.create_consumer_group_if_not_exists, + self.stream_id.clone(), + self.topic_id.clone(), + self.consumer.clone(), + &self.consumer_name, + self.joined_consumer_group.clone(), + ) + .await + } + + async fn subscribe_events(&self) { + trace!("Subscribing to diagnostic events"); + let mut receiver; + { + let client = self.client.read().await; + receiver = client.subscribe_events().await; + } + + let is_consumer_group = self.is_consumer_group; + let can_join_consumer_group = is_consumer_group && self.auto_join_consumer_group; + let client = self.client.clone(); + let create_consumer_group_if_not_exists = self.create_consumer_group_if_not_exists; + let stream_id = self.stream_id.clone(); + let topic_id = self.topic_id.clone(); + let consumer = self.consumer.clone(); + let consumer_name = self.consumer_name.clone(); + let can_poll = self.can_poll.clone(); + let joined_consumer_group = self.joined_consumer_group.clone(); + let mut reconnected = false; + let mut disconnected = false; + + tokio::spawn(async move { + while let Some(event) = receiver.next().await { + trace!("Received diagnostic event: {event}"); + match event { + DiagnosticEvent::Connected => { + trace!("Connected to the server"); + joined_consumer_group.store(false, ORDERING); + if disconnected { + reconnected = true; + disconnected = false; + } + } + DiagnosticEvent::Disconnected => { + disconnected = true; + reconnected = false; + joined_consumer_group.store(false, ORDERING); + can_poll.store(false, ORDERING); + warn!("Disconnected from the server"); + } + DiagnosticEvent::SignedIn => { + if !is_consumer_group { + can_poll.store(true, ORDERING); + continue; + } + + if !can_join_consumer_group { + can_poll.store(true, ORDERING); + trace!("Auto join consumer group is disabled"); + continue; + } + + if !reconnected { + can_poll.store(true, ORDERING); + continue; + } + + if joined_consumer_group.load(ORDERING) { + can_poll.store(true, ORDERING); + continue; + } + + info!("Rejoining consumer group"); + if let Err(error) = Self::initialize_consumer_group( + client.clone(), + create_consumer_group_if_not_exists, + stream_id.clone(), + topic_id.clone(), + consumer.clone(), + &consumer_name, + joined_consumer_group.clone(), + ) + .await + { + error!("Failed to join consumer group: {error}"); + continue; + } + info!("Rejoined consumer group"); + can_poll.store(true, ORDERING); + } + DiagnosticEvent::SignedOut => { + joined_consumer_group.store(false, ORDERING); + can_poll.store(false, ORDERING); + } + } + } + }); + } + + fn create_poll_messages_future( + &self, + ) -> impl Future> { + let stream_id = self.stream_id.clone(); + let topic_id = self.topic_id.clone(); + let partition_id = self.partition_id; + let consumer = self.consumer.clone(); + let polling_strategy = self.polling_strategy; + let client = self.client.clone(); + let count = self.batch_size; + let auto_commit = self.auto_commit_after_polling; + let interval = self.poll_interval_micros; + let last_polled_at = self.last_polled_at.clone(); + let can_poll = self.can_poll.clone(); + let retry_interval = self.retry_interval; + + async move { + if interval > 0 { + Self::wait_before_polling(interval, last_polled_at.load(ORDERING)).await; + } + + if !can_poll.load(ORDERING) { + trace!("Trying to poll messages in {retry_interval}..."); + sleep(retry_interval.get_duration()).await; + } + + trace!("Sending poll messages request"); + last_polled_at.store(IggyTimestamp::now().into(), ORDERING); + let polled_messages = client + .read() + .await + .poll_messages( + &stream_id, + &topic_id, + partition_id, + &consumer, + &polling_strategy, + count, + auto_commit, + ) + .await; + if let Ok(polled_messages) = polled_messages { + return Ok(polled_messages); + } + + let error = polled_messages.unwrap_err(); + error!("Failed to poll messages: {error}"); + if matches!(error, IggyError::Disconnected | IggyError::Unauthenticated) { + trace!("Retrying to poll messages in {retry_interval}..."); + sleep(retry_interval.get_duration()).await; + } + Err(error) + } + } + + async fn wait_before_polling(interval: u64, last_sent_at: u64) { + if interval == 0 { + return; + } + + let now: u64 = IggyTimestamp::now().into(); + let elapsed = now - last_sent_at; + if elapsed >= interval { + trace!("No need to wait before polling messages. {now} - {last_sent_at} = {elapsed}"); + return; + } + + let remaining = interval - elapsed; + trace!("Waiting for {remaining} microseconds before polling messages... {interval} - {elapsed} = {remaining}"); + sleep(Duration::from_micros(remaining)).await; + } + + async fn initialize_consumer_group( + client: IggySharedMut>, + create_consumer_group_if_not_exists: bool, + stream_id: Arc, + topic_id: Arc, + consumer: Arc, + consumer_name: &str, + joined_consumer_group: Arc, + ) -> Result<(), IggyError> { + if joined_consumer_group.load(ORDERING) { + return Ok(()); + } + + let client = client.read().await; + let (name, id) = match consumer.id.kind { + IdKind::Numeric => (consumer_name.to_owned(), Some(consumer.id.get_u32_value()?)), + IdKind::String => (consumer.id.get_string_value()?, None), + }; + + let consumer_group_id = name.to_owned().try_into()?; + trace!("Validating consumer group: {consumer_group_id} for topic: {topic_id}, stream: {stream_id}"); + if let Err(error) = client + .get_consumer_group(&stream_id, &topic_id, &consumer_group_id) + .await + { + if !create_consumer_group_if_not_exists { + error!("Consumer group does not exist and auto-creation is disabled."); + return Err(error); + } + + info!("Creating consumer group: {consumer_group_id} for topic: {topic_id}, stream: {stream_id}"); + client + .create_consumer_group(&stream_id, &topic_id, &name, id) + .await?; + } + + info!("Joining consumer group: {consumer_group_id} for topic: {topic_id}, stream: {stream_id}",); + if let Err(error) = client + .join_consumer_group(&stream_id, &topic_id, &consumer_group_id) + .await + { + joined_consumer_group.store(false, ORDERING); + error!("Failed to join consumer group: {consumer_group_id} for topic: {topic_id}, stream: {stream_id}: {error}"); + return Err(error); + } + + joined_consumer_group.store(true, ORDERING); + info!( + "Joined consumer group: {consumer_group_id} for topic: {topic_id}, stream: {stream_id}" + ); + Ok(()) + } +} + +pub struct ReceivedMessage { + pub message: PolledMessage, + pub current_offset: u64, + pub partition_id: u32, +} + +impl ReceivedMessage { + pub fn new(message: PolledMessage, current_offset: u64, partition_id: u32) -> Self { + Self { + message, + current_offset, + partition_id, + } + } +} + +impl Stream for IggyConsumer { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if let Some(message) = self.buffered_messages.pop_front() { + let next_offset = message.offset + 1; + self.next_offset.store(next_offset, ORDERING); + let consumed_messages_count = self.polled_messages_count.fetch_add(1, ORDERING); + if self.store_after_every_nth_message > 0 + && consumed_messages_count % self.store_after_every_nth_message == 0 + { + self.store_offset(message.offset); + } + + if self.buffered_messages.is_empty() { + if self.polling_strategy.kind == PollingKind::Offset { + self.polling_strategy = PollingStrategy::offset(next_offset); + } + + if self.store_offset_after_each_message || self.store_offset_after_all_messages { + self.store_offset(message.offset); + } + } else if self.store_offset_after_each_message { + self.store_offset(message.offset); + } + + return Poll::Ready(Some(Ok(ReceivedMessage::new( + message, + self.current_offset.load(ORDERING), + self.current_partition_id.load(ORDERING), + )))); + } + + if self.poll_future.is_none() { + let future = self.create_poll_messages_future(); + self.poll_future = Some(Box::pin(future)); + } + + while let Some(future) = self.poll_future.as_mut() { + match future.poll_unpin(cx) { + Poll::Ready(Ok(mut polled_messages)) => { + if polled_messages.messages.is_empty() { + self.poll_future = Some(Box::pin(self.create_poll_messages_future())); + } else { + if let Some(ref encryptor) = self.encryptor { + for message in &mut polled_messages.messages { + let payload = encryptor.decrypt(&message.payload); + if payload.is_err() { + self.poll_future = None; + error!("Failed to decrypt the message payload at offset: {}, partition ID: {}", message.offset, polled_messages.partition_id); + let error = payload.unwrap_err(); + return Poll::Ready(Some(Err(error))); + } + + let payload = payload.unwrap(); + message.payload = Bytes::from(payload); + message.length = message.payload.len() as u32; + } + } + self.current_offset + .store(polled_messages.current_offset, ORDERING); + self.current_partition_id + .store(polled_messages.partition_id, ORDERING); + + let message = polled_messages.messages.remove(0); + let next_offset = message.offset + 1; + self.next_offset.store(next_offset, ORDERING); + self.buffered_messages.extend(polled_messages.messages); + + if self.polling_strategy.kind == PollingKind::Offset { + self.polling_strategy = PollingStrategy::offset(next_offset); + } + + let consumed_messages_count = + self.polled_messages_count.fetch_add(1, ORDERING); + if self.store_after_every_nth_message > 0 + && consumed_messages_count % self.store_after_every_nth_message == 0 + { + self.store_offset(message.offset); + } else if self.buffered_messages.is_empty() { + if self.store_offset_after_all_messages { + self.store_offset(message.offset); + } + } else if self.store_offset_after_each_message { + self.store_offset(message.offset); + } + + self.poll_future = None; + return Poll::Ready(Some(Ok(ReceivedMessage::new( + message, + self.current_offset.load(ORDERING), + self.current_partition_id.load(ORDERING), + )))); + } + } + Poll::Ready(Err(err)) => { + self.poll_future = None; + return Poll::Ready(Some(Err(err))); + } + Poll::Pending => return Poll::Pending, + } + } + + Poll::Pending + } +} + +#[derive(Debug)] +pub struct IggyConsumerBuilder { + client: IggySharedMut>, + consumer_name: String, + consumer: Consumer, + stream: Identifier, + topic: Identifier, + partition: Option, + polling_strategy: PollingStrategy, + polling_interval: Option, + batch_size: u32, + auto_commit: AutoCommit, + auto_join_consumer_group: bool, + create_consumer_group_if_not_exists: bool, + encryptor: Option>, + retry_interval: IggyDuration, +} + +impl IggyConsumerBuilder { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + client: IggySharedMut>, + consumer_name: String, + consumer: Consumer, + stream_id: Identifier, + topic_id: Identifier, + partition_id: Option, + encryptor: Option>, + polling_interval: Option, + ) -> Self { + Self { + client, + consumer_name, + consumer, + stream: stream_id, + topic: topic_id, + partition: partition_id, + polling_strategy: PollingStrategy::next(), + batch_size: 1000, + auto_commit: AutoCommit::IntervalOrAfter( + IggyDuration::from(SEC_IN_MICRO), + AutoCommitAfter::PollingMessages, + ), + auto_join_consumer_group: true, + create_consumer_group_if_not_exists: true, + encryptor, + polling_interval, + retry_interval: IggyDuration::from(SEC_IN_MICRO), + } + } + + /// Sets the stream identifier. + pub fn stream(self, stream: Identifier) -> Self { + Self { stream, ..self } + } + + /// Sets the topic identifier. + pub fn topic(self, topic: Identifier) -> Self { + Self { topic, ..self } + } + + /// Sets the partition identifier. + pub fn partition(self, partition: Option) -> Self { + Self { partition, ..self } + } + + /// Sets the polling strategy. + pub fn polling_strategy(self, polling_strategy: PollingStrategy) -> Self { + Self { + polling_strategy, + ..self + } + } + + /// Sets the batch size for polling messages. + pub fn batch_size(self, batch_size: u32) -> Self { + Self { batch_size, ..self } + } + + /// Sets the auto-commit configuration for storing the offset on the server. + pub fn auto_commit(self, auto_commit: AutoCommit) -> Self { + Self { + auto_commit, + ..self + } + } + + /// Automatically joins the consumer group if the consumer is a part of a consumer group. + pub fn auto_join_consumer_group(self) -> Self { + Self { + auto_join_consumer_group: true, + ..self + } + } + + /// Does not automatically join the consumer group if the consumer is a part of a consumer group. + pub fn do_not_auto_join_consumer_group(self) -> Self { + Self { + auto_join_consumer_group: false, + ..self + } + } + + /// Automatically creates the consumer group if it does not exist. + pub fn create_consumer_group_if_not_exists(self) -> Self { + Self { + create_consumer_group_if_not_exists: true, + ..self + } + } + + /// Does not automatically create the consumer group if it does not exist. + pub fn do_not_create_consumer_group_if_not_exists(self) -> Self { + Self { + create_consumer_group_if_not_exists: false, + ..self + } + } + + /// Sets the polling interval for messages. + pub fn poll_interval(self, interval: IggyDuration) -> Self { + Self { + polling_interval: Some(interval), + ..self + } + } + + /// Clears the polling interval for messages. + pub fn without_poll_interval(self) -> Self { + Self { + polling_interval: None, + ..self + } + } + + /// Sets the encryptor for decrypting the messages' payloads. + pub fn encryptor(self, encryptor: Arc) -> Self { + Self { + encryptor: Some(encryptor), + ..self + } + } + + /// Clears the encryptor for decrypting the messages' payloads. + pub fn without_encryptor(self) -> Self { + Self { + encryptor: None, + ..self + } + } + + /// Sets the retry interval in case of server disconnection. + pub fn retry_interval(self, interval: IggyDuration) -> Self { + Self { + retry_interval: interval, + ..self + } + } + + pub fn build(self) -> IggyConsumer { + IggyConsumer::new( + self.client, + self.consumer_name, + self.consumer, + self.stream, + self.topic, + self.partition, + self.polling_interval, + self.polling_strategy, + self.batch_size, + self.auto_commit, + self.auto_join_consumer_group, + self.create_consumer_group_if_not_exists, + self.encryptor, + self.retry_interval, + ) + } +} diff --git a/sdk/src/clients/mod.rs b/sdk/src/clients/mod.rs index 0c2e48b0e..c8cb0767a 100644 --- a/sdk/src/clients/mod.rs +++ b/sdk/src/clients/mod.rs @@ -1,2 +1,4 @@ pub mod builder; pub mod client; +pub mod consumer; +pub mod producer; diff --git a/sdk/src/clients/producer.rs b/sdk/src/clients/producer.rs new file mode 100644 index 000000000..d633b47cf --- /dev/null +++ b/sdk/src/clients/producer.rs @@ -0,0 +1,813 @@ +use crate::client::Client; +use crate::compression::compression_algorithm::CompressionAlgorithm; +use crate::diagnostic::DiagnosticEvent; +use crate::error::IggyError; +use crate::identifier::{IdKind, Identifier}; +use crate::locking::{IggySharedMut, IggySharedMutFn}; +use crate::messages::send_messages::{Message, Partitioning}; +use crate::partitioner::Partitioner; +use crate::utils::crypto::Encryptor; +use crate::utils::duration::{IggyDuration, SEC_IN_MICRO}; +use crate::utils::expiry::IggyExpiry; +use crate::utils::timestamp::IggyTimestamp; +use crate::utils::topic_size::MaxTopicSize; +use async_dropper::AsyncDrop; +use async_trait::async_trait; +use bytes::Bytes; +use futures_util::StreamExt; +use std::collections::{HashMap, VecDeque}; +use std::hash::{Hash, Hasher}; +use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Mutex; +use tokio::time::sleep; +use tracing::{error, info, trace, warn}; + +const ORDERING: std::sync::atomic::Ordering = std::sync::atomic::Ordering::SeqCst; +const MAX_BATCH_SIZE: usize = 1000000; + +unsafe impl Send for IggyProducer {} +unsafe impl Sync for IggyProducer {} + +pub struct IggyProducer { + initialized: bool, + can_send: Arc, + client: Arc>>, + stream_id: Arc, + stream_name: String, + topic_id: Arc, + topic_name: String, + batch_size: Option, + partitioning: Option>, + encryptor: Option>, + partitioner: Option>, + send_interval_micros: u64, + create_stream_if_not_exists: bool, + create_topic_if_not_exists: bool, + topic_partitions_count: u32, + topic_replication_factor: Option, + send_order: Arc>>>, + buffered_messages: Arc, MessagesBatch>>>, + background_sender_initialized: bool, + default_partitioning: Arc, + can_send_immediately: bool, + last_sent_at: Arc, + retry_interval: IggyDuration, +} + +#[derive(Eq, PartialEq)] +struct Destination { + stream: Arc, + topic: Arc, + partitioning: Arc, +} + +impl Hash for Destination { + fn hash(&self, state: &mut H) { + self.stream.hash(state); + self.topic.hash(state); + self.partitioning.hash(state); + } +} + +struct MessagesBatch { + destination: Arc, + messages: Vec, +} + +impl IggyProducer { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + client: IggySharedMut>, + stream: Identifier, + stream_name: String, + topic: Identifier, + topic_name: String, + batch_size: Option, + partitioning: Option, + encryptor: Option>, + partitioner: Option>, + interval: Option, + create_stream_if_not_exists: bool, + create_topic_if_not_exists: bool, + topic_partitions_count: u32, + topic_replication_factor: Option, + retry_interval: IggyDuration, + ) -> Self { + Self { + initialized: false, + client: Arc::new(client), + can_send: Arc::new(AtomicBool::new(true)), + stream_id: Arc::new(stream), + stream_name, + topic_id: Arc::new(topic), + topic_name, + batch_size, + partitioning: partitioning.map(Arc::new), + encryptor, + partitioner, + send_interval_micros: interval.map_or(0, |i| i.as_micros()), + create_stream_if_not_exists, + create_topic_if_not_exists, + topic_partitions_count, + topic_replication_factor, + buffered_messages: Arc::new(Mutex::new(HashMap::new())), + send_order: Arc::new(Mutex::new(VecDeque::new())), + background_sender_initialized: false, + default_partitioning: Arc::new(Partitioning::balanced()), + can_send_immediately: interval.is_none(), + last_sent_at: Arc::new(AtomicU64::new(0)), + retry_interval, + } + } + + pub fn stream(&self) -> &Identifier { + &self.stream_id + } + + pub fn topic(&self) -> &Identifier { + &self.topic_id + } + + /// Initializes the producer by subscribing to diagnostic events, creating the stream and topic if they do not exist etc. + pub async fn init(&mut self) -> Result<(), IggyError> { + if self.initialized { + return Ok(()); + } + + self.subscribe_events().await; + let client = self.client.clone(); + let client = client.read().await; + if let Err(error) = client.get_stream(&self.stream_id).await { + if !self.create_stream_if_not_exists { + error!("Stream does not exist and auto-creation is disabled."); + return Err(error); + } + + let (name, id) = match self.stream_id.kind { + IdKind::Numeric => ( + self.stream_name.to_owned(), + Some(self.stream_id.get_u32_value()?), + ), + IdKind::String => (self.stream_id.get_string_value()?, None), + }; + info!("Creating stream: {name}"); + client.create_stream(&name, id).await?; + } + + if let Err(error) = client.get_topic(&self.stream_id, &self.topic_id).await { + if !self.create_topic_if_not_exists { + error!("Topic does not exist and auto-creation is disabled."); + return Err(error); + } + + let (name, id) = match self.topic_id.kind { + IdKind::Numeric => ( + self.topic_name.to_owned(), + Some(self.topic_id.get_u32_value()?), + ), + IdKind::String => (self.topic_id.get_string_value()?, None), + }; + info!("Creating topic: {name} for stream: {}", self.stream_name); + client + .create_topic( + &self.stream_id, + &self.topic_name, + self.topic_partitions_count, + CompressionAlgorithm::None, + self.topic_replication_factor, + id, + IggyExpiry::ServerDefault, + MaxTopicSize::ServerDefault, + ) + .await?; + } + + if self.send_interval_micros == 0 { + return Ok(()); + }; + + if self.background_sender_initialized { + return Ok(()); + } + + self.background_sender_initialized = true; + let interval_micros = self.send_interval_micros; + let client = self.client.clone(); + let send_order = self.send_order.clone(); + let buffered_messages = self.buffered_messages.clone(); + let last_sent_at = self.last_sent_at.clone(); + let retry_interval = self.retry_interval; + + tokio::spawn(async move { + loop { + if interval_micros > 0 { + Self::wait_before_sending(interval_micros, last_sent_at.load(ORDERING)).await; + } + + let client = client.read().await; + let mut buffered_messages = buffered_messages.lock().await; + let mut send_order = send_order.lock().await; + while let Some(partitioning) = send_order.pop_front() { + let Some(mut batch) = buffered_messages.remove(&partitioning) else { + continue; + }; + + let messages_count = batch.messages.len(); + if messages_count == 0 { + continue; + } + + trace!("Sending {messages_count} buffered messages in the background..."); + last_sent_at.store(IggyTimestamp::now().into(), ORDERING); + if let Err(error) = client + .send_messages( + &batch.destination.stream, + &batch.destination.topic, + &batch.destination.partitioning, + &mut batch.messages, + ) + .await + { + error!( + "Failed to send {messages_count} messages in the background: {error}" + ); + send_order.push_front(partitioning.clone()); + buffered_messages.insert(partitioning, batch); + if matches!(error, IggyError::Disconnected | IggyError::Unauthenticated) { + trace!("Retrying to send {messages_count} buffered messages in the background in {retry_interval}..."); + sleep(retry_interval.get_duration()).await; + } + continue; + } + + trace!("Sent {messages_count} buffered messages in the background."); + } + } + }); + Ok(()) + } + + async fn subscribe_events(&self) { + trace!("Subscribing to diagnostic events"); + let mut receiver; + { + let client = self.client.read().await; + receiver = client.subscribe_events().await; + } + + let can_send = self.can_send.clone(); + + tokio::spawn(async move { + while let Some(event) = receiver.next().await { + trace!("Received diagnostic event: {event}"); + match event { + DiagnosticEvent::Connected => { + can_send.store(false, ORDERING); + trace!("Connected to the server"); + } + DiagnosticEvent::Disconnected => { + can_send.store(false, ORDERING); + warn!("Disconnected from the server"); + } + DiagnosticEvent::SignedIn => { + can_send.store(true, ORDERING); + } + DiagnosticEvent::SignedOut => { + can_send.store(false, ORDERING); + } + } + } + }); + } + + pub async fn send(&self, messages: Vec) -> Result<(), IggyError> { + if messages.is_empty() { + trace!("No messages to send."); + return Ok(()); + } + + if !self.can_send.load(ORDERING) { + trace!("Trying to send messages in {}...", self.retry_interval); + sleep(self.retry_interval.get_duration()).await; + } + + if !self.can_send.load(ORDERING) { + trace!("Trying to send messages in {}...", self.retry_interval); + sleep(self.retry_interval.get_duration()).await; + } + + if self.can_send_immediately { + return self + .send_immediately(&self.stream_id, &self.topic_id, messages, None) + .await; + } + + self.send_buffered( + self.stream_id.clone(), + self.topic_id.clone(), + messages, + None, + ) + .await + } + + pub async fn send_one(&self, message: Message) -> Result<(), IggyError> { + if !self.can_send.load(ORDERING) { + trace!("Trying to send message in {}...", self.retry_interval); + sleep(self.retry_interval.get_duration()).await; + } + + self.send_immediately(&self.stream_id, &self.topic_id, vec![message], None) + .await + } + + pub async fn send_with_partitioning( + &self, + messages: Vec, + partitioning: Option>, + ) -> Result<(), IggyError> { + if messages.is_empty() { + trace!("No messages to send."); + return Ok(()); + } + + if !self.can_send.load(ORDERING) { + trace!("Trying to send messages in {}...", self.retry_interval); + sleep(self.retry_interval.get_duration()).await; + } + + if self.can_send_immediately { + return self + .send_immediately(&self.stream_id, &self.topic_id, messages, partitioning) + .await; + } + + self.send_buffered( + self.stream_id.clone(), + self.topic_id.clone(), + messages, + partitioning, + ) + .await + } + + pub async fn send_to( + &self, + stream: Arc, + topic: Arc, + messages: Vec, + partitioning: Option>, + ) -> Result<(), IggyError> { + if messages.is_empty() { + trace!("No messages to send."); + return Ok(()); + } + + if !self.can_send.load(ORDERING) { + trace!("Trying to send messages in {}...", self.retry_interval); + sleep(self.retry_interval.get_duration()).await; + } + + if self.can_send_immediately { + return self + .send_immediately(&self.stream_id, &self.topic_id, messages, partitioning) + .await; + } + + self.send_buffered(stream, topic, messages, partitioning) + .await + } + + async fn send_buffered( + &self, + stream: Arc, + topic: Arc, + mut messages: Vec, + partitioning: Option>, + ) -> Result<(), IggyError> { + self.encrypt_messages(&mut messages)?; + let partitioning = self.get_partitioning(&stream, &topic, &messages, partitioning)?; + let batch_size = self.batch_size.unwrap_or(MAX_BATCH_SIZE); + let destination = Arc::new(Destination { + stream, + topic, + partitioning, + }); + let mut buffered_messages = self.buffered_messages.lock().await; + let messages_batch = buffered_messages.get_mut(&destination); + if let Some(messages_batch) = messages_batch { + let messages_count = messages.len(); + trace!("Extending buffer with {messages_count} messages..."); + messages_batch.messages.extend(messages); + if messages_batch.messages.len() >= batch_size { + let mut messages = messages_batch + .messages + .drain(..batch_size) + .collect::>(); + let messages_count = messages.len(); + + if self.send_interval_micros > 0 { + Self::wait_before_sending( + self.send_interval_micros, + self.last_sent_at.load(ORDERING), + ) + .await; + } + + trace!("Sending {messages_count} buffered messages..."); + self.last_sent_at + .store(IggyTimestamp::now().into(), ORDERING); + let client = self.client.read().await; + client + .send_messages( + &self.stream_id, + &self.topic_id, + &destination.partitioning, + &mut messages, + ) + .await?; + trace!("Sent {messages_count} buffered messages."); + } + + if messages_batch.messages.is_empty() { + trace!("Removing empty messages batch."); + buffered_messages.remove(&destination); + self.send_order.lock().await.retain(|p| p != &destination); + } + return Ok(()); + } + + if messages.len() >= batch_size { + trace!( + "Sending messages {} exceeding the batch size of {batch_size}...", + messages.len() + ); + let mut messages_batch = messages.drain(..batch_size).collect::>(); + if self.send_interval_micros > 0 { + Self::wait_before_sending( + self.send_interval_micros, + self.last_sent_at.load(ORDERING), + ) + .await; + } + + self.last_sent_at + .store(IggyTimestamp::now().into(), ORDERING); + let client = self.client.read().await; + client + .send_messages( + &destination.stream, + &destination.topic, + &destination.partitioning, + &mut messages_batch, + ) + .await?; + trace!("Sent messages exceeding the batch size of {batch_size}."); + } + + if messages.is_empty() { + trace!("No messages to buffer."); + return Ok(()); + } + + let messages_count = messages.len(); + trace!("Buffering {messages_count} messages..."); + self.send_order.lock().await.push_back(destination.clone()); + buffered_messages.insert( + destination.clone(), + MessagesBatch { + destination, + messages, + }, + ); + trace!("Buffered {messages_count} messages."); + Ok(()) + } + + async fn send_immediately( + &self, + stream: &Identifier, + topic: &Identifier, + mut messages: Vec, + partitioning: Option>, + ) -> Result<(), IggyError> { + trace!("No batch size specified, sending messages immediately."); + self.encrypt_messages(&mut messages)?; + let partitioning = self.get_partitioning(stream, topic, &messages, partitioning)?; + let batch_size = self.batch_size.unwrap_or(MAX_BATCH_SIZE); + let client = self.client.read().await; + if messages.len() <= batch_size { + self.last_sent_at + .store(IggyTimestamp::now().into(), ORDERING); + client + .send_messages(stream, topic, &partitioning, &mut messages) + .await?; + return Ok(()); + } + + for batch in messages.chunks_mut(batch_size) { + self.last_sent_at + .store(IggyTimestamp::now().into(), ORDERING); + client + .send_messages(stream, topic, &partitioning, batch) + .await?; + } + Ok(()) + } + + async fn wait_before_sending(interval: u64, last_sent_at: u64) { + if interval == 0 { + return; + } + + let now: u64 = IggyTimestamp::now().into(); + let elapsed = now - last_sent_at; + if elapsed >= interval { + trace!("No need to wait before sending messages. {now} - {last_sent_at} = {elapsed}"); + return; + } + + let remaining = interval - elapsed; + trace!("Waiting for {remaining} microseconds before sending messages... {interval} - {elapsed} = {remaining}"); + sleep(Duration::from_micros(remaining)).await; + } + + fn encrypt_messages(&self, messages: &mut [Message]) -> Result<(), IggyError> { + if let Some(encryptor) = &self.encryptor { + for message in messages { + message.payload = Bytes::from(encryptor.encrypt(&message.payload)?); + message.length = message.payload.len() as u32; + } + } + Ok(()) + } + + fn get_partitioning( + &self, + stream: &Identifier, + topic: &Identifier, + messages: &[Message], + partitioning: Option>, + ) -> Result, IggyError> { + if let Some(partitioner) = &self.partitioner { + trace!("Calculating partition id using custom partitioner."); + let partition_id = partitioner.calculate_partition_id(stream, topic, messages)?; + Ok(Arc::new(Partitioning::partition_id(partition_id))) + } else { + trace!("Using the provided partitioning."); + Ok(partitioning.unwrap_or_else(|| { + self.partitioning + .clone() + .unwrap_or_else(|| self.default_partitioning.clone()) + })) + } + } +} + +#[async_trait] +impl AsyncDrop for IggyProducer { + async fn async_drop(&mut self) { + let mut buffered_messages = self.buffered_messages.lock().await; + if buffered_messages.is_empty() { + return; + } + + let client = self.client.read().await; + let mut send_order = self.send_order.lock().await; + while let Some(partitioning) = send_order.pop_front() { + let Some(mut batch) = buffered_messages.remove(&partitioning) else { + continue; + }; + + let messages_count = batch.messages.len(); + self.last_sent_at + .store(IggyTimestamp::now().into(), ORDERING); + trace!("Sending {messages_count} remaining messages..."); + if let Err(error) = client + .send_messages( + &batch.destination.stream, + &batch.destination.topic, + &batch.destination.partitioning, + &mut batch.messages, + ) + .await + { + error!("Failed to send {messages_count} remaining messages: {error}"); + } + + trace!("Sent {messages_count} remaining messages."); + } + } +} + +#[derive(Debug)] +pub struct IggyProducerBuilder { + client: IggySharedMut>, + stream: Identifier, + stream_name: String, + topic: Identifier, + topic_name: String, + batch_size: Option, + partitioning: Option, + encryptor: Option>, + partitioner: Option>, + send_interval: Option, + create_stream_if_not_exists: bool, + create_topic_if_not_exists: bool, + topic_partitions_count: u32, + topic_replication_factor: Option, + retry_interval: IggyDuration, +} + +impl IggyProducerBuilder { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + client: IggySharedMut>, + stream: Identifier, + stream_name: String, + topic: Identifier, + topic_name: String, + encryptor: Option>, + partitioner: Option>, + ) -> Self { + Self { + client, + stream, + stream_name, + topic, + topic_name, + batch_size: Some(1000), + partitioning: None, + encryptor, + partitioner, + send_interval: Some(IggyDuration::from(1000)), + create_stream_if_not_exists: true, + create_topic_if_not_exists: true, + topic_partitions_count: 1, + topic_replication_factor: None, + retry_interval: IggyDuration::from(SEC_IN_MICRO), + } + } + + /// Sets the stream identifier. + pub fn stream(self, stream: Identifier) -> Self { + Self { stream, ..self } + } + + /// Sets the stream name. + pub fn topic(self, topic: Identifier) -> Self { + Self { topic, ..self } + } + + /// Sets the number of messages to batch before sending them, can be combined with `interval`. + pub fn batch_size(self, batch_size: u32) -> Self { + Self { + batch_size: if batch_size == 0 { + None + } else { + Some(batch_size.min(MAX_BATCH_SIZE as u32) as usize) + }, + ..self + } + } + + /// Clears the batch size. + pub fn without_batch_size(self) -> Self { + Self { + batch_size: None, + ..self + } + } + + /// Sets the interval between sending the messages, can be combined with `batch_size`. + pub fn send_interval(self, interval: IggyDuration) -> Self { + Self { + send_interval: Some(interval), + ..self + } + } + + /// Clears the interval. + pub fn without_send_interval(self) -> Self { + Self { + send_interval: None, + ..self + } + } + + /// Sets the encryptor for encrypting the messages' payloads. + pub fn encryptor(self, encryptor: Arc) -> Self { + Self { + encryptor: Some(encryptor), + ..self + } + } + + /// Clears the encryptor for encrypting the messages' payloads. + pub fn without_encryptor(self) -> Self { + Self { + encryptor: None, + ..self + } + } + + /// Sets the partitioning strategy for messages. + pub fn partitioning(self, partitioning: Partitioning) -> Self { + Self { + partitioning: Some(partitioning), + ..self + } + } + + /// Clears the partitioning strategy. + pub fn without_partitioning(self) -> Self { + Self { + partitioning: None, + ..self + } + } + + /// Sets the partitioner for messages. + pub fn partitioner(self, partitioner: Arc) -> Self { + Self { + partitioner: Some(partitioner), + ..self + } + } + + /// Clears the partitioner. + pub fn without_partitioner(self) -> Self { + Self { + partitioner: None, + ..self + } + } + + /// Creates the stream if it does not exist - requires user to have the necessary permissions. + pub fn create_stream_if_not_exists(self) -> Self { + Self { + create_stream_if_not_exists: true, + ..self + } + } + + /// Does not create the stream if it does not exist. + pub fn do_not_create_stream_if_not_exists(self) -> Self { + Self { + create_stream_if_not_exists: false, + ..self + } + } + + /// Creates the topic if it does not exist - requires user to have the necessary permissions. + pub fn create_topic_if_not_exists( + self, + partitions_count: u32, + replication_factor: Option, + ) -> Self { + Self { + create_topic_if_not_exists: true, + topic_partitions_count: partitions_count, + topic_replication_factor: replication_factor, + ..self + } + } + + /// Does not create the topic if it does not exist. + pub fn do_not_create_topic_if_not_exists(self) -> Self { + Self { + create_topic_if_not_exists: false, + ..self + } + } + + /// Sets the retry interval in case of server disconnection. + pub fn retry_interval(self, interval: IggyDuration) -> Self { + Self { + retry_interval: interval, + ..self + } + } + + pub fn build(self) -> IggyProducer { + IggyProducer::new( + self.client, + self.stream, + self.stream_name, + self.topic, + self.topic_name, + self.batch_size, + self.partitioning, + self.encryptor, + self.partitioner, + self.send_interval, + self.create_stream_if_not_exists, + self.create_topic_if_not_exists, + self.topic_partitions_count, + self.topic_replication_factor, + self.retry_interval, + ) + } +} diff --git a/sdk/src/diagnostic.rs b/sdk/src/diagnostic.rs new file mode 100644 index 000000000..c88950059 --- /dev/null +++ b/sdk/src/diagnostic.rs @@ -0,0 +1,15 @@ +use derive_more::Display; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Display, Copy, Clone)] +#[serde(rename_all = "snake_case")] +pub enum DiagnosticEvent { + #[display(fmt = "disconnected")] + Disconnected, + #[display(fmt = "connected")] + Connected, + #[display(fmt = "signed_in")] + SignedIn, + #[display(fmt = "signed_out")] + SignedOut, +} diff --git a/sdk/src/error.rs b/sdk/src/error.rs index 11e8b296f..18d318d5d 100644 --- a/sdk/src/error.rs +++ b/sdk/src/error.rs @@ -24,6 +24,10 @@ pub enum IggyError { InvalidIdentifier = 6, #[error("Invalid version: {0}")] InvalidVersion(String) = 7, + #[error("Disconnected")] + Disconnected = 8, + #[error("Cannot establish connection")] + CannotEstablishConnection = 9, #[error("Cannot create base directory, Path: {0}")] CannotCreateBaseDirectory(String) = 10, #[error("Cannot create runtime directory, Path: {0}")] @@ -362,6 +366,8 @@ pub enum IggyError { CannotReadMaxTimestamp = 7003, #[error("Cannot read batch payload")] CannotReadBatchPayload = 7004, + #[error("Invalid connection string")] + InvalidConnectionString = 8000, } impl IggyError { diff --git a/sdk/src/http/client.rs b/sdk/src/http/client.rs index 3b6347d39..21e82ea00 100644 --- a/sdk/src/http/client.rs +++ b/sdk/src/http/client.rs @@ -1,9 +1,11 @@ use crate::client::Client; +use crate::diagnostic::DiagnosticEvent; use crate::error::IggyError; use crate::http::config::HttpClientConfig; use crate::http::HttpTransport; use crate::locking::{IggySharedMut, IggySharedMutFn}; use crate::models::identity_info::IdentityInfo; +use async_broadcast::{broadcast, Receiver, Sender}; use async_trait::async_trait; use reqwest::{Response, Url}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; @@ -29,6 +31,7 @@ pub struct HttpClient { pub api_url: Url, client: ClientWithMiddleware, access_token: IggySharedMut, + events: (Sender, Receiver), } #[async_trait] @@ -40,6 +43,10 @@ impl Client for HttpClient { async fn disconnect(&self) -> Result<(), IggyError> { HttpClient::disconnect(self).await } + + async fn subscribe_events(&self) -> Receiver { + self.events.1.clone() + } } unsafe impl Send for HttpClient {} @@ -238,6 +245,7 @@ impl HttpClient { api_url, client, access_token: IggySharedMut::new("".to_string()), + events: broadcast(1000), }) } diff --git a/sdk/src/http/streams.rs b/sdk/src/http/streams.rs index a43671ca1..d579e0961 100644 --- a/sdk/src/http/streams.rs +++ b/sdk/src/http/streams.rs @@ -24,16 +24,22 @@ impl StreamClient for HttpClient { Ok(streams) } - async fn create_stream(&self, name: &str, stream_id: Option) -> Result<(), IggyError> { - self.post( - PATH, - &CreateStream { - name: name.to_string(), - stream_id, - }, - ) - .await?; - Ok(()) + async fn create_stream( + &self, + name: &str, + stream_id: Option, + ) -> Result { + let response = self + .post( + PATH, + &CreateStream { + name: name.to_string(), + stream_id, + }, + ) + .await?; + let stream = response.json().await?; + Ok(stream) } async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError> { diff --git a/sdk/src/http/topics.rs b/sdk/src/http/topics.rs index 1de12f517..f16e32ff8 100644 --- a/sdk/src/http/topics.rs +++ b/sdk/src/http/topics.rs @@ -44,22 +44,24 @@ impl TopicClient for HttpClient { topic_id: Option, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - ) -> Result<(), IggyError> { - self.post( - &get_path(&stream_id.as_cow_str()), - &CreateTopic { - stream_id: stream_id.clone(), - name: name.to_string(), - partitions_count, - compression_algorithm, - replication_factor, - topic_id, - message_expiry, - max_topic_size, - }, - ) - .await?; - Ok(()) + ) -> Result { + let response = self + .post( + &get_path(&stream_id.as_cow_str()), + &CreateTopic { + stream_id: stream_id.clone(), + name: name.to_string(), + partitions_count, + compression_algorithm, + replication_factor, + topic_id, + message_expiry, + max_topic_size, + }, + ) + .await?; + let topic = response.json().await?; + Ok(topic) } async fn update_topic( diff --git a/sdk/src/http/users.rs b/sdk/src/http/users.rs index 2bec95525..5d68eff72 100644 --- a/sdk/src/http/users.rs +++ b/sdk/src/http/users.rs @@ -36,18 +36,20 @@ impl UserClient for HttpClient { password: &str, status: UserStatus, permissions: Option, - ) -> Result<(), IggyError> { - self.post( - PATH, - &CreateUser { - username: username.to_string(), - password: password.to_string(), - status, - permissions, - }, - ) - .await?; - Ok(()) + ) -> Result { + let response = self + .post( + PATH, + &CreateUser { + username: username.to_string(), + password: password.to_string(), + status, + permissions, + }, + ) + .await?; + let user = response.json().await?; + Ok(user) } async fn delete_user(&self, user_id: &Identifier) -> Result<(), IggyError> { @@ -115,7 +117,7 @@ impl UserClient for HttpClient { &LoginUser { username: username.to_string(), password: password.to_string(), - version: Some("0.5.0".to_string()), + version: Some(env!("CARGO_PKG_VERSION").to_string()), context: Some("".to_string()), }, ) diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 0519a90e8..47cecd0ff 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -16,11 +16,11 @@ pub mod compression; pub mod consumer; pub mod consumer_groups; pub mod consumer_offsets; +pub mod diagnostic; pub mod error; pub mod http; pub mod identifier; pub mod locking; -pub mod message_handler; pub mod messages; pub mod models; pub mod partitioner; diff --git a/sdk/src/message_handler.rs b/sdk/src/message_handler.rs deleted file mode 100644 index b7b2cedf1..000000000 --- a/sdk/src/message_handler.rs +++ /dev/null @@ -1,7 +0,0 @@ -use crate::models::messages::PolledMessage; -use std::fmt::Debug; - -/// The trait represent the logic responsible for handling the message and is used by the `IggyClient`. -pub trait MessageHandler: Send + Sync + Debug { - fn handle(&self, message: PolledMessage); -} diff --git a/sdk/src/messages/poll_messages.rs b/sdk/src/messages/poll_messages.rs index 4f447c674..2acab5dcd 100644 --- a/sdk/src/messages/poll_messages.rs +++ b/sdk/src/messages/poll_messages.rs @@ -78,7 +78,7 @@ pub enum PollingKind { First, /// Start polling from the last message in the partition. Last, - /// Start polling from the next message after the last polled message based on the stored consumer offset. + /// Start polling from the next message after the last polled message based on the stored consumer offset. Should be used with `auto_commit` set to `true`. Next, } @@ -170,7 +170,7 @@ impl PollingStrategy { } } - /// Poll messages from the next message after the last polled message based on the stored consumer offset. + /// Poll messages from the next message after the last polled message based on the stored consumer offset. Should be used with `auto_commit` set to `true`. pub fn next() -> Self { Self { kind: PollingKind::Next, diff --git a/sdk/src/messages/send_messages.rs b/sdk/src/messages/send_messages.rs index bc78f16ee..8a62b4906 100644 --- a/sdk/src/messages/send_messages.rs +++ b/sdk/src/messages/send_messages.rs @@ -12,6 +12,7 @@ use serde_with::base64::Base64; use serde_with::serde_as; use std::collections::HashMap; use std::fmt::Display; +use std::hash::{Hash, Hasher}; use std::str::FromStr; use uuid::Uuid; @@ -43,7 +44,7 @@ pub struct SendMessages { /// - `PartitionId` - the partition ID is provided by the client. /// - `MessagesKey` - the partition ID is calculated by the server using the hash of the provided messages key. #[serde_as] -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] pub struct Partitioning { /// The kind of partitioning. pub kind: PartitioningKind, @@ -55,6 +56,14 @@ pub struct Partitioning { pub value: Vec, } +impl Hash for Partitioning { + fn hash(&self, state: &mut H) { + self.kind.hash(state); + self.length.hash(state); + self.value.hash(state); + } +} + /// The single message to be sent. It has the following payload: /// - `id` - unique message ID, if not specified by the client (has value = 0), it will be generated by the server. /// - `length` - length of the payload. @@ -77,7 +86,7 @@ pub struct Message { } /// `PartitioningKind` is an enum which specifies the kind of partitioning and is used by `Partitioning`. -#[derive(Debug, Serialize, Deserialize, PartialEq, Default, Copy, Clone)] +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Default, Copy, Clone)] #[serde(rename_all = "snake_case")] pub enum PartitioningKind { /// The partition ID is calculated by the server using the round-robin algorithm. @@ -89,6 +98,12 @@ pub enum PartitioningKind { MessagesKey, } +impl Hash for PartitioningKind { + fn hash(&self, state: &mut H) { + self.as_code().hash(state); + } +} + fn default_message_id() -> u128 { 0 } diff --git a/sdk/src/models/permissions.rs b/sdk/src/models/permissions.rs index e1cb6f239..b79ee6834 100644 --- a/sdk/src/models/permissions.rs +++ b/sdk/src/models/permissions.rs @@ -66,10 +66,16 @@ pub struct GlobalPermissions { /// - delete_topic pub manage_topics: bool, - /// `read_topics` permission allows to read the topics and includes all the permissions of `read_messages`. + /// `read_topics` permission allows to read the topics, manage consumer groups, and includes all the permissions of `poll_messages`. /// Additionally, the following methods can be invoked: /// - get_topic /// - get_topics + /// - get_consumer_group + /// - get_consumer_groups + /// - join_consumer_group + /// - leave_consumer_group + /// - create_consumer_group + /// - delete_consumer_group pub read_topics: bool, /// `poll_messages` permission allows to poll messages from all the streams and theirs topics. @@ -92,7 +98,7 @@ pub struct StreamPermissions { pub manage_stream: bool, /// `read_stream` permission allows to read the stream and includes all the permissions of `read_topics`. - /// Also, it allows to read all the messages of a topic, thus it has all the permissions of `read_messages`. + /// Also, it allows to read all the messages of a topic, thus it has all the permissions of `poll_messages`. /// Additionally, the following methods can be invoked: /// - get_stream /// - get_streams @@ -106,7 +112,7 @@ pub struct StreamPermissions { /// - delete_topic pub manage_topics: bool, - /// `read_topics` permission allows to read the topics and includes all the permissions of `read_messages`. + /// `read_topics` permission allows to read the topics, manage consumer groups, and includes all the permissions of `poll_messages`. pub read_topics: bool, /// `poll_messages` permission allows to poll messages from the stream and its topics. @@ -125,7 +131,7 @@ pub struct TopicPermissions { /// `manage_topic` permission allows to manage the topic and includes all the permissions of `read_topic`. pub manage_topic: bool, - /// `read_topic` permission allows to read the topic and includes all the permissions of `read_messages`. + /// `read_topic` permission allows to read the topic, manage consumer groups, and includes all the permissions of `poll_messages`. pub read_topic: bool, /// `poll_messages` permission allows to poll messages from the topic. diff --git a/sdk/src/partitioner.rs b/sdk/src/partitioner.rs index eac0dbb49..8d2145c41 100644 --- a/sdk/src/partitioner.rs +++ b/sdk/src/partitioner.rs @@ -1,6 +1,6 @@ use crate::error::IggyError; use crate::identifier::Identifier; -use crate::messages::send_messages::{Message, Partitioning}; +use crate::messages::send_messages::Message; use std::fmt::Debug; /// The trait represent the logic responsible for calculating the partition ID and is used by the `IggyClient`. @@ -10,7 +10,6 @@ pub trait Partitioner: Send + Sync + Debug { &self, stream_id: &Identifier, topic_id: &Identifier, - partitioning: &Partitioning, messages: &[Message], ) -> Result; } diff --git a/sdk/src/quic/client.rs b/sdk/src/quic/client.rs index a56ebbf98..7d47fe530 100644 --- a/sdk/src/quic/client.rs +++ b/sdk/src/quic/client.rs @@ -1,9 +1,13 @@ use crate::binary::binary_client::BinaryClient; use crate::binary::{BinaryTransport, ClientState}; -use crate::client::Client; +use crate::client::{AutoLogin, Client, Credentials, PersonalAccessTokenClient, UserClient}; use crate::command::Command; +use crate::diagnostic::DiagnosticEvent; use crate::error::IggyError; use crate::quic::config::QuicClientConfig; +use crate::utils::duration::IggyDuration; +use crate::utils::timestamp::IggyTimestamp; +use async_broadcast::{broadcast, Receiver, Sender}; use async_trait::async_trait; use bytes::Bytes; use quinn::crypto::rustls::QuicClientConfig as QuinnQuicClientConfig; @@ -17,7 +21,7 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::Mutex; use tokio::time::sleep; -use tracing::{error, info, trace}; +use tracing::{error, info, trace, warn}; const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; @@ -31,6 +35,8 @@ pub struct QuicClient { pub(crate) config: Arc, pub(crate) server_address: SocketAddr, pub(crate) state: Mutex, + events: (Sender, Receiver), + connected_at: Mutex>, } unsafe impl Send for QuicClient {} @@ -51,6 +57,10 @@ impl Client for QuicClient { async fn disconnect(&self) -> Result<(), IggyError> { QuicClient::disconnect(self).await } + + async fn subscribe_events(&self) -> Receiver { + self.events.1.clone() + } } #[async_trait] @@ -70,26 +80,33 @@ impl BinaryTransport for QuicClient { } async fn send_raw_with_response(&self, code: u32, payload: Bytes) -> Result { - if self.get_state().await == ClientState::Disconnected { - return Err(IggyError::NotConnected); + let result = self.send_raw(code, payload.clone()).await; + if result.is_ok() { + return result; } - let connection = self.connection.lock().await; - if let Some(connection) = connection.as_ref() { - let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; - let (mut send, mut recv) = connection.open_bi().await?; - trace!("Sending a QUIC request..."); - send.write_all(&(payload_length as u32).to_le_bytes()) - .await?; - send.write_all(&code.to_le_bytes()).await?; - send.write_all(&payload).await?; - send.finish()?; - trace!("Sent a QUIC request, waiting for a response..."); - return self.handle_response(&mut recv).await; + let error = result.unwrap_err(); + if !matches!( + error, + IggyError::Disconnected | IggyError::EmptyResponse | IggyError::Unauthenticated + ) { + return Err(error); } - error!("Cannot send data. Client is not connected."); - Err(IggyError::NotConnected) + if !self.config.reconnection.enabled { + return Err(IggyError::Disconnected); + } + + self.disconnect().await?; + info!("Reconnecting to the server..."); + self.connect().await?; + self.send_raw(code, payload).await + } + + async fn publish_event(&self, event: DiagnosticEvent) { + if let Err(error) = self.events.0.broadcast(event).await { + error!("Failed to send a QUIC diagnostic event: {error}"); + } } } @@ -102,12 +119,14 @@ impl QuicClient { server_address: &str, server_name: &str, validate_certificate: bool, + auto_sign_in: AutoLogin, ) -> Result { Self::create(Arc::new(QuicClientConfig { client_address: client_address.to_string(), server_address: server_address.to_string(), server_name: server_name.to_string(), validate_certificate, + auto_login: auto_sign_in, ..Default::default() })) } @@ -140,6 +159,8 @@ impl QuicClient { server_address, connection: Mutex::new(None), state: Mutex::new(ClientState::Disconnected), + events: broadcast(1000), + connected_at: Mutex::new(None), }) } @@ -183,16 +204,41 @@ impl QuicClient { } async fn connect(&self) -> Result<(), IggyError> { - if self.get_state().await == ClientState::Connected { - return Ok(()); + match self.get_state().await { + ClientState::Connected | ClientState::Authenticating | ClientState::Authenticated => { + trace!("Client is already connected."); + return Ok(()); + } + ClientState::Connecting => { + trace!("Client is already connecting."); + return Ok(()); + } + _ => {} + } + + self.set_state(ClientState::Connecting).await; + if let Some(connected_at) = self.connected_at.lock().await.as_ref() { + let now = IggyTimestamp::now(); + let elapsed = now.as_micros() - connected_at.as_micros(); + let interval = self.config.reconnection.re_establish_after.as_micros(); + trace!( + "Elapsed time since last connection: {}", + IggyDuration::from(elapsed) + ); + if elapsed < interval { + let remaining = IggyDuration::from(interval - elapsed); + info!("Trying to connect to the server in: {remaining}",); + sleep(remaining.get_duration()).await; + } } let mut retry_count = 0; let connection; + let remote_address; loop { info!( - "{} client is connecting to server: {}...", - NAME, self.config.server_address + "{NAME} client is connecting to server: {}...", + self.config.server_address ); let connection_result = self .endpoint @@ -205,30 +251,72 @@ impl QuicClient { "Failed to connect to server: {}", self.config.server_address ); - if retry_count < self.config.reconnection_retries { + if !self.config.reconnection.enabled { + warn!("Automatic reconnection is disabled."); + return Err(IggyError::CannotEstablishConnection); + } + + let unlimited_retries = self.config.reconnection.max_retries.is_none(); + let max_retries = self.config.reconnection.max_retries.unwrap_or_default(); + let max_retries_str = + if let Some(max_retries) = self.config.reconnection.max_retries { + max_retries.to_string() + } else { + "unlimited".to_string() + }; + + let interval_str = self.config.reconnection.interval.as_human_time_string(); + if unlimited_retries || retry_count < max_retries { retry_count += 1; info!( - "Retrying to connect to server ({}/{}): {} in: {} ms...", - retry_count, - self.config.reconnection_retries, + "Retrying to connect to server ({retry_count}/{max_retries_str}): {} in: {interval_str}", self.config.server_address, - self.config.reconnection_interval ); - sleep(Duration::from_millis(self.config.reconnection_interval)).await; + sleep(self.config.reconnection.interval.get_duration()).await; continue; } - return Err(IggyError::NotConnected); + self.set_state(ClientState::Disconnected).await; + self.publish_event(DiagnosticEvent::Disconnected).await; + return Err(IggyError::CannotEstablishConnection); } connection = connection_result.unwrap(); + remote_address = connection.remote_address(); break; } + let now = IggyTimestamp::now(); + info!("{NAME} client has connected to server: {remote_address} at {now}",); self.set_state(ClientState::Connected).await; self.connection.lock().await.replace(connection); + self.connected_at.lock().await.replace(now); + self.publish_event(DiagnosticEvent::Connected).await; - Ok(()) + match &self.config.auto_login { + AutoLogin::Disabled => { + info!("Automatic sign-in is disabled."); + Ok(()) + } + AutoLogin::Enabled(credentials) => { + info!("{NAME} client is signing in..."); + self.set_state(ClientState::Authenticating).await; + match credentials { + Credentials::UsernamePassword(username, password) => { + self.login_user(username, password).await?; + self.publish_event(DiagnosticEvent::SignedIn).await; + info!("{NAME} client has signed in with the user credentials, username: {username}",); + Ok(()) + } + Credentials::PersonalAccessToken(token) => { + self.login_with_personal_access_token(token).await?; + self.publish_event(DiagnosticEvent::SignedIn).await; + info!("{NAME} client has signed in with a personal access token.",); + Ok(()) + } + } + } + } } async fn disconnect(&self) -> Result<(), IggyError> { @@ -236,13 +324,46 @@ impl QuicClient { return Ok(()); } - info!("{} client is disconnecting from server...", NAME); + info!("{NAME} client is disconnecting from server..."); self.set_state(ClientState::Disconnected).await; self.connection.lock().await.take(); self.endpoint.wait_idle().await; - info!("{} client has disconnected from server.", NAME); + self.publish_event(DiagnosticEvent::Disconnected).await; + let now = IggyTimestamp::now(); + info!("{NAME} client has disconnected from server at: {now}."); Ok(()) } + + async fn send_raw(&self, code: u32, payload: Bytes) -> Result { + match self.get_state().await { + ClientState::Disconnected => { + trace!("Cannot send data. Client is not connected."); + return Err(IggyError::NotConnected); + } + ClientState::Connecting => { + trace!("Cannot send data. Client is still connecting."); + return Err(IggyError::NotConnected); + } + _ => {} + } + + let connection = self.connection.lock().await; + if let Some(connection) = connection.as_ref() { + let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; + let (mut send, mut recv) = connection.open_bi().await?; + trace!("Sending a QUIC request with code: {code}"); + send.write_all(&(payload_length as u32).to_le_bytes()) + .await?; + send.write_all(&code.to_le_bytes()).await?; + send.write_all(&payload).await?; + send.finish()?; + trace!("Sent a QUIC request with code: {code}, waiting for a response..."); + return self.handle_response(&mut recv).await; + } + + error!("Cannot send data. Client is not connected."); + Err(IggyError::NotConnected) + } } fn configure(config: &QuicClientConfig) -> Result { diff --git a/sdk/src/quic/config.rs b/sdk/src/quic/config.rs index 743fb005f..a4583b08c 100644 --- a/sdk/src/quic/config.rs +++ b/sdk/src/quic/config.rs @@ -1,3 +1,7 @@ +use crate::client::AutoLogin; +use crate::utils::duration::IggyDuration; +use std::str::FromStr; + /// Configuration for the QUIC client. #[derive(Debug, Clone)] pub struct QuicClientConfig { @@ -7,10 +11,10 @@ pub struct QuicClientConfig { pub server_address: String, /// The server name to use. pub server_name: String, - /// The number of reconnection retries. - pub reconnection_retries: u32, - /// The interval between reconnection retries. - pub reconnection_interval: u64, + /// Whether to automatically login user after establishing connection. + pub auto_login: AutoLogin, + // Whether to automatically reconnect when disconnected. + pub reconnection: QuicClientReconnectionConfig, /// The size of the response buffer. pub response_buffer_size: u64, /// The maximum number of concurrent bidirectional streams. @@ -31,14 +35,33 @@ pub struct QuicClientConfig { pub validate_certificate: bool, } +#[derive(Debug, Clone)] +pub struct QuicClientReconnectionConfig { + pub enabled: bool, + pub max_retries: Option, + pub interval: IggyDuration, + pub re_establish_after: IggyDuration, +} + +impl Default for QuicClientReconnectionConfig { + fn default() -> QuicClientReconnectionConfig { + QuicClientReconnectionConfig { + enabled: true, + max_retries: None, + interval: IggyDuration::from_str("1s").unwrap(), + re_establish_after: IggyDuration::from_str("5s").unwrap(), + } + } +} + impl Default for QuicClientConfig { fn default() -> QuicClientConfig { QuicClientConfig { client_address: "127.0.0.1:0".to_string(), server_address: "127.0.0.1:8080".to_string(), server_name: "localhost".to_string(), - reconnection_retries: 3, - reconnection_interval: 1000, + auto_login: AutoLogin::Disabled, + reconnection: QuicClientReconnectionConfig::default(), response_buffer_size: 1000 * 1000 * 10, max_concurrent_bidi_streams: 10000, datagram_send_buffer_size: 100_000, @@ -58,8 +81,8 @@ impl Default for QuicClientConfig { /// - `client_address`: Default is "127.0.0.1:0" (binds to any available port). /// - `server_address`: Default is "127.0.0.1:8080". /// - `server_name`: Default is "localhost". -/// - `reconnection_retries`: Default is 3 attempts. -/// - `reconnection_interval`: Default is 1000 milliseconds. +/// - `auto_login`: Default is AutoLogin::Disabled. +/// - `reconnection`: Default is enabled unlimited retries and 1 second interval. /// - `response_buffer_size`: Default is 10MB (10,000,000 bytes). /// - `max_concurrent_bidi_streams`: Default is 10,000 streams. /// - `datagram_send_buffer_size`: Default is 100,000 bytes. @@ -92,21 +115,32 @@ impl QuicClientConfigBuilder { self } + /// Sets the auto sign in during connection. + pub fn with_auto_sign_in(mut self, auto_sign_in: AutoLogin) -> Self { + self.config.auto_login = auto_sign_in; + self + } + /// Sets the server name. Defaults to "localhost". pub fn with_server_name(mut self, server_name: String) -> Self { self.config.server_name = server_name; self } - /// Sets the number of reconnection retries. Defaults to 3. - pub fn with_reconnection_retries(mut self, reconnection_retries: u32) -> Self { - self.config.reconnection_retries = reconnection_retries; + pub fn with_enabled_reconnection(mut self) -> Self { + self.config.reconnection.enabled = true; + self + } + + /// Sets the number of retries when connecting to the server. + pub fn with_reconnection_max_retries(mut self, max_retries: Option) -> Self { + self.config.reconnection.max_retries = max_retries; self } - /// Sets the reconnection interval in milliseconds. Defaults to 1000ms. - pub fn with_reconnection_interval(mut self, reconnection_interval: u64) -> Self { - self.config.reconnection_interval = reconnection_interval; + /// Sets the interval between retries when connecting to the server. + pub fn with_reconnection_interval(mut self, interval: IggyDuration) -> Self { + self.config.reconnection.interval = interval; self } diff --git a/sdk/src/tcp/client.rs b/sdk/src/tcp/client.rs index f566b9150..9ce909cce 100644 --- a/sdk/src/tcp/client.rs +++ b/sdk/src/tcp/client.rs @@ -1,14 +1,20 @@ use crate::binary::binary_client::BinaryClient; use crate::binary::{BinaryTransport, ClientState}; -use crate::client::Client; +use crate::client::{ + AutoLogin, Client, ConnectionString, Credentials, PersonalAccessTokenClient, UserClient, +}; use crate::command::Command; +use crate::diagnostic::DiagnosticEvent; use crate::error::{IggyError, IggyErrorDiscriminants}; use crate::tcp::config::TcpClientConfig; +use crate::utils::duration::IggyDuration; +use crate::utils::timestamp::IggyTimestamp; +use async_broadcast::{broadcast, Receiver, Sender}; use async_trait::async_trait; use bytes::{Buf, BufMut, Bytes, BytesMut}; use std::fmt::Debug; +use std::str::FromStr; use std::sync::Arc; -use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader, BufWriter}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; @@ -16,7 +22,7 @@ use tokio::sync::Mutex; use tokio::time::sleep; use tokio_native_tls::native_tls::TlsConnector; use tokio_native_tls::TlsStream; -use tracing::{error, info, trace}; +use tracing::{error, info, trace, warn}; const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; @@ -29,6 +35,8 @@ pub struct TcpClient { pub(crate) stream: Mutex>>, pub(crate) config: Arc, pub(crate) state: Mutex, + events: (Sender, Receiver), + connected_at: Mutex>, } unsafe impl Send for TcpClient {} @@ -71,12 +79,10 @@ unsafe impl Sync for TcpTlsConnectionStream {} #[async_trait] impl ConnectionStream for TcpConnectionStream { async fn read(&mut self, buf: &mut [u8]) -> Result { - let result = self.reader.read_exact(buf).await; - if let Err(error) = result { - return Err(IggyError::from(error)); - } - - Ok(result.unwrap()) + self.reader.read_exact(buf).await.map_err(|error| { + error!("Failed to read data from the TCP connection: {error}"); + IggyError::from(error) + }) } async fn write(&mut self, buf: &[u8]) -> Result<(), IggyError> { @@ -91,21 +97,14 @@ impl ConnectionStream for TcpConnectionStream { #[async_trait] impl ConnectionStream for TcpTlsConnectionStream { async fn read(&mut self, buf: &mut [u8]) -> Result { - let result = self.stream.read_exact(buf).await; - if let Err(error) = result { - return Err(IggyError::from(error)); - } - - Ok(result.unwrap()) + self.stream.read_exact(buf).await.map_err(|error| { + error!("Failed to read data from the TCP connection: {error}"); + IggyError::from(error) + }) } async fn write(&mut self, buf: &[u8]) -> Result<(), IggyError> { - let result = self.stream.write_all(buf).await; - if let Err(error) = result { - return Err(IggyError::from(error)); - } - - Ok(()) + Ok(self.stream.write_all(buf).await?) } async fn flush(&mut self) -> Result<(), IggyError> { @@ -128,6 +127,10 @@ impl Client for TcpClient { async fn disconnect(&self) -> Result<(), IggyError> { TcpClient::disconnect(self).await } + + async fn subscribe_events(&self) -> Receiver { + self.events.1.clone() + } } #[async_trait] @@ -147,34 +150,33 @@ impl BinaryTransport for TcpClient { } async fn send_raw_with_response(&self, code: u32, payload: Bytes) -> Result { - if self.get_state().await == ClientState::Disconnected { - return Err(IggyError::NotConnected); + let result = self.send_raw(code, payload.clone()).await; + if result.is_ok() { + return result; } - let mut stream = self.stream.lock().await; - if let Some(stream) = stream.as_mut() { - let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; - trace!("Sending a TCP request..."); - stream.write(&(payload_length as u32).to_le_bytes()).await?; - stream.write(&code.to_le_bytes()).await?; - stream.write(&payload).await?; - stream.flush().await?; - trace!("Sent a TCP request, waiting for a response..."); - - let mut response_buffer = [0u8; RESPONSE_INITIAL_BYTES_LENGTH]; - let read_bytes = stream.read(&mut response_buffer).await?; - if read_bytes != RESPONSE_INITIAL_BYTES_LENGTH { - error!("Received an invalid or empty response."); - return Err(IggyError::EmptyResponse); - } + let error = result.unwrap_err(); + if !matches!( + error, + IggyError::Disconnected | IggyError::EmptyResponse | IggyError::Unauthenticated + ) { + return Err(error); + } - let status = u32::from_le_bytes(response_buffer[..4].try_into().unwrap()); - let length = u32::from_le_bytes(response_buffer[4..].try_into().unwrap()); - return self.handle_response(status, length, stream.as_mut()).await; + if !self.config.reconnection.enabled { + return Err(IggyError::Disconnected); } - error!("Cannot send data. Client is not connected."); - Err(IggyError::NotConnected) + self.disconnect().await?; + info!("Reconnecting to the server..."); + self.connect().await?; + self.send_raw(code, payload).await + } + + async fn publish_event(&self, event: DiagnosticEvent) { + if let Err(error) = self.events.0.broadcast(event).await { + error!("Failed to send a TCP diagnostic event: {error}"); + } } } @@ -182,29 +184,43 @@ impl BinaryClient for TcpClient {} impl TcpClient { /// Create a new TCP client for the provided server address. - pub fn new(server_address: &str) -> Result { + pub fn new(server_address: &str, auto_sign_in: AutoLogin) -> Result { Self::create(Arc::new(TcpClientConfig { server_address: server_address.to_string(), + auto_login: auto_sign_in, ..Default::default() })) } /// Create a new TCP client for the provided server address using TLS. - pub fn new_tls(server_address: &str, domain: &str) -> Result { + pub fn new_tls( + server_address: &str, + domain: &str, + auto_sign_in: AutoLogin, + ) -> Result { Self::create(Arc::new(TcpClientConfig { server_address: server_address.to_string(), tls_enabled: true, tls_domain: domain.to_string(), + auto_login: auto_sign_in, ..Default::default() })) } + pub fn from_connection_string(connection_string: &str) -> Result { + Self::create(Arc::new( + ConnectionString::from_str(connection_string)?.into(), + )) + } + /// Create a new TCP client based on the provided configuration. pub fn create(config: Arc) -> Result { Ok(Self { config, stream: Mutex::new(None), state: Mutex::new(ClientState::Disconnected), + events: broadcast(1000), + connected_at: Mutex::new(None), }) } @@ -264,8 +280,32 @@ impl TcpClient { } async fn connect(&self) -> Result<(), IggyError> { - if self.get_state().await == ClientState::Connected { - return Ok(()); + match self.get_state().await { + ClientState::Connected | ClientState::Authenticating | ClientState::Authenticated => { + trace!("Client is already connected."); + return Ok(()); + } + ClientState::Connecting => { + trace!("Client is already connecting."); + return Ok(()); + } + _ => {} + } + + self.set_state(ClientState::Connecting).await; + if let Some(connected_at) = self.connected_at.lock().await.as_ref() { + let now = IggyTimestamp::now(); + let elapsed = now.as_micros() - connected_at.as_micros(); + let interval = self.config.reconnection.re_establish_after.as_micros(); + trace!( + "Elapsed time since last connection: {}", + IggyDuration::from(elapsed) + ); + if elapsed < interval { + let remaining = IggyDuration::from(interval - elapsed); + info!("Trying to connect to the server in: {remaining}",); + sleep(remaining.get_duration()).await; + } } let tls_enabled = self.config.tls_enabled; @@ -274,8 +314,8 @@ impl TcpClient { let remote_address; loop { info!( - "{} client is connecting to server: {}...", - NAME, self.config.server_address + "{NAME} client is connecting to server: {}...", + self.config.server_address ); let connection = TcpStream::connect(&self.config.server_address).await; @@ -284,20 +324,34 @@ impl TcpClient { "Failed to connect to server: {}", self.config.server_address ); - if retry_count < self.config.reconnection_retries { + if !self.config.reconnection.enabled { + warn!("Automatic reconnection is disabled."); + return Err(IggyError::CannotEstablishConnection); + } + + let unlimited_retries = self.config.reconnection.max_retries.is_none(); + let max_retries = self.config.reconnection.max_retries.unwrap_or_default(); + let max_retries_str = + if let Some(max_retries) = self.config.reconnection.max_retries { + max_retries.to_string() + } else { + "unlimited".to_string() + }; + + let interval_str = self.config.reconnection.interval.as_human_time_string(); + if unlimited_retries || retry_count < max_retries { retry_count += 1; info!( - "Retrying to connect to server ({}/{}): {} in: {} ms...", - retry_count, - self.config.reconnection_retries, + "Retrying to connect to server ({retry_count}/{max_retries_str}): {} in: {interval_str}", self.config.server_address, - self.config.reconnection_interval ); - sleep(Duration::from_millis(self.config.reconnection_interval)).await; + sleep(self.config.reconnection.interval.get_duration()).await; continue; } - return Err(IggyError::NotConnected); + self.set_state(ClientState::Disconnected).await; + self.publish_event(DiagnosticEvent::Disconnected).await; + return Err(IggyError::CannotEstablishConnection); } let stream = connection.unwrap(); @@ -308,28 +362,55 @@ impl TcpClient { break; } - let connector = - tokio_native_tls::TlsConnector::from(TlsConnector::builder().build().unwrap()); + let connector = tokio_native_tls::TlsConnector::from( + TlsConnector::builder().build().map_err(|error| { + error!("Failed to create a TLS connector: {error}"); + IggyError::CannotEstablishConnection + })?, + ); let stream = tokio_native_tls::TlsConnector::connect( &connector, &self.config.tls_domain, stream, ) .await - .unwrap(); + .map_err(|error| { + error!("Failed to establish a TLS connection: {error}"); + IggyError::CannotEstablishConnection + })?; + connection_stream = Box::new(TcpTlsConnectionStream { stream }); break; } + let now = IggyTimestamp::now(); + info!("{NAME} client has connected to server: {remote_address} at: {now}",); self.stream.lock().await.replace(connection_stream); self.set_state(ClientState::Connected).await; - - info!( - "{} client has connected to server: {}", - NAME, remote_address - ); - - Ok(()) + self.connected_at.lock().await.replace(now); + self.publish_event(DiagnosticEvent::Connected).await; + match &self.config.auto_login { + AutoLogin::Disabled => { + info!("Automatic sign-in is disabled."); + Ok(()) + } + AutoLogin::Enabled(credentials) => { + info!("{NAME} client is signing in..."); + self.set_state(ClientState::Authenticating).await; + match credentials { + Credentials::UsernamePassword(username, password) => { + self.login_user(username, password).await?; + info!("{NAME} client has signed in with the user credentials, username: {username}",); + Ok(()) + } + Credentials::PersonalAccessToken(token) => { + self.login_with_personal_access_token(token).await?; + info!("{NAME} client has signed in with a personal access token.",); + Ok(()) + } + } + } + } } async fn disconnect(&self) -> Result<(), IggyError> { @@ -337,10 +418,59 @@ impl TcpClient { return Ok(()); } - info!("{} client is disconnecting from server...", NAME); + info!("{NAME} client is disconnecting from server..."); self.set_state(ClientState::Disconnected).await; self.stream.lock().await.take(); - info!("{} client has disconnected from server.", NAME); + self.publish_event(DiagnosticEvent::Disconnected).await; + let now = IggyTimestamp::now(); + info!("{NAME} client has disconnected from server at: {now}."); Ok(()) } + + async fn send_raw(&self, code: u32, payload: Bytes) -> Result { + match self.get_state().await { + ClientState::Disconnected => { + trace!("Cannot send data. Client is not connected."); + return Err(IggyError::NotConnected); + } + ClientState::Connecting => { + trace!("Cannot send data. Client is still connecting."); + return Err(IggyError::NotConnected); + } + _ => {} + } + + let mut stream = self.stream.lock().await; + if let Some(stream) = stream.as_mut() { + let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; + trace!("Sending a TCP request with code: {code}"); + stream.write(&(payload_length as u32).to_le_bytes()).await?; + stream.write(&code.to_le_bytes()).await?; + stream.write(&payload).await?; + stream.flush().await?; + trace!("Sent a TCP request with code: {code}, waiting for a response..."); + + let mut response_buffer = [0u8; RESPONSE_INITIAL_BYTES_LENGTH]; + let read_bytes = stream.read(&mut response_buffer).await.map_err(|error| { + error!( + "Failed to read response for TCP request with code: {code}: {error}", + code = code, + error = error + ); + IggyError::Disconnected + })?; + + if read_bytes != RESPONSE_INITIAL_BYTES_LENGTH { + error!("Received an invalid or empty response."); + return Err(IggyError::EmptyResponse); + } + + let status = u32::from_le_bytes(response_buffer[..4].try_into()?); + let length = u32::from_le_bytes(response_buffer[4..].try_into()?); + return self.handle_response(status, length, stream.as_mut()).await; + } + + error!("Cannot send data. Client is not connected."); + Err(IggyError::NotConnected) + } } diff --git a/sdk/src/tcp/config.rs b/sdk/src/tcp/config.rs index 358b75403..c81e8be65 100644 --- a/sdk/src/tcp/config.rs +++ b/sdk/src/tcp/config.rs @@ -1,26 +1,49 @@ +use crate::client::AutoLogin; +use crate::utils::duration::IggyDuration; +use std::str::FromStr; + /// Configuration for the TCP client. #[derive(Debug, Clone)] pub struct TcpClientConfig { /// The address of the Iggy server. pub server_address: String, - /// The number of retries when connecting to the server. - pub reconnection_retries: u32, - /// The interval between retries when connecting to the server. - pub reconnection_interval: u64, /// Whether to use TLS when connecting to the server. pub tls_enabled: bool, /// The domain to use for TLS when connecting to the server. pub tls_domain: String, + /// Whether to automatically login user after establishing connection. + pub auto_login: AutoLogin, + // Whether to automatically reconnect when disconnected. + pub reconnection: TcpClientReconnectionConfig, +} + +#[derive(Debug, Clone)] +pub struct TcpClientReconnectionConfig { + pub enabled: bool, + pub max_retries: Option, + pub interval: IggyDuration, + pub re_establish_after: IggyDuration, } impl Default for TcpClientConfig { fn default() -> TcpClientConfig { TcpClientConfig { server_address: "127.0.0.1:8090".to_string(), - reconnection_retries: 3, - reconnection_interval: 1000, tls_enabled: false, tls_domain: "localhost".to_string(), + auto_login: AutoLogin::Disabled, + reconnection: TcpClientReconnectionConfig::default(), + } + } +} + +impl Default for TcpClientReconnectionConfig { + fn default() -> TcpClientReconnectionConfig { + TcpClientReconnectionConfig { + enabled: true, + max_retries: None, + interval: IggyDuration::from_str("1s").unwrap(), + re_establish_after: IggyDuration::from_str("5s").unwrap(), } } } @@ -28,8 +51,8 @@ impl Default for TcpClientConfig { /// Builder for the TCP client configuration. /// Allows configuring the TCP client with custom settings or using defaults: /// - `server_address`: Default is "127.0.0.1:8090" -/// - `reconnection_retries`: Default is 3. -/// - `reconnection_interval`: Default is 1000 ms. +/// - `auto_login`: Default is AutoLogin::Disabled. +/// - `reconnection`: Default is enabled unlimited retries and 1 second interval. /// - `tls_enabled`: Default is false. /// - `tls_domain`: Default is "localhost". #[derive(Debug, Default)] @@ -48,15 +71,26 @@ impl TcpClientConfigBuilder { self } + /// Sets the auto sign in during connection. + pub fn with_auto_sign_in(mut self, auto_sign_in: AutoLogin) -> Self { + self.config.auto_login = auto_sign_in; + self + } + + pub fn with_enabled_reconnection(mut self) -> Self { + self.config.reconnection.enabled = true; + self + } + /// Sets the number of retries when connecting to the server. - pub fn with_reconnection_retries(mut self, reconnection_retries: u32) -> Self { - self.config.reconnection_retries = reconnection_retries; + pub fn with_reconnection_max_retries(mut self, max_retries: Option) -> Self { + self.config.reconnection.max_retries = max_retries; self } /// Sets the interval between retries when connecting to the server. - pub fn with_reconnection_interval(mut self, reconnection_interval: u64) -> Self { - self.config.reconnection_interval = reconnection_interval; + pub fn with_reconnection_interval(mut self, interval: IggyDuration) -> Self { + self.config.reconnection.interval = interval; self } diff --git a/server/Cargo.toml b/server/Cargo.toml index c73544545..bff959525 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "server" -version = "0.3.6" +version = "0.4.0" edition = "2021" build = "src/build.rs" diff --git a/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs index a324150c7..cf8e76a18 100644 --- a/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs +++ b/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs @@ -15,7 +15,7 @@ pub async fn handle( system: &SharedSystem, ) -> Result<(), IggyError> { debug!("session: {session}, command: {command}"); - let consumer_group_bytes; + let response; { let mut system = system.write().await; let consumer_group = system @@ -28,7 +28,7 @@ pub async fn handle( ) .await?; let consumer_group = consumer_group.read().await; - consumer_group_bytes = mapper::map_consumer_group(&consumer_group).await; + response = mapper::map_consumer_group(&consumer_group).await; } let system = system.read().await; system @@ -38,6 +38,6 @@ pub async fn handle( EntryCommand::CreateConsumerGroup(command), ) .await?; - sender.send_ok_response(&consumer_group_bytes).await?; + sender.send_ok_response(&response).await?; Ok(()) } diff --git a/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs b/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs index 8ccdbec76..96e4b2bae 100644 --- a/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs +++ b/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs @@ -1,6 +1,5 @@ use crate::binary::mapper; use crate::binary::sender::Sender; -use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::session::Session; use crate::streaming::systems::system::SharedSystem; use anyhow::Result; @@ -16,10 +15,14 @@ pub async fn handle( ) -> Result<(), IggyError> { debug!("session: {session}, command: {command}"); let system = system.read().await; - let consumer = - PollingConsumer::from_consumer(&command.consumer, session.client_id, command.partition_id); let offset = system - .get_consumer_offset(session, consumer, &command.stream_id, &command.topic_id) + .get_consumer_offset( + session, + &command.consumer, + &command.stream_id, + &command.topic_id, + command.partition_id, + ) .await?; let offset = mapper::map_consumer_offset(&offset); sender.send_ok_response(&offset).await?; diff --git a/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs b/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs index 736b2ea5e..5c7ab4033 100644 --- a/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs +++ b/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs @@ -1,5 +1,4 @@ use crate::binary::sender::Sender; -use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::session::Session; use crate::streaming::systems::system::SharedSystem; use anyhow::Result; @@ -15,15 +14,13 @@ pub async fn handle( ) -> Result<(), IggyError> { debug!("session: {session}, command: {command}"); let system = system.read().await; - let consumer = - PollingConsumer::from_consumer(&command.consumer, session.client_id, command.partition_id); - system .store_consumer_offset( session, - consumer, + command.consumer, &command.stream_id, &command.topic_id, + command.partition_id, command.offset, ) .await?; diff --git a/server/src/binary/handlers/messages/poll_messages_handler.rs b/server/src/binary/handlers/messages/poll_messages_handler.rs index a419674a2..24174ffdf 100644 --- a/server/src/binary/handlers/messages/poll_messages_handler.rs +++ b/server/src/binary/handlers/messages/poll_messages_handler.rs @@ -1,6 +1,5 @@ use crate::binary::mapper; use crate::binary::sender::Sender; -use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::session::Session; use crate::streaming::systems::messages::PollingArgs; use crate::streaming::systems::system::SharedSystem; @@ -16,15 +15,14 @@ pub async fn handle( system: &SharedSystem, ) -> Result<(), IggyError> { debug!("session: {session}, command: {command}"); - let consumer = - PollingConsumer::from_consumer(&command.consumer, session.client_id, command.partition_id); let system = system.read().await; let messages = system .poll_messages( session, - consumer, + &command.consumer, &command.stream_id, &command.topic_id, + command.partition_id, PollingArgs::new(command.strategy, command.count, command.auto_commit), ) .await?; diff --git a/server/src/binary/handlers/streams/create_stream_handler.rs b/server/src/binary/handlers/streams/create_stream_handler.rs index 5fdb4376f..e9320ab80 100644 --- a/server/src/binary/handlers/streams/create_stream_handler.rs +++ b/server/src/binary/handlers/streams/create_stream_handler.rs @@ -1,3 +1,4 @@ +use crate::binary::mapper; use crate::binary::sender::Sender; use crate::state::command::EntryCommand; use crate::streaming::session::Session; @@ -14,11 +15,13 @@ pub async fn handle( system: &SharedSystem, ) -> Result<(), IggyError> { debug!("session: {session}, command: {command}"); + let response; { let mut system = system.write().await; - system + let stream = system .create_stream(session, command.stream_id, &command.name) .await?; + response = mapper::map_stream(stream); } let system = system.read().await; @@ -26,6 +29,6 @@ pub async fn handle( .state .apply(session.get_user_id(), EntryCommand::CreateStream(command)) .await?; - sender.send_empty_ok_response().await?; + sender.send_ok_response(&response).await?; Ok(()) } diff --git a/server/src/binary/handlers/streams/get_stream_handler.rs b/server/src/binary/handlers/streams/get_stream_handler.rs index 798828dc7..976fa6661 100644 --- a/server/src/binary/handlers/streams/get_stream_handler.rs +++ b/server/src/binary/handlers/streams/get_stream_handler.rs @@ -16,7 +16,7 @@ pub async fn handle( debug!("session: {session}, command: {command}"); let system = system.read().await; let stream = system.find_stream(session, &command.stream_id)?; - let stream = mapper::map_stream(stream).await; - sender.send_ok_response(&stream).await?; + let response = mapper::map_stream(stream); + sender.send_ok_response(&response).await?; Ok(()) } diff --git a/server/src/binary/handlers/streams/get_streams_handler.rs b/server/src/binary/handlers/streams/get_streams_handler.rs index 0b6a723aa..6009b7d00 100644 --- a/server/src/binary/handlers/streams/get_streams_handler.rs +++ b/server/src/binary/handlers/streams/get_streams_handler.rs @@ -16,7 +16,7 @@ pub async fn handle( debug!("session: {session}, command: {command}"); let system = system.read().await; let streams = system.find_streams(session)?; - let streams = mapper::map_streams(&streams).await; - sender.send_ok_response(&streams).await?; + let response = mapper::map_streams(&streams); + sender.send_ok_response(&response).await?; Ok(()) } diff --git a/server/src/binary/handlers/system/get_client_handler.rs b/server/src/binary/handlers/system/get_client_handler.rs index d77c54418..088ba7847 100644 --- a/server/src/binary/handlers/system/get_client_handler.rs +++ b/server/src/binary/handlers/system/get_client_handler.rs @@ -20,7 +20,7 @@ pub async fn handle( let client = system.get_client(session, command.client_id).await?; { let client = client.read().await; - bytes = mapper::map_client(&client).await; + bytes = mapper::map_client(&client); } } sender.send_ok_response(&bytes).await?; diff --git a/server/src/binary/handlers/system/get_me_handler.rs b/server/src/binary/handlers/system/get_me_handler.rs index a6d62af89..834c2ac79 100644 --- a/server/src/binary/handlers/system/get_me_handler.rs +++ b/server/src/binary/handlers/system/get_me_handler.rs @@ -20,7 +20,7 @@ pub async fn handle( let client = system.get_client(session, session.client_id).await?; { let client = client.read().await; - bytes = mapper::map_client(&client).await; + bytes = mapper::map_client(&client); } } sender.send_ok_response(&bytes).await?; diff --git a/server/src/binary/handlers/topics/create_topic_handler.rs b/server/src/binary/handlers/topics/create_topic_handler.rs index 5decd41bd..c9e53bb4c 100644 --- a/server/src/binary/handlers/topics/create_topic_handler.rs +++ b/server/src/binary/handlers/topics/create_topic_handler.rs @@ -1,3 +1,4 @@ +use crate::binary::mapper; use crate::binary::sender::Sender; use crate::state::command::EntryCommand; use crate::streaming::session::Session; @@ -14,9 +15,10 @@ pub async fn handle( system: &SharedSystem, ) -> Result<(), IggyError> { debug!("session: {session}, command: {command}"); + let response; { let mut system = system.write().await; - system + let topic = system .create_topic( session, &command.stream_id, @@ -29,6 +31,7 @@ pub async fn handle( command.replication_factor, ) .await?; + response = mapper::map_topic(topic).await; } let system = system.read().await; @@ -36,6 +39,6 @@ pub async fn handle( .state .apply(session.get_user_id(), EntryCommand::CreateTopic(command)) .await?; - sender.send_empty_ok_response().await?; + sender.send_ok_response(&response).await?; Ok(()) } diff --git a/server/src/binary/handlers/topics/get_topics_handler.rs b/server/src/binary/handlers/topics/get_topics_handler.rs index 282f28e04..891c440e1 100644 --- a/server/src/binary/handlers/topics/get_topics_handler.rs +++ b/server/src/binary/handlers/topics/get_topics_handler.rs @@ -16,7 +16,7 @@ pub async fn handle( debug!("session: {session}, command: {command}"); let system = system.read().await; let topics = system.find_topics(session, &command.stream_id)?; - let topics = mapper::map_topics(&topics).await; - sender.send_ok_response(&topics).await?; + let response = mapper::map_topics(&topics); + sender.send_ok_response(&response).await?; Ok(()) } diff --git a/server/src/binary/handlers/users/create_user_handler.rs b/server/src/binary/handlers/users/create_user_handler.rs index e570c94ed..4aecc9545 100644 --- a/server/src/binary/handlers/users/create_user_handler.rs +++ b/server/src/binary/handlers/users/create_user_handler.rs @@ -1,3 +1,4 @@ +use crate::binary::mapper; use crate::binary::sender::Sender; use crate::state::command::EntryCommand; use crate::streaming::session::Session; @@ -15,9 +16,10 @@ pub async fn handle( system: &SharedSystem, ) -> Result<(), IggyError> { debug!("session: {session}, command: {command}"); + let response; { let mut system = system.write().await; - system + let user = system .create_user( session, &command.username, @@ -26,6 +28,7 @@ pub async fn handle( command.permissions.clone(), ) .await?; + response = mapper::map_user(user); } // For the security of the system, we hash the password before storing it in metadata. @@ -42,6 +45,6 @@ pub async fn handle( }), ) .await?; - sender.send_empty_ok_response().await?; + sender.send_ok_response(&response).await?; Ok(()) } diff --git a/server/src/binary/mapper.rs b/server/src/binary/mapper.rs index 4d042eeba..3491bfd72 100644 --- a/server/src/binary/mapper.rs +++ b/server/src/binary/mapper.rs @@ -53,7 +53,7 @@ pub fn map_consumer_offset(offset: &ConsumerOffsetInfo) -> Bytes { bytes.freeze() } -pub async fn map_client(client: &Client) -> Bytes { +pub fn map_client(client: &Client) -> Bytes { let mut bytes = BytesMut::new(); extend_client(client, &mut bytes); for consumer_group in &client.consumer_groups { @@ -136,34 +136,34 @@ pub fn map_polled_messages(polled_messages: &PolledMessages) -> Bytes { bytes.freeze() } -pub async fn map_stream(stream: &Stream) -> Bytes { +pub fn map_stream(stream: &Stream) -> Bytes { let mut bytes = BytesMut::new(); - extend_stream(stream, &mut bytes).await; + extend_stream(stream, &mut bytes); for topic in stream.get_topics() { - extend_topic(topic, &mut bytes).await; + extend_topic(topic, &mut bytes); } bytes.freeze() } -pub async fn map_streams(streams: &[&Stream]) -> Bytes { +pub fn map_streams(streams: &[&Stream]) -> Bytes { let mut bytes = BytesMut::new(); for stream in streams { - extend_stream(stream, &mut bytes).await; + extend_stream(stream, &mut bytes); } bytes.freeze() } -pub async fn map_topics(topics: &[&Topic]) -> Bytes { +pub fn map_topics(topics: &[&Topic]) -> Bytes { let mut bytes = BytesMut::new(); for topic in topics { - extend_topic(topic, &mut bytes).await; + extend_topic(topic, &mut bytes); } bytes.freeze() } pub async fn map_topic(topic: &Topic) -> Bytes { let mut bytes = BytesMut::new(); - extend_topic(topic, &mut bytes).await; + extend_topic(topic, &mut bytes); for partition in topic.get_partitions() { let partition = partition.read().await; extend_partition(&partition, &mut bytes); @@ -196,7 +196,7 @@ pub async fn map_consumer_groups(consumer_groups: &[&RwLock]) -> bytes.freeze() } -async fn extend_stream(stream: &Stream, bytes: &mut BytesMut) { +fn extend_stream(stream: &Stream, bytes: &mut BytesMut) { bytes.put_u32_le(stream.stream_id); bytes.put_u64_le(stream.created_at.into()); bytes.put_u32_le(stream.get_topics().len() as u32); @@ -206,7 +206,7 @@ async fn extend_stream(stream: &Stream, bytes: &mut BytesMut) { bytes.put_slice(stream.name.as_bytes()); } -async fn extend_topic(topic: &Topic, bytes: &mut BytesMut) { +fn extend_topic(topic: &Topic, bytes: &mut BytesMut) { bytes.put_u32_le(topic.topic_id); bytes.put_u64_le(topic.created_at.into()); bytes.put_u32_le(topic.get_partitions().len() as u32); diff --git a/server/src/configs/defaults.rs b/server/src/configs/defaults.rs index 71fea1c39..b150bf465 100644 --- a/server/src/configs/defaults.rs +++ b/server/src/configs/defaults.rs @@ -10,7 +10,7 @@ use crate::configs::server::{ use crate::configs::system::{ BackupConfig, CacheConfig, CompatibilityConfig, CompressionConfig, EncryptionConfig, LoggingConfig, MessageDeduplicationConfig, PartitionConfig, RuntimeConfig, SegmentConfig, - StreamConfig, SystemConfig, TopicConfig, + StateConfig, StreamConfig, SystemConfig, TopicConfig, }; use crate::configs::tcp::{TcpConfig, TcpTlsConfig}; use std::sync::Arc; @@ -279,6 +279,7 @@ impl Default for SystemConfig { topic: TopicConfig::default(), partition: PartitionConfig::default(), segment: SegmentConfig::default(), + state: StateConfig::default(), compression: CompressionConfig::default(), message_deduplication: MessageDeduplicationConfig::default(), } @@ -407,6 +408,14 @@ impl Default for SegmentConfig { } } +impl Default for StateConfig { + fn default() -> StateConfig { + StateConfig { + enforce_fsync: SERVER_CONFIG.system.state.enforce_fsync, + } + } +} + impl Default for MessageDeduplicationConfig { fn default() -> MessageDeduplicationConfig { MessageDeduplicationConfig { diff --git a/server/src/configs/system.rs b/server/src/configs/system.rs index c02c76a55..5660dc91d 100644 --- a/server/src/configs/system.rs +++ b/server/src/configs/system.rs @@ -14,6 +14,7 @@ pub struct SystemConfig { pub path: String, pub backup: BackupConfig, pub database: Option, + pub state: StateConfig, pub runtime: RuntimeConfig, pub logging: LoggingConfig, pub cache: CacheConfig, @@ -119,6 +120,11 @@ pub struct SegmentConfig { pub archive_expired: bool, } +#[derive(Debug, Deserialize, Serialize)] +pub struct StateConfig { + pub enforce_fsync: bool, +} + impl SystemConfig { pub fn get_system_path(&self) -> String { self.path.to_string() diff --git a/server/src/http/consumer_offsets.rs b/server/src/http/consumer_offsets.rs index 925b87360..16da2a618 100644 --- a/server/src/http/consumer_offsets.rs +++ b/server/src/http/consumer_offsets.rs @@ -1,12 +1,12 @@ use crate::http::error::CustomError; use crate::http::jwt::json_web_token::Identity; use crate::http::shared::AppState; -use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::session::Session; use axum::extract::{Path, Query, State}; use axum::http::StatusCode; use axum::routing::get; use axum::{Extension, Json, Router}; +use iggy::consumer::Consumer; use iggy::consumer_offsets::get_consumer_offset::GetConsumerOffset; use iggy::consumer_offsets::store_consumer_offset::StoreConsumerOffset; use iggy::identifier::Identifier; @@ -32,15 +32,15 @@ async fn get_consumer_offset( query.stream_id = Identifier::from_str_value(&stream_id)?; query.topic_id = Identifier::from_str_value(&topic_id)?; query.validate()?; - let consumer_id = PollingConsumer::resolve_consumer_id(&query.consumer.id); - let consumer = PollingConsumer::Consumer(consumer_id, query.partition_id.unwrap_or(0)); + let consumer = Consumer::new(query.0.consumer.id); let system = state.system.read().await; let offset = system .get_consumer_offset( &Session::stateless(identity.user_id, identity.ip_address), - consumer, - &query.stream_id, - &query.topic_id, + &consumer, + &query.0.stream_id, + &query.0.topic_id, + query.0.partition_id, ) .await?; Ok(Json(offset)) @@ -55,16 +55,16 @@ async fn store_consumer_offset( command.stream_id = Identifier::from_str_value(&stream_id)?; command.topic_id = Identifier::from_str_value(&topic_id)?; command.validate()?; - let consumer_id = PollingConsumer::resolve_consumer_id(&command.consumer.id); - let consumer = PollingConsumer::Consumer(consumer_id, command.partition_id.unwrap_or(0)); + let consumer = Consumer::new(command.0.consumer.id); let system = state.system.read().await; system .store_consumer_offset( &Session::stateless(identity.user_id, identity.ip_address), consumer, - &command.stream_id, - &command.topic_id, - command.offset, + &command.0.stream_id, + &command.0.topic_id, + command.0.partition_id, + command.0.offset, ) .await?; Ok(StatusCode::NO_CONTENT) diff --git a/server/src/http/mapper.rs b/server/src/http/mapper.rs index e14a6d39f..aac1c9a0e 100644 --- a/server/src/http/mapper.rs +++ b/server/src/http/mapper.rs @@ -16,8 +16,8 @@ use iggy::models::topic::TopicDetails; use iggy::models::user_info::{UserInfo, UserInfoDetails}; use tokio::sync::RwLock; -pub async fn map_stream(stream: &Stream) -> StreamDetails { - let topics = map_topics(&stream.get_topics()).await; +pub fn map_stream(stream: &Stream) -> StreamDetails { + let topics = map_topics(&stream.get_topics()); let mut stream_details = StreamDetails { id: stream.stream_id, created_at: stream.created_at, @@ -31,7 +31,7 @@ pub async fn map_stream(stream: &Stream) -> StreamDetails { stream_details } -pub async fn map_streams(streams: &[&Stream]) -> Vec { +pub fn map_streams(streams: &[&Stream]) -> Vec { let mut streams_data = Vec::with_capacity(streams.len()); for stream in streams { let stream = iggy::models::stream::Stream { @@ -49,7 +49,7 @@ pub async fn map_streams(streams: &[&Stream]) -> Vec Vec { +pub fn map_topics(topics: &[&Topic]) -> Vec { let mut topics_data = Vec::with_capacity(topics.len()); for topic in topics { let topic = iggy::models::topic::Topic { @@ -141,7 +141,7 @@ pub fn map_personal_access_tokens( personal_access_tokens_data } -pub async fn map_client(client: &Client) -> iggy::models::client_info::ClientInfoDetails { +pub fn map_client(client: &Client) -> iggy::models::client_info::ClientInfoDetails { let client = iggy::models::client_info::ClientInfoDetails { client_id: client.client_id, user_id: client.user_id, diff --git a/server/src/http/messages.rs b/server/src/http/messages.rs index b1f75546b..5f660cde7 100644 --- a/server/src/http/messages.rs +++ b/server/src/http/messages.rs @@ -1,7 +1,6 @@ use crate::http::error::CustomError; use crate::http::jwt::json_web_token::Identity; use crate::http::shared::AppState; -use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::session::Session; use crate::streaming::systems::messages::PollingArgs; use crate::streaming::utils::random_id; @@ -9,6 +8,7 @@ use axum::extract::{Path, Query, State}; use axum::http::StatusCode; use axum::routing::get; use axum::{Extension, Json, Router}; +use iggy::consumer::Consumer; use iggy::identifier::Identifier; use iggy::messages::poll_messages::PollMessages; use iggy::messages::send_messages::SendMessages; @@ -35,17 +35,16 @@ async fn poll_messages( query.topic_id = Identifier::from_str_value(&topic_id)?; query.validate()?; - let partition_id = query.partition_id.unwrap_or(0); - let consumer_id = PollingConsumer::resolve_consumer_id(&query.consumer.id); - let consumer = PollingConsumer::Consumer(consumer_id, partition_id); + let consumer = Consumer::new(query.0.consumer.id); let system = state.system.read().await; let polled_messages = system .poll_messages( &Session::stateless(identity.user_id, identity.ip_address), - consumer, - &query.stream_id, - &query.topic_id, - PollingArgs::new(query.strategy, query.count, query.auto_commit), + &consumer, + &query.0.stream_id, + &query.0.topic_id, + query.0.partition_id, + PollingArgs::new(query.0.strategy, query.0.count, query.0.auto_commit), ) .await?; Ok(Json(polled_messages)) diff --git a/server/src/http/streams.rs b/server/src/http/streams.rs index 27a13a551..582497219 100644 --- a/server/src/http/streams.rs +++ b/server/src/http/streams.rs @@ -40,7 +40,7 @@ async fn get_stream( &Session::stateless(identity.user_id, identity.ip_address), &stream_id, )?; - let stream = mapper::map_stream(stream).await; + let stream = mapper::map_stream(stream); Ok(Json(stream)) } @@ -51,7 +51,7 @@ async fn get_streams( let system = state.system.read().await; let streams = system.find_streams(&Session::stateless(identity.user_id, identity.ip_address))?; - let streams = mapper::map_streams(&streams).await; + let streams = mapper::map_streams(&streams); Ok(Json(streams)) } @@ -59,17 +59,19 @@ async fn create_stream( State(state): State>, Extension(identity): Extension, Json(command): Json, -) -> Result { +) -> Result, CustomError> { command.validate()?; + let response; { let mut system = state.system.write().await; - system + let stream = system .create_stream( &Session::stateless(identity.user_id, identity.ip_address), command.stream_id, &command.name, ) .await?; + response = Json(mapper::map_stream(stream)); } let system = state.system.read().await; @@ -77,7 +79,7 @@ async fn create_stream( .state .apply(identity.user_id, EntryCommand::CreateStream(command)) .await?; - Ok(StatusCode::CREATED) + Ok(response) } async fn update_stream( diff --git a/server/src/http/system.rs b/server/src/http/system.rs index fad7f31c7..a6c81079a 100644 --- a/server/src/http/system.rs +++ b/server/src/http/system.rs @@ -58,7 +58,7 @@ async fn get_client( ) .await?; let client = client.read().await; - let client = mapper::map_client(&client).await; + let client = mapper::map_client(&client); Ok(Json(client)) } diff --git a/server/src/http/topics.rs b/server/src/http/topics.rs index b767ca3fb..859ebfcb5 100644 --- a/server/src/http/topics.rs +++ b/server/src/http/topics.rs @@ -62,7 +62,7 @@ async fn get_topics( &Session::stateless(identity.user_id, identity.ip_address), &stream_id, )?; - let topics = mapper::map_topics(&topics).await; + let topics = mapper::map_topics(&topics); Ok(Json(topics)) } @@ -71,12 +71,13 @@ async fn create_topic( Extension(identity): Extension, Path(stream_id): Path, Json(mut command): Json, -) -> Result { +) -> Result, CustomError> { command.stream_id = Identifier::from_str_value(&stream_id)?; command.validate()?; + let response; { let mut system = state.system.write().await; - system + let topic = system .create_topic( &Session::stateless(identity.user_id, identity.ip_address), &command.stream_id, @@ -89,6 +90,7 @@ async fn create_topic( command.replication_factor, ) .await?; + response = Json(mapper::map_topic(topic).await); } let system = state.system.read().await; @@ -96,7 +98,7 @@ async fn create_topic( .state .apply(identity.user_id, EntryCommand::CreateTopic(command)) .await?; - Ok(StatusCode::CREATED) + Ok(response) } async fn update_topic( diff --git a/server/src/http/users.rs b/server/src/http/users.rs index 34dd24246..dd3636791 100644 --- a/server/src/http/users.rs +++ b/server/src/http/users.rs @@ -69,11 +69,12 @@ async fn create_user( State(state): State>, Extension(identity): Extension, Json(command): Json, -) -> Result { +) -> Result, CustomError> { command.validate()?; + let response; { let mut system = state.system.write().await; - system + let user = system .create_user( &Session::stateless(identity.user_id, identity.ip_address), &command.username, @@ -82,6 +83,7 @@ async fn create_user( command.permissions.clone(), ) .await?; + response = Json(mapper::map_user(user)); } // For the security of the system, we hash the password before storing it in metadata. @@ -98,7 +100,8 @@ async fn create_user( }), ) .await?; - Ok(StatusCode::NO_CONTENT) + + Ok(response) } async fn update_user( diff --git a/server/src/streaming/polling_consumer.rs b/server/src/streaming/polling_consumer.rs index ebcd6431c..253bc9401 100644 --- a/server/src/streaming/polling_consumer.rs +++ b/server/src/streaming/polling_consumer.rs @@ -1,5 +1,4 @@ use crate::streaming::utils::hash; -use iggy::consumer::{Consumer, ConsumerKind}; use iggy::identifier::{IdKind, Identifier}; use std::fmt::{Display, Formatter}; @@ -10,14 +9,12 @@ pub enum PollingConsumer { } impl PollingConsumer { - pub fn from_consumer(consumer: &Consumer, client_id: u32, partition_id: Option) -> Self { - let consumer_id = Self::resolve_consumer_id(&consumer.id); - match consumer.kind { - ConsumerKind::Consumer => { - PollingConsumer::Consumer(consumer_id, partition_id.unwrap_or(0)) - } - ConsumerKind::ConsumerGroup => PollingConsumer::ConsumerGroup(consumer_id, client_id), - } + pub fn consumer(consumer_id: &Identifier, partition_id: u32) -> Self { + PollingConsumer::Consumer(Self::resolve_consumer_id(consumer_id), partition_id) + } + + pub fn consumer_group(consumer_group_id: u32, member_id: u32) -> Self { + PollingConsumer::ConsumerGroup(consumer_group_id, member_id) } pub fn resolve_consumer_id(identifier: &Identifier) -> u32 { @@ -50,40 +47,35 @@ impl Display for PollingConsumer { #[cfg(test)] mod tests { use super::*; + use iggy::consumer::Consumer; #[test] fn given_consumer_with_numeric_id_polling_consumer_should_be_created() { - let consumer_id = 1; - let client_id = 2; + let consumer_id_value = 1; let partition_id = 3; - let consumer = Consumer::new(Identifier::numeric(consumer_id).unwrap()); - let polling_consumer = - PollingConsumer::from_consumer(&consumer, client_id, Some(partition_id)); + let consumer_id = Identifier::numeric(consumer_id_value).unwrap(); + let consumer = Consumer::new(consumer_id); + let polling_consumer = PollingConsumer::consumer(&consumer.id, partition_id); assert_eq!( polling_consumer, - PollingConsumer::Consumer(consumer_id, partition_id) - ); - - assert_eq!( - consumer_id, - PollingConsumer::resolve_consumer_id(&consumer.id) + PollingConsumer::Consumer(consumer_id_value, partition_id) ); } #[test] fn given_consumer_with_named_id_polling_consumer_should_be_created() { let consumer_name = "consumer"; - let client_id = 2; let partition_id = 3; - let consumer = Consumer::new(Identifier::named(consumer_name).unwrap()); - let polling_consumer = - PollingConsumer::from_consumer(&consumer, client_id, Some(partition_id)); + let consumer_id = Identifier::named(consumer_name).unwrap(); + let consumer = Consumer::new(consumer_id); + + let resolved_consumer_id = PollingConsumer::resolve_consumer_id(&consumer.id); + let polling_consumer = PollingConsumer::consumer(&consumer.id, partition_id); - let consumer_id = PollingConsumer::resolve_consumer_id(&consumer.id); assert_eq!( polling_consumer, - PollingConsumer::Consumer(consumer_id, partition_id) + PollingConsumer::Consumer(resolved_consumer_id, partition_id) ); } @@ -91,28 +83,15 @@ mod tests { fn given_consumer_group_with_numeric_id_polling_consumer_group_should_be_created() { let group_id = 1; let client_id = 2; - let consumer = Consumer::group(Identifier::numeric(group_id).unwrap()); - let polling_consumer = PollingConsumer::from_consumer(&consumer, client_id, None); - - assert_eq!( - polling_consumer, - PollingConsumer::ConsumerGroup(group_id, client_id) - ); - assert_eq!(group_id, PollingConsumer::resolve_consumer_id(&consumer.id)); - } - - #[test] - fn given_consumer_group_with_named_id_polling_consumer_group_should_be_created() { - let consumer_group_name = "consumer_group"; - let client_id = 2; - let consumer = Consumer::group(Identifier::named(consumer_group_name).unwrap()); - let polling_consumer = PollingConsumer::from_consumer(&consumer, client_id, None); + let polling_consumer = PollingConsumer::consumer_group(group_id, client_id); - let consumer_id = PollingConsumer::resolve_consumer_id(&consumer.id); - assert_eq!( - polling_consumer, - PollingConsumer::ConsumerGroup(consumer_id, client_id) - ); + match polling_consumer { + PollingConsumer::ConsumerGroup(consumer_group_id, member_id) => { + assert_eq!(consumer_group_id, group_id); + assert_eq!(member_id, client_id); + } + _ => panic!("Expected ConsumerGroup"), + } } #[test] diff --git a/server/src/streaming/segments/storage.rs b/server/src/streaming/segments/storage.rs index 27fbe9e22..6c7419850 100644 --- a/server/src/streaming/segments/storage.rs +++ b/server/src/streaming/segments/storage.rs @@ -581,7 +581,7 @@ async fn load_batches_by_range( .seek(SeekFrom::Start(index_range.start.position as u64)) .await?; - let mut read_bytes: u64 = 0; + let mut read_bytes = index_range.start.position as u64; let mut last_batch_to_read = false; while !last_batch_to_read { let batch_base_offset = reader @@ -607,13 +607,15 @@ async fn load_batches_by_range( let payload_len = batch_length as usize; let mut payload = BytesMut::with_capacity(payload_len); payload.put_bytes(0, payload_len); - reader - .read_exact(&mut payload) - .await - .map_err(|_| IggyError::CannotReadBatchPayload)?; + if let Err(error) = reader.read_exact(&mut payload).await { + warn!( + "Cannot read batch payload for batch with base offset: {batch_base_offset}, last offset delta: {last_offset_delta}, max timestamp: {max_timestamp}, batch length: {batch_length} and payload length: {payload_len}.\nProbably OS hasn't flushed the data yet, try setting `enforce_fsync = true` for partition configuration if this issue occurs again.\n{error}", + ); + break; + } read_bytes += 8 + 4 + 4 + 8 + payload_len as u64; - last_batch_to_read = read_bytes == file_size || last_offset == index_last_offset; + last_batch_to_read = read_bytes >= file_size || last_offset == index_last_offset; let batch = RetainedMessageBatch::new( batch_base_offset, diff --git a/server/src/streaming/streams/topics.rs b/server/src/streaming/streams/topics.rs index a9464327a..ae4759c94 100644 --- a/server/src/streaming/streams/topics.rs +++ b/server/src/streaming/streams/topics.rs @@ -25,7 +25,7 @@ impl Stream { compression_algorithm: CompressionAlgorithm, max_topic_size: MaxTopicSize, replication_factor: u8, - ) -> Result<(), IggyError> { + ) -> Result { let name = text::to_lowercase_non_whitespace(name); if self.topics_ids.contains_key(&name) { return Err(IggyError::TopicNameAlreadyExists(name, self.stream_id)); @@ -71,8 +71,7 @@ impl Stream { info!("Created topic {}", topic); self.topics_ids.insert(name, id); self.topics.insert(id, topic); - - Ok(()) + Ok(id) } pub async fn update_topic( diff --git a/server/src/streaming/systems/consumer_offsets.rs b/server/src/streaming/systems/consumer_offsets.rs index bb96f809a..3ae9434ea 100644 --- a/server/src/streaming/systems/consumer_offsets.rs +++ b/server/src/streaming/systems/consumer_offsets.rs @@ -1,6 +1,6 @@ -use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::session::Session; use crate::streaming::systems::system::System; +use iggy::consumer::Consumer; use iggy::error::IggyError; use iggy::identifier::Identifier; use iggy::models::consumer_offset_info::ConsumerOffsetInfo; @@ -9,9 +9,10 @@ impl System { pub async fn store_consumer_offset( &self, session: &Session, - consumer: PollingConsumer, + consumer: Consumer, stream_id: &Identifier, topic_id: &Identifier, + partition_id: Option, offset: u64, ) -> Result<(), IggyError> { self.ensure_authenticated(session)?; @@ -23,15 +24,18 @@ impl System { topic.topic_id, )?; - topic.store_consumer_offset(consumer, offset).await + topic + .store_consumer_offset(consumer, offset, partition_id, session.client_id) + .await } pub async fn get_consumer_offset( &self, session: &Session, - consumer: PollingConsumer, + consumer: &Consumer, stream_id: &Identifier, topic_id: &Identifier, + partition_id: Option, ) -> Result { self.ensure_authenticated(session)?; let stream = self.get_stream(stream_id)?; @@ -42,6 +46,8 @@ impl System { topic.topic_id, )?; - topic.get_consumer_offset(consumer).await + topic + .get_consumer_offset(consumer, partition_id, session.client_id) + .await } } diff --git a/server/src/streaming/systems/messages.rs b/server/src/streaming/systems/messages.rs index dce89c668..78b69a32b 100644 --- a/server/src/streaming/systems/messages.rs +++ b/server/src/streaming/systems/messages.rs @@ -1,8 +1,8 @@ use crate::streaming::cache::memory_tracker::CacheMemoryTracker; -use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::session::Session; use crate::streaming::systems::system::System; use bytes::Bytes; +use iggy::consumer::Consumer; use iggy::messages::poll_messages::PollingStrategy; use iggy::messages::send_messages::Message; use iggy::messages::send_messages::Partitioning; @@ -14,9 +14,10 @@ impl System { pub async fn poll_messages( &self, session: &Session, - consumer: PollingConsumer, + consumer: &Consumer, stream_id: &Identifier, topic_id: &Identifier, + partition_id: Option, args: PollingArgs, ) -> Result { self.ensure_authenticated(session)?; @@ -33,16 +34,12 @@ impl System { return Err(IggyError::NoPartitions(topic.topic_id, topic.stream_id)); } - let partition_id = match consumer { - PollingConsumer::Consumer(_, partition_id) => partition_id, - PollingConsumer::ConsumerGroup(group_id, member_id) => { - let consumer_group = topic.get_consumer_group_by_id(group_id)?.read().await; - consumer_group.calculate_partition_id(member_id).await? - } - }; + let (polling_consumer, partition_id) = topic + .resolve_consumer_with_partition_id(consumer, session.client_id, partition_id, true) + .await?; let mut polled_messages = topic - .get_messages(consumer, partition_id, args.strategy, args.count) + .get_messages(polling_consumer, partition_id, args.strategy, args.count) .await?; if polled_messages.messages.is_empty() { @@ -52,7 +49,9 @@ impl System { let offset = polled_messages.messages.last().unwrap().offset; if args.auto_commit { trace!("Last offset: {} will be automatically stored for {}, stream: {}, topic: {}, partition: {}", offset, consumer, stream_id, topic_id, partition_id); - topic.store_consumer_offset(consumer, offset).await?; + topic + .store_consumer_offset_internal(polling_consumer, offset, partition_id) + .await?; } if self.encryptor.is_none() { @@ -111,6 +110,7 @@ impl System { match payload { Ok(payload) => { message.payload = Bytes::from(payload); + message.length = message.payload.len() as u32; batch_size_bytes += message.get_size_bytes() as u64; } Err(error) => { diff --git a/server/src/streaming/systems/streams.rs b/server/src/streaming/systems/streams.rs index 9d2e6ea95..7f1178604 100644 --- a/server/src/streaming/systems/streams.rs +++ b/server/src/streaming/systems/streams.rs @@ -205,7 +205,7 @@ impl System { session: &Session, stream_id: Option, name: &str, - ) -> Result<(), IggyError> { + ) -> Result<&Stream, IggyError> { self.ensure_authenticated(session)?; self.permissioner.create_stream(session.get_user_id())?; let name = text::to_lowercase_non_whitespace(name); @@ -240,7 +240,7 @@ impl System { self.streams_ids.insert(name, stream.stream_id); self.streams.insert(stream.stream_id, stream); self.metrics.increment_streams(1); - Ok(()) + self.get_stream_by_id(id) } pub async fn update_stream( diff --git a/server/src/streaming/systems/system.rs b/server/src/streaming/systems/system.rs index 598a88474..ca5638bd3 100644 --- a/server/src/streaming/systems/system.rs +++ b/server/src/streaming/systems/system.rs @@ -99,19 +99,18 @@ impl System { false => None, }; - let persister: Arc = match config.partition.enforce_fsync { - true => Arc::new(FileWithSyncPersister {}), - false => Arc::new(FilePersister {}), - }; + let state_persister = Self::resolve_persister(config.state.enforce_fsync); + let partition_persister = Self::resolve_persister(config.partition.enforce_fsync); + let state = Arc::new(FileState::new( &config.get_state_log_path(), &version, - persister.clone(), + state_persister, encryptor.clone(), )); Self::create( config.clone(), - SystemStorage::new(config, persister), + SystemStorage::new(config, partition_persister), state, encryptor, data_maintenance_config, @@ -119,6 +118,13 @@ impl System { ) } + fn resolve_persister(enforce_fsync: bool) -> Arc { + match enforce_fsync { + true => Arc::new(FileWithSyncPersister), + false => Arc::new(FilePersister), + } + } + pub fn create( system_config: Arc, storage: SystemStorage, diff --git a/server/src/streaming/systems/topics.rs b/server/src/streaming/systems/topics.rs index 5db5f2398..bfc9c5c1c 100644 --- a/server/src/streaming/systems/topics.rs +++ b/server/src/streaming/systems/topics.rs @@ -47,7 +47,7 @@ impl System { compression_algorithm: CompressionAlgorithm, max_topic_size: MaxTopicSize, replication_factor: Option, - ) -> Result<(), IggyError> { + ) -> Result<&Topic, IggyError> { self.ensure_authenticated(session)?; { let stream = self.get_stream(stream_id)?; @@ -55,7 +55,8 @@ impl System { .create_topic(session.get_user_id(), stream.stream_id)?; } - self.get_stream_mut(stream_id)? + let created_topic_id = self + .get_stream_mut(stream_id)? .create_topic( topic_id, name, @@ -66,10 +67,13 @@ impl System { replication_factor.unwrap_or(1), ) .await?; + self.metrics.increment_topics(1); self.metrics.increment_partitions(partitions_count); self.metrics.increment_segments(partitions_count); - Ok(()) + + self.get_stream(stream_id)? + .get_topic(&created_topic_id.try_into()?) } #[allow(clippy::too_many_arguments)] diff --git a/server/src/streaming/systems/users.rs b/server/src/streaming/systems/users.rs index 64ad47046..d823d5c6d 100644 --- a/server/src/streaming/systems/users.rs +++ b/server/src/streaming/systems/users.rs @@ -168,7 +168,7 @@ impl System { password: &str, status: UserStatus, permissions: Option, - ) -> Result<(), IggyError> { + ) -> Result<&User, IggyError> { self.ensure_authenticated(session)?; self.permissioner.create_user(session.get_user_id())?; let username = text::to_lowercase_non_whitespace(username); @@ -190,7 +190,7 @@ impl System { self.users.insert(user.id, user); info!("Created user: {username} with ID: {user_id}."); self.metrics.increment_users(1); - Ok(()) + self.get_user(&user_id.try_into()?) } pub async fn delete_user( diff --git a/server/src/streaming/topics/consumer_groups.rs b/server/src/streaming/topics/consumer_groups.rs index ebc278073..bb4c955c9 100644 --- a/server/src/streaming/topics/consumer_groups.rs +++ b/server/src/streaming/topics/consumer_groups.rs @@ -105,12 +105,11 @@ impl Topic { ConsumerGroup::new(self.topic_id, id, &name, self.partitions.len() as u32); self.consumer_groups.insert(id, RwLock::new(consumer_group)); self.consumer_groups_ids.insert(name, id); - let consumer_group = self.get_consumer_group_by_id(id)?; info!( "Created consumer group with ID: {} for topic with ID: {} and stream with ID: {}.", id, self.topic_id, self.stream_id ); - Ok(consumer_group) + self.get_consumer_group_by_id(id) } pub async fn delete_consumer_group( diff --git a/server/src/streaming/topics/consumer_offsets.rs b/server/src/streaming/topics/consumer_offsets.rs index 441f8f86f..4b612fe59 100644 --- a/server/src/streaming/topics/consumer_offsets.rs +++ b/server/src/streaming/topics/consumer_offsets.rs @@ -1,58 +1,55 @@ -use crate::streaming::partitions::partition::Partition; use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::topics::topic::Topic; +use iggy::consumer::Consumer; use iggy::error::IggyError; -use iggy::locking::IggySharedMut; use iggy::locking::IggySharedMutFn; use iggy::models::consumer_offset_info::ConsumerOffsetInfo; impl Topic { pub async fn store_consumer_offset( + &self, + consumer: Consumer, + offset: u64, + partition_id: Option, + client_id: u32, + ) -> Result<(), IggyError> { + let (polling_consumer, partition_id) = self + .resolve_consumer_with_partition_id(&consumer, client_id, partition_id, false) + .await?; + let partition = self.get_partition(partition_id)?; + let partition = partition.read().await; + partition + .store_consumer_offset(polling_consumer, offset) + .await + } + + pub async fn store_consumer_offset_internal( &self, consumer: PollingConsumer, offset: u64, + partition_id: u32, ) -> Result<(), IggyError> { - let partition = self.resolve_partition(consumer).await?; + let partition = self.get_partition(partition_id)?; let partition = partition.read().await; partition.store_consumer_offset(consumer, offset).await } pub async fn get_consumer_offset( &self, - consumer: PollingConsumer, + consumer: &Consumer, + partition_id: Option, + client_id: u32, ) -> Result { - let partition = self.resolve_partition(consumer).await?; + let (polling_consumer, partition_id) = self + .resolve_consumer_with_partition_id(consumer, client_id, partition_id, false) + .await?; + let partition = self.get_partition(partition_id)?; let partition = partition.read().await; - let offset = partition.get_consumer_offset(consumer).await?; + let offset = partition.get_consumer_offset(polling_consumer).await?; Ok(ConsumerOffsetInfo { partition_id: partition.partition_id, current_offset: partition.current_offset, stored_offset: offset, }) } - - async fn resolve_partition( - &self, - consumer: PollingConsumer, - ) -> Result<&IggySharedMut, IggyError> { - let partition_id = match consumer { - PollingConsumer::Consumer(_, partition_id) => Ok(partition_id), - PollingConsumer::ConsumerGroup(group_id, member_id) => { - let consumer_group = self.get_consumer_group_by_id(group_id)?.read().await; - consumer_group.get_current_partition_id(member_id).await - } - }?; - - let partition = self.partitions.get(&partition_id); - if partition.is_none() { - return Err(IggyError::PartitionNotFound( - partition_id, - self.topic_id, - self.stream_id, - )); - } - - let partition = partition.unwrap(); - Ok(partition) - } } diff --git a/server/src/streaming/topics/storage.rs b/server/src/streaming/topics/storage.rs index 466eb2148..bc739e0eb 100644 --- a/server/src/streaming/topics/storage.rs +++ b/server/src/streaming/topics/storage.rs @@ -44,18 +44,6 @@ impl TopicStorage for FileTopicStorage { topic.max_topic_size = state.max_topic_size; topic.replication_factor = state.replication_factor.unwrap_or(1); - for consumer_group in state.consumer_groups.into_values() { - let consumer_group = ConsumerGroup::new( - topic.topic_id, - consumer_group.id, - &consumer_group.name, - topic.get_partitions_count(), - ); - topic - .consumer_groups - .insert(consumer_group.group_id, RwLock::new(consumer_group)); - } - let dir_entries = fs::read_dir(&topic.partitions_path).await .with_context(|| format!("Failed to read partition with ID: {} for stream with ID: {} for topic with ID: {} and path: {}", topic.topic_id, topic.stream_id, topic.topic_id, &topic.partitions_path)); @@ -165,6 +153,21 @@ impl TopicStorage for FileTopicStorage { .insert(partition.partition_id, IggySharedMut::new(partition)); } + for consumer_group in state.consumer_groups.into_values() { + let consumer_group = ConsumerGroup::new( + topic.topic_id, + consumer_group.id, + &consumer_group.name, + topic.get_partitions_count(), + ); + topic + .consumer_groups_ids + .insert(consumer_group.name.to_owned(), consumer_group.group_id); + topic + .consumer_groups + .insert(consumer_group.group_id, RwLock::new(consumer_group)); + } + topic.load_messages_from_disk_to_cache().await?; info!("Loaded topic {topic}"); diff --git a/server/src/streaming/topics/topic.rs b/server/src/streaming/topics/topic.rs index 0ccaf3afc..c1d34a8dc 100644 --- a/server/src/streaming/topics/topic.rs +++ b/server/src/streaming/topics/topic.rs @@ -1,9 +1,11 @@ use crate::configs::system::SystemConfig; use crate::streaming::partitions::partition::Partition; +use crate::streaming::polling_consumer::PollingConsumer; use crate::streaming::storage::SystemStorage; use crate::streaming::topics::consumer_group::ConsumerGroup; use core::fmt; use iggy::compression::compression_algorithm::CompressionAlgorithm; +use iggy::consumer::{Consumer, ConsumerKind}; use iggy::error::IggyError; use iggy::locking::IggySharedMut; use iggy::utils::byte_size::IggyByteSize; @@ -169,6 +171,36 @@ impl Topic { )), } } + + pub async fn resolve_consumer_with_partition_id( + &self, + consumer: &Consumer, + client_id: u32, + partition_id: Option, + calculate_partition_id: bool, + ) -> Result<(PollingConsumer, u32), IggyError> { + match consumer.kind { + ConsumerKind::Consumer => { + let partition_id = partition_id.unwrap_or(1); + Ok(( + PollingConsumer::consumer(&consumer.id, partition_id), + partition_id, + )) + } + ConsumerKind::ConsumerGroup => { + let consumer_group = self.get_consumer_group(&consumer.id)?.read().await; + let partition_id = if calculate_partition_id { + consumer_group.calculate_partition_id(client_id).await? + } else { + consumer_group.get_current_partition_id(client_id).await? + }; + Ok(( + PollingConsumer::consumer_group(consumer_group.group_id, client_id), + partition_id, + )) + } + } + } } impl fmt::Display for Topic { diff --git a/server/src/streaming/users/permissioner_rules/consumer_groups.rs b/server/src/streaming/users/permissioner_rules/consumer_groups.rs index 5f73400d0..6c81d9410 100644 --- a/server/src/streaming/users/permissioner_rules/consumer_groups.rs +++ b/server/src/streaming/users/permissioner_rules/consumer_groups.rs @@ -8,7 +8,7 @@ impl Permissioner { stream_id: u32, topic_id: u32, ) -> Result<(), IggyError> { - self.update_topic(user_id, stream_id, topic_id) + self.get_topic(user_id, stream_id, topic_id) } pub fn delete_consumer_group( @@ -17,7 +17,7 @@ impl Permissioner { stream_id: u32, topic_id: u32, ) -> Result<(), IggyError> { - self.update_topic(user_id, stream_id, topic_id) + self.get_topic(user_id, stream_id, topic_id) } pub fn get_consumer_group( diff --git a/server/src/streaming/users/permissioner_rules/messages.rs b/server/src/streaming/users/permissioner_rules/messages.rs index 54705ef0e..7daef8b1e 100644 --- a/server/src/streaming/users/permissioner_rules/messages.rs +++ b/server/src/streaming/users/permissioner_rules/messages.rs @@ -28,6 +28,18 @@ impl Permissioner { } let stream_permissions = stream_permissions.unwrap(); + if stream_permissions.read_stream { + return Ok(()); + } + + if stream_permissions.manage_topics { + return Ok(()); + } + + if stream_permissions.read_topics { + return Ok(()); + } + if stream_permissions.poll_messages { return Ok(()); } @@ -38,7 +50,10 @@ impl Permissioner { let topic_permissions = stream_permissions.topics.as_ref().unwrap(); if let Some(topic_permissions) = topic_permissions.get(&topic_id) { - return match topic_permissions.poll_messages { + return match topic_permissions.poll_messages + | topic_permissions.read_topic + | topic_permissions.manage_topic + { true => Ok(()), false => Err(IggyError::Unauthorized), }; @@ -73,6 +88,14 @@ impl Permissioner { } let stream_permissions = stream_permissions.unwrap(); + if stream_permissions.manage_stream { + return Ok(()); + } + + if stream_permissions.manage_topics { + return Ok(()); + } + if stream_permissions.send_messages { return Ok(()); } @@ -83,7 +106,7 @@ impl Permissioner { let topic_permissions = stream_permissions.topics.as_ref().unwrap(); if let Some(topic_permissions) = topic_permissions.get(&topic_id) { - return match topic_permissions.send_messages { + return match topic_permissions.send_messages | topic_permissions.manage_topic { true => Ok(()), false => Err(IggyError::Unauthorized), }; diff --git a/server/src/streaming/utils/file.rs b/server/src/streaming/utils/file.rs index 5c5cb5e1b..57e993ef2 100644 --- a/server/src/streaming/utils/file.rs +++ b/server/src/streaming/utils/file.rs @@ -14,7 +14,7 @@ pub async fn overwrite(path: &str) -> Result { OpenOptions::new() .create(true) .write(true) - .truncate(true) + .truncate(false) .open(path) .await } diff --git a/tools/src/data-seeder/main.rs b/tools/src/data-seeder/main.rs index c8f51c88d..a51a43959 100644 --- a/tools/src/data-seeder/main.rs +++ b/tools/src/data-seeder/main.rs @@ -6,7 +6,7 @@ use iggy::args::{Args, ArgsOptional}; use iggy::client::UserClient; use iggy::client_provider; use iggy::client_provider::ClientProviderConfig; -use iggy::clients::client::{IggyClient, IggyClientBackgroundConfig}; +use iggy::clients::client::IggyClient; use iggy::utils::crypto::{Aes256GcmEncryptor, Encryptor}; use std::error::Error; use std::sync::Arc; @@ -31,9 +31,9 @@ async fn main() -> Result<(), Box> { let iggy_args = Args::from(vec![args.iggy.clone()]); tracing_subscriber::fmt::init(); - let encryptor: Option> = match iggy_args.encryption_key.is_empty() { + let encryptor: Option> = match iggy_args.encryption_key.is_empty() { true => None, - false => Some(Box::new( + false => Some(Arc::new( Aes256GcmEncryptor::from_base64_key(&iggy_args.encryption_key).unwrap(), )), }; @@ -42,13 +42,7 @@ async fn main() -> Result<(), Box> { let password = args.password.clone(); let client_provider_config = Arc::new(ClientProviderConfig::from_args(iggy_args)?); let client = client_provider::get_raw_connected_client(client_provider_config).await?; - let client = IggyClient::create( - client, - IggyClientBackgroundConfig::default(), - None, - None, - encryptor, - ); + let client = IggyClient::create(client, None, encryptor); client.login_user(&username, &password).await.unwrap(); info!("Data seeder has started..."); seeder::seed(&client).await.unwrap();