From 53e2cfe16e405b6e1968373792538c114938caee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Thu, 24 Jul 2025 12:30:28 +0100 Subject: [PATCH 1/5] initialise nym-signers-monitor --- Cargo.lock | 16 ++++++ Cargo.toml | 2 +- nym-signers-monitor/Cargo.toml | 27 +++++++++ nym-signers-monitor/src/cli/build_info.rs | 15 +++++ nym-signers-monitor/src/cli/env.rs | 11 ++++ nym-signers-monitor/src/cli/mod.rs | 44 ++++++++++++++ nym-signers-monitor/src/cli/run.rs | 54 +++++++++++++++++ nym-signers-monitor/src/main.rs | 23 ++++++++ nym-signers-monitor/src/monitor.rs | 70 +++++++++++++++++++++++ 9 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 nym-signers-monitor/Cargo.toml create mode 100644 nym-signers-monitor/src/cli/build_info.rs create mode 100644 nym-signers-monitor/src/cli/env.rs create mode 100644 nym-signers-monitor/src/cli/mod.rs create mode 100644 nym-signers-monitor/src/cli/run.rs create mode 100644 nym-signers-monitor/src/main.rs create mode 100644 nym-signers-monitor/src/monitor.rs diff --git a/Cargo.lock b/Cargo.lock index bf12ab61cf6..8feaecfd021 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6711,6 +6711,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "nym-signers-monitor" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "humantime", + "nym-bin-common 0.6.0", + "nym-ecash-signer-check", + "nym-task 0.1.0", + "tokio", + "tracing", + "url", + "zulip-client", +] + [[package]] name = "nym-socks5-client" version = "1.1.61" diff --git a/Cargo.toml b/Cargo.toml index ab1266ba269..c6d0eb9509e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,7 +119,7 @@ members = [ "nym-node-status-api/nym-node-status-client", "nym-node/nym-node-metrics", "nym-node/nym-node-requests", - "nym-outfox", + "nym-outfox", "nym-signers-monitor", "nym-statistics-api", "nym-validator-rewarder", "nyx-chain-watcher", diff --git a/nym-signers-monitor/Cargo.toml b/nym-signers-monitor/Cargo.toml new file mode 100644 index 00000000000..3f10085ef8a --- /dev/null +++ b/nym-signers-monitor/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "nym-signers-monitor" +version = "0.1.0" +authors.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +readme.workspace = true + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true, features = ["cargo", "derive", "env", "string"] } +humantime = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +tracing = { workspace = true } +url = { workspace = true } + +nym-bin-common = { path = "../common/bin-common", features = ["output_format", "basic_tracing"] } +nym-ecash-signer-check = { path = "../common/ecash-signer-check" } +nym-task = { path = "../common/task" } +zulip-client = { path = "../common/zulip-client" } + +[lints] +workspace = true diff --git a/nym-signers-monitor/src/cli/build_info.rs b/nym-signers-monitor/src/cli/build_info.rs new file mode 100644 index 00000000000..077d989975f --- /dev/null +++ b/nym-signers-monitor/src/cli/build_info.rs @@ -0,0 +1,15 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use nym_bin_common::bin_info_owned; +use nym_bin_common::output_format::OutputFormat; + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + #[arg(short, long, default_value_t = OutputFormat::default())] + output: OutputFormat, +} + +pub(crate) fn execute(args: Args) { + println!("{}", args.output.format(&bin_info_owned!())) +} diff --git a/nym-signers-monitor/src/cli/env.rs b/nym-signers-monitor/src/cli/env.rs new file mode 100644 index 00000000000..acd0783401a --- /dev/null +++ b/nym-signers-monitor/src/cli/env.rs @@ -0,0 +1,11 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +pub(crate) mod vars { + pub(crate) const ZULIP_BOT_EMAIL_ARG: &str = "ZULIP_BOT_EMAIL"; + pub(crate) const ZULIP_BOT_API_KEY_ARG: &str = "ZULIP_BOT_API_KEY"; + pub(crate) const ZULIP_SERVER_URL_ARG: &str = "ZULIP_SERVER_URL"; + pub(crate) const ZULIP_NOTIFICATION_CHANNEL_ID_ARG: &str = "ZULIP_NOTIFICATION_CHANNEL_ID"; + + pub(crate) const SIGNERS_MONITOR_CHECK_INTERVAL_ARG: &str = "SIGNERS_MONITOR_CHECK_INTERVAL"; +} diff --git a/nym-signers-monitor/src/cli/mod.rs b/nym-signers-monitor/src/cli/mod.rs new file mode 100644 index 00000000000..788eb1f2eea --- /dev/null +++ b/nym-signers-monitor/src/cli/mod.rs @@ -0,0 +1,44 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use clap::{Parser, Subcommand}; +use env::vars::*; +use nym_bin_common::bin_info; +use std::sync::OnceLock; + +pub(crate) mod build_info; +pub(crate) mod env; +pub(crate) mod run; + +// Helper for passing LONG_VERSION to clap +fn pretty_build_info_static() -> &'static str { + static PRETTY_BUILD_INFORMATION: OnceLock = OnceLock::new(); + PRETTY_BUILD_INFORMATION.get_or_init(|| bin_info!().pretty_print()) +} + +#[derive(Parser, Debug)] +#[clap(author = "Nymtech", version, long_version = pretty_build_info_static(), about)] +pub(crate) struct Cli { + #[clap(subcommand)] + command: Commands, +} + +impl Cli { + pub async fn execute(self) -> anyhow::Result<()> { + match self.command { + Commands::BuildInfo(args) => build_info::execute(args), + Commands::Run(args) => run::execute(args).await?, + } + + Ok(()) + } +} + +#[derive(Subcommand, Debug)] +pub(crate) enum Commands { + /// Show build information of this binary + BuildInfo(build_info::Args), + + /// Start signers monitor and send notifications on any failures + Run(run::Args), +} diff --git a/nym-signers-monitor/src/cli/run.rs b/nym-signers-monitor/src/cli/run.rs new file mode 100644 index 00000000000..e4d4c014b73 --- /dev/null +++ b/nym-signers-monitor/src/cli/run.rs @@ -0,0 +1,54 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::cli::env::vars::*; +use crate::monitor::SignersMonitor; +use std::time::Duration; +use url::Url; + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + /// Specify email address for the bot responsible for sending notifications to the zulip server + /// in case 'upgrade' mode is detected + #[clap( + long, + env = ZULIP_BOT_EMAIL_ARG + )] + pub(crate) zulip_bot_email: String, + + /// Specify the API key for the bot responsible for sending notifications to the zulip server + /// in case 'upgrade' mode is detected + #[clap( + long, + env = ZULIP_BOT_API_KEY_ARG + )] + pub(crate) zulip_bot_api_key: String, + + /// Specify the sever endpoint for the bot responsible for sending notifications + /// in case 'upgrade' mode is detected + #[clap( + long, + env = ZULIP_SERVER_URL_ARG + )] + pub(crate) zulip_server_url: Url, + + /// Specify the channel id for where the notification is going to be sent + #[clap( + long, + env = ZULIP_NOTIFICATION_CHANNEL_ID_ARG + )] + pub(crate) zulip_notification_channel_id: u32, + + /// Specify the delay between subsequent signers checks + #[clap( + long, + env = SIGNERS_MONITOR_CHECK_INTERVAL_ARG, + value_parser = humantime::parse_duration, + default_value = "15m" + )] + pub(crate) signers_check_interval: Duration, +} + +pub(crate) async fn execute(args: Args) -> anyhow::Result<()> { + SignersMonitor::new(args)?.run().await +} diff --git a/nym-signers-monitor/src/main.rs b/nym-signers-monitor/src/main.rs new file mode 100644 index 00000000000..b030079fb63 --- /dev/null +++ b/nym-signers-monitor/src/main.rs @@ -0,0 +1,23 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::cli::Cli; +use clap::Parser; +use nym_bin_common::bin_info_owned; +use nym_bin_common::logging::setup_tracing_logger; +use tracing::{info, trace}; + +mod cli; +mod monitor; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + setup_tracing_logger(); + let cli = Cli::parse(); + trace!("args: {cli:#?}"); + + let bin_info = bin_info_owned!(); + info!("using the following version: {bin_info}"); + + cli.execute().await +} diff --git a/nym-signers-monitor/src/monitor.rs b/nym-signers-monitor/src/monitor.rs new file mode 100644 index 00000000000..b5a2acd8a2a --- /dev/null +++ b/nym-signers-monitor/src/monitor.rs @@ -0,0 +1,70 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::cli::run; +use nym_task::ShutdownManager; +use std::time::Duration; +use tokio::time::interval; +use tracing::{error, info}; + +pub(crate) struct SignersMonitor { + zulip_client: zulip_client::Client, + check_interval: Duration, +} + +impl SignersMonitor { + pub(crate) fn new(args: run::Args) -> anyhow::Result { + let zulip_client = zulip_client::Client::builder( + args.zulip_bot_email, + args.zulip_bot_api_key, + args.zulip_server_url, + )? + .build()?; + Ok(SignersMonitor { + zulip_client, + check_interval: args.signers_check_interval, + }) + } + + async fn check_signers(&self) -> anyhow::Result<()> { + println!("here be checking the signers"); + Ok(()) + } + + async fn send_shutdown_notification(&self) -> anyhow::Result<()> { + println!("here be sending shutdown notification"); + Ok(()) + } + + pub(crate) async fn run(&mut self) -> anyhow::Result<()> { + let mut shutdown_manager = + ShutdownManager::new("nym-signers-monitor").with_default_shutdown_signals()?; + + let mut check_interval = interval(self.check_interval); + + while !shutdown_manager.root_token.is_cancelled() { + tokio::select! { + biased; + _ = shutdown_manager.root_token.cancelled() => { + info!("received shutdown"); + break; + } + _ = check_interval.tick() => { + if let Err(err) = self.check_signers().await { + error!("failed to check signers: {err}"); + } + } + + } + } + + shutdown_manager.close(); + shutdown_manager.run_until_shutdown().await; + + if let Err(err) = self.send_shutdown_notification().await { + error!("failed to send shutdown notification: {err}"); + } + + Ok(()) + } +} From df6ca25e3317658e424734736370691880e4fa28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Fri, 25 Jul 2025 11:33:21 +0100 Subject: [PATCH 2/5] creating nyxd client --- Cargo.lock | 3 + nym-signers-monitor/Cargo.toml | 3 + nym-signers-monitor/src/cli/env.rs | 5 ++ nym-signers-monitor/src/cli/run.rs | 97 ++++++++++++++++++++++++++++++ nym-signers-monitor/src/monitor.rs | 7 +++ 5 files changed, 115 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 8feaecfd021..fe611296da2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6720,7 +6720,10 @@ dependencies = [ "humantime", "nym-bin-common 0.6.0", "nym-ecash-signer-check", + "nym-network-defaults 0.1.0", "nym-task 0.1.0", + "nym-validator-client 0.1.0", + "time", "tokio", "tracing", "url", diff --git a/nym-signers-monitor/Cargo.toml b/nym-signers-monitor/Cargo.toml index 3f10085ef8a..264176d054f 100644 --- a/nym-signers-monitor/Cargo.toml +++ b/nym-signers-monitor/Cargo.toml @@ -14,13 +14,16 @@ readme.workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["cargo", "derive", "env", "string"] } humantime = { workspace = true } +time = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tracing = { workspace = true } url = { workspace = true } nym-bin-common = { path = "../common/bin-common", features = ["output_format", "basic_tracing"] } nym-ecash-signer-check = { path = "../common/ecash-signer-check" } +nym-network-defaults = { path = "../common/network-defaults" } nym-task = { path = "../common/task" } +nym-validator-client = { path = "../common/client-libs/validator-client" } zulip-client = { path = "../common/zulip-client" } [lints] diff --git a/nym-signers-monitor/src/cli/env.rs b/nym-signers-monitor/src/cli/env.rs index acd0783401a..fe02be9386a 100644 --- a/nym-signers-monitor/src/cli/env.rs +++ b/nym-signers-monitor/src/cli/env.rs @@ -8,4 +8,9 @@ pub(crate) mod vars { pub(crate) const ZULIP_NOTIFICATION_CHANNEL_ID_ARG: &str = "ZULIP_NOTIFICATION_CHANNEL_ID"; pub(crate) const SIGNERS_MONITOR_CHECK_INTERVAL_ARG: &str = "SIGNERS_MONITOR_CHECK_INTERVAL"; + + pub(crate) const KNOWN_NETWORK_NAME_ARG: &str = "KNOWN_NETWORK_NAME"; + pub(crate) const NYXD_CLIENT_CONFIG_ENV_FILE_ARG: &str = "NYXD_CLIENT_CONFIG_ENV_FILE"; + pub(crate) const NYXD_RPC_ENDPOINT_ARG: &str = "NYXD_RPC_ENDPOINT"; + pub(crate) const NYXD_DKG_CONTRACT_ADDRESS_ARG: &str = "NYXD_DKG_CONTRACT_ADDRESS"; } diff --git a/nym-signers-monitor/src/cli/run.rs b/nym-signers-monitor/src/cli/run.rs index e4d4c014b73..e5ff8829170 100644 --- a/nym-signers-monitor/src/cli/run.rs +++ b/nym-signers-monitor/src/cli/run.rs @@ -3,10 +3,104 @@ use crate::cli::env::vars::*; use crate::monitor::SignersMonitor; +use anyhow::{bail, Context}; +use clap::ArgGroup; +use nym_network_defaults::{setup_env, NymNetworkDetails}; +use nym_validator_client::nyxd::AccountId; +use nym_validator_client::QueryHttpRpcNyxdClient; use std::time::Duration; use url::Url; +#[derive(Debug, clap::Args)] +pub(crate) struct NyxdConnectionArgs { + // for well-known networks, such mainnet, we can use hardcoded values + /// Name of a well known network (such as 'mainnet') that has well-known + /// pre-configured setup values + #[clap(long, env = KNOWN_NETWORK_NAME_ARG)] + pub(crate) known_network_name: Option, + + /// Path pointing to an env file that configures the nyxd client. + #[clap( + short, + long, + env = NYXD_CLIENT_CONFIG_ENV_FILE_ARG + )] + pub(crate) config_env_file: Option, + + /// For unknown networks (or if one wishes to override defaults), + /// specify the RPC endpoint of a node from which signer information should be retrieved + #[clap(long, env = NYXD_RPC_ENDPOINT_ARG)] + pub(crate) nyxd_rpc_endpoint: Option, + + /// For unknown networks, specify address of the DKG contract to pull signer information from. + #[clap( + long, + requires("nyxd_rpc_endpoint"), + env = NYXD_DKG_CONTRACT_ADDRESS_ARG + )] + pub(crate) dkg_contract_address: Option, + // if needed down the line (not sure why), we could define additional args + // for specifying denoms, etc. + // #[clap(long, requires("dkg_contract_address"))] + // pub(crate) mix_denom: Option, +} + +impl NyxdConnectionArgs { + fn get_minimal_nym_network_details(&self) -> anyhow::Result { + if let Some(known_network_name) = &self.known_network_name { + match known_network_name.as_str() { + "mainnet" => return Ok(NymNetworkDetails::new_mainnet()), + other => bail!("{other} is not a known network name - please use another method of setting up chain connection"), + } + } + + if let Some(config_env_file) = &self.config_env_file { + setup_env(Some(config_env_file)); + return Ok(NymNetworkDetails::new_from_env()); + } + + // SAFETY: clap ensures at least one of the fields is set + #[allow(clippy::unwrap_used)] + let dkg_contract = self.dkg_contract_address.as_ref().unwrap(); + + // use mainnet's chain details (i.e. prefixes, denoms, etc) + let mainnet_chain_details = NymNetworkDetails::new_mainnet().chain_details; + Ok(NymNetworkDetails::new_empty() + .with_chain_details(mainnet_chain_details) + .with_coconut_dkg_contract(Some(dkg_contract.to_string()))) + } + + pub(crate) fn try_create_nyxd_client(&self) -> anyhow::Result { + let network_details = self.get_minimal_nym_network_details()?; + + let nyxd_endpoint = match &self.nyxd_rpc_endpoint { + Some(nyxd_rpc_endpoint) => nyxd_rpc_endpoint.clone(), + None => network_details + .endpoints + .first() + .context("no nyxd endpoints provided")? + .nyxd_url + .parse()?, + }; + + Ok(QueryHttpRpcNyxdClient::connect_with_network_details( + nyxd_endpoint.as_str(), + network_details, + )?) + } +} + #[derive(clap::Args, Debug)] +#[command(group( + ArgGroup::new("nyxd_connection") + .multiple(true) + .required(true) + .args([ + "nyxd_connection.known_network_name", + "nyxd_connection.config_env_file", + "nyxd_connection.nyxd_rpc_endpoint" + ]) +))] pub(crate) struct Args { /// Specify email address for the bot responsible for sending notifications to the zulip server /// in case 'upgrade' mode is detected @@ -47,6 +141,9 @@ pub(crate) struct Args { default_value = "15m" )] pub(crate) signers_check_interval: Duration, + + #[clap(flatten)] + pub(crate) nyxd_connection: NyxdConnectionArgs, } pub(crate) async fn execute(args: Args) -> anyhow::Result<()> { diff --git a/nym-signers-monitor/src/monitor.rs b/nym-signers-monitor/src/monitor.rs index b5a2acd8a2a..3630676e82c 100644 --- a/nym-signers-monitor/src/monitor.rs +++ b/nym-signers-monitor/src/monitor.rs @@ -2,13 +2,17 @@ // SPDX-License-Identifier: GPL-3.0-only use crate::cli::run; +use nym_ecash_signer_check::check_signers; use nym_task::ShutdownManager; +use nym_validator_client::QueryHttpRpcNyxdClient; use std::time::Duration; +use time::OffsetDateTime; use tokio::time::interval; use tracing::{error, info}; pub(crate) struct SignersMonitor { zulip_client: zulip_client::Client, + nyxd_client: QueryHttpRpcNyxdClient, check_interval: Duration, } @@ -20,8 +24,11 @@ impl SignersMonitor { args.zulip_server_url, )? .build()?; + let nyxd_client = args.nyxd_connection.try_create_nyxd_client()?; + Ok(SignersMonitor { zulip_client, + nyxd_client, check_interval: args.signers_check_interval, }) } From 1578c585789ba44e5be0213ca48b562a80ba874b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Fri, 25 Jul 2025 13:11:51 +0100 Subject: [PATCH 3/5] performing checks --- nym-signers-monitor/src/cli/mod.rs | 1 - nym-signers-monitor/src/monitor.rs | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/nym-signers-monitor/src/cli/mod.rs b/nym-signers-monitor/src/cli/mod.rs index 788eb1f2eea..ea675cba82f 100644 --- a/nym-signers-monitor/src/cli/mod.rs +++ b/nym-signers-monitor/src/cli/mod.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-3.0-only use clap::{Parser, Subcommand}; -use env::vars::*; use nym_bin_common::bin_info; use std::sync::OnceLock; diff --git a/nym-signers-monitor/src/monitor.rs b/nym-signers-monitor/src/monitor.rs index 3630676e82c..99b8e3c995d 100644 --- a/nym-signers-monitor/src/monitor.rs +++ b/nym-signers-monitor/src/monitor.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-only use crate::cli::run; -use nym_ecash_signer_check::check_signers; +use nym_ecash_signer_check::{check_signers, check_signers_with_client}; use nym_task::ShutdownManager; use nym_validator_client::QueryHttpRpcNyxdClient; use std::time::Duration; @@ -10,6 +10,20 @@ use time::OffsetDateTime; use tokio::time::interval; use tracing::{error, info}; +struct DisplayableSigner { + url: String, + version: Option, + built_on: Option, + signing_available: Option, + chain_not_stalled: Option, +} + +enum CheckOutcome { + QuorumAvailable, + QuorumUnavailable, + UnknownQuorumStatus, +} + pub(crate) struct SignersMonitor { zulip_client: zulip_client::Client, nyxd_client: QueryHttpRpcNyxdClient, @@ -34,6 +48,8 @@ impl SignersMonitor { } async fn check_signers(&self) -> anyhow::Result<()> { + info!("starting signer check..."); + let check_result = check_signers_with_client(&self.nyxd_client).await?; println!("here be checking the signers"); Ok(()) } From 198dd8cc6efc143c38affc8d2618147b735063b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Fri, 25 Jul 2025 16:25:04 +0100 Subject: [PATCH 4/5] sending notifications on failure --- Cargo.lock | 1 + common/ecash-signer-check-types/src/status.rs | 8 + common/zulip-client/src/client.rs | 16 +- common/zulip-client/src/message/mod.rs | 70 +++++++- nym-signers-monitor/Cargo.toml | 1 + nym-signers-monitor/src/cli/env.rs | 2 + nym-signers-monitor/src/cli/run.rs | 13 +- nym-signers-monitor/src/main.rs | 1 + nym-signers-monitor/src/monitor.rs | 158 +++++++++++++++--- nym-signers-monitor/src/test_result.rs | 115 +++++++++++++ 10 files changed, 352 insertions(+), 33 deletions(-) create mode 100644 nym-signers-monitor/src/test_result.rs diff --git a/Cargo.lock b/Cargo.lock index fe611296da2..b86273d5efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6718,6 +6718,7 @@ dependencies = [ "anyhow", "clap", "humantime", + "itertools 0.14.0", "nym-bin-common 0.6.0", "nym-ecash-signer-check", "nym-network-defaults 0.1.0", diff --git a/common/ecash-signer-check-types/src/status.rs b/common/ecash-signer-check-types/src/status.rs index 2a506d6c969..ea078626330 100644 --- a/common/ecash-signer-check-types/src/status.rs +++ b/common/ecash-signer-check-types/src/status.rs @@ -157,6 +157,14 @@ impl SignerResult { pub fn malformed_details(&self) -> bool { self.information.parse().is_err() } + + pub fn try_get_test_result(&self) -> Option<&SignerTestResult> { + if let SignerStatus::Tested { result } = &self.status { + Some(result) + } else { + None + } + } } impl SignerResult diff --git a/common/zulip-client/src/client.rs b/common/zulip-client/src/client.rs index 5de0a18b5cb..60554e5c25c 100644 --- a/common/zulip-client/src/client.rs +++ b/common/zulip-client/src/client.rs @@ -25,7 +25,7 @@ //! ``` use crate::error::ZulipClientError; -use crate::message::{SendMessageResponse, SendableMessage}; +use crate::message::{DirectMessage, SendMessageResponse, SendableMessage, StreamMessage}; use nym_bin_common::bin_info; use nym_http_api_client::UserAgent; use reqwest::{header, Method, RequestBuilder}; @@ -92,6 +92,20 @@ impl Client { .map_err(|source| ZulipClientError::RequestDecodeFailure { source }) } + pub async fn send_direct_message( + &self, + msg: impl Into, + ) -> Result { + self.send_message(msg.into()).await + } + + pub async fn send_channel_message( + &self, + msg: impl Into, + ) -> Result { + self.send_message(msg.into()).await + } + fn build_request(&self, method: Method, endpoint: &'static str) -> RequestBuilder { let url = format!("{}{endpoint}", self.server_url); trace!("posting to {url}"); diff --git a/common/zulip-client/src/message/mod.rs b/common/zulip-client/src/message/mod.rs index 307ef0222cc..9611a0d1095 100644 --- a/common/zulip-client/src/message/mod.rs +++ b/common/zulip-client/src/message/mod.rs @@ -22,7 +22,7 @@ pub enum SendMessageResponse { }, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "snake_case")] #[serde(tag = "type")] pub enum SendableMessageContent { @@ -40,7 +40,7 @@ pub enum SendableMessageContent { }, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub struct SendableMessage { #[serde(flatten)] @@ -117,11 +117,11 @@ impl StreamMessage { pub fn new( to: impl Into, content: impl Into, - topic: Option, + topic: impl IntoMaybeTopic, ) -> Self { StreamMessage { to: to.into().to_string(), - topic, + topic: topic.into_maybe_topic(), content: content.into(), } } @@ -194,22 +194,74 @@ impl From for SendableMessageContent { } } -impl From<(T, S, Option)> for StreamMessage +impl From<(T, S, U)> for StreamMessage where T: Into, S: Into, + U: IntoMaybeTopic, { - fn from((to, content, topic): (T, S, Option)) -> Self { - StreamMessage::new(to, content, topic.map(Into::into)) + fn from((to, content, topic): (T, S, U)) -> Self { + StreamMessage::new(to, content, topic) } } -impl From<(T, S, Option)> for SendableMessage +impl From<(T, S)> for StreamMessage where T: Into, S: Into, { - fn from(inner: (T, S, Option)) -> Self { + fn from((to, content): (T, S)) -> Self { + StreamMessage::no_topic(to, content) + } +} + +impl From<(T, S, U)> for SendableMessage +where + T: Into, + S: Into, + U: IntoMaybeTopic, +{ + fn from(inner: (T, S, U)) -> Self { StreamMessage::from(inner).into() } } + +pub trait IntoMaybeTopic { + fn into_maybe_topic(self) -> Option; +} + +impl IntoMaybeTopic for &Option +where + S: Into + Clone, +{ + fn into_maybe_topic(self) -> Option { + self.clone().map(|s| s.into()) + } +} + +impl IntoMaybeTopic for Option +where + S: Into, +{ + fn into_maybe_topic(self) -> Option { + self.map(Into::into) + } +} + +impl IntoMaybeTopic for String { + fn into_maybe_topic(self) -> Option { + Some(self) + } +} + +impl IntoMaybeTopic for &String { + fn into_maybe_topic(self) -> Option { + Some(self.clone()) + } +} + +impl IntoMaybeTopic for &str { + fn into_maybe_topic(self) -> Option { + Some(self.to_string()) + } +} diff --git a/nym-signers-monitor/Cargo.toml b/nym-signers-monitor/Cargo.toml index 264176d054f..d043aa28717 100644 --- a/nym-signers-monitor/Cargo.toml +++ b/nym-signers-monitor/Cargo.toml @@ -14,6 +14,7 @@ readme.workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["cargo", "derive", "env", "string"] } humantime = { workspace = true } +itertools = { workspace = true } time = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tracing = { workspace = true } diff --git a/nym-signers-monitor/src/cli/env.rs b/nym-signers-monitor/src/cli/env.rs index fe02be9386a..a768634bee2 100644 --- a/nym-signers-monitor/src/cli/env.rs +++ b/nym-signers-monitor/src/cli/env.rs @@ -6,6 +6,8 @@ pub(crate) mod vars { pub(crate) const ZULIP_BOT_API_KEY_ARG: &str = "ZULIP_BOT_API_KEY"; pub(crate) const ZULIP_SERVER_URL_ARG: &str = "ZULIP_SERVER_URL"; pub(crate) const ZULIP_NOTIFICATION_CHANNEL_ID_ARG: &str = "ZULIP_NOTIFICATION_CHANNEL_ID"; + pub(crate) const ZULIP_NOTIFICATION_CHANNEL_TOPIC_ARG: &str = + "ZULIP_NOTIFICATION_CHANNEL_TOPIC"; pub(crate) const SIGNERS_MONITOR_CHECK_INTERVAL_ARG: &str = "SIGNERS_MONITOR_CHECK_INTERVAL"; diff --git a/nym-signers-monitor/src/cli/run.rs b/nym-signers-monitor/src/cli/run.rs index e5ff8829170..dcb1329faa8 100644 --- a/nym-signers-monitor/src/cli/run.rs +++ b/nym-signers-monitor/src/cli/run.rs @@ -96,9 +96,9 @@ impl NyxdConnectionArgs { .multiple(true) .required(true) .args([ - "nyxd_connection.known_network_name", - "nyxd_connection.config_env_file", - "nyxd_connection.nyxd_rpc_endpoint" + "known_network_name", + "config_env_file", + "nyxd_rpc_endpoint" ]) ))] pub(crate) struct Args { @@ -133,6 +133,13 @@ pub(crate) struct Args { )] pub(crate) zulip_notification_channel_id: u32, + /// Optionally specify the channel topic for where the notification is going to be sent + #[clap( + long, + env = ZULIP_NOTIFICATION_CHANNEL_TOPIC_ARG + )] + pub(crate) zulip_notification_topic: Option, + /// Specify the delay between subsequent signers checks #[clap( long, diff --git a/nym-signers-monitor/src/main.rs b/nym-signers-monitor/src/main.rs index b030079fb63..b2955277e05 100644 --- a/nym-signers-monitor/src/main.rs +++ b/nym-signers-monitor/src/main.rs @@ -9,6 +9,7 @@ use tracing::{info, trace}; mod cli; mod monitor; +pub(crate) mod test_result; #[tokio::main] async fn main() -> anyhow::Result<()> { diff --git a/nym-signers-monitor/src/monitor.rs b/nym-signers-monitor/src/monitor.rs index 99b8e3c995d..3079a186c33 100644 --- a/nym-signers-monitor/src/monitor.rs +++ b/nym-signers-monitor/src/monitor.rs @@ -2,32 +2,23 @@ // SPDX-License-Identifier: GPL-3.0-only use crate::cli::run; -use nym_ecash_signer_check::{check_signers, check_signers_with_client}; +use crate::test_result::{DisplayableSignerResult, Summary, TestResult}; +use nym_ecash_signer_check::check_signers_with_client; use nym_task::ShutdownManager; use nym_validator_client::QueryHttpRpcNyxdClient; use std::time::Duration; -use time::OffsetDateTime; use tokio::time::interval; use tracing::{error, info}; -struct DisplayableSigner { - url: String, - version: Option, - built_on: Option, - signing_available: Option, - chain_not_stalled: Option, -} - -enum CheckOutcome { - QuorumAvailable, - QuorumUnavailable, - UnknownQuorumStatus, -} - pub(crate) struct SignersMonitor { zulip_client: zulip_client::Client, nyxd_client: QueryHttpRpcNyxdClient, + + notification_channel_id: u32, + notification_topic: Option, check_interval: Duration, + // min_notification_delay: Duration, + // last_notification_sent: Option, } impl SignersMonitor { @@ -43,14 +34,141 @@ impl SignersMonitor { Ok(SignersMonitor { zulip_client, nyxd_client, + notification_channel_id: args.zulip_notification_channel_id, + notification_topic: args.zulip_notification_topic, check_interval: args.signers_check_interval, }) } - async fn check_signers(&self) -> anyhow::Result<()> { + async fn check_signers(&self) -> anyhow::Result { info!("starting signer check..."); let check_result = check_signers_with_client(&self.nyxd_client).await?; - println!("here be checking the signers"); + + let mut unreachable_signers = 0; + let mut unknown_local_chain_status = 0; + let mut stalled_local_chain = 0; + let mut working_local_chain = 0; + let mut unknown_credential_issuance_status = 0; + let mut working_credential_issuance = 0; + let mut unavailable_credential_issuance = 0; + + let mut fully_working = 0; + + let mut signers = Vec::new(); + for result in &check_result.results { + if result.signer_unreachable() { + unreachable_signers += 1; + } + + if result.unknown_chain_status() { + unknown_local_chain_status += 1; + } + if result.chain_available() { + working_local_chain += 1; + } + if result.chain_provably_stalled() || result.chain_unprovably_stalled() { + stalled_local_chain += 1; + } + + if result.unknown_signing_status() { + unknown_credential_issuance_status += 1; + } + if result.signing_available() { + working_credential_issuance += 1; + } + if result.signing_provably_unavailable() || result.signing_unprovably_unavailable() { + unavailable_credential_issuance += 1; + } + + let signing_available = if result.unknown_signing_status() { + None + } else { + Some(result.signing_available()) + }; + + let chain_not_stalled = if result.unknown_chain_status() { + None + } else { + Some(result.chain_available()) + }; + + if (result.chain_available()) && (result.signing_available()) { + fully_working += 1; + } + + signers.push(DisplayableSignerResult { + version: result + .try_get_test_result() + .map(|r| r.reported_version.clone()), + url: result.information.announce_address.clone(), + signing_available, + chain_not_stalled, + }) + } + + let signing_quorum_available = check_result.threshold.map(|threshold| { + (working_local_chain as u64) >= threshold + && (working_credential_issuance as u64) >= threshold + }); + signers.sort_by_key(|s| s.version); + + let summary = Summary { + signing_quorum_available, + fully_working, + unreachable_signers, + registered_signers: check_result.results.len(), + unknown_local_chain_status, + stalled_local_chain, + working_local_chain, + unknown_credential_issuance_status, + working_credential_issuance, + unavailable_credential_issuance, + threshold: check_result.threshold, + }; + + Ok(TestResult { summary, signers }) + } + + async fn perform_signer_check(&self) -> anyhow::Result<()> { + let result = self.check_signers().await?; + + if result.quorum_unavailable() { + let message = format!( + r#" +# 🔥🔥🔥 LOST SIGNING QUORUM 🔥🔥🔥 +We seem to have lost the signing quorum - check if we should enable the 'upgrade' mode! + +{} + "#, + result.results_to_markdown_message() + ); + return self.send_zulip_notification(&message).await; + } + + if result.quorum_unknown() { + let message = format!( + r#" +# ❓❓❓ UNKNOWN SIGNING QUORUM ❓❓❓ +We can't determine the signing quroum - if we're not undergoing DKG exchange check if we should enable the 'upgrade' mode! + +{} + "#, + result.results_to_markdown_message() + ); + return self.send_zulip_notification(&message).await; + } + + Ok(()) + } + + async fn send_zulip_notification(&self, message: &String) -> anyhow::Result<()> { + self.zulip_client + .send_channel_message(( + self.notification_channel_id, + message, + &self.notification_topic, + )) + .await?; Ok(()) } @@ -60,7 +178,7 @@ impl SignersMonitor { } pub(crate) async fn run(&mut self) -> anyhow::Result<()> { - let mut shutdown_manager = + let shutdown_manager = ShutdownManager::new("nym-signers-monitor").with_default_shutdown_signals()?; let mut check_interval = interval(self.check_interval); @@ -73,7 +191,7 @@ impl SignersMonitor { break; } _ = check_interval.tick() => { - if let Err(err) = self.check_signers().await { + if let Err(err) = self.perform_signer_check().await { error!("failed to check signers: {err}"); } } diff --git a/nym-signers-monitor/src/test_result.rs b/nym-signers-monitor/src/test_result.rs new file mode 100644 index 00000000000..1fb091ec144 --- /dev/null +++ b/nym-signers-monitor/src/test_result.rs @@ -0,0 +1,115 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use itertools::Itertools; + +fn maybe_bool_to_emoji_string(maybe_bool: Option) -> String { + match maybe_bool { + None => "⚠️ unknown".into(), + Some(true) => "✅ yes".into(), + Some(false) => "❌ no".into(), + } +} + +pub(crate) struct DisplayableSignerResult { + pub(crate) url: String, + pub(crate) version: Option, + pub(crate) signing_available: Option, + pub(crate) chain_not_stalled: Option, +} + +impl DisplayableSignerResult { + fn to_markdown_table_row(&self) -> String { + format!( + "| {} | {} | {} | {} |", + self.url, + self.version.as_deref().unwrap_or("unknown"), + maybe_bool_to_emoji_string(self.signing_available), + maybe_bool_to_emoji_string(self.chain_not_stalled) + ) + } +} + +pub(crate) struct TestResult { + pub(crate) summary: Summary, + pub(crate) signers: Vec, +} + +impl TestResult { + pub(crate) fn quorum_unavailable(&self) -> bool { + self.summary.signing_quorum_available.unwrap_or(false) + } + + pub(crate) fn quorum_unknown(&self) -> bool { + self.summary.signing_quorum_available.is_none() + } + + pub(crate) fn results_to_markdown_message(&self) -> String { + let p_available = format!( + "{:.2}", + (self.summary.fully_working as f32 / self.summary.registered_signers as f32) * 100. + ); + + format!( + r#" +## Summary +- quorum available: {} ({p_available}% of signers fully available) +- signers fully working: {} +- signing threshold: {} +- registered signers: {} +- unreachable signers: {} + +### Chain Status +- unknown status: {} +- working chain: {} +- stalled chain: {} + +### Credential Issuance Status +(note: signers below 1.1.64 do not return fully reliable results) +- unknown status: {} +- working issuance: {} +- unavailable issuance: {} + +## Detailed Results +| address | version | chain working | issuance (maybe) available | +| - | - | - | - | +{} + "#, + maybe_bool_to_emoji_string(self.summary.signing_quorum_available), + self.summary.fully_working, + self.summary + .threshold + .map(|threshold| threshold.to_string()) + .unwrap_or("???".to_string()), + self.summary.registered_signers, + self.summary.unreachable_signers, + self.summary.unknown_local_chain_status, + self.summary.working_local_chain, + self.summary.stalled_local_chain, + self.summary.unknown_credential_issuance_status, + self.summary.working_credential_issuance, + self.summary.unavailable_credential_issuance, + self.signers + .iter() + .map(|r| r.to_markdown_table_row()) + .join("\n") + ) + } +} + +pub(crate) struct Summary { + pub(crate) signing_quorum_available: Option, + pub(crate) fully_working: usize, + pub(crate) threshold: Option, + + pub(crate) registered_signers: usize, + pub(crate) unreachable_signers: usize, + + pub(crate) unknown_local_chain_status: usize, + pub(crate) stalled_local_chain: usize, + pub(crate) working_local_chain: usize, + + pub(crate) unknown_credential_issuance_status: usize, + pub(crate) working_credential_issuance: usize, + pub(crate) unavailable_credential_issuance: usize, +} From 44bb704397fe5602d253facc2f795f1508b47653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Tue, 5 Aug 2025 12:43:16 +0100 Subject: [PATCH 5/5] rate limitting on notifications + clippy --- Cargo.lock | 8 ++-- common/zulip-client/src/message/mod.rs | 2 +- nym-signers-monitor/src/cli/env.rs | 2 + nym-signers-monitor/src/cli/mod.rs | 4 +- nym-signers-monitor/src/cli/run.rs | 9 ++++ nym-signers-monitor/src/monitor.rs | 66 +++++++++++++++----------- 6 files changed, 57 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b86273d5efb..d3739bb3309 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6719,11 +6719,11 @@ dependencies = [ "clap", "humantime", "itertools 0.14.0", - "nym-bin-common 0.6.0", + "nym-bin-common", "nym-ecash-signer-check", - "nym-network-defaults 0.1.0", - "nym-task 0.1.0", - "nym-validator-client 0.1.0", + "nym-network-defaults", + "nym-task", + "nym-validator-client", "time", "tokio", "tracing", diff --git a/common/zulip-client/src/message/mod.rs b/common/zulip-client/src/message/mod.rs index 9611a0d1095..2877c724eed 100644 --- a/common/zulip-client/src/message/mod.rs +++ b/common/zulip-client/src/message/mod.rs @@ -127,7 +127,7 @@ impl StreamMessage { } pub fn no_topic(to: impl Into, content: impl Into) -> Self { - Self::new(to, content, None) + Self::new(to, content, None::) } #[must_use] diff --git a/nym-signers-monitor/src/cli/env.rs b/nym-signers-monitor/src/cli/env.rs index a768634bee2..2be0cbbb1e8 100644 --- a/nym-signers-monitor/src/cli/env.rs +++ b/nym-signers-monitor/src/cli/env.rs @@ -10,6 +10,8 @@ pub(crate) mod vars { "ZULIP_NOTIFICATION_CHANNEL_TOPIC"; pub(crate) const SIGNERS_MONITOR_CHECK_INTERVAL_ARG: &str = "SIGNERS_MONITOR_CHECK_INTERVAL"; + pub(crate) const SIGNERS_MONITOR_MIN_NOTIFICATION_DELAY_ARG: &str = + "SIGNERS_MONITOR_MIN_NOTIFICATION_DELAY"; pub(crate) const KNOWN_NETWORK_NAME_ARG: &str = "KNOWN_NETWORK_NAME"; pub(crate) const NYXD_CLIENT_CONFIG_ENV_FILE_ARG: &str = "NYXD_CLIENT_CONFIG_ENV_FILE"; diff --git a/nym-signers-monitor/src/cli/mod.rs b/nym-signers-monitor/src/cli/mod.rs index ea675cba82f..7a59f459fd7 100644 --- a/nym-signers-monitor/src/cli/mod.rs +++ b/nym-signers-monitor/src/cli/mod.rs @@ -26,7 +26,7 @@ impl Cli { pub async fn execute(self) -> anyhow::Result<()> { match self.command { Commands::BuildInfo(args) => build_info::execute(args), - Commands::Run(args) => run::execute(args).await?, + Commands::Run(args) => run::execute(*args).await?, } Ok(()) @@ -39,5 +39,5 @@ pub(crate) enum Commands { BuildInfo(build_info::Args), /// Start signers monitor and send notifications on any failures - Run(run::Args), + Run(Box), } diff --git a/nym-signers-monitor/src/cli/run.rs b/nym-signers-monitor/src/cli/run.rs index dcb1329faa8..b0b0855b49b 100644 --- a/nym-signers-monitor/src/cli/run.rs +++ b/nym-signers-monitor/src/cli/run.rs @@ -149,6 +149,15 @@ pub(crate) struct Args { )] pub(crate) signers_check_interval: Duration, + /// Specify the minimum delay between two subsequent notifications + #[clap( + long, + env = SIGNERS_MONITOR_MIN_NOTIFICATION_DELAY_ARG, + value_parser = humantime::parse_duration, + default_value = "1h" + )] + pub(crate) minimum_notification_delay: Duration, + #[clap(flatten)] pub(crate) nyxd_connection: NyxdConnectionArgs, } diff --git a/nym-signers-monitor/src/monitor.rs b/nym-signers-monitor/src/monitor.rs index 3079a186c33..caae9cbe1f6 100644 --- a/nym-signers-monitor/src/monitor.rs +++ b/nym-signers-monitor/src/monitor.rs @@ -7,9 +7,20 @@ use nym_ecash_signer_check::check_signers_with_client; use nym_task::ShutdownManager; use nym_validator_client::QueryHttpRpcNyxdClient; use std::time::Duration; +use time::OffsetDateTime; use tokio::time::interval; use tracing::{error, info}; +const LOST_QUORUM_MSG: &str = r#" +# 🔥🔥🔥 LOST SIGNING QUORUM 🔥🔥🔥 +We seem to have lost the signing quorum - check if we should enable the 'upgrade' mode! +"#; + +const UNKNOWN_QUORUM_MSG: &str = r#" +# ❓❓❓ UNKNOWN SIGNING QUORUM ❓❓❓ +We can't determine the signing quroum - if we're not undergoing DKG exchange check if we should enable the 'upgrade' mode! +"#; + pub(crate) struct SignersMonitor { zulip_client: zulip_client::Client, nyxd_client: QueryHttpRpcNyxdClient, @@ -17,8 +28,8 @@ pub(crate) struct SignersMonitor { notification_channel_id: u32, notification_topic: Option, check_interval: Duration, - // min_notification_delay: Duration, - // last_notification_sent: Option, + min_notification_delay: Duration, + last_notification_sent: Option, } impl SignersMonitor { @@ -37,10 +48,12 @@ impl SignersMonitor { notification_channel_id: args.zulip_notification_channel_id, notification_topic: args.zulip_notification_topic, check_interval: args.signers_check_interval, + min_notification_delay: args.minimum_notification_delay, + last_notification_sent: None, }) } - async fn check_signers(&self) -> anyhow::Result { + async fn check_signers(&mut self) -> anyhow::Result { info!("starting signer check..."); let check_result = check_signers_with_client(&self.nyxd_client).await?; @@ -110,7 +123,7 @@ impl SignersMonitor { (working_local_chain as u64) >= threshold && (working_credential_issuance as u64) >= threshold }); - signers.sort_by_key(|s| s.version); + signers.sort_by_key(|s| s.version.clone()); let summary = Summary { signing_quorum_available, @@ -129,35 +142,34 @@ impl SignersMonitor { Ok(TestResult { summary, signers }) } - async fn perform_signer_check(&self) -> anyhow::Result<()> { + async fn perform_signer_check(&mut self) -> anyhow::Result<()> { let result = self.check_signers().await?; + let result_md = result.results_to_markdown_message(); + + let msg = if result.quorum_unavailable() { + Some(format!("{LOST_QUORUM_MSG}\n\n{result_md}",)) + } else if result.quorum_unknown() { + Some(format!("{UNKNOWN_QUORUM_MSG}\n\n{result_md}",)) + } else { + None + }; - if result.quorum_unavailable() { - let message = format!( - r#" -# 🔥🔥🔥 LOST SIGNING QUORUM 🔥🔥🔥 -We seem to have lost the signing quorum - check if we should enable the 'upgrade' mode! - -{} - "#, - result.results_to_markdown_message() - ); - return self.send_zulip_notification(&message).await; + if let Some(msg) = msg { + self.maybe_notify_about_failure(&msg).await? } - if result.quorum_unknown() { - let message = format!( - r#" -# ❓❓❓ UNKNOWN SIGNING QUORUM ❓❓❓ -We can't determine the signing quroum - if we're not undergoing DKG exchange check if we should enable the 'upgrade' mode! + Ok(()) + } -{} - "#, - result.results_to_markdown_message() - ); - return self.send_zulip_notification(&message).await; + async fn maybe_notify_about_failure(&mut self, message: &String) -> anyhow::Result<()> { + if let Some(last_notification_sent) = self.last_notification_sent { + if last_notification_sent + self.min_notification_delay > OffsetDateTime::now_utc() { + info!("too soon to send another notification"); + return Ok(()); + } } - + self.send_zulip_notification(message).await?; + self.last_notification_sent = Some(OffsetDateTime::now_utc()); Ok(()) }