From 7ed135f9bf72f25b51c89e6aa7fc553778e7db9d Mon Sep 17 00:00:00 2001 From: Mablr <59505383+mablr@users.noreply.github.com> Date: Wed, 22 Oct 2025 23:33:03 +0200 Subject: [PATCH 1/5] feat(metrics): add push-gateway support for metrics reporting with configurable interval --- crates/node/builder/src/launch/common.rs | 2 +- crates/node/core/src/args/metric.rs | 22 +++++- crates/node/core/src/node_config.rs | 2 +- crates/node/metrics/Cargo.toml | 1 + crates/node/metrics/src/server.rs | 99 ++++++++++++++++++++++-- docs/vocs/docs/pages/cli/reth/node.mdx | 12 +++ 6 files changed, 128 insertions(+), 10 deletions(-) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 190cfdc8817..214abd2bb5f 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -610,7 +610,7 @@ where } }) .build(), - ); + ).with_pushgateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval.clone()); MetricServer::new(config).serve().await?; } diff --git a/crates/node/core/src/args/metric.rs b/crates/node/core/src/args/metric.rs index d46018b8e77..9c257c449d7 100644 --- a/crates/node/core/src/args/metric.rs +++ b/crates/node/core/src/args/metric.rs @@ -1,6 +1,6 @@ use clap::Parser; -use reth_cli_util::parse_socket_address; -use std::net::SocketAddr; +use reth_cli_util::{parse_duration_from_secs, parse_socket_address}; +use std::{net::SocketAddr, time::Duration}; /// Metrics configuration. #[derive(Debug, Clone, Default, Parser)] @@ -10,4 +10,22 @@ pub struct MetricArgs { /// The metrics will be served at the given interface and port. #[arg(long="metrics", alias = "metrics.prometheus", value_name = "PROMETHEUS", value_parser = parse_socket_address, help_heading = "Metrics")] pub prometheus: Option, + + /// URL for pushing Prometheus metrics to a push gateway. + /// + /// If set, the node will periodically push metrics to the specified push gateway URL. + #[arg(long = "metrics.push.url", value_name = "PUSH_GATEWAY_URL", help_heading = "Metrics")] + pub push_gateway_url: Option, + + /// Interval in seconds for pushing metrics to push gateway. + /// + /// Default: 5 seconds + #[arg( + long = "metrics.push.interval", + default_value = "5", + value_parser = parse_duration_from_secs, + value_name = "SECONDS", + help_heading = "Metrics" + )] + pub push_gateway_interval: Duration, } diff --git a/crates/node/core/src/node_config.rs b/crates/node/core/src/node_config.rs index 61eb29db38b..c69593adf07 100644 --- a/crates/node/core/src/node_config.rs +++ b/crates/node/core/src/node_config.rs @@ -234,7 +234,7 @@ impl NodeConfig { } /// Set the metrics address for the node - pub const fn with_metrics(mut self, metrics: MetricArgs) -> Self { + pub fn with_metrics(mut self, metrics: MetricArgs) -> Self { self.metrics = metrics; self } diff --git a/crates/node/metrics/Cargo.toml b/crates/node/metrics/Cargo.toml index 39884fa73ef..9687c9c20ac 100644 --- a/crates/node/metrics/Cargo.toml +++ b/crates/node/metrics/Cargo.toml @@ -21,6 +21,7 @@ tokio.workspace = true jsonrpsee-server.workspace = true http.workspace = true tower.workspace = true +reqwest.workspace = true tracing.workspace = true eyre.workspace = true diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index c029b773718..01999b4940e 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -10,7 +10,7 @@ use metrics::describe_gauge; use metrics_process::Collector; use reth_metrics::metrics::Unit; use reth_tasks::TaskExecutor; -use std::{convert::Infallible, net::SocketAddr, sync::Arc}; +use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration}; /// Configuration for the [`MetricServer`] #[derive(Debug)] @@ -20,6 +20,8 @@ pub struct MetricServerConfig { chain_spec_info: ChainSpecInfo, task_executor: TaskExecutor, hooks: Hooks, + push_gateway_url: Option, + push_gateway_interval: Duration, } impl MetricServerConfig { @@ -31,7 +33,22 @@ impl MetricServerConfig { task_executor: TaskExecutor, hooks: Hooks, ) -> Self { - Self { listen_addr, hooks, task_executor, version_info, chain_spec_info } + Self { + listen_addr, + hooks, + task_executor, + version_info, + chain_spec_info, + push_gateway_url: None, + push_gateway_interval: Duration::from_secs(5), + } + } + + /// Set the push gateway URL for pushing metrics + pub fn with_pushgateway(mut self, url: Option, interval: Duration) -> Self { + self.push_gateway_url = url; + self.push_gateway_interval = interval; + self } } @@ -49,18 +66,35 @@ impl MetricServer { /// Spawns the metrics server pub async fn serve(&self) -> eyre::Result<()> { - let MetricServerConfig { listen_addr, hooks, task_executor, version_info, chain_spec_info } = - &self.config; + let MetricServerConfig { + listen_addr, + hooks, + task_executor, + version_info, + chain_spec_info, + push_gateway_url, + push_gateway_interval, + } = &self.config; - let hooks = hooks.clone(); + let hooks_for_endpoint = hooks.clone(); self.start_endpoint( *listen_addr, - Arc::new(move || hooks.iter().for_each(|hook| hook())), + Arc::new(move || hooks_for_endpoint.iter().for_each(|hook| hook())), task_executor.clone(), ) .await .wrap_err_with(|| format!("Could not start Prometheus endpoint at {listen_addr}"))?; + // Start push-gateway task if configured + if let Some(url) = push_gateway_url { + self.start_push_gateway_task( + url.clone(), + *push_gateway_interval, + hooks.clone(), + task_executor.clone(), + ); + } + // Describe metrics after recorder installation describe_db_metrics(); describe_static_file_metrics(); @@ -128,6 +162,59 @@ impl MetricServer { Ok(()) } + + /// Starts a background task to push metrics to a push gateway + fn start_push_gateway_task( + &self, + url: String, + interval: Duration, + hooks: Hooks, + task_executor: TaskExecutor, + ) { + task_executor.spawn_with_graceful_shutdown_signal(move |mut signal| { + Box::pin(async move { + let client = reqwest::Client::builder() + .build(); + + let client = match client { + Ok(c) => c, + Err(err) => { + tracing::error!(%err, "Failed to create HTTP client for PushGateway"); + return; + } + }; + + tracing::info!(url = %url, interval = ?interval, "Starting PushGateway metrics push task"); + + loop { + tokio::select! { + _ = &mut signal => { + tracing::info!("Shutting down PushGateway push task"); + break; + } + _ = tokio::time::sleep(interval) => { + hooks.iter().for_each(|hook| hook()); + let handle = install_prometheus_recorder(); + let metrics = handle.handle().render(); + match client.put(&url).header("Content-Type", "text/plain").body(metrics).send().await { + Ok(response) => { + if !response.status().is_success() { + tracing::warn!( + status = %response.status(), + "Failed to push metrics to PushGateway" + ); + } + } + Err(err) => { + tracing::warn!(%err, "Failed to push metrics to PushGateway"); + } + } + } + } + } + }) + }); + } } fn describe_db_metrics() { diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 3fc6988dc69..12a8d1810c9 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -44,6 +44,18 @@ Metrics: The metrics will be served at the given interface and port. + --metrics.push.url + URL for pushing Prometheus metrics to a push gateway. + + If set, the node will periodically push metrics to the specified push gateway URL. + + --metrics.push.interval + Interval in seconds for pushing metrics to push gateway. + + Default: 5 seconds + + [default: 5] + Datadir: --datadir The path to the data dir for all reth files and subdirectories. From 8233888838752306d71ea33cc8ccf6f573e1ee07 Mon Sep 17 00:00:00 2001 From: Mablr <59505383+mablr@users.noreply.github.com> Date: Wed, 22 Oct 2025 23:42:58 +0200 Subject: [PATCH 2/5] fix: clippy --- crates/node/builder/src/launch/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 214abd2bb5f..9ba5b986fa6 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -610,7 +610,7 @@ where } }) .build(), - ).with_pushgateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval.clone()); + ).with_pushgateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval); MetricServer::new(config).serve().await?; } From 4fbfc3f2fb260a0420eb50fe4a761d5bb4ed2cdb Mon Sep 17 00:00:00 2001 From: Mablr <59505383+mablr@users.noreply.github.com> Date: Wed, 22 Oct 2025 23:48:09 +0200 Subject: [PATCH 3/5] fix: style nit --- crates/node/builder/src/launch/common.rs | 2 +- crates/node/metrics/src/server.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 9ba5b986fa6..92e3a7aa811 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -610,7 +610,7 @@ where } }) .build(), - ).with_pushgateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval); + ).with_push_gateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval); MetricServer::new(config).serve().await?; } diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index 01999b4940e..f48c2348ec5 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -44,8 +44,8 @@ impl MetricServerConfig { } } - /// Set the push gateway URL for pushing metrics - pub fn with_pushgateway(mut self, url: Option, interval: Duration) -> Self { + /// Set the gateway URL and interval for pushing metrics + pub fn with_push_gateway(mut self, url: Option, interval: Duration) -> Self { self.push_gateway_url = url; self.push_gateway_interval = interval; self @@ -163,7 +163,7 @@ impl MetricServer { Ok(()) } - /// Starts a background task to push metrics to a push gateway + /// Starts a background task to push metrics to a metrics gateway fn start_push_gateway_task( &self, url: String, @@ -179,17 +179,17 @@ impl MetricServer { let client = match client { Ok(c) => c, Err(err) => { - tracing::error!(%err, "Failed to create HTTP client for PushGateway"); + tracing::error!(%err, "Failed to create HTTP client to push metrics to gateway"); return; } }; - tracing::info!(url = %url, interval = ?interval, "Starting PushGateway metrics push task"); + tracing::info!(url = %url, interval = ?interval, "Starting task to push metrics to gateway"); loop { tokio::select! { _ = &mut signal => { - tracing::info!("Shutting down PushGateway push task"); + tracing::info!("Shutting down task to push metrics to gateway"); break; } _ = tokio::time::sleep(interval) => { @@ -201,12 +201,12 @@ impl MetricServer { if !response.status().is_success() { tracing::warn!( status = %response.status(), - "Failed to push metrics to PushGateway" + "Failed to push metrics to gateway" ); } } Err(err) => { - tracing::warn!(%err, "Failed to push metrics to PushGateway"); + tracing::warn!(%err, "Failed to push metrics to gateway"); } } } From 26bd31f43ec2080e76647d5f0e0d9c60a82e67b1 Mon Sep 17 00:00:00 2001 From: Mablr <59505383+mablr@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:04:04 +0200 Subject: [PATCH 4/5] fix: client init outside task + install recorder outside the loop --- crates/node/metrics/src/server.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index f48c2348ec5..d7beb6c3a1d 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -8,6 +8,7 @@ use eyre::WrapErr; use http::{header::CONTENT_TYPE, HeaderValue, Response}; use metrics::describe_gauge; use metrics_process::Collector; +use reqwest::Client; use reth_metrics::metrics::Unit; use reth_tasks::TaskExecutor; use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration}; @@ -92,7 +93,7 @@ impl MetricServer { *push_gateway_interval, hooks.clone(), task_executor.clone(), - ); + )?; } // Describe metrics after recorder installation @@ -170,22 +171,14 @@ impl MetricServer { interval: Duration, hooks: Hooks, task_executor: TaskExecutor, - ) { + ) -> eyre::Result<()> { + let client = Client::builder() + .build() + .wrap_err("Could not create HTTP client to push metrics to gateway")?; task_executor.spawn_with_graceful_shutdown_signal(move |mut signal| { Box::pin(async move { - let client = reqwest::Client::builder() - .build(); - - let client = match client { - Ok(c) => c, - Err(err) => { - tracing::error!(%err, "Failed to create HTTP client to push metrics to gateway"); - return; - } - }; - tracing::info!(url = %url, interval = ?interval, "Starting task to push metrics to gateway"); - + let handle = install_prometheus_recorder(); loop { tokio::select! { _ = &mut signal => { @@ -194,7 +187,6 @@ impl MetricServer { } _ = tokio::time::sleep(interval) => { hooks.iter().for_each(|hook| hook()); - let handle = install_prometheus_recorder(); let metrics = handle.handle().render(); match client.put(&url).header("Content-Type", "text/plain").body(metrics).send().await { Ok(response) => { @@ -214,6 +206,7 @@ impl MetricServer { } }) }); + Ok(()) } } From 48dce07c4d16f3f38fb4b22a889ac20bad34bc39 Mon Sep 17 00:00:00 2001 From: Mablr <59505383+mablr@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:22:07 +0200 Subject: [PATCH 5/5] fix: update CLI args --- crates/node/core/src/args/metric.rs | 8 ++++++-- docs/vocs/docs/pages/cli/reth/node.mdx | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/node/core/src/args/metric.rs b/crates/node/core/src/args/metric.rs index 9c257c449d7..5ef18787a81 100644 --- a/crates/node/core/src/args/metric.rs +++ b/crates/node/core/src/args/metric.rs @@ -14,14 +14,18 @@ pub struct MetricArgs { /// URL for pushing Prometheus metrics to a push gateway. /// /// If set, the node will periodically push metrics to the specified push gateway URL. - #[arg(long = "metrics.push.url", value_name = "PUSH_GATEWAY_URL", help_heading = "Metrics")] + #[arg( + long = "metrics.prometheus.push.url", + value_name = "PUSH_GATEWAY_URL", + help_heading = "Metrics" + )] pub push_gateway_url: Option, /// Interval in seconds for pushing metrics to push gateway. /// /// Default: 5 seconds #[arg( - long = "metrics.push.interval", + long = "metrics.prometheus.push.interval", default_value = "5", value_parser = parse_duration_from_secs, value_name = "SECONDS", diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 12a8d1810c9..80310fc1138 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -44,12 +44,12 @@ Metrics: The metrics will be served at the given interface and port. - --metrics.push.url + --metrics.prometheus.push.url URL for pushing Prometheus metrics to a push gateway. If set, the node will periodically push metrics to the specified push gateway URL. - --metrics.push.interval + --metrics.prometheus.push.interval Interval in seconds for pushing metrics to push gateway. Default: 5 seconds