diff --git a/database/src/base/rpc_server.rs b/database/src/base/rpc_server.rs index 68f1e6c0..1dc710c4 100644 --- a/database/src/base/rpc_server.rs +++ b/database/src/base/rpc_server.rs @@ -90,23 +90,26 @@ pub trait ReaderDbManager { ) -> anyhow::Result; /// Returns the readnode_primitives::TransactionDetails at the given transaction hash + /// TODO: rewrite this method to return a Result like a struct with block_height and transaction_details instead of a tuple async fn get_transaction_by_hash( &self, transaction_hash: &str, method_name: &str, - ) -> anyhow::Result; + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)>; /// Returns the readnode_primitives::TransactionDetails at the given transaction hash + /// TODO: rewrite this method to return a Result like a struct with block_height and transaction_details instead of a tuple async fn get_indexed_transaction_by_hash( &self, transaction_hash: &str, method_name: &str, - ) -> anyhow::Result; + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)>; /// Returns the readnode_primitives::TransactionDetails at the given transaction hash + /// TODO: rewrite this method to return a Result like a struct with block_height and transaction_details instead of a tuple async fn get_indexing_transaction_by_hash( &self, transaction_hash: &str, method_name: &str, - ) -> anyhow::Result; + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)>; /// Returns the block height and shard id by the given block height async fn get_block_by_height_and_shard_id( diff --git a/database/src/postgres/models.rs b/database/src/postgres/models.rs index d6824b61..df2e6e1e 100644 --- a/database/src/postgres/models.rs +++ b/database/src/postgres/models.rs @@ -433,14 +433,14 @@ impl TransactionDetail { pub async fn get_transaction_by_hash( mut conn: crate::postgres::PgAsyncConn, transaction_hash: &str, - ) -> anyhow::Result> { + ) -> anyhow::Result<(bigdecimal::BigDecimal, Vec)> { let response = transaction_detail::table .filter(transaction_detail::transaction_hash.eq(transaction_hash)) .select(Self::as_select()) .first(&mut conn) .await?; - Ok(response.transaction_details) + Ok((response.block_height, response.transaction_details)) } } diff --git a/database/src/postgres/rpc_server.rs b/database/src/postgres/rpc_server.rs index 17bc32f8..21ce11bc 100644 --- a/database/src/postgres/rpc_server.rs +++ b/database/src/postgres/rpc_server.rs @@ -286,7 +286,7 @@ impl crate::ReaderDbManager for PostgresDBManager { &self, transaction_hash: &str, method_name: &str, - ) -> anyhow::Result { + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)> { if let Ok(transaction) = self .get_indexed_transaction_by_hash(transaction_hash, method_name) .await @@ -302,20 +302,26 @@ impl crate::ReaderDbManager for PostgresDBManager { &self, transaction_hash: &str, _method_name: &str, - ) -> anyhow::Result { - let transaction_data = crate::models::TransactionDetail::get_transaction_by_hash( - Self::get_connection(&self.pg_pool).await?, - transaction_hash, - ) - .await?; - Ok(borsh::from_slice::(&transaction_data)?) + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)> { + let (block_height, transaction_data) = + crate::models::TransactionDetail::get_transaction_by_hash( + Self::get_connection(&self.pg_pool).await?, + transaction_hash, + ) + .await?; + Ok(( + block_height + .to_u64() + .expect("Failed to parse `block_height` to u64"), + borsh::from_slice::(&transaction_data)?, + )) } async fn get_indexing_transaction_by_hash( &self, transaction_hash: &str, _method_name: &str, - ) -> anyhow::Result { + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)> { let data_value = crate::models::TransactionCache::get_transaction_by_hash( Self::get_connection(&self.pg_pool).await?, transaction_hash, @@ -342,7 +348,7 @@ impl crate::ReaderDbManager for PostgresDBManager { .push(execution_outcome) } - Ok(transaction_details.into()) + Ok((transaction_details.block_height, transaction_details.into())) } async fn get_block_by_height_and_shard_id( diff --git a/database/src/scylladb/rpc_server.rs b/database/src/scylladb/rpc_server.rs index 5f153f28..b9f8e4f8 100644 --- a/database/src/scylladb/rpc_server.rs +++ b/database/src/scylladb/rpc_server.rs @@ -124,7 +124,7 @@ impl ScyllaStorageManager for ScyllaDBManager { // ref: https://github.com/near/near-indexer-for-explorer/issues/84 get_transaction_by_hash: Self::prepare_read_query( &scylla_db_session, - "SELECT transaction_details FROM tx_indexer.transactions_details WHERE transaction_hash = ? LIMIT 1", + "SELECT block_height, transaction_details FROM tx_indexer.transactions_details WHERE transaction_hash = ? LIMIT 1", ).await?, get_indexing_transaction_by_hash: Self::prepare_read_query( &scylla_db_session, @@ -485,7 +485,7 @@ impl crate::ReaderDbManager for ScyllaDBManager { &self, transaction_hash: &str, method_name: &str, - ) -> anyhow::Result { + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)> { if let Ok(transaction) = self .get_indexed_transaction_by_hash(transaction_hash, method_name) .await @@ -503,20 +503,20 @@ impl crate::ReaderDbManager for ScyllaDBManager { &self, transaction_hash: &str, method_name: &str, - ) -> anyhow::Result { + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)> { crate::metrics::DATABASE_READ_QUERIES .with_label_values(&[method_name, "tx_indexer.transactions_details"]) .inc(); - let (data_value,) = Self::execute_prepared_query( + let (block_height, data_value) = Self::execute_prepared_query( &self.scylla_session, &self.get_transaction_by_hash, (transaction_hash.to_string(),), ) .await? .single_row()? - .into_typed::<(Vec,)>()?; + .into_typed::<(num_bigint::BigInt, Vec)>()?; match borsh::from_slice::(&data_value) { - Ok(tx) => Ok(tx), + Ok(tx) => Ok((u64::try_from(block_height)?, tx)), Err(e) => { tracing::warn!( "Failed tx_details borsh deserialize: TX_HASH - {}, ERROR: {:?}", @@ -534,7 +534,7 @@ impl crate::ReaderDbManager for ScyllaDBManager { &self, transaction_hash: &str, method_name: &str, - ) -> anyhow::Result { + ) -> anyhow::Result<(u64, readnode_primitives::TransactionDetails)> { crate::metrics::DATABASE_READ_QUERIES .with_label_values(&[method_name, "tx_indexer_cache.transactions"]) .inc(); @@ -574,7 +574,7 @@ impl crate::ReaderDbManager for ScyllaDBManager { .push(execution_outcome); } - Ok(transaction_details.into()) + Ok((transaction_details.block_height, transaction_details.into())) } /// Returns the block height and shard id by the given block height diff --git a/rpc-server/src/config.rs b/rpc-server/src/config.rs index f59a0d45..4c582102 100644 --- a/rpc-server/src/config.rs +++ b/rpc-server/src/config.rs @@ -26,6 +26,7 @@ impl GenesisInfo { let genesis_config = near_rpc_client .call( near_jsonrpc_client::methods::EXPERIMENTAL_genesis_config::RpcGenesisConfigRequest, + None, ) .await .expect("Error to get genesis config"); diff --git a/rpc-server/src/metrics.rs b/rpc-server/src/metrics.rs index dbfd6135..06397fa3 100644 --- a/rpc-server/src/metrics.rs +++ b/rpc-server/src/metrics.rs @@ -96,10 +96,10 @@ lazy_static! { &["method_name"] // This declares a label named `method name` ).unwrap(); - pub(crate) static ref REQUESTS_COUNTER: IntCounterVec = register_int_counter_vec( - "requests_counter", - "Total number of requests", - &["request_type"] // This declares a label named `request_type` + pub(crate) static ref TOTAL_REQUESTS_COUNTER: IntCounterVec = register_int_counter_vec( + "total_requests_counter", + "Total number of method requests by type", + &["method_name", "request_type"] // This declares a label named `method_name` and `request_type` ).unwrap(); pub(crate) static ref OPTIMISTIC_STATUS: IntGauge = try_create_int_gauge( @@ -128,6 +128,7 @@ lazy_static! { pub async fn increase_request_category_metrics( data: &jsonrpc_v2::Data, block_reference: &near_primitives::types::BlockReference, + method_name: &str, block_height: Option, ) { match block_reference { @@ -135,38 +136,49 @@ pub async fn increase_request_category_metrics( let final_block = data.blocks_info_by_finality.final_cache_block().await; let expected_earliest_available_block = final_block.block_height - 5 * data.genesis_info.genesis_config.epoch_length; - if block_height.unwrap_or_default() > expected_earliest_available_block { + // By default, all requests should be historical, therefore + // if block_height is None we use `genesis.block_height` by default + if block_height.unwrap_or(data.genesis_info.genesis_block_cache.block_height) + > expected_earliest_available_block + { // This is request to regular nodes which includes 5 last epochs - REQUESTS_COUNTER.with_label_values(&["regular"]).inc(); + TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "regular"]) + .inc(); } else { // This is a request to archival nodes which include blocks from genesis (later than 5 epochs ago) - REQUESTS_COUNTER.with_label_values(&["historical"]).inc(); + TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "historical"]) + .inc(); } } near_primitives::types::BlockReference::Finality(finality) => { + // All Finality is requests to regular nodes which includes 5 last epochs + TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "regular"]) + .inc(); match finality { - // Increase the REQUESTS_COUNTER `final` metric + // Increase the TOTAL_REQUESTS_COUNTER `final` metric // if the request has final finality near_primitives::types::Finality::DoomSlug | near_primitives::types::Finality::Final => { - REQUESTS_COUNTER.with_label_values(&["final"]).inc(); + TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "final"]) + .inc(); } - // Increase the REQUESTS_COUNTER `optimistic` metric + // Increase the TOTAL_REQUESTS_COUNTER `optimistic` metric // if the request has optimistic finality near_primitives::types::Finality::None => { - REQUESTS_COUNTER.with_label_values(&["optimistic"]).inc(); - // Increase the REQUESTS_COUNTER `proxy_optimistic` metric - // if the optimistic is not updating and proxy to native JSON-RPC - if crate::metrics::OPTIMISTIC_UPDATING.is_not_working() { - REQUESTS_COUNTER - .with_label_values(&["proxy_optimistic"]) - .inc(); - } + TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "optimistic"]) + .inc(); } } } near_primitives::types::BlockReference::SyncCheckpoint(_) => { - REQUESTS_COUNTER.with_label_values(&["historical"]).inc(); + TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "historical"]) + .inc(); } } } diff --git a/rpc-server/src/middlewares.rs b/rpc-server/src/middlewares.rs index cd74ff55..daf40935 100644 --- a/rpc-server/src/middlewares.rs +++ b/rpc-server/src/middlewares.rs @@ -1,4 +1,4 @@ -use crate::metrics::{METHOD_CALLS_COUNTER, REQUESTS_COUNTER}; +use crate::metrics::METHOD_CALLS_COUNTER; use actix_web::dev::{forward_ready, Service, ServiceRequest, ServiceResponse, Transform}; use futures::future::LocalBoxFuture; use futures::StreamExt; @@ -53,8 +53,6 @@ where return service_clone.call(request).await; } - REQUESTS_COUNTER.with_label_values(&["total"]).inc(); - let (req, mut payload) = request.into_parts(); let mut body = actix_web::web::BytesMut::new(); while let Some(chunk) = &payload.next().await { diff --git a/rpc-server/src/modules/blocks/methods.rs b/rpc-server/src/modules/blocks/methods.rs index afb3c6ca..bf09a05a 100644 --- a/rpc-server/src/modules/blocks/methods.rs +++ b/rpc-server/src/modules/blocks/methods.rs @@ -24,15 +24,11 @@ pub async fn block( ) = &block_request.block_reference { if crate::metrics::OPTIMISTIC_UPDATING.is_not_working() { - // increase metrics before proxy request - crate::metrics::increase_request_category_metrics( - &data, - &block_request.block_reference, - None, - ) - .await; // Proxy if the optimistic updating is not working - let block_view = data.near_rpc_client.call(block_request).await?; + let block_view = data + .near_rpc_client + .call(block_request, Some("optimistic")) + .await?; return Ok(near_jsonrpc::primitives::types::blocks::RpcBlockResponse { block_view }); } }; @@ -81,15 +77,11 @@ pub async fn changes_in_block_by_type( ) = &changes_in_block_request.block_reference { if crate::metrics::OPTIMISTIC_UPDATING.is_not_working() { - // increase metrics before proxy request - crate::metrics::increase_request_category_metrics( - &data, - &changes_in_block_request.block_reference, - None, - ) - .await; // Proxy if the optimistic updating is not working - return Ok(data.near_rpc_client.call(changes_in_block_request).await?); + return Ok(data + .near_rpc_client + .call(changes_in_block_request, Some("optimistic")) + .await?); } }; @@ -114,15 +106,11 @@ pub async fn changes_in_block( ) = &changes_in_block_request.block_reference { if crate::metrics::OPTIMISTIC_UPDATING.is_not_working() { - // increase metrics before proxy request - crate::metrics::increase_request_category_metrics( - &data, - &changes_in_block_request.block_reference, - None, - ) - .await; // Proxy if the optimistic updating is not working - return Ok(data.near_rpc_client.call(changes_in_block_request).await?); + return Ok(data + .near_rpc_client + .call(changes_in_block_request, Some("optimistic")) + .await?); } }; @@ -143,6 +131,7 @@ async fn block_call( crate::metrics::increase_request_category_metrics( &data, &block_request.block_reference, + "block", Some(block.block_view.header.height), ) .await; @@ -192,14 +181,6 @@ async fn changes_in_block_call( .await .map_err(near_jsonrpc::primitives::errors::RpcError::from)?; - // increase block category metrics - crate::metrics::increase_request_category_metrics( - &data, - ¶ms.block_reference, - Some(cache_block.block_height), - ) - .await; - let result = fetch_changes_in_block(&data, cache_block, ¶ms.block_reference).await; #[cfg(feature = "shadow_data_consistency")] { @@ -233,14 +214,6 @@ async fn changes_in_block_by_type_call( .await .map_err(near_jsonrpc::primitives::errors::RpcError::from)?; - // increase block category metrics - crate::metrics::increase_request_category_metrics( - &data, - ¶ms.block_reference, - Some(cache_block.block_height), - ) - .await; - let result = fetch_changes_in_block_by_type( &data, cache_block, @@ -402,6 +375,16 @@ pub async fn fetch_chunk( shard_id, ) .await?; + // increase block category metrics + crate::metrics::increase_request_category_metrics( + data, + &near_primitives::types::BlockReference::BlockId(near_primitives::types::BlockId::Height( + block_height, + )), + "chunk", + Some(block_height), + ) + .await; Ok(near_jsonrpc::primitives::types::chunks::RpcChunkResponse { chunk_view }) } diff --git a/rpc-server/src/modules/blocks/utils.rs b/rpc-server/src/modules/blocks/utils.rs index b4ec01e0..0f8040e0 100644 --- a/rpc-server/src/modules/blocks/utils.rs +++ b/rpc-server/src/modules/blocks/utils.rs @@ -122,16 +122,25 @@ pub async fn fetch_block_from_cache_or_get( Some(data.genesis_info.genesis_block_cache) } }; - match block { - Some(block) => Ok(block), + let cache_block = match block { + Some(block) => block, None => { let block_from_s3 = fetch_block(data, block_reference, method_name).await?; let block = CacheBlock::from(&block_from_s3.block_view); data.blocks_cache.put(block.block_height, block).await; - Ok(block) + block } - } + }; + // increase block category metrics + crate::metrics::increase_request_category_metrics( + data, + block_reference, + method_name, + Some(cache_block.block_height), + ) + .await; + Ok(cache_block) } /// Determines whether a given `StateChangeWithCauseView` object matches a set of criteria diff --git a/rpc-server/src/modules/clients/methods.rs b/rpc-server/src/modules/clients/methods.rs index 1bb51949..63b721a8 100644 --- a/rpc-server/src/modules/clients/methods.rs +++ b/rpc-server/src/modules/clients/methods.rs @@ -14,10 +14,10 @@ pub async fn light_client_proof( near_jsonrpc::primitives::types::light_client::RpcLightClientExecutionProofRequest::parse( params, )?; - crate::metrics::REQUESTS_COUNTER - .with_label_values(&["archive_proxy_light_client_proof"]) - .inc(); - Ok(data.near_rpc_client.archival_call(request).await?) + Ok(data + .near_rpc_client + .archival_call(request, Some("light_client_proof")) + .await?) } pub async fn next_light_client_block( @@ -29,7 +29,11 @@ pub async fn next_light_client_block( near_jsonrpc::primitives::types::light_client::RpcLightClientNextBlockRequest::parse( params, )?; - match data.near_rpc_client.call(request).await? { + match data + .near_rpc_client + .call(request, Some("next_light_client_block")) + .await? + { Some(light_client_block) => Ok( near_jsonrpc::primitives::types::light_client::RpcLightClientNextBlockResponse { light_client_block: Some(std::sync::Arc::new(light_client_block)), diff --git a/rpc-server/src/modules/network/methods.rs b/rpc-server/src/modules/network/methods.rs index 89c63ede..65922e52 100644 --- a/rpc-server/src/modules/network/methods.rs +++ b/rpc-server/src/modules/network/methods.rs @@ -89,7 +89,10 @@ pub async fn network_info( ) -> Result { Ok(data .near_rpc_client - .call(near_jsonrpc_client::methods::network_info::RpcNetworkInfoRequest) + .call( + near_jsonrpc_client::methods::network_info::RpcNetworkInfoRequest, + Some("network_info"), + ) .await?) } @@ -101,7 +104,10 @@ pub async fn validators( tracing::debug!("`validators` called with parameters: {:?}", request); // Latest epoch validators fetches from the Near RPC node if let near_primitives::types::EpochReference::Latest = &request.epoch_reference { - let validator_info = data.near_rpc_client.call(request).await?; + let validator_info = data + .near_rpc_client + .call(request, Some("validators")) + .await?; return Ok( near_jsonrpc::primitives::types::validator::RpcValidatorResponse { validator_info }, ); @@ -116,7 +122,10 @@ pub async fn validators( .epoch_id == epoch_id.0 { - let validator_info = data.near_rpc_client.call(request).await?; + let validator_info = data + .near_rpc_client + .call(request, Some("validators")) + .await?; return Ok( near_jsonrpc::primitives::types::validator::RpcValidatorResponse { validator_info }, ); @@ -170,16 +179,38 @@ pub async fn validators_ordered( final_block.block_height - 5 * data.genesis_info.genesis_config.epoch_length; if block.block_height > expected_earliest_available_block { // Proxy to regular rpc if the block is available - Ok(data.near_rpc_client.call(request).await?) + Ok(data + .near_rpc_client + .call(request, Some("EXPERIMENTAL_validators_ordered")) + .await?) } else { // Proxy to archival rpc if the block garbage collected - Ok(data.near_rpc_client.archival_call(request).await?) + Ok(data + .near_rpc_client + .archival_call(request, Some("EXPERIMENTAL_validators_ordered")) + .await?) } } else { - Ok(data.near_rpc_client.call(request).await?) + Ok(data + .near_rpc_client + .call(request, Some("EXPERIMENTAL_validators_ordered")) + .await?) } } else { - Ok(data.near_rpc_client.call(request).await?) + // increase block category metrics + crate::metrics::increase_request_category_metrics( + &data, + &near_primitives::types::BlockReference::Finality( + near_primitives::types::Finality::Final, + ), + "EXPERIMENTAL_validators_ordered", + None, + ) + .await; + Ok(data + .near_rpc_client + .call(request, Some("EXPERIMENTAL_validators_ordered")) + .await?) } } @@ -231,13 +262,26 @@ async fn validators_call( near_jsonrpc::primitives::types::validator::RpcValidatorError, > { let validators = match &validator_request.epoch_reference { - near_primitives::types::EpochReference::EpochId(epoch_id) => data - .db_manager - .get_validators_by_epoch_id(epoch_id.0, "validators") - .await - .map_err(|_err| { - near_jsonrpc::primitives::types::validator::RpcValidatorError::UnknownEpoch - })?, + near_primitives::types::EpochReference::EpochId(epoch_id) => { + let validators = data + .db_manager + .get_validators_by_epoch_id(epoch_id.0, "validators") + .await + .map_err(|_err| { + near_jsonrpc::primitives::types::validator::RpcValidatorError::UnknownEpoch + })?; + // increase block category metrics + crate::metrics::increase_request_category_metrics( + data, + &near_primitives::types::BlockReference::BlockId( + near_primitives::types::BlockId::Height(validators.epoch_start_height), + ), + "validators", + Some(validators.epoch_start_height), + ) + .await; + validators + } near_primitives::types::EpochReference::BlockId(block_id) => { let block_reference = near_primitives::types::BlockReference::BlockId(block_id.clone()); let block = fetch_block_from_cache_or_get(data, &block_reference, "validators") diff --git a/rpc-server/src/modules/queries/methods.rs b/rpc-server/src/modules/queries/methods.rs index 820a0de5..484bb784 100644 --- a/rpc-server/src/modules/queries/methods.rs +++ b/rpc-server/src/modules/queries/methods.rs @@ -26,15 +26,11 @@ pub async fn query( ) = &query_request.block_reference { return if crate::metrics::OPTIMISTIC_UPDATING.is_not_working() { - // increase metrics before proxy request - crate::metrics::increase_request_category_metrics( - &data, - &query_request.block_reference, - None, - ) - .await; // Proxy if the optimistic updating is not working - Ok(data.near_rpc_client.call(query_request).await?) + Ok(data + .near_rpc_client + .call(query_request, Some("optimistic")) + .await?) } else { // query_call with optimistic block query_call(&data, query_request, true).await @@ -53,18 +49,25 @@ async fn query_call( is_optimistic: bool, ) -> Result { tracing::debug!("`query` call. Params: {:?}", query_request,); - - let block = fetch_block_from_cache_or_get(data, &query_request.block_reference, "query") + // When the data check fails, we want to emit the log message and increment the + // corresponding metric. Despite the metrics have "proxies" in their names, we + // are not proxying the requests anymore and respond with the error to the client. + // Since we already have the dashboard using these metric names, we don't want to + // change them and reuse them for the observability of the shadow data consistency checks. + let method_name = match &query_request.request { + near_primitives::views::QueryRequest::ViewAccount { .. } => "query_view_account", + near_primitives::views::QueryRequest::ViewCode { .. } => "query_view_code", + near_primitives::views::QueryRequest::ViewAccessKey { .. } => "query_view_access_key", + near_primitives::views::QueryRequest::ViewState { .. } => "query_view_state", + near_primitives::views::QueryRequest::CallFunction { .. } => "query_call_function", + near_primitives::views::QueryRequest::ViewAccessKeyList { .. } => { + "query_view_access_key_list" + } + }; + let block = fetch_block_from_cache_or_get(data, &query_request.block_reference, method_name) .await .map_err(near_jsonrpc::primitives::errors::RpcError::from)?; - // increase block category metrics - crate::metrics::increase_request_category_metrics( - data, - &query_request.block_reference, - Some(block.block_height), - ) - .await; let result = match &query_request.request { near_primitives::views::QueryRequest::ViewAccount { account_id } => { view_account(data, block, account_id, is_optimistic).await @@ -92,14 +95,16 @@ async fn query_call( final_block.block_height - 5 * data.genesis_info.genesis_config.epoch_length; return if block.block_height > expected_earliest_available_block { // Proxy to regular rpc if the block is available - Ok(data.near_rpc_client.call(query_request).await?) + Ok(data + .near_rpc_client + .call(query_request, Some("query_view_state_proofs")) + .await?) } else { - // Increase the QUERY_VIEW_STATE_INCLUDE_PROOFS metric if we proxy to archival rpc - crate::metrics::REQUESTS_COUNTER - .with_label_values(&["archive_proxy_view_state_proofs"]) - .inc(); // Proxy to archival rpc if the block garbage collected - Ok(data.near_rpc_client.archival_call(query_request).await?) + Ok(data + .near_rpc_client + .archival_call(query_request, Some("query_view_state_proofs")) + .await?) }; } else { view_state(data, block, account_id, prefix, is_optimistic).await @@ -113,6 +118,14 @@ async fn query_call( // TODO: Temporary solution to proxy for poolv1.near // It should be removed after migration to the postgres db if account_id.to_string().ends_with("poolv1.near") { + // increase block category metrics + crate::metrics::increase_request_category_metrics( + data, + &query_request.block_reference, + "poolv1.near", + Some(block.block_height), + ) + .await; let final_block = data.blocks_info_by_finality.final_cache_block().await; // `expected_earliest_available_block` calculated by formula: // `final_block_height` - `node_epoch_count` * `epoch_length` @@ -122,10 +135,16 @@ async fn query_call( final_block.block_height - 5 * data.genesis_info.genesis_config.epoch_length; return if block.block_height > expected_earliest_available_block { // Proxy to regular rpc if the block is available - Ok(data.near_rpc_client.call(query_request).await?) + Ok(data + .near_rpc_client + .call(query_request, Some("poolv1.near")) + .await?) } else { // Proxy to archival rpc if the block garbage collected - Ok(data.near_rpc_client.archival_call(query_request).await?) + Ok(data + .near_rpc_client + .archival_call(query_request, Some("poolv1.near")) + .await?) }; } else { function_call(data, block, account_id, method_name, args, is_optimistic).await @@ -145,10 +164,16 @@ async fn query_call( final_block.block_height - 5 * data.genesis_info.genesis_config.epoch_length; return if block.block_height > expected_earliest_available_block { // Proxy to regular rpc if the block is available - Ok(data.near_rpc_client.call(query_request).await?) + Ok(data + .near_rpc_client + .call(query_request, Some("query_view_access_key_list")) + .await?) } else { // Proxy to archival rpc if the block garbage collected - Ok(data.near_rpc_client.archival_call(query_request).await?) + Ok(data + .near_rpc_client + .archival_call(query_request, Some("query_view_access_key_list")) + .await?) }; } #[cfg(feature = "account_access_keys")] @@ -176,21 +201,6 @@ async fn query_call( near_primitives::types::BlockId::Height(block_height), ); - // When the data check fails, we want to emit the log message and increment the - // corresponding metric. Despite the metrics have "proxies" in their names, we - // are not proxying the requests anymore and respond with the error to the client. - // Since we already have the dashboard using these metric names, we don't want to - // change them and reuse them for the observability of the shadow data consistency checks. - let method_name = match &query_request.request { - near_primitives::views::QueryRequest::ViewAccount { .. } => "query_view_account", - near_primitives::views::QueryRequest::ViewCode { .. } => "query_view_code", - near_primitives::views::QueryRequest::ViewAccessKey { .. } => "query_view_access_key", - near_primitives::views::QueryRequest::ViewState { .. } => "query_view_state", - near_primitives::views::QueryRequest::CallFunction { .. } => "query_call_function", - near_primitives::views::QueryRequest::ViewAccessKeyList { .. } => { - "query_view_access_key_list" - } - }; crate::utils::shadow_compare_results_handler( data.shadow_data_consistency_rate, &result, diff --git a/rpc-server/src/modules/receipts/methods.rs b/rpc-server/src/modules/receipts/methods.rs index 422b5291..346ba988 100644 --- a/rpc-server/src/modules/receipts/methods.rs +++ b/rpc-server/src/modules/receipts/methods.rs @@ -61,7 +61,7 @@ async fn fetch_receipt( let receipt_record = fetch_receipt_record(data, request, "EXPERIMENTAL_receipt").await?; // Getting the raw Vec of the TransactionDetails from ScyllaDB - let transaction_details = data + let (_, transaction_details) = data .db_manager .get_transaction_by_hash( &receipt_record.parent_transaction_hash.to_string(), @@ -104,8 +104,8 @@ async fn fetch_receipt_record( near_jsonrpc::primitives::types::receipts::RpcReceiptError, > { let receipt_id = request.receipt_reference.receipt_id; - - data.db_manager + let result = data + .db_manager .get_receipt_by_id(receipt_id, method_name) .await .map_err(|err| { @@ -113,5 +113,18 @@ async fn fetch_receipt_record( near_jsonrpc::primitives::types::receipts::RpcReceiptError::UnknownReceipt { receipt_id, } - }) + }); + if let Ok(receipt_record) = &result { + // increase block category metrics + crate::metrics::increase_request_category_metrics( + data, + &near_primitives::types::BlockReference::BlockId( + near_primitives::types::BlockId::Height(receipt_record.block_height), + ), + method_name, + Some(receipt_record.block_height), + ) + .await; + }; + result } diff --git a/rpc-server/src/modules/transactions/methods.rs b/rpc-server/src/modules/transactions/methods.rs index 8f689dc8..b547a992 100644 --- a/rpc-server/src/modules/transactions/methods.rs +++ b/rpc-server/src/modules/transactions/methods.rs @@ -11,7 +11,7 @@ pub async fn send_tx( Params(params): Params, ) -> Result { let request = near_jsonrpc_client::methods::send_tx::RpcSendTransactionRequest::parse(params)?; - Ok(data.near_rpc_client.call(request).await?) + Ok(data.near_rpc_client.call(request, Some("send_tx")).await?) } /// Queries status of a transaction by hash and returns the final transaction result. @@ -104,7 +104,11 @@ pub async fn broadcast_tx_async( near_jsonrpc_client::methods::broadcast_tx_async::RpcBroadcastTxAsyncRequest { signed_transaction: tx_async_request.signed_transaction, }; - match data.near_rpc_client.call(proxy_params).await { + match data + .near_rpc_client + .call(proxy_params, Some("broadcast_tx_async")) + .await + { Ok(resp) => Ok(resp), Err(err) => Err(RPCError::internal_error(&err.to_string())), } @@ -130,7 +134,10 @@ pub async fn broadcast_tx_commit( near_jsonrpc_client::methods::broadcast_tx_commit::RpcBroadcastTxCommitRequest { signed_transaction: tx_commit_request.signed_transaction, }; - let result = data.near_rpc_client.call(proxy_params).await?; + let result = data + .near_rpc_client + .call(proxy_params, Some("broadcast_tx_commit")) + .await?; Ok( near_jsonrpc::primitives::types::transactions::RpcTransactionResponse { final_execution_outcome: Some(FinalExecutionOutcome(result)), @@ -168,7 +175,7 @@ async fn tx_status_common( } => *tx_hash, }; - let transaction_details = data + let (block_height, transaction_details) = data .db_manager .get_transaction_by_hash(&tx_hash.to_string(), method_name) .await @@ -178,6 +185,17 @@ async fn tx_status_common( } })?; + // increase block category metrics + crate::metrics::increase_request_category_metrics( + data, + &near_primitives::types::BlockReference::BlockId(near_primitives::types::BlockId::Height( + block_height, + )), + method_name, + Some(block_height), + ) + .await; + if fetch_receipt { Ok( near_jsonrpc::primitives::types::transactions::RpcTransactionResponse { diff --git a/rpc-server/src/utils.rs b/rpc-server/src/utils.rs index 56d2ebe8..1df95040 100644 --- a/rpc-server/src/utils.rs +++ b/rpc-server/src/utils.rs @@ -61,14 +61,17 @@ impl JsonRpcClient { pub async fn call( &self, params: M, + method_name: Option<&str>, ) -> near_jsonrpc_client::MethodCallResult where M: near_jsonrpc_client::methods::RpcMethod + std::fmt::Debug, { tracing::debug!("PROXY call. {:?}", params); - crate::metrics::REQUESTS_COUNTER - .with_label_values(&["regular_proxy"]) - .inc(); + if let Some(method_name) = method_name { + crate::metrics::TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "proxy"]) + .inc(); + } self.rpc_call(params, false).await } @@ -76,14 +79,17 @@ impl JsonRpcClient { pub async fn archival_call( &self, params: M, + method_name: Option<&str>, ) -> near_jsonrpc_client::MethodCallResult where M: near_jsonrpc_client::methods::RpcMethod + std::fmt::Debug, { tracing::debug!("ARCHIVAL PROXY call. {:?}", params); - crate::metrics::REQUESTS_COUNTER - .with_label_values(&["archive_proxy"]) - .inc(); + if let Some(method_name) = method_name { + crate::metrics::TOTAL_REQUESTS_COUNTER + .with_label_values(&[method_name, "archive_proxy"]) + .inc(); + } self.rpc_call(params, true).await } @@ -112,7 +118,7 @@ pub async fn get_final_block( near_primitives::types::Finality::Final }), }; - let block_view = near_rpc_client.call(block_request_method).await?; + let block_view = near_rpc_client.call(block_request_method, None).await?; // Updating the metric to expose the block height considered as final by the server // this metric can be used to calculate the lag between the server and the network @@ -138,7 +144,7 @@ pub async fn get_current_validators( let params = near_jsonrpc_client::methods::validators::RpcValidatorRequest { epoch_reference: near_primitives::types::EpochReference::Latest, }; - Ok(near_rpc_client.call(params).await?) + Ok(near_rpc_client.call(params, None).await?) } pub async fn get_current_epoch_config( @@ -150,7 +156,7 @@ pub async fn get_current_epoch_config( near_primitives::types::Finality::Final, ), }; - let protocol_config_view = near_rpc_client.call(params).await?; + let protocol_config_view = near_rpc_client.call(params, None).await?; Ok(epoch_config_from_protocol_config_view(protocol_config_view).await) }