metric: add stale read count metric (tikv#11051)

Signed-off-by: linning <[email protected]> Co-authored-by: Ti Chi Robot <[email protected]>
longfangsong · Nov 30, 2021 · 3b15b89 · 3b15b89
1 parent 57e5d7c
commit 3b15b89
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 0 deletions.
diff --git a/components/raftstore/src/store/worker/metrics.rs b/components/raftstore/src/store/worker/metrics.rs
@@ -123,6 +123,11 @@ lazy_static! {
         "Total number of requests directly executed by local reader."
     )
     .unwrap();
+    pub static ref LOCAL_READ_EXECUTED_STALE_READ_REQUESTS: IntCounter = register_int_counter!(
+        "tikv_raftstore_local_read_executed_stale_read_requests",
+        "Total number of stale read requests directly executed by local reader."
+    )
+    .unwrap();
     pub static ref RAFT_LOG_GC_WRITE_DURATION_HISTOGRAM: Histogram = register_histogram!(
         "tikv_raftstore_raft_log_gc_write_duration_secs",
         "Bucketed histogram of write duration of raft log gc.",

diff --git a/components/raftstore/src/store/worker/read.rs b/components/raftstore/src/store/worker/read.rs
@@ -596,6 +596,7 @@ where
                             cb.invoke_read(resp);
                             return;
                         }
+                        self.metrics.local_executed_stale_read_requests += 1;
                         response
                     }
                     _ => unreachable!(),
@@ -704,6 +705,7 @@ const METRICS_FLUSH_INTERVAL: u64 = 15_000; // 15s
 #[derive(Clone)]
 struct ReadMetrics {
     local_executed_requests: u64,
+    local_executed_stale_read_requests: u64,
     local_executed_snapshot_cache_hit: u64,
     // TODO: record rejected_by_read_quorum.
     rejected_by_store_id_mismatch: u64,
@@ -725,6 +727,7 @@ impl Default for ReadMetrics {
     fn default() -> ReadMetrics {
         ReadMetrics {
             local_executed_requests: 0,
+            local_executed_stale_read_requests: 0,
             local_executed_snapshot_cache_hit: 0,
             rejected_by_store_id_mismatch: 0,
             rejected_by_peer_id_mismatch: 0,
@@ -817,6 +820,10 @@ impl ReadMetrics {
             LOCAL_READ_EXECUTED_REQUESTS.inc_by(self.local_executed_requests);
             self.local_executed_requests = 0;
         }
+        if self.local_executed_stale_read_requests > 0 {
+            LOCAL_READ_EXECUTED_STALE_READ_REQUESTS.inc_by(self.local_executed_stale_read_requests);
+            self.local_executed_stale_read_requests = 0;
+        }
     }
 }
 

diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
@@ -16690,6 +16690,13 @@
               "intervalFactor": 2,
               "legendFormat": "{{instance}}-total",
               "refId": "B"
+            },
+            {
+              "expr": "sum(rate(tikv_raftstore_local_read_executed_stale_read_requests{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{instance}}-stale-read",
+              "refId": "C"
             }
           ],
           "thresholds": [],