persist: drop per-shard gauges only the raw persist dashboard plots

`mz_persist_shard_since`, `mz_persist_shard_upper`,
`mz_persist_shard_stale_version`,
`mz_persist_shard_schema_registry_version_count`,
`mz_persist_shard_inline_part_bytes` and
`mz_persist_shard_batch_part_version_bytes` appear on no curated
dashboard, only on the everything dump. Frontiers per collection are in
`mz_internal.mz_frontiers`, `mz_persist_shard_count` already counts
shards, and each of the two `_bytes` families has a `_count` sibling on
the shard dashboard.

The release sign-off reference counted `mz_persist_shard_upper` series
to get a shard count and now points at `mz_persist_shard_count`.

Author: SangJunBak

.agents/skills/mz-release-signoff/SKILL.md

@@ -290,4 +290,4 @@ The per-area references list the dead panels found so far. Check for new ones wh
 
 **An implausibly constant quantile is a bucket artifact, not stability.** A `histogram_quantile` landing inside one wide bucket returns the bucket boundary and cannot move, which reads as a rock-steady latency. Two adapter panels do this. When a quantile is stable to five significant figures while its counters advance, switch to `rate(_sum) / rate(_count)`.
 
-**Counting series is sometimes the measurement.** Several metrics are per-entity gauges whose only useful aggregate is a series count: `mz_persist_shard_upper` for shards, `mz_balancer_metadata_seconds` for balancers, `mz_persist_metadata_seconds` by `version` for persist client builds. Similarly `mz_source_progress` is a millisecond frontier timestamp, so a healthy series contributes exactly 1000 to its rate and the panel's absolute value is really a series count in disguise.
+**Counting series is sometimes the measurement.** Several metrics are per-entity gauges whose only useful aggregate is a series count: `mz_balancer_metadata_seconds` for balancers, `mz_persist_metadata_seconds` by `version` for persist client builds. Similarly `mz_source_progress` is a millisecond frontier timestamp, so a healthy series contributes exactly 1000 to its rate and the panel's absolute value is really a series count in disguise.

.agents/skills/mz-release-signoff/references/persist.md

@@ -34,7 +34,7 @@ Worth adding to the sweep even though the panel omits them: `mz_persist_compacti
 | `mz_persist_compaction_seconds`, `_requested`, `_applied`, `_bytes`, `_goodbytes` | counter | Compaction. Requested minus applied equals the noop and dropped counts. |
 | `mz_persist_gc_seconds`, `_started`, `_finished`, `_noop`, `_skipped`, `_merged_reqs` | counter | Garbage collection. |
 | `mz_persist_retry_retries_count`, `_started_count`, `_finished_count`, `_sleep_seconds` | counter | The panel excludes `op="next_listen_batch"`, which has its own panel because it retries by design. |
-| `mz_persist_shard_upper` | gauge, per shard | Counting distinct `shard` labels gives the shard count. |
+| `mz_persist_shard_count` | gauge, per process | Shards the process holds a handle to. environmentd holds one per collection, so its value is the environment's shard count. |
 | `mz_persist_read_batch_part_bytes`, `_count` | counter, by `op` | The `op="unindexed"` slice is the `unindexed reads` panel. Rises by roughly 3x during rehydration. |
 | `mz_persist_pushdown_parts_filtered_count`, `_fetched_count`, `_inline_count`, `_faked_count` and their `_bytes` variants | counter | The ratio panels divide one by the sum of all four. |
 | `mz_persist_blob_cache_hits_bytes`, `_hits_blobs`, `_evictions` | counter | |
@@ -65,7 +65,7 @@ Each entry states a property that holds at any fleet size, followed by the measu
 
 **Persist reports its own build independently of `mz_version`.** `mz_persist_metadata_seconds` carries a `version` label, and the `# processes by version` panel counts by it. Use it to confirm a rollout reached the persist clients.
 
-**Counting distinct `shard` labels on `mz_persist_shard_upper` is how the shard count is obtained.** There is no shard-count gauge.
+**`mz_persist_shard_count` on the environmentd pod is the shard count.** Summing it across pods overcounts, because clusterd pods hold handles to the same shards.
 
 ## Order of magnitude
 

doc/user/data/metrics.yml

@@ -2479,14 +2479,6 @@ metrics:
   - shard
   source: src/persist-client/src/internal/metrics.rs
   visibility: internal
-- name: mz_persist_shard_batch_part_version_bytes
-  help: total bytes in batch parts by shard and version
-  labels:
-  - name
-  - shard
-  - version
-  source: src/persist-client/src/internal/metrics.rs
-  visibility: internal
 - name: mz_persist_shard_batch_part_version_count
   help: count of batch parts by shard and version
   labels:
@@ -2576,13 +2568,6 @@ metrics:
   - shard
   source: src/persist-client/src/internal/metrics.rs
   visibility: internal
-- name: mz_persist_shard_inline_part_bytes
-  help: total size of parts inline in shard metadata
-  labels:
-  - name
-  - shard
-  source: src/persist-client/src/internal/metrics.rs
-  visibility: internal
 - name: mz_persist_shard_inline_part_count
   help: count of parts inline in shard metadata
   labels:
@@ -2618,13 +2603,6 @@ metrics:
   - shard
   source: src/persist-client/src/internal/metrics.rs
   visibility: internal
-- name: mz_persist_shard_schema_registry_version_count
-  help: count of versions in the schema registry
-  labels:
-  - name
-  - shard
-  source: src/persist-client/src/internal/metrics.rs
-  visibility: internal
 - name: mz_persist_shard_seqnos_held
   help: maximum count of gc-ineligible states by shard
   labels:
@@ -2639,27 +2617,13 @@ metrics:
   - shard
   source: src/persist-client/src/internal/metrics.rs
   visibility: internal
-- name: mz_persist_shard_since
-  help: since by shard
-  labels:
-  - name
-  - shard
-  source: src/persist-client/src/internal/metrics.rs
-  visibility: internal
 - name: mz_persist_shard_spine_batch_count
   help: count of spine batches by shard
   labels:
   - name
   - shard
   source: src/persist-client/src/internal/metrics.rs
   visibility: internal
-- name: mz_persist_shard_stale_version
-  help: indicates whether the current version of the shard is less than the current version of the code
-  labels:
-  - name
-  - shard
-  source: src/persist-client/src/internal/metrics.rs
-  visibility: internal
 - name: mz_persist_shard_unconsolidated_snapshot
   help: in snapshot_and_read, the number of times consolidating the raw data wasn't enough to produce consolidated output
   labels:
@@ -2674,13 +2638,6 @@ metrics:
   - shard
   source: src/persist-client/src/internal/metrics.rs
   visibility: internal
-- name: mz_persist_shard_upper
-  help: upper by shard
-  labels:
-  - name
-  - shard
-  source: src/persist-client/src/internal/metrics.rs
-  visibility: internal
 - name: mz_persist_shard_usage_current_state_batches_bytes
   help: data in batches/parts referenced by current version of state
   labels:

src/persist-client/src/internal/metrics.rs

@@ -35,7 +35,7 @@ use mz_persist::metrics::{BlobHedgeMetrics, ColumnarMetrics, S3BlobMetrics};
 use mz_persist::retry::RetryStream;
 use mz_persist_types::Codec64;
 use mz_postgres_client::metrics::PostgresClientMetrics;
-use prometheus::core::{AtomicI64, AtomicU64, Collector, Desc, GenericGauge};
+use prometheus::core::{AtomicU64, Collector, Desc, GenericGauge};
 use prometheus::proto::MetricFamily;
 use prometheus::{CounterVec, Gauge, GaugeVec, Histogram, HistogramVec, IntCounterVec};
 use timely::progress::Antichain;
@@ -1261,15 +1261,12 @@ pub struct ShardsMetrics {
     // the DeleteOnDrop wrappers. A process might stop using a shard (drop all
     // handles to it) but e.g. the set of commands never changes.
     _count: ComputedIntGauge,
-    since: mz_ore::metrics::IntGaugeVec,
-    upper: mz_ore::metrics::IntGaugeVec,
     encoded_rollup_size: mz_ore::metrics::UIntGaugeVec,
     encoded_diff_size: mz_ore::metrics::IntCounterVec,
     hollow_batch_count: mz_ore::metrics::UIntGaugeVec,
     spine_batch_count: mz_ore::metrics::UIntGaugeVec,
     batch_part_count: mz_ore::metrics::UIntGaugeVec,
     batch_part_version_count: mz_ore::metrics::UIntGaugeVec,
-    batch_part_version_bytes: mz_ore::metrics::UIntGaugeVec,
     update_count: mz_ore::metrics::UIntGaugeVec,
     rollup_count: mz_ore::metrics::UIntGaugeVec,
     largest_batch_size: mz_ore::metrics::UIntGaugeVec,
@@ -1285,16 +1282,13 @@ pub struct ShardsMetrics {
     usage_referenced_not_current_state_bytes: mz_ore::metrics::UIntGaugeVec,
     usage_not_leaked_not_referenced_bytes: mz_ore::metrics::UIntGaugeVec,
     usage_leaked_bytes: mz_ore::metrics::UIntGaugeVec,
-    stale_version: mz_ore::metrics::UIntGaugeVec,
     blob_gets: mz_ore::metrics::IntCounterVec,
     blob_sets: mz_ore::metrics::IntCounterVec,
     unconsolidated_snapshot: mz_ore::metrics::IntCounterVec,
     inline_part_count: UIntGaugeVec,
-    inline_part_bytes: UIntGaugeVec,
     compact_batches: UIntGaugeVec,
     compacting_batches: UIntGaugeVec,
     noncompact_batches: UIntGaugeVec,
-    schema_registry_version_count: UIntGaugeVec,
     // We hand out `Arc<ShardMetrics>` to read and write handles, but store it
     // here as `Weak`. This allows us to discover if it's no longer in use and
     // so we can remove it from the map.
@@ -1317,16 +1311,6 @@ impl ShardsMetrics {
                     ret
                 },
             ),
-            since: registry.register(metric!(
-                name: "mz_persist_shard_since",
-                help: "since by shard",
-                var_labels: ["shard", "name"],
-            )),
-            upper: registry.register(metric!(
-                name: "mz_persist_shard_upper",
-                help: "upper by shard",
-                var_labels: ["shard", "name"],
-            )),
             encoded_rollup_size: registry.register(metric!(
                 name: "mz_persist_shard_rollup_size_bytes",
                 help: "total encoded rollup size by shard",
@@ -1357,11 +1341,6 @@ impl ShardsMetrics {
                 help: "count of batch parts by shard and version",
                 var_labels: ["shard", "name", "version"],
             )),
-            batch_part_version_bytes: registry.register(metric!(
-                name: "mz_persist_shard_batch_part_version_bytes",
-                help: "total bytes in batch parts by shard and version",
-                var_labels: ["shard", "name", "version"],
-            )),
             update_count: registry.register(metric!(
                 name: "mz_persist_shard_update_count",
                 help: "count of updates by shard",
@@ -1437,11 +1416,6 @@ impl ShardsMetrics {
                 help: "data reclaimable by a leaked blob detector",
                 var_labels: ["shard", "name"],
             )),
-            stale_version: registry.register(metric!(
-                name: "mz_persist_shard_stale_version",
-                help: "indicates whether the current version of the shard is less than the current version of the code",
-                var_labels: ["shard", "name"],
-            )),
             blob_gets: registry.register(metric!(
                 name: "mz_persist_shard_blob_gets",
                 help: "number of Blob::get calls for this shard",
@@ -1462,11 +1436,6 @@ impl ShardsMetrics {
                 help: "count of parts inline in shard metadata",
                 var_labels: ["shard", "name"],
             )),
-            inline_part_bytes: registry.register(metric!(
-                name: "mz_persist_shard_inline_part_bytes",
-                help: "total size of parts inline in shard metadata",
-                var_labels: ["shard", "name"],
-            )),
             compact_batches: registry.register(metric!(
                 name: "mz_persist_shard_compact_batches",
                 help: "number of fully compact batches in the shard",
@@ -1482,11 +1451,6 @@ impl ShardsMetrics {
                 help: "number of batches in the shard that aren't compact and have no ongoing compaction",
                 var_labels: ["shard", "name"],
             )),
-            schema_registry_version_count: registry.register(metric!(
-                name: "mz_persist_shard_schema_registry_version_count",
-                help: "count of versions in the schema registry",
-                var_labels: ["shard", "name"],
-            )),
             shards,
         }
     }
@@ -1532,16 +1496,13 @@ impl ShardsMetrics {
 pub struct ShardMetrics {
     pub shard_id: ShardId,
     pub name: String,
-    pub since: DeleteOnDropGauge<AtomicI64, Vec<String>>,
-    pub upper: DeleteOnDropGauge<AtomicI64, Vec<String>>,
     pub largest_batch_size: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub latest_rollup_size: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub encoded_diff_size: DeleteOnDropCounter<AtomicU64, Vec<String>>,
     pub hollow_batch_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub spine_batch_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub batch_part_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     batch_part_version_count: mz_ore::metrics::UIntGaugeVec,
-    batch_part_version_bytes: mz_ore::metrics::UIntGaugeVec,
     batch_part_version_map: Mutex<BTreeMap<String, BatchPartVersionMetrics>>,
     pub update_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub rollup_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
@@ -1557,16 +1518,13 @@ pub struct ShardMetrics {
     pub gc_finished: DeleteOnDropCounter<AtomicU64, Vec<String>>,
     pub compaction_applied: DeleteOnDropCounter<AtomicU64, Vec<String>>,
     pub cmd_succeeded: DeleteOnDropCounter<AtomicU64, Vec<String>>,
-    pub stale_version: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub blob_gets: DeleteOnDropCounter<AtomicU64, Vec<String>>,
     pub blob_sets: DeleteOnDropCounter<AtomicU64, Vec<String>>,
     pub unconsolidated_snapshot: DeleteOnDropCounter<AtomicU64, Vec<String>>,
     pub inline_part_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
-    pub inline_part_bytes: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub compact_batches: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub compacting_batches: DeleteOnDropGauge<AtomicU64, Vec<String>>,
     pub noncompact_batches: DeleteOnDropGauge<AtomicU64, Vec<String>>,
-    pub schema_registry_version_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
 }
 
 impl ShardMetrics {
@@ -1575,12 +1533,6 @@ impl ShardMetrics {
         ShardMetrics {
             shard_id: *shard_id,
             name: name.to_string(),
-            since: shards_metrics
-                .since
-                .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
-            upper: shards_metrics
-                .upper
-                .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
             latest_rollup_size: shards_metrics
                 .encoded_rollup_size
                 .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
@@ -1597,7 +1549,6 @@ impl ShardMetrics {
                 .batch_part_count
                 .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
             batch_part_version_count: shards_metrics.batch_part_version_count.clone(),
-            batch_part_version_bytes: shards_metrics.batch_part_version_bytes.clone(),
             batch_part_version_map: Mutex::new(BTreeMap::new()),
             update_count: shards_metrics
                 .update_count
@@ -1644,9 +1595,6 @@ impl ShardMetrics {
             usage_leaked_bytes: shards_metrics
                 .usage_leaked_bytes
                 .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
-            stale_version: shards_metrics
-                .stale_version
-                .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
             blob_gets: shards_metrics
                 .blob_gets
                 .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
@@ -1659,9 +1607,6 @@ impl ShardMetrics {
             inline_part_count: shards_metrics
                 .inline_part_count
                 .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
-            inline_part_bytes: shards_metrics
-                .inline_part_bytes
-                .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
             compact_batches: shards_metrics
                 .compact_batches
                 .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
@@ -1670,24 +1615,13 @@ impl ShardMetrics {
                 .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
             noncompact_batches: shards_metrics
                 .noncompact_batches
-                .get_delete_on_drop_metric(vec![shard.clone(), name.to_string()]),
-            schema_registry_version_count: shards_metrics
-                .schema_registry_version_count
                 .get_delete_on_drop_metric(vec![shard, name.to_string()]),
         }
     }
 
-    pub fn set_since<T: Codec64>(&self, since: &Antichain<T>) {
-        self.since.set(encode_ts_metric(since))
-    }
-
-    pub fn set_upper<T: Codec64>(&self, upper: &Antichain<T>) {
-        self.upper.set(encode_ts_metric(upper))
-    }
-
     pub(crate) fn set_batch_part_versions<'a>(
         &self,
-        batch_parts_by_version: impl Iterator<Item = (&'a str, usize)>,
+        batch_parts_by_version: impl Iterator<Item = &'a str>,
     ) {
         let mut map = self
             .batch_part_version_map
@@ -1701,12 +1635,11 @@ impl ShardMetrics {
         // map). First reset everything.
         for x in map.values() {
             x.batch_part_version_count.set(0);
-            x.batch_part_version_bytes.set(0);
         }
 
         // Then go through the iterator, creating new entries as necessary and
         // adding.
-        for (key, bytes) in batch_parts_by_version {
+        for key in batch_parts_by_version {
             if !map.contains_key(key) {
                 map.insert(
                     key.to_owned(),
@@ -1718,27 +1651,18 @@ impl ShardMetrics {
                                 self.name.clone(),
                                 key.to_owned(),
                             ]),
-                        batch_part_version_bytes: self
-                            .batch_part_version_bytes
-                            .get_delete_on_drop_metric(vec![
-                                self.shard_id.to_string(),
-                                self.name.clone(),
-                                key.to_owned(),
-                            ]),
                     },
                 );
             }
             let value = map.get(key).expect("inserted above");
             value.batch_part_version_count.inc();
-            value.batch_part_version_bytes.add(u64::cast_from(bytes));
         }
     }
 }
 
 #[derive(Debug)]
 pub struct BatchPartVersionMetrics {
     pub batch_part_version_count: DeleteOnDropGauge<AtomicU64, Vec<String>>,
-    pub batch_part_version_bytes: DeleteOnDropGauge<AtomicU64, Vec<String>>,
 }
 
 /// Metrics recorded by audits of persist usage

src/persist-client/src/internal/state.rs

@@ -2488,7 +2488,6 @@ where
                 for x in x.parts.iter() {
                     if x.is_inline() {
                         ret.inline_part_count += 1;
-                        ret.inline_part_bytes += x.inline_bytes();
                     }
                 }
                 ret.largest_batch_bytes = std::cmp::max(ret.largest_batch_bytes, batch_size);
@@ -2860,7 +2859,6 @@ pub struct StateSizeMetrics {
     pub state_rollups_bytes: usize,
     pub state_rollup_count: usize,
     pub inline_part_count: usize,
-    pub inline_part_bytes: usize,
 }
 
 #[derive(Default)]