From ef005c6947a9efced7cb39c8ae0a32f6607b2fad Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Mon, 13 Jan 2025 16:31:36 -0800 Subject: [PATCH 1/3] fix pd client metrics registration Signed-off-by: artem_danilov --- client/metrics/metrics.go | 22 ++++++++++++++++++++ client/pkg/circuitbreaker/circuit_breaker.go | 10 +++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/client/metrics/metrics.go b/client/metrics/metrics.go index 67268c826f5..1c8c152ea2d 100644 --- a/client/metrics/metrics.go +++ b/client/metrics/metrics.go @@ -15,6 +15,7 @@ package metrics import ( + "sync" "sync/atomic" "github.com/prometheus/client_golang/prometheus" @@ -26,6 +27,26 @@ var initialized int32 func init() { initMetrics(prometheus.Labels{}) initCmdDurations() + initRegisteredConsumers() +} + +var mutex sync.Mutex +var consumersInitializers []func() + +// RegisterConsumer registers a consumer to be initialized when the metrics are (re)initialized. +func RegisterConsumer(initConsumer func()) { + mutex.Lock() + defer mutex.Unlock() + consumersInitializers = append(consumersInitializers, initConsumer) + initConsumer() +} + +func initRegisteredConsumers() { + mutex.Lock() + defer mutex.Unlock() + for _, initConsumer := range consumersInitializers { + initConsumer() + } } // InitAndRegisterMetrics initializes and registers the metrics manually. @@ -34,6 +55,7 @@ func InitAndRegisterMetrics(constLabels prometheus.Labels) { // init metrics with constLabels initMetrics(constLabels) initCmdDurations() + initRegisteredConsumers() // register metrics registerMetrics() } diff --git a/client/pkg/circuitbreaker/circuit_breaker.go b/client/pkg/circuitbreaker/circuit_breaker.go index 0acee5d5c8d..f2abed9f80c 100644 --- a/client/pkg/circuitbreaker/circuit_breaker.go +++ b/client/pkg/circuitbreaker/circuit_breaker.go @@ -109,12 +109,18 @@ func NewCircuitBreaker(name string, st Settings) *CircuitBreaker { cb.config = &st cb.state = cb.newState(time.Now(), StateClosed) - metricName := replacer.Replace(name) + m.RegisterConsumer(func() { + registerMetrics(cb) + }) + return cb +} + +func registerMetrics(cb *CircuitBreaker) { + metricName := replacer.Replace(cb.name) cb.successCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "success") cb.errorCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "error") cb.overloadCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "overload") cb.fastFailCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "fast_fail") - return cb } // ChangeSettings changes the CircuitBreaker settings. From 029233b845c1e72de10b9a3cbb434134340af8e5 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Wed, 22 Jan 2025 17:18:24 -0800 Subject: [PATCH 2/3] wrap list and mutex into struct Signed-off-by: artem_danilov --- client/metrics/metrics.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/client/metrics/metrics.go b/client/metrics/metrics.go index 1c8c152ea2d..2ced64c2d48 100644 --- a/client/metrics/metrics.go +++ b/client/metrics/metrics.go @@ -30,21 +30,23 @@ func init() { initRegisteredConsumers() } -var mutex sync.Mutex -var consumersInitializers []func() +var consumersInitializers = struct { + sync.Mutex + value []func() +}{} // RegisterConsumer registers a consumer to be initialized when the metrics are (re)initialized. func RegisterConsumer(initConsumer func()) { - mutex.Lock() - defer mutex.Unlock() - consumersInitializers = append(consumersInitializers, initConsumer) + consumersInitializers.Lock() + defer consumersInitializers.Unlock() + consumersInitializers.value = append(consumersInitializers.value, initConsumer) initConsumer() } func initRegisteredConsumers() { - mutex.Lock() - defer mutex.Unlock() - for _, initConsumer := range consumersInitializers { + consumersInitializers.Lock() + defer consumersInitializers.Unlock() + for _, initConsumer := range consumersInitializers.value { initConsumer() } } From f9a6540e14dde20583958a7c30ced5d05f3087f1 Mon Sep 17 00:00:00 2001 From: artem_danilov Date: Wed, 22 Jan 2025 17:30:03 -0800 Subject: [PATCH 3/3] retest Signed-off-by: artem_danilov --- client/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/metrics/metrics.go b/client/metrics/metrics.go index 2ced64c2d48..da7637b19be 100644 --- a/client/metrics/metrics.go +++ b/client/metrics/metrics.go @@ -35,7 +35,7 @@ var consumersInitializers = struct { value []func() }{} -// RegisterConsumer registers a consumer to be initialized when the metrics are (re)initialized. +// RegisterConsumer registers a consumer to be initialized when the metrics are (re)initialized func RegisterConsumer(initConsumer func()) { consumersInitializers.Lock() defer consumersInitializers.Unlock()