From 6cfcd6eb324806efede2d8f18399d53580c558b8 Mon Sep 17 00:00:00 2001 From: Spencer Amann Date: Tue, 11 Jul 2023 10:13:07 -0400 Subject: [PATCH] emit a metric when we fail to gather ping metric --- pkg/monitor/cluster/cluster.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/monitor/cluster/cluster.go b/pkg/monitor/cluster/cluster.go index 87d5d50a33e..228405061b7 100644 --- a/pkg/monitor/cluster/cluster.go +++ b/pkg/monitor/cluster/cluster.go @@ -141,16 +141,14 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) { statusCode, err := mon.emitAPIServerHealthzCode(ctx) if err != nil { errs = append(errs, err) - friendlyFuncName := steps.FriendlyName(mon.emitAPIServerHealthzCode) - mon.log.Printf("%s: %s", friendlyFuncName, err) - mon.emitGauge("monitor.clustererrors", 1, map[string]string{"monitor": friendlyFuncName}) + mon.emitFailureToGatherMetric(steps.FriendlyName(mon.emitAPIServerHealthzCode), err) } // If API is not returning 200, fallback to checking ping and short circuit the rest of the checks if statusCode != http.StatusOK { err := mon.emitAPIServerPingCode(ctx) if err != nil { errs = append(errs, err) - mon.log.Printf("%s: %s", steps.FriendlyName(mon.emitAPIServerPingCode), err) + mon.emitFailureToGatherMetric(steps.FriendlyName(mon.emitAPIServerPingCode), err) } return } @@ -181,9 +179,7 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) { err = f(ctx) if err != nil { errs = append(errs, err) - friendlyFuncName := steps.FriendlyName(f) - mon.log.Printf("%s: %s", friendlyFuncName, err) - mon.emitGauge("monitor.clustererrors", 1, map[string]string{"monitor": friendlyFuncName}) + mon.emitFailureToGatherMetric(steps.FriendlyName(f), err) // keep going } } @@ -191,6 +187,11 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) { return } +func (mon *Monitor) emitFailureToGatherMetric(friendlyFuncName string, err error) { + mon.log.Printf("%s: %s", friendlyFuncName, err) + mon.emitGauge("monitor.clustererrors", 1, map[string]string{"monitor": friendlyFuncName}) +} + func (mon *Monitor) emitGauge(m string, value int64, dims map[string]string) { if dims == nil { dims = map[string]string{}