check ping if healthz returns non 200

pkg/monitor/cluster/cluster.go

@@ -137,7 +137,7 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) {
 		})
 	}
 
-	// If API is not returning 200, don't need to run the next checks
+	// If API is not returning 200, fallback to checking ping and short circuit the rest of the checks
 	statusCode, err := mon.emitAPIServerHealthzCode(ctx)
 	if err != nil {
 		errs = append(errs, err)
@@ -146,6 +146,11 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) {
 		mon.emitGauge("monitor.clustererrors", 1, map[string]string{"monitor": friendlyFuncName})
 	}
 	if statusCode != http.StatusOK {
+		err := mon.emitAPIServerPingCode(ctx)
+		if err != nil {
+			errs = append(errs, err)
+			mon.log.Printf("%s: %s", steps.FriendlyName(mon.emitAPIServerPingCode), err)
+		}
 		return
 	}
 	for _, f := range []func(context.Context) error{

pkg/monitor/cluster/healthz.go

@@ -23,3 +23,19 @@ func (mon *Monitor) emitAPIServerHealthzCode(ctx context.Context) (int, error) {
 
 	return statusCode, err
 }
+
+func (mon *Monitor) emitAPIServerPingCode(ctx context.Context) error {
+	var statusCode int
+	err := mon.cli.Discovery().RESTClient().
+		Get().
+		AbsPath("/healthz/ping").
+		Do(ctx).
+		StatusCode(&statusCode).
+		Error()
+
+	mon.emitGauge("apiserver.healthz.ping.code", 1, map[string]string{
+		"code": strconv.FormatInt(int64(statusCode), 10),
+	})
+
+	return err
+}