[ISSUE-5705] Support for degraded health

core/src/main/java/com/linecorp/armeria/server/healthcheck/CpuHealthChecker.java

@@ -51,9 +51,9 @@ final class CpuHealthChecker implements HealthChecker {
 
     private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup();
 
-    private static final DoubleSupplier currentSystemCpuUsageSupplier;
+    private static final DoubleSupplier DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER;
 
-    private static final DoubleSupplier currentProcessCpuUsageSupplier;
+    private static final DoubleSupplier DEFAULT_PROCESS_CPU_USAGE_SUPPLIER;
 
     @Nullable
     private static final OperatingSystemMXBean operatingSystemBean;
@@ -78,8 +78,8 @@ final class CpuHealthChecker implements HealthChecker {
         final MethodHandle getCpuLoad = detectMethod("getCpuLoad");
         systemCpuLoad = getCpuLoad != null ? getCpuLoad : detectMethod("getSystemCpuLoad");
         processCpuLoad = detectMethod("getProcessCpuLoad");
-        currentSystemCpuUsageSupplier = () -> invoke(systemCpuLoad);
-        currentProcessCpuUsageSupplier = () -> invoke(processCpuLoad);
+        DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER = () -> invoke(systemCpuLoad);
+        DEFAULT_PROCESS_CPU_USAGE_SUPPLIER = () -> invoke(processCpuLoad);
     }
 
     private final DoubleSupplier systemCpuUsageSupplier;
@@ -92,6 +92,12 @@ final class CpuHealthChecker implements HealthChecker {
     @VisibleForTesting
     final double targetSystemCpuUsage;
 
+    @VisibleForTesting
+    final double degradedTargetProcessCpuLoad;
+
+    @VisibleForTesting
+    final double degradedTargetSystemCpuUsage;
+
     /**
      * Instantiates a new Default cpu health checker.
      *
@@ -100,23 +106,50 @@ final class CpuHealthChecker implements HealthChecker {
      */
     CpuHealthChecker(double targetSystemCpuUsage, double targetProcessCpuUsage) {
         this(targetSystemCpuUsage, targetProcessCpuUsage,
-             currentSystemCpuUsageSupplier, currentProcessCpuUsageSupplier);
+             targetSystemCpuUsage, targetProcessCpuUsage,
+             DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER, DEFAULT_PROCESS_CPU_USAGE_SUPPLIER);
+    }
+
+    /**
+     * Instantiates a new Default cpu health checker.
+     *
+     * @param targetSystemCpuUsage the target cpu usage
+     * @param targetProcessCpuLoad the target process cpu load
+     * @param degradedTargetSystemCpuUsage the degraded target cpu usage
+     * @param degradedTargetProcessCpuLoad the degraded target process cpu load
+     */
+    CpuHealthChecker(double targetSystemCpuUsage, double targetProcessCpuLoad,
+                     double degradedTargetSystemCpuUsage, double degradedTargetProcessCpuLoad) {
+        this(targetSystemCpuUsage, targetProcessCpuLoad,
+             degradedTargetSystemCpuUsage, degradedTargetProcessCpuLoad,
+             DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER, DEFAULT_PROCESS_CPU_USAGE_SUPPLIER);
     }
 
     private CpuHealthChecker(double targetSystemCpuUsage, double targetProcessCpuLoad,
+                             double degradedTargetSystemCpuUsage, double degradedTargetProcessCpuLoad,
                              DoubleSupplier systemCpuUsageSupplier, DoubleSupplier processCpuUsageSupplier) {
         checkArgument(targetSystemCpuUsage >= 0 && targetSystemCpuUsage <= 1.0,
                       "cpuUsage: %s (expected: 0 <= cpuUsage <= 1)", targetSystemCpuUsage);
         checkArgument(targetProcessCpuLoad >= 0 && targetProcessCpuLoad <= 1.0,
                       "processCpuLoad: %s (expected: 0 <= processCpuLoad <= 1)", targetProcessCpuLoad);
+        checkArgument(degradedTargetSystemCpuUsage >= targetSystemCpuUsage &&
+                      degradedTargetSystemCpuUsage <= 1.0,
+                      "degradedCpuUsage: %s (expected: %s <= degradedCpuUsage <= 1)",
+                      degradedTargetSystemCpuUsage, targetSystemCpuUsage);
+        checkArgument(degradedTargetProcessCpuLoad >= targetProcessCpuLoad &&
+                      degradedTargetProcessCpuLoad <= 1.0,
+                      "degradedProcessCpuLoad: %s (expected: %s <= degradedProcessCpuLoad <= 1)",
+                      degradedTargetProcessCpuLoad, targetProcessCpuLoad);
         this.targetSystemCpuUsage = targetSystemCpuUsage;
         this.targetProcessCpuLoad = targetProcessCpuLoad;
+        this.degradedTargetSystemCpuUsage = degradedTargetSystemCpuUsage;
+        this.degradedTargetProcessCpuLoad = degradedTargetProcessCpuLoad;
         this.systemCpuUsageSupplier = systemCpuUsageSupplier;
         this.processCpuUsageSupplier = processCpuUsageSupplier;
         checkState(operatingSystemBeanClass != null, "Unable to find an 'OperatingSystemMXBean' class");
         checkState(operatingSystemBean != null, "Unable to find an 'OperatingSystemMXBean'");
         checkState(systemCpuLoad != null, "Unable to find the method 'OperatingSystemMXBean.getCpuLoad'" +
-                                           " or 'OperatingSystemMXBean.getSystemCpuLoad'");
+                                          " or 'OperatingSystemMXBean.getSystemCpuLoad'");
         checkState(processCpuLoad != null,
                    "Unable to find the method 'OperatingSystemMXBean.getProcessCpuLoad'");
     }
@@ -167,13 +200,26 @@ private static Class<?> getFirstClassFound(final List<String> classNames) {
      */
     @Override
     public boolean isHealthy() {
-        return isHealthy(systemCpuUsageSupplier, processCpuUsageSupplier);
+        return healthStatus().isHealthy();
     }
 
-    private boolean isHealthy(
-            DoubleSupplier currentSystemCpuUsageSupplier, DoubleSupplier currentProcessCpuUsageSupplier) {
-        final double currentSystemCpuUsage = currentSystemCpuUsageSupplier.getAsDouble();
-        final double currentProcessCpuUsage = currentProcessCpuUsageSupplier.getAsDouble();
-        return currentSystemCpuUsage <= targetSystemCpuUsage && currentProcessCpuUsage <= targetProcessCpuLoad;
+    /**
+     * Returns the {@link HealthStatus} representing the System CPU Usage and Processes cpu usage.
+     */
+    @Override
+    public HealthStatus healthStatus() {
+        final double currentSystemCpuUsage = systemCpuUsageSupplier.getAsDouble();
+        final double currentProcessCpuUsage = processCpuUsageSupplier.getAsDouble();
+
+        if (currentSystemCpuUsage <= targetSystemCpuUsage && currentProcessCpuUsage <= targetProcessCpuLoad) {
+            return HealthStatus.HEALTHY;
+        }
+
+        if (currentSystemCpuUsage <= degradedTargetSystemCpuUsage &&
+            currentProcessCpuUsage <= degradedTargetProcessCpuLoad) {
+            return HealthStatus.DEGRADED;
+        }
+
+        return HealthStatus.UNHEALTHY;
     }
 }

core/src/main/java/com/linecorp/armeria/server/healthcheck/DefaultHealthCheckUpdateHandler.java

@@ -32,8 +32,13 @@
 import com.linecorp.armeria.common.MediaType;
 import com.linecorp.armeria.common.util.UnmodifiableFuture;
 import com.linecorp.armeria.server.HttpStatusException;
+import com.linecorp.armeria.server.Server;
 import com.linecorp.armeria.server.ServiceRequestContext;
 
+/**
+ * Handler which updates the healthiness of the {@link Server}. Supports {@code PUT}, {@code POST} and
+ * {@code PATCH} requests and tells if the {@link Server} needs to be marked as healthy or unhealthy.
+ */
 enum DefaultHealthCheckUpdateHandler implements HealthCheckUpdateHandler {
 
     INSTANCE;
@@ -62,16 +67,30 @@ private static HealthCheckUpdateResult handlePut(AggregatedHttpRequest req) {
             throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
         }
 
-        final JsonNode healthy = json.get("healthy");
-        if (healthy == null) {
-            throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
-        }
-        if (healthy.getNodeType() != JsonNodeType.BOOLEAN) {
+        if (json.has("healthy")) {
+            final JsonNode jsonNode = json.get("healthy");
+            if (jsonNode == null) {
+                throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
+            }
+            if (jsonNode.getNodeType() != JsonNodeType.BOOLEAN) {
+                throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
+            }
+
+            return jsonNode.booleanValue() ? HealthCheckUpdateResult.HEALTHY
+                                           : HealthCheckUpdateResult.UNHEALTHY;
+        } else if (json.has("status")) {
+            final JsonNode status = json.get("status");
+            if (status == null) {
+                throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
+            }
+            if (status.getNodeType() != JsonNodeType.STRING) {
+                throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
+            }
+
+            return getHealthCheckUpdateResult(status);
+        } else {
             throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
         }
-
-        return healthy.booleanValue() ? HealthCheckUpdateResult.HEALTHY
-                                      : HealthCheckUpdateResult.UNHEALTHY;
     }
 
     private static HealthCheckUpdateResult handlePatch(AggregatedHttpRequest req) {
@@ -86,14 +105,21 @@ private static HealthCheckUpdateResult handlePatch(AggregatedHttpRequest req) {
         final JsonNode path = patchCommand.get("path");
         final JsonNode value = patchCommand.get("value");
         if (op == null || path == null || value == null ||
-            !"replace".equals(op.textValue()) ||
-            !"/healthy".equals(path.textValue()) ||
-            !value.isBoolean()) {
+            !"replace".equals(op.textValue())) {
             throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
         }
 
-        return value.booleanValue() ? HealthCheckUpdateResult.HEALTHY
-                                    : HealthCheckUpdateResult.UNHEALTHY;
+        if ("/healthy".equals(path.textValue()) && value.isBoolean()) {
+            return value.isBoolean() ? value.booleanValue() ? HealthCheckUpdateResult.HEALTHY
+                                                            : HealthCheckUpdateResult.UNHEALTHY
+                                     : HealthCheckUpdateResult.UNHEALTHY;
+        }
+
+        if ("/status".equals(path.textValue()) && value.isTextual()) {
+            return getHealthCheckUpdateResult(value);
+        }
+
+        throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
     }
 
     private static JsonNode toJsonNode(AggregatedHttpRequest req) {
@@ -111,4 +137,21 @@ private static JsonNode toJsonNode(AggregatedHttpRequest req) {
             throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
         }
     }
+
+    private static HealthCheckUpdateResult getHealthCheckUpdateResult(JsonNode status) {
+        switch (status.textValue()) {
+            case "HEALTHY":
+                return HealthCheckUpdateResult.HEALTHY;
+            case "DEGRADED":
+                return HealthCheckUpdateResult.DEGRADED;
+            case "STOPPING":
+                return HealthCheckUpdateResult.STOPPING;
+            case "UNHEALTHY":
+                return HealthCheckUpdateResult.UNHEALTHY;
+            case "UNDER_MAINTENANCE":
+                return HealthCheckUpdateResult.UNDER_MAINTENANCE;
+            default:
+                throw HttpStatusException.of(HttpStatus.BAD_REQUEST);
+        }
+    }
 }