diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/CpuHealthChecker.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/CpuHealthChecker.java index bc040182a72..d8ad50c87eb 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/CpuHealthChecker.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/CpuHealthChecker.java @@ -51,9 +51,9 @@ final class CpuHealthChecker implements HealthChecker { private static final MethodHandles.Lookup LOOKUP = MethodHandles.lookup(); - private static final DoubleSupplier currentSystemCpuUsageSupplier; + private static final DoubleSupplier DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER; - private static final DoubleSupplier currentProcessCpuUsageSupplier; + private static final DoubleSupplier DEFAULT_PROCESS_CPU_USAGE_SUPPLIER; @Nullable private static final OperatingSystemMXBean operatingSystemBean; @@ -78,8 +78,8 @@ final class CpuHealthChecker implements HealthChecker { final MethodHandle getCpuLoad = detectMethod("getCpuLoad"); systemCpuLoad = getCpuLoad != null ? getCpuLoad : detectMethod("getSystemCpuLoad"); processCpuLoad = detectMethod("getProcessCpuLoad"); - currentSystemCpuUsageSupplier = () -> invoke(systemCpuLoad); - currentProcessCpuUsageSupplier = () -> invoke(processCpuLoad); + DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER = () -> invoke(systemCpuLoad); + DEFAULT_PROCESS_CPU_USAGE_SUPPLIER = () -> invoke(processCpuLoad); } private final DoubleSupplier systemCpuUsageSupplier; @@ -92,6 +92,12 @@ final class CpuHealthChecker implements HealthChecker { @VisibleForTesting final double targetSystemCpuUsage; + @VisibleForTesting + final double degradedTargetProcessCpuLoad; + + @VisibleForTesting + final double degradedTargetSystemCpuUsage; + /** * Instantiates a new Default cpu health checker. * @@ -100,23 +106,50 @@ final class CpuHealthChecker implements HealthChecker { */ CpuHealthChecker(double targetSystemCpuUsage, double targetProcessCpuUsage) { this(targetSystemCpuUsage, targetProcessCpuUsage, - currentSystemCpuUsageSupplier, currentProcessCpuUsageSupplier); + targetSystemCpuUsage, targetProcessCpuUsage, + DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER, DEFAULT_PROCESS_CPU_USAGE_SUPPLIER); + } + + /** + * Instantiates a new Default cpu health checker. + * + * @param targetSystemCpuUsage the target cpu usage + * @param targetProcessCpuLoad the target process cpu load + * @param degradedTargetSystemCpuUsage the degraded target cpu usage + * @param degradedTargetProcessCpuLoad the degraded target process cpu load + */ + CpuHealthChecker(double targetSystemCpuUsage, double targetProcessCpuLoad, + double degradedTargetSystemCpuUsage, double degradedTargetProcessCpuLoad) { + this(targetSystemCpuUsage, targetProcessCpuLoad, + degradedTargetSystemCpuUsage, degradedTargetProcessCpuLoad, + DEFAULT_SYSTEM_CPU_USAGE_SUPPLIER, DEFAULT_PROCESS_CPU_USAGE_SUPPLIER); } private CpuHealthChecker(double targetSystemCpuUsage, double targetProcessCpuLoad, + double degradedTargetSystemCpuUsage, double degradedTargetProcessCpuLoad, DoubleSupplier systemCpuUsageSupplier, DoubleSupplier processCpuUsageSupplier) { checkArgument(targetSystemCpuUsage >= 0 && targetSystemCpuUsage <= 1.0, "cpuUsage: %s (expected: 0 <= cpuUsage <= 1)", targetSystemCpuUsage); checkArgument(targetProcessCpuLoad >= 0 && targetProcessCpuLoad <= 1.0, "processCpuLoad: %s (expected: 0 <= processCpuLoad <= 1)", targetProcessCpuLoad); + checkArgument(degradedTargetSystemCpuUsage >= targetSystemCpuUsage && + degradedTargetSystemCpuUsage <= 1.0, + "degradedCpuUsage: %s (expected: %s <= degradedCpuUsage <= 1)", + degradedTargetSystemCpuUsage, targetSystemCpuUsage); + checkArgument(degradedTargetProcessCpuLoad >= targetProcessCpuLoad && + degradedTargetProcessCpuLoad <= 1.0, + "degradedProcessCpuLoad: %s (expected: %s <= degradedProcessCpuLoad <= 1)", + degradedTargetProcessCpuLoad, targetProcessCpuLoad); this.targetSystemCpuUsage = targetSystemCpuUsage; this.targetProcessCpuLoad = targetProcessCpuLoad; + this.degradedTargetSystemCpuUsage = degradedTargetSystemCpuUsage; + this.degradedTargetProcessCpuLoad = degradedTargetProcessCpuLoad; this.systemCpuUsageSupplier = systemCpuUsageSupplier; this.processCpuUsageSupplier = processCpuUsageSupplier; checkState(operatingSystemBeanClass != null, "Unable to find an 'OperatingSystemMXBean' class"); checkState(operatingSystemBean != null, "Unable to find an 'OperatingSystemMXBean'"); checkState(systemCpuLoad != null, "Unable to find the method 'OperatingSystemMXBean.getCpuLoad'" + - " or 'OperatingSystemMXBean.getSystemCpuLoad'"); + " or 'OperatingSystemMXBean.getSystemCpuLoad'"); checkState(processCpuLoad != null, "Unable to find the method 'OperatingSystemMXBean.getProcessCpuLoad'"); } @@ -167,13 +200,26 @@ private static Class getFirstClassFound(final List classNames) { */ @Override public boolean isHealthy() { - return isHealthy(systemCpuUsageSupplier, processCpuUsageSupplier); + return healthStatus().isHealthy(); } - private boolean isHealthy( - DoubleSupplier currentSystemCpuUsageSupplier, DoubleSupplier currentProcessCpuUsageSupplier) { - final double currentSystemCpuUsage = currentSystemCpuUsageSupplier.getAsDouble(); - final double currentProcessCpuUsage = currentProcessCpuUsageSupplier.getAsDouble(); - return currentSystemCpuUsage <= targetSystemCpuUsage && currentProcessCpuUsage <= targetProcessCpuLoad; + /** + * Returns the {@link HealthStatus} representing the System CPU Usage and Processes cpu usage. + */ + @Override + public HealthStatus healthStatus() { + final double currentSystemCpuUsage = systemCpuUsageSupplier.getAsDouble(); + final double currentProcessCpuUsage = processCpuUsageSupplier.getAsDouble(); + + if (currentSystemCpuUsage <= targetSystemCpuUsage && currentProcessCpuUsage <= targetProcessCpuLoad) { + return HealthStatus.HEALTHY; + } + + if (currentSystemCpuUsage <= degradedTargetSystemCpuUsage && + currentProcessCpuUsage <= degradedTargetProcessCpuLoad) { + return HealthStatus.DEGRADED; + } + + return HealthStatus.UNHEALTHY; } } diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/DefaultHealthCheckUpdateHandler.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/DefaultHealthCheckUpdateHandler.java index 726ad94a7ea..526b26f6cf5 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/DefaultHealthCheckUpdateHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/DefaultHealthCheckUpdateHandler.java @@ -32,8 +32,13 @@ import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.server.HttpStatusException; +import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServiceRequestContext; +/** + * Handler which updates the healthiness of the {@link Server}. Supports {@code PUT}, {@code POST} and + * {@code PATCH} requests and tells if the {@link Server} needs to be marked as healthy or unhealthy. + */ enum DefaultHealthCheckUpdateHandler implements HealthCheckUpdateHandler { INSTANCE; @@ -62,16 +67,30 @@ private static HealthCheckUpdateResult handlePut(AggregatedHttpRequest req) { throw HttpStatusException.of(HttpStatus.BAD_REQUEST); } - final JsonNode healthy = json.get("healthy"); - if (healthy == null) { - throw HttpStatusException.of(HttpStatus.BAD_REQUEST); - } - if (healthy.getNodeType() != JsonNodeType.BOOLEAN) { + if (json.has("healthy")) { + final JsonNode jsonNode = json.get("healthy"); + if (jsonNode == null) { + throw HttpStatusException.of(HttpStatus.BAD_REQUEST); + } + if (jsonNode.getNodeType() != JsonNodeType.BOOLEAN) { + throw HttpStatusException.of(HttpStatus.BAD_REQUEST); + } + + return jsonNode.booleanValue() ? HealthCheckUpdateResult.HEALTHY + : HealthCheckUpdateResult.UNHEALTHY; + } else if (json.has("status")) { + final JsonNode status = json.get("status"); + if (status == null) { + throw HttpStatusException.of(HttpStatus.BAD_REQUEST); + } + if (status.getNodeType() != JsonNodeType.STRING) { + throw HttpStatusException.of(HttpStatus.BAD_REQUEST); + } + + return getHealthCheckUpdateResult(status); + } else { throw HttpStatusException.of(HttpStatus.BAD_REQUEST); } - - return healthy.booleanValue() ? HealthCheckUpdateResult.HEALTHY - : HealthCheckUpdateResult.UNHEALTHY; } private static HealthCheckUpdateResult handlePatch(AggregatedHttpRequest req) { @@ -86,14 +105,21 @@ private static HealthCheckUpdateResult handlePatch(AggregatedHttpRequest req) { final JsonNode path = patchCommand.get("path"); final JsonNode value = patchCommand.get("value"); if (op == null || path == null || value == null || - !"replace".equals(op.textValue()) || - !"/healthy".equals(path.textValue()) || - !value.isBoolean()) { + !"replace".equals(op.textValue())) { throw HttpStatusException.of(HttpStatus.BAD_REQUEST); } - return value.booleanValue() ? HealthCheckUpdateResult.HEALTHY - : HealthCheckUpdateResult.UNHEALTHY; + if ("/healthy".equals(path.textValue()) && value.isBoolean()) { + return value.isBoolean() ? value.booleanValue() ? HealthCheckUpdateResult.HEALTHY + : HealthCheckUpdateResult.UNHEALTHY + : HealthCheckUpdateResult.UNHEALTHY; + } + + if ("/status".equals(path.textValue()) && value.isTextual()) { + return getHealthCheckUpdateResult(value); + } + + throw HttpStatusException.of(HttpStatus.BAD_REQUEST); } private static JsonNode toJsonNode(AggregatedHttpRequest req) { @@ -111,4 +137,21 @@ private static JsonNode toJsonNode(AggregatedHttpRequest req) { throw HttpStatusException.of(HttpStatus.BAD_REQUEST); } } + + private static HealthCheckUpdateResult getHealthCheckUpdateResult(JsonNode status) { + switch (status.textValue()) { + case "HEALTHY": + return HealthCheckUpdateResult.HEALTHY; + case "DEGRADED": + return HealthCheckUpdateResult.DEGRADED; + case "STOPPING": + return HealthCheckUpdateResult.STOPPING; + case "UNHEALTHY": + return HealthCheckUpdateResult.UNHEALTHY; + case "UNDER_MAINTENANCE": + return HealthCheckUpdateResult.UNDER_MAINTENANCE; + default: + throw HttpStatusException.of(HttpStatus.BAD_REQUEST); + } + } } diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckService.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckService.java index a45b3a5514b..eead1306630 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckService.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckService.java @@ -23,6 +23,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,6 +92,12 @@ *
  • {@code Prefer: wait=}
    • * * + *

    To wait for specific server {@link HealthStatus}, send an HTTP request with two additional headers: + *

    + * *

    The {@link Server} will wait up to the amount of seconds specified in the {@code "Prefer"} header * and respond with {@code "200 OK"}, {@code "503 Service Unavailable"} or {@code "304 Not Modified"}. * {@code "304 Not Modifies"} signifies that the healthiness of the {@link Server} did not change. @@ -106,6 +115,8 @@ public final class HealthCheckService implements TransientHttpService { private static final Logger logger = LoggerFactory.getLogger(HealthCheckService.class); private static final AsciiString ARMERIA_LPHC = HttpHeaderNames.of("armeria-lphc"); private static final PendingResponse[] EMPTY_PENDING_RESPONSES = new PendingResponse[0]; + private static final Pattern IF_NONE_MATCH_SPECIFIC_STATUS = + Pattern.compile("\"status=(healthy|degraded|stopping|unhealthy|under_maintenance)\""); /** * Returns a newly created {@link HealthCheckService} with the specified {@link HealthChecker}s. @@ -131,7 +142,9 @@ public static HealthCheckServiceBuilder builder() { private final SettableHealthChecker serverHealth; private final Set healthCheckers; private final AggregatedHttpResponse healthyResponse; + private final AggregatedHttpResponse degradedResponse; private final AggregatedHttpResponse unhealthyResponse; + private final AggregatedHttpResponse underMaintenanceResponse; private final AggregatedHttpResponse stoppingResponse; private final ResponseHeaders ping; private final ResponseHeaders notModifiedHeaders; @@ -143,10 +156,7 @@ public static HealthCheckServiceBuilder builder() { private final Consumer healthCheckerListener; @Nullable @VisibleForTesting - final Set pendingHealthyResponses; - @Nullable - @VisibleForTesting - final Set pendingUnhealthyResponses; + final Set pendingResponses; @Nullable private final HealthCheckUpdateHandler updateHandler; private final boolean startHealthy; @@ -154,20 +164,21 @@ public static HealthCheckServiceBuilder builder() { @Nullable private Server server; - private boolean serverStopping; HealthCheckService(Set healthCheckers, - AggregatedHttpResponse healthyResponse, AggregatedHttpResponse unhealthyResponse, - long maxLongPollingTimeoutMillis, double longPollingTimeoutJitterRate, - long pingIntervalMillis, @Nullable HealthCheckUpdateHandler updateHandler, + AggregatedHttpResponse healthyResponse, AggregatedHttpResponse degradedResponse, + AggregatedHttpResponse stoppingResponse, AggregatedHttpResponse unhealthyResponse, + AggregatedHttpResponse underMaintenanceResponse, long maxLongPollingTimeoutMillis, + double longPollingTimeoutJitterRate, long pingIntervalMillis, + @Nullable HealthCheckUpdateHandler updateHandler, List updateListeners, boolean startHealthy, Set transientServiceOptions) { - serverHealth = new SettableHealthChecker(false); + serverHealth = new SettableHealthChecker(HealthStatus.UNHEALTHY); if (!updateListeners.isEmpty()) { addServerHealthUpdateListener(ImmutableList.copyOf(updateListeners)); } this.healthCheckers = ImmutableSet.builder() - .add(serverHealth).addAll(healthCheckers).build(); + .add(serverHealth).addAll(healthCheckers).build(); this.updateHandler = updateHandler; this.startHealthy = startHealthy; this.transientServiceOptions = transientServiceOptions; @@ -178,15 +189,13 @@ public static HealthCheckServiceBuilder builder() { this.longPollingTimeoutJitterRate = longPollingTimeoutJitterRate; this.pingIntervalMillis = pingIntervalMillis; healthCheckerListener = this::onHealthCheckerUpdate; - pendingHealthyResponses = new ObjectLinkedOpenHashSet<>(); - pendingUnhealthyResponses = new ObjectLinkedOpenHashSet<>(); + pendingResponses = new ObjectLinkedOpenHashSet<>(); } else { this.maxLongPollingTimeoutMillis = 0; this.longPollingTimeoutJitterRate = 0; this.pingIntervalMillis = 0; healthCheckerListener = null; - pendingHealthyResponses = null; - pendingUnhealthyResponses = null; + pendingResponses = null; if (maxLongPollingTimeoutMillis > 0 && logger.isWarnEnabled()) { logger.warn("Long-polling support has been disabled " + @@ -200,8 +209,10 @@ public static HealthCheckServiceBuilder builder() { } this.healthyResponse = setCommonHeaders(healthyResponse); + this.degradedResponse = setCommonHeaders(degradedResponse); this.unhealthyResponse = setCommonHeaders(unhealthyResponse); - stoppingResponse = clearCommonHeaders(unhealthyResponse); + this.underMaintenanceResponse = setCommonHeaders(underMaintenanceResponse); + this.stoppingResponse = clearCommonHeaders(stoppingResponse); notModifiedHeaders = ResponseHeaders.builder() .add(this.unhealthyResponse.headers()) .endOfStream(true) @@ -216,7 +227,7 @@ private void addServerHealthUpdateListener(ImmutableList { updateListeners.forEach(updateListener -> { try { - updateListener.healthUpdated(healthChecker.isHealthy()); + updateListener.healthStatusUpdated(healthChecker.healthStatus()); } catch (Throwable t) { logger.warn("Unexpected exception from HealthCheckUpdateListener.healthUpdated():", t); } @@ -280,7 +291,6 @@ public void serviceAdded(ServiceConfig cfg) throws Exception { server.addListener(new ServerListenerAdapter() { @Override public void serverStarting(Server server) throws Exception { - serverStopping = false; if (healthCheckerListener != null) { healthCheckers.stream().map(ListenableHealthChecker.class::cast).forEach(c -> { c.addListener(healthCheckerListener); @@ -295,14 +305,13 @@ public void serverStarting(Server server) throws Exception { @Override public void serverStarted(Server server) { if (startHealthy) { - serverHealth.setHealthy(true); + serverHealth.setHealthStatus(HealthStatus.HEALTHY); } } @Override public void serverStopping(Server server) { - serverStopping = true; - serverHealth.setHealthy(false); + serverHealth.setHealthStatus(HealthStatus.STOPPING); } @Override @@ -323,17 +332,24 @@ public void serverStopped(Server server) throws Exception { @Override public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exception { final long longPollingTimeoutMillis = getLongPollingTimeoutMillis(req); - final boolean isHealthy = isHealthy(); + final HealthStatus healthStatus = healthStatus(); final boolean useLongPolling; + boolean pollHealthStatus = false; if (longPollingTimeoutMillis > 0) { final String expectedState = Ascii.toLowerCase(req.headers().get(HttpHeaderNames.IF_NONE_MATCH, "")); if ("\"healthy\"".equals(expectedState) || "w/\"healthy\"".equals(expectedState)) { - useLongPolling = isHealthy; + useLongPolling = healthStatus.isHealthy(); } else if ("\"unhealthy\"".equals(expectedState) || "w/\"unhealthy\"".equals(expectedState)) { - useLongPolling = !isHealthy; + useLongPolling = !healthStatus.isHealthy(); } else { - useLongPolling = false; + final Matcher matcher = IF_NONE_MATCH_SPECIFIC_STATUS.matcher(expectedState); + if (matcher.matches()) { + useLongPolling = matcher.group(1).equalsIgnoreCase(healthStatus.name()); + pollHealthStatus = true; + } else { + useLongPolling = false; + } } } else { useLongPolling = false; @@ -351,19 +367,15 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exc } assert healthCheckerListener != null : "healthCheckerListener is null."; - assert pendingHealthyResponses != null : "pendingHealthyResponses is null."; - assert pendingUnhealthyResponses != null : "pendingUnhealthyResponses is null."; + assert pendingResponses != null : "pendingResponses is null."; // If healthy, wait until it becomes unhealthy, and vice versa. lock.lock(); try { - final boolean currentHealthiness = isHealthy(); - if (isHealthy == currentHealthiness) { + final HealthStatus currentHealthStatus = healthStatus(); + if (healthStatus == currentHealthStatus) { final HttpResponseWriter res = HttpResponse.streaming(); - final Set pendingResponses = isHealthy ? pendingUnhealthyResponses - : pendingHealthyResponses; - // Send the initial ack (102 Processing) to let the client know that the request // was accepted. res.write(ping); @@ -382,8 +394,8 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exc final ScheduledFuture timeoutFuture = ctx.eventLoop().withoutContext().schedule( new TimeoutTask(res), longPollingTimeoutMillis, TimeUnit.MILLISECONDS); - final PendingResponse pendingResponse = - new PendingResponse(method, res, pingFuture, timeoutFuture); + final PendingResponse pendingResponse = new PendingResponse( + method, res, pingFuture, timeoutFuture, healthStatus, pollHealthStatus); pendingResponses.add(pendingResponse); timeoutFuture.addListener((FutureListener) f -> { lock.lock(); @@ -415,7 +427,7 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exc switch (method) { case HEAD: case GET: - return newResponse(method, isHealthy); + return newResponse(method, healthStatus); case CONNECT: case DELETE: case OPTIONS: @@ -435,26 +447,26 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exc if (updateResult != null) { switch (updateResult) { case HEALTHY: - serverHealth.setHealthy(true); + serverHealth.setHealthStatus(HealthStatus.HEALTHY); + break; + case DEGRADED: + serverHealth.setHealthStatus(HealthStatus.DEGRADED); + break; + case STOPPING: + serverHealth.setHealthStatus(HealthStatus.STOPPING); break; case UNHEALTHY: - serverHealth.setHealthy(false); + serverHealth.setHealthStatus(HealthStatus.UNHEALTHY); + break; + case UNDER_MAINTENANCE: + serverHealth.setHealthStatus(HealthStatus.UNDER_MAINTENANCE); break; } } - return HttpResponse.of(newResponse(method, isHealthy())); + return HttpResponse.of(newResponse(method, healthStatus())); })); } - private boolean isHealthy() { - for (HealthChecker healthChecker : healthCheckers) { - if (!healthChecker.isHealthy()) { - return false; - } - } - return true; - } - private long getLongPollingTimeoutMillis(HttpRequest req) { if (!isLongPollingEnabled()) { return 0; @@ -491,6 +503,17 @@ private long getLongPollingTimeoutMillis(HttpRequest req) { return (long) (Math.min(timeoutMillisHolder.value, maxLongPollingTimeoutMillis) * multiplier); } + private HealthStatus healthStatus() { + HealthStatus status = HealthStatus.HEALTHY; + for (HealthChecker healthChecker : healthCheckers) { + if (healthChecker.healthStatus().priority() < status.priority()) { + status = healthChecker.healthStatus(); + } + } + + return status; + } + private boolean isLongPollingEnabled() { return healthCheckerListener != null; } @@ -507,8 +530,8 @@ private static void updateRequestTimeout(ServiceRequestContext ctx, long longPol } } - private HttpResponse newResponse(HttpMethod method, boolean isHealthy) { - final AggregatedHttpResponse aRes = getResponse(isHealthy); + private HttpResponse newResponse(HttpMethod method, HealthStatus healthStatus) { + final AggregatedHttpResponse aRes = getResponse(healthStatus); if (method == HttpMethod.HEAD) { return HttpResponse.of(aRes.headers()); @@ -517,32 +540,46 @@ private HttpResponse newResponse(HttpMethod method, boolean isHealthy) { } } - private AggregatedHttpResponse getResponse(boolean isHealthy) { - if (isHealthy) { - return healthyResponse; - } - - if (serverStopping) { - return stoppingResponse; + private AggregatedHttpResponse getResponse(HealthStatus healthStatus) { + switch (healthStatus) { + case HEALTHY: + return healthyResponse; + case DEGRADED: + return degradedResponse; + case STOPPING: + return stoppingResponse; + case UNDER_MAINTENANCE: + return underMaintenanceResponse; + default: + return unhealthyResponse; } - - return unhealthyResponse; } private void onHealthCheckerUpdate(HealthChecker unused) { assert healthCheckerListener != null : "healthCheckerListener is null."; - assert pendingHealthyResponses != null : "pendingHealthyResponses is null."; - assert pendingUnhealthyResponses != null : "pendingUnhealthyResponses is null."; + assert pendingResponses != null : "pendingResponses is null."; - final boolean isHealthy = isHealthy(); + final HealthStatus healthStatus = healthStatus(); final PendingResponse[] pendingResponses; lock.lock(); try { - final Set set = isHealthy ? pendingHealthyResponses - : pendingUnhealthyResponses; + final List set = + this.pendingResponses.stream() + .filter(res -> { + if (res.pollHealthStatus) { + return res.interestedStatus != healthStatus; + } else { + return res.interestedStatus.isHealthy() != + healthStatus.isHealthy(); + } + }) + .collect(Collectors.toList()); + if (!set.isEmpty()) { pendingResponses = set.toArray(EMPTY_PENDING_RESPONSES); - set.clear(); + for (PendingResponse res : pendingResponses) { + this.pendingResponses.remove(res); + } } else { pendingResponses = EMPTY_PENDING_RESPONSES; } @@ -550,7 +587,7 @@ private void onHealthCheckerUpdate(HealthChecker unused) { lock.unlock(); } - final AggregatedHttpResponse res = getResponse(isHealthy); + final AggregatedHttpResponse res = getResponse(healthStatus); for (PendingResponse e : pendingResponses) { if (e.cancelAllScheduledFutures()) { if (e.method == HttpMethod.HEAD) { @@ -575,15 +612,21 @@ private static final class PendingResponse { @Nullable private final ScheduledFuture pingFuture; private final ScheduledFuture timeoutFuture; + final HealthStatus interestedStatus; + final boolean pollHealthStatus; PendingResponse(HttpMethod method, HttpResponseWriter res, @Nullable ScheduledFuture pingFuture, - ScheduledFuture timeoutFuture) { + ScheduledFuture timeoutFuture, + HealthStatus interestedStatus, + boolean pollHealthStatus) { this.method = method; this.res = res; this.pingFuture = pingFuture; this.timeoutFuture = timeoutFuture; + this.interestedStatus = interestedStatus; + this.pollHealthStatus = pollHealthStatus; } boolean cancelAllScheduledFutures() { diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceBuilder.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceBuilder.java index 853847adbf8..2aa3b6b42b9 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceBuilder.java @@ -30,6 +30,7 @@ import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.annotation.UnstableApi; import com.linecorp.armeria.internal.server.TransientServiceOptionsBuilder; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.Service; @@ -42,17 +43,35 @@ */ public final class HealthCheckServiceBuilder implements TransientServiceBuilder { + private static final AggregatedHttpResponse DEFAULT_HEALTHY_RESPONSE = + AggregatedHttpResponse.of(HttpStatus.OK, MediaType.JSON_UTF_8, + "{\"healthy\":true,\"status\":\"HEALTHY\"}"); + private static final AggregatedHttpResponse DEFAULT_DEGRADED_RESPONSE = + AggregatedHttpResponse.of(HttpStatus.OK, MediaType.JSON_UTF_8, + "{\"healthy\":true,\"status\":\"DEGRADED\"}"); + private static final AggregatedHttpResponse DEFAULT_STOPPING_RESPONSE = + AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE, MediaType.JSON_UTF_8, + "{\"healthy\":false,\"status\":\"STOPPING\"}"); + private static final AggregatedHttpResponse DEFAULT_UNHEALTHY_RESPONSE = + AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE, MediaType.JSON_UTF_8, + "{\"healthy\":false,\"status\":\"UNHEALTHY\"}"); + + private static final AggregatedHttpResponse DEFAULT_UNDER_MAINTENANCE_RESPONSE = + AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE, MediaType.JSON_UTF_8, + "{\"healthy\":false,\"status\":\"UNDER_MAINTENANCE\"}"); + private static final int DEFAULT_LONG_POLLING_TIMEOUT_SECONDS = 60; private static final int DEFAULT_PING_INTERVAL_SECONDS = 5; private static final double DEFAULT_LONG_POLLING_TIMEOUT_JITTER_RATE = 0.2; private final ImmutableSet.Builder healthCheckers = ImmutableSet.builder(); - private AggregatedHttpResponse healthyResponse = AggregatedHttpResponse.of(HttpStatus.OK, - MediaType.JSON_UTF_8, - "{\"healthy\":true}"); - private AggregatedHttpResponse unhealthyResponse = AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE, - MediaType.JSON_UTF_8, - "{\"healthy\":false}"); + + private AggregatedHttpResponse healthyResponse = DEFAULT_HEALTHY_RESPONSE; + private AggregatedHttpResponse degradedResponse = DEFAULT_DEGRADED_RESPONSE; + private AggregatedHttpResponse stoppingResponse = DEFAULT_STOPPING_RESPONSE; + private AggregatedHttpResponse unhealthyResponse = DEFAULT_UNHEALTHY_RESPONSE; + private AggregatedHttpResponse underMaintenanceResponse = DEFAULT_UNDER_MAINTENANCE_RESPONSE; + private long maxLongPollingTimeoutMillis = TimeUnit.SECONDS.toMillis(DEFAULT_LONG_POLLING_TIMEOUT_SECONDS); private double longPollingTimeoutJitterRate = DEFAULT_LONG_POLLING_TIMEOUT_JITTER_RATE; private long pingIntervalMillis = TimeUnit.SECONDS.toMillis(DEFAULT_PING_INTERVAL_SECONDS); @@ -91,10 +110,10 @@ public HealthCheckServiceBuilder checkers(Iterable heal * response is sent by default: * * 
    {@code
-     * HTTP/1.1 200 OK
+     * HTTP/2 200 OK
      * Content-Type: application/json; charset=utf-8
      *
-     * { "healthy": true }
+     * { "healthy": true, "status": "HEALTHY" }
      * }
    * * @return {@code this} @@ -105,15 +124,55 @@ public HealthCheckServiceBuilder healthyResponse(AggregatedHttpResponse healthyR return this; } + /** + * Sets the {@link AggregatedHttpResponse} to send when the {@link Service} is degraded. The following + * response is sent by default: + * + * 
    {@code
+     * HTTP/2 200 OK
+     * Content-Type: application/json; charset=utf-8
+     *
+     * { "healthy": true, "status": "DEGRADED" }
+     * }
    + * + * @return {@code this} + */ + @UnstableApi + public HealthCheckServiceBuilder degradedResponse(AggregatedHttpResponse degradedResponse) { + requireNonNull(degradedResponse, "degradedResponse"); + this.degradedResponse = copyResponse(degradedResponse); + return this; + } + + /** + * Sets the {@link AggregatedHttpResponse} to send when the {@link Service} is stopping. The following + * response is sent by default: + * + * 
    {@code
+     * HTTP/2 503 Service Unavailable
+     * Content-Type: application/json; charset=utf-8
+     *
+     * { "healthy": false, "status": "STOPPING" }
+     * }
    + * + * @return {@code this} + */ + @UnstableApi + public HealthCheckServiceBuilder stoppingResponse(AggregatedHttpResponse stoppingResponse) { + requireNonNull(stoppingResponse, "stoppingResponse"); + this.stoppingResponse = copyResponse(stoppingResponse); + return this; + } + /** * Sets the {@link AggregatedHttpResponse} to send when the {@link Service} is unhealthy. The following * response is sent by default: * * 
    {@code
-     * HTTP/1.1 503 Service Unavailable
+     * HTTP/2 503 Service Unavailable
      * Content-Type: application/json; charset=utf-8
      *
-     * { "healthy": false }
+     * { "healthy": false, "status": "UNHEALTHY" }
      * }
    * * @return {@code this} @@ -124,6 +183,26 @@ public HealthCheckServiceBuilder unhealthyResponse(AggregatedHttpResponse unheal return this; } + /** + * Sets the {@link AggregatedHttpResponse} to send when the {@link Service} is under maintenance. The + * following response is sent by default: + * + * 
    {@code
+     * HTTP/2 503 Service Unavailable
+     * Content-Type: application/json; charset=utf-8
+     *
+     * { "healthy": false, "status": "UNDER_MAINTENANCE" }
+     * }
    + * + * @return {@code this} + */ + @UnstableApi + public HealthCheckServiceBuilder underMaintenanceResponse(AggregatedHttpResponse underMaintenanceResponse) { + requireNonNull(underMaintenanceResponse, "underMaintenanceResponse"); + this.underMaintenanceResponse = copyResponse(underMaintenanceResponse); + return this; + } + /** * Make a copy just in case the content is modified by the caller or is backed by ByteBuf. */ @@ -238,7 +317,11 @@ public HealthCheckServiceBuilder longPolling(long maxLongPollingTimeoutMillis, */ public HealthCheckServiceBuilder updatable(boolean updatable) { if (updatable) { - return updatable(HealthCheckUpdateHandler.of()); + if (updateHandler == null) { + return updatable(HealthCheckUpdateHandler.of()); + } + + return this; } updateHandler = null; @@ -306,8 +389,8 @@ public HealthCheckServiceBuilder transientServiceOptions( public HealthCheckService build() { checkState(startHealthy || updateHandler != null, "Healthiness must be updatable by server listener or update handler."); - return new HealthCheckService(healthCheckers.build(), - healthyResponse, unhealthyResponse, + return new HealthCheckService(healthCheckers.build(), healthyResponse, degradedResponse, + stoppingResponse, unhealthyResponse, underMaintenanceResponse, maxLongPollingTimeoutMillis, longPollingTimeoutJitterRate, pingIntervalMillis, updateHandler, updateListenersBuilder.build(), startHealthy, transientServiceOptionsBuilder.build()); diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckStatus.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckStatus.java index 11ddb8593d2..bde76c72b8b 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckStatus.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckStatus.java @@ -27,7 +27,7 @@ */ @UnstableApi public final class HealthCheckStatus { - private final boolean isHealthy; + private final HealthStatus healthStatus; private final long ttlMillis; /** @@ -37,8 +37,18 @@ public final class HealthCheckStatus { * @param ttlMillis interval for scheduling the next check */ public HealthCheckStatus(boolean isHealthy, long ttlMillis) { + this(isHealthy ? HealthStatus.HEALTHY : HealthStatus.UNHEALTHY, ttlMillis); + } + + /** + * Create the result of the health check. + * + * @param healthStatus health check result + * @param ttlMillis interval for scheduling the next check + */ + public HealthCheckStatus(HealthStatus healthStatus, long ttlMillis) { checkArgument(ttlMillis > 0, "ttlMillis: %s (expected: > 0)", ttlMillis); - this.isHealthy = isHealthy; + this.healthStatus = healthStatus; this.ttlMillis = ttlMillis; } @@ -46,7 +56,15 @@ public HealthCheckStatus(boolean isHealthy, long ttlMillis) { * Return the result of health check. */ public boolean isHealthy() { - return isHealthy; + return healthStatus.isHealthy(); + } + + /** + * Return the result of health check. + */ + @UnstableApi + public HealthStatus healthStatus() { + return healthStatus; } /** @@ -59,7 +77,7 @@ public long ttlMillis() { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("isHealthy", isHealthy) + .add("healthStatus", healthStatus) .add("ttlMillis", ttlMillis) .toString(); } diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateListener.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateListener.java index 73472e7dce5..f6e9444bfd3 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateListener.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateListener.java @@ -16,6 +16,8 @@ package com.linecorp.armeria.server.healthcheck; +import static java.util.Objects.requireNonNull; + /** * A listener interface for receiving {@link HealthCheckService} update events. */ @@ -26,4 +28,13 @@ public interface HealthCheckUpdateListener { * Invoked when the healthiness is updated. */ void healthUpdated(boolean isHealthy) throws Exception; + + /** + * Invoked when the health status is updated. Override this method for more fine-grained health status + * updates. + */ + default void healthStatusUpdated(HealthStatus healthStatus) throws Exception { + requireNonNull(healthStatus, "healthStatus"); + healthUpdated(healthStatus.isHealthy()); + } } diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateResult.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateResult.java index 2590a137f33..ce560ec7405 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateResult.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthCheckUpdateResult.java @@ -25,10 +25,22 @@ public enum HealthCheckUpdateResult { * Tells {@link HealthCheckService} to mark the {@link Server} as 'healthy'. */ HEALTHY, + /** + * Tells {@link HealthCheckService} to mark the {@link Server} as 'degraded'. + */ + DEGRADED, + /** + * Tells {@link HealthCheckService} to mark the {@link Server} as 'stopping'. + */ + STOPPING, /** * Tells {@link HealthCheckService} to mark the {@link Server} as 'unhealthy'. */ UNHEALTHY, + /** + * Tells {@link HealthCheckService} to mark the {@link Server} as 'under maintenance'. + */ + UNDER_MAINTENANCE, /** * Tells {@link HealthCheckService} to leave the {@link Server} healthiness unchanged. */ diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthChecker.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthChecker.java index 61e5f84d84a..1b00e03ad19 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthChecker.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthChecker.java @@ -26,6 +26,7 @@ import com.sun.management.OperatingSystemMXBean; import com.linecorp.armeria.common.CommonPools; +import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.annotation.UnstableApi; import com.linecorp.armeria.server.Server; @@ -105,8 +106,37 @@ static HealthChecker ofCpu(double targetSystemCpuUsage, double targetProcessCpuU return new CpuHealthChecker(targetSystemCpuUsage, targetProcessCpuUsage); } + /** + * Creates a new instance of {@link HealthChecker} which reports health based on + * cpu usage reported by {@link OperatingSystemMXBean}. + * Both system and process cpu usage must be (inclusive) lower than the specified threshold in order + * for the {@link HealthChecker} to report as healthy. If the {@link HealthChecker} is unable + * to find a suitable {@link OperatingSystemMXBean}, an exception is thrown on construction. + * + * @param targetSystemCpuUsage Target system CPU usage as a percentage (0 - 1). + * @param targetProcessCpuUsage Target process CPU usage as a percentage (0 - 1). + * @param degradedTargetSystemCpuUsage Degraded target system CPU usage as a percentage + * (targetSystemCpuUsage - 1). + * @param degradedTargetProcessCpuLoad Degraded target process CPU usage as a percentage + * (targetProcessCpuUsage - 1). + * @return an instance of {@link HealthChecker} configured with the provided CPU usage targets. + */ + static HealthChecker ofCpu(double targetSystemCpuUsage, double targetProcessCpuUsage, + double degradedTargetSystemCpuUsage, double degradedTargetProcessCpuLoad) { + return new CpuHealthChecker(targetSystemCpuUsage, targetProcessCpuUsage, + degradedTargetSystemCpuUsage, degradedTargetProcessCpuLoad); + } + /** * Returns {@code true} if and only if the {@link Server} is healthy. */ boolean isHealthy(); + + /** + * Returns the {@link HttpStatus} representing the health status of the {@link Server}. + * Override below method if you want more fine-grained health status. + */ + default HealthStatus healthStatus() { + return isHealthy() ? HealthStatus.HEALTHY : HealthStatus.UNHEALTHY; + } } diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthStatus.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthStatus.java new file mode 100644 index 00000000000..43647898ff7 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/HealthStatus.java @@ -0,0 +1,72 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.server.healthcheck; + +import com.linecorp.armeria.common.annotation.UnstableApi; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerListener; + +/** + * The health status of a {@link Server}. + */ +@UnstableApi +public enum HealthStatus { + /** + * The {@link Server} is healthy and able to serve requests. + */ + HEALTHY(500, true), + /** + * The {@link Server} is degraded and may not be able to handle requests as much as the server with + * {@link HealthStatus#HEALTHY} status. + */ + DEGRADED(400, true), + /** + * The {@link Server} is stopping and unable to serve requests. This status is set when + * {@link ServerListener#serverStopping(Server)} is called by default. + */ + STOPPING(300, false), + /** + * The {@link Server} is unhealthy and unable to serve requests. + */ + UNHEALTHY(200, false), + /** + * The {@link Server} is under maintenance and unable to serve requests. + */ + UNDER_MAINTENANCE(100, false); + + private final int priority; + private final boolean isHealthy; + + HealthStatus(int priority, boolean isHealthy) { + this.priority = priority; + this.isHealthy = isHealthy; + } + + /** + * Returns the priority of this {@link HealthStatus}. + */ + public int priority() { + return priority; + } + + /** + * Returns whether this {@link HealthStatus} is considered healthy. + */ + public boolean isHealthy() { + return isHealthy; + } +} diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthChecker.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthChecker.java index d095f08fc94..4cf68b6cd66 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthChecker.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthChecker.java @@ -20,7 +20,6 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -46,7 +45,7 @@ final class ScheduledHealthChecker extends AbstractListenable private final Duration fallbackTtl; private final EventExecutor eventExecutor; private final Consumer onHealthCheckerUpdate; - private final AtomicBoolean isHealthy = new AtomicBoolean(); + private final AtomicReference healthStatusAtomicReference; private final AtomicInteger requestCount = new AtomicInteger(); private final AtomicReference impl = new AtomicReference<>(); @@ -55,16 +54,26 @@ final class ScheduledHealthChecker extends AbstractListenable this.healthChecker = healthChecker; this.fallbackTtl = fallbackTtl; this.eventExecutor = eventExecutor; + healthStatusAtomicReference = new AtomicReference<>(HealthStatus.UNHEALTHY); onHealthCheckerUpdate = latestValue -> { - isHealthy.set(latestValue.isHealthy()); + healthStatusAtomicReference.set(latestValue.healthStatus()); notifyListeners(latestValue); }; } @Override public boolean isHealthy() { - return isHealthy.get(); + final HealthStatus healthStatus = healthStatusAtomicReference.get(); + assert healthStatus != null; + return healthStatus.isHealthy(); + } + + @Override + public HealthStatus healthStatus() { + final HealthStatus healthStatus = healthStatusAtomicReference.get(); + assert healthStatus != null; + return healthStatus; } void startHealthChecker() { @@ -177,24 +186,24 @@ private void runHealthCheck() { } try { healthChecker.get().handle((result, throwable) -> { - final boolean isHealthy; + final HealthStatus healthStatus; final long intervalMillis; if (throwable != null) { logger.warn("Health checker throws an exception, schedule the next check after {}ms.", fallbackTtl.toMillis(), throwable); - isHealthy = false; + healthStatus = HealthStatus.UNHEALTHY; intervalMillis = fallbackTtl.toMillis(); } else if (result == null) { logger.warn("Health checker returns an unexpected null result, " + "schedule the next check after {}ms.", fallbackTtl.toMillis()); - isHealthy = false; + healthStatus = HealthStatus.UNHEALTHY; intervalMillis = fallbackTtl.toMillis(); } else { - isHealthy = result.isHealthy(); + healthStatus = result.healthStatus(); intervalMillis = result.ttlMillis(); } - settableHealthChecker.setHealthy(isHealthy); + settableHealthChecker.setHealthStatus(healthStatus); scheduledFuture = eventExecutor.schedule(this::runHealthCheck, intervalMillis, TimeUnit.MILLISECONDS); return null; @@ -202,7 +211,7 @@ private void runHealthCheck() { } catch (Throwable throwable) { logger.warn("Health checker throws an exception, schedule the next check after {}ms.", fallbackTtl.toMillis(), throwable); - settableHealthChecker.setHealthy(false); + settableHealthChecker.setHealthStatus(HealthStatus.UNHEALTHY); scheduledFuture = eventExecutor.schedule(this::runHealthCheck, fallbackTtl.toMillis(), TimeUnit.MILLISECONDS); } diff --git a/core/src/main/java/com/linecorp/armeria/server/healthcheck/SettableHealthChecker.java b/core/src/main/java/com/linecorp/armeria/server/healthcheck/SettableHealthChecker.java index 92cb3da0a4e..105aa9cbb15 100644 --- a/core/src/main/java/com/linecorp/armeria/server/healthcheck/SettableHealthChecker.java +++ b/core/src/main/java/com/linecorp/armeria/server/healthcheck/SettableHealthChecker.java @@ -16,10 +16,11 @@ package com.linecorp.armeria.server.healthcheck; -import java.util.concurrent.atomic.AtomicBoolean; +import static java.util.Objects.requireNonNull; import javax.annotation.Nonnull; +import com.linecorp.armeria.common.annotation.UnstableApi; import com.linecorp.armeria.common.util.AbstractListenable; import com.linecorp.armeria.server.Server; @@ -31,7 +32,7 @@ public final class SettableHealthChecker extends AbstractListenable implements ListenableHealthChecker { - private final AtomicBoolean isHealthy; + private volatile HealthStatus healthStatus; /** * Constructs a new {@link SettableHealthChecker} which starts out in a healthy state and can be changed @@ -46,22 +47,48 @@ public SettableHealthChecker() { * changed using {@link #setHealthy(boolean)}. */ public SettableHealthChecker(boolean isHealthy) { - this.isHealthy = new AtomicBoolean(isHealthy); + healthStatus = isHealthy ? HealthStatus.HEALTHY : HealthStatus.UNHEALTHY; + } + + /** + * Constructs a new {@link SettableHealthChecker} which starts out in the specified health status and can + * be changed using {@link #setHealthStatus(HealthStatus)}. + */ + @UnstableApi + public SettableHealthChecker(HealthStatus healthStatus) { + this.healthStatus = requireNonNull(healthStatus, "healthStatus"); } @Override public boolean isHealthy() { - return isHealthy.get(); + return healthStatus.isHealthy(); } /** * Sets if the {@link Server} is healthy or not. */ public SettableHealthChecker setHealthy(boolean isHealthy) { - final boolean oldValue = this.isHealthy.getAndSet(isHealthy); - if (oldValue != isHealthy) { - notifyListeners(this); + final HealthStatus newValue = isHealthy ? HealthStatus.HEALTHY : HealthStatus.UNHEALTHY; + return setHealthStatus(newValue); + } + + @Override + public HealthStatus healthStatus() { + return healthStatus; + } + + /** + * Sets the {@link HealthStatus} of the {@link Server}. + */ + @UnstableApi + public SettableHealthChecker setHealthStatus(HealthStatus healthStatus) { + requireNonNull(healthStatus, "healthStatus"); + final HealthStatus oldValue = this.healthStatus; + if (oldValue == healthStatus) { + return this; } + this.healthStatus = healthStatus; + notifyListeners(this); return this; } @@ -73,6 +100,6 @@ protected HealthChecker latestValue() { @Override public String toString() { - return "SettableHealthChecker: " + (isHealthy.get() ? "healthy" : "not healthy"); + return "SettableHealthChecker: " + healthStatus; } } diff --git a/core/src/test/java/com/linecorp/armeria/server/healthcheck/CpuHealthCheckerTest.java b/core/src/test/java/com/linecorp/armeria/server/healthcheck/CpuHealthCheckerTest.java index d5bf3765c9f..902c18f3629 100644 --- a/core/src/test/java/com/linecorp/armeria/server/healthcheck/CpuHealthCheckerTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/healthcheck/CpuHealthCheckerTest.java @@ -16,6 +16,8 @@ package com.linecorp.armeria.server.healthcheck; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; import org.junit.jupiter.api.Test; @@ -28,4 +30,11 @@ void shouldGatherCpuUsageInformation() { assertThat(healthChecker.targetSystemCpuUsage).isNotNaN(); assertThat(healthChecker.targetProcessCpuLoad).isNotNaN(); } + + @Test + void degradedCpuUsageShouldNotBeLowerThanTargetCpuUsage() { + assertDoesNotThrow(() -> HealthChecker.ofCpu(0.5, 0.5, 0.6, 0.6)); + assertThrows(IllegalArgumentException.class, + () -> HealthChecker.ofCpu(0.5, 0.5, 0.4, 0.4)); + } } diff --git a/core/src/test/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceTest.java b/core/src/test/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceTest.java index 448df756faf..78217a352a7 100644 --- a/core/src/test/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/healthcheck/HealthCheckServiceTest.java @@ -41,6 +41,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import com.google.common.collect.ImmutableList; @@ -76,7 +79,7 @@ class HealthCheckServiceTest { @RegisterExtension static final ServerExtension server = new ServerExtension() { @Override - protected void configure(ServerBuilder sb) throws Exception { + protected void configure(ServerBuilder sb) { sb.service("/hc", HealthCheckService.of(checker)); sb.service("/hc_unfinished_1", HealthCheckService.of(unfinishedHealthChecker)); sb.service("/hc_unfinished_2", HealthCheckService.of(unfinishedHealthChecker)); @@ -94,6 +97,10 @@ protected void configure(ServerBuilder sb) throws Exception { .updatable(true) .startUnhealthy() .build()); + sb.service("/hc_status", + HealthCheckService.builder() + .updatable(DefaultHealthCheckUpdateHandler.INSTANCE) + .build()); sb.service("/hc_custom", HealthCheckService.builder() .healthyResponse(AggregatedHttpResponse.of( @@ -152,11 +159,8 @@ void ensureNoPendingResponses() { final HealthCheckService service = cfg.service().as(HealthCheckService.class); if (service != null) { await().untilAsserted(() -> { - if (service.pendingHealthyResponses != null) { - assertThat(service.pendingHealthyResponses).isEmpty(); - } - if (service.pendingUnhealthyResponses != null) { - assertThat(service.pendingUnhealthyResponses).isEmpty(); + if (service.pendingResponses != null) { + assertThat(service.pendingResponses).isEmpty(); } }); } @@ -168,12 +172,11 @@ void getWhenHealthy() throws Exception { assertResponseEquals("GET /hc HTTP/1.0", "HTTP/1.1 200 OK\r\n" + "content-type: application/json; charset=utf-8\r\n" + - "armeria-lphc: 60, 5\r\n" + - "content-length: 16\r\n" + + "armeria-lphc: 60, 5\r\n" + "content-length: 35\r\n" + // As Armeria is not fully compatible with HTTP/1.0, // an HTTP/1.1 response including "connection: close" header is returned. "connection: close\r\n\r\n" + - "{\"healthy\":true}"); + "{\"healthy\":true,\"status\":\"HEALTHY\"}"); verifyDebugEnabled(logger); verifyNoMoreInteractions(logger); } @@ -184,10 +187,9 @@ void getWhenUnhealthy() throws Exception { assertResponseEquals("GET /hc HTTP/1.0", "HTTP/1.1 503 Service Unavailable\r\n" + "content-type: application/json; charset=utf-8\r\n" + - "armeria-lphc: 60, 5\r\n" + - "content-length: 17\r\n" + + "armeria-lphc: 60, 5\r\n" + "content-length: 38\r\n" + "connection: close\r\n\r\n" + - "{\"healthy\":false}"); + "{\"healthy\":false,\"status\":\"UNHEALTHY\"}"); await().untilAsserted(() -> { verify(logger).debug(anyString()); }); @@ -198,8 +200,7 @@ void headWhenHealthy() throws Exception { assertResponseEquals("HEAD /hc HTTP/1.0", "HTTP/1.1 200 OK\r\n" + "content-type: application/json; charset=utf-8\r\n" + - "armeria-lphc: 60, 5\r\n" + - "content-length: 16\r\n" + + "armeria-lphc: 60, 5\r\n" + "content-length: 35\r\n" + "connection: close\r\n\r\n"); verifyDebugEnabled(logger); verifyNoMoreInteractions(logger); @@ -212,7 +213,7 @@ void headWhenUnhealthy() throws Exception { "HTTP/1.1 503 Service Unavailable\r\n" + "content-type: application/json; charset=utf-8\r\n" + "armeria-lphc: 60, 5\r\n" + - "content-length: 17\r\n" + + "content-length: 38\r\n" + "connection: close\r\n\r\n"); verify(logger).debug(anyString()); } @@ -233,7 +234,7 @@ private static void assertResponseEquals(String request, String expectedResponse @Test void waitUntilUnhealthy() { - final CompletableFuture f = sendLongPollingGet("healthy"); + final CompletableFuture f = sendLongPollingGet("healthy", "/hc"); // Should not wake up until the server becomes unhealthy. assertThatThrownBy(() -> f.get(1, TimeUnit.SECONDS)) @@ -248,7 +249,7 @@ void waitUntilUnhealthy() { ResponseHeaders.of(HttpStatus.SERVICE_UNAVAILABLE, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":false}"), + HttpData.ofUtf8("{\"healthy\":false,\"status\":\"UNHEALTHY\"}"), HttpHeaders.of())); } @@ -257,14 +258,14 @@ void waitUntilUnhealthyWithImmediateWakeup() throws Exception { // Make the server unhealthy. checker.setHealthy(false); - final CompletableFuture f = sendLongPollingGet("healthy"); + final CompletableFuture f = sendLongPollingGet("healthy", "/hc"); // The server is unhealthy already, so the response has to come in immediately. assertThat(f.get()).isEqualTo(AggregatedHttpResponse.of( ResponseHeaders.of(HttpStatus.SERVICE_UNAVAILABLE, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":false}"))); + HttpData.ofUtf8("{\"healthy\":false,\"status\":\"UNHEALTHY\"}"))); } @Test @@ -272,7 +273,7 @@ void waitUntilHealthy() throws Exception { // Make the server unhealthy. checker.setHealthy(false); - final CompletableFuture f = sendLongPollingGet("unhealthy"); + final CompletableFuture f = sendLongPollingGet("unhealthy", "/hc"); // Should not wake up until the server becomes unhealthy. assertThatThrownBy(() -> f.get(1, TimeUnit.SECONDS)) @@ -285,27 +286,29 @@ void waitUntilHealthy() throws Exception { .set("armeria-lphc", "60, 5") .build()), ResponseHeaders.of(HttpStatus.OK, - HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, - "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":true}"), + HttpHeaderNames.CONTENT_TYPE, + MediaType.JSON_UTF_8, + "armeria-lphc", + "60, 5"), + HttpData.ofUtf8("{\"healthy\":true,\"status\":\"HEALTHY\"}"), HttpHeaders.of())); } @Test void waitUntilHealthyWithImmediateWakeUp() throws Exception { - final CompletableFuture f = sendLongPollingGet("unhealthy"); + final CompletableFuture f = sendLongPollingGet("unhealthy", "/hc"); // The server is healthy already, so the response has to come in immediately. assertThat(f.get()).isEqualTo(AggregatedHttpResponse.of( ResponseHeaders.of(HttpStatus.OK, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":true}"))); + HttpData.ofUtf8("{\"healthy\":true,\"status\":\"HEALTHY\"}"))); } @Test void waitTimeout() throws Exception { - final AggregatedHttpResponse res = sendLongPollingGet("healthy", 1).get(); + final AggregatedHttpResponse res = sendLongPollingGet("healthy", 1, "/hc").get(); assertThat(res).isEqualTo(AggregatedHttpResponse.of( ImmutableList.of(ResponseHeaders.builder(HttpStatus.PROCESSING) .set("armeria-lphc", "60, 5") @@ -315,8 +318,7 @@ void waitTimeout() throws Exception { .status(HttpStatus.NOT_MODIFIED) .contentType(MediaType.JSON_UTF_8) .set("armeria-lphc", "60, 5") - .build(), - HttpData.empty(), + .build(), HttpData.empty(), HttpHeaders.of())); } @@ -334,7 +336,7 @@ void waitWithWrongMethod() throws Exception { @Test void waitWithWrongTimeout() throws Exception { - final AggregatedHttpResponse res = sendLongPollingGet("healthy", -1).get(); + final AggregatedHttpResponse res = sendLongPollingGet("healthy", -1, "/hc").get(); assertThat(res.status()).isEqualTo(HttpStatus.BAD_REQUEST); verifyDebugEnabled(logger); verifyNoMoreInteractions(logger); @@ -343,7 +345,7 @@ void waitWithWrongTimeout() throws Exception { @Test void waitWithOtherETag() throws Exception { // A never-matching etag must disable polling. - final AggregatedHttpResponse res = sendLongPollingGet("whatever", 1).get(); + final AggregatedHttpResponse res = sendLongPollingGet("whatever", 1, "/hc").get(); assertThat(res.status()).isEqualTo(HttpStatus.OK); verifyDebugEnabled(logger); verifyNoMoreInteractions(logger); @@ -357,19 +359,18 @@ void longPollingDisabled() throws Exception { HttpHeaderNames.PREFER, "wait=60", HttpHeaderNames.IF_NONE_MATCH, "\"healthy\"")).aggregate(); assertThat(f.get(10, TimeUnit.SECONDS)).isEqualTo(AggregatedHttpResponse.of( - ResponseHeaders.of(HttpStatus.OK, - HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, + ResponseHeaders.of(HttpStatus.OK, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "0, 0"), - HttpData.ofUtf8("{\"healthy\":true}"))); + HttpData.ofUtf8("{\"healthy\":true,\"status\":\"HEALTHY\"}"))); verifyDebugEnabled(logger); verifyNoMoreInteractions(logger); } @Test - void notUpdatableByDefault() throws Exception { + void notUpdatableByDefault() { final BlockingWebClient client = BlockingWebClient.of(server.httpUri()); final AggregatedHttpResponse res = client.execute(RequestHeaders.of(HttpMethod.POST, "/hc"), - "{\"healthy\":false}"); + "{\"healthy\":false,\"status\":\"UNHEALTHY\"}"); assertThat(res.status()).isEqualTo(HttpStatus.METHOD_NOT_ALLOWED); verifyDebugEnabled(logger); verifyNoMoreInteractions(logger); @@ -386,7 +387,7 @@ void updateUsingPutOrPost() { ResponseHeaders.of(HttpStatus.SERVICE_UNAVAILABLE, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":false}"))); + HttpData.ofUtf8("{\"healthy\":false,\"status\":\"UNHEALTHY\"}"))); // Make healthy. final AggregatedHttpResponse res2 = client.execute(RequestHeaders.of(HttpMethod.POST, "/hc_updatable"), @@ -395,7 +396,7 @@ void updateUsingPutOrPost() { ResponseHeaders.of(HttpStatus.OK, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":true}"))); + HttpData.ofUtf8("{\"healthy\":true,\"status\":\"HEALTHY\"}"))); } @Test @@ -410,7 +411,7 @@ void updateUsingPatch() { ResponseHeaders.of(HttpStatus.SERVICE_UNAVAILABLE, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":false}"))); + HttpData.ofUtf8("{\"healthy\":false,\"status\":\"UNHEALTHY\"}"))); // Make healthy. final AggregatedHttpResponse res2 = client.execute( @@ -420,7 +421,7 @@ void updateUsingPatch() { ResponseHeaders.of(HttpStatus.OK, HttpHeaderNames.CONTENT_TYPE, MediaType.JSON_UTF_8, "armeria-lphc", "60, 5"), - HttpData.ofUtf8("{\"healthy\":true}"))); + HttpData.ofUtf8("{\"healthy\":true,\"status\":\"HEALTHY\"}"))); } @Test @@ -430,13 +431,15 @@ void updateListener() { capturedHealthy.set(null); // Make unhealthy. - client.execute(RequestHeaders.of(HttpMethod.POST, "/hc_update_listener"), "{\"healthy\":false}"); + client.execute(RequestHeaders.of(HttpMethod.POST, "/hc_update_listener"), + "{\"healthy\":false,\"status\":\"UNHEALTHY\"}"); assertThat(capturedHealthy.get()).isFalse(); capturedHealthy.set(null); // Make healthy. - client.execute(RequestHeaders.of(HttpMethod.POST, "/hc_update_listener"), "{\"healthy\":true}"); + client.execute(RequestHeaders.of(HttpMethod.POST, "/hc_update_listener"), + "{\"healthy\":true,\"status\":\"HEALTHY\"}"); assertThat(capturedHealthy.get()).isTrue(); } @@ -446,9 +449,22 @@ void startUnhealthy() throws Exception { "HTTP/1.1 503 Service Unavailable\r\n" + "content-type: application/json; charset=utf-8\r\n" + "armeria-lphc: 60, 5\r\n" + - "content-length: 17\r\n" + + "content-length: 38\r\n" + "connection: close\r\n\r\n" + - "{\"healthy\":false}"); + "{\"healthy\":false,\"status\":\"UNHEALTHY\"}"); + } + + @ParameterizedTest + @ValueSource(strings = { "HEALTHY", "DEGRADED", "STOPPING", "UNHEALTHY", "UNDER_MAINTENANCE" }) + void useDefaultHealthCheckUpdateHandlerToUpdateStatus(String status) { + final HealthStatus healthStatus = HealthStatus.valueOf(status); + final BlockingWebClient client = BlockingWebClient.of(server.httpUri()); + + final AggregatedHttpResponse res = client.execute(RequestHeaders.of( + HttpMethod.POST, "/hc_status"), "{\"status\":\"" + healthStatus.name() + "\"}"); + final String healthy = healthStatus.isHealthy() ? "true" : "false"; + assertThat(res.content()).isEqualTo( + HttpData.ofUtf8("{\"healthy\":" + healthy + ",\"status\":\"" + healthStatus.name() + "\"}")); } @Test @@ -497,21 +513,103 @@ void customError() { assertThat(res2.status()).isEqualTo(HttpStatus.BAD_REQUEST); } + @ParameterizedTest + @ValueSource(strings = { "DEGRADED", "STOPPING", "UNHEALTHY", "UNDER_MAINTENANCE" }) + void checkHealthStatus(String status) { + final HealthStatus healthStatus = HealthStatus.valueOf(status); + checker.setHealthStatus(healthStatus); + final BlockingWebClient client = BlockingWebClient.of(server.httpUri()); + final AggregatedHttpResponse res = client.execute(RequestHeaders.of(HttpMethod.GET, "/hc")); + assertThat(res.content()) + .isEqualTo(HttpData.ofUtf8("{\"healthy\":" + healthStatus.isHealthy() + ',' + + "\"status\":\"" + healthStatus.name() + "\"}")); + } + + @Test + void healthStateChangeNotificationByHealthStatusUpdate() throws Exception { + checker.setHealthStatus(HealthStatus.HEALTHY); + final CompletableFuture f = sendLongPollingGet("healthy", "/hc"); + assertThatThrownBy(() -> f.get(1, TimeUnit.SECONDS)).isInstanceOf(TimeoutException.class); + + // Because the health state is unchanged(from healthy to healthy), response is not received. + checker.setHealthStatus(HealthStatus.DEGRADED); + assertThatThrownBy(() -> f.get(1, TimeUnit.SECONDS)).isInstanceOf(TimeoutException.class); + + // Because the health state is changed(from healthy to unhealthy), response is received. + checker.setHealthStatus(HealthStatus.UNHEALTHY); + assertThat(f.get().contentUtf8()).isEqualTo( + "{\"healthy\":" + "false" + ",\"status\":\"" + "UNHEALTHY" + "\"}"); + } + + @ParameterizedTest + @CsvSource({ + "healthy, STOPPING", + "healthy, UNHEALTHY", + "healthy, UNDER_MAINTENANCE", + "unhealthy, HEALTHY", + "unhealthy, DEGRADED", + }) + void pollHealthStateChangeAndUpdateHealthStatus(String longPollingState, String targetStatus) + throws Exception { + checker.setHealthy("healthy".equals(longPollingState)); + final HealthStatus healthStatus = HealthStatus.valueOf(targetStatus); + final CompletableFuture f = sendLongPollingGet(longPollingState, "/hc"); + assertThatThrownBy(() -> f.get(1, TimeUnit.SECONDS)).isInstanceOf(TimeoutException.class); + + checker.setHealthStatus(healthStatus); + final String changedState = healthStatus.isHealthy() ? "true" : "false"; + assertThat(f.get().contentUtf8()).isEqualTo( + "{\"healthy\":" + changedState + ",\"status\":\"" + healthStatus.name() + "\"}"); + } + + @ParameterizedTest + @ValueSource(strings = { "DEGRADED", "STOPPING", "UNHEALTHY", "UNDER_MAINTENANCE" }) + void pollSpecificHealthStatus(String status) throws Exception { + final HealthStatus healthStatus = HealthStatus.valueOf(status); + checker.setHealthStatus(HealthStatus.HEALTHY); + final CompletableFuture f = + sendLongPollingGet(HealthStatus.HEALTHY, "/hc"); + assertThatThrownBy(() -> f.get(1, TimeUnit.SECONDS)).isInstanceOf(TimeoutException.class); + + // change to different health status and get notified + checker.setHealthStatus(healthStatus); + final String healthy = healthStatus.isHealthy() ? "true" : "false"; + assertThat(f.get().contentUtf8()).isEqualTo( + "{\"healthy\":" + healthy + ",\"status\":\"" + healthStatus.name() + "\"}"); + } + private static void verifyDebugEnabled(Logger logger) { // Do not log for health check requests unless TransientServiceOption is enabled. verify(logger, never()).isDebugEnabled(); } - private static CompletableFuture sendLongPollingGet(String healthiness) { - return sendLongPollingGet(healthiness, 120); + private static CompletableFuture sendLongPollingGet(String healthiness, + String path) { + return sendLongPollingGet(healthiness, 120, path); } private static CompletableFuture sendLongPollingGet(String healthiness, - int timeoutSeconds) { + int timeoutSeconds, + String path) { final WebClient client = WebClient.of(server.httpUri()); - return client.execute(RequestHeaders.of(HttpMethod.GET, "/hc", + return client.execute(RequestHeaders.of(HttpMethod.GET, path, HttpHeaderNames.PREFER, "wait=" + timeoutSeconds, HttpHeaderNames.IF_NONE_MATCH, '"' + healthiness + '"')).aggregate(); } + + private static CompletableFuture sendLongPollingGet(HealthStatus healthStatus, + String path) { + return sendLongPollingGet(healthStatus, 120, path); + } + + private static CompletableFuture sendLongPollingGet(HealthStatus healthStatus, + int timeoutSeconds, + String path) { + final WebClient client = WebClient.of(server.httpUri()); + return client.execute(RequestHeaders.of(HttpMethod.GET, path, + HttpHeaderNames.PREFER, "wait=" + timeoutSeconds, + HttpHeaderNames.IF_NONE_MATCH, + "\"status=" + healthStatus + '"')).aggregate(); + } } diff --git a/core/src/test/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthCheckerTest.java b/core/src/test/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthCheckerTest.java index 5c3adac3c19..cd07b426d7e 100644 --- a/core/src/test/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthCheckerTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/healthcheck/ScheduledHealthCheckerTest.java @@ -54,11 +54,11 @@ void stopSchedulingAfterStop() throws InterruptedException { server.start().join(); await().untilAsserted(() -> assertThat(invokedCount.get()).isOne()); - holder.get().complete(new HealthCheckStatus(true, 100)); + holder.get().complete(new HealthCheckStatus(HealthStatus.HEALTHY, 100)); await().untilAsserted(() -> assertThat(invokedCount.get()).isEqualTo(2)); server.stop().join(); - holder.get().complete(new HealthCheckStatus(true, 100)); + holder.get().complete(new HealthCheckStatus(HealthStatus.HEALTHY, 100)); // Wait for a while to verify health checker is not invoked anymore. Thread.sleep(1000); assertThat(invokedCount.get()).isEqualTo(2); diff --git a/core/src/test/java/com/linecorp/armeria/server/healthcheck/SettableHealthCheckerTest.java b/core/src/test/java/com/linecorp/armeria/server/healthcheck/SettableHealthCheckerTest.java index 9e9d155cfea..4dee53085fd 100644 --- a/core/src/test/java/com/linecorp/armeria/server/healthcheck/SettableHealthCheckerTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/healthcheck/SettableHealthCheckerTest.java @@ -15,6 +15,7 @@ */ package com.linecorp.armeria.server.healthcheck; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -25,26 +26,34 @@ public class SettableHealthCheckerTest { @Test public void justCreated() { final SettableHealthChecker checker = new SettableHealthChecker(); - assertTrue(checker.isHealthy()); + assertTrue(checker.healthStatus().isHealthy()); } @Test public void justCreatedExplicit() { final SettableHealthChecker checker = new SettableHealthChecker(false); - assertFalse(checker.isHealthy()); + assertFalse(checker.healthStatus().isHealthy()); } @Test public void setHealthy() { final SettableHealthChecker checker = new SettableHealthChecker(); checker.setHealthy(true); - assertTrue(checker.isHealthy()); + assertTrue(checker.healthStatus().isHealthy()); } @Test public void setUnHealthy() { final SettableHealthChecker checker = new SettableHealthChecker(); checker.setHealthy(false); - assertFalse(checker.isHealthy()); + assertFalse(checker.healthStatus().isHealthy()); + } + + @Test + public void setHealthStatus() { + final SettableHealthChecker checker = new SettableHealthChecker(); + checker.setHealthStatus(HealthStatus.DEGRADED); + assertTrue(checker.healthStatus().isHealthy()); + assertEquals(HealthStatus.DEGRADED, checker.healthStatus()); } } diff --git a/examples/spring-boot-jetty/src/test/java/example/springframework/boot/jetty/HelloIntegrationTest.java b/examples/spring-boot-jetty/src/test/java/example/springframework/boot/jetty/HelloIntegrationTest.java index 8e1ce99e559..8f48ae6baec 100644 --- a/examples/spring-boot-jetty/src/test/java/example/springframework/boot/jetty/HelloIntegrationTest.java +++ b/examples/spring-boot-jetty/src/test/java/example/springframework/boot/jetty/HelloIntegrationTest.java @@ -55,6 +55,6 @@ void hello() throws Exception { void healthCheck() throws Exception { final AggregatedHttpResponse res = client.get("/internal/healthcheck").aggregate().join(); assertThat(res.status()).isEqualTo(HttpStatus.OK); - assertThat(res.contentUtf8()).isEqualTo("{\"healthy\":true}"); + assertThat(res.contentUtf8()).isEqualTo("{\"healthy\":true,\"status\":\"HEALTHY\"}"); } } diff --git a/examples/spring-boot-tomcat/src/test/java/example/springframework/boot/tomcat/HelloIntegrationTest.java b/examples/spring-boot-tomcat/src/test/java/example/springframework/boot/tomcat/HelloIntegrationTest.java index c6256bbbdf5..c750e1473d4 100644 --- a/examples/spring-boot-tomcat/src/test/java/example/springframework/boot/tomcat/HelloIntegrationTest.java +++ b/examples/spring-boot-tomcat/src/test/java/example/springframework/boot/tomcat/HelloIntegrationTest.java @@ -55,6 +55,6 @@ void hello() throws Exception { void healthCheck() throws Exception { final AggregatedHttpResponse res = client.get("/internal/healthcheck").aggregate().join(); assertThat(res.status()).isEqualTo(HttpStatus.OK); - assertThat(res.contentUtf8()).isEqualTo("{\"healthy\":true}"); + assertThat(res.contentUtf8()).isEqualTo("{\"healthy\":true,\"status\":\"HEALTHY\"}"); } } diff --git a/grpc/src/main/java/com/linecorp/armeria/server/grpc/GrpcHealthCheckService.java b/grpc/src/main/java/com/linecorp/armeria/server/grpc/GrpcHealthCheckService.java index 3c1810437c6..b4f855486bf 100644 --- a/grpc/src/main/java/com/linecorp/armeria/server/grpc/GrpcHealthCheckService.java +++ b/grpc/src/main/java/com/linecorp/armeria/server/grpc/GrpcHealthCheckService.java @@ -231,7 +231,7 @@ ServingStatus checkServingStatus(String serviceName) { if (listenableHealthChecker == null) { return ServingStatus.SERVICE_UNKNOWN; } - if (listenableHealthChecker.isHealthy()) { + if (listenableHealthChecker.healthStatus().isHealthy()) { return ServingStatus.SERVING; } return ServingStatus.NOT_SERVING; @@ -247,7 +247,7 @@ private void addServerHealthUpdateListener(List updat serverHealthChecker.addListener(healthChecker -> { updateListeners.forEach(updateListener -> { try { - updateListener.healthUpdated(healthChecker.isHealthy()); + updateListener.healthStatusUpdated(healthChecker.healthStatus()); } catch (Throwable t) { logger.warn("Unexpected exception from HealthCheckUpdateListener.healthUpdated():", t); } @@ -268,7 +268,7 @@ private Consumer watcherHealthUpdater() { private boolean isServerHealthy() { for (HealthChecker healthChecker : serverHealthCheckers) { - if (!healthChecker.isHealthy()) { + if (!healthChecker.healthStatus().isHealthy()) { return false; } }