diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/pom.xml b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/pom.xml new file mode 100644 index 000000000..88c5076c4 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + com.example + java-hc-consumer + 1.0.0-SNAPSHOT + jar + + uc41 consumer that reports which provider served it + uc41 consumer that reports which provider served it (issue #1480) + + + org.springframework.boot + spring-boot-starter-parent + 4.0.5 + + + + + 17 + 3.5.1 + + + + + io.mcp-mesh + mcp-mesh-spring-boot-starter + ${mcp-mesh.version} + + + + org.springframework.boot + spring-boot-starter-web + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.example.hcconsumer.HcConsumerApplication + + + + + + diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/java/com/example/hcconsumer/HcConsumerApplication.java b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/java/com/example/hcconsumer/HcConsumerApplication.java new file mode 100644 index 000000000..98f31d1c3 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/java/com/example/hcconsumer/HcConsumerApplication.java @@ -0,0 +1,64 @@ +package com.example.hcconsumer; + +import io.mcpmesh.MeshAgent; +import io.mcpmesh.MeshTool; +import io.mcpmesh.Selector; +import io.mcpmesh.types.McpMeshTool; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * java-hc-consumer — reports which provider the mesh routed it to (#1480). + * + *

{@code who_served} injects {@code hc_probe_java} and returns the + * provider's payload verbatim. Started ONCE and never restarted, so the only + * way its answer can move from provider A to provider B and back is a genuine + * re-resolution — the withdrawal / recovery chain under test. + * + *

The MCP tool name is the METHOD name verbatim — {@code @MeshTool} has no + * {@code name()} attribute and does not snake_case anything — so the test + * calls {@code hc-consumer-java:whoServed}, not {@code who_served}. Same + * convention as uc38's {@code getRejectCount}. + * + *

An unresolved dependency reports {@code served_by: "UNRESOLVED"} rather + * than throwing. The test's failover poll then keeps polling through a + * transient gap instead of tripping on it, and a PERMANENT gap still fails the + * run because {@code "UNRESOLVED"} never becomes a provider name. + */ +@SpringBootApplication +@MeshAgent( + name = "hc-consumer-java", + version = "1.0.0", + description = "Consumer that must fail over when provider A withdraws (#1480)", + port = 3433 +) +public class HcConsumerApplication { + + public static void main(String[] args) { + SpringApplication.run(HcConsumerApplication.class, args); + } + + @MeshTool( + capability = "who_served_java", + description = "Call hc_probe_java and report which provider answered", + tags = {"hc-withdrawal"}, + dependencies = {@Selector(capability = "hc_probe_java")} + ) + public Map whoServed(McpMeshTool> probe) { + if (probe == null || !probe.isAvailable()) { + return Map.of("served_by", "UNRESOLVED"); + } + try { + Map payload = probe.call(Map.of()); + if (payload == null || !payload.containsKey("served_by")) { + return Map.of("served_by", "UNRESOLVED", "raw", String.valueOf(payload)); + } + return new LinkedHashMap<>(payload); + } catch (Exception e) { + return Map.of("served_by", "UNRESOLVED", "error", String.valueOf(e.getMessage())); + } + } +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/resources/application.yml b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/resources/application.yml new file mode 100644 index 000000000..dd92af92f --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/resources/application.yml @@ -0,0 +1,20 @@ +# uc41 consumer that reports which provider served it (issue #1480) + +spring: + application: + name: java-hc-consumer + +server: + port: ${MCP_MESH_HTTP_PORT:3433} + +mesh: + registry: + url: ${MCP_MESH_REGISTRY_URL:http://localhost:8000} + agent: + name: ${MCP_MESH_AGENT_NAME:hc-consumer-java} + namespace: ${MCP_MESH_NAMESPACE:default} + +logging: + level: + io.mcpmesh: DEBUG + com.example: DEBUG diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/pom.xml b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/pom.xml new file mode 100644 index 000000000..9dbe8c1fe --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + com.example + java-hc-provider-a + 1.0.0-SNAPSHOT + jar + + uc41 provider whose health check withdraws it from resolution + uc41 provider whose health check withdraws it from resolution (issue #1480) + + + org.springframework.boot + spring-boot-starter-parent + 4.0.5 + + + + + 17 + 3.5.1 + + + + + io.mcp-mesh + mcp-mesh-spring-boot-starter + ${mcp-mesh.version} + + + + org.springframework.boot + spring-boot-starter-web + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.example.hcprovidera.HcProviderAApplication + + + + + + diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/java/com/example/hcprovidera/HcProviderAApplication.java b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/java/com/example/hcprovidera/HcProviderAApplication.java new file mode 100644 index 000000000..cc4c6dba3 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/java/com/example/hcprovidera/HcProviderAApplication.java @@ -0,0 +1,125 @@ +package com.example.hcprovidera; + +import io.mcpmesh.MeshAgent; +import io.mcpmesh.MeshHealth; +import io.mcpmesh.MeshHealthCheck; +import io.mcpmesh.MeshTool; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Instant; +import java.util.Map; + +/** + * java-hc-provider-a — the provider whose health check the test drives (#1480). + * + *

Java twin of py-hc-provider-a / ts-hc-provider-a. Same contract, same flag + * file, same trace file: the three runtimes are meant to be behaviourally + * identical here, and divergence is exactly what keeps getting found. + * + *

File-toggled, not invocation-counting

+ * + *

{@code @MeshHealthCheck} re-reads {@code /workspace/health-flag} on every + * tick so the test controls WHEN the transition happens and can poll for it: + * + * 

+ *   ok (or file absent) -> healthy    heartbeats, stays resolvable
+ *   fail                -> unhealthy  heartbeat suppressed -> registry withdraws
+ *   throw               -> throws     must map to DEGRADED, must NOT withdraw
+ *
+ * + *

Every invocation is traced

+ * + *

One line is appended to {@code /workspace/hc-invocations.log} BEFORE the + * throw branch throws. Without it the negative test would pass vacuously: a + * health check that stopped running also fails to withdraw the agent, and "the + * agent is still resolvable" cannot tell that apart from what we want. + */ +@SpringBootApplication +@MeshAgent( + name = "hc-provider-a-java", + version = "1.0.0", + description = "Provider whose health check withdraws it from resolution (#1480)", + port = 3431 +) +public class HcProviderAApplication { + + static final String AGENT_NAME = "hc-provider-a-java"; + + private static final Path FLAG_FILE = Path.of( + System.getenv().getOrDefault("HC_FLAG_FILE", "/workspace/health-flag")); + private static final Path TRACE_FILE = Path.of( + System.getenv().getOrDefault("HC_TRACE_FILE", "/workspace/hc-invocations.log")); + + public static void main(String[] args) { + SpringApplication.run(HcProviderAApplication.class, args); + } + + @MeshTool( + capability = "hc_probe_java", + description = "Report which provider instance served this call", + tags = {"hc-withdrawal"} + ) + public Map probeA() { + // pid is self-reported from inside the JVM, so it cannot go stale the + // way a pid FILE can — and unlike meshctl's pid file it names the JVM + // rather than the `mvn spring-boot:run` wrapper that forked it. + // Baseline and post-recovery answers carrying the same pid is the + // proof that recovery did not restart anything. + return Map.of("served_by", AGENT_NAME, "pid", ProcessHandle.current().pid()); + } + + /** + * Simulated upstream-vendor probe, driven by the flag file. + * + *

ttlSeconds = 2 so a withdrawal costs ~1 TTL plus the registry staleness + * window rather than the 15s default; the test's registry runs at a matching + * 5s/2s. + */ + @MeshHealthCheck(ttlSeconds = 2) + public MeshHealth vendorHealth() { + String flag = readFlag(); + + if ("fail".equals(flag)) { + trace(flag, "unhealthy"); + return MeshHealth.unhealthy("simulated vendor outage (health-flag=fail)") + .withCheck("vendor_api_reachable", false); + } + + if ("throw".equals(flag)) { + // Traced BEFORE throwing — see the class javadoc. + trace(flag, "raised"); + throw new IllegalStateException( + "simulated broken health check (health-flag=throw)"); + } + + trace(flag, "healthy"); + return MeshHealth.healthy().withCheck("vendor_api_reachable", true); + } + + /** Current fault state. A missing file means healthy, so the agent boots green. */ + private static String readFlag() { + try { + String raw = Files.readString(FLAG_FILE, StandardCharsets.UTF_8).trim(); + return raw.isEmpty() ? "ok" : raw.toLowerCase(); + } catch (Exception e) { + return "ok"; + } + } + + /** Best-effort: a trace write that fails must not change the verdict. */ + private static void trace(String flag, String verdict) { + String line = Instant.now() + " agent=" + AGENT_NAME + + " flag=" + flag + " verdict=" + verdict + System.lineSeparator(); + try { + Files.writeString(TRACE_FILE, line, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } catch (Exception ignored) { + // ignore + } + } +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/resources/application.yml b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/resources/application.yml new file mode 100644 index 000000000..8971b9b89 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/resources/application.yml @@ -0,0 +1,20 @@ +# uc41 provider whose health check withdraws it from resolution (issue #1480) + +spring: + application: + name: java-hc-provider-a + +server: + port: ${MCP_MESH_HTTP_PORT:3431} + +mesh: + registry: + url: ${MCP_MESH_REGISTRY_URL:http://localhost:8000} + agent: + name: ${MCP_MESH_AGENT_NAME:hc-provider-a-java} + namespace: ${MCP_MESH_NAMESPACE:default} + +logging: + level: + io.mcpmesh: DEBUG + com.example: DEBUG diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/pom.xml b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/pom.xml new file mode 100644 index 000000000..b8535fb52 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + com.example + java-hc-provider-b + 1.0.0-SNAPSHOT + jar + + uc41 survivor provider the consumer fails over to + uc41 survivor provider the consumer fails over to (issue #1480) + + + org.springframework.boot + spring-boot-starter-parent + 4.0.5 + + + + + 17 + 3.5.1 + + + + + io.mcp-mesh + mcp-mesh-spring-boot-starter + ${mcp-mesh.version} + + + + org.springframework.boot + spring-boot-starter-web + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.example.hcproviderb.HcProviderBApplication + + + + + + diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/src/main/java/com/example/hcproviderb/HcProviderBApplication.java b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/src/main/java/com/example/hcproviderb/HcProviderBApplication.java new file mode 100644 index 000000000..39a7fa611 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/src/main/java/com/example/hcproviderb/HcProviderBApplication.java @@ -0,0 +1,47 @@ +package com.example.hcproviderb; + +import io.mcpmesh.MeshAgent; +import io.mcpmesh.MeshTool; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +import java.util.Map; + +/** + * java-hc-provider-b — the survivor (issue #1480). + * + *

Second provider of {@code hc_probe_java}. Deliberately has NO + * {@code @MeshHealthCheck}: it is the control. It must keep heartbeating + * throughout, so a run where BOTH providers go unhealthy (dead registry, + * stalled sweep, container-wide stall) is distinguishable from a genuine + * withdrawal of A. + * + *

Loses the resolver tiebreak to A while A is healthy — equal tag score, + * equal version, then agent ID ASC, and {@code hc-provider-a-java-} sorts + * before {@code hc-provider-b-java-}. So the consumer deterministically + * starts on A and any answer naming B is a real re-resolution. + */ +@SpringBootApplication +@MeshAgent( + name = "hc-provider-b-java", + version = "1.0.0", + description = "Survivor provider that the consumer fails over to (#1480)", + port = 3432 +) +public class HcProviderBApplication { + + static final String AGENT_NAME = "hc-provider-b-java"; + + public static void main(String[] args) { + SpringApplication.run(HcProviderBApplication.class, args); + } + + @MeshTool( + capability = "hc_probe_java", + description = "Report which provider instance served this call", + tags = {"hc-withdrawal"} + ) + public Map probeB() { + return Map.of("served_by", AGENT_NAME, "pid", ProcessHandle.current().pid()); + } +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/src/main/resources/application.yml b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/src/main/resources/application.yml new file mode 100644 index 000000000..8e97c0f28 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/src/main/resources/application.yml @@ -0,0 +1,20 @@ +# uc41 survivor provider the consumer fails over to (issue #1480) + +spring: + application: + name: java-hc-provider-b + +server: + port: ${MCP_MESH_HTTP_PORT:3432} + +mesh: + registry: + url: ${MCP_MESH_REGISTRY_URL:http://localhost:8000} + agent: + name: ${MCP_MESH_AGENT_NAME:hc-provider-b-java} + namespace: ${MCP_MESH_NAMESPACE:default} + +logging: + level: + io.mcpmesh: DEBUG + com.example: DEBUG diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-consumer/main.py b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-consumer/main.py new file mode 100644 index 000000000..beab5e92b --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-consumer/main.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""py-hc-consumer — reports which provider the mesh routed it to (issue #1480). + +``who_served`` injects ``hc_probe_py`` and returns the provider's payload +verbatim. The consumer process is started ONCE and never restarted, so the +only way its answer can move from provider A to provider B and back is if +the runtime re-resolved the dependency on its own — which is exactly the +withdrawal / recovery chain under test. +""" + +import mesh +from fastmcp import FastMCP +from mesh.types import McpMeshTool + +app = FastMCP("HC Consumer (python)") + + +@app.tool() +@mesh.tool( + capability="who_served_py", + description="Call hc_probe_py and report which provider answered", + tags=["hc-withdrawal"], + dependencies=["hc_probe_py"], +) +async def who_served(probe: McpMeshTool = None) -> dict: + if probe is None: + return {"error": "hc_probe_py dependency not injected"} + return await probe() + + +@mesh.agent( + name="hc-consumer-py", + version="1.0.0", + description="Consumer that must fail over when provider A withdraws (#1480)", + http_port=0, # actual port comes from MCP_MESH_HTTP_PORT + enable_http=True, + auto_run=True, +) +class HcConsumer: + pass diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-consumer/requirements.txt b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-consumer/requirements.txt new file mode 100644 index 000000000..385ef4086 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-consumer/requirements.txt @@ -0,0 +1,2 @@ +# py-hc-consumer dependencies +# mcp-mesh + fastmcp are baked into the tsuite-mesh image; nothing to install. diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-a/main.py b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-a/main.py new file mode 100644 index 000000000..3f089599c --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-a/main.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""py-hc-provider-a — the provider whose health check the test drives (issue #1480). + +Provides ``hc_probe_py`` alongside py-hc-provider-b. Its ``probe_a`` tool +reports its own agent name, so the consumer's answer says WHICH provider +served the call — that string is the whole failover signal. + +## The health check is FILE-TOGGLED, not invocation-counting + +uc02/tc20 counts invocations because what it asks is "does the refresh loop +fire at all". Here the test needs to control WHEN the transition happens — +withdrawal has to be gated on a fault the test injects at a known moment, so +polling can bound it. So the check reads ``/workspace/health-flag`` on every +invocation: + + ok (or file absent) -> healthy heartbeats, stays resolvable + fail -> unhealthy heartbeat suppressed -> registry withdraws + throw -> raises must map to DEGRADED, must NOT withdraw + +## Every invocation is traced to a file + +``/workspace/hc-invocations.log`` gets one line per invocation, written +BEFORE the ``throw`` branch raises. Without it the negative test +(tc02_python_throwing_check_degrades) would pass vacuously: a health check +that stopped running entirely also fails to withdraw the agent, and "the +agent is still resolvable" cannot tell the two apart. The trace is what +proves the loop kept running and kept seeing ``throw``. +""" + +import os +from datetime import UTC, datetime + +import mesh +from fastmcp import FastMCP + +AGENT_NAME = "hc-provider-a-py" +FLAG_FILE = os.environ.get("HC_FLAG_FILE", "/workspace/health-flag") +TRACE_FILE = os.environ.get("HC_TRACE_FILE", "/workspace/hc-invocations.log") + +app = FastMCP("HC Provider A (python)") + + +def _read_flag() -> str: + """Current fault state. A missing file means healthy, so the agent boots + green without the test having to seed anything.""" + try: + with open(FLAG_FILE) as handle: + return handle.read().strip().lower() or "ok" + except OSError: + return "ok" + + +def _trace(flag: str, verdict: str) -> None: + """Append one line per invocation. Best-effort: a trace write that fails + must never be the reason the health check reports something different.""" + try: + with open(TRACE_FILE, "a") as handle: + handle.write( + f"{datetime.now(UTC).isoformat()} agent={AGENT_NAME} " + f"flag={flag} verdict={verdict}\n" + ) + except OSError: + pass + + +async def vendor_health() -> dict: + """Simulated upstream-vendor probe, driven by the flag file.""" + flag = _read_flag() + + if flag == "fail": + _trace(flag, "unhealthy") + return { + "status": "unhealthy", + "checks": {"vendor_api_reachable": False}, + "errors": ["simulated vendor outage (health-flag=fail)"], + } + + if flag == "throw": + # Traced BEFORE raising — see the module docstring. + _trace(flag, "raised") + raise RuntimeError("simulated broken health check (health-flag=throw)") + + _trace(flag, "healthy") + return { + "status": "healthy", + "checks": {"vendor_api_reachable": True}, + } + + +@app.tool() +@mesh.tool( + capability="hc_probe_py", + description="Report which provider instance served this call", + tags=["hc-withdrawal"], +) +async def probe_a() -> dict: + # pid is self-reported from inside the process, so it cannot go stale the + # way a pid FILE can. Baseline and post-recovery answers carrying the same + # pid is the proof that recovery did not restart anything. + return {"served_by": AGENT_NAME, "pid": os.getpid()} + + +@mesh.agent( + name=AGENT_NAME, + version="1.0.0", + description="Provider whose health check withdraws it from resolution (#1480)", + http_port=0, # actual port comes from MCP_MESH_HTTP_PORT + enable_http=True, + auto_run=True, + health_check=vendor_health, + # 2s so a withdrawal costs ~1 TTL + the registry staleness window rather + # than the 15s default; the test's registry runs at a matching 5s/2s. + health_check_ttl=2, +) +class HcProviderA: + pass diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-a/requirements.txt b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-a/requirements.txt new file mode 100644 index 000000000..191b5dd55 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-a/requirements.txt @@ -0,0 +1,2 @@ +# py-hc-provider-a dependencies +# mcp-mesh + fastmcp are baked into the tsuite-mesh image; nothing to install. diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-b/main.py b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-b/main.py new file mode 100644 index 000000000..9e2cc245b --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-b/main.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""py-hc-provider-b — the survivor (issue #1480). + +Second provider of ``hc_probe_py``. Deliberately has NO health check: it is +the control. Two things depend on that: + + - it must keep heartbeating throughout, so a run where BOTH providers go + unhealthy is distinguishable from a genuine withdrawal of A (a dead + registry, a stalled sweep or a container-wide stall would take both down); + - it is the failover target the consumer must land on. + +Loses the resolver tiebreak to A while A is healthy: equal tag score, equal +version, and the last tiebreak is agent ID ASC — ``hc-provider-a-py-`` +sorts before ``hc-provider-b-py-``. So the consumer deterministically +starts on A, and any answer naming B is a real re-resolution. +""" + +import os + +import mesh +from fastmcp import FastMCP + +AGENT_NAME = "hc-provider-b-py" + +app = FastMCP("HC Provider B (python)") + + +@app.tool() +@mesh.tool( + capability="hc_probe_py", + description="Report which provider instance served this call", + tags=["hc-withdrawal"], +) +async def probe_b() -> dict: + return {"served_by": AGENT_NAME, "pid": os.getpid()} + + +@mesh.agent( + name=AGENT_NAME, + version="1.0.0", + description="Survivor provider that the consumer fails over to (#1480)", + http_port=0, # actual port comes from MCP_MESH_HTTP_PORT + enable_http=True, + auto_run=True, +) +class HcProviderB: + pass diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-b/requirements.txt b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-b/requirements.txt new file mode 100644 index 000000000..8871b788d --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/py-hc-provider-b/requirements.txt @@ -0,0 +1,2 @@ +# py-hc-provider-b dependencies +# mcp-mesh + fastmcp are baked into the tsuite-mesh image; nothing to install. diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/package.json b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/package.json new file mode 100644 index 000000000..a1bcdb3aa --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/package.json @@ -0,0 +1,23 @@ +{ + "name": "ts-hc-consumer", + "version": "1.0.0", + "description": "uc41 consumer that reports which provider served it (issue #1480)", + "type": "module", + "engines": { + "node": ">=22.0.0" + }, + "scripts": { + "start": "tsx src/index.ts", + "build": "tsc", + "dev": "tsx watch src/index.ts" + }, + "dependencies": { + "@mcpmesh/sdk": "^3.5.1", + "zod": "^3.24.1" + }, + "devDependencies": { + "@types/node": "^22.18.0", + "tsx": "^4.0.0", + "typescript": "^5.7.0" + } +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/src/index.ts b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/src/index.ts new file mode 100644 index 000000000..26a136c10 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/src/index.ts @@ -0,0 +1,44 @@ +#!/usr/bin/env npx tsx +/** + * ts-hc-consumer — reports which provider the mesh routed it to (#1480). + * + * `who_served` injects `hc_probe_ts` positionally and returns the provider's + * payload verbatim. Started ONCE and never restarted, so the only way its + * answer can move from provider A to provider B and back is a genuine + * re-resolution — the withdrawal / recovery chain under test. + */ + +import { FastMCP, mesh, McpMeshTool } from "@mcpmesh/sdk"; +import { z } from "zod"; + +const server = new FastMCP({ + name: "HC Consumer (typescript)", + version: "1.0.0", +}); + +const agent = mesh(server, { + name: "hc-consumer-ts", + version: "1.0.0", + description: "Consumer that must fail over when provider A withdraws (#1480)", + httpPort: Number(process.env.MCP_MESH_HTTP_PORT ?? "3423"), +}); + +agent.addTool({ + name: "who_served", + capability: "who_served_ts", + description: "Call hc_probe_ts and report which provider answered", + tags: ["hc-withdrawal"], + dependencies: ["hc_probe_ts"], + parameters: z.object({}), + execute: async ( + _args, + probe: McpMeshTool | null = null, // positional: dependencies[0] + ) => { + if (!probe) { + return JSON.stringify({ error: "hc_probe_ts dependency not injected" }); + } + return JSON.stringify(await probe({})); + }, +}); + +console.log("hc-consumer-ts defined. Waiting for auto-start..."); diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/tsconfig.json b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/tsconfig.json new file mode 100644 index 000000000..94f0ab505 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-consumer/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "outDir": "dist", + "declaration": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/package.json b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/package.json new file mode 100644 index 000000000..ae509a50b --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/package.json @@ -0,0 +1,23 @@ +{ + "name": "ts-hc-provider-a", + "version": "1.0.0", + "description": "uc41 provider whose health check withdraws it from resolution (issue #1480)", + "type": "module", + "engines": { + "node": ">=22.0.0" + }, + "scripts": { + "start": "tsx src/index.ts", + "build": "tsc", + "dev": "tsx watch src/index.ts" + }, + "dependencies": { + "@mcpmesh/sdk": "^3.5.1", + "zod": "^3.24.1" + }, + "devDependencies": { + "@types/node": "^22.18.0", + "tsx": "^4.0.0", + "typescript": "^5.7.0" + } +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/src/index.ts b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/src/index.ts new file mode 100644 index 000000000..10e160113 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/src/index.ts @@ -0,0 +1,113 @@ +#!/usr/bin/env npx tsx +/** + * ts-hc-provider-a — the provider whose health check the test drives (#1480). + * + * TypeScript twin of py-hc-provider-a. Same contract, same flag file, same + * trace file: the three runtimes are meant to be behaviourally identical + * here, and divergence is exactly what keeps getting found. + * + * ## File-toggled, not invocation-counting + * + * `healthCheck` re-reads `/workspace/health-flag` on every tick so the test + * controls WHEN the transition happens and can poll for it: + * + * ok (or file absent) -> healthy heartbeats, stays resolvable + * fail -> unhealthy heartbeat suppressed -> registry withdraws + * throw -> throws must map to DEGRADED, must NOT withdraw + * + * ## Every invocation is traced + * + * One line is appended to `/workspace/hc-invocations.log` BEFORE the throw + * branch throws. Without it the negative test would pass vacuously: a health + * check that stopped running also fails to withdraw the agent, and "the + * agent is still resolvable" cannot tell that apart from the behaviour we + * actually want. + * + * Reads are sync on purpose. This is a test fixture: an async read would add + * a scheduling hop between "the test wrote the flag" and "the check saw it", + * for no coverage. + */ + +import { FastMCP, mesh } from "@mcpmesh/sdk"; +import { appendFileSync, readFileSync } from "node:fs"; +import { z } from "zod"; + +const AGENT_NAME = "hc-provider-a-ts"; +const FLAG_FILE = process.env.HC_FLAG_FILE ?? "/workspace/health-flag"; +const TRACE_FILE = process.env.HC_TRACE_FILE ?? "/workspace/hc-invocations.log"; + +const server = new FastMCP({ + name: "HC Provider A (typescript)", + version: "1.0.0", +}); + +/** Current fault state. A missing file means healthy, so the agent boots green. */ +function readFlag(): string { + try { + return readFileSync(FLAG_FILE, "utf8").trim().toLowerCase() || "ok"; + } catch { + return "ok"; + } +} + +/** Best-effort: a trace write that fails must not change the verdict. */ +function trace(flag: string, verdict: string): void { + try { + appendFileSync( + TRACE_FILE, + `${new Date().toISOString()} agent=${AGENT_NAME} flag=${flag} verdict=${verdict}\n`, + ); + } catch { + /* ignore */ + } +} + +const agent = mesh(server, { + name: AGENT_NAME, + version: "1.0.0", + description: + "Provider whose health check withdraws it from resolution (#1480)", + httpPort: Number(process.env.MCP_MESH_HTTP_PORT ?? "3421"), + healthCheck: () => { + const flag = readFlag(); + + if (flag === "fail") { + trace(flag, "unhealthy"); + return { + status: "unhealthy", + checks: { vendor_api_reachable: false }, + errors: ["simulated vendor outage (health-flag=fail)"], + }; + } + + if (flag === "throw") { + // Traced BEFORE throwing — see the file header. + trace(flag, "raised"); + throw new Error("simulated broken health check (health-flag=throw)"); + } + + trace(flag, "healthy"); + return { + status: "healthy", + checks: { vendor_api_reachable: true }, + }; + }, + // 2s so a withdrawal costs ~1 TTL + the registry staleness window rather + // than the 15s default; the test's registry runs at a matching 5s/2s. + healthCheckTtl: 2, +}); + +agent.addTool({ + name: "probe_a", + capability: "hc_probe_ts", + description: "Report which provider instance served this call", + tags: ["hc-withdrawal"], + parameters: z.object({}), + // pid is self-reported from inside the process, so it cannot go stale the + // way a pid FILE can. Baseline and post-recovery answers carrying the same + // pid is the proof that recovery did not restart anything. + execute: async () => + JSON.stringify({ served_by: AGENT_NAME, pid: process.pid }), +}); + +console.log(`${AGENT_NAME} defined. Waiting for auto-start...`); diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/tsconfig.json b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/tsconfig.json new file mode 100644 index 000000000..94f0ab505 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-a/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "outDir": "dist", + "declaration": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/package.json b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/package.json new file mode 100644 index 000000000..c3b86e972 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/package.json @@ -0,0 +1,23 @@ +{ + "name": "ts-hc-provider-b", + "version": "1.0.0", + "description": "uc41 survivor provider the consumer fails over to (issue #1480)", + "type": "module", + "engines": { + "node": ">=22.0.0" + }, + "scripts": { + "start": "tsx src/index.ts", + "build": "tsc", + "dev": "tsx watch src/index.ts" + }, + "dependencies": { + "@mcpmesh/sdk": "^3.5.1", + "zod": "^3.24.1" + }, + "devDependencies": { + "@types/node": "^22.18.0", + "tsx": "^4.0.0", + "typescript": "^5.7.0" + } +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/src/index.ts b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/src/index.ts new file mode 100644 index 000000000..b517b9a95 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/src/index.ts @@ -0,0 +1,43 @@ +#!/usr/bin/env npx tsx +/** + * ts-hc-provider-b — the survivor (issue #1480). + * + * Second provider of `hc_probe_ts`. Deliberately has NO `healthCheck`: it is + * the control. It must keep heartbeating throughout, so a run where BOTH + * providers go unhealthy (dead registry, stalled sweep, container-wide + * stall) is distinguishable from a genuine withdrawal of A. + * + * Loses the resolver tiebreak to A while A is healthy — equal tag score, + * equal version, then agent ID ASC, and `hc-provider-a-ts-` sorts + * before `hc-provider-b-ts-`. So the consumer deterministically starts + * on A and any answer naming B is a real re-resolution. + */ + +import { FastMCP, mesh } from "@mcpmesh/sdk"; +import { z } from "zod"; + +const AGENT_NAME = "hc-provider-b-ts"; + +const server = new FastMCP({ + name: "HC Provider B (typescript)", + version: "1.0.0", +}); + +const agent = mesh(server, { + name: AGENT_NAME, + version: "1.0.0", + description: "Survivor provider that the consumer fails over to (#1480)", + httpPort: Number(process.env.MCP_MESH_HTTP_PORT ?? "3422"), +}); + +agent.addTool({ + name: "probe_b", + capability: "hc_probe_ts", + description: "Report which provider instance served this call", + tags: ["hc-withdrawal"], + parameters: z.object({}), + execute: async () => + JSON.stringify({ served_by: AGENT_NAME, pid: process.pid }), +}); + +console.log(`${AGENT_NAME} defined. Waiting for auto-start...`); diff --git a/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/tsconfig.json b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/tsconfig.json new file mode 100644 index 000000000..94f0ab505 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/artifacts/ts-hc-provider-b/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "outDir": "dist", + "declaration": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/tests/integration/suites/uc41_health_check_withdrawal/routines.yaml b/tests/integration/suites/uc41_health_check_withdrawal/routines.yaml new file mode 100644 index 000000000..45aed61e9 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/routines.yaml @@ -0,0 +1,115 @@ +# UC-level routines for uc41_health_check_withdrawal (issue #1480). +# +# ============================================================================ +# SCENARIO COVERAGE +# ============================================================================ +# +# | Python | TypeScript | Java +# -----------------------+--------+------------+------ +# 1. withdrawal | tc01 | tc03 | tc05 +# 2. recovery, same pid | tc01 | tc03 | tc05 +# 3. a throw does NOT | tc02 | tc04 | tc06 +# withdraw (degraded) | | | +# +# Scenarios 1 and 2 share a test file per runtime because they are one +# continuous timeline, not two: recovery is only meaningful for the process +# that was withdrawn, and the load-bearing claim — the SAME pid across the +# outage AND the restore — cannot be made by a test that re-does the +# withdrawal from scratch. They are separate, separately-asserted phases +# inside the file (Phase A baseline / Phase B withdrawal+failover / Phase C +# recovery), the way uc25/tc03 splits Phase A and Phase B. Splitting them +# into separate test cases would pay the fleet-startup cost twice — three +# JVMs, in the Java case — for a strictly weaker assertion. +# +# Topology is identical in all six: two providers of the same capability plus +# one consumer. Provider A carries the health check the test drives; provider +# B has none and is the control (it must stay healthy throughout, so a +# container-wide stall cannot be mistaken for a withdrawal) and the failover +# target. Artifacts are per-runtime: py-hc-*, ts-hc-*, java-hc-*. +# +# NOT COVERED HERE: the scaffold's emitted vendor probe. Exercising it needs +# a base-URL override that does not exist yet — see #1483. +# +# ============================================================================ +# +# Every test in this UC turns on how fast the registry notices that an agent +# stopped heartbeating, so the timing knobs live here rather than being +# copy-pasted into nine phases. +# +# Default timing would make each test cost ~30s of pure waiting per +# transition: DEFAULT_TIMEOUT_THRESHOLD=20s before a silent agent is marked +# unhealthy, checked by a sweep that ticks every HEALTH_CHECK_INTERVAL=10s. +# At 5s/2s the same transition converges in ~5-7s. +# +# The agents' own knobs have to move with it, and are set per-test rather +# than here because they are per-process: +# - MCP_MESH_HEALTH_INTERVAL=2 heartbeat cadence, must stay well under the +# 5s threshold or a HEALTHY agent flaps +# - health-check TTL = 2s declared in each provider-a artifact +# +# MCP_MESH_RETENTION / MCP_MESH_SWEEP_INTERVAL are deliberately LEFT AT +# THEIR DEFAULTS. The recovery half of these tests needs the registry row to +# still exist and be marked `unhealthy` when heartbeats resume, because that +# is what makes HEAD /heartbeat answer 410 Gone and forces the POST +# re-register (#955). A short retention would purge the row instead and the +# agent would take a different path back. + +routines: + # Fast-sweep registry. Fails loudly if the tuning did not take effect: a + # silently-inert env var would turn every withdrawal poll in this UC into a + # 60s timeout with a confusing "never withdrawn" message. + start_registry_fast_sweep: + description: "Start the registry with a 5s staleness threshold and a 2s sweep" + steps: + - handler: shell + workdir: /workspace + command: | + MCP_MESH_HEALTH_CHECK_INTERVAL=2 \ + DEFAULT_TIMEOUT_THRESHOLD=5 \ + meshctl start --registry-only -d + # BOTH conditions are retried inside the SAME loop. /health can start + # answering before the health monitor's startup line is flushed, so a + # gate that made its verdict on the first successful curl would fail + # on that ordering race rather than on a real fault. + READY="" + for i in $(seq 1 30); do + if [ -z "$READY" ] && curl -sf http://localhost:8000/health > /dev/null 2>&1; then + READY="yes" + echo "registry ready after ${i}s" + fi + # The health monitor stamps its effective timing on startup: + # "Starting agent health monitor (timeout: 5s, interval: 2s)" + # Assert it rather than trusting the env var, so a rename or a + # dropped passthrough fails HERE instead of as a slow timeout. + if [ -n "$READY" ] && tail -n 200 ~/.mcp-mesh/logs/registry.log 2>/dev/null \ + | grep -qE 'health monitor \(timeout: 5s, interval: 2s\)'; then + echo "[setup] confirmed fast-sweep timing active" + exit 0 + fi + sleep 1 + done + # Loop expired. Distinguish "never came up" from "came up with the + # wrong timing" — the second is the deliberate loud failure: a + # silently-inert env var would turn every withdrawal poll in this UC + # into a confusing timeout instead of failing here. + if [ -n "$READY" ]; then + echo "FAIL: registry did not adopt the 5s/2s health-monitor timing" + grep -iE 'health monitor' ~/.mcp-mesh/logs/registry.log 2>/dev/null | tail -5 + else + echo "ERROR: registry did not become ready in 30s" + tail -50 ~/.mcp-mesh/logs/registry.log 2>/dev/null || true + fi + exit 1 + capture: registry_start + timeout: 60 + + # Stop everything this test started (agents + registry). `meshctl stop` is + # global, which is correct here and only here: the test owns the whole + # container. + stop_all: + description: "Stop all mesh processes started by this test" + steps: + - handler: shell + workdir: /workspace + command: meshctl stop 2>/dev/null || true + ignore_errors: true diff --git a/tests/integration/suites/uc41_health_check_withdrawal/tc01_python_withdraw_failover_recover/test.yaml b/tests/integration/suites/uc41_health_check_withdrawal/tc01_python_withdraw_failover_recover/test.yaml new file mode 100644 index 000000000..8d9cd2204 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/tc01_python_withdraw_failover_recover/test.yaml @@ -0,0 +1,437 @@ +# tc01 — Python: health-check withdrawal, consumer failover, and recovery +# without a restart (issue #1480; feature landed in #1472/#1473). +# +# FAILURE MODE THIS CATCHES +# +# Before #1472 a failing `health_check` affected NOTHING but the /health and +# /ready bodies: the heartbeat hard-coded HEALTHY, so the registry kept the +# provider in resolution and consumers kept being routed to an agent whose +# upstream was down. Every unit test in that area stubs the publish, and +# uc02/tc20 only proves the refresh LOOP fires — neither can see a runtime +# that computes the right verdict and then throws it away. This test fails if +# any link in the real chain breaks: +# +# health_check -> publish_health_status_to_core -> pyo3 update_health +# -> Rust heartbeat suppression -> registry staleness sweep +# -> resolution excludes A -> consumer re-resolves to B +# -> (flag cleared) heartbeat resumes -> HEAD 410 Gone -> POST re-register +# -> resolution includes A again -> consumer re-resolves back to A +# +# It also fails if withdrawal is implemented as anything OTHER than going +# quiet: `/livez` is polled every second across the whole outage and any gap +# fails the run. Withdrawn-not-dead is the entire point of the feature — a +# "fix" that exits the process, tears down the HTTP server or unregisters +# would satisfy the failover assertions and be caught here and by the pid. +# +# WHY THE FLAG FILE +# +# uc02/tc20 counts invocations, which is right for "does the loop fire at +# all". Here the test must control WHEN the transition happens so each phase +# can be gated on a real condition. The provider re-reads /workspace/health-flag +# on every tick; the test writes it. +# +# WHY NO FIXED WAITS FOR STATE TRANSITIONS +# +# Every phase polls for the actual condition — the registry's own status for +# each agent, and the consumer's answer naming a provider. A fixed sleep that +# happens to pass is the defect #1459 was about. Registry timing is tuned to +# 5s/2s by the uc routine and the agents beat every 2s with a 2s health-check +# TTL, so each transition converges in well under the 60s poll bound. + +name: "Python: health-check withdrawal, failover and recovery" +description: "A failing Python health check withdraws the provider, the consumer fails over, and clearing the fault restores it in the same process (issue #1480)" +tags: + - health-check + - withdrawal + - failover + - recovery + - lifecycle + - python + - integration +timeout: 420 + +pre_run: + - routine: global.setup_for_python_agent + params: + meshctl_version: "${config.packages.cli_version}" + mcpmesh_version: "${config.packages.sdk_python_version}" + +test: + - name: "Copy artifacts" + handler: shell + workdir: /workspace + command: | + cp -rL /uc-artifacts/py-hc-provider-a /workspace/ + cp -rL /uc-artifacts/py-hc-provider-b /workspace/ + cp -rL /uc-artifacts/py-hc-consumer /workspace/ + # Seed the flag explicitly rather than relying on the artifact's + # absent-file default, so a phase that fails to write it later is + # distinguishable from one that never had a file at all. + echo ok > /workspace/health-flag + : > /workspace/hc-invocations.log + echo "seeded health-flag=$(cat /workspace/health-flag)" + + - name: "Install provider A deps" + handler: pip-install + path: /workspace/py-hc-provider-a + + - name: "Install provider B deps" + handler: pip-install + path: /workspace/py-hc-provider-b + + - name: "Install consumer deps" + handler: pip-install + path: /workspace/py-hc-consumer + + - routine: start_registry_fast_sweep + + # MCP_MESH_HEALTH_INTERVAL=2 on every agent: the registry's staleness + # threshold is 5s, so a 5s default heartbeat would leave a HEALTHY agent one + # scheduling hiccup away from being swept. + - name: "Start provider A (port 3411, health-check driven)" + handler: shell + workdir: /workspace + command: meshctl start py-hc-provider-a/main.py --env MCP_MESH_HTTP_PORT=3411 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_a + + # ORDERING GATE — provider A must be registered and healthy BEFORE provider + # B is started, and both before the consumer. + # + # The resolver's last tiebreak is agent ID ASC, so hc-provider-a-* beats + # hc-provider-b-* — but only among candidates that EXIST when the consumer + # first resolves. Nothing later makes it move: the rewire is diff-gated, so + # a consumer that settled on B because A had not registered yet stays on B + # for the rest of the run, and the test loses the property it is built on + # ("an answer naming B means a re-resolution happened"). Observed flaking + # exactly this way under parallel load; a fixed sleep here would only make + # the flake rarer, so the gate polls the registry for the real condition. + - name: "Gate: provider A must be healthy before provider B starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 90); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-a-py") | .status') + echo "t=${i}s hc-provider-a-py=$S" + if [ "$S" = "healthy" ]; then echo "GATE_A: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_A: TIMEOUT" + exit 1 + capture: gate_a + timeout: 120 + + - name: "Start provider B (port 3412, survivor)" + handler: shell + workdir: /workspace + command: meshctl start py-hc-provider-b/main.py --env MCP_MESH_HTTP_PORT=3412 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_b + + - name: "Gate: provider B must be healthy before the consumer starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 90); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-b-py") | .status') + echo "t=${i}s hc-provider-b-py=$S" + if [ "$S" = "healthy" ]; then echo "GATE_B: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_B: TIMEOUT" + exit 1 + capture: gate_b + timeout: 120 + + - name: "Start consumer (port 3413, started once and never restarted)" + handler: shell + workdir: /workspace + command: meshctl start py-hc-consumer/main.py --env MCP_MESH_HTTP_PORT=3413 --env MCP_MESH_HEALTH_INTERVAL=2 --env MCP_MESH_SETTLE_TIMEOUT=60 -d + capture: start_consumer + + - name: "Wait for all three agents to be healthy in the registry" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 90); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-py") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-py") | .status') + C=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-consumer-py") | .status') + echo "t=${i}s A=$A B=$B C=$C" + if [ "$A" = "healthy" ] && [ "$B" = "healthy" ] && [ "$C" = "healthy" ]; then + echo "FLEET: OK after ~${i}s" + exit 0 + fi + sleep 1 + done + echo "FLEET: TIMEOUT" + meshctl list 2>/dev/null || true + exit 1 + capture: fleet_ready + timeout: 120 + + # ===== Phase A: baseline — the consumer starts on provider A ===== + # + # Deterministic, not lucky: equal tag score and equal version leave agent ID + # ASC as the resolver's last tiebreak, and hc-provider-a-py- sorts + # before hc-provider-b-py-. That is what makes "the answer named B" + # mean "a re-resolution happened" later on. + - name: "Phase A: poll until the consumer answers, and record which provider" + handler: shell + workdir: /workspace + command: | + # ELAPSED-TIME DEADLINE, NOT AN ITERATION COUNT. Each iteration costs a + # `meshctl call` plus a 1s sleep, so a fixed 60 iterations can run well + # past the 90s step timeout — and a step killed by the harness never + # prints `BASELINE: TIMEOUT`, losing the diagnostic exactly when the poll + # failed. The 70s budget leaves ~20s of headroom for an in-flight call + # and the verdict line. Same shape in the failover and failback loops. + DEADLINE=$(( $(date +%s) + 70 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-py:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-a-py*) + # Escape-tolerant on purpose: meshctl renders the Java/Python answer + # with an unescaped `structuredContent` block ("pid": 643) but the + # TypeScript answer only as the escaped text payload (\"pid\":278). + # A pattern anchored on the literal `"pid":` silently extracts + # NOTHING for TS, and an empty baseline would make the same-pid + # check below vacuously compare "" with "" — which is exactly what + # the BASELINE_PID assertion guards against. + printf '%s' "$R" | grep -o 'pid[^0-9]*[0-9][0-9]*' | grep -o '[0-9][0-9]*' | head -1 > /workspace/pid-a-baseline + echo "BASELINE: OK after ~${T}s served_by=hc-provider-a-py pid=$(cat /workspace/pid-a-baseline)" + exit 0 + ;; + *hc-provider-b-py*) + echo "BASELINE: WRONG_PROVIDER (resolved to B before any fault)" + exit 0 + ;; + esac + sleep 1 + done + echo "BASELINE: TIMEOUT" + capture: baseline + timeout: 90 + + # ===== Phase B: inject the fault ===== + - name: "Phase B: flip the health flag to fail" + handler: shell + workdir: /workspace + command: | + echo fail > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_injected + + # The registry status poll and the /livez liveness poll run in the SAME loop + # on purpose: /livez is then sampled once per second across the WHOLE outage + # window rather than at two convenient instants. A single failure fails the + # run — withdrawn must not mean dead. + # + # Provider B's status is polled alongside A's so that a run where BOTH go + # unhealthy (dead registry, stalled sweep, container-wide stall) cannot be + # mistaken for a genuine withdrawal of A. + - name: "Phase B: poll until the registry withdraws A, sampling /livez every second" + handler: shell + workdir: /workspace + command: | + LIVEZ_FAIL=0 + for i in $(seq 1 60); do + curl -sf --max-time 2 http://localhost:3411/livez > /dev/null 2>&1 || LIVEZ_FAIL=$((LIVEZ_FAIL + 1)) + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-py") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-py") | .status') + echo "t=${i}s A=$A B=$B livez_failures=$LIVEZ_FAIL" + if [ "$A" = "unhealthy" ] && [ "$B" = "healthy" ]; then + # One more livez sample after the transition lands. + curl -sf --max-time 2 http://localhost:3411/livez > /dev/null 2>&1 || LIVEZ_FAIL=$((LIVEZ_FAIL + 1)) + echo "WITHDRAWAL: OK after ~${i}s" + echo "LIVEZ_FAILURES: $LIVEZ_FAIL" + exit 0 + fi + sleep 1 + done + echo "WITHDRAWAL: TIMEOUT" + echo "LIVEZ_FAILURES: $LIVEZ_FAIL" + capture: withdrawal + timeout: 120 + + - name: "Phase B: A's own /health must report the unhealthy verdict, not a crash" + handler: shell + workdir: /workspace + command: curl -s --max-time 5 http://localhost:3411/health || echo 'HEALTH_UNREACHABLE' + capture: provider_a_health_during_outage + + - name: "Phase B: poll until the consumer fails over to provider B" + handler: shell + workdir: /workspace + command: | + # Elapsed-time deadline — see the note on the Phase A baseline poll. + DEADLINE=$(( $(date +%s) + 70 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-py:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-b-py*) echo "FAILOVER: OK after ~${T}s"; exit 0 ;; + esac + sleep 1 + done + echo "FAILOVER: TIMEOUT" + capture: failover + timeout: 90 + + - name: "Phase B: the health check must actually have seen the fault" + handler: shell + workdir: /workspace + command: | + # `grep -c` prints 0 AND exits 1 when the file exists with no matches, so + # a `|| echo 0` fallback would append a SECOND 0 and emit "FAIL_TICKS: 0 0". + # `|| true` adds no output; the ${:-0} default covers the missing-file case, + # where grep prints nothing at all. + FAIL_TICKS=$(grep -c 'flag=fail' /workspace/hc-invocations.log 2>/dev/null || true) + echo "FAIL_TICKS: ${FAIL_TICKS:-0}" + tail -12 /workspace/hc-invocations.log 2>/dev/null || echo 'NO_TRACE' + capture: fault_trace + + # ===== Phase C: clear the fault ===== + - name: "Phase C: flip the health flag back to ok" + handler: shell + workdir: /workspace + command: | + echo ok > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_cleared + + # Going back to `healthy` in the registry is not a formality: per #955 a HEAD + # heartbeat from an agent whose row is `unhealthy` is answered 410 Gone + # precisely so a bare ping cannot revive it. The only route back to healthy + # is a full POST re-register, so this poll turning green IS the 410 path. + - name: "Phase C: poll until the registry restores A (410 Gone -> POST re-register)" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 60); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-py") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-py") | .status') + echo "t=${i}s A=$A B=$B" + if [ "$A" = "healthy" ]; then echo "RECOVERY_REGISTRY: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "RECOVERY_REGISTRY: TIMEOUT" + capture: recovery_registry + timeout: 90 + + - name: "Phase C: poll until the consumer is routed back to A, and compare pids" + handler: shell + workdir: /workspace + command: | + BASE=$(cat /workspace/pid-a-baseline 2>/dev/null || echo '') + echo "BASELINE_PID: ${BASE:-NONE}" + # Elapsed-time deadline — see the note on the Phase A baseline poll. The + # TIMEOUT branch here also has to print PID_VERDICT: UNKNOWN, so being + # killed by the step timeout would leave the pid verdict entirely absent. + DEADLINE=$(( $(date +%s) + 70 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-py:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-a-py*) + NOW=$(printf '%s' "$R" | grep -o 'pid[^0-9]*[0-9][0-9]*' | grep -o '[0-9][0-9]*' | head -1) + echo "FAILBACK: OK after ~${T}s" + echo "RECOVERED_PID: ${NOW:-NONE}" + if [ -n "$BASE" ] && [ "$BASE" = "$NOW" ]; then + echo "PID_VERDICT: SAME ($BASE)" + else + echo "PID_VERDICT: CHANGED (baseline=${BASE:-NONE} recovered=${NOW:-NONE})" + fi + exit 0 + ;; + esac + sleep 1 + done + echo "FAILBACK: TIMEOUT" + echo "PID_VERDICT: UNKNOWN" + capture: failback + timeout: 90 + + # ===== Diagnostics (captured for the report, not asserted on) ===== + - name: "Diagnostic: provider A log" + handler: shell + workdir: /workspace + command: | + # Not `meshctl logs `: meshctl names the log file from its own + # static analysis of the entrypoint, which does not always match the + # registered agent name (py-hc-provider-a.log, probe_a.log ...). + tail -n 60 ~/.mcp-mesh/logs/*.log 2>/dev/null || echo 'NO_LOGS' + capture: provider_a_logs + ignore_errors: true + + - name: "Diagnostic: registry view of all agents" + handler: shell + workdir: /workspace + command: curl -s --max-time 5 http://localhost:8000/agents | jq -c '.agents[]? | {name, status, endpoint}' || echo 'NO_AGENTS' + capture: agents_final + ignore_errors: true + +assertions: + # ---- setup ---- + - expr: "${captured.fleet_ready} contains 'FLEET: OK'" + message: "SETUP: both providers and the consumer must be healthy in the registry before the fault is injected" + + # ---- Phase A: baseline ---- + - expr: "${captured.baseline} contains 'BASELINE: OK'" + message: "BASELINE: the consumer must resolve hc_probe_py and answer before any fault is injected" + - expr: "${captured.baseline} contains 'served_by=hc-provider-a-py'" + message: "BASELINE: the consumer must start on provider A (agent-ID tiebreak) — otherwise the later flip to B proves nothing" + - expr: "${captured.baseline} not contains 'BASELINE: WRONG_PROVIDER'" + message: "BASELINE: resolving to provider B before any fault means the tiebreak assumption no longer holds and this test cannot discriminate" + + # ---- Phase B: withdrawal ---- + # THE assertion. Fails if the verdict never reaches the Rust core, if the + # core keeps heartbeating while Unhealthy, or if the registry never sweeps. + - expr: "${captured.withdrawal} contains 'WITHDRAWAL: OK'" + message: "WITHDRAWAL: a health check reporting unhealthy must stop the heartbeat so the registry marks provider A unhealthy" + + # Withdrawn, not dead. A single missed /livez across the whole outage + # window fails this. + - expr: "${captured.withdrawal} contains 'LIVEZ_FAILURES: 0'" + message: "WITHDRAWAL: provider A's process must stay alive and serving /livez for the ENTIRE outage — withdrawn is not dead" + + - expr: "${captured.provider_a_health_during_outage} contains 'unhealthy'" + message: "WITHDRAWAL: provider A's own /health must report the unhealthy verdict while withdrawn" + - expr: "${captured.provider_a_health_during_outage} contains 'simulated vendor outage'" + message: "WITHDRAWAL: /health must carry the health check's own error string, proving the verdict came from the user check" + - expr: "${captured.provider_a_health_during_outage} not contains 'HEALTH_UNREACHABLE'" + message: "WITHDRAWAL: provider A's HTTP server must still answer /health while withdrawn" + + # The check really did observe the fault — rules out a passing run where the + # loop simply stopped. + - expr: "${captured.fault_trace} not contains 'FAIL_TICKS: 0'" + message: "WITHDRAWAL: the health check must have been invoked at least once while the flag said fail" + + # ---- Phase B: failover ---- + - expr: "${captured.failover} contains 'FAILOVER: OK'" + message: "FAILOVER: with provider A withdrawn, the never-restarted consumer must re-resolve hc_probe_py to provider B" + + # ---- Phase C: recovery ---- + - expr: "${captured.recovery_registry} contains 'RECOVERY_REGISTRY: OK'" + message: "RECOVERY: clearing the fault must resume the heartbeat and restore provider A to healthy via the 410 Gone re-register path (#955)" + - expr: "${captured.failback} contains 'FAILBACK: OK'" + message: "RECOVERY: the consumer must be routed back to the restored provider A" + + # No restart. The pid is reported from inside the provider process, so this + # compares the SAME process across withdrawal and recovery. + - expr: "${captured.failback} contains 'PID_VERDICT: SAME'" + message: "RECOVERY: provider A must recover in the SAME process — a changed pid means it was restarted, not restored" + - expr: "${captured.failback} not contains 'BASELINE_PID: NONE'" + message: "RECOVERY: the baseline pid must have been captured, otherwise the same-pid check is vacuous" + +post_run: + - routine: stop_all + - routine: global.cleanup_workspace diff --git a/tests/integration/suites/uc41_health_check_withdrawal/tc02_python_throwing_check_degrades/test.yaml b/tests/integration/suites/uc41_health_check_withdrawal/tc02_python_throwing_check_degrades/test.yaml new file mode 100644 index 000000000..4393fe109 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/tc02_python_throwing_check_degrades/test.yaml @@ -0,0 +1,320 @@ +# tc02 — Python: a health check that THROWS must degrade, never withdraw +# (issue #1480; rule established in #1472 and mirrored into #1474/#1476). +# +# FAILURE MODE THIS CATCHES +# +# The withdraw / don't-withdraw split has to be real. If any non-healthy +# verdict suppressed the heartbeat, then a NullPointerException, a typo in a +# probe, or a transient exception inside the user's own health check would +# silently take a perfectly good provider out of the mesh — the exact +# outage-amplifying behaviour this feature exists to prevent. tc01 proves +# `unhealthy` withdraws; this proves a THROW does not. Only both together +# say the split exists rather than "anything that isn't healthy withdraws". +# +# WHY THIS TEST IS NOT VACUOUS +# +# "The agent was not withdrawn" is trivially true of a runtime whose health +# loop died, never started, or never observed the fault — so three separate +# things are asserted, and the negative alone is never enough: +# +# 1. the invocation trace shows the check ran REPEATEDLY while the flag +# said `throw` (the loop is alive and seeing the fault); +# 2. /health reports `degraded` carrying the thrown exception's own message +# (the runtime observed the throw and classified it); +# 3. only then: the registry never moves A off `healthy`, and the consumer +# is still routed to A. +# +# WHY THE WATCH IS 30s AND WHY IT IS NOT A SLEEP +# +# The watch loop polls the registry every second and FAILS THE INSTANT A +# leaves `healthy` — it is an invariant watch with early exit, not a fixed +# wait that hopes. Its length is calibrated against tc01: withdrawal there +# converges in ~7s at this registry's 5s/2s timing, so 30s is over four times +# the latency a real withdrawal needs. Provider B is watched alongside A so a +# container-wide stall cannot masquerade as a held invariant. + +name: "Python: a throwing health check degrades and keeps serving" +description: "A Python health check that raises must map to degraded, keep heartbeating and stay resolvable (issue #1480)" +tags: + - health-check + - withdrawal + - degraded + - lifecycle + - python + - integration +timeout: 420 + +pre_run: + - routine: global.setup_for_python_agent + params: + meshctl_version: "${config.packages.cli_version}" + mcpmesh_version: "${config.packages.sdk_python_version}" + +test: + - name: "Copy artifacts" + handler: shell + workdir: /workspace + command: | + cp -rL /uc-artifacts/py-hc-provider-a /workspace/ + cp -rL /uc-artifacts/py-hc-provider-b /workspace/ + cp -rL /uc-artifacts/py-hc-consumer /workspace/ + echo ok > /workspace/health-flag + : > /workspace/hc-invocations.log + echo "seeded health-flag=$(cat /workspace/health-flag)" + + - name: "Install provider A deps" + handler: pip-install + path: /workspace/py-hc-provider-a + + - name: "Install provider B deps" + handler: pip-install + path: /workspace/py-hc-provider-b + + - name: "Install consumer deps" + handler: pip-install + path: /workspace/py-hc-consumer + + - routine: start_registry_fast_sweep + + - name: "Start provider A (port 3411, health-check driven)" + handler: shell + workdir: /workspace + command: meshctl start py-hc-provider-a/main.py --env MCP_MESH_HTTP_PORT=3411 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_a + + # ORDERING GATE — provider A must be registered and healthy BEFORE provider + # B is started, and both before the consumer. + # + # The resolver's last tiebreak is agent ID ASC, so hc-provider-a-* beats + # hc-provider-b-* — but only among candidates that EXIST when the consumer + # first resolves. Nothing later makes it move: the rewire is diff-gated, so + # a consumer that settled on B because A had not registered yet stays on B + # for the rest of the run — and then "the consumer is still routed to A" + # can never be true, whatever the runtime does with the throwing check. + # Observed flaking exactly this way under parallel load; a fixed sleep here + # would only make the flake rarer, so the gate polls for the real condition. + - name: "Gate: provider A must be healthy before provider B starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 90); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-a-py") | .status') + echo "t=${i}s hc-provider-a-py=$S" + if [ "$S" = "healthy" ]; then echo "GATE_A: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_A: TIMEOUT" + exit 1 + capture: gate_a + timeout: 120 + + - name: "Start provider B (port 3412, survivor)" + handler: shell + workdir: /workspace + command: meshctl start py-hc-provider-b/main.py --env MCP_MESH_HTTP_PORT=3412 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_b + + - name: "Gate: provider B must be healthy before the consumer starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 90); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-b-py") | .status') + echo "t=${i}s hc-provider-b-py=$S" + if [ "$S" = "healthy" ]; then echo "GATE_B: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_B: TIMEOUT" + exit 1 + capture: gate_b + timeout: 120 + + - name: "Start consumer (port 3413)" + handler: shell + workdir: /workspace + command: meshctl start py-hc-consumer/main.py --env MCP_MESH_HTTP_PORT=3413 --env MCP_MESH_HEALTH_INTERVAL=2 --env MCP_MESH_SETTLE_TIMEOUT=60 -d + capture: start_consumer + + - name: "Wait for all three agents to be healthy in the registry" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 90); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-py") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-py") | .status') + C=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-consumer-py") | .status') + echo "t=${i}s A=$A B=$B C=$C" + if [ "$A" = "healthy" ] && [ "$B" = "healthy" ] && [ "$C" = "healthy" ]; then + echo "FLEET: OK after ~${i}s" + exit 0 + fi + sleep 1 + done + echo "FLEET: TIMEOUT" + meshctl list 2>/dev/null || true + exit 1 + capture: fleet_ready + timeout: 120 + + - name: "Baseline: the consumer starts on provider A" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 60); do + R=$(meshctl call hc-consumer-py:who_served '{}' 2>&1 || true) + echo "t=${i}s -> $R" + case "$R" in + *hc-provider-a-py*) echo "BASELINE: OK after ~${i}s served_by=hc-provider-a-py"; exit 0 ;; + *hc-provider-b-py*) echo "BASELINE: WRONG_PROVIDER (resolved to B before any fault)"; exit 0 ;; + esac + sleep 1 + done + echo "BASELINE: TIMEOUT" + capture: baseline + timeout: 90 + + - name: "Break the health check: make it raise on every invocation" + handler: shell + workdir: /workspace + command: | + echo throw > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_injected + + # Invariant watch: fails the INSTANT A leaves healthy, rather than sleeping + # and hoping. 30s is >4x the ~7s a real withdrawal takes at this registry's + # 5s/2s timing (see tc01). + - name: "Watch for 30s: A must never leave healthy while its check keeps throwing" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 30); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-py") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-py") | .status') + echo "t=${i}s A=$A B=$B" + if [ "$A" != "healthy" ]; then + echo "DEGRADE_WATCH: WITHDRAWN at ~${i}s (status=$A) — a throwing check must not withdraw the agent" + exit 0 + fi + if [ "$B" != "healthy" ]; then + echo "DEGRADE_WATCH: CONTROL_LOST at ~${i}s (provider B status=$B) — container-wide stall, verdict unusable" + exit 0 + fi + sleep 1 + done + echo "DEGRADE_WATCH: HELD for 30s" + capture: degrade_watch + timeout: 90 + + # Non-vacuity, part 1: the loop kept running and kept seeing the fault. + - name: "The health check must have been invoked repeatedly while throwing" + handler: shell + workdir: /workspace + command: | + # `grep -c` prints 0 AND exits 1 when the file exists with no matches, so + # a `|| echo 0` fallback would append a SECOND 0, make N="0 0" and turn the + # `-ge 3` test below into a bash "integer expression expected" error — + # exactly in the failure path where a clean verdict matters most. + # `|| true` adds no output; the ${:-0} default covers the missing-file case, + # where grep prints nothing at all. + N=$(grep -c 'flag=throw' /workspace/hc-invocations.log 2>/dev/null || true) + N=${N:-0} + echo "THROW_TICKS: $N" + if [ "$N" -ge 3 ]; then echo "THROW_TICKS_VERDICT: SUFFICIENT"; else echo "THROW_TICKS_VERDICT: TOO_FEW"; fi + tail -12 /workspace/hc-invocations.log 2>/dev/null || echo 'NO_TRACE' + capture: throw_trace + + # Non-vacuity, part 2: the runtime saw the throw and classified it. + - name: "A's /health must report degraded, carrying the thrown message" + handler: shell + workdir: /workspace + command: curl -s --max-time 5 http://localhost:3411/health || echo 'HEALTH_UNREACHABLE' + capture: provider_a_health + + # Poll, don't assume — the same principle every other phase in this UC is + # built on. A single call turns one transient dispatch error into a bare + # CALL_FAILED that is indistinguishable from a real failover. Retrying does + # NOT weaken the claim: an answer naming provider B is a verdict on the + # spot, and a window that expires with no answer naming A is a failure too, + # so neither outcome can hide behind a retry. + - name: "Poll: the consumer must still be routed to provider A" + handler: shell + workdir: /workspace + command: | + DEADLINE=$(( $(date +%s) + 40 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-py:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + case "$R" in + *hc-provider-a-py*) + echo "t=${T}s -> $R" + echo "STILL_ON_A: OK after ~${T}s" + exit 0 + ;; + *hc-provider-b-py*) + echo "t=${T}s -> $R" + echo "STILL_ON_A: FAILED_OVER (answer named provider B — the throwing check withdrew A)" + exit 0 + ;; + esac + # Safe to echo verbatim: the cases above have already claimed every + # response that names either provider, so this can only be an error. + echo "t=${T}s (no provider named, retrying) -> $R" + sleep 2 + done + echo "STILL_ON_A: FAILED_OVER (no answer named provider A within the poll window)" + capture: still_on_a + timeout: 60 + + - name: "Diagnostic: provider A log" + handler: shell + workdir: /workspace + command: | + # Not `meshctl logs `: meshctl names the log file from its own + # static analysis of the entrypoint, which does not always match the + # registered agent name (py-hc-provider-a.log, probe_a.log ...). + tail -n 40 ~/.mcp-mesh/logs/*.log 2>/dev/null || echo 'NO_LOGS' + capture: provider_a_logs + ignore_errors: true + +assertions: + - expr: "${captured.fleet_ready} contains 'FLEET: OK'" + message: "SETUP: both providers and the consumer must be healthy before the check is broken" + - expr: "${captured.baseline} contains 'BASELINE: OK'" + message: "BASELINE: the consumer must resolve hc_probe_py and answer before the check is broken" + - expr: "${captured.baseline} contains 'served_by=hc-provider-a-py'" + message: "BASELINE: the consumer must start on provider A, so 'still on A' later is a meaningful statement" + + # ---- non-vacuity FIRST: without these the negative below proves nothing ---- + - expr: "${captured.throw_trace} contains 'THROW_TICKS_VERDICT: SUFFICIENT'" + message: "NON-VACUITY: the health check must have been invoked at least 3 times while raising — otherwise 'not withdrawn' just means the refresh loop died" + - expr: "${captured.provider_a_health} contains 'degraded'" + message: "CLASSIFICATION: a health check that raises must be recorded as DEGRADED, not healthy and not unhealthy" + - expr: "${captured.provider_a_health} contains 'simulated broken health check'" + message: "CLASSIFICATION: /health must carry the thrown exception's own message, proving the runtime caught THIS throw" + - expr: "${captured.provider_a_health} not contains 'HEALTH_UNREACHABLE'" + message: "CLASSIFICATION: provider A's HTTP server must still answer /health" + + # ---- the negative ---- + - expr: "${captured.degrade_watch} contains 'DEGRADE_WATCH: HELD'" + message: "DEGRADED MUST NOT WITHDRAW: a throwing health check must keep heartbeating — a buggy check must never be able to remove a working provider from the mesh" + - expr: "${captured.degrade_watch} not contains 'CONTROL_LOST'" + message: "CONTROL: provider B must stay healthy throughout, otherwise the watch verdict reflects a container-wide stall rather than the feature" + + # Explicit verdict markers, not a bare substring: the poll only prints + # STILL_ON_A: OK when an answer actually named provider A, and prints + # STILL_ON_A: FAILED_OVER for both ways this can go wrong (an answer naming + # B, or a window that expired with no answer at all). + - expr: "${captured.still_on_a} contains 'STILL_ON_A: OK'" + message: "RESOLVABLE: a degraded provider must remain selectable — the consumer must still be routed to A" + - expr: "${captured.still_on_a} not contains 'STILL_ON_A: FAILED_OVER'" + message: "RESOLVABLE: the consumer must NOT have failed over to B, and must have answered at all — either would mean the throwing check withdrew A" + - expr: "${captured.still_on_a} not contains 'hc-provider-b-py'" + message: "RESOLVABLE: provider B must not appear anywhere in the answer — the consumer must never have been re-routed to it" + +post_run: + - routine: stop_all + - routine: global.cleanup_workspace diff --git a/tests/integration/suites/uc41_health_check_withdrawal/tc03_typescript_withdraw_failover_recover/test.yaml b/tests/integration/suites/uc41_health_check_withdrawal/tc03_typescript_withdraw_failover_recover/test.yaml new file mode 100644 index 000000000..1e278e1db --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/tc03_typescript_withdraw_failover_recover/test.yaml @@ -0,0 +1,412 @@ +# tc03 — TypeScript: health-check withdrawal, consumer failover, and recovery +# without a restart (issue #1480; feature landed in #1476/#1481). +# +# FAILURE MODE THIS CATCHES +# +# TypeScript shipped the withdrawal mechanism with no integration coverage at +# all. What existed was `health-check.spec.ts`, which stubs `publish`, and a +# Rust unit test asserting a command was ENQUEUED. Neither can see the chain +# the feature exists for, and every link in it is TypeScript-specific code +# that Python's passing tests say nothing about: +# +# healthCheck -> napi updateHealth -> Rust heartbeat suppression +# -> registry staleness sweep -> resolution excludes A +# -> consumer re-resolves to B +# -> (flag cleared) heartbeat resumes -> HEAD 410 Gone -> POST re-register +# -> resolution includes A again -> consumer re-resolves back to A +# +# The napi binding was written for this release; a stubbed publish would keep +# passing if it were wired to nothing. +# +# It also fails if withdrawal is implemented as anything OTHER than going +# quiet: `/livez` is polled every second across the whole outage and any gap +# fails the run. Withdrawn-not-dead is the entire point — a "fix" that exits +# the process or tears down the server would satisfy the failover assertions +# and be caught here and by the pid. +# +# WHY THE VERDICT IS READ FROM THE LOG AND NOT FROM /health +# +# Unlike Python and Java, TypeScript's `/health` and `/ready` are FastMCP's +# built-ins and do NOT reflect the user health check's verdict — that is +# #1478, known and deliberately out of scope for #1481. So the TS tests read +# the verdict from the runtime's own `[mesh-health]` log line instead. If +# #1478 lands, this is the assertion to move onto /health. +# +# The rest of the design (file-toggled flag, poll-for-the-condition rather +# than fixed waits, provider B as the control) is documented in tc01. + +name: "TypeScript: health-check withdrawal, failover and recovery" +description: "A failing TypeScript health check withdraws the provider, the consumer fails over, and clearing the fault restores it in the same process (issue #1480)" +tags: + - health-check + - withdrawal + - failover + - recovery + - lifecycle + - typescript + - integration +timeout: 420 + +pre_run: + - routine: global.setup_for_typescript_agent + params: + meshctl_version: "${config.packages.cli_version}" + +test: + - name: "Copy artifacts" + handler: shell + workdir: /workspace + command: | + cp -rL /uc-artifacts/ts-hc-provider-a /workspace/ + cp -rL /uc-artifacts/ts-hc-provider-b /workspace/ + cp -rL /uc-artifacts/ts-hc-consumer /workspace/ + echo ok > /workspace/health-flag + : > /workspace/hc-invocations.log + echo "seeded health-flag=$(cat /workspace/health-flag)" + + - name: "Install provider A deps" + handler: npm-install + path: /workspace/ts-hc-provider-a + + - name: "Install provider B deps" + handler: npm-install + path: /workspace/ts-hc-provider-b + + - name: "Install consumer deps" + handler: npm-install + path: /workspace/ts-hc-consumer + + - routine: start_registry_fast_sweep + + # MCP_MESH_HEALTH_INTERVAL=2 on every agent: the registry's staleness + # threshold is 5s, so a 5s default heartbeat would leave a HEALTHY agent one + # scheduling hiccup away from being swept. + - name: "Start provider A (port 3421, health-check driven)" + handler: shell + workdir: /workspace + command: meshctl start ts-hc-provider-a/src/index.ts --env MCP_MESH_HTTP_PORT=3421 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_a + + # ORDERING GATE — provider A must be registered and healthy BEFORE provider + # B is started, and both before the consumer. + # + # The resolver's last tiebreak is agent ID ASC, so hc-provider-a-* beats + # hc-provider-b-* — but only among candidates that EXIST when the consumer + # first resolves. Nothing later makes it move: the rewire is diff-gated, so + # a consumer that settled on B because A had not registered yet stays on B + # for the rest of the run, and the test loses the property it is built on + # ("an answer naming B means a re-resolution happened"). Observed flaking + # exactly this way under parallel load; a fixed sleep here would only make + # the flake rarer, so the gate polls the registry for the real condition. + - name: "Gate: provider A must be healthy before provider B starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 120); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-a-ts") | .status') + echo "t=${i}s hc-provider-a-ts=$S" + if [ "$S" = "healthy" ]; then echo "GATE_A: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_A: TIMEOUT" + exit 1 + capture: gate_a + timeout: 150 + + - name: "Start provider B (port 3422, survivor)" + handler: shell + workdir: /workspace + command: meshctl start ts-hc-provider-b/src/index.ts --env MCP_MESH_HTTP_PORT=3422 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_b + + - name: "Gate: provider B must be healthy before the consumer starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 120); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-b-ts") | .status') + echo "t=${i}s hc-provider-b-ts=$S" + if [ "$S" = "healthy" ]; then echo "GATE_B: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_B: TIMEOUT" + exit 1 + capture: gate_b + timeout: 150 + + - name: "Start consumer (port 3423, started once and never restarted)" + handler: shell + workdir: /workspace + command: meshctl start ts-hc-consumer/src/index.ts --env MCP_MESH_HTTP_PORT=3423 --env MCP_MESH_HEALTH_INTERVAL=2 --env MCP_MESH_SETTLE_TIMEOUT=60 -d + capture: start_consumer + + - name: "Wait for all three agents to be healthy in the registry" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 120); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-ts") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-ts") | .status') + C=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-consumer-ts") | .status') + echo "t=${i}s A=$A B=$B C=$C" + if [ "$A" = "healthy" ] && [ "$B" = "healthy" ] && [ "$C" = "healthy" ]; then + echo "FLEET: OK after ~${i}s" + exit 0 + fi + sleep 1 + done + echo "FLEET: TIMEOUT" + meshctl list 2>/dev/null || true + exit 1 + capture: fleet_ready + timeout: 150 + + # ===== Phase A: baseline — the consumer starts on provider A ===== + - name: "Phase A: poll until the consumer answers, and record which provider" + handler: shell + workdir: /workspace + command: | + # ELAPSED-TIME DEADLINE, NOT AN ITERATION COUNT. Each iteration costs a + # `meshctl call` plus a 1s sleep, so a fixed 60 iterations can run well + # past the 90s step timeout — and a step killed by the harness never + # prints `BASELINE: TIMEOUT`, losing the diagnostic exactly when the poll + # failed. The 70s budget leaves ~20s of headroom for an in-flight call + # and the verdict line. Same shape in the failover and failback loops. + DEADLINE=$(( $(date +%s) + 70 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-ts:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-a-ts*) + # Escape-tolerant on purpose: meshctl renders the Java/Python answer + # with an unescaped `structuredContent` block ("pid": 643) but the + # TypeScript answer only as the escaped text payload (\"pid\":278). + # A pattern anchored on the literal `"pid":` silently extracts + # NOTHING for TS, and an empty baseline would make the same-pid + # check below vacuously compare "" with "" — which is exactly what + # the BASELINE_PID assertion guards against. + printf '%s' "$R" | grep -o 'pid[^0-9]*[0-9][0-9]*' | grep -o '[0-9][0-9]*' | head -1 > /workspace/pid-a-baseline + echo "BASELINE: OK after ~${T}s served_by=hc-provider-a-ts pid=$(cat /workspace/pid-a-baseline)" + exit 0 + ;; + *hc-provider-b-ts*) + echo "BASELINE: WRONG_PROVIDER (resolved to B before any fault)" + exit 0 + ;; + esac + sleep 1 + done + echo "BASELINE: TIMEOUT" + capture: baseline + timeout: 90 + + # ===== Phase B: inject the fault ===== + - name: "Phase B: flip the health flag to fail" + handler: shell + workdir: /workspace + command: | + echo fail > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_injected + + - name: "Phase B: poll until the registry withdraws A, sampling /livez every second" + handler: shell + workdir: /workspace + command: | + LIVEZ_FAIL=0 + for i in $(seq 1 60); do + curl -sf --max-time 2 http://localhost:3421/livez > /dev/null 2>&1 || LIVEZ_FAIL=$((LIVEZ_FAIL + 1)) + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-ts") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-ts") | .status') + echo "t=${i}s A=$A B=$B livez_failures=$LIVEZ_FAIL" + if [ "$A" = "unhealthy" ] && [ "$B" = "healthy" ]; then + curl -sf --max-time 2 http://localhost:3421/livez > /dev/null 2>&1 || LIVEZ_FAIL=$((LIVEZ_FAIL + 1)) + echo "WITHDRAWAL: OK after ~${i}s" + echo "LIVEZ_FAILURES: $LIVEZ_FAIL" + exit 0 + fi + sleep 1 + done + echo "WITHDRAWAL: TIMEOUT" + echo "LIVEZ_FAILURES: $LIVEZ_FAIL" + capture: withdrawal + timeout: 120 + + # The verdict, read from the runtime's own log rather than /health — see the + # #1478 note in the header. + # + # Every log is grepped rather than `meshctl logs `, because meshctl + # names the log file from its own static analysis of the entrypoint and for + # this TS agent that comes out as `probe_a.log` (the first tool's name), not + # the registered agent name. Grepping all of them is name-proof AND still + # specific: provider A is the only agent here with a health check, so a + # `[mesh-health]` line can only be its output. + - name: "Phase B: the runtime must have logged the UNHEALTHY verdict" + handler: shell + workdir: /workspace + command: | + LOG=$(cat ~/.mcp-mesh/logs/*.log 2>/dev/null || true) + N=$(printf '%s' "$LOG" | grep -c "reports UNHEALTHY" || true) + echo "UNHEALTHY_LOG_LINES: $N" + printf '%s' "$LOG" | grep 'mesh-health' | tail -10 || echo 'NO_MESH_HEALTH_LINES' + capture: verdict_log + ignore_errors: true + + - name: "Phase B: poll until the consumer fails over to provider B" + handler: shell + workdir: /workspace + command: | + # Elapsed-time deadline — see the note on the Phase A baseline poll. + DEADLINE=$(( $(date +%s) + 70 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-ts:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-b-ts*) echo "FAILOVER: OK after ~${T}s"; exit 0 ;; + esac + sleep 1 + done + echo "FAILOVER: TIMEOUT" + capture: failover + timeout: 90 + + - name: "Phase B: the health check must actually have seen the fault" + handler: shell + workdir: /workspace + command: | + # `grep -c` prints 0 AND exits 1 when the file exists with no matches, so + # a `|| echo 0` fallback would append a SECOND 0 and emit "FAIL_TICKS: 0 0". + # `|| true` adds no output; the ${:-0} default covers the missing-file case, + # where grep prints nothing at all. + FAIL_TICKS=$(grep -c 'flag=fail' /workspace/hc-invocations.log 2>/dev/null || true) + echo "FAIL_TICKS: ${FAIL_TICKS:-0}" + tail -12 /workspace/hc-invocations.log 2>/dev/null || echo 'NO_TRACE' + capture: fault_trace + + # ===== Phase C: clear the fault ===== + - name: "Phase C: flip the health flag back to ok" + handler: shell + workdir: /workspace + command: | + echo ok > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_cleared + + # Per #955 a HEAD heartbeat from an agent whose row is `unhealthy` is + # answered 410 Gone precisely so a bare ping cannot revive it. The only + # route back to healthy is a full POST re-register, so this poll turning + # green IS the 410 path. + - name: "Phase C: poll until the registry restores A (410 Gone -> POST re-register)" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 60); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-ts") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-ts") | .status') + echo "t=${i}s A=$A B=$B" + if [ "$A" = "healthy" ]; then echo "RECOVERY_REGISTRY: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "RECOVERY_REGISTRY: TIMEOUT" + capture: recovery_registry + timeout: 90 + + - name: "Phase C: poll until the consumer is routed back to A, and compare pids" + handler: shell + workdir: /workspace + command: | + BASE=$(cat /workspace/pid-a-baseline 2>/dev/null || echo '') + echo "BASELINE_PID: ${BASE:-NONE}" + # Elapsed-time deadline — see the note on the Phase A baseline poll. The + # TIMEOUT branch here also has to print PID_VERDICT: UNKNOWN, so being + # killed by the step timeout would leave the pid verdict entirely absent. + DEADLINE=$(( $(date +%s) + 70 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-ts:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-a-ts*) + NOW=$(printf '%s' "$R" | grep -o 'pid[^0-9]*[0-9][0-9]*' | grep -o '[0-9][0-9]*' | head -1) + echo "FAILBACK: OK after ~${T}s" + echo "RECOVERED_PID: ${NOW:-NONE}" + if [ -n "$BASE" ] && [ "$BASE" = "$NOW" ]; then + echo "PID_VERDICT: SAME ($BASE)" + else + echo "PID_VERDICT: CHANGED (baseline=${BASE:-NONE} recovered=${NOW:-NONE})" + fi + exit 0 + ;; + esac + sleep 1 + done + echo "FAILBACK: TIMEOUT" + echo "PID_VERDICT: UNKNOWN" + capture: failback + timeout: 90 + + - name: "Diagnostic: provider A log" + handler: shell + workdir: /workspace + command: | + # Not `meshctl logs `: meshctl names the log file from its own + # static analysis of the entrypoint, which does not always match the + # registered agent name (py-hc-provider-a.log, probe_a.log ...). + tail -n 60 ~/.mcp-mesh/logs/*.log 2>/dev/null || echo 'NO_LOGS' + capture: provider_a_logs + ignore_errors: true + + - name: "Diagnostic: registry view of all agents" + handler: shell + workdir: /workspace + command: curl -s --max-time 5 http://localhost:8000/agents | jq -c '.agents[]? | {name, status, endpoint}' || echo 'NO_AGENTS' + capture: agents_final + ignore_errors: true + +assertions: + - expr: "${captured.fleet_ready} contains 'FLEET: OK'" + message: "SETUP: both providers and the consumer must be healthy in the registry before the fault is injected" + + - expr: "${captured.baseline} contains 'BASELINE: OK'" + message: "BASELINE: the consumer must resolve hc_probe_ts and answer before any fault is injected" + - expr: "${captured.baseline} contains 'served_by=hc-provider-a-ts'" + message: "BASELINE: the consumer must start on provider A (agent-ID tiebreak) — otherwise the later flip to B proves nothing" + - expr: "${captured.baseline} not contains 'BASELINE: WRONG_PROVIDER'" + message: "BASELINE: resolving to provider B before any fault means the tiebreak assumption no longer holds and this test cannot discriminate" + + # THE assertion. Fails if the verdict never crosses the napi boundary, if the + # core keeps heartbeating while Unhealthy, or if the registry never sweeps. + - expr: "${captured.withdrawal} contains 'WITHDRAWAL: OK'" + message: "WITHDRAWAL: a healthCheck reporting unhealthy must stop the heartbeat so the registry marks provider A unhealthy" + - expr: "${captured.withdrawal} contains 'LIVEZ_FAILURES: 0'" + message: "WITHDRAWAL: provider A's process must stay alive and serving /livez for the ENTIRE outage — withdrawn is not dead" + + - expr: "${captured.verdict_log} not contains 'UNHEALTHY_LOG_LINES: 0'" + message: "WITHDRAWAL: the runtime must log the UNHEALTHY verdict, proving the withdrawal came from the user's healthCheck (TS /health cannot be used — see #1478)" + + - expr: "${captured.fault_trace} not contains 'FAIL_TICKS: 0'" + message: "WITHDRAWAL: the health check must have been invoked at least once while the flag said fail" + + - expr: "${captured.failover} contains 'FAILOVER: OK'" + message: "FAILOVER: with provider A withdrawn, the never-restarted consumer must re-resolve hc_probe_ts to provider B" + + - expr: "${captured.recovery_registry} contains 'RECOVERY_REGISTRY: OK'" + message: "RECOVERY: clearing the fault must resume the heartbeat and restore provider A to healthy via the 410 Gone re-register path (#955)" + - expr: "${captured.failback} contains 'FAILBACK: OK'" + message: "RECOVERY: the consumer must be routed back to the restored provider A" + - expr: "${captured.failback} contains 'PID_VERDICT: SAME'" + message: "RECOVERY: provider A must recover in the SAME process — a changed pid means it was restarted, not restored" + - expr: "${captured.failback} not contains 'BASELINE_PID: NONE'" + message: "RECOVERY: the baseline pid must have been captured, otherwise the same-pid check is vacuous" + +post_run: + - routine: stop_all + - routine: global.cleanup_workspace diff --git a/tests/integration/suites/uc41_health_check_withdrawal/tc04_typescript_throwing_check_degrades/test.yaml b/tests/integration/suites/uc41_health_check_withdrawal/tc04_typescript_throwing_check_degrades/test.yaml new file mode 100644 index 000000000..dccafe12f --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/tc04_typescript_throwing_check_degrades/test.yaml @@ -0,0 +1,358 @@ +# tc04 — TypeScript: a health check that THROWS must degrade, never withdraw +# (issue #1480; rule established in #1472 and mirrored into #1476/#1481). +# +# FAILURE MODE THIS CATCHES +# +# The withdraw / don't-withdraw split has to be real. If any non-healthy +# verdict suppressed the heartbeat, then a rejected promise, an undefined +# property access, or a transient error inside the user's own healthCheck +# would silently take a perfectly good provider out of the mesh — the exact +# outage-amplifying behaviour this feature exists to prevent. tc03 proves +# `unhealthy` withdraws; this proves a THROW does not. Only both together +# say the split exists rather than "anything that isn't healthy withdraws". +# +# TypeScript's containment story is also the most elaborate of the three +# runtimes — `runHealthCheck` catches, `tick` is total, and rescheduling is +# in a `finally` — and every one of those guards exists so that a throwing +# check leaves a LIVE loop reporting degraded. That is what is asserted here. +# +# WHY THIS TEST IS NOT VACUOUS +# +# "The agent was not withdrawn" is trivially true of a runtime whose health +# loop died, never started, or never observed the fault — so the non-vacuity +# assertions come FIRST and the negative only counts alongside them: +# +# 1. the invocation trace shows the check ran REPEATEDLY while the flag +# said `throw` (the loop survived the throw and kept rescheduling); +# 2. the runtime logged the degrade decision naming this agent; +# 3. only then: the registry never moves A off `healthy`, and the consumer +# is still routed to A. +# +# Unlike Python and Java, TS `/health` is FastMCP's built-in and does not +# reflect the verdict (#1478), so (2) is read from the `[mesh-health]` log. +# +# WHY THE WATCH IS 30s AND WHY IT IS NOT A SLEEP +# +# The watch loop polls the registry every second and FAILS THE INSTANT A +# leaves `healthy` — an invariant watch with early exit, not a fixed wait +# that hopes. Its length is calibrated against tc03: withdrawal there +# converges in ~7s at this registry's 5s/2s timing, so 30s is over four times +# the latency a real withdrawal needs. Provider B is watched alongside A so a +# container-wide stall cannot masquerade as a held invariant. + +name: "TypeScript: a throwing health check degrades and keeps serving" +description: "A TypeScript healthCheck that throws must map to degraded, keep heartbeating and stay resolvable (issue #1480)" +tags: + - health-check + - withdrawal + - degraded + - lifecycle + - typescript + - integration +timeout: 420 + +pre_run: + - routine: global.setup_for_typescript_agent + params: + meshctl_version: "${config.packages.cli_version}" + +test: + - name: "Copy artifacts" + handler: shell + workdir: /workspace + command: | + cp -rL /uc-artifacts/ts-hc-provider-a /workspace/ + cp -rL /uc-artifacts/ts-hc-provider-b /workspace/ + cp -rL /uc-artifacts/ts-hc-consumer /workspace/ + echo ok > /workspace/health-flag + : > /workspace/hc-invocations.log + echo "seeded health-flag=$(cat /workspace/health-flag)" + + - name: "Install provider A deps" + handler: npm-install + path: /workspace/ts-hc-provider-a + + - name: "Install provider B deps" + handler: npm-install + path: /workspace/ts-hc-provider-b + + - name: "Install consumer deps" + handler: npm-install + path: /workspace/ts-hc-consumer + + - routine: start_registry_fast_sweep + + - name: "Start provider A (port 3421, health-check driven)" + handler: shell + workdir: /workspace + command: meshctl start ts-hc-provider-a/src/index.ts --env MCP_MESH_HTTP_PORT=3421 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_a + + # ORDERING GATE — provider A must be registered and healthy BEFORE provider + # B is started, and both before the consumer. + # + # The resolver's last tiebreak is agent ID ASC, so hc-provider-a-* beats + # hc-provider-b-* — but only among candidates that EXIST when the consumer + # first resolves. Nothing later makes it move: the rewire is diff-gated, so + # a consumer that settled on B because A had not registered yet stays on B + # for the rest of the run — and then "the consumer is still routed to A" + # can never be true, whatever the runtime does with the throwing check. + # Observed flaking exactly this way under parallel load; a fixed sleep here + # would only make the flake rarer, so the gate polls for the real condition. + - name: "Gate: provider A must be healthy before provider B starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 120); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-a-ts") | .status') + echo "t=${i}s hc-provider-a-ts=$S" + if [ "$S" = "healthy" ]; then echo "GATE_A: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_A: TIMEOUT" + exit 1 + capture: gate_a + timeout: 150 + + - name: "Start provider B (port 3422, survivor)" + handler: shell + workdir: /workspace + command: meshctl start ts-hc-provider-b/src/index.ts --env MCP_MESH_HTTP_PORT=3422 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_b + + - name: "Gate: provider B must be healthy before the consumer starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 120); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-b-ts") | .status') + echo "t=${i}s hc-provider-b-ts=$S" + if [ "$S" = "healthy" ]; then echo "GATE_B: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_B: TIMEOUT" + exit 1 + capture: gate_b + timeout: 150 + + - name: "Start consumer (port 3423)" + handler: shell + workdir: /workspace + command: meshctl start ts-hc-consumer/src/index.ts --env MCP_MESH_HTTP_PORT=3423 --env MCP_MESH_HEALTH_INTERVAL=2 --env MCP_MESH_SETTLE_TIMEOUT=60 -d + capture: start_consumer + + - name: "Wait for all three agents to be healthy in the registry" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 120); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-ts") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-ts") | .status') + C=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-consumer-ts") | .status') + echo "t=${i}s A=$A B=$B C=$C" + if [ "$A" = "healthy" ] && [ "$B" = "healthy" ] && [ "$C" = "healthy" ]; then + echo "FLEET: OK after ~${i}s" + exit 0 + fi + sleep 1 + done + echo "FLEET: TIMEOUT" + meshctl list 2>/dev/null || true + exit 1 + capture: fleet_ready + timeout: 150 + + - name: "Baseline: the consumer starts on provider A" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 60); do + R=$(meshctl call hc-consumer-ts:who_served '{}' 2>&1 || true) + echo "t=${i}s -> $R" + case "$R" in + *hc-provider-a-ts*) echo "BASELINE: OK after ~${i}s served_by=hc-provider-a-ts"; exit 0 ;; + *hc-provider-b-ts*) echo "BASELINE: WRONG_PROVIDER (resolved to B before any fault)"; exit 0 ;; + esac + sleep 1 + done + echo "BASELINE: TIMEOUT" + capture: baseline + timeout: 90 + + - name: "Break the health check: make it throw on every invocation" + handler: shell + workdir: /workspace + command: | + echo throw > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_injected + + - name: "Watch for 30s: A must never leave healthy while its check keeps throwing" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 30); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-ts") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-ts") | .status') + echo "t=${i}s A=$A B=$B" + if [ "$A" != "healthy" ]; then + echo "DEGRADE_WATCH: WITHDRAWN at ~${i}s (status=$A) — a throwing check must not withdraw the agent" + exit 0 + fi + if [ "$B" != "healthy" ]; then + echo "DEGRADE_WATCH: CONTROL_LOST at ~${i}s (provider B status=$B) — container-wide stall, verdict unusable" + exit 0 + fi + sleep 1 + done + echo "DEGRADE_WATCH: HELD for 30s" + capture: degrade_watch + timeout: 90 + + # Non-vacuity, part 1: the loop survived the throw and kept rescheduling. + - name: "The health check must have been invoked repeatedly while throwing" + handler: shell + workdir: /workspace + command: | + # `grep -c` prints 0 AND exits 1 when the file exists with no matches, so + # a `|| echo 0` fallback would append a SECOND 0, make N="0 0" and turn the + # `-ge 3` test below into a bash "integer expression expected" error — + # exactly in the failure path where a clean verdict matters most. + # `|| true` adds no output; the ${:-0} default covers the missing-file case, + # where grep prints nothing at all. + N=$(grep -c 'flag=throw' /workspace/hc-invocations.log 2>/dev/null || true) + N=${N:-0} + echo "THROW_TICKS: $N" + if [ "$N" -ge 3 ]; then echo "THROW_TICKS_VERDICT: SUFFICIENT"; else echo "THROW_TICKS_VERDICT: TOO_FEW"; fi + tail -12 /workspace/hc-invocations.log 2>/dev/null || echo 'NO_TRACE' + capture: throw_trace + + # Non-vacuity, part 2: the runtime saw the throw and classified it as + # degraded. Read from the log because TS /health does not carry the verdict + # (#1478). + # + # Every log is grepped rather than `meshctl logs `, because meshctl + # names the log file from its own static analysis of the entrypoint and for + # this TS agent that comes out as `probe_a.log` (the first tool's name), not + # the registered agent name. Grepping all of them is name-proof AND still + # specific: provider A is the only agent here with a health check, so a + # `[mesh-health]` line can only be its output. + - name: "The runtime must have logged the degrade decision for this agent" + handler: shell + workdir: /workspace + command: | + LOG=$(cat ~/.mcp-mesh/logs/*.log 2>/dev/null || true) + THREW=$(printf '%s' "$LOG" | grep -c "threw" || true) + DEGRADED=$(printf '%s' "$LOG" | grep -c "reporting degraded" || true) + CORE_DEGRADE=$(printf '%s' "$LOG" | grep -c "Health status changed: Healthy -> Degraded" || true) + UNHEALTHY=$(printf '%s' "$LOG" | grep -c "reports UNHEALTHY" || true) + echo "THREW_LOG_LINES: $THREW" + echo "DEGRADE_CLASSIFIED_LINES: $DEGRADED" + echo "CORE_DEGRADE_TRANSITIONS: $CORE_DEGRADE" + echo "UNHEALTHY_LOG_LINES: $UNHEALTHY" + printf '%s' "$LOG" | grep 'mesh-health' | tail -10 || echo 'NO_MESH_HEALTH_LINES' + printf '%s' "$LOG" | grep 'Health status changed' | tail -5 || echo 'NO_CORE_HEALTH_TRANSITIONS' + capture: degrade_log + ignore_errors: true + + # Poll, don't assume — the same principle every other phase in this UC is + # built on. A single call turns one transient dispatch error into a bare + # CALL_FAILED that is indistinguishable from a real failover. Retrying does + # NOT weaken the claim: an answer naming provider B is a verdict on the + # spot, and a window that expires with no answer naming A is a failure too, + # so neither outcome can hide behind a retry. + - name: "Poll: the consumer must still be routed to provider A" + handler: shell + workdir: /workspace + command: | + DEADLINE=$(( $(date +%s) + 40 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-ts:who_served '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + case "$R" in + *hc-provider-a-ts*) + echo "t=${T}s -> $R" + echo "STILL_ON_A: OK after ~${T}s" + exit 0 + ;; + *hc-provider-b-ts*) + echo "t=${T}s -> $R" + echo "STILL_ON_A: FAILED_OVER (answer named provider B — the throwing check withdrew A)" + exit 0 + ;; + esac + # Safe to echo verbatim: the cases above have already claimed every + # response that names either provider, so this can only be an error. + echo "t=${T}s (no provider named, retrying) -> $R" + sleep 2 + done + echo "STILL_ON_A: FAILED_OVER (no answer named provider A within the poll window)" + capture: still_on_a + timeout: 60 + + - name: "Diagnostic: provider A log" + handler: shell + workdir: /workspace + command: | + # Not `meshctl logs `: meshctl names the log file from its own + # static analysis of the entrypoint, which does not always match the + # registered agent name (py-hc-provider-a.log, probe_a.log ...). + tail -n 40 ~/.mcp-mesh/logs/*.log 2>/dev/null || echo 'NO_LOGS' + capture: provider_a_logs + ignore_errors: true + +assertions: + - expr: "${captured.fleet_ready} contains 'FLEET: OK'" + message: "SETUP: both providers and the consumer must be healthy before the check is broken" + - expr: "${captured.baseline} contains 'BASELINE: OK'" + message: "BASELINE: the consumer must resolve hc_probe_ts and answer before the check is broken" + - expr: "${captured.baseline} contains 'served_by=hc-provider-a-ts'" + message: "BASELINE: the consumer must start on provider A, so 'still on A' later is a meaningful statement" + + # ---- non-vacuity FIRST: without these the negative below proves nothing ---- + - expr: "${captured.throw_trace} contains 'THROW_TICKS_VERDICT: SUFFICIENT'" + message: "NON-VACUITY: the health check must have been invoked at least 3 times while throwing — otherwise 'not withdrawn' just means the refresh loop died" + # POSITIVE, not merely "never unhealthy". A runtime that classified the + # throw as HEALTHY would also emit no UNHEALTHY line, and that is precisely + # one of the neuters this file is meant to catch. So the verdict has to be + # asserted in the affirmative, at two points on the chain: + # + # - `[mesh-health] ... threw — reporting degraded ...` (health-check.ts's + # catch): the loop caught THIS throw and took the degrade branch; + # - `Health status changed: Healthy -> Degraded` (mcp_mesh_core::heartbeat, + # logged only on a real transition): the degraded verdict crossed the + # napi boundary and became the core's state. This one is computed from + # the verdict rather than printed alongside it, so a throw branch that + # returns `healthy` produces NO such line and fails here. + - expr: "${captured.degrade_log} not contains 'THREW_LOG_LINES: 0'" + message: "CLASSIFICATION: the runtime must log that the check threw — proving it caught THIS throw rather than never running" + - expr: "${captured.degrade_log} not contains 'DEGRADE_CLASSIFIED_LINES: 0'" + message: "CLASSIFICATION: a throwing healthCheck must be classified DEGRADED by the runtime — 'not unhealthy' is also true of a runtime that wrongly called it healthy" + - expr: "${captured.degrade_log} not contains 'CORE_DEGRADE_TRANSITIONS: 0'" + message: "CLASSIFICATION: the degraded verdict must reach the mesh core (Health status changed: Healthy -> Degraded) — otherwise the runtime only printed the word and published something else" + - expr: "${captured.degrade_log} contains 'UNHEALTHY_LOG_LINES: 0'" + message: "CLASSIFICATION: a throwing check must never be classified UNHEALTHY — that is the verdict that withdraws" + + # ---- the negative ---- + - expr: "${captured.degrade_watch} contains 'DEGRADE_WATCH: HELD'" + message: "DEGRADED MUST NOT WITHDRAW: a throwing healthCheck must keep heartbeating — a buggy check must never be able to remove a working provider from the mesh" + - expr: "${captured.degrade_watch} not contains 'CONTROL_LOST'" + message: "CONTROL: provider B must stay healthy throughout, otherwise the watch verdict reflects a container-wide stall rather than the feature" + + # Explicit verdict markers, not a bare substring: the poll only prints + # STILL_ON_A: OK when an answer actually named provider A, and prints + # STILL_ON_A: FAILED_OVER for both ways this can go wrong (an answer naming + # B, or a window that expired with no answer at all). + - expr: "${captured.still_on_a} contains 'STILL_ON_A: OK'" + message: "RESOLVABLE: a degraded provider must remain selectable — the consumer must still be routed to A" + - expr: "${captured.still_on_a} not contains 'STILL_ON_A: FAILED_OVER'" + message: "RESOLVABLE: the consumer must NOT have failed over to B, and must have answered at all — either would mean the throwing check withdrew A" + - expr: "${captured.still_on_a} not contains 'hc-provider-b-ts'" + message: "RESOLVABLE: provider B must not appear anywhere in the answer — the consumer must never have been re-routed to it" + +post_run: + - routine: stop_all + - routine: global.cleanup_workspace diff --git a/tests/integration/suites/uc41_health_check_withdrawal/tc05_java_withdraw_failover_recover/test.yaml b/tests/integration/suites/uc41_health_check_withdrawal/tc05_java_withdraw_failover_recover/test.yaml new file mode 100644 index 000000000..10ce8ad44 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/tc05_java_withdraw_failover_recover/test.yaml @@ -0,0 +1,417 @@ +# tc05 — Java: health-check withdrawal, consumer failover, and recovery +# without a restart (issue #1480; feature landed in #1474/#1475). +# +# FAILURE MODE THIS CATCHES +# +# Java shipped the withdrawal mechanism with no integration coverage. The +# chain it has to complete is longer than Python's and every extra link is +# Java-only code written for this release — the C entry point for +# `mesh_update_health` did not exist before #1475, and there is a +# similarly-named `mesh_report_health` right next to it that touches shared +# state and CANNOT gate the heartbeat. Wiring the scheduler to that one would +# leave every Java unit test green and every provider un-withdrawable: +# +# @MeshHealthCheck -> MeshHealthCheckScheduler -> MeshRuntime.updateHealth +# -> C ABI mesh_update_health -> Rust heartbeat suppression +# -> registry staleness sweep -> resolution excludes A +# -> consumer re-resolves to B +# -> (flag cleared) heartbeat resumes -> HEAD 410 Gone -> POST re-register +# -> resolution includes A again -> consumer re-resolves back to A +# +# It also fails if withdrawal is implemented as anything OTHER than going +# quiet: `/livez` is polled every second across the whole outage and any gap +# fails the run. Withdrawn-not-dead is the entire point — a "fix" that exits +# the JVM or stops the web context would satisfy the failover assertions and +# be caught here and by the pid. +# +# The pid is reported by the provider's own tool via +# `ProcessHandle.current().pid()` rather than read from meshctl's pid file, +# which names the `mvn spring-boot:run` wrapper and not the JVM the plugin +# forks. A restarted JVM under a surviving wrapper would be invisible to the +# pid file and is caught here. +# +# The rest of the design (file-toggled flag, poll-for-the-condition rather +# than fixed waits, provider B as the control) is documented in tc01. + +name: "Java: health-check withdrawal, failover and recovery" +description: "A failing Java @MeshHealthCheck withdraws the provider, the consumer fails over, and clearing the fault restores it in the same JVM (issue #1480)" +tags: + - health-check + - withdrawal + - failover + - recovery + - lifecycle + - java + - integration + - slow +timeout: 900 + +pre_run: + - routine: global.setup_for_java_agent + params: + meshctl_version: "${config.packages.cli_version}" + +test: + - name: "Copy artifacts" + handler: shell + workdir: /workspace + command: | + cp -rL /uc-artifacts/java-hc-provider-a /workspace/ + cp -rL /uc-artifacts/java-hc-provider-b /workspace/ + cp -rL /uc-artifacts/java-hc-consumer /workspace/ + echo ok > /workspace/health-flag + : > /workspace/hc-invocations.log + echo "seeded health-flag=$(cat /workspace/health-flag)" + + - name: "Install provider A deps" + handler: maven-install + path: /workspace/java-hc-provider-a + timeout: 600 + + - name: "Install provider B deps" + handler: maven-install + path: /workspace/java-hc-provider-b + timeout: 600 + + - name: "Install consumer deps" + handler: maven-install + path: /workspace/java-hc-consumer + timeout: 600 + + - routine: start_registry_fast_sweep + + # MCP_MESH_HEALTH_INTERVAL=2 on every agent: the registry's staleness + # threshold is 5s, so a 5s default heartbeat would leave a HEALTHY agent one + # scheduling hiccup away from being swept. + - name: "Start provider A (port 3431, health-check driven)" + handler: shell + workdir: /workspace + command: meshctl start java-hc-provider-a --env MCP_MESH_HTTP_PORT=3431 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_a + + # ORDERING GATE — provider A must be registered and healthy BEFORE provider + # B is started, and both before the consumer. + # + # The resolver's last tiebreak is agent ID ASC, so hc-provider-a-* beats + # hc-provider-b-* — but only among candidates that EXIST when the consumer + # first resolves. Nothing later makes it move: the rewire is diff-gated, so + # a consumer that settled on B because A had not registered yet stays on B + # for the rest of the run, and the test loses the property it is built on + # ("an answer naming B means a re-resolution happened"). Observed flaking + # exactly this way under parallel load; a fixed sleep here would only make + # the flake rarer, so the gate polls the registry for the real condition. + - name: "Gate: provider A must be healthy before provider B starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 240); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-a-java") | .status') + echo "t=${i}s hc-provider-a-java=$S" + if [ "$S" = "healthy" ]; then echo "GATE_A: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_A: TIMEOUT" + exit 1 + capture: gate_a + timeout: 300 + + - name: "Start provider B (port 3432, survivor)" + handler: shell + workdir: /workspace + command: meshctl start java-hc-provider-b --env MCP_MESH_HTTP_PORT=3432 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_b + + - name: "Gate: provider B must be healthy before the consumer starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 240); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-b-java") | .status') + echo "t=${i}s hc-provider-b-java=$S" + if [ "$S" = "healthy" ]; then echo "GATE_B: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_B: TIMEOUT" + exit 1 + capture: gate_b + timeout: 300 + + - name: "Start consumer (port 3433, started once and never restarted)" + handler: shell + workdir: /workspace + command: meshctl start java-hc-consumer --env MCP_MESH_HTTP_PORT=3433 --env MCP_MESH_HEALTH_INTERVAL=2 --env MCP_MESH_SETTLE_TIMEOUT=60 -d + capture: start_consumer + + - name: "Wait for all three agents to be healthy in the registry" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 240); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-java") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-java") | .status') + C=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-consumer-java") | .status') + echo "t=${i}s A=$A B=$B C=$C" + if [ "$A" = "healthy" ] && [ "$B" = "healthy" ] && [ "$C" = "healthy" ]; then + echo "FLEET: OK after ~${i}s" + exit 0 + fi + sleep 1 + done + echo "FLEET: TIMEOUT" + meshctl list 2>/dev/null || true + tail -n 40 ~/.mcp-mesh/logs/*.log 2>/dev/null || true + exit 1 + capture: fleet_ready + timeout: 300 + + # ===== Phase A: baseline — the consumer starts on provider A ===== + - name: "Phase A: poll until the consumer answers, and record which provider" + handler: shell + workdir: /workspace + command: | + # ELAPSED-TIME DEADLINE, NOT AN ITERATION COUNT. Each iteration costs a + # `meshctl call` plus a 1s sleep, so a fixed 90 iterations can run well + # past the 120s step timeout — and a step killed by the harness never + # prints `BASELINE: TIMEOUT`, losing the diagnostic exactly when the poll + # failed. The 95s budget leaves ~25s of headroom for an in-flight call + # and the verdict line. Same shape in the failover and failback loops. + DEADLINE=$(( $(date +%s) + 95 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-java:whoServed '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-a-java*) + # Escape-tolerant on purpose: meshctl renders the Java/Python answer + # with an unescaped `structuredContent` block ("pid": 643) but the + # TypeScript answer only as the escaped text payload (\"pid\":278). + # A pattern anchored on the literal `"pid":` silently extracts + # NOTHING for TS, and an empty baseline would make the same-pid + # check below vacuously compare "" with "" — which is exactly what + # the BASELINE_PID assertion guards against. + printf '%s' "$R" | grep -o 'pid[^0-9]*[0-9][0-9]*' | grep -o '[0-9][0-9]*' | head -1 > /workspace/pid-a-baseline + echo "BASELINE: OK after ~${T}s served_by=hc-provider-a-java pid=$(cat /workspace/pid-a-baseline)" + exit 0 + ;; + *hc-provider-b-java*) + echo "BASELINE: WRONG_PROVIDER (resolved to B before any fault)" + exit 0 + ;; + esac + sleep 1 + done + echo "BASELINE: TIMEOUT" + capture: baseline + timeout: 120 + + # ===== Phase B: inject the fault ===== + - name: "Phase B: flip the health flag to fail" + handler: shell + workdir: /workspace + command: | + echo fail > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_injected + + - name: "Phase B: poll until the registry withdraws A, sampling /livez every second" + handler: shell + workdir: /workspace + command: | + # Each probe is retried ONCE, immediately, before it is allowed to count + # as a failure. This is the heaviest fleet in the UC — three JVMs — and a + # GC pause or a momentarily busy accept queue can lose a single probe, + # which would read as "the process died" and fail the run on a blip + # rather than on the feature. A genuinely dead process fails both + # attempts, so LIVEZ_FAILURES: 0 keeps exactly its old meaning. + livez_probe() { + curl -sf --max-time 2 http://localhost:3431/livez > /dev/null 2>&1 && return 0 + curl -sf --max-time 2 http://localhost:3431/livez > /dev/null 2>&1 + } + LIVEZ_FAIL=0 + for i in $(seq 1 60); do + livez_probe || LIVEZ_FAIL=$((LIVEZ_FAIL + 1)) + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-java") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-java") | .status') + echo "t=${i}s A=$A B=$B livez_failures=$LIVEZ_FAIL" + if [ "$A" = "unhealthy" ] && [ "$B" = "healthy" ]; then + # Final confirmation once the transition lands — same retry, same + # accounting. + livez_probe || LIVEZ_FAIL=$((LIVEZ_FAIL + 1)) + echo "WITHDRAWAL: OK after ~${i}s" + echo "LIVEZ_FAILURES: $LIVEZ_FAIL" + exit 0 + fi + sleep 1 + done + echo "WITHDRAWAL: TIMEOUT" + echo "LIVEZ_FAILURES: $LIVEZ_FAIL" + capture: withdrawal + timeout: 120 + + - name: "Phase B: A's own /health must report the unhealthy verdict, not a crash" + handler: shell + workdir: /workspace + command: curl -s --max-time 5 http://localhost:3431/health || echo 'HEALTH_UNREACHABLE' + capture: provider_a_health_during_outage + + - name: "Phase B: poll until the consumer fails over to provider B" + handler: shell + workdir: /workspace + command: | + # Elapsed-time deadline — see the note on the Phase A baseline poll. + DEADLINE=$(( $(date +%s) + 95 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-java:whoServed '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-b-java*) echo "FAILOVER: OK after ~${T}s"; exit 0 ;; + esac + sleep 1 + done + echo "FAILOVER: TIMEOUT" + capture: failover + timeout: 120 + + - name: "Phase B: the health check must actually have seen the fault" + handler: shell + workdir: /workspace + command: | + # `grep -c` prints 0 AND exits 1 when the file exists with no matches, so + # a `|| echo 0` fallback would append a SECOND 0 and emit "FAIL_TICKS: 0 0". + # `|| true` adds no output; the ${:-0} default covers the missing-file case, + # where grep prints nothing at all. + FAIL_TICKS=$(grep -c 'flag=fail' /workspace/hc-invocations.log 2>/dev/null || true) + echo "FAIL_TICKS: ${FAIL_TICKS:-0}" + tail -12 /workspace/hc-invocations.log 2>/dev/null || echo 'NO_TRACE' + capture: fault_trace + + # ===== Phase C: clear the fault ===== + - name: "Phase C: flip the health flag back to ok" + handler: shell + workdir: /workspace + command: | + echo ok > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_cleared + + # Per #955 a HEAD heartbeat from an agent whose row is `unhealthy` is + # answered 410 Gone precisely so a bare ping cannot revive it. The only + # route back to healthy is a full POST re-register, so this poll turning + # green IS the 410 path. + - name: "Phase C: poll until the registry restores A (410 Gone -> POST re-register)" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 60); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-java") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-java") | .status') + echo "t=${i}s A=$A B=$B" + if [ "$A" = "healthy" ]; then echo "RECOVERY_REGISTRY: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "RECOVERY_REGISTRY: TIMEOUT" + capture: recovery_registry + timeout: 90 + + - name: "Phase C: poll until the consumer is routed back to A, and compare pids" + handler: shell + workdir: /workspace + command: | + BASE=$(cat /workspace/pid-a-baseline 2>/dev/null || echo '') + echo "BASELINE_PID: ${BASE:-NONE}" + # Elapsed-time deadline — see the note on the Phase A baseline poll. The + # TIMEOUT branch here also has to print PID_VERDICT: UNKNOWN, so being + # killed by the step timeout would leave the pid verdict entirely absent. + DEADLINE=$(( $(date +%s) + 95 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-java:whoServed '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + echo "t=${T}s -> $R" + case "$R" in + *hc-provider-a-java*) + NOW=$(printf '%s' "$R" | grep -o 'pid[^0-9]*[0-9][0-9]*' | grep -o '[0-9][0-9]*' | head -1) + echo "FAILBACK: OK after ~${T}s" + echo "RECOVERED_PID: ${NOW:-NONE}" + if [ -n "$BASE" ] && [ "$BASE" = "$NOW" ]; then + echo "PID_VERDICT: SAME ($BASE)" + else + echo "PID_VERDICT: CHANGED (baseline=${BASE:-NONE} recovered=${NOW:-NONE})" + fi + exit 0 + ;; + esac + sleep 1 + done + echo "FAILBACK: TIMEOUT" + echo "PID_VERDICT: UNKNOWN" + capture: failback + timeout: 120 + + - name: "Diagnostic: provider A log" + handler: shell + workdir: /workspace + command: | + # Not `meshctl logs `: meshctl names the log file from its own + # static analysis of the entrypoint, which does not always match the + # registered agent name (py-hc-provider-a.log, probe_a.log ...). + tail -n 60 ~/.mcp-mesh/logs/*.log 2>/dev/null || echo 'NO_LOGS' + capture: provider_a_logs + ignore_errors: true + + - name: "Diagnostic: registry view of all agents" + handler: shell + workdir: /workspace + command: curl -s --max-time 5 http://localhost:8000/agents | jq -c '.agents[]? | {name, status, endpoint}' || echo 'NO_AGENTS' + capture: agents_final + ignore_errors: true + +assertions: + - expr: "${captured.fleet_ready} contains 'FLEET: OK'" + message: "SETUP: both providers and the consumer must be healthy in the registry before the fault is injected" + + - expr: "${captured.baseline} contains 'BASELINE: OK'" + message: "BASELINE: the consumer must resolve hc_probe_java and answer before any fault is injected" + - expr: "${captured.baseline} contains 'served_by=hc-provider-a-java'" + message: "BASELINE: the consumer must start on provider A (agent-ID tiebreak) — otherwise the later flip to B proves nothing" + - expr: "${captured.baseline} not contains 'BASELINE: WRONG_PROVIDER'" + message: "BASELINE: resolving to provider B before any fault means the tiebreak assumption no longer holds and this test cannot discriminate" + + # THE assertion. Fails if the scheduler calls the wrong FFI entry point, if + # the core keeps heartbeating while Unhealthy, or if the registry never sweeps. + - expr: "${captured.withdrawal} contains 'WITHDRAWAL: OK'" + message: "WITHDRAWAL: a @MeshHealthCheck reporting unhealthy must stop the heartbeat so the registry marks provider A unhealthy" + - expr: "${captured.withdrawal} contains 'LIVEZ_FAILURES: 0'" + message: "WITHDRAWAL: provider A's JVM must stay alive and serving /livez for the ENTIRE outage — withdrawn is not dead" + + - expr: "${captured.provider_a_health_during_outage} contains 'unhealthy'" + message: "WITHDRAWAL: provider A's own /health must report the unhealthy verdict while withdrawn" + - expr: "${captured.provider_a_health_during_outage} contains 'simulated vendor outage'" + message: "WITHDRAWAL: /health must carry the health check's own error string, proving the verdict came from the user check" + - expr: "${captured.provider_a_health_during_outage} not contains 'HEALTH_UNREACHABLE'" + message: "WITHDRAWAL: provider A's HTTP server must still answer /health while withdrawn" + + - expr: "${captured.fault_trace} not contains 'FAIL_TICKS: 0'" + message: "WITHDRAWAL: the health check must have been invoked at least once while the flag said fail" + + - expr: "${captured.failover} contains 'FAILOVER: OK'" + message: "FAILOVER: with provider A withdrawn, the never-restarted consumer must re-resolve hc_probe_java to provider B" + + - expr: "${captured.recovery_registry} contains 'RECOVERY_REGISTRY: OK'" + message: "RECOVERY: clearing the fault must resume the heartbeat and restore provider A to healthy via the 410 Gone re-register path (#955)" + - expr: "${captured.failback} contains 'FAILBACK: OK'" + message: "RECOVERY: the consumer must be routed back to the restored provider A" + - expr: "${captured.failback} contains 'PID_VERDICT: SAME'" + message: "RECOVERY: provider A must recover in the SAME JVM — a changed pid means it was restarted, not restored" + - expr: "${captured.failback} not contains 'BASELINE_PID: NONE'" + message: "RECOVERY: the baseline pid must have been captured, otherwise the same-pid check is vacuous" + +post_run: + - routine: stop_all + - routine: global.cleanup_workspace diff --git a/tests/integration/suites/uc41_health_check_withdrawal/tc06_java_throwing_check_degrades/test.yaml b/tests/integration/suites/uc41_health_check_withdrawal/tc06_java_throwing_check_degrades/test.yaml new file mode 100644 index 000000000..d6308cd45 --- /dev/null +++ b/tests/integration/suites/uc41_health_check_withdrawal/tc06_java_throwing_check_degrades/test.yaml @@ -0,0 +1,332 @@ +# tc06 — Java: a health check that THROWS must degrade, never withdraw +# (issue #1480; rule established in #1472 and mirrored into #1474/#1475). +# +# FAILURE MODE THIS CATCHES +# +# The withdraw / don't-withdraw split has to be real. If any non-healthy +# verdict suppressed the heartbeat, then a NullPointerException, a lazily +# initialised bean, or a transient exception inside the user's own +# @MeshHealthCheck would silently take a perfectly good provider out of the +# mesh — the exact outage-amplifying behaviour this feature exists to +# prevent. tc05 proves `unhealthy` withdraws; this proves a THROW does not. +# Only both together say the split exists rather than "anything that isn't +# healthy withdraws". +# +# Java reaches the check through reflection, so the throw arrives wrapped in +# an InvocationTargetException. Unwrapping it into DEGRADED rather than +# letting it fall through to some other branch is Java-specific code with no +# Python counterpart, and it is what this test exercises. +# +# WHY THIS TEST IS NOT VACUOUS +# +# "The agent was not withdrawn" is trivially true of a runtime whose +# scheduler thread died, never started, or never observed the fault — so the +# non-vacuity assertions come FIRST and the negative only counts alongside +# them: +# +# 1. the invocation trace shows the check ran REPEATEDLY while the flag +# said `throw` (the ScheduledExecutorService survived the throw — note +# that a raw `scheduleWithFixedDelay` task that lets an exception escape +# is CANCELLED for the rest of the JVM's life, so this is a real risk +# and not a hypothetical one); +# 2. /health reports `degraded` carrying the thrown exception's own +# message; +# 3. only then: the registry never moves A off `healthy`, and the consumer +# is still routed to A. +# +# WHY THE WATCH IS 30s AND WHY IT IS NOT A SLEEP +# +# The watch loop polls the registry every second and FAILS THE INSTANT A +# leaves `healthy` — an invariant watch with early exit, not a fixed wait +# that hopes. Its length is calibrated against tc05: withdrawal there +# converges in ~7s at this registry's 5s/2s timing, so 30s is over four times +# the latency a real withdrawal needs. Provider B is watched alongside A so a +# container-wide stall cannot masquerade as a held invariant. + +name: "Java: a throwing health check degrades and keeps serving" +description: "A Java @MeshHealthCheck that throws must map to degraded, keep heartbeating and stay resolvable (issue #1480)" +tags: + - health-check + - withdrawal + - degraded + - lifecycle + - java + - integration + - slow +timeout: 900 + +pre_run: + - routine: global.setup_for_java_agent + params: + meshctl_version: "${config.packages.cli_version}" + +test: + - name: "Copy artifacts" + handler: shell + workdir: /workspace + command: | + cp -rL /uc-artifacts/java-hc-provider-a /workspace/ + cp -rL /uc-artifacts/java-hc-provider-b /workspace/ + cp -rL /uc-artifacts/java-hc-consumer /workspace/ + echo ok > /workspace/health-flag + : > /workspace/hc-invocations.log + echo "seeded health-flag=$(cat /workspace/health-flag)" + + - name: "Install provider A deps" + handler: maven-install + path: /workspace/java-hc-provider-a + timeout: 600 + + - name: "Install provider B deps" + handler: maven-install + path: /workspace/java-hc-provider-b + timeout: 600 + + - name: "Install consumer deps" + handler: maven-install + path: /workspace/java-hc-consumer + timeout: 600 + + - routine: start_registry_fast_sweep + + - name: "Start provider A (port 3431, health-check driven)" + handler: shell + workdir: /workspace + command: meshctl start java-hc-provider-a --env MCP_MESH_HTTP_PORT=3431 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_a + + # ORDERING GATE — provider A must be registered and healthy BEFORE provider + # B is started, and both before the consumer. + # + # The resolver's last tiebreak is agent ID ASC, so hc-provider-a-* beats + # hc-provider-b-* — but only among candidates that EXIST when the consumer + # first resolves. Nothing later makes it move: the rewire is diff-gated, so + # a consumer that settled on B because A had not registered yet stays on B + # for the rest of the run — and then "the consumer is still routed to A" + # can never be true, whatever the runtime does with the throwing check. + # Observed flaking exactly this way under parallel load; a fixed sleep here + # would only make the flake rarer, so the gate polls for the real condition. + - name: "Gate: provider A must be healthy before provider B starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 240); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-a-java") | .status') + echo "t=${i}s hc-provider-a-java=$S" + if [ "$S" = "healthy" ]; then echo "GATE_A: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_A: TIMEOUT" + exit 1 + capture: gate_a + timeout: 300 + + - name: "Start provider B (port 3432, survivor)" + handler: shell + workdir: /workspace + command: meshctl start java-hc-provider-b --env MCP_MESH_HTTP_PORT=3432 --env MCP_MESH_HEALTH_INTERVAL=2 -d + capture: start_provider_b + + - name: "Gate: provider B must be healthy before the consumer starts" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 240); do + S=$(curl -s --max-time 3 http://localhost:8000/agents | jq -r '.agents[]? | select(.name=="hc-provider-b-java") | .status') + echo "t=${i}s hc-provider-b-java=$S" + if [ "$S" = "healthy" ]; then echo "GATE_B: OK after ~${i}s"; exit 0; fi + sleep 1 + done + echo "GATE_B: TIMEOUT" + exit 1 + capture: gate_b + timeout: 300 + + - name: "Start consumer (port 3433)" + handler: shell + workdir: /workspace + command: meshctl start java-hc-consumer --env MCP_MESH_HTTP_PORT=3433 --env MCP_MESH_HEALTH_INTERVAL=2 --env MCP_MESH_SETTLE_TIMEOUT=60 -d + capture: start_consumer + + - name: "Wait for all three agents to be healthy in the registry" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 240); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-java") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-java") | .status') + C=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-consumer-java") | .status') + echo "t=${i}s A=$A B=$B C=$C" + if [ "$A" = "healthy" ] && [ "$B" = "healthy" ] && [ "$C" = "healthy" ]; then + echo "FLEET: OK after ~${i}s" + exit 0 + fi + sleep 1 + done + echo "FLEET: TIMEOUT" + meshctl list 2>/dev/null || true + tail -n 40 ~/.mcp-mesh/logs/*.log 2>/dev/null || true + exit 1 + capture: fleet_ready + timeout: 300 + + - name: "Baseline: the consumer starts on provider A" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 90); do + R=$(meshctl call hc-consumer-java:whoServed '{}' 2>&1 || true) + echo "t=${i}s -> $R" + case "$R" in + *hc-provider-a-java*) echo "BASELINE: OK after ~${i}s served_by=hc-provider-a-java"; exit 0 ;; + *hc-provider-b-java*) echo "BASELINE: WRONG_PROVIDER (resolved to B before any fault)"; exit 0 ;; + esac + sleep 1 + done + echo "BASELINE: TIMEOUT" + capture: baseline + timeout: 120 + + - name: "Break the health check: make it throw on every invocation" + handler: shell + workdir: /workspace + command: | + echo throw > /workspace/health-flag + echo "health-flag=$(cat /workspace/health-flag) at $(date -u +%H:%M:%S)" + capture: fault_injected + + - name: "Watch for 30s: A must never leave healthy while its check keeps throwing" + handler: shell + workdir: /workspace + command: | + for i in $(seq 1 30); do + BODY=$(curl -s --max-time 3 http://localhost:8000/agents || echo '{}') + A=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-a-java") | .status') + B=$(echo "$BODY" | jq -r '.agents[]? | select(.name=="hc-provider-b-java") | .status') + echo "t=${i}s A=$A B=$B" + if [ "$A" != "healthy" ]; then + echo "DEGRADE_WATCH: WITHDRAWN at ~${i}s (status=$A) — a throwing check must not withdraw the agent" + exit 0 + fi + if [ "$B" != "healthy" ]; then + echo "DEGRADE_WATCH: CONTROL_LOST at ~${i}s (provider B status=$B) — container-wide stall, verdict unusable" + exit 0 + fi + sleep 1 + done + echo "DEGRADE_WATCH: HELD for 30s" + capture: degrade_watch + timeout: 90 + + # Non-vacuity, part 1: the scheduler survived the throw and kept ticking. + - name: "The health check must have been invoked repeatedly while throwing" + handler: shell + workdir: /workspace + command: | + # `grep -c` prints 0 AND exits 1 when the file exists with no matches, so + # a `|| echo 0` fallback would append a SECOND 0, make N="0 0" and turn the + # `-ge 3` test below into a bash "integer expression expected" error — + # exactly in the failure path where a clean verdict matters most. + # `|| true` adds no output; the ${:-0} default covers the missing-file case, + # where grep prints nothing at all. + N=$(grep -c 'flag=throw' /workspace/hc-invocations.log 2>/dev/null || true) + N=${N:-0} + echo "THROW_TICKS: $N" + if [ "$N" -ge 3 ]; then echo "THROW_TICKS_VERDICT: SUFFICIENT"; else echo "THROW_TICKS_VERDICT: TOO_FEW"; fi + tail -12 /workspace/hc-invocations.log 2>/dev/null || echo 'NO_TRACE' + capture: throw_trace + + # Non-vacuity, part 2: the runtime unwrapped the reflective throw and + # classified it. + - name: "A's /health must report degraded, carrying the thrown message" + handler: shell + workdir: /workspace + command: curl -s --max-time 5 http://localhost:3431/health || echo 'HEALTH_UNREACHABLE' + capture: provider_a_health + + # Poll, don't assume — the same principle every other phase in this UC is + # built on. A single call turns one transient dispatch error into a bare + # CALL_FAILED that is indistinguishable from a real failover. Retrying does + # NOT weaken the claim: an answer naming provider B is a verdict on the + # spot, and a window that expires with no answer naming A is a failure too, + # so neither outcome can hide behind a retry. + - name: "Poll: the consumer must still be routed to provider A" + handler: shell + workdir: /workspace + command: | + DEADLINE=$(( $(date +%s) + 40 )) + START=$(date +%s) + while [ "$(date +%s)" -lt "$DEADLINE" ]; do + R=$(meshctl call hc-consumer-java:whoServed '{}' 2>&1 || true) + T=$(( $(date +%s) - START )) + case "$R" in + *hc-provider-a-java*) + echo "t=${T}s -> $R" + echo "STILL_ON_A: OK after ~${T}s" + exit 0 + ;; + *hc-provider-b-java*) + echo "t=${T}s -> $R" + echo "STILL_ON_A: FAILED_OVER (answer named provider B — the throwing check withdrew A)" + exit 0 + ;; + esac + # Safe to echo verbatim: the cases above have already claimed every + # response that names either provider, so this can only be an error. + echo "t=${T}s (no provider named, retrying) -> $R" + sleep 2 + done + echo "STILL_ON_A: FAILED_OVER (no answer named provider A within the poll window)" + capture: still_on_a + timeout: 60 + + - name: "Diagnostic: provider A log" + handler: shell + workdir: /workspace + command: | + # Not `meshctl logs `: meshctl names the log file from its own + # static analysis of the entrypoint, which does not always match the + # registered agent name (py-hc-provider-a.log, probe_a.log ...). + tail -n 40 ~/.mcp-mesh/logs/*.log 2>/dev/null || echo 'NO_LOGS' + capture: provider_a_logs + ignore_errors: true + +assertions: + - expr: "${captured.fleet_ready} contains 'FLEET: OK'" + message: "SETUP: both providers and the consumer must be healthy before the check is broken" + - expr: "${captured.baseline} contains 'BASELINE: OK'" + message: "BASELINE: the consumer must resolve hc_probe_java and answer before the check is broken" + - expr: "${captured.baseline} contains 'served_by=hc-provider-a-java'" + message: "BASELINE: the consumer must start on provider A, so 'still on A' later is a meaningful statement" + + # ---- non-vacuity FIRST: without these the negative below proves nothing ---- + - expr: "${captured.throw_trace} contains 'THROW_TICKS_VERDICT: SUFFICIENT'" + message: "NON-VACUITY: the health check must have been invoked at least 3 times while throwing — otherwise 'not withdrawn' just means the scheduled task was cancelled by the escaping exception" + - expr: "${captured.provider_a_health} contains 'degraded'" + message: "CLASSIFICATION: a @MeshHealthCheck that throws must be recorded as DEGRADED, not healthy and not unhealthy" + - expr: "${captured.provider_a_health} contains 'simulated broken health check'" + message: "CLASSIFICATION: /health must carry the thrown exception's own message, proving the runtime unwrapped and caught THIS throw" + - expr: "${captured.provider_a_health} not contains 'HEALTH_UNREACHABLE'" + message: "CLASSIFICATION: provider A's HTTP server must still answer /health" + + # ---- the negative ---- + - expr: "${captured.degrade_watch} contains 'DEGRADE_WATCH: HELD'" + message: "DEGRADED MUST NOT WITHDRAW: a throwing @MeshHealthCheck must keep heartbeating — a buggy check must never be able to remove a working provider from the mesh" + - expr: "${captured.degrade_watch} not contains 'CONTROL_LOST'" + message: "CONTROL: provider B must stay healthy throughout, otherwise the watch verdict reflects a container-wide stall rather than the feature" + + # Explicit verdict markers, not a bare substring: the poll only prints + # STILL_ON_A: OK when an answer actually named provider A, and prints + # STILL_ON_A: FAILED_OVER for both ways this can go wrong (an answer naming + # B, or a window that expired with no answer at all). + - expr: "${captured.still_on_a} contains 'STILL_ON_A: OK'" + message: "RESOLVABLE: a degraded provider must remain selectable — the consumer must still be routed to A" + - expr: "${captured.still_on_a} not contains 'STILL_ON_A: FAILED_OVER'" + message: "RESOLVABLE: the consumer must NOT have failed over to B, and must have answered at all — either would mean the throwing check withdrew A" + - expr: "${captured.still_on_a} not contains 'hc-provider-b-java'" + message: "RESOLVABLE: provider B must not appear anywhere in the answer — the consumer must never have been re-routed to it" + +post_run: + - routine: stop_all + - routine: global.cleanup_workspace