test(integration): cover health-check withdrawal and recovery in all 3 runtimes

tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/pom.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.example</groupId>
+    <artifactId>java-hc-consumer</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>uc41 consumer that reports which provider served it</name>
+    <description>uc41 consumer that reports which provider served it (issue #1480)</description>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>4.0.5</version>
+        <relativePath/>
+    </parent>
+
+    <properties>
+        <java.version>17</java.version>
+        <mcp-mesh.version>3.5.1</mcp-mesh.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.mcp-mesh</groupId>
+            <artifactId>mcp-mesh-spring-boot-starter</artifactId>
+            <version>${mcp-mesh.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <mainClass>com.example.hcconsumer.HcConsumerApplication</mainClass>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/java/com/example/hcconsumer/HcConsumerApplication.java

@@ -0,0 +1,64 @@
+package com.example.hcconsumer;
+
+import io.mcpmesh.MeshAgent;
+import io.mcpmesh.MeshTool;
+import io.mcpmesh.Selector;
+import io.mcpmesh.types.McpMeshTool;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * java-hc-consumer — reports which provider the mesh routed it to (#1480).
+ *
+ * <p>{@code who_served} injects {@code hc_probe_java} and returns the
+ * provider's payload verbatim. Started ONCE and never restarted, so the only
+ * way its answer can move from provider A to provider B and back is a genuine
+ * re-resolution — the withdrawal / recovery chain under test.
+ *
+ * <p>The MCP tool name is the METHOD name verbatim — {@code @MeshTool} has no
+ * {@code name()} attribute and does not snake_case anything — so the test
+ * calls {@code hc-consumer-java:whoServed}, not {@code who_served}. Same
+ * convention as uc38's {@code getRejectCount}.
+ *
+ * <p>An unresolved dependency reports {@code served_by: "UNRESOLVED"} rather
+ * than throwing. The test's failover poll then keeps polling through a
+ * transient gap instead of tripping on it, and a PERMANENT gap still fails the
+ * run because {@code "UNRESOLVED"} never becomes a provider name.
+ */
+@SpringBootApplication
+@MeshAgent(
+    name = "hc-consumer-java",
+    version = "1.0.0",
+    description = "Consumer that must fail over when provider A withdraws (#1480)",
+    port = 3433
+)
+public class HcConsumerApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(HcConsumerApplication.class, args);
+    }
+
+    @MeshTool(
+        capability = "who_served_java",
+        description = "Call hc_probe_java and report which provider answered",
+        tags = {"hc-withdrawal"},
+        dependencies = {@Selector(capability = "hc_probe_java")}
+    )
+    public Map<String, Object> whoServed(McpMeshTool<Map<String, Object>> probe) {
+        if (probe == null || !probe.isAvailable()) {
+            return Map.of("served_by", "UNRESOLVED");
+        }
+        try {
+            Map<String, Object> payload = probe.call(Map.of());
+            if (payload == null || !payload.containsKey("served_by")) {
+                return Map.of("served_by", "UNRESOLVED", "raw", String.valueOf(payload));
+            }
+            return new LinkedHashMap<>(payload);
+        } catch (Exception e) {
+            return Map.of("served_by", "UNRESOLVED", "error", String.valueOf(e.getMessage()));
+        }
+    }
+}

tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-consumer/src/main/resources/application.yml

@@ -0,0 +1,20 @@
+# uc41 consumer that reports which provider served it (issue #1480)
+
+spring:
+  application:
+    name: java-hc-consumer
+
+server:
+  port: ${MCP_MESH_HTTP_PORT:3433}
+
+mesh:
+  registry:
+    url: ${MCP_MESH_REGISTRY_URL:http://localhost:8000}
+  agent:
+    name: ${MCP_MESH_AGENT_NAME:hc-consumer-java}
+    namespace: ${MCP_MESH_NAMESPACE:default}
+
+logging:
+  level:
+    io.mcpmesh: DEBUG
+    com.example: DEBUG

tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/pom.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.example</groupId>
+    <artifactId>java-hc-provider-a</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>uc41 provider whose health check withdraws it from resolution</name>
+    <description>uc41 provider whose health check withdraws it from resolution (issue #1480)</description>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>4.0.5</version>
+        <relativePath/>
+    </parent>
+
+    <properties>
+        <java.version>17</java.version>
+        <mcp-mesh.version>3.5.1</mcp-mesh.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.mcp-mesh</groupId>
+            <artifactId>mcp-mesh-spring-boot-starter</artifactId>
+            <version>${mcp-mesh.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <mainClass>com.example.hcprovidera.HcProviderAApplication</mainClass>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/java/com/example/hcprovidera/HcProviderAApplication.java

@@ -0,0 +1,125 @@
+package com.example.hcprovidera;
+
+import io.mcpmesh.MeshAgent;
+import io.mcpmesh.MeshHealth;
+import io.mcpmesh.MeshHealthCheck;
+import io.mcpmesh.MeshTool;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.Instant;
+import java.util.Map;
+
+/**
+ * java-hc-provider-a — the provider whose health check the test drives (#1480).
+ *
+ * <p>Java twin of py-hc-provider-a / ts-hc-provider-a. Same contract, same flag
+ * file, same trace file: the three runtimes are meant to be behaviourally
+ * identical here, and divergence is exactly what keeps getting found.
+ *
+ * <h2>File-toggled, not invocation-counting</h2>
+ *
+ * <p>{@code @MeshHealthCheck} re-reads {@code /workspace/health-flag} on every
+ * tick so the test controls WHEN the transition happens and can poll for it:
+ *
+ * <pre>
+ *   ok (or file absent) -&gt; healthy    heartbeats, stays resolvable
+ *   fail                -&gt; unhealthy  heartbeat suppressed -&gt; registry withdraws
+ *   throw               -&gt; throws     must map to DEGRADED, must NOT withdraw
+ * </pre>
+ *
+ * <h2>Every invocation is traced</h2>
+ *
+ * <p>One line is appended to {@code /workspace/hc-invocations.log} BEFORE the
+ * throw branch throws. Without it the negative test would pass vacuously: a
+ * health check that stopped running also fails to withdraw the agent, and "the
+ * agent is still resolvable" cannot tell that apart from what we want.
+ */
+@SpringBootApplication
+@MeshAgent(
+    name = "hc-provider-a-java",
+    version = "1.0.0",
+    description = "Provider whose health check withdraws it from resolution (#1480)",
+    port = 3431
+)
+public class HcProviderAApplication {
+
+    static final String AGENT_NAME = "hc-provider-a-java";
+
+    private static final Path FLAG_FILE = Path.of(
+        System.getenv().getOrDefault("HC_FLAG_FILE", "/workspace/health-flag"));
+    private static final Path TRACE_FILE = Path.of(
+        System.getenv().getOrDefault("HC_TRACE_FILE", "/workspace/hc-invocations.log"));
+
+    public static void main(String[] args) {
+        SpringApplication.run(HcProviderAApplication.class, args);
+    }
+
+    @MeshTool(
+        capability = "hc_probe_java",
+        description = "Report which provider instance served this call",
+        tags = {"hc-withdrawal"}
+    )
+    public Map<String, Object> probeA() {
+        // pid is self-reported from inside the JVM, so it cannot go stale the
+        // way a pid FILE can — and unlike meshctl's pid file it names the JVM
+        // rather than the `mvn spring-boot:run` wrapper that forked it.
+        // Baseline and post-recovery answers carrying the same pid is the
+        // proof that recovery did not restart anything.
+        return Map.of("served_by", AGENT_NAME, "pid", ProcessHandle.current().pid());
+    }
+
+    /**
+     * Simulated upstream-vendor probe, driven by the flag file.
+     *
+     * <p>ttlSeconds = 2 so a withdrawal costs ~1 TTL plus the registry staleness
+     * window rather than the 15s default; the test's registry runs at a matching
+     * 5s/2s.
+     */
+    @MeshHealthCheck(ttlSeconds = 2)
+    public MeshHealth vendorHealth() {
+        String flag = readFlag();
+
+        if ("fail".equals(flag)) {
+            trace(flag, "unhealthy");
+            return MeshHealth.unhealthy("simulated vendor outage (health-flag=fail)")
+                .withCheck("vendor_api_reachable", false);
+        }
+
+        if ("throw".equals(flag)) {
+            // Traced BEFORE throwing — see the class javadoc.
+            trace(flag, "raised");
+            throw new IllegalStateException(
+                "simulated broken health check (health-flag=throw)");
+        }
+
+        trace(flag, "healthy");
+        return MeshHealth.healthy().withCheck("vendor_api_reachable", true);
+    }
+
+    /** Current fault state. A missing file means healthy, so the agent boots green. */
+    private static String readFlag() {
+        try {
+            String raw = Files.readString(FLAG_FILE, StandardCharsets.UTF_8).trim();
+            return raw.isEmpty() ? "ok" : raw.toLowerCase();
+        } catch (Exception e) {
+            return "ok";
+        }
+    }
+
+    /** Best-effort: a trace write that fails must not change the verdict. */
+    private static void trace(String flag, String verdict) {
+        String line = Instant.now() + " agent=" + AGENT_NAME
+            + " flag=" + flag + " verdict=" + verdict + System.lineSeparator();
+        try {
+            Files.writeString(TRACE_FILE, line, StandardCharsets.UTF_8,
+                StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+        } catch (Exception ignored) {
+            // ignore
+        }
+    }
+}

tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-a/src/main/resources/application.yml

@@ -0,0 +1,20 @@
+# uc41 provider whose health check withdraws it from resolution (issue #1480)
+
+spring:
+  application:
+    name: java-hc-provider-a
+
+server:
+  port: ${MCP_MESH_HTTP_PORT:3431}
+
+mesh:
+  registry:
+    url: ${MCP_MESH_REGISTRY_URL:http://localhost:8000}
+  agent:
+    name: ${MCP_MESH_AGENT_NAME:hc-provider-a-java}
+    namespace: ${MCP_MESH_NAMESPACE:default}
+
+logging:
+  level:
+    io.mcpmesh: DEBUG
+    com.example: DEBUG

tests/integration/suites/uc41_health_check_withdrawal/artifacts/java-hc-provider-b/pom.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.example</groupId>
+    <artifactId>java-hc-provider-b</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>uc41 survivor provider the consumer fails over to</name>
+    <description>uc41 survivor provider the consumer fails over to (issue #1480)</description>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>4.0.5</version>
+        <relativePath/>
+    </parent>
+
+    <properties>
+        <java.version>17</java.version>
+        <mcp-mesh.version>3.5.1</mcp-mesh.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.mcp-mesh</groupId>
+            <artifactId>mcp-mesh-spring-boot-starter</artifactId>
+            <version>${mcp-mesh.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <mainClass>com.example.hcproviderb.HcProviderBApplication</mainClass>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>