diff --git a/vllm_sidecar/README.md b/vllm_sidecar/README.md index 7a82eef..c7926af 100644 --- a/vllm_sidecar/README.md +++ b/vllm_sidecar/README.md @@ -97,9 +97,48 @@ kill && sleep 7 etcdctl get --prefix XLLM:DEFAULT: # empty ``` +## Heartbeat (load metrics) + +Alongside lease keepalive, the sidecar scrapes vLLM's Prometheus `/metrics` and +POSTs a heartbeat to the master every `--heartbeat-interval` seconds: + + POST /v1/internal/heartbeat + X-Internal-Token: # only if configured + {"name","incarnation_id","load_metrics":{...},"latency_metrics":{...}} + +The master reuses the same `handle_instance_heartbeat` path as the brpc backends: +the JSON is parsed into a `proto::HeartbeatRequest` and feeds the scheduler's +in-memory load metrics (which the master then publishes to `XLLM:LOADMETRICS:` +for non-master nodes). Mapping (pinned to vLLM 0.21): + +| vLLM metric | heartbeat field | +|---|---| +| `vllm:num_requests_waiting` | `load_metrics.waiting_requests_num` | +| `vllm:gpu_cache_usage_perc` | `load_metrics.gpu_cache_usage_perc` | +| `vllm:time_to_first_token_seconds` | `latency_metrics.recent_max_ttft` (ms, interval avg) | +| `vllm:time_per_output_token_seconds` | `latency_metrics.recent_max_tbt` (ms, interval avg) | + +Heartbeat is **metrics-only**: liveness stays with the lease. A `409` (unknown / +stale incarnation) makes the sidecar re-register; a transient `/metrics` or POST +failure is logged and skipped — the lease still holds the instance up. + +### Heartbeat flags + +| flag | default | notes | +|---|---|---| +| `--xllm-service-url` | `http://127.0.0.1:9998` | master HTTP base for `/v1/internal/heartbeat` | +| `--internal-token` | `""` | `X-Internal-Token`; must match master `--internal_api_token`; also `XLLM_INTERNAL_TOKEN` | +| `--heartbeat-interval` | `3` | seconds between metrics heartbeats | +| `--metrics-url` | `/metrics` | vLLM Prometheus endpoint | + ## Scope -This PR provides **liveness-only** auto-registration via the etcd lease. Load -metrics (queue depth, KV-cache usage), an HTTP `/register`+`/heartbeat` contract -on the master, and `X-Internal-Token` auth are intentionally left to a follow-up -PR — none are needed for single-instance routing today. +Registration is lease-based (liveness); heartbeat adds load/latency metrics for +load-aware routing. The KV-cache event bridge (CacheAwareRouting) and +disaggregated-PD linking remain out of scope. + + + +## Internal heartbeat security + +The sidecar posts load metrics to `/v1/internal/heartbeat`. Expose this endpoint only on trusted networks. For production deployments, set `--internal_api_token` on the master and pass the same value to the sidecar with `--internal-token` or `XLLM_INTERNAL_TOKEN`. diff --git a/vllm_sidecar/metrics.py b/vllm_sidecar/metrics.py new file mode 100644 index 0000000..06994aa --- /dev/null +++ b/vllm_sidecar/metrics.py @@ -0,0 +1,112 @@ +# Copyright 2026 The xLLM Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/jd-opensource/xllm-service/blob/main/LICENSE +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Scrape vLLM's Prometheus `/metrics` into the xllm-service heartbeat schema. + +Maps vLLM gauges/histograms onto proto HeartbeatRequest fields (see +proto/xllm_rpc_service.proto LoadMetrics / LatencyMetrics): + + vllm:num_requests_waiting -> load_metrics.waiting_requests_num + vllm:gpu_cache_usage_perc -> load_metrics.gpu_cache_usage_perc + vllm:time_to_first_token_seconds -> latency_metrics.recent_max_ttft (ms) + vllm:time_per_output_token_seconds -> latency_metrics.recent_max_tbt (ms) + +Latency histograms only expose _sum/_count, so we report the per-interval +average (delta_sum/delta_count, in ms) as a pragmatic proxy for the proto's +"recent max" -- good enough for load-aware routing; a true max would need +bucket analysis. Metric names are pinned to vLLM 0.21. +""" + +import logging +import math +import re + +import requests + +logger = logging.getLogger("vllm_sidecar.metrics") + +# `name{labels} value [timestamp]` -> (base_name, value) +_SAMPLE_RE = re.compile( + r"^(?P[a-zA-Z_:][\w:]*)(?:\{[^}]*\})?\s+" + r"(?P[-+]?[0-9.eE+-]+|NaN|[-+]?Inf)\s*" +) + +_WAITING = "vllm:num_requests_waiting" +_GPU_CACHE = ("vllm:gpu_cache_usage_perc", "vllm:kv_cache_usage_perc") +_TTFT = "vllm:time_to_first_token_seconds" +_TBT = "vllm:time_per_output_token_seconds" + + +def _parse_samples(text: str) -> dict[str, float]: + """Sum sample values by base metric name (ignoring labels / HELP lines).""" + totals = {} + for line in text.splitlines(): + if not line or line[0] == "#": + continue + m = _SAMPLE_RE.match(line) + if not m: + continue + try: + val = float(m.group("value")) + except ValueError: + continue + # Drop NaN/Inf so downstream int()/JSON on these samples can't crash. + if not math.isfinite(val): + continue + totals[m.group("name")] = totals.get(m.group("name"), 0.0) + val + return totals + + +class VllmMetricsScraper: + def __init__(self, metrics_url: str, timeout: float = 3.0) -> None: + self._url = metrics_url + self._timeout = timeout + self._prev = {} # histogram base -> (sum, count) from the last scrape + # Reuse one connection across periodic scrapes (HTTP keep-alive). + self._session = requests.Session() + + def _interval_avg_ms(self, samples: dict[str, float], base: str) -> int: + """Average over the interval since the last scrape, in ms (0 if none).""" + cur = (samples.get(base + "_sum", 0.0), samples.get(base + "_count", 0.0)) + prev = self._prev.get(base) + self._prev[base] = cur + if prev is None: + return 0 # first observation: establish baseline, no interval yet + d_count = cur[1] - prev[1] + if d_count <= 0: + return 0 + d_sum = cur[0] - prev[0] + return max(0, int((d_sum / d_count) * 1000.0)) + + def scrape(self) -> dict | None: + """Return the heartbeat metrics dict, or None if /metrics unreachable.""" + try: + r = self._session.get(self._url, timeout=self._timeout) + r.raise_for_status() + except requests.RequestException as e: + logger.debug("metrics scrape failed: %s", e) + return None + + s = _parse_samples(r.text) + gpu_cache = next((s[n] for n in _GPU_CACHE if n in s), 0.0) + return { + "load_metrics": { + "waiting_requests_num": int(s.get(_WAITING, 0.0)), + "gpu_cache_usage_perc": float(gpu_cache), + }, + "latency_metrics": { + "recent_max_ttft": self._interval_avg_ms(s, _TTFT), + "recent_max_tbt": self._interval_avg_ms(s, _TBT), + }, + } diff --git a/vllm_sidecar/sidecar.py b/vllm_sidecar/sidecar.py index 889de56..5865f36 100644 --- a/vllm_sidecar/sidecar.py +++ b/vllm_sidecar/sidecar.py @@ -30,12 +30,16 @@ import os import signal import threading +import time import uuid from types import FrameType +import requests + from .etcd_registry import EtcdGatewayClient, EtcdError from .health import VllmHealthProbe from .meta import InstanceType, build_instance_key, build_instance_meta +from .metrics import VllmMetricsScraper logger = logging.getLogger("vllm_sidecar") @@ -57,6 +61,14 @@ def __init__(self, args: argparse.Namespace) -> None: self._stop = threading.Event() self._lease_id = None self._incarnation_id = None + # heartbeat (LoadMetrics/LatencyMetrics) -> master HTTP endpoint + self._hb_url = args.xllm_service_url.rstrip("/") + "/v1/internal/heartbeat" + self._metrics = VllmMetricsScraper( + args.metrics_url, timeout=args.health_timeout + ) + self._last_hb = 0.0 + # Reuse one connection for the periodic heartbeat POSTs (keep-alive). + self._hb_session = requests.Session() # --- registration primitives ------------------------------------------ @@ -131,6 +143,7 @@ def run(self) -> None: if self._health.is_healthy(): fail = 0 self._keepalive_or_reregister() + self._maybe_heartbeat() else: fail += 1 logger.warning( @@ -173,6 +186,51 @@ def _on_signal(self, signum: int, _frame: FrameType | None) -> None: logger.info("received signal %d, shutting down", signum) self._stop.set() + # --- heartbeat (metrics) ---------------------------------------------- + + def _maybe_heartbeat(self) -> None: + """POST LoadMetrics/LatencyMetrics on cadence while registered.""" + if not self._registered: + return + now = time.monotonic() + if now - self._last_hb < self._args.heartbeat_interval: + return + self._last_hb = now + self._send_heartbeat() + + def _send_heartbeat(self) -> None: + metrics = self._metrics.scrape() + if metrics is None: + return # /metrics transiently unreachable; lease still holds liveness + body = { + "name": self._args.register_addr, + "incarnation_id": self._incarnation_id, + "load_metrics": metrics["load_metrics"], + "latency_metrics": metrics["latency_metrics"], + } + headers = {"Content-Type": "application/json"} + if self._args.internal_token: + headers["X-Internal-Token"] = self._args.internal_token + try: + r = self._hb_session.post( + self._hb_url, + json=body, + headers=headers, + timeout=self._args.health_timeout, + ) + except requests.RequestException as e: + logger.debug("heartbeat POST failed: %s", e) + return + if r.status_code == 409: + # master doesn't know this incarnation -> re-register and resync id + logger.warning("heartbeat 409 (stale/unknown), re-registering") + self._lease_id = None + self._register() + elif r.status_code == 401: + logger.error("heartbeat 401: invalid --internal-token") + elif r.status_code != 200: + logger.warning("heartbeat -> HTTP %d: %s", r.status_code, r.text[:120]) + def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( @@ -234,6 +292,28 @@ def build_parser() -> argparse.ArgumentParser: default=3, help="consecutive failed probes before deregistering", ) + # heartbeat (LoadMetrics/LatencyMetrics reporting) + p.add_argument( + "--xllm-service-url", + default="http://127.0.0.1:9998", + help="master HTTP base URL for /v1/internal/heartbeat", + ) + p.add_argument( + "--internal-token", + default=os.environ.get("XLLM_INTERNAL_TOKEN", ""), + help="X-Internal-Token; must match master --internal_api_token", + ) + p.add_argument( + "--heartbeat-interval", + type=float, + default=3.0, + help="seconds between metrics heartbeats", + ) + p.add_argument( + "--metrics-url", + default=None, + help="vLLM Prometheus endpoint (default: /metrics)", + ) p.add_argument("--log-level", default="INFO") return p @@ -252,6 +332,8 @@ def main(argv: list[str] | None = None) -> int: ) if not args.register_addr: args.register_addr = _derive_addr(args.vllm_url) + if not args.metrics_url: + args.metrics_url = args.vllm_url.rstrip("/") + "/metrics" logger.info( "sidecar starting: register %s as backend_type=%s type=%s", args.register_addr, diff --git a/vllm_sidecar/tests/test_metrics.py b/vllm_sidecar/tests/test_metrics.py new file mode 100644 index 0000000..ca1bb06 --- /dev/null +++ b/vllm_sidecar/tests/test_metrics.py @@ -0,0 +1,81 @@ +# Copyright 2026 The xLLM Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/jd-opensource/xllm-service/blob/main/LICENSE +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Unit tests for vLLM /metrics parsing (no HTTP needed).""" + +from vllm_sidecar.metrics import VllmMetricsScraper, _parse_samples + +SAMPLE = """\ +# HELP vllm:num_requests_waiting Number of requests waiting. +# TYPE vllm:num_requests_waiting gauge +vllm:num_requests_waiting{model_name="qwen2.5-7b"} 4.0 +# TYPE vllm:gpu_cache_usage_perc gauge +vllm:gpu_cache_usage_perc{model_name="qwen2.5-7b"} 0.37 +# TYPE vllm:time_to_first_token_seconds histogram +vllm:time_to_first_token_seconds_sum{model_name="qwen2.5-7b"} 2.0 +vllm:time_to_first_token_seconds_count{model_name="qwen2.5-7b"} 10.0 +vllm:time_per_output_token_seconds_sum{model_name="qwen2.5-7b"} 1.0 +vllm:time_per_output_token_seconds_count{model_name="qwen2.5-7b"} 100.0 +""" + + +def test_parse_samples_strips_labels_and_comments() -> None: + s = _parse_samples(SAMPLE) + assert s["vllm:num_requests_waiting"] == 4.0 + assert s["vllm:gpu_cache_usage_perc"] == 0.37 + assert "vllm:num_requests_waiting" in s + # HELP/TYPE comment lines must not appear + assert all(not k.startswith("#") for k in s) + + +def test_parse_samples_sums_duplicate_label_sets() -> None: + text = ( + 'vllm:num_requests_waiting{m="a"} 2.0\nvllm:num_requests_waiting{m="b"} 3.0\n' + ) + assert _parse_samples(text)["vllm:num_requests_waiting"] == 5.0 + + +def test_parse_samples_handles_scientific_notation() -> None: + # Small latency values are emitted in scientific notation with a negative + # exponent; the value must parse fully, not be truncated at "1.23e" -> 0. + text = 'vllm:time_to_first_token_seconds_sum{m="a"} 1.23e-04\n' + assert _parse_samples(text)["vllm:time_to_first_token_seconds_sum"] == 1.23e-04 + + +def test_parse_samples_drops_nan_and_inf() -> None: + # vLLM may emit NaN/Inf; these must be dropped so downstream int()/JSON + # cannot crash the scrape loop. + text = ( + "vllm:num_requests_waiting NaN\n" + "vllm:gpu_cache_usage_perc +Inf\n" + "vllm:other 2.0\n" + ) + s = _parse_samples(text) + assert "vllm:num_requests_waiting" not in s + assert "vllm:gpu_cache_usage_perc" not in s + assert s["vllm:other"] == 2.0 + + +def test_interval_avg_ms_uses_delta() -> None: + sc = VllmMetricsScraper("http://unused") + # first call establishes the baseline -> 0 + assert sc._interval_avg_ms({"h_sum": 2.0, "h_count": 10.0}, "h") == 0 + # next: delta_sum=3.0 over delta_count=5 -> 0.6s avg -> 600 ms + assert sc._interval_avg_ms({"h_sum": 5.0, "h_count": 15.0}, "h") == 600 + + +def test_interval_avg_ms_zero_when_no_new_samples() -> None: + sc = VllmMetricsScraper("http://unused") + sc._interval_avg_ms({"h_sum": 2.0, "h_count": 10.0}, "h") + assert sc._interval_avg_ms({"h_sum": 2.0, "h_count": 10.0}, "h") == 0 diff --git a/vllm_sidecar/tests/test_sidecar.py b/vllm_sidecar/tests/test_sidecar.py index 0a8f506..9b230d9 100644 --- a/vllm_sidecar/tests/test_sidecar.py +++ b/vllm_sidecar/tests/test_sidecar.py @@ -239,6 +239,46 @@ def test_etcd_gateway_keepalive_handles_null_ttl(monkeypatch): assert EtcdGatewayClient("127.0.0.1:2379").lease_keepalive("lease-1") == 0 +def test_send_heartbeat_posts_metrics_with_token(monkeypatch): + _install_fakes(monkeypatch) + + class _Scraper: + def __init__(self, *a, **k): + pass + + def scrape(self): + return { + "load_metrics": { + "waiting_requests_num": 2, + "gpu_cache_usage_perc": 0.4, + }, + "latency_metrics": {"recent_max_ttft": 12, "recent_max_tbt": 3}, + } + + monkeypatch.setattr(sidecar_mod, "VllmMetricsScraper", _Scraper) + sc = sidecar_mod.Sidecar(_args(internal_token="tok")) + sc._lease_id = "lease-1" + sc._incarnation_id = "vllm-x" + + captured = {} + + def fake_post(url, json, headers, timeout): + captured.update(url=url, json=json, headers=headers) + + class _R: + status_code = 200 + text = "" + + return _R() + + monkeypatch.setattr(sc._hb_session, "post", fake_post) + sc._send_heartbeat() + + assert captured["url"].endswith("/v1/internal/heartbeat") + assert captured["headers"]["X-Internal-Token"] == "tok" + assert captured["json"]["load_metrics"]["waiting_requests_num"] == 2 + + def test_sidecar_registration_keepalive_and_deregister(monkeypatch): etcd = _install_fakes(monkeypatch) sc = sidecar_mod.Sidecar(_args()) diff --git a/xllm_service/common/global_gflags.cpp b/xllm_service/common/global_gflags.cpp index 943d8c3..d1e84d4 100644 --- a/xllm_service/common/global_gflags.cpp +++ b/xllm_service/common/global_gflags.cpp @@ -156,3 +156,9 @@ DEFINE_string(default_backend_type, DEFINE_int32(vllm_http_timeout_ms, 60000, "HTTP timeout (ms) when forwarding to vLLM backend."); + +DEFINE_string(internal_api_token, + "", + "Shared token guarding the internal Heartbeat endpoint via the " + "X-Internal-Token header. Empty value disables the check " + "(backwards compatible)."); diff --git a/xllm_service/common/global_gflags.h b/xllm_service/common/global_gflags.h index 76febf7..bafebc9 100644 --- a/xllm_service/common/global_gflags.h +++ b/xllm_service/common/global_gflags.h @@ -84,3 +84,5 @@ DECLARE_int32(readiness_check_interval_s); DECLARE_string(default_backend_type); DECLARE_int32(vllm_http_timeout_ms); + +DECLARE_string(internal_api_token); diff --git a/xllm_service/common/options.h b/xllm_service/common/options.h index ead23b9..be8b035 100644 --- a/xllm_service/common/options.h +++ b/xllm_service/common/options.h @@ -92,6 +92,8 @@ class Options { PROPERTY(std::string, default_backend_type) = "xllm"; PROPERTY(int32_t, vllm_http_timeout_ms) = 60000; + + PROPERTY(std::string, internal_api_token); }; } // namespace xllm_service diff --git a/xllm_service/examples/vllm_backend/README.md b/xllm_service/examples/vllm_backend/README.md index 6bc9913..252b50f 100644 --- a/xllm_service/examples/vllm_backend/README.md +++ b/xllm_service/examples/vllm_backend/README.md @@ -78,7 +78,7 @@ Set `STOP_VLLM=0` to leave an externally managed vLLM process running. ## Scope This demo covers the minimal vLLM backend path: OpenAI-compatible request -forwarding, model listing, streaming/non-streaming responses, and sidecar-based -etcd lease registration. Cache-aware +forwarding, model listing, streaming/non-streaming responses, sidecar-based +etcd lease registration, and optional vLLM metrics heartbeat reporting. Cache-aware routing, disaggregated PD, and mixed-backend clusters are out of scope for this demo. diff --git a/xllm_service/examples/vllm_backend/start_demo.sh b/xllm_service/examples/vllm_backend/start_demo.sh index b53b726..a67a52c 100755 --- a/xllm_service/examples/vllm_backend/start_demo.sh +++ b/xllm_service/examples/vllm_backend/start_demo.sh @@ -93,6 +93,7 @@ if [ -d "$ROOT/vllm_sidecar" ]; then --etcd-endpoints "$ETCD" \ --vllm-url "http://127.0.0.1:$VLLM_PORT" \ --register-addr "$INSTANCE_ADDR" \ + --xllm-service-url "http://127.0.0.1:$HTTP_PORT" \ > "$SIDECAR_LOG" 2>&1 & echo $! > "$SIDECAR_PID" ok "sidecar started (pid=$(cat "$SIDECAR_PID"))" diff --git a/xllm_service/http_service/service.cpp b/xllm_service/http_service/service.cpp index 762d92b..05a9a1f 100644 --- a/xllm_service/http_service/service.cpp +++ b/xllm_service/http_service/service.cpp @@ -37,6 +37,7 @@ limitations under the License. #include "http_service/anthropic_adapter.h" #include "http_service/chat_json_parser.h" #include "scheduler/scheduler.h" +#include "xllm_rpc_service.pb.h" #include "xllm_service.pb.h" namespace xllm_service { @@ -731,4 +732,62 @@ void XllmHttpServiceImpl::Metrics(::google::protobuf::RpcController* controller, // TODO: implement metrics endpoint } +void XllmHttpServiceImpl::Heartbeat( + ::google::protobuf::RpcController* controller, + const proto::HttpRequest* request, + proto::HttpResponse* response, + ::google::protobuf::Closure* done) { + ClosureGuard done_guard(done); + auto cntl = reinterpret_cast(controller); + if (!cntl) { + return; + } + + auto reply = [&](int32_t status_code, const char* body) { + cntl->http_response().set_status_code(status_code); + cntl->http_response().set_content_type("application/json"); + cntl->response_attachment().append(body); + }; + + // Optional static shared-token auth; skip the check when no token configured + // (backward compatible with deployments that don't set internal_api_token). + const std::string& expected = options_.internal_api_token(); + if (!expected.empty()) { + const std::string* got = cntl->http_request().GetHeader("X-Internal-Token"); + if (got == nullptr || *got != expected) { + LOG(WARNING) << "Heartbeat rejected: invalid X-Internal-Token"; + reply(401, "{\"error\":\"invalid internal token\"}"); + return; + } + } + + // Body is JSON of proto HeartbeatRequest (snake_case field names accepted). + proto::HeartbeatRequest req; + const std::string attachment = cntl->request_attachment().to_string(); + google::protobuf::util::JsonParseOptions opts; + opts.ignore_unknown_fields = true; + const auto status = + google::protobuf::util::JsonStringToMessage(attachment, &req, opts); + if (!status.ok()) { + LOG(ERROR) << "Heartbeat parse failed: " << status.ToString(); + reply(400, "{\"error\":\"invalid heartbeat json\"}"); + return; + } + if (req.name().empty()) { + reply(400, "{\"error\":\"missing instance name\"}"); + return; + } + + // Reuse the RPC heartbeat schema/path so HTTP sidecars and brpc + // backends update scheduler state consistently: liveness/incarnation, + // LoadMetrics, and LatencyMetrics. vLLM cache_event is empty. + if (!scheduler_ || !scheduler_->handle_instance_heartbeat(&req)) { + // Unknown instance or stale incarnation -> ask the sidecar to re-register. + reply(409, "{\"error\":\"instance not registered or stale incarnation\"}"); + return; + } + + reply(200, "{\"ok\":true}"); +} + } // namespace xllm_service diff --git a/xllm_service/http_service/service.h b/xllm_service/http_service/service.h index 2ef311e..160f82e 100644 --- a/xllm_service/http_service/service.h +++ b/xllm_service/http_service/service.h @@ -76,6 +76,13 @@ class XllmHttpServiceImpl : public proto::XllmHttpService { proto::HttpResponse* response, ::google::protobuf::Closure* done) override; + // Internal heartbeat from non-brpc backends (vLLM sidecar): JSON body of + // proto::HeartbeatRequest carrying LoadMetrics/LatencyMetrics. + void Heartbeat(::google::protobuf::RpcController* controller, + const proto::HttpRequest* request, + proto::HttpResponse* response, + ::google::protobuf::Closure* done) override; + private: template std::shared_ptr generate_request(T* req_pb, diff --git a/xllm_service/master.cpp b/xllm_service/master.cpp index e548758..64769b5 100644 --- a/xllm_service/master.cpp +++ b/xllm_service/master.cpp @@ -74,7 +74,8 @@ bool Master::setup_http_server() { "/v1/messages => AnthropicMessages," "/v1/embeddings => Embeddings," "/v1/models => Models," - "/metrics => Metrics,") != 0) { + "/metrics => Metrics," + "/v1/internal/heartbeat => Heartbeat,") != 0) { LOG(FATAL) << "Fail to add http service"; return false; } @@ -236,7 +237,8 @@ int main(int argc, char* argv[]) { .tool_call_parser(FLAGS_tool_call_parser) .reasoning_parser(FLAGS_reasoning_parser) .default_backend_type(FLAGS_default_backend_type) - .vllm_http_timeout_ms(FLAGS_vllm_http_timeout_ms); + .vllm_http_timeout_ms(FLAGS_vllm_http_timeout_ms) + .internal_api_token(FLAGS_internal_api_token); xllm_service::Master master(options); diff --git a/xllm_service/proto/xllm_http_service.proto b/xllm_service/proto/xllm_http_service.proto index 3e5899a..1df6ade 100644 --- a/xllm_service/proto/xllm_http_service.proto +++ b/xllm_service/proto/xllm_http_service.proto @@ -26,4 +26,7 @@ service XllmHttpService { rpc Embeddings (HttpRequest) returns (HttpResponse) {} rpc Models (HttpRequest) returns (HttpResponse) {} rpc Metrics (HttpRequest) returns (HttpResponse) {} + // Internal: backends (e.g. the vLLM sidecar) POST a heartbeat carrying + // LoadMetrics/LatencyMetrics here. Body is JSON of proto HeartbeatRequest. + rpc Heartbeat (HttpRequest) returns (HttpResponse) {} }