Add vLLM v2.27.0 worker source for Holo4 Hub

.github/workflows/verify.yml

@@ -0,0 +1,21 @@
+name: Verify Holo4 worker
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install pytest aiohttp
+      - run: python -m pytest -q tests
+      - run: python -m compileall -q src handler.py
+      - run: cmp handler.py src/handler.py
+      - run: python -m json.tool .runpod/hub.json > /dev/null
+      - run: python -m json.tool .runpod/tests.json > /dev/null

.gitignore

@@ -0,0 +1,7 @@
+
+runpod.toml
+*.pyc
+.env
+test/*
+vllm-base/vllm-*
+.DS_Store

Dockerfile

@@ -0,0 +1,65 @@
+# Worker image = official vLLM OpenAI server image + RunPod serverless wrapper.
+# vLLM upgrades are now a single build ARG:
+#   docker buildx build --build-arg VLLM_VERSION=v0.23.0 ...
+ARG VLLM_VERSION=v0.29.0
+FROM vllm/vllm-openai:${VLLM_VERSION}
+# Re-declare so the stage can reference it in RUN steps below.
+ARG VLLM_VERSION
+
+# RunPod serverless SDK + HTTP proxy deps (vLLM itself comes from the base image).
+COPY builder/requirements.txt /requirements.txt
+RUN python3 -m ensurepip --upgrade 2>/dev/null || true \
+    && python3 -m pip install --no-cache-dir -r /requirements.txt
+
+# bitsandbytes moved out of vLLM core into the out-of-tree vllm-bnb-plugin in
+# v0.28.0 (vllm-project/vllm#43529). The --no-deps is load-bearing: the plugin
+# declares an UNPINNED `vllm` dependency, and letting pip resolve it replaces
+# the base image's vLLM — it once backtracked to vllm 0.19.1, producing an
+# image tagged v0.28.0 that actually contained 0.19.1 (broken deep_ep/NCCL
+# mix). Its only real runtime need is the `bitsandbytes` package, pinned in
+# builder/requirements.txt.
+RUN python3 -m pip install --no-cache-dir --no-deps "vllm-bnb-plugin>=0.0.3"
+
+# Guard rail: the base image's vLLM must survive our pip installs — fail the
+# build loudly if the installed version no longer matches the requested tag.
+RUN test "$(python3 -c 'import vllm; print(vllm.__version__)')" = "${VLLM_VERSION#v}"
+
+# Setup for Option 2: building the image with the model baked in.
+ARG MODEL_NAME=""
+ARG MODEL_REVISION=""
+ARG TOKENIZER_NAME=""
+ARG TOKENIZER_REVISION=""
+ARG QUANTIZATION=""
+ARG BASE_PATH="/runpod-volume"
+
+ENV MODEL_NAME=$MODEL_NAME \
+    MODEL_REVISION=$MODEL_REVISION \
+    TOKENIZER_NAME=$TOKENIZER_NAME \
+    TOKENIZER_REVISION=$TOKENIZER_REVISION \
+    QUANTIZATION=$QUANTIZATION \
+    BASE_PATH=$BASE_PATH \
+    # The RunPod network volume mounts at $BASE_PATH; keep the HF cache there so
+    # model downloads persist across worker cold starts.
+    HF_HOME="${BASE_PATH}/huggingface-cache/hub" \
+    HUGGINGFACE_HUB_CACHE="${BASE_PATH}/huggingface-cache/hub" \
+    HF_DATASETS_CACHE="${BASE_PATH}/huggingface-cache/datasets" \
+    HF_HUB_ENABLE_HF_TRANSFER=0 \
+    TOKENIZERS_PARALLELISM=false
+
+COPY src /src
+
+# Optionally bake the model into the image at build time. Pass the
+# HF_TOKEN build secret if the repo is gated:
+#   docker buildx build --secret id=HF_TOKEN ... --build-arg MODEL_NAME=...
+RUN --mount=type=secret,id=HF_TOKEN,required=false \
+    if [ -n "$MODEL_NAME" ]; then \
+        if [ -f /run/secrets/HF_TOKEN ]; then \
+            export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
+        fi && \
+        python3 /src/download_model.py; \
+    fi
+
+# main.py spawns `vllm serve` with args built from the environment, waits for
+# /health, then starts the RunPod serverless loop. Explicit ENTRYPOINT so the
+# base image's own entrypoint can never swallow our command.
+ENTRYPOINT ["python3", "/src/main.py"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Runpod
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

UPSTREAM.md

@@ -0,0 +1,7 @@
+# Worker source
+
+The worker implementation (`Dockerfile`, `builder/`, `src/`, and `tests/`) is copied from [`runpod-workers/worker-vllm` tag `v2.27.0`](https://github.com/runpod-workers/worker-vllm/tree/v2.27.0), commit `76054c22c79c515f07065f523598d8efb2f9b682`. Its MIT license is in `LICENSE`.
+
+The root `handler.py` is a byte-for-byte copy of `src/handler.py` to meet the Runpod Hub publishing guide's root-file requirement. The Dockerfile runs the `src/` copy. Keep the copies identical when changing the handler.
+
+The three Holo4 bundle files in `.runpod/` and `README.md` were copied unchanged from the newer bundle comment on [DR-1532](https://linear.app/runpod/issue/DR-1532/review-serverless-worker-and-hub-listing-tests-green-marked-official).

builder/requirements.txt

@@ -0,0 +1,10 @@
+# vLLM itself comes from the vllm/vllm-openai base image; this is only what
+# the RunPod wrapper adds on top.
+runpod~=1.12.0
+aiohttp>=3.9
+huggingface-hub>=0.24
+# bitsandbytes moved out of vLLM core into an out-of-tree plugin in v0.28.0
+# (vllm-project/vllm#43529); without it LOAD_FORMAT/QUANTIZATION=bitsandbytes
+# configs die at model load. This is its runtime library — the plugin itself
+# is installed in the Dockerfile with --no-deps (see the comment there).
+bitsandbytes>=0.50.2

handler.py

@@ -0,0 +1,164 @@
+"""RunPod Serverless handler that proxies jobs to the local vLLM OpenAI server.
+
+`main.py` starts `vllm serve` on 127.0.0.1:VLLM_PORT and only then starts the
+RunPod serverless loop, so by the time a job arrives the vLLM HTTP server is up
+and fully backwards/forwards compatible — we never import vLLM. The one
+exception: when vLLM could not start for a reason a restart cannot fix,
+`main.py` sets `startup_error` and every job is answered with that message.
+
+Accepted job input shapes (all under job["input"]):
+
+1. RunPod OpenAI passthrough (what the platform sends on /openai/v1/...):
+       {"openai_route": "/v1/chat/completions", "openai_input": {...}}
+2. Generic proxy to any vLLM route:
+       {"route": "/v1/completions", "body": {...}, "method": "POST"}
+       ("method" is optional — bodies POST, body-less requests GET)
+3. Legacy shorthand:
+       {"prompt": "...", "sampling_params": {...}, "stream": false}
+       {"messages": [...], "sampling_params": {...}, "stream": true}
+
+When the request body has "stream": true, raw SSE chunks are yielded as they
+arrive; otherwise the parsed JSON response is yielded once.
+"""
+
+import logging
+import os
+from typing import Any, AsyncGenerator, Optional, Tuple
+
+import aiohttp
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+
+VLLM_PORT = os.getenv("VLLM_PORT", "8000")
+VLLM_BASE_URL = os.getenv("VLLM_BASE_URL", f"http://127.0.0.1:{VLLM_PORT}")
+REQUEST_TIMEOUT = float(os.getenv("REQUEST_TIMEOUT", "3600"))
+
+DEFAULT_CHAT_ROUTE = "/v1/chat/completions"
+DEFAULT_COMPLETION_ROUTE = "/v1/completions"
+
+# Set by main.py once the vLLM subprocess is running, so we can fail fast
+# instead of hanging on a dead server.
+vllm_process = None
+
+# Set by main.py when vLLM failed to start for a reason a restart cannot fix
+# (see startup_errors.py). Every job then answers with the cause instead of
+# dialling a server that is not there.
+startup_error: Optional[str] = None
+
+_default_model_cache: Optional[str] = None
+
+
+def _is_vllm_alive() -> bool:
+    return vllm_process is None or vllm_process.poll() is None
+
+
+async def _default_model(session: aiohttp.ClientSession) -> Optional[str]:
+    """Best-effort model id for legacy shorthand requests that omit "model"."""
+    global _default_model_cache
+    if _default_model_cache is not None:
+        return _default_model_cache
+
+    served = os.getenv("SERVED_MODEL_NAME") or os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE")
+    if served:
+        _default_model_cache = served.split(",")[0].strip()
+        return _default_model_cache
+
+    try:
+        async with session.get(f"{VLLM_BASE_URL}/v1/models") as resp:
+            data = await resp.json(content_type=None)
+            model_id = (data.get("data") or [{}])[0].get("id")
+            if model_id:
+                _default_model_cache = model_id
+                return model_id
+    except Exception as e:
+        logging.warning("Could not resolve default model from /v1/models: %s", e)
+
+    return os.getenv("MODEL_NAME")
+
+
+def _normalize_job_input(job_input: dict) -> Tuple[str, str, Optional[dict]]:
+    """Return (route, method, body) for any accepted job input shape."""
+    if job_input.get("openai_input"):
+        return job_input.get("openai_route") or DEFAULT_CHAT_ROUTE, "POST", job_input["openai_input"]
+
+    if job_input.get("openai_route"):
+        # Bare route without a body (e.g. /v1/models) is proxied as a GET.
+        return job_input["openai_route"], "GET", None
+
+    if job_input.get("route"):
+        body = job_input.get("body")
+        # The verb defaults to the payload: bodies POST, body-less requests GET
+        # so read-only routes (/v1/models, /health) don't need "method" spelled out.
+        method = (job_input.get("method") or ("POST" if body else "GET")).upper()
+        return job_input["route"], method, body
+
+    # Legacy shorthand: prompt/messages + sampling_params at the top level.
+    messages = job_input.get("messages")
+    prompt = job_input.get("prompt")
+    if messages is None and prompt is None:
+        raise ValueError(
+            "Job input must contain one of: openai_input (+openai_route), "
+            "route (+body), or prompt/messages."
+        )
+
+    sampling_params = dict(job_input.get("sampling_params") or {})
+    body = {**sampling_params, "stream": bool(job_input.get("stream", False))}
+    if messages is not None:
+        body["messages"] = messages
+        return DEFAULT_CHAT_ROUTE, "POST", body
+    body["prompt"] = prompt
+    return DEFAULT_COMPLETION_ROUTE, "POST", body
+
+
+def _error(message: str, error_type: str = "worker_error") -> dict:
+    return {"error": {"message": message, "type": error_type, "code": None}}
+
+
+async def handler(job: dict) -> AsyncGenerator[Any, None]:
+    if startup_error:
+        yield _error(startup_error, error_type="startup_error")
+        return
+
+    job_input = job.get("input") or {}
+
+    try:
+        route, method, body = _normalize_job_input(job_input)
+    except ValueError as e:
+        yield _error(str(e))
+        return
+
+    if not _is_vllm_alive():
+        yield _error("vLLM server process is not running; worker is unhealthy")
+        return
+
+    headers = {"Content-Type": "application/json"}
+
+    if method != "GET" and body is not None and "model" not in body:
+        timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT)
+        try:
+            async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
+                model = await _default_model(session)
+                if model:
+                    body = {**body, "model": model}
+        except Exception:
+            pass
+
+    timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT)
+    try:
+        async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
+            async with session.request(method, f"{VLLM_BASE_URL}{route}", json=body) as resp:
+                if resp.status >= 400:
+                    detail = await resp.text()
+                    logging.error("vLLM %s %s returned HTTP %s: %s", method, route, resp.status, detail)
+                    yield _error(f"vLLM returned HTTP {resp.status}: {detail}")
+                    return
+
+                wants_stream = isinstance(body, dict) and body.get("stream") is True
+                if wants_stream:
+                    async for chunk in resp.content.iter_any():
+                        yield chunk.decode("utf-8", errors="replace")
+                else:
+                    yield await resp.json(content_type=None)
+    except aiohttp.ClientError as e:
+        logging.exception("Request to vLLM failed")
+        yield _error(f"Request to vLLM failed: {e}")