cachebackend-cpu-override.yaml

# SPDX-FileCopyrightText: 2026 The inference-cache Authors
#
# SPDX-License-Identifier: Apache-2.0

# Typed vLLM PodLocal LMCache MP example that uses
# spec.integration.engineOverrides to amend the inference-owner's engine
# container. The manifest is admission-valid without a GPU; actual engine
# startup remains the authoritative connector/package compatibility check.
#
# The override surface is engine-agnostic K8s vocabulary (args + env), so it
# also extends to future runtime adapters and remote bindings with no CRD churn.
#
# Admission HARD-REJECTS overrides that overlap the typed MP adapter's reserved
# args/env: `--kv-transfer-config`, `--disable-hybrid-kv-cache-manager`,
# `PYTHONHASHSEED`, and `INFERENCECACHE_FAIL_OPEN`.
apiVersion: inferencecache.io/v1alpha1
kind: CacheBackend
metadata:
  labels:
    app.kubernetes.io/name: inference-cache
  name: cachebackend-cpu-override
spec:
  runtime: VLLM
  type: LMCache
  integration:
    role: ReadWrite
    # Per-engine overrides applied AFTER the runtime adapter produces its
    # canonical injection. Reserved entries (above) cannot be overridden or
    # suppressed; admission rejects with a field-scoped error that names the
    # offending flag/env and the adapter.
    engineOverrides:
      # Add a CPU-friendly --max-model-len that the canonical injection does
      # not set. The override surface always appends (not replaces) for
      # flags the adapter does not declare itself.
      args:
        - "--max-model-len"
        - "8192"
      env:
        # Free-form engine env appended after the typed MP wire.
        - name: FOO
          value: bar
  engineSelector:
    matchLabels:
      inferencecache.io/cache-domain: vllm-cpu-override
  lmCache:
    topology: PodLocal
    chunkSizeTokens: 512
    podLocal:
      server:
        image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13
        port: 5555
        l1Capacity: 4Gi
        maxWorkers: 4
        resources:
          requests:
            cpu: "1"
            memory: 5Gi
          limits:
            cpu: "2"
            memory: 6Gi
  observation:
    # Served model identifier the matched engine pods are loaded with.
    modelID: Qwen/Qwen2.5-0.5B-Instruct