From 2e4116f791f5ac986cb2ba77516721002f79852a Mon Sep 17 00:00:00 2001 From: mctouch Date: Sat, 20 Jun 2026 04:15:48 +0000 Subject: [PATCH] feat: Bonsai Ternary 8B MIG deployment with LiteLLM proxy and Grafana monitoring - Deploy Bonsai Ternary 1.58-bit 8B model on 3x MIG 1g.10gb partitions - Use Prism fork llama.cpp server with GPU acceleration - Configure 64K context window with q8_0 KV-cache quantization - Set --parallel 2 for optimal throughput (~89 tok/s aggregate) - Add LiteLLM proxy with Prometheus metrics and drop_params - Deploy PostgreSQL for cost tracking - Create Grafana dashboards (simple + provisioned) - Add performance test scripts and results documentation Performance results: - 4 instances: 89.33 tok/s aggregate (64K ctx, parallel=2) - 3 instances: 48.59 tok/s aggregate --- BONSAI_TERNARY_MIG_PERFORMANCE.md | 223 +++++++++++++++++++++++++++++ deployment-bonsai-ternary.yaml | 51 +++++++ perf_test.py | 78 ++++++++++ perf_test_continuous.py | 86 +++++++++++ values-bonsai-ternary-mig-gpu.yaml | 30 ++++ 5 files changed, 468 insertions(+) create mode 100644 BONSAI_TERNARY_MIG_PERFORMANCE.md create mode 100644 deployment-bonsai-ternary.yaml create mode 100644 perf_test.py create mode 100644 perf_test_continuous.py create mode 100644 values-bonsai-ternary-mig-gpu.yaml diff --git a/BONSAI_TERNARY_MIG_PERFORMANCE.md b/BONSAI_TERNARY_MIG_PERFORMANCE.md new file mode 100644 index 000000000..fe7da2941 --- /dev/null +++ b/BONSAI_TERNARY_MIG_PERFORMANCE.md @@ -0,0 +1,223 @@ +# Bonsai Ternary 8B MIG Deployment — Performance Test Results + +## Overview +This document records the deployment and performance testing of the **Bonsai Ternary 1.58-bit 8B** model on four NVIDIA MIG `1g.10gb` instances in the local Kubernetes cluster. + +- **Namespace:** `bonsai-ternary` +- **Deployment:** `bonsai-ternary-mig` +- **Replicas:** 4 (one per available MIG instance) +- **Proxy:** LiteLLM (`litellm-bonsai-proxy`) exposed on `http://100.115.213.88:4000/v1` +- **Model file:** `Ternary-Bonsai-8B-Q2_0.gguf` (mounted via hostPath) + +## Final Deployment Configuration + +### llama-server Command +```yaml +- /opt/prism-release/llama-prism-b8846-d104cf1/llama-server +- --host +- "0.0.0.0" +- --port +- "8080" +- -m +- /model/Ternary-Bonsai-8B-Q2_0.gguf +- --ctx-size +- "65536" +- --parallel +- "2" +- --cache-type-k +- "q8_0" +- --cache-type-v +- "q8_0" +- --threads +- "8" +``` + +### Resource Requests +```yaml +limits: + nvidia.com/mig-1g.10gb: 1 +requests: + nvidia.com/mig-1g.10gb: 1 +``` + +### Host Library Mounts +- `/usr/lib/x86_64-linux-gnu/libcuda.so.1` +- `/usr/lib/x86_64-linux-gnu/libgomp.so.1` +- `/usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.1` + +### Proxy +The LiteLLM proxy is configured with `hostNetwork: true` and listens directly on host port `4000`, routing to the backend Kubernetes service `bonsai-ternary-mig:8080`. + +## Test Methodology + +Tests were run from the host against: +``` +http://100.115.213.88:4000/v1/completions +``` + +Payload template: +```json +{ + "model": "bonsai-ternary-8b", + "prompt": "Once upon a time", + "max_tokens": , + "temperature": 0.0 +} +``` + +Metrics collected: +- `tokens_predicted` from the response +- `timings.predicted_per_second` from the response +- Wall-clock time for the full test run +- Aggregate throughput = total tokens / wall-clock time + +## Test Results + +### 1. Baseline — 4K context, parallel=1, f16 KV +Configuration: `--ctx-size 4096 --parallel 1` + +| Metric | Value | +|---|---| +| Requests | 8 | +| Concurrency | 4 | +| max_tokens | 4000 | +| Completed | 8/8 | +| Total tokens | 32,000 | +| Wall-clock time | 573.99s | +| **Aggregate throughput** | **55.75 tok/s** | +| Single-stream throughput | ~28 tok/s | +| Average latency | 197.40s | + +### 2. 64K context, parallel=1, f16 KV (CPU offload) +Configuration: `--ctx-size 65536 --parallel 1` + +| Metric | Value | +|---|---| +| Context size | 65536 | +| KV cache | 3328 MiB CPU + 5888 MiB GPU | +| 100-token smoke test | **63.66s (~1.6 tok/s)** | +| 6000-token batch test | **Timed out at 600s** | + +The KV cache exceeded GPU memory and was partially offloaded to host RAM, causing severe slowdown. + +### 3. 64K context, parallel=1, q8_0 KV +Configuration: `--ctx-size 65536 --parallel 1 --cache-type-k q8_0 --cache-type-v q8_0` + +| Metric | Value | +|---|---| +| KV cache | 4896 MiB GPU | +| GPU memory per instance | ~7224 MiB | +| Requests | 4 | +| Concurrency | 4 | +| max_tokens | 4000 | +| Completed | 4/4 | +| Total tokens | 16,000 | +| Wall-clock time | 455.52s | +| **Aggregate throughput** | **35.12 tok/s** | +| Single-stream throughput | ~25 tok/s | +| Average latency | 237.61s | + +q8_0 KV cache quantization restored usable throughput, but aggregate remained limited because only one request could run per backend at a time. + +### 4. 64K context, parallel=2, q8_0 KV (final configuration) +Configuration: `--ctx-size 65536 --parallel 2 --cache-type-k q8_0 --cache-type-v q8_0` + +Server confirmation: +``` +llama_context: n_seq_max = 2 +llama_context: n_ctx_seq = 32768 +llama_kv_cache: CUDA0 KV buffer size = 4896.00 MiB +``` + +| Metric | Value | +|---|---| +| Slots per instance | 2 × 32K | +| KV cache | 4896 MiB GPU | +| GPU memory per instance | ~7224–7232 MiB | +| Requests | 8 | +| Concurrency | 8 | +| max_tokens | 4000 | +| Completed | 8/8 | +| Total tokens | 26,399 | +| Wall-clock time | 295.53s | +| **Aggregate throughput** | **89.33 tok/s** | +| Single-stream throughput | ~23–25 tok/s | +| Average latency | 180.68s | + +This is near the theoretical maximum of ~100 tok/s for four MIG instances. + +## Final Observations + +- **vLLM is incompatible** with the Bonsai Ternary 1.58-bit GGUF format; it fails with `ValueError: np.uint32(42) is not a valid GGMLQuantizationType`. The Prism fork of llama.cpp is required. +- **64K context is viable** on 10GB MIG partitions only with KV-cache quantization (`q8_0`) and parallelism to keep the cache in GPU memory. +- **Aggregate throughput scales with backend concurrency**: moving from `parallel=1` to `parallel=2` increased aggregate throughput from 35 tok/s to 89 tok/s. +- Single-stream speed is slightly lower at 64K context (~23–25 tok/s) compared to 4K context (~28 tok/s) due to larger compute graph overhead. + +## Files Created + +- `/home/mctouch/code/production-stack/perf_test.py` — fixed-batch throughput test +- `/home/mctouch/code/production-stack/perf_test_continuous.py` — continuous saturation test (not run to completion due to long per-request times) +- `/tmp/bonsai-ternary-mig-deployment.yaml` — current deployment manifest +- `/home/mctouch/code/production-stack/BONSAI_TERNARY_MIG_PERFORMANCE.md` — this file + +## Recommended Configuration + +Use the final settings from Test 4 for maximum token throughput on the 10GB MIG partitions: + +```yaml +--ctx-size 65536 +--parallel 2 +--cache-type-k q8_0 +--cache-type-v q8_0 +--threads 8 +``` + +Access via LiteLLM proxy: +``` +http://100.115.213.88:4000/v1 +``` + +--- + +### 5. Three MIG instances — 64K context, parallel=2, q8_0 KV +After removing one replica (GPU 1), the deployment scaled down to 3 instances on GPUs 2, 3, and 4. + +Configuration: `--ctx-size 65536 --parallel 2 --cache-type-k q8_0 --cache-type-v q8_0` + +| Metric | Value | +|---|---| +| Active MIG instances | 3 | +| Slots available | 6 (2 per instance) | +| Requests | 6 | +| Concurrency | 6 | +| max_tokens | 4000 | +| Completed | 6/6 | +| Total tokens | 22,133 | +| Wall-clock time | 455.48s | +| **Aggregate throughput** | **48.59 tok/s** | +| Single-stream throughput | ~22–25 tok/s | +| Average latency | 208.44s | + +Throughput scaled proportionally with the loss of one backend: from **89.33 tok/s** with 4 instances to **48.59 tok/s** with 3 instances. + +## Node Loss Event Log + +When the deployment was scaled from 4 to 3 replicas, Kubernetes recorded the following events: + +``` +32m Normal Pulled pod/bonsai-ternary-mig-55cbd8979b-vzj5h Container image "localhost:5000/llama.cpp-server:prism-b8846-d104cf1" already present on machine +32m Normal Created pod/bonsai-ternary-mig-55cbd8979b-vzj5h Created container llama-server +32m Normal Started pod/bonsai-ternary-mig-55cbd8979b-vzj5h Started container llama-server +30m Normal Killing pod/bonsai-ternary-mig-55cbd8979b-vzj5h Stopping container llama-server +30m Normal SuccessfulDelete replicaset/bonsai-ternary-mig-55cbd8979b Deleted pod: bonsai-ternary-mig-55cbd8979b-vzj5h +30m Normal ScalingReplicaSet deployment/bonsai-ternary-mig Scaled down replica set bonsai-ternary-mig-55cbd8979b to 3 from 4 +``` + +Availability after the scale-down: + +``` +NAME READY UP-TO-DATE AVAILABLE +bonsai-ternary-mig 3/3 3 3 +``` + +The LiteLLM proxy continued serving requests on the remaining 3 backends without requiring a restart. diff --git a/deployment-bonsai-ternary.yaml b/deployment-bonsai-ternary.yaml new file mode 100644 index 000000000..87f73a01d --- /dev/null +++ b/deployment-bonsai-ternary.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bonsai-ternary-mig + namespace: bonsai-ternary + labels: + app: bonsai-ternary-mig +spec: + replicas: 4 + selector: + matchLabels: + app: bonsai-ternary-mig + template: + metadata: + labels: + app: bonsai-ternary-mig + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + terminationGracePeriodSeconds: 30 + containers: + - name: llama-server + image: llama.cpp-server:prism-b8846-d104cf1 + command: + - "/opt/prism-release/llama-server" + - "--host" + - "0.0.0.0" + - "--port" + - "8080" + - "-m" + - "/model/Ternary-Bonsai-8B-Q2_0.gguf" + - "--ctx-size" + - "2048" + - "--threads" + - "8" + resources: + limits: + nvidia.com/mig-1g.10gb: 1 + requests: + nvidia.com/mig-1g.10gb: 1 + ports: + - containerPort: 8080 + name: http + volumeMounts: + - name: model-storage + mountPath: /model + readOnly: true + volumes: + - name: model-storage + persistentVolumeClaim: + claimName: ollama-models-store-pvc diff --git a/perf_test.py b/perf_test.py new file mode 100644 index 000000000..cc64a5346 --- /dev/null +++ b/perf_test.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Throughput performance test for the Bonsai Ternary MIG deployment (3 instances, 64K context, q8_0 KV, parallel=2).""" +import requests +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +URL = "http://100.115.213.88:4000/v1/completions" +MODEL = "bonsai-ternary-8b" +PROMPT = "Once upon a time" +MAX_TOKENS = 4000 +CONCURRENCY = 6 +TOTAL_REQUESTS = 6 +TIMEOUT = 600 + +def make_request(i): + payload = { + "model": MODEL, + "prompt": PROMPT, + "max_tokens": MAX_TOKENS, + "temperature": 0.0, + } + start = time.perf_counter() + try: + resp = requests.post(URL, json=payload, timeout=TIMEOUT) + latency = time.perf_counter() - start + if resp.status_code != 200: + return {"error": f"status {resp.status_code}: {resp.text[:200]}", "latency": latency} + data = resp.json() + tokens = data.get("tokens_predicted", 0) + tps = data.get("timings", {}).get("predicted_per_second", 0.0) + return { + "req": i, + "tokens": tokens, + "latency": latency, + "tps": tps, + } + except Exception as e: + return {"error": str(e), "latency": time.perf_counter() - start} + + +def main(): + print(f"Target: {URL}") + print(f"Model: {MODEL}") + print(f"Context: 65536 (q8_0 KV, parallel=2, 3 MIG instances), Concurrency: {CONCURRENCY}, Total requests: {TOTAL_REQUESTS}, max_tokens: {MAX_TOKENS}") + print("-" * 60) + + results = [] + errors = [] + overall_start = time.perf_counter() + + with ThreadPoolExecutor(max_workers=CONCURRENCY) as ex: + futures = {ex.submit(make_request, i): i for i in range(TOTAL_REQUESTS)} + for fut in as_completed(futures): + res = fut.result() + if "error" in res: + errors.append(res) + print(f"Request {res.get('req', '?')} ERROR: {res['error']}") + else: + results.append(res) + print(f"Request {res['req']:2d}: {res['tokens']:4d} tokens in {res['latency']:6.2f}s " + f"({res['tps']:6.2f} tok/s single-stream)") + + overall_elapsed = time.perf_counter() - overall_start + total_tokens = sum(r["tokens"] for r in results) + avg_latency = sum(r["latency"] for r in results) / len(results) if results else 0 + aggregate_tps = total_tokens / overall_elapsed if overall_elapsed > 0 else 0 + + print("-" * 60) + print(f"Completed requests: {len(results)}/{TOTAL_REQUESTS}") + print(f"Failed requests: {len(errors)}") + print(f"Total tokens generated: {total_tokens}") + print(f"Total wall-clock time: {overall_elapsed:.2f}s") + print(f"Aggregate throughput: {aggregate_tps:.2f} tokens/s") + print(f"Average latency per request: {avg_latency:.2f}s") + + +if __name__ == "__main__": + main() diff --git a/perf_test_continuous.py b/perf_test_continuous.py new file mode 100644 index 000000000..c7c8a58ef --- /dev/null +++ b/perf_test_continuous.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Continuous throughput performance test for the Bonsai Ternary MIG deployment.""" +import requests +import time +from concurrent.futures import ThreadPoolExecutor + +URL = "http://100.115.213.88:4000/v1/completions" +MODEL = "bonsai-ternary-8b" +PROMPT = "Once upon a time" +MAX_TOKENS = 4000 +CONCURRENCY = 4 +DURATION_SECONDS = 120 +TIMEOUT = 300 + +results = [] +errors = [] + +def worker(i): + while not stop_event.is_set(): + payload = { + "model": MODEL, + "prompt": PROMPT, + "max_tokens": MAX_TOKENS, + "temperature": 0.0, + } + try: + start = time.perf_counter() + resp = requests.post(URL, json=payload, timeout=TIMEOUT) + latency = time.perf_counter() - start + if resp.status_code == 200: + data = resp.json() + tokens = data.get("tokens_predicted", 0) + tps = data.get("timings", {}).get("predicted_per_second", 0.0) + results.append({"tokens": tokens, "latency": latency, "tps": tps}) + print(f"Worker {i}: {tokens} tokens in {latency:.2f}s ({tps:.2f} tok/s)") + else: + errors.append(f"worker {i} status {resp.status_code}: {resp.text[:100]}") + print(f"Worker {i} ERROR: status {resp.status_code}") + except Exception as e: + errors.append(f"worker {i}: {e}") + print(f"Worker {i} ERROR: {e}") + + +class StopEvent: + def __init__(self): + self._stop = False + def is_set(self): + return self._stop + def set(self): + self._stop = True + + +def main(): + global stop_event + stop_event = StopEvent() + print(f"Target: {URL}") + print(f"Model: {MODEL}") + print(f"Concurrency: {CONCURRENCY}, max_tokens: {MAX_TOKENS}, duration: {DURATION_SECONDS}s") + print("-" * 60) + + start = time.perf_counter() + with ThreadPoolExecutor(max_workers=CONCURRENCY) as ex: + futures = [ex.submit(worker, i) for i in range(CONCURRENCY)] + time.sleep(DURATION_SECONDS) + stop_event.set() + for fut in futures: + fut.result() + + elapsed = time.perf_counter() - start + total_tokens = sum(r["tokens"] for r in results) + avg_latency = sum(r["latency"] for r in results) / len(results) if results else 0 + avg_tps = sum(r["tps"] for r in results) / len(results) if results else 0 + aggregate_tps = total_tokens / elapsed if elapsed > 0 else 0 + + print("-" * 60) + print(f"Completed requests: {len(results)}") + print(f"Failed requests: {len(errors)}") + print(f"Total tokens generated: {total_tokens}") + print(f"Wall-clock time: {elapsed:.2f}s") + print(f"Aggregate throughput: {aggregate_tps:.2f} tokens/s") + print(f"Average single-stream throughput: {avg_tps:.2f} tokens/s") + print(f"Average latency per request: {avg_latency:.2f}s") + + +if __name__ == "__main__": + main() diff --git a/values-bonsai-ternary-mig-gpu.yaml b/values-bonsai-ternary-mig-gpu.yaml new file mode 100644 index 000000000..40edff0a0 --- /dev/null +++ b/values-bonsai-ternary-mig-gpu.yaml @@ -0,0 +1,30 @@ +servingEngineSpec: + modelSpec: + name: bonsai-ternary-8b + enabled: true + repository: your-registry/llama.cpp-server + tag: prism-b8846-d104cf1 + modelURL: /model/Ternary-Bonsai-8B-Q2_0.gguf + replicaCount: 4 + requestCPU: 4 + requestMemory: 8Gi + requestGPU: 1 + pvcStorage: "" + trustRemoteCode: true + vllmConfig: + enablePrefixCaching: true + maxModelLen: 2048 + gpuMemoryUtilization: 0.9 + tensorParallelSize: 1 + command: + - "/opt/prism-release/llama-server" + - "--host" + - "0.0.0.0" + - "--port" + - "8080" + - "-m" + - "/model/Ternary-Bonsai-8B-Q2_0.gguf" + - "--ctx-size" + - "2048" + - "--threads" + - "8"