From ca205647b94642d39f54a9ea81baec10477239d0 Mon Sep 17 00:00:00 2001 From: enjoyandlove Date: Sun, 17 May 2026 01:29:57 -0400 Subject: [PATCH 1/4] deploy: pin genie-* services to dedicated CPU cores for voice latency (#25) --- deploy/systemd/genie-api.service | 4 +++ deploy/systemd/genie-core.service | 9 ++++++ deploy/systemd/genie-governor.service | 4 +++ deploy/systemd/genie-health.service | 4 +++ deploy/systemd/genie-llm.service | 8 +++++ deploy/systemd/genie-mqtt.service | 4 +++ deploy/systemd/genie-wakeword.service | 8 +++-- deploy/systemd/genie-whisper.service | 6 ++++ doc/deployment-and-ops.md | 42 +++++++++++++++++++++++++++ 9 files changed, 87 insertions(+), 2 deletions(-) diff --git a/deploy/systemd/genie-api.service b/deploy/systemd/genie-api.service index f8f9475a..8f3b2069 100644 --- a/deploy/systemd/genie-api.service +++ b/deploy/systemd/genie-api.service @@ -12,6 +12,10 @@ Restart=always RestartSec=5 MemoryMax=32M +# Pin to the system bucket (issue #25) so the dashboard's HTTP loop doesn't +# steal cycles from whisper (2-3), llama-server (4), or genie-core (5). +CPUAffinity=0 1 + ProtectSystem=strict ProtectHome=yes ReadWritePaths=/opt/geniepod/data diff --git a/deploy/systemd/genie-core.service b/deploy/systemd/genie-core.service index ea40a63d..2a4f2e1a 100644 --- a/deploy/systemd/genie-core.service +++ b/deploy/systemd/genie-core.service @@ -15,6 +15,15 @@ StandardInput=null Restart=always RestartSec=3 +# CPU pinning for voice latency stability (issue #25). Core 5 hosts the +# orchestrator and every audio child it spawns (piper, sox, deep-filter, +# arecord, aplay) — they inherit this affinity. Keeping the TTS pipeline on +# one core shields whisper (2-3) and llama-server (4) from preemption during +# decode. arecord runs on core 5 by inheritance (not 0-1 as the issue's +# system-bucket bullet suggests); revisit with per-spawn taskset only if +# capture jitter shows up under load. +CPUAffinity=5 + # Needs GPU (Whisper TRT, Piper TTS), audio (ALSA/I2S), and network. ProtectSystem=no SupplementaryGroups=video render audio diff --git a/deploy/systemd/genie-governor.service b/deploy/systemd/genie-governor.service index 1a985c7b..cb6515b7 100644 --- a/deploy/systemd/genie-governor.service +++ b/deploy/systemd/genie-governor.service @@ -15,6 +15,10 @@ WatchdogSec=30 MemoryMax=32M CPUWeight=50 +# Pin to the system bucket (issue #25) — governor only polls memory and writes +# drop-in configs, keep it off the inference cores (2-5). +CPUAffinity=0 1 + # Security hardening ProtectSystem=strict ProtectHome=yes diff --git a/deploy/systemd/genie-health.service b/deploy/systemd/genie-health.service index eefa44bb..c6ace954 100644 --- a/deploy/systemd/genie-health.service +++ b/deploy/systemd/genie-health.service @@ -14,6 +14,10 @@ WatchdogSec=60 MemoryMax=32M CPUWeight=20 +# Pin to the system bucket (issue #25) — health monitor is a periodic poller, +# keep it off the inference cores (2-5). +CPUAffinity=0 1 + # Security hardening ProtectSystem=strict ProtectHome=yes diff --git a/deploy/systemd/genie-llm.service b/deploy/systemd/genie-llm.service index 3caef2b0..a22af468 100644 --- a/deploy/systemd/genie-llm.service +++ b/deploy/systemd/genie-llm.service @@ -32,6 +32,14 @@ Restart=on-failure RestartSec=5 TimeoutStartSec=120 +# CPU pinning for voice latency stability (issue #25). Inference is GPU-bound +# (--n-gpu-layers 999), so one dedicated core is enough to host CUDA dispatch +# + sampler + tokenizer without contending with whisper (2-3) or genie-core +# (5). --threads=4 above stays as-is for now; with a single-core affinity the +# four worker threads time-share core 4 — fine for the GPU-bound critical path +# but worth revisiting (--threads 1 or 2) if prompt-eval shows scheduler churn. +CPUAffinity=4 + # GPU needs full system access ProtectSystem=no SupplementaryGroups=video render diff --git a/deploy/systemd/genie-mqtt.service b/deploy/systemd/genie-mqtt.service index bbfd5896..85e0fe0a 100644 --- a/deploy/systemd/genie-mqtt.service +++ b/deploy/systemd/genie-mqtt.service @@ -9,6 +9,10 @@ Restart=always RestartSec=3 MemoryMax=16M +# Pin to the system bucket (issue #25) — broker traffic is light and event- +# driven, keep it off the inference cores (2-5). +CPUAffinity=0 1 + ProtectSystem=strict ProtectHome=yes ReadWritePaths=/opt/geniepod/data/mqtt diff --git a/deploy/systemd/genie-wakeword.service b/deploy/systemd/genie-wakeword.service index 858e2071..15e4d334 100644 --- a/deploy/systemd/genie-wakeword.service +++ b/deploy/systemd/genie-wakeword.service @@ -10,8 +10,12 @@ ExecStart=/usr/bin/env python3 /opt/geniepod/bin/genie-wakeword.py Restart=always RestartSec=1 -# Pin to isolated CPU cores for real-time audio processing. -CPUAffinity=4 5 +# Pin to audio/system cores for real-time wake-word detection (issue #25). +# Cores 4 and 5 are now reserved for llama-server and genie-core respectively; +# the wake-word listener belongs alongside ALSA + capture on the system bucket. +# SCHED_FIFO at priority 50 is kept so the continuous audio loop isn't +# preempted by best-effort system tasks sharing these cores. +CPUAffinity=0 1 CPUSchedulingPolicy=fifo CPUSchedulingPriority=50 diff --git a/deploy/systemd/genie-whisper.service b/deploy/systemd/genie-whisper.service index 2e64f81f..2d008c51 100644 --- a/deploy/systemd/genie-whisper.service +++ b/deploy/systemd/genie-whisper.service @@ -22,6 +22,12 @@ Restart=on-failure RestartSec=5 TimeoutStartSec=120 +# CPU pinning for voice latency stability (issue #25). Dedicate two cores to +# whisper-server so STT decode does not contend with llama-server (core 4) or +# the audio capture / TTS pipeline (cores 0-1 and 5). --threads=2 matches the +# affinity so each decode thread gets its own core. +CPUAffinity=2 3 + # GPU needs full system access (same posture as genie-llm). ProtectSystem=no SupplementaryGroups=video render diff --git a/doc/deployment-and-ops.md b/doc/deployment-and-ops.md index 0a623249..de233ed7 100644 --- a/doc/deployment-and-ops.md +++ b/doc/deployment-and-ops.md @@ -120,6 +120,48 @@ Operator decisions before enabling stricter policy: - Use `[core.tool_policy]` allowlists/denylists per channel when a surface should be less capable than local dashboard/API. - Keep `unknown` out of physical actuation origins unless there is a controlled reason to allow it. +## CPU Pinning (Voice Latency Stability) + +The Jetson Orin Nano has six CPU cores. The voice path (wake → STT → LLM → TTS +→ playback) is sensitive to scheduler jitter when multiple inference servers +and audio subprocesses share cores. Each `genie-*` systemd unit ships with a +`CPUAffinity=` directive that partitions the six cores into four buckets +(issue #25): + +| Cores | Workload | +| --- | --- | +| 0–1 | Kernel, ALSA, MQTT broker, `genie-api`, `genie-governor`, `genie-health`, `genie-wakeword` | +| 2–3 | `whisper-server` (STT decode, two threads) | +| 4 | `llama-server` (GPU-bound; one core hosts CUDA dispatch + sampler) | +| 5 | `genie-core` and all audio children it spawns (`piper`, `sox`, `deep-filter`, `arecord`, `aplay`) | + +`genie-wakeword` retains `SCHED_FIFO` at priority 50 so the continuous audio +loop is not preempted by best-effort work sharing cores 0–1. + +Verify pinning after a deploy / restart: + +```bash +# Per-service: confirm the unit and its children are on the expected cores. +for svc in genie-core genie-llm genie-whisper genie-wakeword genie-api \ + genie-governor genie-health genie-mqtt; do + pid=$(systemctl show -p MainPID --value "${svc}.service") + [ "$pid" != "0" ] && printf "%-18s PID=%s affinity=%s\n" \ + "$svc" "$pid" "$(taskset -pc "$pid" | awk -F': ' '{print $2}')" +done + +# All threads of one service (useful for whisper / llama with multi-threading): +ps -L -o pid,tid,psr,comm -p "$(pidof whisper-server)" + +# Live core distribution while a voice cycle runs (Jetson-specific): +sudo tegrastats --interval 250 +``` + +Acceptance signal (issue #25): ten consecutive voice cycles should hold STT +latency within ±100 ms of the median once warmup has completed. If variance +persists after Option 1, the next step is kernel-level `isolcpus=2,3,4,5` on +the bootloader command line — that is intentionally out of scope here because +it requires a Jetson reflash / extlinux.conf edit. + ## Runtime Data And State Default production data location: From 360c7b3c88af6e96b68058f55a43447d8bf8e721 Mon Sep 17 00:00:00 2001 From: enjoyandlove Date: Mon, 18 May 2026 05:31:07 -0400 Subject: [PATCH 2/4] docs: update deployment guide for configurable LLM backend and genie-ai-runtime service --- deploy/systemd/genie-ai-runtime.service | 43 +++++++++++++++++++++++++ doc/deployment-and-ops.md | 4 +-- 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 deploy/systemd/genie-ai-runtime.service diff --git a/deploy/systemd/genie-ai-runtime.service b/deploy/systemd/genie-ai-runtime.service new file mode 100644 index 00000000..a19632e7 --- /dev/null +++ b/deploy/systemd/genie-ai-runtime.service @@ -0,0 +1,43 @@ +[Unit] +Description=GeniePod LLM Server (genie-ai-runtime / jllm-server) +Documentation=https://github.com/GeniePod/genie-ai-runtime +Documentation=https://github.com/GeniePod/genie-claw/issues/27 +After=network.target +ConditionPathExists=/opt/geniepod/bin/jllm-server + +[Service] +Type=simple +# Drop page cache before loading model — Jetson NvMap needs contiguous blocks. +ExecStartPre=/bin/sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +ExecStart=/opt/geniepod/bin/jllm-server \ + --model ${GENIEPOD_LLM_MODEL} \ + --host 0.0.0.0 \ + --port 8080 \ + --ctx-size 2048 \ + --n-gpu-layers 999 \ + --threads 4 \ + --parallel 1 +# The jllm-server flag surface intentionally mirrors llama-server so the +# OpenAI-compatible client in genie-core works against either backend without +# code changes. Issue #27 tracks the opt-in rollout; flip +# [services.llm].backend = "genie_ai_runtime" in geniepod.toml after +# installing /opt/geniepod/bin/jllm-server. + +Environment=GENIEPOD_LLM_MODEL=/opt/geniepod/models/phi-4-mini-instruct-q4_k_m.gguf +Restart=on-failure +RestartSec=5 +TimeoutStartSec=120 + +# CPU pinning for voice latency stability (issue #25). Inference is GPU-bound +# (--n-gpu-layers 999), so one dedicated core is enough to host CUDA dispatch +# + sampler + tokenizer without contending with whisper (2-3) or genie-core +# (5). --threads=4 above stays as-is; with a single-core affinity the four +# worker threads time-share core 4 — fine for the GPU-bound critical path. +CPUAffinity=4 + +# GPU needs full system access +ProtectSystem=no +SupplementaryGroups=video render + +[Install] +WantedBy=geniepod.target diff --git a/doc/deployment-and-ops.md b/doc/deployment-and-ops.md index de233ed7..d0558744 100644 --- a/doc/deployment-and-ops.md +++ b/doc/deployment-and-ops.md @@ -132,7 +132,7 @@ and audio subprocesses share cores. Each `genie-*` systemd unit ships with a | --- | --- | | 0–1 | Kernel, ALSA, MQTT broker, `genie-api`, `genie-governor`, `genie-health`, `genie-wakeword` | | 2–3 | `whisper-server` (STT decode, two threads) | -| 4 | `llama-server` (GPU-bound; one core hosts CUDA dispatch + sampler) | +| 4 | `llama-server` / `jllm-server` (GPU-bound; one core hosts CUDA dispatch + sampler — whichever LLM backend is active) | | 5 | `genie-core` and all audio children it spawns (`piper`, `sox`, `deep-filter`, `arecord`, `aplay`) | `genie-wakeword` retains `SCHED_FIFO` at priority 50 so the continuous audio @@ -142,7 +142,7 @@ Verify pinning after a deploy / restart: ```bash # Per-service: confirm the unit and its children are on the expected cores. -for svc in genie-core genie-llm genie-whisper genie-wakeword genie-api \ +for svc in genie-core genie-llm genie-ai-runtime genie-whisper genie-wakeword genie-api \ genie-governor genie-health genie-mqtt; do pid=$(systemctl show -p MainPID --value "${svc}.service") [ "$pid" != "0" ] && printf "%-18s PID=%s affinity=%s\n" \ From 7ccf14d92af4a95ffaaadc0b1ed5ca7459a67c89 Mon Sep 17 00:00:00 2001 From: enjoyandlove Date: Mon, 18 May 2026 09:09:00 -0400 Subject: [PATCH 3/4] perf(deploy): pin genie-ai-runtime to CPU core 4 for voice latency stability (#25) --- deploy/systemd/genie-ai-runtime.service | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/systemd/genie-ai-runtime.service b/deploy/systemd/genie-ai-runtime.service index b3c36915..be84c90a 100644 --- a/deploy/systemd/genie-ai-runtime.service +++ b/deploy/systemd/genie-ai-runtime.service @@ -31,6 +31,13 @@ Restart=on-failure RestartSec=5 TimeoutStartSec=120 +# CPU pinning for voice latency stability (issue #25). Inference is GPU-bound +# (jetson-llm-server offloads to Jetson iGPU), so one dedicated core is enough +# to host CUDA dispatch + sampler + tokenizer without contending with whisper +# (2-3) or genie-core (5). Mirrors genie-llm.service; only one of the two units +# runs at a time (Conflicts=genie-llm.service above), so both can claim core 4. +CPUAffinity=4 + # GPU needs full system access ProtectSystem=no SupplementaryGroups=video render From 5920e310772c64e92bdc6e87f25e268ff6e0d359 Mon Sep 17 00:00:00 2001 From: enjoyandlove Date: Mon, 18 May 2026 09:57:23 -0400 Subject: [PATCH 4/4] docs(deploy): correct jllm-server to jetson-llm-server in core-affinity table --- doc/deployment-and-ops.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/deployment-and-ops.md b/doc/deployment-and-ops.md index d0558744..0f836a46 100644 --- a/doc/deployment-and-ops.md +++ b/doc/deployment-and-ops.md @@ -132,7 +132,7 @@ and audio subprocesses share cores. Each `genie-*` systemd unit ships with a | --- | --- | | 0–1 | Kernel, ALSA, MQTT broker, `genie-api`, `genie-governor`, `genie-health`, `genie-wakeword` | | 2–3 | `whisper-server` (STT decode, two threads) | -| 4 | `llama-server` / `jllm-server` (GPU-bound; one core hosts CUDA dispatch + sampler — whichever LLM backend is active) | +| 4 | `llama-server` / `jetson-llm-server` (GPU-bound; one core hosts CUDA dispatch + sampler — whichever LLM backend is active) | | 5 | `genie-core` and all audio children it spawns (`piper`, `sox`, `deep-filter`, `arecord`, `aplay`) | `genie-wakeword` retains `SCHED_FIFO` at priority 50 so the continuous audio