feat: optimize ONNX peak RSS with mmap'd external-data weights

.github/workflows/ci.yml

@@ -7,6 +7,28 @@ on:
     branches: [main]
 
 jobs:
+  # The model publishing helper is independent of the C++ build. Exercise it
+  # on Linux and Windows because external-data path and rename behavior differs.
+  onnx-repacker-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install ONNX tooling dependencies
+        run: python -m pip install -r scripts/requirements-onnx-tools.txt
+
+      - name: Test external-data repacker
+        run: python -m unittest discover -s tests -p "test_export_external_data.py" -v
+
   # Fast lane: orchestration-only build, no ML deps. Catches anything that
   # breaks the core library on macOS, Linux, or Windows.
   unit-tests:

AGENTS.md

@@ -127,6 +127,7 @@ Known: `test_pipeline_e2e` is intermittently flaky (SIGTRAP under load) — not
 ## Guidelines
 
 - **TTS delivery modes and post-processing.** `TTSInterface::synthesize()` and standalone C `sc_*_synthesize()` are the streaming APIs and are not deprecated. `TTSInterface::synthesize_with_options()` is interface-wide via the base default; models override it only when they need custom buffering around internal decoder chunks (currently VoxCPM/VoxCPM2). Standalone C `*_synthesize_with_options()` exists only for the direct TTS wrappers, not the main pipeline C ABI, and `VoicePipeline` still calls `synthesize()` unless a future pipeline config threads options through. Any offline post-processing must run only through `Buffered` synthesis options after accumulating all PCM for the single submitted text input. Do not apply offline processing to individual decoder flush chunks or internal text chunks. `Streaming` plus non-zero postprocess flags is invalid unless a future flag is explicitly streaming-safe.
+- **Heavy ONNX models** (e.g. >= 100MB) should be benchmarked as new 64 KiB-aligned external-data variants using `scripts/export_external_data.py`. Alignment makes CPU memory mapping possible but does not guarantee zero-copy loading. Preserve published inline files; use a new filename or versioned revision and update consumers to fetch both the model and sidecar.
 - **C++17 only.** No external dependencies in the orchestration core. ONNX Runtime is the only third-party dep, and only in `speech_core_models`.
 - **No platform-specific code in `speech_core`.** Platform features (NNAPI, QNN, `__system_property_get`) live in `models/` and are gated by `__ANDROID__` ifdefs inside `onnx_engine.h`.
 - **Interfaces are the contract.** Public headers in `include/speech_core/` are the API surface — minimize churn. Add to `interfaces.h` only when the abstraction is needed; don't anticipate.

CMakeLists.txt

@@ -689,6 +689,7 @@ if(SPEECH_CORE_BUILD_TESTS)
            OR TEST_NAME STREQUAL "test_onnx_moss_model"
            OR TEST_NAME STREQUAL "test_onnx_localvqe_model"
            OR TEST_NAME STREQUAL "test_onnx_redimnet_model"
+           OR TEST_NAME STREQUAL "test_onnx_sortformer_model"
            OR TEST_NAME STREQUAL "test_redimnet_audio_preparation"
            OR TEST_NAME STREQUAL "test_litert_functiongemma_llm"
            OR TEST_NAME STREQUAL "test_openai_tts_server"
@@ -848,6 +849,16 @@ if(SPEECH_CORE_BUILD_TESTS)
                  COMMAND test_onnx_redimnet_model)
     endif()
 
+    if(SPEECH_CORE_WITH_ONNX AND
+       EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_onnx_sortformer_model.cpp)
+        add_executable(test_onnx_sortformer_model
+            tests/test_onnx_sortformer_model.cpp)
+        target_link_libraries(test_onnx_sortformer_model
+            PRIVATE speech_core_models)
+        add_test(NAME test_onnx_sortformer_model
+                 COMMAND test_onnx_sortformer_model)
+    endif()
+
     if(SPEECH_CORE_WITH_ONNX AND
        EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_openai_stt_server_e2e.cpp)
         add_executable(test_openai_stt_server_e2e

docs/models.md

@@ -182,6 +182,70 @@ the still-separate microphone and playback streams through
 `TimestampedEchoCancellationStream`; alignment or inference failures must be
 handled explicitly rather than substituting raw microphone PCM.
 
+### Aligned ONNX external data
+
+Large ONNX models can be repacked into a small model protobuf plus one aligned
+weight sidecar. Install the publishing-tool dependencies and write a new
+variant; the command refuses to overwrite either output file:
+
+```bash
+python3 -m pip install -r scripts/requirements-onnx-tools.txt
+python3 scripts/export_external_data.py \
+    model.onnx model-external-v2.onnx
+```
+
+The paired output is `model-external-v2.onnx` and
+`model-external-v2.onnx.data`. Tensor offsets default to 64 KiB alignment,
+which satisfies the Windows allocation-granularity requirement and is page
+aligned on supported Unix targets. Empty tensors remain inline. Existing
+external sidecars are copied in bounded 8 MiB chunks; converting a large
+inline protobuf still requires ONNX to deserialize that input model.
+
+The default 64 KiB threshold leaves smaller inline tensors in the model so
+their alignment padding cannot dominate the bundle size. Lower `--threshold`
+only after measuring the resulting disk-size and memory tradeoff.
+
+Alignment makes CPU memory mapping possible; it does not guarantee zero-copy
+loading for every ONNX Runtime version, execution provider, model transform,
+or platform. Benchmark session creation, first inference, and peak process
+memory with the oldest runtime shipped on each target before publishing a
+variant.
+
+Never replace an already-published inline filename in place. Publish a new
+variant filename or versioned model revision, keep the inline files available,
+and update consumers to download the model protobuf and its sidecar together.
+
+The additive `external-v2/` variants were selected with isolated, alternating
+three-process measurements on an Apple M5 Pro using ONNX Runtime 1.19.2 CPU,
+two inference threads, and disabled CPU arena and memory pattern. All compared
+inference-output hashes were identical:
+
+| Bundle | Load peak RSS saved | Post-warm USS saved | Load time | First inference | Warm inference | Disk |
+|---|---:|---:|---:|---:|---:|---:|
+| Parakeet INT8 | 701 MiB | 688 MiB | -15.9% | +10.8 ms | -0.2% | +0.015% |
+| Whisper Small INT8 | 511 MiB | 511 MiB | -10.7% | -1.1 ms | -0.9% | +0.044% |
+| Canary INT8 | 281 MiB | 268 MiB | -9.2% | +2.9 ms | -1.1% | +0.007% |
+| Canary FP32 | 259 MiB | 255 MiB | -13.7% | +2.1 ms | -0.9% | +0.002% |
+| Sortformer | 208 MiB | 199 MiB | -1.8% | +36.2 ms | -1.6% | +0.774% |
+| ReDimNet | 79 MiB | 75 MiB | -14.9% | -22.6 ms | +0.2% | +4.523% |
+
+Negative latency changes are faster; first-inference values are external minus
+inline, so a positive value is slower.
+
+Paired tests on the same sorted 50-utterance FLEURS English slice also kept
+every ASR hypothesis unchanged: Parakeet INT8 5.92% WER, Whisper Small INT8
+7.14%, and Canary INT8 10.42% for both layouts. These storage-layout results do
+not justify changing the general ASR default from Parakeet to Canary; Canary
+remains useful when its smaller bundle or transcription/translation contract
+fits the application better.
+
+The download helpers pin the audited model-repository commits, fetch each
+sidecar before its small graph protobuf, and retain the filenames expected by
+the runtime. Revisions can be overridden for an explicitly tested bundle with
+`SPEECH_PARAKEET_ONNX_REVISION`, `SPEECH_WHISPER_ONNX_REVISION`,
+`SPEECH_CANARY_ONNX_REVISION`, `SPEECH_SORTFORMER_ONNX_REVISION`, or
+`SPEECH_REDIMNET_ONNX_REVISION`.
+
 ## Building with ONNX support
 
 ```bash
@@ -272,7 +336,7 @@ auto result = stt.transcribe(audio, length, 16000);
   and the published v3 exports do not emit a language token in greedy decode,
   so the wrapper deliberately does not expose a language-forcing API.
 - Streaming supported via `begin_stream` / `push_chunk` / `end_stream` (accumulates audio and re-transcribes each chunk; not a true streaming decoder)
-- Model files: [soniqo/Parakeet-TDT-0.6B-ONNX](https://huggingface.co/soniqo/Parakeet-TDT-0.6B-ONNX) — `parakeet-encoder.onnx` (FP32, plus external `.onnx.data`) or `parakeet-encoder-int8.onnx` (~840 MB / ~100 MB INT8), `parakeet-decoder-joint.onnx` / `parakeet-decoder-joint-int8.onnx`, `vocab.json`. Decoder-joint inputs `targets` + `target_length` are INT32; encoder length input stays INT64.
+- Model files: [soniqo/Parakeet-TDT-0.6B-ONNX](https://huggingface.co/soniqo/Parakeet-TDT-0.6B-ONNX) — `parakeet-encoder.onnx` (FP32, plus external `.onnx.data`) or `parakeet-encoder-int8.onnx` (~840 MB / ~100 MB INT8), `parakeet-decoder-joint.onnx` / `parakeet-decoder-joint-int8.onnx`, `vocab.json`. The download helper selects the aligned `external-v2/` INT8 graph/sidecar pairs while the root inline files remain available. Decoder-joint inputs `targets` + `target_length` are INT32; encoder length input stays INT64.
 
 ## OnnxWhisperStt
 
@@ -318,6 +382,7 @@ SPEECH_CORE_WHISPER_ORT_THREADS=16 \
 - Download helper:
 
 ```bash
+scripts/download_whisper_onnx.sh small int8
 scripts/download_whisper_onnx.sh turbo int8
 scripts/download_whisper_onnx.sh medium fp16
 ```
@@ -326,7 +391,7 @@ scripts/download_whisper_onnx.sh medium fp16
   [soniqo/Whisper-Small-ONNX](https://huggingface.co/soniqo/Whisper-Small-ONNX),
   [soniqo/Whisper-Medium-ONNX](https://huggingface.co/soniqo/Whisper-Medium-ONNX),
   [soniqo/Whisper-Large-v3-ONNX](https://huggingface.co/soniqo/Whisper-Large-v3-ONNX),
-  [soniqo/Whisper-Large-v3-Turbo-ONNX](https://huggingface.co/soniqo/Whisper-Large-v3-Turbo-ONNX).
+  [soniqo/Whisper-Large-v3-Turbo-ONNX](https://huggingface.co/soniqo/Whisper-Large-v3-Turbo-ONNX). Small INT8 uses its aligned `external-v2/` graph/sidecar pairs; the other model/precision combinations are unchanged.
 
 ## OnnxCanaryStt
 
@@ -354,7 +419,7 @@ scripts/download_canary_onnx.sh int8
 scripts/download_canary_onnx.sh fp32
 ```
 
-- Model files: [soniqo/Canary-180M-Flash-ONNX](https://huggingface.co/soniqo/Canary-180M-Flash-ONNX) — `canary-encoder-int8.onnx` + `canary-decoder-int8.onnx` (273 MB) or the FP32 pair (778 MB), plus `vocab.json` and `config.json`.
+- Model files: [soniqo/Canary-180M-Flash-ONNX](https://huggingface.co/soniqo/Canary-180M-Flash-ONNX) — `canary-encoder-int8.onnx` + `canary-decoder-int8.onnx` (273 MB) or the FP32 pair (778 MB), plus `vocab.json` and `config.json`. The helper installs the corresponding aligned `external-v2/` graph/sidecar pairs under those local graph names.
 
 ## LiteRTSileroVad
 

scripts/download_canary_onnx.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 PRECISION="${1:-int8}"
 DEST="${2:-scripts/models/canary-180m-flash}"
 REPO="soniqo/Canary-180M-Flash-ONNX"
+REVISION="${SPEECH_CANARY_ONNX_REVISION:-d2801ebd34204c093e32eaf5092f2e5be5e22567}"
 
 case "$PRECISION" in
   int8|fp32|all) ;;
@@ -21,15 +22,17 @@ if [ -n "${HF_TOKEN:-}" ]; then
 fi
 
 download() {
-  local name="$1"
+  local remote_name="$1"
   local required="${2:-1}"
-  local url="https://huggingface.co/${REPO}/resolve/main/${name}"
-  local out="${DEST}/${name}"
-  if [ -f "$out" ]; then
+  local local_name="${3:-$remote_name}"
+  local force="${4:-0}"
+  local url="https://huggingface.co/${REPO}/resolve/${REVISION}/${remote_name}"
+  local out="${DEST}/${local_name}"
+  if [ -f "$out" ] && [ "$force" != "1" ]; then
     echo "exists ${out}"
     return
   fi
-  echo "download ${REPO}/${name}"
+  echo "download ${REPO}@${REVISION}/${remote_name}"
   # The +"${...}" guard keeps set -u from tripping over an empty array, which
   # is an error in the bash 3.2 that ships with macOS.
   if ! curl -L --fail --retry 3 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} \
@@ -38,7 +41,7 @@ download() {
     if [ "$required" = "1" ]; then
       exit 1
     fi
-    echo "optional missing ${name}"
+    echo "optional missing ${remote_name}"
     return
   fi
   mv "${out}.part" "$out"
@@ -52,12 +55,16 @@ download vocab.json 1
 download_precision() {
   case "$1" in
     int8)
-      download canary-encoder-int8.onnx 1
-      download canary-decoder-int8.onnx 1
+      download external-v2/canary-encoder-int8.onnx.data 1 canary-encoder-int8.onnx.data
+      download external-v2/canary-encoder-int8.onnx 1 canary-encoder-int8.onnx 1
+      download external-v2/canary-decoder-int8.onnx.data 1 canary-decoder-int8.onnx.data
+      download external-v2/canary-decoder-int8.onnx 1 canary-decoder-int8.onnx 1
       ;;
     fp32)
-      download canary-encoder.onnx 1
-      download canary-decoder.onnx 1
+      download external-v2/canary-encoder.onnx.data 1 canary-encoder.onnx.data
+      download external-v2/canary-encoder.onnx 1 canary-encoder.onnx 1
+      download external-v2/canary-decoder.onnx.data 1 canary-decoder.onnx.data
+      download external-v2/canary-decoder.onnx 1 canary-decoder.onnx 1
       ;;
   esac
 }

scripts/download_models.ps1

@@ -27,9 +27,27 @@ $outputRoot = [System.IO.Path]::GetFullPath($OutputDirectory)
 $voicesDirectory = Join-Path $outputRoot "voices"
 New-Item -ItemType Directory -Force -Path $voicesDirectory | Out-Null
 
+$parakeetRevision = if ([string]::IsNullOrWhiteSpace($env:SPEECH_PARAKEET_ONNX_REVISION)) {
+    "c1652ab21826e26dfc2be5273fe69edc7f9cc938"
+} else {
+    $env:SPEECH_PARAKEET_ONNX_REVISION
+}
+$redimnetRevision = if ([string]::IsNullOrWhiteSpace($env:SPEECH_REDIMNET_ONNX_REVISION)) {
+    "e911e3f063899805f3d94ee5d1db53fff8e9f3e8"
+} else {
+    $env:SPEECH_REDIMNET_ONNX_REVISION
+}
+$sortformerRevision = if ([string]::IsNullOrWhiteSpace($env:SPEECH_SORTFORMER_ONNX_REVISION)) {
+    "a7176b247fb7df5588414c20632f584d4f8562c8"
+} else {
+    $env:SPEECH_SORTFORMER_ONNX_REVISION
+}
+
 $defaultFiles = @(
     "Silero-VAD-v5-ONNX/silero-vad.onnx",
+    "Parakeet-TDT-0.6B-ONNX/parakeet-encoder-int8.onnx.data",
     "Parakeet-TDT-0.6B-ONNX/parakeet-encoder-int8.onnx",
+    "Parakeet-TDT-0.6B-ONNX/parakeet-decoder-joint-int8.onnx.data",
     "Parakeet-TDT-0.6B-ONNX/parakeet-decoder-joint-int8.onnx",
     "Parakeet-TDT-0.6B-ONNX/vocab.json",
     "Kokoro-82M-ONNX/kokoro-e2e.onnx",
@@ -70,12 +88,14 @@ $stenografFiles = @(
     "MOSS-Transcribe-Diarize-0.9B-ONNX-INT8-ENC/processor_config.json",
     "MOSS-Transcribe-Diarize-0.9B-ONNX-INT8-ENC/preprocessor_config.json",
     "MOSS-Transcribe-Diarize-0.9B-ONNX-INT8-ENC/vocab.json",
+    "ReDimNet2-B6-ONNX-FP32/ReDimNet2B6.onnx.data",
     "ReDimNet2-B6-ONNX-FP32/ReDimNet2B6.onnx",
     "ReDimNet2-B6-ONNX-FP32/config.json",
     # Only the light transcription pipeline loads this, and which pipeline runs
     # is not known until a session has tried the CUDA provider. It is listed
     # here because the application asks for the manifest a second time when it
     # turns out to need it, and already-present files are skipped.
+    "Sortformer-Diarization-4spk-ONNX/sortformer-default.onnx.data",
     "Sortformer-Diarization-4spk-ONNX/sortformer-default.onnx",
     "Sortformer-Diarization-4spk-ONNX/config.json",
     "LocalVQE-v1.4-AEC-200K-ONNX-FP32/LocalVQEAECResidualMask.onnx",
@@ -93,6 +113,29 @@ foreach ($entry in $files) {
     $slash = $entry.IndexOf('/')
     $repository = $entry.Substring(0, $slash)
     $relativePath = $entry.Substring($slash + 1)
+    $remoteRelativePath = $relativePath
+    $revision = "main"
+    $forceGraph = $false
+    if ($repository -eq "Parakeet-TDT-0.6B-ONNX") {
+        $revision = $parakeetRevision
+    } elseif ($repository -eq "ReDimNet2-B6-ONNX-FP32") {
+        $revision = $redimnetRevision
+    } elseif ($repository -eq "Sortformer-Diarization-4spk-ONNX") {
+        $revision = $sortformerRevision
+    }
+    if ($repository -eq "Parakeet-TDT-0.6B-ONNX" -and
+        $relativePath -like "parakeet-*-int8.onnx*") {
+        $remoteRelativePath = "external-v2/$relativePath"
+        $forceGraph = $relativePath.EndsWith(".onnx")
+    } elseif ($repository -eq "ReDimNet2-B6-ONNX-FP32" -and
+              $relativePath -like "ReDimNet2B6.onnx*") {
+        $remoteRelativePath = "external-v2/$relativePath"
+        $forceGraph = $relativePath.EndsWith(".onnx")
+    } elseif ($repository -eq "Sortformer-Diarization-4spk-ONNX" -and
+              $relativePath -like "sortformer-default.onnx*") {
+        $remoteRelativePath = "external-v2/$relativePath"
+        $forceGraph = $relativePath.EndsWith(".onnx")
+    }
     if ($ModelSet -eq "stenograf") {
         # Keep each application model in its repository directory. Nemotron
         # external-data references and common config/vocab filenames require
@@ -104,7 +147,8 @@ foreach ($entry in $files) {
         $destination = Join-Path $outputRoot ([System.IO.Path]::GetFileName($relativePath))
     }
 
-    if ((Test-Path -LiteralPath $destination) -and
+    if (-not $forceGraph -and
+        (Test-Path -LiteralPath $destination) -and
         (Get-Item -LiteralPath $destination).Length -gt 0) {
         Write-Host "[skip] $relativePath (already exists)"
         continue
@@ -114,8 +158,8 @@ foreach ($entry in $files) {
     New-Item -ItemType Directory -Force -Path $parent | Out-Null
     $temporary = "$destination.part"
     Remove-Item -Force -ErrorAction SilentlyContinue -LiteralPath $temporary
-    $url = "https://huggingface.co/soniqo/$repository/resolve/main/$relativePath"
-    Write-Host "[fetch] $relativePath"
+    $url = "https://huggingface.co/soniqo/$repository/resolve/$revision/$remoteRelativePath"
+    Write-Host "[fetch] $repository@$revision/$remoteRelativePath"
 
     & curl.exe --fail --location --retry 3 --output $temporary $url
     if ($LASTEXITCODE -ne 0) {