feat(coding-agent): add Apple SpeechAnalyzer STT

Author: audreyt

.github/workflows/bazel-cache-warm.yml

@@ -70,8 +70,8 @@ jobs:
          fail-fast: false
          matrix:
             include:
-               - { os: macos-15-intel, target: darwin-x64-baseline, scope: release-darwin-x64 }
-               - { os: macos-14, target: darwin-arm64, scope: release-darwin-arm64 }
+               - { os: macos-26-intel, target: darwin-x64-baseline, scope: release-darwin-x64 }
+               - { os: macos-26, target: darwin-arm64, scope: release-darwin-arm64 }
       runs-on: ${{ matrix.os }}
       steps:
          - uses: actions/checkout@v4

.github/workflows/ci.yml

@@ -621,13 +621,13 @@ jobs:
          matrix:
             include:
                - {
-                    os: macos-15-intel,
+                    os: macos-26-intel,
                     target_id: darwin-x64,
                     binary_path: packages/coding-agent/binaries/omp-darwin-x64,
                     native_targets: darwin-x64-baseline,
                  }
                - {
-                    os: macos-14,
+                    os: macos-26,
                     target_id: darwin-arm64,
                     binary_path: packages/coding-agent/binaries/omp-darwin-arm64,
                     native_targets: darwin-arm64,
@@ -693,6 +693,12 @@ jobs:
            with:
               name: omp-binary-${{ matrix.target_id }}
               path: ${{ matrix.binary_path }}
+         - name: Upload Apple speech sidecar artifact
+           uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+           with:
+              name: apple-speech-sidecar-${{ matrix.target_id }}
+              path: packages/coding-agent/dist/omp-speech-analyzer-*
+              if-no-files-found: error
 
    # Publishes the five @oh-my-pi/pi-natives-<tag> leaf packages once
    # validation passes and every binary built. Runs on one linux runner: leaf
@@ -894,6 +900,16 @@ jobs:
            uses: ./.github/actions/native-artifacts
            with:
               targets: linux-x64-baseline linux-x64-modern
+         - name: Install Apple speech sidecars
+           uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+           with:
+              pattern: apple-speech-sidecar-*
+              path: packages/coding-agent/dist
+              merge-multiple: true
+         - name: Verify Apple speech sidecars
+           run: |
+              test -s packages/coding-agent/dist/omp-speech-analyzer-arm64
+              test -s packages/coding-agent/dist/omp-speech-analyzer-x64
          - name: Publish to npm
            env:
               # Fallback auth: setup-node wrote an .npmrc referencing

packages/coding-agent/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+- Added opt-in Apple SpeechAnalyzer speech-to-text on macOS 26+, with live partials and system-managed locale assets.
+
 ## [18.0.8] - 2026-08-27
 
 ### Added

packages/coding-agent/package.json

@@ -88,6 +88,7 @@
     "dist/cli.js",
     "dist/docs-index.generated.txt",
     "dist/CHANGELOG-*.md",
+    "dist/omp-speech-analyzer-*",
     "dist/*.node",
     "dist/template-*.css",
     "dist/template-*.html",

packages/coding-agent/scripts/apple-speech-sidecar.ts

@@ -0,0 +1,80 @@
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
+import * as path from "node:path";
+import { $which } from "@oh-my-pi/pi-utils";
+
+export type AppleSpeechArchitecture = "arm64" | "x64";
+
+const packageDir = path.join(import.meta.dir, "..");
+const sourcePath = path.join(packageDir, "src", "stt", "speech-analyzer.swift");
+
+/** Compile the SpeechAnalyzer helper for one Darwin release architecture. */
+export async function buildAppleSpeechSidecar(
+	architecture: AppleSpeechArchitecture,
+	outputPath: string,
+): Promise<void> {
+	if (process.platform !== "darwin") {
+		throw new Error("Apple SpeechAnalyzer sidecars can only be built on macOS.");
+	}
+	const swiftc = $which("swiftc");
+	if (!swiftc) throw new Error("Swift compiler not found; Xcode 26 or newer is required.");
+	await fs.mkdir(path.dirname(outputPath), { recursive: true });
+	const targetArchitecture = architecture === "x64" ? "x86_64" : architecture;
+	const proc = Bun.spawn(
+		[
+			swiftc,
+			"-parse-as-library",
+			"-O",
+			"-target",
+			`${targetArchitecture}-apple-macos26.0`,
+			"-framework",
+			"Speech",
+			"-framework",
+			"AVFAudio",
+			"-framework",
+			"CoreMedia",
+			"-o",
+			outputPath,
+			sourcePath,
+		],
+		{ stdin: "ignore", stdout: "inherit", stderr: "pipe" },
+	);
+	const [exitCode, stderr] = await Promise.all([
+		proc.exited,
+		new Response(proc.stderr as ReadableStream<Uint8Array>).text(),
+	]);
+	if (exitCode !== 0) {
+		throw new Error(`SpeechAnalyzer sidecar build failed: ${stderr.trim() || `swiftc exited ${exitCode}`}`);
+	}
+	await fs.chmod(outputPath, 0o755);
+
+	const codesign = $which("codesign");
+	if (codesign) {
+		const sign = Bun.spawn([codesign, "--force", "--sign", "-", outputPath], {
+			stdin: "ignore",
+			stdout: "ignore",
+			stderr: "pipe",
+		});
+		const [signExitCode, signStderr] = await Promise.all([
+			sign.exited,
+			new Response(sign.stderr as ReadableStream<Uint8Array>).text(),
+		]);
+		if (signExitCode !== 0) {
+			throw new Error(
+				`SpeechAnalyzer sidecar signing failed: ${signStderr.trim() || `codesign exited ${signExitCode}`}`,
+			);
+		}
+	}
+}
+
+/** Build a temporary helper and return its bytes for Bun's compiled-binary embed. */
+export async function buildAppleSpeechSidecarBase64(architecture: AppleSpeechArchitecture): Promise<string> {
+	const directory = await fs.mkdtemp(path.join(os.tmpdir(), "omp-speech-analyzer-build-"));
+	const outputPath = path.join(directory, "omp-speech-analyzer");
+	try {
+		await buildAppleSpeechSidecar(architecture, outputPath);
+		return Buffer.from(await Bun.file(outputPath).bytes()).toString("base64");
+	} finally {
+		await fs.rm(directory, { recursive: true, force: true });
+	}
+}

packages/coding-agent/scripts/build-binary.ts

@@ -2,6 +2,7 @@
 
 import { createRequire } from "node:module";
 import * as path from "node:path";
+import { buildAppleSpeechSidecarBase64 } from "./apple-speech-sidecar";
 import { compileCodingAgent } from "./compile-binary";
 
 const packageDir = path.join(import.meta.dir, "..");
@@ -88,12 +89,22 @@ async function main(): Promise<void> {
 			["bun", "--cwd=../natives", "run", "gen:native"],
 			crossBuild ? { ...Bun.env, TARGET_PLATFORM: crossBuild.platform, TARGET_ARCH: crossBuild.arch } : Bun.env,
 		);
+		const targetPlatform = crossBuild?.platform ?? process.platform;
+		const targetArchitecture = crossBuild?.arch ?? process.arch;
+		let appleSpeechSidecarBase64: string | undefined;
+		if (targetPlatform === "darwin") {
+			if (targetArchitecture !== "arm64" && targetArchitecture !== "x64") {
+				throw new Error(`Unsupported Darwin architecture for SpeechAnalyzer: ${targetArchitecture}`);
+			}
+			appleSpeechSidecarBase64 = await buildAppleSpeechSidecarBase64(targetArchitecture);
+		}
 		try {
 			await compileCodingAgent({
 				repoRoot,
 				entrypoint: path.join(packageDir, "src", "cli.ts"),
 				outfile: outputPath,
 				transformersVersion,
+				appleSpeechSidecarBase64,
 				target: crossBuild?.target,
 				executablePath: Bun.env.BUN_COMPILE_EXECUTABLE_PATH || undefined,
 				skipBuiltinCodesign: shouldAdhocSign,

packages/coding-agent/scripts/bundle-dist.ts

@@ -3,6 +3,7 @@
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { isEnoent } from "@oh-my-pi/pi-utils";
+import { buildAppleSpeechSidecar } from "./apple-speech-sidecar";
 import { buildDocsIndexPayload } from "./generate-docs-index";
 
 const packageDir = path.join(import.meta.dir, "..");
@@ -70,6 +71,7 @@ async function cleanBundleOutputs(): Promise<void> {
 					entry === "cli.js" ||
 					entry === "docs-index.generated.txt" ||
 					entry.endsWith(".node") ||
+					(process.platform === "darwin" && entry.startsWith("omp-speech-analyzer-")) ||
 					entry.endsWith(".js.map") ||
 					(entry.startsWith("CHANGELOG-") && entry.endsWith(".md")) ||
 					legacyHtmlExportAssetPattern.test(entry),
@@ -81,6 +83,12 @@ async function cleanBundleOutputs(): Promise<void> {
 async function main(): Promise<void> {
 	const start = Bun.nanoseconds();
 	await cleanBundleOutputs();
+	if (process.platform === "darwin") {
+		await Promise.all([
+			buildAppleSpeechSidecar("arm64", path.join(outDir, "omp-speech-analyzer-arm64")),
+			buildAppleSpeechSidecar("x64", path.join(outDir, "omp-speech-analyzer-x64")),
+		]);
+	}
 	// The npm bundle ships no stats dashboard sources, so embed the dashboard
 	// archive the same way compiled binaries do (scripts/build-binary.ts). Reset
 	// afterwards to keep the checked-in placeholder empty.
@@ -101,6 +109,7 @@ async function main(): Promise<void> {
 			external: [...ALWAYS_EXTERNAL, ...RUNTIME_EXTERNAL],
 			define: {
 				"process.env.PI_BUNDLED": JSON.stringify("true"),
+				"process.env.PI_APPLE_SPEECH_SIDECAR_BASE64": JSON.stringify(""),
 				"process.env.PI_DOCS_EMBED": JSON.stringify(docsPayload.payload),
 			},
 			minify: {

packages/coding-agent/scripts/compile-binary.ts

@@ -14,6 +14,8 @@ export interface CodingAgentCompileOptions {
 	readonly outfile: string;
 	/** Concrete Transformers.js version baked into the tiny-model worker. */
 	readonly transformersVersion: string;
+	/** Base64-encoded native SpeechAnalyzer helper for the selected Darwin architecture. */
+	readonly appleSpeechSidecarBase64?: string;
 	/** Optional cross-compilation runtime target. */
 	readonly target?: Bun.Build.CompileTarget;
 	/** Optional unmodified Bun executable used as the standalone runtime template. */
@@ -41,6 +43,7 @@ export async function compileCodingAgent(options: CodingAgentCompileOptions): Pr
 			define: {
 				"process.env.PI_COMPILED": JSON.stringify("true"),
 				"process.env.PI_TINY_TRANSFORMERS_VERSION": JSON.stringify(options.transformersVersion),
+				"process.env.PI_APPLE_SPEECH_SIDECAR_BASE64": JSON.stringify(options.appleSpeechSidecarBase64 ?? ""),
 				"process.env.PI_DOCS_EMBED": JSON.stringify((await buildDocsIndexPayload()).payload),
 			},
 			minify: {

packages/coding-agent/src/cli.ts

@@ -35,6 +35,7 @@ import type { WorkerInbound as JsWorkerInbound, WorkerOutbound as JsWorkerOutbou
 import { DAEMON_BROKER_WORKER_ARG } from "./launch/protocol";
 import { TERMINAL_OUTPUT_WORKER_ARG } from "./launch/terminal-output-worker-protocol";
 import { LSP_MUX_WORKER_ARG } from "./lsp/mux/protocol";
+import { smokeTestAppleSpeechSidecar } from "./stt/apple-speech-client";
 import rootLicense from "./tools/browser/relay/extension-assets/LICENSE.txt" with { type: "text" };
 import thirdPartyNotices from "./tools/browser/relay/extension-assets/THIRD-PARTY-NOTICES.txt" with { type: "text" };
 import { COMPUTER_WORKER_ARG } from "./tools/computer/protocol";
@@ -120,6 +121,7 @@ async function runSmokeTest(): Promise<void> {
 
 	await smokeTestTinyTitleWorker();
 	await smokeTestSttWorker();
+	await smokeTestAppleSpeechSidecar();
 	await smokeTestJsEvalWorker();
 	await smokeTestComputerWorker();
 	await smokeTestTtsWorker();

packages/coding-agent/src/cli/setup-cli.ts

@@ -9,8 +9,9 @@ import chalk from "@oh-my-pi/pi-utils/chalk";
 import { Settings, settings } from "../config/settings";
 import { checkPythonKernelAvailability } from "../eval/py/kernel";
 import { theme } from "../modes/theme/theme";
+import { appleSpeechClient } from "../stt/apple-speech-client";
 import { downloadSttModel, isSttModelCached } from "../stt/downloader";
-import { isSttModelKey, STT_MODEL_OPTIONS } from "../stt/models";
+import { isSttModelKey, resolveSttModelSpec, STT_MODEL_OPTIONS } from "../stt/models";
 import { downloadTtsModel, isTtsLocalModelKey, isTtsModelCached, TTS_LOCAL_MODEL_OPTIONS } from "../tts";
 import { selectSetupModel } from "./setup-model-picker";
 
@@ -147,9 +148,9 @@ async function handlePythonSetup(flags: { json?: boolean; check?: boolean }): Pr
 }
 
 /**
- * One installable speech dependency. `isReady`/`status` are read-only probes;
- * `pick` (optional) lets an interactive user choose + persist a model; `ensure`
- * performs the download, streaming a normalized progress event.
+ * One speech dependency. `isReady`/`status` are read-only probes; `pick`
+ * (optional) lets an interactive user choose + persist an engine; `ensure`
+ * prepares its application-managed model or system-managed locale asset.
  */
 interface SpeechComponent {
 	name: string;
@@ -163,10 +164,23 @@ function buildSpeechComponents(): SpeechComponent[] {
 	return [
 		{
 			name: "Speech-to-Text model",
-			isReady: () => isSttModelCached(settings.get("stt.modelName")),
+			isReady: async () => {
+				const spec = resolveSttModelSpec(settings.get("stt.modelName"));
+				if (spec.engine === "speech-analyzer") {
+					return (await appleSpeechClient.status(settings.get("stt.language"))).installed;
+				}
+				return await isSttModelCached(spec.key);
+			},
 			status: async () => {
-				const key = settings.get("stt.modelName");
-				return (await isSttModelCached(key)) ? key : `${key} — not downloaded`;
+				const spec = resolveSttModelSpec(settings.get("stt.modelName"));
+				if (spec.engine === "speech-analyzer") {
+					const status = await appleSpeechClient.status(settings.get("stt.language"));
+					if (status.installed) {
+						return `${spec.key} — ${status.locale ?? "system locale"} (system-managed)`;
+					}
+					return `${spec.key} — ${status.error ?? "locale asset not prepared"}`;
+				}
+				return (await isSttModelCached(spec.key)) ? spec.key : `${spec.key} — not downloaded`;
 			},
 			pick: async () => {
 				const chosen = await selectSetupModel(
@@ -181,10 +195,21 @@ function buildSpeechComponents(): SpeechComponent[] {
 				}
 				return true;
 			},
-			ensure: onProgress =>
-				downloadSttModel(settings.get("stt.modelName"), progress =>
+			ensure: async onProgress => {
+				const spec = resolveSttModelSpec(settings.get("stt.modelName"));
+				if (spec.engine === "speech-analyzer") {
+					onProgress({ stage: "Preparing system-managed Apple speech recognition" });
+					const status = await appleSpeechClient.prepare(settings.get("stt.language"));
+					onProgress({
+						stage: `Apple speech recognition ready${status.locale ? ` (${status.locale})` : ""}`,
+						percent: 100,
+					});
+					return;
+				}
+				await downloadSttModel(spec.key, progress =>
 					onProgress({ stage: `Downloading ${progress.label} model`, percent: progress.percent }),
-				),
+				);
+			},
 		},
 		{
 			name: "Text-to-Speech model",