Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,4 @@ benchmark-results/
*.xcodeproj/project.pbxproj
.DS_Store
.claude/scheduled_tasks.lock
/.idea/speech-swift.git-personal.iml
9 changes: 9 additions & 0 deletions Examples/SpeechDemo/SpeechDemo/EchoMicrophoneGate.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import Foundation

/// Keeps speaker playback out of the VAD stream while preserving audio timing.
struct EchoMicrophoneGate {
static func samplesToPush(_ samples: [Float], muted: Bool) -> [Float] {
guard muted else { return samples }
return [Float](repeating: 0, count: samples.count)
}
}
23 changes: 21 additions & 2 deletions Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ final class EchoViewModel {
private var debugMicBuffer: [Float] = []
private var debugTTSBuffer: [Float] = []
private var speechStartTime: Date?
private var isSpeaking = false

var modelsLoaded: Bool { vad != nil && asr != nil && tts != nil }

Expand All @@ -56,7 +57,14 @@ final class EchoViewModel {
loadingStatus = "Loading TTS (Qwen3 Base)..."
tts = try await Task.detached {
try await Qwen3TTSModel.fromPretrained(
modelId: TTSModelVariant.base.rawValue)
modelId: TTSModelVariant.base.rawValue
) { (progress: Double, status: String) in
DispatchQueue.main.async { [weak self] in
self?.loadingStatus = status.isEmpty
? "Loading TTS... \(Int(progress * 100))%"
: "\(status) (\(Int(progress * 100))%)"
}
}
}.value

appendLog("All models loaded.")
Expand Down Expand Up @@ -89,6 +97,7 @@ final class EchoViewModel {

player.onPlaybackFinished = { [weak self] in
guard let self, self.isRunning else { return }
self.isSpeaking = false
self.pipeline?.resumeListening()
self.pipelineState = "listening"
self.appendLog("Listening...")
Expand All @@ -108,6 +117,7 @@ final class EchoViewModel {
pipeline?.stop()
pipeline = nil
isRunning = false
isSpeaking = false
pipelineState = "idle"
saveDebugFiles()
appendLog("Pipeline stopped.")
Expand All @@ -133,14 +143,19 @@ final class EchoViewModel {
}
case .transcriptionCompleted(let text, let language, _):
pipelineState = "synthesizing..."
if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
isSpeaking = true
}
lastTranscription = text
lastLanguage = language ?? ""
appendLog("[STT\(language.map { " [\($0)]" } ?? "")] \(text)")
case .responseCreated:
pipelineState = "speaking..."
isSpeaking = true
player.resetGeneration()
case .responseInterrupted:
player.stop()
isSpeaking = false
pipelineState = "listening"
case .responseAudioDelta(let samples):
debugTTSBuffer.append(contentsOf: samples)
Expand All @@ -152,6 +167,7 @@ final class EchoViewModel {
break
case .error(let msg):
pipelineState = "error"
isSpeaking = false
appendLog("[ERROR] \(msg)")
pipeline?.resumeListening()
}
Expand Down Expand Up @@ -213,7 +229,10 @@ final class EchoViewModel {
}

self.debugMicBuffer.append(contentsOf: samples)
self.pipeline?.pushAudio(samples)
// Keep the C++ pipeline's audio clock continuous, but do not let
// speaker playback become a new VAD speech turn.
self.pipeline?.pushAudio(
EchoMicrophoneGate.samplesToPush(samples, muted: self.isSpeaking))
}

do {
Expand Down
15 changes: 8 additions & 7 deletions Examples/SpeechDemo/Tests/AudioPlayerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@
import AVFoundation
import XCTest
@testable import SpeechDemo
import AudioCommon

final class AudioPlayerTests: XCTestCase {

// MARK: - State machine tests (no audio hardware needed)

/// markGenerationComplete with zero pending buffers fires callback (via main queue).
func testMarkGenerationCompleteFiresWhenNoPendingBuffers() {
let player = AudioPlayer()
let player = StreamingAudioPlayer()

let exp = expectation(description: "playback finished")
player.onPlaybackFinished = { exp.fulfill() }
Expand All @@ -20,7 +21,7 @@ final class AudioPlayerTests: XCTestCase {

/// Without markGenerationComplete, callback never fires (even with no buffers).
func testNoCallbackWithoutMarkGenerationComplete() {
let player = AudioPlayer()
let player = StreamingAudioPlayer()

var finished = false
player.onPlaybackFinished = { finished = true }
Expand All @@ -32,7 +33,7 @@ final class AudioPlayerTests: XCTestCase {

/// resetGeneration prevents stale generationComplete from firing.
func testResetGenerationClearsFlag() {
let player = AudioPlayer()
let player = StreamingAudioPlayer()

var finishCount = 0
let exp = expectation(description: "two finishes")
Expand All @@ -57,7 +58,7 @@ final class AudioPlayerTests: XCTestCase {

/// stop() resets generationComplete, allowing clean next cycle.
func testStopResetsGenerationComplete() {
let player = AudioPlayer()
let player = StreamingAudioPlayer()

var finishCount = 0
let exp = expectation(description: "two finishes")
Expand All @@ -82,7 +83,7 @@ final class AudioPlayerTests: XCTestCase {

/// Without markGenerationComplete, play() alone never triggers callback.
func testRaceConditionPrevented() throws {
let player = AudioPlayer()
let player = StreamingAudioPlayer()

var callbackFired = false
player.onPlaybackFinished = { callbackFired = true }
Expand All @@ -105,7 +106,7 @@ final class AudioPlayerTests: XCTestCase {

/// Two full cycles back-to-back (simulates two Echo responses).
func testTwoCyclesBackToBack() {
let player = AudioPlayer()
let player = StreamingAudioPlayer()

var finishCount = 0
player.onPlaybackFinished = { finishCount += 1 }
Expand All @@ -129,7 +130,7 @@ final class AudioPlayerTests: XCTestCase {

/// Interrupt during playback: stop() mid-cycle, then new cycle works.
func testInterruptThenNewCycle() {
let player = AudioPlayer()
let player = StreamingAudioPlayer()

var finishCount = 0
player.onPlaybackFinished = { finishCount += 1 }
Expand Down
22 changes: 22 additions & 0 deletions Examples/SpeechDemo/Tests/EchoMicrophoneGateTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#if os(macOS)
import XCTest
@testable import SpeechDemo

final class EchoMicrophoneGateTests: XCTestCase {
func testMutedInputPreservesTimingWithSilence() {
let samples: [Float] = [0.2, -0.4, 0.6]

XCTAssertEqual(
EchoMicrophoneGate.samplesToPush(samples, muted: true),
[0, 0, 0])
}

func testUnmutedInputPassesThrough() {
let samples: [Float] = [0.2, -0.4, 0.6]

XCTAssertEqual(
EchoMicrophoneGate.samplesToPush(samples, muted: false),
samples)
}
}
#endif
105 changes: 88 additions & 17 deletions Sources/AudioCommon/HuggingFaceDownloader.swift
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import os
/// Download errors
public enum DownloadError: Error, LocalizedError {
case failedToDownload(String)
case remoteFileNotFound(modelId: String, file: String)
case invalidRemoteFileName(String)
/// A download attempt made no progress for `seconds` and was aborted
/// so the caller's retry loop can fire instead of hanging.
Expand All @@ -14,6 +15,8 @@ public enum DownloadError: Error, LocalizedError {
switch self {
case .failedToDownload(let file):
return "Failed to download: \(file)"
case .remoteFileNotFound(let modelId, let file):
return "Remote file not found: \(modelId)/\(file)"
case .invalidRemoteFileName(let file):
return "Refusing to write unsafe remote file name: \(file)"
case .stalled(let modelId, let seconds):
Expand Down Expand Up @@ -244,6 +247,7 @@ public enum HuggingFaceDownloader {
modelId: String,
to directory: URL,
files: [String],
optionalFiles: [String] = [],
expectedSizes: [String: Int64]? = nil,
offlineMode: Bool = false,
retryDelaysSeconds: [Int]? = nil,
Expand All @@ -257,9 +261,15 @@ public enum HuggingFaceDownloader {
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)

let safeFiles = try files.map(validatedRemoteFileName)
let safeOptionalFiles = Set(try optionalFiles.map(validatedRemoteFileName))
guard safeOptionalFiles.isSubset(of: Set(safeFiles)) else {
throw DownloadError.failedToDownload(
"optionalFiles must be included in files for \(modelId)")
}
if offlineMode {
let missing = safeFiles.first {
!FileManager.default.fileExists(atPath: directory.appendingPathComponent($0).path)
!safeOptionalFiles.contains($0)
&& !FileManager.default.fileExists(atPath: directory.appendingPathComponent($0).path)
}
if let missing {
throw DownloadError.failedToDownload(
Expand All @@ -277,6 +287,7 @@ public enum HuggingFaceDownloader {
let remoteFiles = try await resolveRemoteFiles(
modelId: modelId,
files: safeFiles,
optionalFiles: safeOptionalFiles,
expectedSizes: expectedSizes)
try await downloadResolvedFilesByteWeighted(
remoteFiles,
Expand Down Expand Up @@ -552,6 +563,38 @@ public enum HuggingFaceDownloader {
}
}

extension HuggingFaceDownloader {
static func makeHubRequest(
url: URL,
method: String? = nil,
range: String? = nil,
timeout: TimeInterval? = nil
) -> URLRequest {
var request = URLRequest(url: url)
request.httpMethod = method
if let range {
request.setValue(range, forHTTPHeaderField: "Range")
}
if let timeout {
request.timeoutInterval = timeout
}
applyHubAuth(to: &request)
return request
}

static func contentRangeTotal(_ response: HTTPURLResponse) -> Int64? {
for (rawKey, rawValue) in response.allHeaderFields {
guard String(describing: rawKey).caseInsensitiveCompare("Content-Range") == .orderedSame else {
continue
}
let value = String(describing: rawValue)
guard let total = value.split(separator: "/").last else { return nil }
return Int64(total)
}
return nil
}
}

// MARK: - Byte-weighted explicit downloads

private struct ResolvedRemoteFile: Sendable {
Expand All @@ -567,11 +610,13 @@ private extension HuggingFaceDownloader {
static func resolveRemoteFiles(
modelId: String,
files: [String],
optionalFiles: Set<String>,
expectedSizes: [String: Int64]?
) async throws -> [ResolvedRemoteFile] {
if let expectedSizes {
return try files.map { file in
return try files.compactMap { file in
guard let size = expectedSizes[file], size > 0 else {
if optionalFiles.contains(file) { return nil }
throw DownloadError.failedToDownload("\(modelId)/\(file): missing expected size")
}
return ResolvedRemoteFile(
Expand All @@ -581,32 +626,58 @@ private extension HuggingFaceDownloader {
}
}

var result: [ResolvedRemoteFile] = []
result.reserveCapacity(files.count)
for file in files {
result.append(try await resolveRemoteFile(modelId: modelId, file: file))
return try await withThrowingTaskGroup(of: (Int, ResolvedRemoteFile?).self) { group in
for (index, file) in files.enumerated() {
group.addTask {
do {
return (index, try await resolveRemoteFile(modelId: modelId, file: file))
} catch DownloadError.remoteFileNotFound where optionalFiles.contains(file) {
return (index, nil)
}
}
}

var result = Array<ResolvedRemoteFile?>(repeating: nil, count: files.count)
for try await (index, file) in group {
result[index] = file
}
return result.compactMap { $0 }
}
return result
}

static func resolveRemoteFile(modelId: String, file: String) async throws -> ResolvedRemoteFile {
let url = try resolveURL(modelId: modelId, file: file)
var request = URLRequest(url: url)
request.httpMethod = "HEAD"
applyHubAuth(to: &request)
let request = makeHubRequest(url: url, method: "HEAD", timeout: 30)
let (_, response) = try await URLSession.shared.data(for: request)
guard let http = response as? HTTPURLResponse else {
throw DownloadError.failedToDownload("\(modelId)/\(file): missing HTTP response")
}
if http.statusCode == 404 {
throw DownloadError.remoteFileNotFound(modelId: modelId, file: file)
}
guard (200..<300).contains(http.statusCode) else {
throw DownloadError.failedToDownload("\(modelId)/\(file): HTTP \(http.statusCode)")
}
guard let resolved = http.url else {
guard let headResolved = http.url else {
throw DownloadError.failedToDownload("\(modelId)/\(file): missing resolved URL")
}
let size = headerInt64(http, "Content-Length")
var resolved = headResolved
var size = headerInt64(http, "Content-Length")
?? headerInt64(http, "x-linked-size")
?? http.expectedContentLength
if size <= 0 {
var probe = makeHubRequest(url: url, range: "bytes=0-0", timeout: 30)
probe.cachePolicy = .reloadIgnoringLocalCacheData
let (probeData, probeResponse) = try await URLSession.shared.data(for: probe)
guard let probeHTTP = probeResponse as? HTTPURLResponse,
probeHTTP.statusCode == 206,
!probeData.isEmpty,
let total = contentRangeTotal(probeHTTP) else {
throw DownloadError.failedToDownload("\(modelId)/\(file): missing Content-Range size")
}
resolved = probeHTTP.url ?? resolved
size = total
}
guard size > 0 else {
throw DownloadError.failedToDownload("\(modelId)/\(file): unknown remote size")
}
Expand Down Expand Up @@ -680,8 +751,7 @@ private extension HuggingFaceDownloader {
totalBytes: Int64,
progressHandler: ((Double, Int64, Int64, String) -> Void)?
) async throws {
var request = URLRequest(url: file.url)
applyHubAuth(to: &request)
let request = makeHubRequest(url: file.url, timeout: 120)
let tempURL = destination
.deletingLastPathComponent()
.appendingPathComponent(".\(destination.lastPathComponent).download")
Expand Down Expand Up @@ -813,9 +883,10 @@ private extension HuggingFaceDownloader {
state: RangedDownloadProgress,
session: URLSession
) async throws {
var request = URLRequest(url: file.url)
request.setValue("bytes=\(chunk.start)-\(chunk.end)", forHTTPHeaderField: "Range")
applyHubAuth(to: &request)
let request = makeHubRequest(
url: file.url,
range: "bytes=\(chunk.start)-\(chunk.end)",
timeout: 120)

let (data, response) = try await session.data(for: request)
guard let http = response as? HTTPURLResponse else {
Expand Down
Loading