From ce0cf50bb46d2690210a4835f6c28b146aca9214 Mon Sep 17 00:00:00 2001 From: "maryna.kryvko" Date: Sun, 26 Jul 2026 12:24:57 +0200 Subject: [PATCH 1/4] Fix Qwen3-TTS model downloads --- .../SpeechDemo/SpeechDemo/EchoViewModel.swift | 9 +++- .../AudioCommon/HuggingFaceDownloader.swift | 54 ++++++++++++++++--- Sources/Qwen3TTS/Qwen3TTS.swift | 25 ++++++--- .../HuggingFaceDownloaderTests.swift | 20 +++++++ 4 files changed, 93 insertions(+), 15 deletions(-) diff --git a/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift b/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift index 78688084..82a1c9c0 100644 --- a/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift +++ b/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift @@ -56,7 +56,14 @@ final class EchoViewModel { loadingStatus = "Loading TTS (Qwen3 Base)..." tts = try await Task.detached { try await Qwen3TTSModel.fromPretrained( - modelId: TTSModelVariant.base.rawValue) + modelId: TTSModelVariant.base.rawValue + ) { (progress: Double, status: String) in + DispatchQueue.main.async { [weak self] in + self?.loadingStatus = status.isEmpty + ? "Loading TTS... \(Int(progress * 100))%" + : "\(status) (\(Int(progress * 100))%)" + } + } }.value appendLog("All models loaded.") diff --git a/Sources/AudioCommon/HuggingFaceDownloader.swift b/Sources/AudioCommon/HuggingFaceDownloader.swift index f9cb8985..81de31c8 100644 --- a/Sources/AudioCommon/HuggingFaceDownloader.swift +++ b/Sources/AudioCommon/HuggingFaceDownloader.swift @@ -552,6 +552,20 @@ public enum HuggingFaceDownloader { } } +extension HuggingFaceDownloader { + static func contentRangeTotal(_ response: HTTPURLResponse) -> Int64? { + for (rawKey, rawValue) in response.allHeaderFields { + guard String(describing: rawKey).caseInsensitiveCompare("Content-Range") == .orderedSame else { + continue + } + let value = String(describing: rawValue) + guard let total = value.split(separator: "/").last else { return nil } + return Int64(total) + } + return nil + } +} + // MARK: - Byte-weighted explicit downloads private struct ResolvedRemoteFile: Sendable { @@ -581,18 +595,26 @@ private extension HuggingFaceDownloader { } } - var result: [ResolvedRemoteFile] = [] - result.reserveCapacity(files.count) - for file in files { - result.append(try await resolveRemoteFile(modelId: modelId, file: file)) + return try await withThrowingTaskGroup(of: (Int, ResolvedRemoteFile).self) { group in + for (index, file) in files.enumerated() { + group.addTask { + (index, try await resolveRemoteFile(modelId: modelId, file: file)) + } + } + + var result = Array(repeating: nil, count: files.count) + for try await (index, file) in group { + result[index] = file + } + return result.compactMap { $0 } } - return result } static func resolveRemoteFile(modelId: String, file: String) async throws -> ResolvedRemoteFile { let url = try resolveURL(modelId: modelId, file: file) var request = URLRequest(url: url) request.httpMethod = "HEAD" + request.timeoutInterval = 30 applyHubAuth(to: &request) let (_, response) = try await URLSession.shared.data(for: request) guard let http = response as? HTTPURLResponse else { @@ -601,12 +623,28 @@ private extension HuggingFaceDownloader { guard (200..<300).contains(http.statusCode) else { throw DownloadError.failedToDownload("\(modelId)/\(file): HTTP \(http.statusCode)") } - guard let resolved = http.url else { + guard let headResolved = http.url else { throw DownloadError.failedToDownload("\(modelId)/\(file): missing resolved URL") } - let size = headerInt64(http, "Content-Length") + var resolved = headResolved + var size = headerInt64(http, "Content-Length") ?? headerInt64(http, "x-linked-size") ?? http.expectedContentLength + if size <= 0 { + var probe = URLRequest(url: url) + probe.setValue("bytes=0-0", forHTTPHeaderField: "Range") + probe.timeoutInterval = 30 + probe.cachePolicy = .reloadIgnoringLocalCacheData + let (probeData, probeResponse) = try await URLSession.shared.data(for: probe) + guard let probeHTTP = probeResponse as? HTTPURLResponse, + probeHTTP.statusCode == 206, + !probeData.isEmpty, + let total = contentRangeTotal(probeHTTP) else { + throw DownloadError.failedToDownload("\(modelId)/\(file): missing Content-Range size") + } + resolved = probeHTTP.url ?? resolved + size = total + } guard size > 0 else { throw DownloadError.failedToDownload("\(modelId)/\(file): unknown remote size") } @@ -681,6 +719,7 @@ private extension HuggingFaceDownloader { progressHandler: ((Double, Int64, Int64, String) -> Void)? ) async throws { var request = URLRequest(url: file.url) + request.timeoutInterval = 120 applyHubAuth(to: &request) let tempURL = destination .deletingLastPathComponent() @@ -815,6 +854,7 @@ private extension HuggingFaceDownloader { ) async throws { var request = URLRequest(url: file.url) request.setValue("bytes=\(chunk.start)-\(chunk.end)", forHTTPHeaderField: "Range") + request.timeoutInterval = 120 applyHubAuth(to: &request) let (data, response) = try await session.data(for: request) diff --git a/Sources/Qwen3TTS/Qwen3TTS.swift b/Sources/Qwen3TTS/Qwen3TTS.swift index b1016c10..289653fb 100644 --- a/Sources/Qwen3TTS/Qwen3TTS.swift +++ b/Sources/Qwen3TTS/Qwen3TTS.swift @@ -1628,14 +1628,24 @@ public extension Qwen3TTSModel { // Download main model weights let mainCacheDir = try cacheDir ?? HuggingFaceDownloader.getCacheDirectory(for: modelId) if !HuggingFaceDownloader.weightsExist(in: mainCacheDir) { - progressHandler?(0.1, "Downloading TTS model weights...") - try await HuggingFaceDownloader.downloadWeights( + progressHandler?(0.1, "Resolving TTS model files...") + try await HuggingFaceDownloader.downloadFilesByteWeighted( modelId: modelId, to: mainCacheDir, - additionalFiles: ["vocab.json", "merges.txt", "tokenizer_config.json"], + files: [ + "config.json", + "merges.txt", + "model.safetensors", + "model.safetensors.index.json", + "tokenizer_config.json", + "vocab.json", + ], offlineMode: offlineMode, - progressHandler: { progress in - progressHandler?(0.1 + progress * 0.3, "Downloading TTS model...") + progressHandler: { progress, _, _, fileName in + let status = fileName == "model.safetensors" + ? "Downloading TTS model weights..." + : "Downloading TTS model..." + progressHandler?(0.1 + progress * 0.3, status) }) } @@ -1643,11 +1653,12 @@ public extension Qwen3TTSModel { let tokenizerCacheDir = try HuggingFaceDownloader.getCacheDirectory(for: tokenizerModelId) if !HuggingFaceDownloader.weightsExist(in: tokenizerCacheDir) { progressHandler?(0.4, "Downloading speech tokenizer...") - try await HuggingFaceDownloader.downloadWeights( + try await HuggingFaceDownloader.downloadFilesByteWeighted( modelId: tokenizerModelId, to: tokenizerCacheDir, + files: ["config.json", "model.safetensors"], offlineMode: offlineMode, - progressHandler: { progress in + progressHandler: { progress, _, _, _ in progressHandler?(0.4 + progress * 0.2, "Downloading speech tokenizer...") }) } diff --git a/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift b/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift index 35aca56b..c46981b2 100644 --- a/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift +++ b/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift @@ -257,6 +257,26 @@ final class HuggingFaceDownloaderTests: XCTestCase { XCTAssertTrue(HuggingFaceDownloader.weightsExist(in: tmpDir)) } + func testContentRangeTotalParsesByteRange() throws { + let response = try XCTUnwrap(HTTPURLResponse( + url: URL(string: "https://example.com/model.safetensors")!, + statusCode: 206, + httpVersion: nil, + headerFields: ["Content-Range": "bytes 0-0/1304461214"])) + + XCTAssertEqual(HuggingFaceDownloader.contentRangeTotal(response), 1_304_461_214) + } + + func testContentRangeTotalRejectsMalformedHeader() throws { + let response = try XCTUnwrap(HTTPURLResponse( + url: URL(string: "https://example.com/model.safetensors")!, + statusCode: 206, + httpVersion: nil, + headerFields: ["Content-Range": "bytes 0-0/*"])) + + XCTAssertNil(HuggingFaceDownloader.contentRangeTotal(response)) + } + // MARK: - Download stall guard /// A stalled operation (reports progress once, then sleeps forever) From 5d8921a1e69385360b741739a2ce43d8ad4f6802 Mon Sep 17 00:00:00 2001 From: "maryna.kryvko" Date: Sun, 26 Jul 2026 16:57:47 +0200 Subject: [PATCH 2/4] Cover Qwen3-TTS download compatibility --- .../AudioCommon/HuggingFaceDownloader.swift | 67 ++++++++++++++----- Sources/Qwen3TTS/Qwen3TTS.swift | 1 + .../HuggingFaceDownloaderTests.swift | 33 +++++++++ Tests/Qwen3TTSTests/Qwen3TTSTests.swift | 22 ++++++ 4 files changed, 105 insertions(+), 18 deletions(-) diff --git a/Sources/AudioCommon/HuggingFaceDownloader.swift b/Sources/AudioCommon/HuggingFaceDownloader.swift index 81de31c8..eb1a3453 100644 --- a/Sources/AudioCommon/HuggingFaceDownloader.swift +++ b/Sources/AudioCommon/HuggingFaceDownloader.swift @@ -5,6 +5,7 @@ import os /// Download errors public enum DownloadError: Error, LocalizedError { case failedToDownload(String) + case remoteFileNotFound(modelId: String, file: String) case invalidRemoteFileName(String) /// A download attempt made no progress for `seconds` and was aborted /// so the caller's retry loop can fire instead of hanging. @@ -14,6 +15,8 @@ public enum DownloadError: Error, LocalizedError { switch self { case .failedToDownload(let file): return "Failed to download: \(file)" + case .remoteFileNotFound(let modelId, let file): + return "Remote file not found: \(modelId)/\(file)" case .invalidRemoteFileName(let file): return "Refusing to write unsafe remote file name: \(file)" case .stalled(let modelId, let seconds): @@ -244,6 +247,7 @@ public enum HuggingFaceDownloader { modelId: String, to directory: URL, files: [String], + optionalFiles: [String] = [], expectedSizes: [String: Int64]? = nil, offlineMode: Bool = false, retryDelaysSeconds: [Int]? = nil, @@ -257,9 +261,15 @@ public enum HuggingFaceDownloader { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) let safeFiles = try files.map(validatedRemoteFileName) + let safeOptionalFiles = Set(try optionalFiles.map(validatedRemoteFileName)) + guard safeOptionalFiles.isSubset(of: Set(safeFiles)) else { + throw DownloadError.failedToDownload( + "optionalFiles must be included in files for \(modelId)") + } if offlineMode { let missing = safeFiles.first { - !FileManager.default.fileExists(atPath: directory.appendingPathComponent($0).path) + !safeOptionalFiles.contains($0) + && !FileManager.default.fileExists(atPath: directory.appendingPathComponent($0).path) } if let missing { throw DownloadError.failedToDownload( @@ -277,6 +287,7 @@ public enum HuggingFaceDownloader { let remoteFiles = try await resolveRemoteFiles( modelId: modelId, files: safeFiles, + optionalFiles: safeOptionalFiles, expectedSizes: expectedSizes) try await downloadResolvedFilesByteWeighted( remoteFiles, @@ -553,6 +564,24 @@ public enum HuggingFaceDownloader { } extension HuggingFaceDownloader { + static func makeHubRequest( + url: URL, + method: String? = nil, + range: String? = nil, + timeout: TimeInterval? = nil + ) -> URLRequest { + var request = URLRequest(url: url) + request.httpMethod = method + if let range { + request.setValue(range, forHTTPHeaderField: "Range") + } + if let timeout { + request.timeoutInterval = timeout + } + applyHubAuth(to: &request) + return request + } + static func contentRangeTotal(_ response: HTTPURLResponse) -> Int64? { for (rawKey, rawValue) in response.allHeaderFields { guard String(describing: rawKey).caseInsensitiveCompare("Content-Range") == .orderedSame else { @@ -581,11 +610,13 @@ private extension HuggingFaceDownloader { static func resolveRemoteFiles( modelId: String, files: [String], + optionalFiles: Set, expectedSizes: [String: Int64]? ) async throws -> [ResolvedRemoteFile] { if let expectedSizes { - return try files.map { file in + return try files.compactMap { file in guard let size = expectedSizes[file], size > 0 else { + if optionalFiles.contains(file) { return nil } throw DownloadError.failedToDownload("\(modelId)/\(file): missing expected size") } return ResolvedRemoteFile( @@ -595,10 +626,14 @@ private extension HuggingFaceDownloader { } } - return try await withThrowingTaskGroup(of: (Int, ResolvedRemoteFile).self) { group in + return try await withThrowingTaskGroup(of: (Int, ResolvedRemoteFile?).self) { group in for (index, file) in files.enumerated() { group.addTask { - (index, try await resolveRemoteFile(modelId: modelId, file: file)) + do { + return (index, try await resolveRemoteFile(modelId: modelId, file: file)) + } catch DownloadError.remoteFileNotFound where optionalFiles.contains(file) { + return (index, nil) + } } } @@ -612,14 +647,14 @@ private extension HuggingFaceDownloader { static func resolveRemoteFile(modelId: String, file: String) async throws -> ResolvedRemoteFile { let url = try resolveURL(modelId: modelId, file: file) - var request = URLRequest(url: url) - request.httpMethod = "HEAD" - request.timeoutInterval = 30 - applyHubAuth(to: &request) + let request = makeHubRequest(url: url, method: "HEAD", timeout: 30) let (_, response) = try await URLSession.shared.data(for: request) guard let http = response as? HTTPURLResponse else { throw DownloadError.failedToDownload("\(modelId)/\(file): missing HTTP response") } + if http.statusCode == 404 { + throw DownloadError.remoteFileNotFound(modelId: modelId, file: file) + } guard (200..<300).contains(http.statusCode) else { throw DownloadError.failedToDownload("\(modelId)/\(file): HTTP \(http.statusCode)") } @@ -631,9 +666,7 @@ private extension HuggingFaceDownloader { ?? headerInt64(http, "x-linked-size") ?? http.expectedContentLength if size <= 0 { - var probe = URLRequest(url: url) - probe.setValue("bytes=0-0", forHTTPHeaderField: "Range") - probe.timeoutInterval = 30 + var probe = makeHubRequest(url: url, range: "bytes=0-0", timeout: 30) probe.cachePolicy = .reloadIgnoringLocalCacheData let (probeData, probeResponse) = try await URLSession.shared.data(for: probe) guard let probeHTTP = probeResponse as? HTTPURLResponse, @@ -718,9 +751,7 @@ private extension HuggingFaceDownloader { totalBytes: Int64, progressHandler: ((Double, Int64, Int64, String) -> Void)? ) async throws { - var request = URLRequest(url: file.url) - request.timeoutInterval = 120 - applyHubAuth(to: &request) + let request = makeHubRequest(url: file.url, timeout: 120) let tempURL = destination .deletingLastPathComponent() .appendingPathComponent(".\(destination.lastPathComponent).download") @@ -852,10 +883,10 @@ private extension HuggingFaceDownloader { state: RangedDownloadProgress, session: URLSession ) async throws { - var request = URLRequest(url: file.url) - request.setValue("bytes=\(chunk.start)-\(chunk.end)", forHTTPHeaderField: "Range") - request.timeoutInterval = 120 - applyHubAuth(to: &request) + let request = makeHubRequest( + url: file.url, + range: "bytes=\(chunk.start)-\(chunk.end)", + timeout: 120) let (data, response) = try await session.data(for: request) guard let http = response as? HTTPURLResponse else { diff --git a/Sources/Qwen3TTS/Qwen3TTS.swift b/Sources/Qwen3TTS/Qwen3TTS.swift index 289653fb..19ec6c62 100644 --- a/Sources/Qwen3TTS/Qwen3TTS.swift +++ b/Sources/Qwen3TTS/Qwen3TTS.swift @@ -1640,6 +1640,7 @@ public extension Qwen3TTSModel { "tokenizer_config.json", "vocab.json", ], + optionalFiles: ["model.safetensors.index.json"], offlineMode: offlineMode, progressHandler: { progress, _, _, fileName in let status = fileName == "model.safetensors" diff --git a/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift b/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift index c46981b2..fc1fd9a9 100644 --- a/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift +++ b/Tests/AudioCommonTests/HuggingFaceDownloaderTests.swift @@ -257,6 +257,39 @@ final class HuggingFaceDownloaderTests: XCTestCase { XCTAssertTrue(HuggingFaceDownloader.weightsExist(in: tmpDir)) } + func testByteWeightedDownloadAllowsMissingOptionalIndexOffline() async throws { + let tmpDir = FileManager.default.temporaryDirectory + .appendingPathComponent("optional_index_\(UUID().uuidString)") + try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmpDir) } + + try Data([0x00]).write(to: tmpDir.appendingPathComponent("model.safetensors")) + + try await HuggingFaceDownloader.downloadFilesByteWeighted( + modelId: "fake/single-file-model", + to: tmpDir, + files: ["model.safetensors", "model.safetensors.index.json"], + optionalFiles: ["model.safetensors.index.json"], + offlineMode: true) + } + + func testHubRequestPropagatesTokenToRangeProbeRequest() { + let previous = ProcessInfo.processInfo.environment["HF_TOKEN"] + setenv("HF_TOKEN", "test-token", 1) + defer { + if let previous { setenv("HF_TOKEN", previous, 1) } + else { unsetenv("HF_TOKEN") } + } + + let request = HuggingFaceDownloader.makeHubRequest( + url: URL(string: "https://huggingface.co/example/model/resolve/main/model.safetensors")!, + range: "bytes=0-0", + timeout: 30) + + XCTAssertEqual(request.value(forHTTPHeaderField: "Authorization"), "Bearer test-token") + XCTAssertEqual(request.value(forHTTPHeaderField: "Range"), "bytes=0-0") + } + func testContentRangeTotalParsesByteRange() throws { let response = try XCTUnwrap(HTTPURLResponse( url: URL(string: "https://example.com/model.safetensors")!, diff --git a/Tests/Qwen3TTSTests/Qwen3TTSTests.swift b/Tests/Qwen3TTSTests/Qwen3TTSTests.swift index bc6bd84a..a4c36a27 100644 --- a/Tests/Qwen3TTSTests/Qwen3TTSTests.swift +++ b/Tests/Qwen3TTSTests/Qwen3TTSTests.swift @@ -1156,6 +1156,28 @@ final class E2ETTS17BTests: XCTestCase { XCTAssertEqual(model.config.talker.hiddenSize, 2048, "Should be 1.7B (hidden=2048)") } + /// A clean cache must load the single-file 1.7B bf16 repository, which has + /// no model.safetensors.index.json. + func testFreshCacheModelLoading17BBf16() async throws { + let cacheDir = FileManager.default.temporaryDirectory + .appendingPathComponent("qwen3-tts-17b-bf16-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: cacheDir) } + + let model = try await Qwen3TTSModel.fromPretrained( + modelId: Self.ttsModelIdBf16, + tokenizerModelId: Self.ttsTokenizerModelId, + cacheDir: cacheDir + ) { progress, status in + print("[TTS-fresh-cache \(Int(progress * 100))%] \(status)") + } + + XCTAssertEqual(model.config.talker.bits, 0, "Should load as bf16 (no quantization)") + XCTAssertTrue(FileManager.default.fileExists( + atPath: cacheDir.appendingPathComponent("model.safetensors").path)) + XCTAssertFalse(FileManager.default.fileExists( + atPath: cacheDir.appendingPathComponent("model.safetensors.index.json").path)) + } + /// 1.7B bf16 -> ASR round-trip func testRoundTrip17BBf16() async throws { let ttsModel = try await loadBf16Model() From 936730ae581f3308c8c6d3de89913b854e1191da Mon Sep 17 00:00:00 2001 From: "maryna.kryvko" Date: Sun, 26 Jul 2026 17:13:06 +0200 Subject: [PATCH 3/4] added IDEA file to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4aa45490..27f190e0 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ benchmark-results/ *.xcodeproj/project.pbxproj .DS_Store .claude/scheduled_tasks.lock +/.idea/speech-swift.git-personal.iml From 1cd436f792c3cd4a4ddee480d3b47bc099d216b0 Mon Sep 17 00:00:00 2001 From: "maryna.kryvko" Date: Sun, 26 Jul 2026 17:49:12 +0200 Subject: [PATCH 4/4] Fix Echo demo one-shot audio --- .../SpeechDemo/EchoMicrophoneGate.swift | 9 ++++++++ .../SpeechDemo/SpeechDemo/EchoViewModel.swift | 14 +++++++++++- .../SpeechDemo/Tests/AudioPlayerTests.swift | 15 +++++++------ .../Tests/EchoMicrophoneGateTests.swift | 22 +++++++++++++++++++ .../AudioCommon/StreamingAudioPlayer.swift | 2 ++ 5 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 Examples/SpeechDemo/SpeechDemo/EchoMicrophoneGate.swift create mode 100644 Examples/SpeechDemo/Tests/EchoMicrophoneGateTests.swift diff --git a/Examples/SpeechDemo/SpeechDemo/EchoMicrophoneGate.swift b/Examples/SpeechDemo/SpeechDemo/EchoMicrophoneGate.swift new file mode 100644 index 00000000..00f62f93 --- /dev/null +++ b/Examples/SpeechDemo/SpeechDemo/EchoMicrophoneGate.swift @@ -0,0 +1,9 @@ +import Foundation + +/// Keeps speaker playback out of the VAD stream while preserving audio timing. +struct EchoMicrophoneGate { + static func samplesToPush(_ samples: [Float], muted: Bool) -> [Float] { + guard muted else { return samples } + return [Float](repeating: 0, count: samples.count) + } +} diff --git a/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift b/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift index 82a1c9c0..a5a83882 100644 --- a/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift +++ b/Examples/SpeechDemo/SpeechDemo/EchoViewModel.swift @@ -30,6 +30,7 @@ final class EchoViewModel { private var debugMicBuffer: [Float] = [] private var debugTTSBuffer: [Float] = [] private var speechStartTime: Date? + private var isSpeaking = false var modelsLoaded: Bool { vad != nil && asr != nil && tts != nil } @@ -96,6 +97,7 @@ final class EchoViewModel { player.onPlaybackFinished = { [weak self] in guard let self, self.isRunning else { return } + self.isSpeaking = false self.pipeline?.resumeListening() self.pipelineState = "listening" self.appendLog("Listening...") @@ -115,6 +117,7 @@ final class EchoViewModel { pipeline?.stop() pipeline = nil isRunning = false + isSpeaking = false pipelineState = "idle" saveDebugFiles() appendLog("Pipeline stopped.") @@ -140,14 +143,19 @@ final class EchoViewModel { } case .transcriptionCompleted(let text, let language, _): pipelineState = "synthesizing..." + if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + isSpeaking = true + } lastTranscription = text lastLanguage = language ?? "" appendLog("[STT\(language.map { " [\($0)]" } ?? "")] \(text)") case .responseCreated: pipelineState = "speaking..." + isSpeaking = true player.resetGeneration() case .responseInterrupted: player.stop() + isSpeaking = false pipelineState = "listening" case .responseAudioDelta(let samples): debugTTSBuffer.append(contentsOf: samples) @@ -159,6 +167,7 @@ final class EchoViewModel { break case .error(let msg): pipelineState = "error" + isSpeaking = false appendLog("[ERROR] \(msg)") pipeline?.resumeListening() } @@ -220,7 +229,10 @@ final class EchoViewModel { } self.debugMicBuffer.append(contentsOf: samples) - self.pipeline?.pushAudio(samples) + // Keep the C++ pipeline's audio clock continuous, but do not let + // speaker playback become a new VAD speech turn. + self.pipeline?.pushAudio( + EchoMicrophoneGate.samplesToPush(samples, muted: self.isSpeaking)) } do { diff --git a/Examples/SpeechDemo/Tests/AudioPlayerTests.swift b/Examples/SpeechDemo/Tests/AudioPlayerTests.swift index 0fb63b80..38824b39 100644 --- a/Examples/SpeechDemo/Tests/AudioPlayerTests.swift +++ b/Examples/SpeechDemo/Tests/AudioPlayerTests.swift @@ -2,6 +2,7 @@ import AVFoundation import XCTest @testable import SpeechDemo +import AudioCommon final class AudioPlayerTests: XCTestCase { @@ -9,7 +10,7 @@ final class AudioPlayerTests: XCTestCase { /// markGenerationComplete with zero pending buffers fires callback (via main queue). func testMarkGenerationCompleteFiresWhenNoPendingBuffers() { - let player = AudioPlayer() + let player = StreamingAudioPlayer() let exp = expectation(description: "playback finished") player.onPlaybackFinished = { exp.fulfill() } @@ -20,7 +21,7 @@ final class AudioPlayerTests: XCTestCase { /// Without markGenerationComplete, callback never fires (even with no buffers). func testNoCallbackWithoutMarkGenerationComplete() { - let player = AudioPlayer() + let player = StreamingAudioPlayer() var finished = false player.onPlaybackFinished = { finished = true } @@ -32,7 +33,7 @@ final class AudioPlayerTests: XCTestCase { /// resetGeneration prevents stale generationComplete from firing. func testResetGenerationClearsFlag() { - let player = AudioPlayer() + let player = StreamingAudioPlayer() var finishCount = 0 let exp = expectation(description: "two finishes") @@ -57,7 +58,7 @@ final class AudioPlayerTests: XCTestCase { /// stop() resets generationComplete, allowing clean next cycle. func testStopResetsGenerationComplete() { - let player = AudioPlayer() + let player = StreamingAudioPlayer() var finishCount = 0 let exp = expectation(description: "two finishes") @@ -82,7 +83,7 @@ final class AudioPlayerTests: XCTestCase { /// Without markGenerationComplete, play() alone never triggers callback. func testRaceConditionPrevented() throws { - let player = AudioPlayer() + let player = StreamingAudioPlayer() var callbackFired = false player.onPlaybackFinished = { callbackFired = true } @@ -105,7 +106,7 @@ final class AudioPlayerTests: XCTestCase { /// Two full cycles back-to-back (simulates two Echo responses). func testTwoCyclesBackToBack() { - let player = AudioPlayer() + let player = StreamingAudioPlayer() var finishCount = 0 player.onPlaybackFinished = { finishCount += 1 } @@ -129,7 +130,7 @@ final class AudioPlayerTests: XCTestCase { /// Interrupt during playback: stop() mid-cycle, then new cycle works. func testInterruptThenNewCycle() { - let player = AudioPlayer() + let player = StreamingAudioPlayer() var finishCount = 0 player.onPlaybackFinished = { finishCount += 1 } diff --git a/Examples/SpeechDemo/Tests/EchoMicrophoneGateTests.swift b/Examples/SpeechDemo/Tests/EchoMicrophoneGateTests.swift new file mode 100644 index 00000000..22ff4614 --- /dev/null +++ b/Examples/SpeechDemo/Tests/EchoMicrophoneGateTests.swift @@ -0,0 +1,22 @@ +#if os(macOS) +import XCTest +@testable import SpeechDemo + +final class EchoMicrophoneGateTests: XCTestCase { + func testMutedInputPreservesTimingWithSilence() { + let samples: [Float] = [0.2, -0.4, 0.6] + + XCTAssertEqual( + EchoMicrophoneGate.samplesToPush(samples, muted: true), + [0, 0, 0]) + } + + func testUnmutedInputPassesThrough() { + let samples: [Float] = [0.2, -0.4, 0.6] + + XCTAssertEqual( + EchoMicrophoneGate.samplesToPush(samples, muted: false), + samples) + } +} +#endif diff --git a/Sources/AudioCommon/StreamingAudioPlayer.swift b/Sources/AudioCommon/StreamingAudioPlayer.swift index 6e8d72d2..53158c73 100644 --- a/Sources/AudioCommon/StreamingAudioPlayer.swift +++ b/Sources/AudioCommon/StreamingAudioPlayer.swift @@ -423,6 +423,7 @@ public final class StreamingAudioPlayer: @unchecked Sendable { totalWritten = 0 totalRead = 0 underflowEvents = 0 + playbackFinishedFired = false ringBuffer?.reset() lock.unlock() isPlaying = false @@ -448,6 +449,7 @@ public final class StreamingAudioPlayer: @unchecked Sendable { totalWritten = 0 totalRead = 0 underflowEvents = 0 + playbackFinishedFired = false ringBuffer?.reset() lock.unlock() isPlaying = false