diff --git a/VoiceAssistant-dev.xcworkspace/contents.xcworkspacedata b/VoiceAssistant-dev.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..eab9f34 --- /dev/null +++ b/VoiceAssistant-dev.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,10 @@ + + + + + + + diff --git a/VoiceAssistant-dev.xcworkspace/xcshareddata/swiftpm/Package.resolved b/VoiceAssistant-dev.xcworkspace/xcshareddata/swiftpm/Package.resolved new file mode 100644 index 0000000..54d4af2 --- /dev/null +++ b/VoiceAssistant-dev.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -0,0 +1,86 @@ +{ + "pins" : [ + { + "identity" : "components-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/components-swift", + "state" : { + "revision" : "f640b51cf38687b77e598df035ce93720d04365a", + "version" : "0.1.1" + } + }, + { + "identity" : "jwt-kit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vapor/jwt-kit.git", + "state" : { + "revision" : "13e7513b3ba0afa13967daf77af2fb4ad087306c", + "version" : "4.13.5" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "ae33e5941bb88d88538d0a6b19ca0b01e6c76dcf", + "version" : "1.3.1" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "ff0f781cf7c6a22d52957e50b104f5768b50c779", + "version" : "3.10.0" + } + }, + { + "identity" : "swift-docc-plugin", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-docc-plugin.git", + "state" : { + "revision" : "85e4bb4e1cd62cec64a4b8e769dcefdf0c5b9d64", + "version" : "1.4.3" + } + }, + { + "identity" : "swift-docc-symbolkit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-docc-symbolkit", + "state" : { + "revision" : "b45d1f2ed151d057b54504d653e0da5552844e34", + "version" : "1.0.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "96a2f8a0fa41e9e09af4585e2724c4e825410b91", + "version" : "1.6.2" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "ebc7251dd5b37f627c93698e4374084d98409633", + "version" : "1.28.2" + } + }, + { + "identity" : "webrtc-xcframework", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/webrtc-xcframework.git", + "state" : { + "revision" : "404510a224284954dc6edcabfe135a7e5dbf580b", + "version" : "125.6422.12" + } + } + ], + "version" : 2 +} diff --git a/VoiceAssistant.xcodeproj/project.pbxproj b/VoiceAssistant.xcodeproj/project.pbxproj index e43030f..6337bf7 100644 --- a/VoiceAssistant.xcodeproj/project.pbxproj +++ b/VoiceAssistant.xcodeproj/project.pbxproj @@ -7,7 +7,6 @@ objects = { /* Begin PBXBuildFile section */ - B577A98A2D16796000B59A0B /* LiveKitKrispNoiseFilter in Frameworks */ = {isa = PBXBuildFile; platformFilters = (ios, macos, ); productRef = B577A9892D16796000B59A0B /* LiveKitKrispNoiseFilter */; }; B5E1B90F2D14E9EC00A38CB6 /* LiveKitComponents in Frameworks */ = {isa = PBXBuildFile; productRef = B5E1B90E2D14E9EC00A38CB6 /* LiveKitComponents */; }; B5E1B9122D14E9F500A38CB6 /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = B5E1B9112D14E9F500A38CB6 /* LiveKit */; }; /* End PBXBuildFile section */ @@ -20,8 +19,6 @@ B577A91F2D15FDB800B59A0B /* Exceptions for "VoiceAssistant" folder in "VoiceAssistant" target */ = { isa = PBXFileSystemSynchronizedBuildFileExceptionSet; membershipExceptions = ( - .env.example.xcconfig, - .env.xcconfig, Info.plist, VoiceAssistant.xcconfig, ); @@ -30,6 +27,11 @@ /* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ + 68AB06302D42885900C61299 /* VoiceAssistantTests */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = VoiceAssistantTests; + sourceTree = ""; + }; B5B5E3B42D124AE00099C9BE /* VoiceAssistant */ = { isa = PBXFileSystemSynchronizedRootGroup; exceptions = ( @@ -45,7 +47,6 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - B577A98A2D16796000B59A0B /* LiveKitKrispNoiseFilter in Frameworks */, B5E1B90F2D14E9EC00A38CB6 /* LiveKitComponents in Frameworks */, B5E1B9122D14E9F500A38CB6 /* LiveKit in Frameworks */, ); @@ -58,6 +59,7 @@ isa = PBXGroup; children = ( B5B5E3B42D124AE00099C9BE /* VoiceAssistant */, + 68AB06302D42885900C61299 /* VoiceAssistantTests */, B5B5E3B32D124AE00099C9BE /* Products */, ); sourceTree = ""; @@ -92,7 +94,6 @@ packageProductDependencies = ( B5E1B90E2D14E9EC00A38CB6 /* LiveKitComponents */, B5E1B9112D14E9F500A38CB6 /* LiveKit */, - B577A9892D16796000B59A0B /* LiveKitKrispNoiseFilter */, ); productName = VoiceAssistant; productReference = B5B5E3B22D124AE00099C9BE /* VoiceAssistant.app */; @@ -125,7 +126,6 @@ packageReferences = ( B5E1B90D2D14E9EC00A38CB6 /* XCRemoteSwiftPackageReference "components-swift" */, B5E1B9102D14E9F500A38CB6 /* XCRemoteSwiftPackageReference "client-sdk-swift" */, - B577A9882D16796000B59A0B /* XCLocalSwiftPackageReference "../../livekit/swift-krisp-noise-filter" */, ); preferredProjectObjectVersion = 77; productRefGroup = B5B5E3B32D124AE00099C9BE /* Products */; @@ -308,6 +308,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; + SUPPORTS_MACCATALYST = YES; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2,7"; @@ -351,6 +352,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; + SUPPORTS_MACCATALYST = YES; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2,7"; @@ -381,13 +383,6 @@ }; /* End XCConfigurationList section */ -/* Begin XCLocalSwiftPackageReference section */ - B577A9882D16796000B59A0B /* XCLocalSwiftPackageReference "../../livekit/swift-krisp-noise-filter" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = "../../livekit/swift-krisp-noise-filter"; - }; -/* End XCLocalSwiftPackageReference section */ - /* Begin XCRemoteSwiftPackageReference section */ B5E1B90D2D14E9EC00A38CB6 /* XCRemoteSwiftPackageReference "components-swift" */ = { isa = XCRemoteSwiftPackageReference; @@ -401,17 +396,13 @@ isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/livekit/client-sdk-swift"; requirement = { - kind = upToNextMajorVersion; - minimumVersion = 2.0.19; + branch = "hiroshi/adm-audioengine2"; + kind = branch; }; }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ - B577A9892D16796000B59A0B /* LiveKitKrispNoiseFilter */ = { - isa = XCSwiftPackageProductDependency; - productName = LiveKitKrispNoiseFilter; - }; B5E1B90E2D14E9EC00A38CB6 /* LiveKitComponents */ = { isa = XCSwiftPackageProductDependency; package = B5E1B90D2D14E9EC00A38CB6 /* XCRemoteSwiftPackageReference "components-swift" */; diff --git a/VoiceAssistant.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/VoiceAssistant.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 7acd0b3..29acc0a 100644 --- a/VoiceAssistant.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/VoiceAssistant.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -4,10 +4,10 @@ { "identity" : "client-sdk-swift", "kind" : "remoteSourceControl", - "location" : "https://github.com/livekit/client-sdk-swift.git", + "location" : "https://github.com/livekit/client-sdk-swift", "state" : { - "revision" : "9c5242f0217f59f5f30fd122e7ac509dcf446201", - "version" : "2.0.19" + "branch" : "hiroshi/adm-audioengine2", + "revision" : "4b84621d2ff4f41b63badf2fe17201eb575dfe64" } }, { @@ -42,8 +42,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/webrtc-xcframework.git", "state" : { - "revision" : "e43fe60fddd27e2d88ba0568ac2a2a15591b3063", - "version" : "125.6422.11" + "revision" : "95e2365aa1623c3fe14ad7eaf18a8a0cb5e46c23", + "version" : "125.6422.12-exp.4" } } ], diff --git a/VoiceAssistant/ContentView.swift b/VoiceAssistant/ContentView.swift index 43f566a..7b1dc3a 100644 --- a/VoiceAssistant/ContentView.swift +++ b/VoiceAssistant/ContentView.swift @@ -1,26 +1,13 @@ import LiveKit import SwiftUI -#if os(iOS) || os(macOS) -import LiveKitKrispNoiseFilter -#endif struct ContentView: View { @StateObject private var room = Room() + @State private var isUserMutedTalking: Bool = false -// Krisp is available only on iOS and macOS right now -// Krisp is also a feature of LiveKit Cloud, so if you're using open-source / self-hosted you should remove this -#if os(iOS) || os(macOS) - private let krispProcessor = LiveKitKrispNoiseFilter() -#endif - - init() { -#if os(iOS) || os(macOS) - AudioManager.shared.capturePostProcessingDelegate = krispProcessor -#endif - } - var body: some View { VStack(spacing: 24) { + Text(isUserMutedTalking ? "Are you talking? Your mic is muted" : "") StatusView() .frame(height: 256) .frame(maxWidth: 512) @@ -30,9 +17,11 @@ struct ContentView: View { .padding() .environmentObject(room) .onAppear { -#if os(iOS) || os(macOS) - room.add(delegate: krispProcessor) -#endif + AudioManager.shared.onMutedSpeechActivity = { _, event in + DispatchQueue.main.async { + isUserMutedTalking = event == .started + } + } } } } diff --git a/VoiceAssistant/ControlBar.swift b/VoiceAssistant/ControlBar.swift index 3b605dd..9ecdaaf 100644 --- a/VoiceAssistant/ControlBar.swift +++ b/VoiceAssistant/ControlBar.swift @@ -2,6 +2,7 @@ import LiveKit import LiveKitComponents import SwiftUI +import Combine /// The ControlBar component handles connection, disconnection, and audio controls /// You can customize this component to fit your app's needs @@ -18,6 +19,8 @@ struct ControlBar: View { // Namespace for view transitions @Namespace private var animation + @State private var cancelSink: AnyCancellable? + @State private var agentState: AgentState = .initializing // These are the overall configurations for this component, based on current app state private enum Configuration { @@ -91,8 +94,24 @@ struct ControlBar: View { Spacer() } .animation(.spring(duration: 0.3), value: currentConfiguration) - } + .onAppear { + cancelSink = room.objectWillChange.sink { _ in + let newAgentState = room.remoteParticipants.first?.value.agentState + guard let newAgentState else { return } + Task { @MainActor in + let oldAgentState = self.agentState + guard newAgentState != oldAgentState else { return } + // Unmute mic when agent starts listening + if newAgentState == .listening { + try await room.localParticipant.setMicrophone(enabled: true) + } + + self.agentState = newAgentState + } + } + } + } /// Fetches a token and connects to the LiveKit room /// This assumes the agent is running and is configured to automatically join new rooms private func connect() { @@ -115,7 +134,8 @@ struct ControlBar: View { // Connect to the room and enable the microphone try await room.connect( url: connectionDetails.serverUrl, token: connectionDetails.participantToken) - try await room.localParticipant.setMicrophone(enabled: true) + // Don't enable mic asap. + // try await room.localParticipant.setMicrophone(enabled: true) } else { print("Failed to fetch connection details") } diff --git a/VoiceAssistant/VoiceAssistantApp.swift b/VoiceAssistant/VoiceAssistantApp.swift index e8a4041..79ee93e 100644 --- a/VoiceAssistant/VoiceAssistantApp.swift +++ b/VoiceAssistant/VoiceAssistantApp.swift @@ -1,9 +1,15 @@ import SwiftUI +import LiveKit @main struct VoiceAssistantApp: App { private var tokenService: TokenService = .init() + init() { + LiveKitSDK.setLoggerStandardOutput() + AudioManager.shared.isRecordingAlwaysPrepared = true + } + var body: some Scene { WindowGroup { ContentView()