diff --git a/LLMonFHIR.xcodeproj/project.pbxproj b/LLMonFHIR.xcodeproj/project.pbxproj index 1248893..abc6068 100644 --- a/LLMonFHIR.xcodeproj/project.pbxproj +++ b/LLMonFHIR.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 60; + objectVersion = 55; objects = { /* Begin PBXBuildFile section */ @@ -27,7 +27,6 @@ 2FC186092AD52FFF0065EBB2 /* Localizable.xcstrings in Resources */ = {isa = PBXBuildFile; fileRef = 2FC186082AD52FFF0065EBB2 /* Localizable.xcstrings */; }; 2FC975A82978F11A00BA99FE /* Home.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FC975A72978F11A00BA99FE /* Home.swift */; }; 2FD024892B116EEF009A682C /* FHIRInterpretationFunction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FD024882B116EEF009A682C /* FHIRInterpretationFunction.swift */; }; - 2FD0248B2B116EF6009A682C /* LLMStreamResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FD0248A2B116EF6009A682C /* LLMStreamResult.swift */; }; 2FD024912B1171EC009A682C /* Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json in Resources */ = {isa = PBXBuildFile; fileRef = 2FD0248D2B1171EC009A682C /* Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json */; }; 2FD024922B1171EC009A682C /* Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json in Resources */ = {isa = PBXBuildFile; fileRef = 2FD0248E2B1171EC009A682C /* Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json */; }; 2FD8E8272A1AAD9B00357F4E /* LLMonFHIRStandard.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FD8E8262A1AAD9B00357F4E /* LLMonFHIRStandard.swift */; }; @@ -63,6 +62,12 @@ 9775720D2B5E718A00FB0286 /* SpeziSpeechRecognizer in Frameworks */ = {isa = PBXBuildFile; productRef = 9775720C2B5E718A00FB0286 /* SpeziSpeechRecognizer */; }; 9775720F2B5E718A00FB0286 /* SpeziSpeechSynthesizer in Frameworks */ = {isa = PBXBuildFile; productRef = 9775720E2B5E718A00FB0286 /* SpeziSpeechSynthesizer */; }; 977572122B5E721F00FB0286 /* SpeziChat in Frameworks */ = {isa = PBXBuildFile; productRef = 977572112B5E721F00FB0286 /* SpeziChat */; }; + 979825532B79717200335095 /* SpeziLLM in Frameworks */ = {isa = PBXBuildFile; productRef = 979825522B79717200335095 /* SpeziLLM */; }; + 979825552B79717200335095 /* SpeziLLMOpenAI in Frameworks */ = {isa = PBXBuildFile; productRef = 979825542B79717200335095 /* SpeziLLMOpenAI */; }; + 979825582B79719B00335095 /* SpeziFHIR in Frameworks */ = {isa = PBXBuildFile; productRef = 979825572B79719B00335095 /* SpeziFHIR */; }; + 9798255A2B79719B00335095 /* SpeziFHIRHealthKit in Frameworks */ = {isa = PBXBuildFile; productRef = 979825592B79719B00335095 /* SpeziFHIRHealthKit */; }; + 9798255C2B79719B00335095 /* SpeziFHIRInterpretation in Frameworks */ = {isa = PBXBuildFile; productRef = 9798255B2B79719B00335095 /* SpeziFHIRInterpretation */; }; + 9798255E2B79719B00335095 /* SpeziFHIRMockPatients in Frameworks */ = {isa = PBXBuildFile; productRef = 9798255D2B79719B00335095 /* SpeziFHIRMockPatients */; }; 97CAB1F72B64A03600D646CE /* SpeziLLM in Frameworks */ = {isa = PBXBuildFile; productRef = 97CAB1F62B64A03600D646CE /* SpeziLLM */; }; 97CAB1F92B64A03600D646CE /* SpeziLLMOpenAI in Frameworks */ = {isa = PBXBuildFile; productRef = 97CAB1F82B64A03600D646CE /* SpeziLLMOpenAI */; }; /* End PBXBuildFile section */ @@ -106,7 +111,6 @@ 2FC94CD4298B0A1D009C8209 /* LLMonFHIR.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = LLMonFHIR.xctestplan; sourceTree = ""; }; 2FC975A72978F11A00BA99FE /* Home.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Home.swift; sourceTree = ""; }; 2FD024882B116EEF009A682C /* FHIRInterpretationFunction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FHIRInterpretationFunction.swift; sourceTree = ""; }; - 2FD0248A2B116EF6009A682C /* LLMStreamResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMStreamResult.swift; sourceTree = ""; }; 2FD0248D2B1171EC009A682C /* Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = "Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json"; sourceTree = ""; }; 2FD0248E2B1171EC009A682C /* Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = "Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json"; sourceTree = ""; }; 2FD8E8262A1AAD9B00357F4E /* LLMonFHIRStandard.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LLMonFHIRStandard.swift; sourceTree = ""; }; @@ -139,18 +143,24 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 9798255A2B79719B00335095 /* SpeziFHIRHealthKit in Frameworks */, 97CAB1F72B64A03600D646CE /* SpeziLLM in Frameworks */, 9733FC592B60E5FB0024F12C /* SpeziFHIRHealthKit in Frameworks */, 9775720F2B5E718A00FB0286 /* SpeziSpeechSynthesizer in Frameworks */, 2FD8E82C2A1AADDA00357F4E /* ModelsR4 in Frameworks */, 2FD8E82A2A1AADDA00357F4E /* ModelsDSTU2 in Frameworks */, 2FD8E8402A1AE3F200357F4E /* SpeziViews in Frameworks */, + 979825552B79717200335095 /* SpeziLLMOpenAI in Frameworks */, + 979825532B79717200335095 /* SpeziLLM in Frameworks */, 2FE5DC7229EDD8D3004B9AB4 /* SpeziHealthKit in Frameworks */, + 9798255C2B79719B00335095 /* SpeziFHIRInterpretation in Frameworks */, 2F49B7762980407C00BCB272 /* Spezi in Frameworks */, 9775720D2B5E718A00FB0286 /* SpeziSpeechRecognizer in Frameworks */, + 9798255E2B79719B00335095 /* SpeziFHIRMockPatients in Frameworks */, 2FD8E8312A1AB00D00357F4E /* HealthKitOnFHIR in Frameworks */, 97CAB1F92B64A03600D646CE /* SpeziLLMOpenAI in Frameworks */, 9733FC5D2B60E5FB0024F12C /* SpeziFHIRMockPatients in Frameworks */, + 979825582B79719B00335095 /* SpeziFHIR in Frameworks */, 977572122B5E721F00FB0286 /* SpeziChat in Frameworks */, 2FE5DC8129EDD91D004B9AB4 /* SpeziOnboarding in Frameworks */, 9733FC5B2B60E5FB0024F12C /* SpeziFHIRInterpretation in Frameworks */, @@ -225,7 +235,6 @@ 2F036EA02B0E4B22009B2745 /* FHIRInterpretationModule.swift */, 433352432A5C96090043A440 /* FHIRMultipleResourceInterpreter.swift */, 2FD024882B116EEF009A682C /* FHIRInterpretationFunction.swift */, - 2FD0248A2B116EF6009A682C /* LLMStreamResult.swift */, ); path = "FHIR Interpretation"; sourceTree = ""; @@ -384,6 +393,12 @@ 9733FC5C2B60E5FB0024F12C /* SpeziFHIRMockPatients */, 97CAB1F62B64A03600D646CE /* SpeziLLM */, 97CAB1F82B64A03600D646CE /* SpeziLLMOpenAI */, + 979825522B79717200335095 /* SpeziLLM */, + 979825542B79717200335095 /* SpeziLLMOpenAI */, + 979825572B79719B00335095 /* SpeziFHIR */, + 979825592B79719B00335095 /* SpeziFHIRHealthKit */, + 9798255B2B79719B00335095 /* SpeziFHIRInterpretation */, + 9798255D2B79719B00335095 /* SpeziFHIRMockPatients */, ); productName = LLMonFHIR; productReference = 653A254D283387FE005D4D48 /* LLMonFHIR.app */; @@ -476,8 +491,8 @@ 2FD8E83E2A1AE3F200357F4E /* XCRemoteSwiftPackageReference "SpeziViews" */, 9775720B2B5E718A00FB0286 /* XCRemoteSwiftPackageReference "SpeziSpeech" */, 977572102B5E721F00FB0286 /* XCRemoteSwiftPackageReference "SpeziChat" */, - 9733FC552B60E5FB0024F12C /* XCLocalSwiftPackageReference "../SpeziFHIR" */, - 97CAB1F52B64A03600D646CE /* XCLocalSwiftPackageReference "../SpeziLLM" */, + 979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */, + 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */, ); productRefGroup = 653A254E283387FE005D4D48 /* Products */; projectDirPath = ""; @@ -571,7 +586,6 @@ 2F4E23832989D51F0013F3D9 /* LLMonFHIRTestingSetup.swift in Sources */, 433352442A5C96090043A440 /* FHIRMultipleResourceInterpreter.swift in Sources */, 2FD8E8272A1AAD9B00357F4E /* LLMonFHIRStandard.swift in Sources */, - 2FD0248B2B116EF6009A682C /* LLMStreamResult.swift in Sources */, 2F5E32BD297E05EA003432F8 /* LLMonFHIRDelegate.swift in Sources */, 653A2551283387FE005D4D48 /* LLMonFHIR.swift in Sources */, 2F036EA52B0ED1F0009B2745 /* FHIRResourcesInstructionsView.swift in Sources */, @@ -1093,17 +1107,6 @@ }; /* End XCConfigurationList section */ -/* Begin XCLocalSwiftPackageReference section */ - 9733FC552B60E5FB0024F12C /* XCLocalSwiftPackageReference "../SpeziFHIR" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = ../SpeziFHIR; - }; - 97CAB1F52B64A03600D646CE /* XCLocalSwiftPackageReference "../SpeziLLM" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = ../SpeziLLM; - }; -/* End XCLocalSwiftPackageReference section */ - /* Begin XCRemoteSwiftPackageReference section */ 2F49B7742980407B00BCB272 /* XCRemoteSwiftPackageReference "Spezi" */ = { isa = XCRemoteSwiftPackageReference; @@ -1185,6 +1188,22 @@ minimumVersion = 0.1.4; }; }; + 979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/StanfordSpezi/SpeziLLM"; + requirement = { + branch = "feat/structural-improvments"; + kind = branch; + }; + }; + 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/StanfordSpezi/SpeziFHIR"; + requirement = { + branch = "feat/lift-to-spezi-llm"; + kind = branch; + }; + }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ @@ -1264,6 +1283,36 @@ package = 977572102B5E721F00FB0286 /* XCRemoteSwiftPackageReference "SpeziChat" */; productName = SpeziChat; }; + 979825522B79717200335095 /* SpeziLLM */ = { + isa = XCSwiftPackageProductDependency; + package = 979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */; + productName = SpeziLLM; + }; + 979825542B79717200335095 /* SpeziLLMOpenAI */ = { + isa = XCSwiftPackageProductDependency; + package = 979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */; + productName = SpeziLLMOpenAI; + }; + 979825572B79719B00335095 /* SpeziFHIR */ = { + isa = XCSwiftPackageProductDependency; + package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */; + productName = SpeziFHIR; + }; + 979825592B79719B00335095 /* SpeziFHIRHealthKit */ = { + isa = XCSwiftPackageProductDependency; + package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */; + productName = SpeziFHIRHealthKit; + }; + 9798255B2B79719B00335095 /* SpeziFHIRInterpretation */ = { + isa = XCSwiftPackageProductDependency; + package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */; + productName = SpeziFHIRInterpretation; + }; + 9798255D2B79719B00335095 /* SpeziFHIRMockPatients */ = { + isa = XCSwiftPackageProductDependency; + package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */; + productName = SpeziFHIRMockPatients; + }; 97CAB1F62B64A03600D646CE /* SpeziLLM */ = { isa = XCSwiftPackageProductDependency; productName = SpeziLLM; diff --git a/LLMonFHIR.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/LLMonFHIR.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 6b42f31..1718e18 100644 --- a/LLMonFHIR.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/LLMonFHIR.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -32,8 +32,16 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/MacPaw/OpenAI", "state" : { - "revision" : "ac5892fd0de8d283362ddc30f8e9f1a0eaba8cc0", - "version" : "0.2.5" + "revision" : "35afc9a6ee127b8f22a85a31aec2036a987478af" + } + }, + { + "identity" : "semaphore", + "kind" : "remoteSourceControl", + "location" : "https://github.com/groue/Semaphore.git", + "state" : { + "revision" : "f1c4a0acabeb591068dea6cffdd39660b86dec28", + "version" : "0.0.8" } }, { @@ -54,6 +62,15 @@ "version" : "0.1.5" } }, + { + "identity" : "spezifhir", + "kind" : "remoteSourceControl", + "location" : "https://github.com/StanfordSpezi/SpeziFHIR", + "state" : { + "branch" : "feat/lift-to-spezi-llm", + "revision" : "b58ba7afc34ff1b83d4f4ab3525d3d13f372b080" + } + }, { "identity" : "spezifoundation", "kind" : "remoteSourceControl", @@ -72,6 +89,15 @@ "version" : "0.5.0" } }, + { + "identity" : "spezillm", + "kind" : "remoteSourceControl", + "location" : "https://github.com/StanfordSpezi/SpeziLLM", + "state" : { + "branch" : "feat/structural-improvments", + "revision" : "f30f4d311cf99a396e76e5b732dca6462463f663" + } + }, { "identity" : "spezionboarding", "kind" : "remoteSourceControl", @@ -93,7 +119,7 @@ { "identity" : "spezistorage", "kind" : "remoteSourceControl", - "location" : "https://github.com/StanfordSpezi/SpeziStorage.git", + "location" : "https://github.com/StanfordSpezi/SpeziStorage", "state" : { "revision" : "eaed2220375c35400aa69d1f96a8d32b7e66b1c7", "version" : "1.0.0" diff --git a/LLMonFHIR/FHIR Display/FHIRResourcesView.swift b/LLMonFHIR/FHIR Display/FHIRResourcesView.swift index 3e3fbb4..14f470c 100644 --- a/LLMonFHIR/FHIR Display/FHIRResourcesView.swift +++ b/LLMonFHIR/FHIR Display/FHIRResourcesView.swift @@ -9,8 +9,8 @@ import ModelsR4 import SpeziFHIR import SpeziFHIRInterpretation -import SpeziOnboarding import SpeziLLMOpenAI +import SpeziOnboarding import SpeziViews import SwiftUI diff --git a/LLMonFHIR/FHIR Display/InspectResourceView.swift b/LLMonFHIR/FHIR Display/InspectResourceView.swift index 03b10a9..b963955 100644 --- a/LLMonFHIR/FHIR Display/InspectResourceView.swift +++ b/LLMonFHIR/FHIR Display/InspectResourceView.swift @@ -8,6 +8,7 @@ import SpeziFHIR import SpeziFHIRInterpretation +import SpeziLLM import SpeziViews import SwiftUI @@ -114,10 +115,10 @@ struct InspectResourceView: View { do { try await fhirResourceSummary.summarize(resource: resource, forceReload: forceReload) loadingSummary = .idle - }/* catch let error as APIErrorResponse { + } catch let error as LLMError { loadingSummary = .error(error) - }*/ catch { - loadingSummary = .error("Unknown error") + } catch { + loadingSummary = .error("Unknown LLM processing error") } } } @@ -129,10 +130,10 @@ struct InspectResourceView: View { do { try await fhirResourceInterpreter.interpret(resource: resource, forceReload: forceReload) interpreting = .idle - } /*catch let error as APIErrorResponse { - loadingSummary = .error(error) - }*/ catch { - loadingSummary = .error("Unknown error") + } catch let error as LLMError { + interpreting = .error(error) + } catch { + interpreting = .error("Unknown LLM processing error") } } } diff --git a/LLMonFHIR/FHIR Display/MultipleResourcesChatView.swift b/LLMonFHIR/FHIR Display/MultipleResourcesChatView.swift index 2209389..e2cdb35 100644 --- a/LLMonFHIR/FHIR Display/MultipleResourcesChatView.swift +++ b/LLMonFHIR/FHIR Display/MultipleResourcesChatView.swift @@ -27,18 +27,27 @@ struct MultipleResourcesChatView: View { var body: some View { @Bindable var multipleResourceInterpreter = multipleResourceInterpreter NavigationStack { - ChatView( - $multipleResourceInterpreter.llm.context, - disableInput: multipleResourceInterpreter.viewState == .processing - ) + Group { + if let llm = multipleResourceInterpreter.llm { + let contextBinding = Binding { llm.context } set: { llm.context = $0 } + ChatView( + contextBinding, + disableInput: multipleResourceInterpreter.viewState == .processing + ) + .onChange(of: llm.context) { + multipleResourceInterpreter.queryLLM() + } + } else { + ChatView( + .constant([]) + ) + } + } .navigationTitle("LLM on FHIR") .toolbar { toolbar } .viewStateAlert(state: $multipleResourceInterpreter.viewState) - .onChange(of: multipleResourceInterpreter.llm.context) { - multipleResourceInterpreter.queryLLM() - } .onAppear { multipleResourceInterpreter.queryLLM() } @@ -47,8 +56,7 @@ struct MultipleResourcesChatView: View { } - @MainActor - @ToolbarContentBuilder private var toolbar: some ToolbarContent { + @MainActor @ToolbarContentBuilder private var toolbar: some ToolbarContent { ToolbarItem(placement: .cancellationAction) { if multipleResourceInterpreter.viewState == .processing { ProgressView() diff --git a/LLMonFHIR/FHIR Interpretation/FHIRInterpretationFunction.swift b/LLMonFHIR/FHIR Interpretation/FHIRInterpretationFunction.swift index 8d22671..5b8b6f5 100644 --- a/LLMonFHIR/FHIR Interpretation/FHIRInterpretationFunction.swift +++ b/LLMonFHIR/FHIR Interpretation/FHIRInterpretationFunction.swift @@ -7,9 +7,9 @@ // import os -import SpeziLLMOpenAI import SpeziFHIR import SpeziFHIRInterpretation +import SpeziLLMOpenAI struct FHIRInterpretationFunction: LLMFunction { @@ -23,8 +23,7 @@ struct FHIRInterpretationFunction: LLMFunction { private let allResourcesFunctionCallIdentifier: [String] - @Parameter - var resources: [String] + @Parameter var resources: [String] init(fhirStore: FHIRStore, resourceSummary: FHIRResourceSummary, allResourcesFunctionCallIdentifier: [String]) { @@ -65,7 +64,7 @@ struct FHIRInterpretationFunction: LLMFunction { // Iterate over fitting resources and summarizing them for resource in fittingResources { innerGroup.addTask { - return try await summarizeResource(fhirResource: resource, resourceType: requestedResource) + try await summarizeResource(fhirResource: resource, resourceType: requestedResource) } } @@ -88,7 +87,7 @@ struct FHIRInterpretationFunction: LLMFunction { private func summarizeResource(fhirResource: FHIRResource, resourceType: String) async throws -> String { let summary = try await resourceSummary.summarize(resource: fhirResource) - Self.logger.debug("Summary of appended resource: \(summary)") + Self.logger.debug("Summary of appended FHIR resource \(resourceType): \(summary.description)") return String(localized: "This is the summary of the requested \(resourceType):\n\n\(summary.description)") } @@ -102,7 +101,8 @@ struct FHIRInterpretationFunction: LLMFunction { Self.logger.debug( """ Reduced to the following 64 resources: \(fittingResources.map { $0.functionCallIdentifier }.joined(separator: ",")) - """) + """ + ) } return fittingResources diff --git a/LLMonFHIR/FHIR Interpretation/FHIRInterpretationModule.swift b/LLMonFHIR/FHIR Interpretation/FHIRInterpretationModule.swift index 7b3e041..021578d 100644 --- a/LLMonFHIR/FHIR Interpretation/FHIRInterpretationModule.swift +++ b/LLMonFHIR/FHIR Interpretation/FHIRInterpretationModule.swift @@ -10,7 +10,7 @@ import Spezi import SpeziFHIR import SpeziFHIRInterpretation import SpeziLLM -import class SpeziLLMOpenAI.LLMOpenAI +import SpeziLLMOpenAI import SpeziLocalStorage import SwiftUI @@ -26,12 +26,12 @@ class FHIRInterpretationModule: Module { func configure() { - let openAIModelType = UserDefaults.standard.string(forKey: StorageKeys.openAIModel) ?? .gpt4_1106_preview + let openAIModelType = UserDefaults.standard.string(forKey: StorageKeys.openAIModel) ?? .gpt4_turbo_preview resourceSummary = FHIRResourceSummary( localStorage: localStorage, llmRunner: llmRunner, - llm: LLMOpenAI( + llmSchema: LLMOpenAISchema( parameters: .init( modelType: openAIModelType, systemPrompt: nil // No system prompt as this will be determined later by the resource interpreter @@ -42,7 +42,7 @@ class FHIRInterpretationModule: Module { resourceInterpreter = FHIRResourceInterpreter( localStorage: localStorage, llmRunner: llmRunner, - llm: LLMOpenAI( + llmSchema: LLMOpenAISchema( parameters: .init( modelType: openAIModelType, systemPrompt: nil // No system prompt as this will be determined later by the resource interpreter @@ -53,12 +53,18 @@ class FHIRInterpretationModule: Module { multipleResourceInterpreter = FHIRMultipleResourceInterpreter( localStorage: localStorage, llmRunner: llmRunner, - llm: LLMOpenAI( + llmSchema: LLMOpenAISchema( parameters: .init( modelType: openAIModelType, systemPrompt: nil // No system prompt as this will be determined later by the resource interpreter ) - ), + ) { + FHIRInterpretationFunction( + fhirStore: self.fhirStore, + resourceSummary: self.resourceSummary, + allResourcesFunctionCallIdentifier: self.fhirStore.allResourcesFunctionCallIdentifier + ) + }, fhirStore: fhirStore, resourceSummary: resourceSummary ) diff --git a/LLMonFHIR/FHIR Interpretation/FHIRMultipleResourceInterpreter.swift b/LLMonFHIR/FHIR Interpretation/FHIRMultipleResourceInterpreter.swift index 2801a8f..a47c9dd 100644 --- a/LLMonFHIR/FHIR Interpretation/FHIRMultipleResourceInterpreter.swift +++ b/LLMonFHIR/FHIR Interpretation/FHIRMultipleResourceInterpreter.swift @@ -25,41 +25,51 @@ private enum FHIRMultipleResourceInterpreterConstants { class FHIRMultipleResourceInterpreter { private let localStorage: LocalStorage private let llmRunner: LLMRunner + private let llmSchema: any LLMSchema private let fhirStore: FHIRStore private let resourceSummary: FHIRResourceSummary @MainActor var viewState: ViewState = .idle - var llm: any LLM + var llm: (any LLMSession)? - required init(localStorage: LocalStorage, llmRunner: LLMRunner, llm: any LLM, fhirStore: FHIRStore, resourceSummary: FHIRResourceSummary) { + required init( + localStorage: LocalStorage, + llmRunner: LLMRunner, + llmSchema: any LLMSchema, + fhirStore: FHIRStore, + resourceSummary: FHIRResourceSummary + ) { self.localStorage = localStorage self.llmRunner = llmRunner - self.llm = llm + self.llmSchema = llmSchema self.fhirStore = fhirStore self.resourceSummary = resourceSummary - - Task { @MainActor in - llm.context = (try? localStorage.read(storageKey: FHIRMultipleResourceInterpreterConstants.chat)) ?? [] - } } @MainActor func resetChat() { - llm.context = [] + llm?.context = [] queryLLM() } - - // TODO: Can we get rid of this main actor annotation? + @MainActor func queryLLM() { - guard viewState == .idle, llm.context.last?.role == .user || !llm.context.contains(where: { $0.role == .assistant }) else { + guard viewState == .idle, llm?.context.last?.role == .user || !(llm?.context.contains(where: { $0.role == .assistant }) ?? false) else { return } Task { do { + var llm: LLMSession + if let llmTemp = self.llm { + llm = llmTemp + } else { + llm = await llmRunner(with: llmSchema) + self.llm = llm + } + viewState = .processing if llm.context.isEmpty { @@ -72,9 +82,8 @@ class FHIRMultipleResourceInterpreter { print("The Multiple Resource Interpreter has access to \(fhirStore.llmRelevantResources.count) resources.") - //try await executeLLMQueries() do { - let stream = try await llmRunner(with: llm).generate() + let stream = try await llm.generate() for try await token in stream { llm.context.append(assistantOutput: token) @@ -82,134 +91,17 @@ class FHIRMultipleResourceInterpreter { } catch let error as LLMError { llm.state = .error(error: error) } catch { - llm.state = .error(error: LLMRunnerError.setupError) + llm.state = .error(error: LLMDefaultError.unknown(error)) } try localStorage.store(llm.context, storageKey: FHIRMultipleResourceInterpreterConstants.chat) viewState = .idle - } - /* Do we need something like that? - catch let error as APIErrorResponse { - viewState = .error(error) - } - */ - catch { + } catch { viewState = .error(error.localizedDescription) } } } - - /* - private func executeLLMQueries() async throws { - while true { - let chatStreamResults = try await openAIModel.queryAPI(withChat: chat, withFunction: functions) - - let currentMessageCount = chat.count - var llmStreamResults: [LLMStreamResult] = [] - - for try await chatStreamResult in chatStreamResults { - // Parse the different elements in mutable llm stream results. - for choice in chatStreamResult.choices { - let existingLLMStreamResult = llmStreamResults.first(where: { $0.id == choice.index }) - let llmStreamResult: LLMStreamResult - - if let existingLLMStreamResult { - llmStreamResult = existingLLMStreamResult - } else { - llmStreamResult = LLMStreamResult(id: choice.index) - llmStreamResults.append(llmStreamResult) - } - - llmStreamResult.append(choice: choice) - } - - // Append assistant messages during the streaming to ensure that they are presented in the UI. - // Limitation: We currently don't really handle multiple llmStreamResults, messages could overwritten. - for llmStreamResult in llmStreamResults where llmStreamResult.role == .assistant && !(llmStreamResult.content?.isEmpty ?? true) { - let newMessage = Chat( - role: .assistant, - content: llmStreamResult.content - ) - - if chat.indices.contains(currentMessageCount) { - chat[currentMessageCount] = newMessage - } else { - chat.append(newMessage) - } - } - } - - let functionCalls = llmStreamResults.compactMap { $0.functionCall } - - // Exit the while loop if we don't have any function calls. - guard !functionCalls.isEmpty else { - break - } - - for functionCall in functionCalls { - print("Function Call - Name: \(functionCall.name ?? ""), Arguments: \(functionCall.arguments ?? "")") - - switch functionCall.name { - case LLMFunction.getResourcesName: - try await callGetResources(functionCall: functionCall) - default: - break - } - } - } - } - - - private func callGetResources(functionCall: LLMStreamResult.FunctionCall) async throws { - struct Response: Codable { - let resources: String - } - - guard let jsonData = functionCall.arguments?.data(using: .utf8), - let response = try? JSONDecoder().decode(Response.self, from: jsonData) else { - return - } - - let requestedResources = response.resources.filter { !$0.isWhitespace }.components(separatedBy: ",") - - print("Parsed Resources: \(requestedResources)") - - for requestedResource in requestedResources { - var fittingResources = fhirStore.llmRelevantResources.filter { $0.functionCallIdentifier.contains(requestedResource) } - - guard !fittingResources.isEmpty else { - chat.append( - Chat( - role: .function, - content: String(localized: "The medical record does not include any FHIR resources for the search term \(requestedResource)."), - name: LLMFunction.getResourcesName - ) - ) - continue - } - - print("Fitting Resources: \(fittingResources.count)") - if fittingResources.count > 64 { - fittingResources = fittingResources.lazy.sorted(by: { $0.date ?? .distantPast < $1.date ?? .distantPast }).suffix(64) - print("Reduced to the following 64 resources: \(fittingResources.map { $0.functionCallIdentifier }.joined(separator: ","))") - } - - for resource in fittingResources { - print("Appending Resource: \(resource)") - let summary = try await resourceSummary.summarize(resource: resource) - print("Summary of Resource generated: \(summary)") - chat.append( - Chat( - role: .function, - content: String(localized: "This is the summary of the requested \(requestedResource):\n\n\(summary.description)"), - name: LLMFunction.getResourcesName - ) - ) - } - } - } - */ } diff --git a/LLMonFHIR/FHIR Interpretation/LLMStreamResult.swift b/LLMonFHIR/FHIR Interpretation/LLMStreamResult.swift deleted file mode 100644 index 93c9c00..0000000 --- a/LLMonFHIR/FHIR Interpretation/LLMStreamResult.swift +++ /dev/null @@ -1,64 +0,0 @@ -// -// This source file is part of the Stanford LLM on FHIR project -// -// SPDX-FileCopyrightText: 2023 Stanford University -// -// SPDX-License-Identifier: MIT -// - -import OpenAI - - -class LLMStreamResult { - class FunctionCall { - var name: String? - var arguments: String? - - - init(name: String? = nil, arguments: String? = nil) { - self.name = name - self.arguments = arguments - } - } - - - let id: Int - var content: String? - var role: Chat.Role? - var functionCall: FunctionCall? - var finishReason: String? - - - init(id: Int, content: String? = nil, role: Chat.Role? = nil, functionCall: FunctionCall? = nil, finishReason: String? = nil) { - self.id = id - self.content = content - self.role = role - self.functionCall = functionCall - self.finishReason = finishReason - } - - - func append(choice: ChatStreamResult.Choice) { - if let deltaContent = choice.delta.content { - self.content = (self.content ?? String()).appending(deltaContent) - } - - if let role = choice.delta.role { - self.role = role - } - - if let deltaName = choice.delta.functionCall?.name { - functionCall = functionCall ?? LLMStreamResult.FunctionCall() - functionCall?.name = (functionCall?.name ?? String()).appending(deltaName) - } - - if let deltaArguments = choice.delta.functionCall?.arguments { - functionCall = functionCall ?? LLMStreamResult.FunctionCall() - functionCall?.arguments = (functionCall?.arguments ?? String()).appending(deltaArguments) - } - - if let finishReason = choice.finishReason { - self.finishReason = (self.finishReason ?? String()).appending(finishReason) - } - } -} diff --git a/LLMonFHIR/LLMonFHIRDelegate.swift b/LLMonFHIR/LLMonFHIRDelegate.swift index 2b6f50f..36d81e8 100644 --- a/LLMonFHIR/LLMonFHIRDelegate.swift +++ b/LLMonFHIR/LLMonFHIRDelegate.swift @@ -22,7 +22,7 @@ class LLMonFHIRDelegate: SpeziAppDelegate { healthKit } LLMRunner { - LLMOpenAIRunnerSetupTask() + LLMOpenAIPlatform(configuration: .init(concurrentStreams: 25)) } FHIRInterpretationModule() } diff --git a/LLMonFHIR/Onboarding/OnboardingFlow.swift b/LLMonFHIR/Onboarding/OnboardingFlow.swift index fdcf5a1..f0f9199 100644 --- a/LLMonFHIR/Onboarding/OnboardingFlow.swift +++ b/LLMonFHIR/Onboarding/OnboardingFlow.swift @@ -6,8 +6,8 @@ // SPDX-License-Identifier: MIT // -import SpeziOnboarding import SpeziLLMOpenAI +import SpeziOnboarding import SwiftUI diff --git a/LLMonFHIR/Resources/Localizable.xcstrings b/LLMonFHIR/Resources/Localizable.xcstrings index 2ed6818..24b76d7 100644 --- a/LLMonFHIR/Resources/Localizable.xcstrings +++ b/LLMonFHIR/Resources/Localizable.xcstrings @@ -1006,7 +1006,7 @@ "en" : { "stringUnit" : { "state" : "translated", - "value" : "Call this function to determine the relevant FHIR health record titles based on the user's question. The titles must have the exact name defined in the initial prompt. \n\nPass in one or more titles separated by commas that you need access to the \"resources\" property as a string. \n\nAlways try to output the least amount of resource titles to be sent to the model to prevent exceeding the token limit." + "value" : "Call this function to determine the relevant FHIR health record titles based on the user's question. The titles must have the exact name defined in the initial prompt. \n\nPass in one or more titles that you need access within the \"resources\" property as an array of Strings.\n\nAlways try to output the least amount of resource titles to be sent to the model to prevent exceeding the token limit." } }, "es" : { @@ -1319,7 +1319,7 @@ "en" : { "stringUnit" : { "state" : "translated", - "value" : "Provide an array of all the FHIR health record titles with the exact same name as given in the list that are applicable to answer the user's questions. It is possible that multiple titles apply to the users's question (e.g for multiple medications). You can also request a larger set of FHIR resources by, e.g., just stating the resource type but this might not include all relevant resources to avoid exceeding the token limit.\n\nThe FHIR resource identifiers are composed of three elements:\n1. The FHIR resource type, e.g., MedicationRequest, Condition, and more.\n2. The title of the FHIR resource\n3. The date associated with the FHIR resource.\n\nUse these informations to determine the best possible FHIR resource for each question. Try to request all the required titles to allow yourself to fully answer the question in a comprehensive manner. \n\nFor example, if the user asks about their Medication it would be recommended to request all MedicationRequest FHIR resource types as well as other related FHIR resources.\n\nA question can build upon a previous question and does not need to be explicit. e.g. if a user says prescribe, this is associated with medication." + "value" : "An array of all the FHIR health record titles with the exact same name as given in the list that are applicable to answer the user's questions. It is possible that multiple titles apply to the users's question (e.g for multiple medications). You can also request a larger set of FHIR resources by, e.g., just stating the resource type but this might not include all relevant resources to avoid exceeding the token limit.\n\nThe FHIR resource identifiers are composed of three elements:\n1. The FHIR resource type, e.g., MedicationRequest, Condition, and more.\n2. The title of the FHIR resource\n3. The date associated with the FHIR resource.\n\nUse these informations to determine the best possible FHIR resource for each question. Try to request all the required titles to allow yourself to fully answer the question in a comprehensive manner. \n\nFor example, if the user asks about their Medication it would be recommended to request all MedicationRequest FHIR resource types as well as other related FHIR resources.\n\nA question can build upon a previous question and does not need to be explicit. e.g. if a user says prescribe, this is associated with medication." } }, "es" : { @@ -1796,6 +1796,16 @@ }, "The medical record does not include any FHIR resources for the search term %@." : { + }, + "This is the summary of the requested %@:\n\n%@" : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "This is the summary of the requested %1$@:\n\n%2$@" + } + } + } }, "Total Number of Resources: %lld" : { diff --git a/LLMonFHIR/Settings/SettingsView.swift b/LLMonFHIR/Settings/SettingsView.swift index 3196a6c..4b067f2 100644 --- a/LLMonFHIR/Settings/SettingsView.swift +++ b/LLMonFHIR/Settings/SettingsView.swift @@ -28,7 +28,6 @@ struct SettingsView: View { @AppStorage(StorageKeys.openAIModel) private var openAIModel = StorageKeys.Defaults.openAIModel - var body: some View { NavigationStack(path: $path) { List { @@ -112,7 +111,7 @@ struct SettingsView: View { case .openAIModel: LLMOpenAIModelOnboardingStep( actionText: "OPEN_AI_MODEL_SAVE_ACTION", - models: [.gpt4, .gpt4_1106_preview] + models: [.gpt4_turbo_preview, .gpt4] ) { chosenModelType in openAIModel = chosenModelType path.removeLast() diff --git a/LLMonFHIR/SharedContext/StorageKeys.swift b/LLMonFHIR/SharedContext/StorageKeys.swift index f81db4b..7dbc9e4 100644 --- a/LLMonFHIR/SharedContext/StorageKeys.swift +++ b/LLMonFHIR/SharedContext/StorageKeys.swift @@ -6,6 +6,7 @@ // SPDX-License-Identifier: MIT // +import struct OpenAI.Model import SpeziLLMOpenAI @@ -14,7 +15,7 @@ enum StorageKeys { enum Defaults { static let enableTextToSpeech = false static let resourceLimit = 250 - static let openAIModel: Model = .gpt4_1106_preview + static let openAIModel: Model = .gpt4_turbo_preview }