Skip to content

Commit

Permalink
Lift to structural LLM improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@philippzagar
philippzagar committed Feb 11, 2024
1 parent fa589d6 commit 76296cd
Show file tree
Hide file tree
Showing 14 changed files with 181 additions and 253 deletions.
85 changes: 67 additions & 18 deletions LLMonFHIR.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
archiveVersion = 1;
classes = {
};
objectVersion = 60;
objectVersion = 55;
objects = {

/* Begin PBXBuildFile section */
Expand All @@ -27,7 +27,6 @@
2FC186092AD52FFF0065EBB2 /* Localizable.xcstrings in Resources */ = {isa = PBXBuildFile; fileRef = 2FC186082AD52FFF0065EBB2 /* Localizable.xcstrings */; };
2FC975A82978F11A00BA99FE /* Home.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FC975A72978F11A00BA99FE /* Home.swift */; };
2FD024892B116EEF009A682C /* FHIRInterpretationFunction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FD024882B116EEF009A682C /* FHIRInterpretationFunction.swift */; };
2FD0248B2B116EF6009A682C /* LLMStreamResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FD0248A2B116EF6009A682C /* LLMStreamResult.swift */; };
2FD024912B1171EC009A682C /* Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json in Resources */ = {isa = PBXBuildFile; fileRef = 2FD0248D2B1171EC009A682C /* Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json */; };
2FD024922B1171EC009A682C /* Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json in Resources */ = {isa = PBXBuildFile; fileRef = 2FD0248E2B1171EC009A682C /* Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json */; };
2FD8E8272A1AAD9B00357F4E /* LLMonFHIRStandard.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FD8E8262A1AAD9B00357F4E /* LLMonFHIRStandard.swift */; };
Expand Down Expand Up @@ -63,6 +62,12 @@
9775720D2B5E718A00FB0286 /* SpeziSpeechRecognizer in Frameworks */ = {isa = PBXBuildFile; productRef = 9775720C2B5E718A00FB0286 /* SpeziSpeechRecognizer */; };
9775720F2B5E718A00FB0286 /* SpeziSpeechSynthesizer in Frameworks */ = {isa = PBXBuildFile; productRef = 9775720E2B5E718A00FB0286 /* SpeziSpeechSynthesizer */; };
977572122B5E721F00FB0286 /* SpeziChat in Frameworks */ = {isa = PBXBuildFile; productRef = 977572112B5E721F00FB0286 /* SpeziChat */; };
979825532B79717200335095 /* SpeziLLM in Frameworks */ = {isa = PBXBuildFile; productRef = 979825522B79717200335095 /* SpeziLLM */; };
979825552B79717200335095 /* SpeziLLMOpenAI in Frameworks */ = {isa = PBXBuildFile; productRef = 979825542B79717200335095 /* SpeziLLMOpenAI */; };
979825582B79719B00335095 /* SpeziFHIR in Frameworks */ = {isa = PBXBuildFile; productRef = 979825572B79719B00335095 /* SpeziFHIR */; };
9798255A2B79719B00335095 /* SpeziFHIRHealthKit in Frameworks */ = {isa = PBXBuildFile; productRef = 979825592B79719B00335095 /* SpeziFHIRHealthKit */; };
9798255C2B79719B00335095 /* SpeziFHIRInterpretation in Frameworks */ = {isa = PBXBuildFile; productRef = 9798255B2B79719B00335095 /* SpeziFHIRInterpretation */; };
9798255E2B79719B00335095 /* SpeziFHIRMockPatients in Frameworks */ = {isa = PBXBuildFile; productRef = 9798255D2B79719B00335095 /* SpeziFHIRMockPatients */; };
97CAB1F72B64A03600D646CE /* SpeziLLM in Frameworks */ = {isa = PBXBuildFile; productRef = 97CAB1F62B64A03600D646CE /* SpeziLLM */; };
97CAB1F92B64A03600D646CE /* SpeziLLMOpenAI in Frameworks */ = {isa = PBXBuildFile; productRef = 97CAB1F82B64A03600D646CE /* SpeziLLMOpenAI */; };
/* End PBXBuildFile section */
Expand Down Expand Up @@ -106,7 +111,6 @@
2FC94CD4298B0A1D009C8209 /* LLMonFHIR.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = LLMonFHIR.xctestplan; sourceTree = "<group>"; };
2FC975A72978F11A00BA99FE /* Home.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Home.swift; sourceTree = "<group>"; };
2FD024882B116EEF009A682C /* FHIRInterpretationFunction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FHIRInterpretationFunction.swift; sourceTree = "<group>"; };
2FD0248A2B116EF6009A682C /* LLMStreamResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMStreamResult.swift; sourceTree = "<group>"; };
2FD0248D2B1171EC009A682C /* Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = "Edythe31_Morar593_9c3df38a-d3b7-2198-3898-51f9153d023d.json"; sourceTree = "<group>"; };
2FD0248E2B1171EC009A682C /* Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = "Milton509_Ortiz186_d66b5418-06cb-fc8a-8c13-85685b6ac939.json"; sourceTree = "<group>"; };
2FD8E8262A1AAD9B00357F4E /* LLMonFHIRStandard.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LLMonFHIRStandard.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -139,18 +143,24 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
9798255A2B79719B00335095 /* SpeziFHIRHealthKit in Frameworks */,
97CAB1F72B64A03600D646CE /* SpeziLLM in Frameworks */,
9733FC592B60E5FB0024F12C /* SpeziFHIRHealthKit in Frameworks */,
9775720F2B5E718A00FB0286 /* SpeziSpeechSynthesizer in Frameworks */,
2FD8E82C2A1AADDA00357F4E /* ModelsR4 in Frameworks */,
2FD8E82A2A1AADDA00357F4E /* ModelsDSTU2 in Frameworks */,
2FD8E8402A1AE3F200357F4E /* SpeziViews in Frameworks */,
979825552B79717200335095 /* SpeziLLMOpenAI in Frameworks */,
979825532B79717200335095 /* SpeziLLM in Frameworks */,
2FE5DC7229EDD8D3004B9AB4 /* SpeziHealthKit in Frameworks */,
9798255C2B79719B00335095 /* SpeziFHIRInterpretation in Frameworks */,
2F49B7762980407C00BCB272 /* Spezi in Frameworks */,
9775720D2B5E718A00FB0286 /* SpeziSpeechRecognizer in Frameworks */,
9798255E2B79719B00335095 /* SpeziFHIRMockPatients in Frameworks */,
2FD8E8312A1AB00D00357F4E /* HealthKitOnFHIR in Frameworks */,
97CAB1F92B64A03600D646CE /* SpeziLLMOpenAI in Frameworks */,
9733FC5D2B60E5FB0024F12C /* SpeziFHIRMockPatients in Frameworks */,
979825582B79719B00335095 /* SpeziFHIR in Frameworks */,
977572122B5E721F00FB0286 /* SpeziChat in Frameworks */,
2FE5DC8129EDD91D004B9AB4 /* SpeziOnboarding in Frameworks */,
9733FC5B2B60E5FB0024F12C /* SpeziFHIRInterpretation in Frameworks */,
Expand Down Expand Up @@ -225,7 +235,6 @@
2F036EA02B0E4B22009B2745 /* FHIRInterpretationModule.swift */,
433352432A5C96090043A440 /* FHIRMultipleResourceInterpreter.swift */,
2FD024882B116EEF009A682C /* FHIRInterpretationFunction.swift */,
2FD0248A2B116EF6009A682C /* LLMStreamResult.swift */,
);
path = "FHIR Interpretation";
sourceTree = "<group>";
Expand Down Expand Up @@ -384,6 +393,12 @@
9733FC5C2B60E5FB0024F12C /* SpeziFHIRMockPatients */,
97CAB1F62B64A03600D646CE /* SpeziLLM */,
97CAB1F82B64A03600D646CE /* SpeziLLMOpenAI */,
979825522B79717200335095 /* SpeziLLM */,
979825542B79717200335095 /* SpeziLLMOpenAI */,
979825572B79719B00335095 /* SpeziFHIR */,
979825592B79719B00335095 /* SpeziFHIRHealthKit */,
9798255B2B79719B00335095 /* SpeziFHIRInterpretation */,
9798255D2B79719B00335095 /* SpeziFHIRMockPatients */,
);
productName = LLMonFHIR;
productReference = 653A254D283387FE005D4D48 /* LLMonFHIR.app */;
Expand Down Expand Up @@ -476,8 +491,8 @@
2FD8E83E2A1AE3F200357F4E /* XCRemoteSwiftPackageReference "SpeziViews" */,
9775720B2B5E718A00FB0286 /* XCRemoteSwiftPackageReference "SpeziSpeech" */,
977572102B5E721F00FB0286 /* XCRemoteSwiftPackageReference "SpeziChat" */,
9733FC552B60E5FB0024F12C /* XCLocalSwiftPackageReference "../SpeziFHIR" */,
97CAB1F52B64A03600D646CE /* XCLocalSwiftPackageReference "../SpeziLLM" */,
979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */,
979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */,
);
productRefGroup = 653A254E283387FE005D4D48 /* Products */;
projectDirPath = "";
Expand Down Expand Up @@ -571,7 +586,6 @@
2F4E23832989D51F0013F3D9 /* LLMonFHIRTestingSetup.swift in Sources */,
433352442A5C96090043A440 /* FHIRMultipleResourceInterpreter.swift in Sources */,
2FD8E8272A1AAD9B00357F4E /* LLMonFHIRStandard.swift in Sources */,
2FD0248B2B116EF6009A682C /* LLMStreamResult.swift in Sources */,
2F5E32BD297E05EA003432F8 /* LLMonFHIRDelegate.swift in Sources */,
653A2551283387FE005D4D48 /* LLMonFHIR.swift in Sources */,
2F036EA52B0ED1F0009B2745 /* FHIRResourcesInstructionsView.swift in Sources */,
Expand Down Expand Up @@ -1093,17 +1107,6 @@
};
/* End XCConfigurationList section */

/* Begin XCLocalSwiftPackageReference section */
9733FC552B60E5FB0024F12C /* XCLocalSwiftPackageReference "../SpeziFHIR" */ = {
isa = XCLocalSwiftPackageReference;
relativePath = ../SpeziFHIR;
};
97CAB1F52B64A03600D646CE /* XCLocalSwiftPackageReference "../SpeziLLM" */ = {
isa = XCLocalSwiftPackageReference;
relativePath = ../SpeziLLM;
};
/* End XCLocalSwiftPackageReference section */

/* Begin XCRemoteSwiftPackageReference section */
2F49B7742980407B00BCB272 /* XCRemoteSwiftPackageReference "Spezi" */ = {
isa = XCRemoteSwiftPackageReference;
Expand Down Expand Up @@ -1185,6 +1188,22 @@
minimumVersion = 0.1.4;
};
};
979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/StanfordSpezi/SpeziLLM";
requirement = {
branch = "feat/structural-improvments";
kind = branch;
};
};
979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/StanfordSpezi/SpeziFHIR";
requirement = {
branch = "feat/lift-to-spezi-llm";
kind = branch;
};
};
/* End XCRemoteSwiftPackageReference section */

/* Begin XCSwiftPackageProductDependency section */
Expand Down Expand Up @@ -1264,6 +1283,36 @@
package = 977572102B5E721F00FB0286 /* XCRemoteSwiftPackageReference "SpeziChat" */;
productName = SpeziChat;
};
979825522B79717200335095 /* SpeziLLM */ = {
isa = XCSwiftPackageProductDependency;
package = 979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */;
productName = SpeziLLM;
};
979825542B79717200335095 /* SpeziLLMOpenAI */ = {
isa = XCSwiftPackageProductDependency;
package = 979825512B79717200335095 /* XCRemoteSwiftPackageReference "SpeziLLM" */;
productName = SpeziLLMOpenAI;
};
979825572B79719B00335095 /* SpeziFHIR */ = {
isa = XCSwiftPackageProductDependency;
package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */;
productName = SpeziFHIR;
};
979825592B79719B00335095 /* SpeziFHIRHealthKit */ = {
isa = XCSwiftPackageProductDependency;
package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */;
productName = SpeziFHIRHealthKit;
};
9798255B2B79719B00335095 /* SpeziFHIRInterpretation */ = {
isa = XCSwiftPackageProductDependency;
package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */;
productName = SpeziFHIRInterpretation;
};
9798255D2B79719B00335095 /* SpeziFHIRMockPatients */ = {
isa = XCSwiftPackageProductDependency;
package = 979825562B79719B00335095 /* XCRemoteSwiftPackageReference "SpeziFHIR" */;
productName = SpeziFHIRMockPatients;
};
97CAB1F62B64A03600D646CE /* SpeziLLM */ = {
isa = XCSwiftPackageProductDependency;
productName = SpeziLLM;
Expand Down
32 changes: 29 additions & 3 deletions LLMonFHIR.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,16 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/MacPaw/OpenAI",
"state" : {
"revision" : "ac5892fd0de8d283362ddc30f8e9f1a0eaba8cc0",
"version" : "0.2.5"
"revision" : "35afc9a6ee127b8f22a85a31aec2036a987478af"
}
},
{
"identity" : "semaphore",
"kind" : "remoteSourceControl",
"location" : "https://github.com/groue/Semaphore.git",
"state" : {
"revision" : "f1c4a0acabeb591068dea6cffdd39660b86dec28",
"version" : "0.0.8"
}
},
{
Expand All @@ -54,6 +62,15 @@
"version" : "0.1.5"
}
},
{
"identity" : "spezifhir",
"kind" : "remoteSourceControl",
"location" : "https://github.com/StanfordSpezi/SpeziFHIR",
"state" : {
"branch" : "feat/lift-to-spezi-llm",
"revision" : "b58ba7afc34ff1b83d4f4ab3525d3d13f372b080"
}
},
{
"identity" : "spezifoundation",
"kind" : "remoteSourceControl",
Expand All @@ -72,6 +89,15 @@
"version" : "0.5.0"
}
},
{
"identity" : "spezillm",
"kind" : "remoteSourceControl",
"location" : "https://github.com/StanfordSpezi/SpeziLLM",
"state" : {
"branch" : "feat/structural-improvments",
"revision" : "f30f4d311cf99a396e76e5b732dca6462463f663"
}
},
{
"identity" : "spezionboarding",
"kind" : "remoteSourceControl",
Expand All @@ -93,7 +119,7 @@
{
"identity" : "spezistorage",
"kind" : "remoteSourceControl",
"location" : "https://github.com/StanfordSpezi/SpeziStorage.git",
"location" : "https://github.com/StanfordSpezi/SpeziStorage",
"state" : {
"revision" : "eaed2220375c35400aa69d1f96a8d32b7e66b1c7",
"version" : "1.0.0"
Expand Down
2 changes: 1 addition & 1 deletion LLMonFHIR/FHIR Display/FHIRResourcesView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
import ModelsR4
import SpeziFHIR
import SpeziFHIRInterpretation
import SpeziOnboarding
import SpeziLLMOpenAI
import SpeziOnboarding
import SpeziViews
import SwiftUI

Expand Down
15 changes: 8 additions & 7 deletions LLMonFHIR/FHIR Display/InspectResourceView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import SpeziFHIR
import SpeziFHIRInterpretation
import SpeziLLM
import SpeziViews
import SwiftUI

Expand Down Expand Up @@ -114,10 +115,10 @@ struct InspectResourceView: View {
do {
try await fhirResourceSummary.summarize(resource: resource, forceReload: forceReload)
loadingSummary = .idle
}/* catch let error as APIErrorResponse {
} catch let error as LLMError {
loadingSummary = .error(error)
}*/ catch {
loadingSummary = .error("Unknown error")
} catch {
loadingSummary = .error("Unknown LLM processing error")
}
}
}
Expand All @@ -129,10 +130,10 @@ struct InspectResourceView: View {
do {
try await fhirResourceInterpreter.interpret(resource: resource, forceReload: forceReload)
interpreting = .idle
} /*catch let error as APIErrorResponse {
loadingSummary = .error(error)
}*/ catch {
loadingSummary = .error("Unknown error")
} catch let error as LLMError {
interpreting = .error(error)
} catch {
interpreting = .error("Unknown LLM processing error")
}
}
}
Expand Down
26 changes: 17 additions & 9 deletions LLMonFHIR/FHIR Display/MultipleResourcesChatView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,27 @@ struct MultipleResourcesChatView: View {
var body: some View {
@Bindable var multipleResourceInterpreter = multipleResourceInterpreter
NavigationStack {
ChatView(
$multipleResourceInterpreter.llm.context,
disableInput: multipleResourceInterpreter.viewState == .processing
)
Group {
if let llm = multipleResourceInterpreter.llm {
let contextBinding = Binding { llm.context } set: { llm.context = $0 }
ChatView(
contextBinding,
disableInput: multipleResourceInterpreter.viewState == .processing
)
.onChange(of: llm.context) {
multipleResourceInterpreter.queryLLM()
}
} else {
ChatView(
.constant([])
)
}
}
.navigationTitle("LLM on FHIR")
.toolbar {
toolbar
}
.viewStateAlert(state: $multipleResourceInterpreter.viewState)
.onChange(of: multipleResourceInterpreter.llm.context) {
multipleResourceInterpreter.queryLLM()
}
.onAppear {
multipleResourceInterpreter.queryLLM()
}
Expand All @@ -47,8 +56,7 @@ struct MultipleResourcesChatView: View {
}


@MainActor
@ToolbarContentBuilder private var toolbar: some ToolbarContent {
@MainActor @ToolbarContentBuilder private var toolbar: some ToolbarContent {
ToolbarItem(placement: .cancellationAction) {
if multipleResourceInterpreter.viewState == .processing {
ProgressView()
Expand Down
12 changes: 6 additions & 6 deletions LLMonFHIR/FHIR Interpretation/FHIRInterpretationFunction.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
//

import os
import SpeziLLMOpenAI
import SpeziFHIR
import SpeziFHIRInterpretation
import SpeziLLMOpenAI


struct FHIRInterpretationFunction: LLMFunction {
Expand All @@ -23,8 +23,7 @@ struct FHIRInterpretationFunction: LLMFunction {
private let allResourcesFunctionCallIdentifier: [String]


@Parameter
var resources: [String]
@Parameter var resources: [String]


init(fhirStore: FHIRStore, resourceSummary: FHIRResourceSummary, allResourcesFunctionCallIdentifier: [String]) {
Expand Down Expand Up @@ -65,7 +64,7 @@ struct FHIRInterpretationFunction: LLMFunction {
// Iterate over fitting resources and summarizing them
for resource in fittingResources {
innerGroup.addTask {
return try await summarizeResource(fhirResource: resource, resourceType: requestedResource)
try await summarizeResource(fhirResource: resource, resourceType: requestedResource)
}
}

Expand All @@ -88,7 +87,7 @@ struct FHIRInterpretationFunction: LLMFunction {

private func summarizeResource(fhirResource: FHIRResource, resourceType: String) async throws -> String {
let summary = try await resourceSummary.summarize(resource: fhirResource)
Self.logger.debug("Summary of appended resource: \(summary)")
Self.logger.debug("Summary of appended FHIR resource \(resourceType): \(summary.description)")
return String(localized: "This is the summary of the requested \(resourceType):\n\n\(summary.description)")
}

Expand All @@ -102,7 +101,8 @@ struct FHIRInterpretationFunction: LLMFunction {
Self.logger.debug(
"""
Reduced to the following 64 resources: \(fittingResources.map { $0.functionCallIdentifier }.joined(separator: ","))
""")
"""
)
}

return fittingResources
Expand Down
Loading

0 comments on commit 76296cd

Please sign in to comment.