diff --git a/auth/build.gradle.kts b/auth/build.gradle.kts index f04d2619d62..9a8b9d8a923 100644 --- a/auth/build.gradle.kts +++ b/auth/build.gradle.kts @@ -1,7 +1,7 @@ apply(from = rootProject.file("buildSrc/shared.gradle.kts")) plugins { - id("org.springframework.boot") version "3.3.2" + id("org.springframework.boot") version "3.3.4" id("io.spring.dependency-management") version "1.1.6" id("reportstream.project-conventions") kotlin("plugin.spring") version "2.0.0" @@ -14,8 +14,8 @@ dependencies { implementation(project(":shared")) implementation("org.jetbrains.kotlin:kotlin-reflect") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.1") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor:1.8.1") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor:1.9.0") /** * Spring WebFlux was chosen for this project to be able to better handle periods of high traffic @@ -24,7 +24,7 @@ dependencies { implementation("org.springframework.cloud:spring-cloud-gateway-webflux") implementation("org.springframework.boot:spring-boot-starter-oauth2-resource-server") - runtimeOnly("com.nimbusds:oauth2-oidc-sdk:11.18") + runtimeOnly("com.nimbusds:oauth2-oidc-sdk:11.19.1") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("org.springframework.security:spring-security-test") @@ -45,7 +45,7 @@ configurations.all { dependencyManagement { imports { - mavenBom("com.azure.spring:spring-cloud-azure-dependencies:5.14.0") + mavenBom("com.azure.spring:spring-cloud-azure-dependencies:5.16.0") mavenBom("org.springframework.cloud:spring-cloud-dependencies:2023.0.3") } } diff --git a/frontend-react/src/content/about/roadmap.mdx b/frontend-react/src/content/about/roadmap.mdx index 94cce91a2c6..c5749c1e094 100644 --- a/frontend-react/src/content/about/roadmap.mdx +++ b/frontend-react/src/content/about/roadmap.mdx @@ -17,7 +17,7 @@ import site from "../../content/site.json" -

Last updated: August 05, 2024

+

Last updated: October 07, 2024

## Objectives for FY24 Learn about our goals and how we are measuring success for this fiscal year (FY), which runs from October 2023 through September 2024. @@ -46,7 +46,7 @@ Learn about our goals and how we are measuring success for this fiscal year (FY) tag: "recently-completed", body: (

- Arizona, Colorado, and Kansas are able to receive flu and RSV data from ReportStream. + Arkansas, Florida, Guam, Idaho, Illinois, Louisiana, Maine, Massachusetts, Minnesota, and Oklahoma are able to receive flu and RSV data from ReportStream.

), }, @@ -54,29 +54,29 @@ Learn about our goals and how we are measuring success for this fiscal year (FY) tag: "working-on-now", body: (

- Onboard or update connections for Arkansas, Connecticut, Florida, Idaho, Illinois, Iowa, Kentucky, Louisiana, Maine, Massachusetts, Michigan, Minnesota, Missouri, Montana, Oklahoma, Virginia, Washington D.C. + Onboard or update connections for Alabama, Connecticut, Delaware, Indiana, Iowa, Kentucky, Michigan, Mississippi, Missouri, Montana, New Hampshire, New Jersey, New Mexico, Ohio, Pennsylvania, Puerto Rico, Republic of Marshall Islands, South Dakota, Tennessee, Vermont, Virginia, Washington, Washington D.C., and Wyoming.

), }, { tag: "next", body: ( -

- Support remaining STLTs to connect or update their connections to be able to receive additional conditions. -

+

Support remaining STLTs to connect or update their connections to be able to receive additional conditions.

), }, { title:

Partner with SimpleReport to help nontraditional and lower-resourced testing sites and facilities report test results and send data to STLTs and the CDC.

, tag: "recently-completed", body: ( -

California, Nevada, and New York are receiving flu A/B or RSV data.

+

Contacted nontraditional and lower-resourced HIV senders in Texas as part of targeted health equity improvement work.

), }, { tag: "recently-completed", body: ( -

Identify nontraditional and lower-resourced HIV senders in Texas as part of targeted health equity improvement work.

+

+ Alaska, Arizona, California, Colorado, Florida, Hawaii, Idaho, Illinois, Louisiana, Massachusetts, Minnesota, Nevada, New York, and Rhode Island are receiving flu A/B or RSV data. +

), }, { @@ -86,45 +86,39 @@ Learn about our goals and how we are measuring success for this fiscal year (FY) ), }, { - tag: "next", + tag: "working-on-now", body: ( -

Identify more regions in California to exchange HIV test result data.

+

Add support for hepatitis C and other STI reporting through SimpleReport.

), }, { tag: "next", body: ( -

Add support for hepatitis C reporting through SimpleReport.

+

Add support for more conditions reporting through SimpleReport.

), }, { title:

Help at-home test takers and test manufacturers easily report results to STLTs and CDC.

, tag: "recently-completed", body: ( -

Support file validation against different HL7 profiles, starting with the RADx MARS custom NIST HL7 2.5.1 profile.

+

Route test results from MakeMyTestCount.org to one STLT in test environment.

), }, { tag: "working-on-now", body: ( -

Route test results from MakeMyTestCount.org to one STLT in test environment.

+

Route test results from MakeMyTestCount.org to one STLT in production.

), }, { tag: "next", body: ( -

Route test results from MakeMyTestCount.org to one STLT in production.

+

Route test results from another OTC testing (point-of-care) sender to one STLT in production.

), }, { title: "Facilitate electronic laboratory reporting (ELR) for the CDC infectious disease labs and transmit to STLTs. ", tag: "recently-completed", - body: ( -

Test data pipeline with two STLT partners to transmit test results.

- ), - }, - { - tag: "working-on-now", body: (

Complete final internal review of test result data transfer before moving to production.

), diff --git a/prime-router/build.gradle.kts b/prime-router/build.gradle.kts index 65ca5665f3f..c8dcd983c41 100644 --- a/prime-router/build.gradle.kts +++ b/prime-router/build.gradle.kts @@ -35,7 +35,7 @@ apply(from = rootProject.file("buildSrc/shared.gradle.kts")) plugins { val kotlinVersion by System.getProperties() id("reportstream.project-conventions") - id("org.flywaydb.flyway") version "10.18.0" + id("org.flywaydb.flyway") version "10.18.2" id("nu.studer.jooq") version "9.0" id("com.github.johnrengelman.shadow") version "8.1.1" id("com.microsoft.azure.azurefunctions") version "1.16.1" @@ -75,7 +75,7 @@ val javaVersion = when (appJvmTarget) { } val ktorVersion = "2.3.12" val kotlinVersion by System.getProperties() -val jacksonVersion = "2.17.2" +val jacksonVersion = "2.18.0" jacoco.toolVersion = "0.8.12" // Local database information, first one wins: @@ -271,9 +271,6 @@ sourceSets.create("testIntegration") { runtimeClasspath += sourceSets["main"].output } -// Add generated version object -sourceSets["main"].java.srcDir("$buildDir/generated-src/version") - val compileTestIntegrationKotlin: KotlinCompile by tasks compileTestIntegrationKotlin.kotlinOptions.jvmTarget = appJvmTarget @@ -281,6 +278,10 @@ val testIntegrationImplementation: Configuration by configurations.getting { extendsFrom(configurations["testImplementation"]) } +tasks.withType { + mustRunAfter("generateVersionObject") +} + configurations["testIntegrationRuntimeOnly"].extendsFrom(configurations["runtimeOnly"]) tasks.register("testIntegration") { @@ -353,7 +354,7 @@ tasks.withType().configureEach { } tasks.processResources { - dependsOn("generateVersionObject") + mustRunAfter("generateVersionObject") // Set the proper build values in the build.properties file filesMatching("build.properties") { val dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd") @@ -427,7 +428,7 @@ tasks.register("primeCLI") { // Use arguments passed by another task in the project.extra["cliArgs"] property. doFirst { if (project.extra.has("cliArgs") && project.extra["cliArgs"] is List<*>) { - args = (project.extra["cliArgs"] as List<*>).filterIsInstance(String::class.java) + args = (project.extra["cliArgs"] as List<*>).filterIsInstance() } else if (args.isNullOrEmpty()) { args = listOf("-h") println("primeCLI Gradle task usage: gradle primeCLI --args=''") @@ -521,12 +522,13 @@ tasks.register("generateVersionFile") { } } -tasks.register("generateVersionObject") { - val sourceDir = file("$buildDir/generated-src/version") - val sourceFile = file("$sourceDir/Version.kt") - sourceDir.mkdirs() - sourceFile.writeText( - """ +val generateVersionObject = tasks.register("generateVersionObject") { + doLast { + val sourceDir = file("$buildDir/generated-src/version/src/main/kotlin/gov/cdc/prime/router") + val sourceFile = file("$sourceDir/Version.kt") + sourceDir.mkdirs() + sourceFile.writeText( + """ package gov.cdc.prime.router.version /** @@ -537,7 +539,12 @@ tasks.register("generateVersionObject") { const val commitId = "$commitId" } """.trimIndent() - ) + ) + } +} +sourceSets.getByName("main").kotlin.srcDir("$buildDir/generated-src/version/src/main/kotlin") +tasks.named("compileKotlin").configure { + dependsOn(generateVersionObject) } val azureResourcesTmpDir = File(buildDir, "$azureFunctionsDir-resources/$azureAppName") @@ -645,7 +652,6 @@ task("uploadSwaggerUI") { } tasks.register("killFunc") { - doLast { val processName = "func" if (org.gradle.internal.os.OperatingSystem.current().isWindows) { exec { @@ -658,7 +664,6 @@ tasks.register("killFunc") { commandLine = listOf("sh", "-c", "pkill -9 $processName || true") } } - } } tasks.register("run") { @@ -772,6 +777,7 @@ tasks.named("generateJooq") { tasks.register("compile") { group = rootProject.description ?: "" description = "Compile the code" + dependsOn("generateVersionObject") dependsOn("compileKotlin") } @@ -833,7 +839,7 @@ buildscript { // will need to be removed once this issue is resolved in Maven. classpath("net.minidev:json-smart:2.5.1") // as per flyway v10 docs the postgres flyway module must be on the project buildpath - classpath("org.flywaydb:flyway-database-postgresql:10.18.0") + classpath("org.flywaydb:flyway-database-postgresql:10.18.2") } } @@ -861,7 +867,7 @@ dependencies { implementation("com.azure:azure-storage-queue:12.22.0") { exclude(group = "com.azure", module = "azure-core") } - implementation("com.azure:azure-security-keyvault-secrets:4.8.6") { + implementation("com.azure:azure-security-keyvault-secrets:4.8.7") { exclude(group = "com.azure", module = "azure-core") exclude(group = "com.azure", module = "azure-core-http-netty") } @@ -891,36 +897,36 @@ dependencies { branch = "master" } } - implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.2.2") + implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.4.2") // https://mvnrepository.com/artifact/ca.uhn.hapi.fhir/hapi-fhir-caching-caffeine - implementation("ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.2.2") - implementation("ca.uhn.hapi.fhir:hapi-fhir-client:7.2.2") + implementation("ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.4.2") + implementation("ca.uhn.hapi.fhir:hapi-fhir-client:7.4.2") // pin - implementation("ca.uhn.hapi.fhir:org.hl7.fhir.utilities:6.3.24") + implementation("ca.uhn.hapi.fhir:org.hl7.fhir.utilities:6.3.29") implementation("ca.uhn.hapi.fhir:org.hl7.fhir.r4:6.3.24") implementation("ca.uhn.hapi:hapi-base:2.5.1") implementation("ca.uhn.hapi:hapi-structures-v251:2.5.1") implementation("ca.uhn.hapi:hapi-structures-v27:2.5.1") implementation("com.googlecode.libphonenumber:libphonenumber:8.13.46") implementation("org.thymeleaf:thymeleaf:3.1.2.RELEASE") - implementation("com.sendgrid:sendgrid-java:4.10.2") + implementation("com.sendgrid:sendgrid-java:4.10.3") implementation("com.okta.jwt:okta-jwt-verifier:0.5.7") implementation("org.json:json:20240303") // DO NOT INCREMENT SSHJ to a newer version without first thoroughly testing it locally. implementation("com.hierynomus:sshj:0.38.0") implementation("com.jcraft:jsch:0.1.55") implementation("org.apache.poi:poi:5.3.0") - implementation("org.apache.commons:commons-csv:1.11.0") + implementation("org.apache.commons:commons-csv:1.12.0") implementation("org.apache.commons:commons-lang3:3.15.0") implementation("org.apache.commons:commons-text:1.12.0") implementation("commons-codec:commons-codec:1.17.1") - implementation("commons-io:commons-io:2.16.1") + implementation("commons-io:commons-io:2.17.0") implementation("org.postgresql:postgresql:42.7.4") - implementation("com.zaxxer:HikariCP:5.1.0") - implementation("org.flywaydb:flyway-core:10.18.0") - implementation("org.flywaydb:flyway-database-postgresql:10.18.0") - implementation("org.commonmark:commonmark:0.22.0") - implementation("com.google.guava:guava:33.3.0-jre") + implementation("com.zaxxer:HikariCP:6.0.0") + implementation("org.flywaydb:flyway-core:10.18.2") + implementation("org.flywaydb:flyway-database-postgresql:10.18.2") + implementation("org.commonmark:commonmark:0.23.0") + implementation("com.google.guava:guava:33.3.1-jre") implementation("com.helger.as2:as2-lib:5.1.2") implementation("org.bouncycastle:bcprov-jdk15to18:1.78.1") implementation("org.bouncycastle:bcprov-jdk18on:1.78.1") @@ -944,7 +950,7 @@ dependencies { implementation("org.apache.poi:poi:5.3.0") implementation("org.apache.poi:poi-ooxml:5.3.0") implementation("org.apache.commons:commons-compress:1.27.1") - implementation("commons-io:commons-io:2.16.1") + implementation("commons-io:commons-io:2.17.0") implementation("com.anyascii:anyascii:0.3.2") // force jsoup since skrapeit-html-parser@1.2.1+ has not updated implementation("org.jsoup:jsoup:1.18.1") @@ -972,7 +978,7 @@ dependencies { implementation("xalan:xalan:2.7.3") // validations - implementation("com.networknt:json-schema-validator:1.5.1") + implementation("com.networknt:json-schema-validator:1.5.2") implementation("io.konform:konform-jvm:0.4.0") runtimeOnly("com.okta.jwt:okta-jwt-verifier-impl:0.5.7") diff --git a/prime-router/docs/universal-pipeline/translate.md b/prime-router/docs/universal-pipeline/translate.md index 6f49a09e37c..aa010074310 100644 --- a/prime-router/docs/universal-pipeline/translate.md +++ b/prime-router/docs/universal-pipeline/translate.md @@ -38,7 +38,8 @@ The two kinds of transforms work the same at a high level. The schema enumerates - contains a FHIR path to the resource that needs to be transformed - a condition specifying whether the resource should be transformed -- how the resource should get transformed +- how the resource should get transformed; a resource can be transformed either by setting it to a value or applying a +FHIR function The primary difference between the FHIR and HL7 schemas is that the HL7 converter has special handling for converting a FHIR resource into an HL7 segment or component. diff --git a/prime-router/metadata/json_schema/fhir/fhir-to-fhir-transform.json b/prime-router/metadata/json_schema/fhir/fhir-to-fhir-transform.json index 3b94072dd9a..487f8fc46e7 100644 --- a/prime-router/metadata/json_schema/fhir/fhir-to-fhir-transform.json +++ b/prime-router/metadata/json_schema/fhir/fhir-to-fhir-transform.json @@ -67,6 +67,9 @@ "type": "string" } }, + "function": { + "type": "string" + }, "valueSet": { "anyOf": [ { diff --git a/prime-router/src/main/kotlin/config/validation/Validations.kt b/prime-router/src/main/kotlin/config/validation/Validations.kt index 1d7b0a41d43..5b32bcb4472 100644 --- a/prime-router/src/main/kotlin/config/validation/Validations.kt +++ b/prime-router/src/main/kotlin/config/validation/Validations.kt @@ -113,6 +113,9 @@ object FhirToFhirTransformValidation : KonformValidation() addConstraint("Invalid FHIR path: {value}", test = ::validFhirPath) } } + addConstraint("Value and function cannot both be set") { element -> + !(element.value != null && element.function != null) + } } } } diff --git a/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt b/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt index ecfe4571007..af338c1b338 100644 --- a/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt +++ b/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt @@ -114,9 +114,13 @@ class FhirTransformer( eligibleFocusResources.forEach { singleFocusResource -> elementContext.focusResource = singleFocusResource val value = getValue(element, bundle, singleFocusResource, elementContext) + val function = element.function + if (value != null && function != null) { + throw SchemaException("Element can only set function or value") + } + val bundleProperty = element.bundleProperty + ?: throw SchemaException("bundleProperty must be set for element ${element.name}") if (value != null) { - val bundleProperty = element.bundleProperty - ?: throw SchemaException("bundleProperty must be set for element ${element.name}") updateBundle( bundleProperty, value, @@ -124,9 +128,18 @@ class FhirTransformer( bundle, singleFocusResource ) + } else if (function != null) { + updateBundle( + bundleProperty, + function, + elementContext, + bundle, + singleFocusResource + ) } else { logger.warn( - "Element ${element.name} is updating a bundle property, but did not specify a value" + "Element ${element.name} is updating a bundle property," + + " but did not specify a value or function" ) } debugMsg += "condition: true, resourceType: ${singleFocusResource.fhirType()}, " + @@ -298,7 +311,32 @@ class FhirTransformer( focusResource, null ) - setBundleProperty(penultimateElements, lastElement, value) + setBundleProperty(penultimateElements, lastElement, value, context) + } + + /** + * Updates a bundle by setting a value at a specified spot + * + * @param bundleProperty the property to update + * @param function the function to apply to the bundle property + * @param context the context to evaluate the bundle under + * @param focusResource the focus resource for any FHIR path evaluations + */ + internal fun updateBundle( + bundleProperty: String, + function: String, + context: CustomContext, + bundle: Bundle, + focusResource: Base, + ) { + val (lastElement, penultimateElements) = createMissingElementsInBundleProperty( + bundleProperty, + context, + bundle, + focusResource, + null + ) + applyFunction(penultimateElements, lastElement, function, context, bundle) } /** @@ -350,7 +388,29 @@ class FhirTransformer( focusResource, appendToElements ) - setBundleProperty(bundlePenultimateElements, lastBundlePropertyElement, value) + setBundleProperty(bundlePenultimateElements, lastBundlePropertyElement, value, context) + } + + /** + * Updates a list of [Base] by applying the passed FHIR [function] + * + * @param elementsToUpdate the list of [Base] to update + * @param propertyName the property to set on each element + * @param function the function to apply + */ + private fun applyFunction( + elementsToUpdate: List, + propertyName: String, + function: String, + context: CustomContext, + bundle: Bundle, + ) { + elementsToUpdate.forEach { penultimateElement -> + val propertyInfo = extractChildProperty(propertyName, context, penultimateElement) + FhirPathUtils.evaluate( + context, penultimateElement, bundle, "%resource.${propertyInfo.propertyString}.$function" + ) + } } /** @@ -364,8 +424,13 @@ class FhirTransformer( elementsToUpdate: List, propertyName: String, value: Base, + context: CustomContext, ) { elementsToUpdate.forEach { penultimateElement -> + val propertyInfo = extractChildProperty(propertyName, context, penultimateElement) + if (propertyInfo.index != null) { + throw SchemaException("Schema is attempting to set a value for a particular index which is not allowed") + } val property = penultimateElement.getNamedProperty(propertyName) val newValue = FhirBundleUtils.convertFhirType(value, value.fhirType(), property.typeCode, logger) penultimateElement.setProperty(propertyName, newValue.copy()) diff --git a/prime-router/src/main/kotlin/fhirengine/translation/hl7/schema/fhirTransform/FhirTransformSchema.kt b/prime-router/src/main/kotlin/fhirengine/translation/hl7/schema/fhirTransform/FhirTransformSchema.kt index 929b22d0a22..991acbb0b99 100644 --- a/prime-router/src/main/kotlin/fhirengine/translation/hl7/schema/fhirTransform/FhirTransformSchema.kt +++ b/prime-router/src/main/kotlin/fhirengine/translation/hl7/schema/fhirTransform/FhirTransformSchema.kt @@ -74,6 +74,7 @@ class FhirTransformSchemaElement( debug: Boolean = false, var bundleProperty: String? = null, val appendToProperty: String? = null, + var function: String? = null, ) : ConfigSchemaElement( name = name, @@ -105,6 +106,7 @@ class FhirTransformSchemaElement( bundleProperty: String? = null, action: FhirTransformSchemaElementAction, appendToProperty: String? = null, + function: String? = null, ) : this( name, condition, @@ -118,7 +120,8 @@ class FhirTransformSchemaElement( valueSet, debug, bundleProperty, - appendToProperty + appendToProperty, + function ) { this.action = action } diff --git a/prime-router/src/main/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctions.kt b/prime-router/src/main/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctions.kt index e6986f75388..c6900d23a94 100644 --- a/prime-router/src/main/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctions.kt +++ b/prime-router/src/main/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctions.kt @@ -9,6 +9,7 @@ import org.hl7.fhir.r4.model.Base import org.hl7.fhir.r4.model.BaseDateTimeType import org.hl7.fhir.r4.model.BooleanType import org.hl7.fhir.r4.model.DateTimeType +import org.hl7.fhir.r4.model.HumanName import org.hl7.fhir.r4.model.IntegerType import org.hl7.fhir.r4.model.StringType import java.time.DateTimeException @@ -37,6 +38,7 @@ object CustomFHIRFunctions : FhirPathFunctions { HasPhoneNumberExtension, ChangeTimezone, ConvertDateToAge, + DeidentifyHumanName, ; companion object { @@ -122,6 +124,14 @@ object CustomFHIRFunctions : FhirPathFunctions { ) } + CustomFHIRFunctionNames.DeidentifyHumanName -> { + FunctionDetails( + "removes PII from a name", + 0, + 1 + ) + } + else -> additionalFunctions?.resolveFunction(functionName) } } @@ -185,6 +195,10 @@ object CustomFHIRFunctions : FhirPathFunctions { convertDateToAge(focus, parameters) } + CustomFHIRFunctionNames.DeidentifyHumanName -> { + deidentifyHumanName(focus, parameters) + } + else -> additionalFunctions?.executeFunction(focus, functionName, parameters) ?: throw IllegalStateException("Tried to execute invalid FHIR Path function $functionName") } @@ -353,6 +367,22 @@ object CustomFHIRFunctions : FhirPathFunctions { return if (type != null) mutableListOf(StringType(type)) else mutableListOf() } + fun deidentifyHumanName(focus: MutableList, parameters: MutableList>?): MutableList { + val deidentifiedValue = parameters?.firstOrNull()?.filterIsInstance()?.firstOrNull()?.value ?: "" + focus.filterIsInstance().forEach { name -> + if (deidentifiedValue.isNotEmpty()) { + val updatedGiven = name.given.map { StringType(deidentifiedValue) } + name.setGiven(updatedGiven.toMutableList()) + } else { + name.setGiven(emptyList()) + } + + name.setFamily(deidentifiedValue) + } + + return focus + } + /** * Get the ID type for the value in [focus]. * @return a list with one value denoting the ID type, or an empty list diff --git a/prime-router/src/main/kotlin/history/SubmissionHistory.kt b/prime-router/src/main/kotlin/history/SubmissionHistory.kt index 15f7af3aaee..3b287cb908b 100644 --- a/prime-router/src/main/kotlin/history/SubmissionHistory.kt +++ b/prime-router/src/main/kotlin/history/SubmissionHistory.kt @@ -182,6 +182,12 @@ class DetailedSubmissionHistory( @JsonIgnore var nextActionScheduled = false + /** + * Flag to check if there's a next action for the newest report in this submission + */ + @JsonIgnore + var hasNextAction = false + /** * The step in the delivery process for a submission * Supported values: @@ -297,17 +303,24 @@ class DetailedSubmissionHistory( // if there is ANY action scheduled on this submission history, ensure this flag is true if (report.nextActionAt != null) nextActionScheduled = true } + val sortedReports = reports.sortedBy { it.createdAt } destinations.forEach { destination -> - val reportsForDestination = reports.filter { + val reportsForDestination = sortedReports.filter { destination.organizationId == it.receivingOrg && destination.service == it.receivingOrgSvc - }.sortedBy { it.createdAt } - val latestAction = reportsForDestination.first().nextAction + } + val latestAction = reportsForDestination.last().nextAction val reportsGroupedByLatestAction = reportsForDestination.groupBy { it.nextAction } val mostRecentReportsForDestination = reportsGroupedByLatestAction[latestAction] ?: emptyList() destination.itemCount = mostRecentReportsForDestination.sumOf { it.itemCount } destination.itemCountBeforeQualFilter = mostRecentReportsForDestination.sumOf { it.itemCountBeforeQualFilter ?: 0 } } + if (destinations.isEmpty() && + sortedReports.isNotEmpty() && + sortedReports.last().nextAction != TaskAction.none + ) { + hasNextAction = true + } errors.addAll(consolidateLogs(ActionLogLevel.error)) warnings.addAll(consolidateLogs(ActionLogLevel.warning)) } @@ -384,7 +397,7 @@ class DetailedSubmissionHistory( * the receivers. */ return if ( - reports.size > 1 + reports.size > 1 && !hasNextAction ) { Status.NOT_DELIVERING } else { diff --git a/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt b/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt index 7dfa4555e90..340261bce85 100644 --- a/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt +++ b/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt @@ -25,6 +25,7 @@ class ReportGraphBuilder { private lateinit var theTopic: Topic private lateinit var theFormat: MimeFormat private lateinit var theSender: Sender + private lateinit var theNextAction: TaskAction fun topic(topic: Topic) { this.theTopic = topic @@ -38,6 +39,10 @@ class ReportGraphBuilder { this.theSender = sender } + fun nextAction(nextAction: TaskAction) { + this.theNextAction = nextAction + } + fun submission(initializer: ReportNodeBuilder.() -> Unit) { this.theSubmission = ReportNodeBuilder().apply(initializer) } @@ -80,7 +85,14 @@ class ReportGraphBuilder { .setItemCount(theSubmission.theItemCount) .setExternalName("test-external-name") .setBodyUrl(theSubmission.theReportBlobUrl) - .setNextAction(theSubmission.reportGraphNodes.firstOrNull()?.theAction) + .setNextAction( + if (::theNextAction.isInitialized) { + theNextAction + } else { + theSubmission.reportGraphNodes.firstOrNull()?.theAction + } + ) + .setCreatedAt(OffsetDateTime.now()) dbAccess.insertReportFile( reportFile, txn, action ) @@ -130,7 +142,14 @@ class ReportGraphBuilder { .setExternalName("test-external-name") .setBodyUrl(node.theReportBlobUrl) .setTransportResult(node.theTransportResult) - .setNextAction(node.reportGraphNodes.firstOrNull()?.theAction) + .setNextAction( + if (node.theNextAction != null) { + node.theNextAction + } else { + node.reportGraphNodes.firstOrNull()?.theAction + } + ) + .setCreatedAt(graph.node.createdAt.plusMinutes(1)) if (node.receiver != null) { childReportFile.setReceivingOrg(node.receiver!!.organizationName) @@ -189,6 +208,7 @@ class ReportNodeBuilder { } } lateinit var theAction: TaskAction + var theNextAction: TaskAction? = null var theReportBlobUrl: String = UUID.randomUUID().toString() var theItemCount: Int = 1 val reportGraphNodes: MutableList = mutableListOf() @@ -204,6 +224,10 @@ class ReportNodeBuilder { this.theAction = action } + fun nextAction(nextAction: TaskAction) { + this.theNextAction = nextAction + } + fun reportBlobUrl(reportBlobUrl: String) { this.theReportBlobUrl = reportBlobUrl } diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt index 6092e53516f..e35dbd95155 100644 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt @@ -368,6 +368,8 @@ class FHIRReceiverFilterIntegrationTests : Logging { assertThat(routedReport.schemaTopic).isEqualTo(Topic.FULL_ELR) assertThat(routedReport.bodyFormat).isEqualTo("FHIR") assertThat(routedReport.itemCount).isZero() + assertThat(routedReport.receivingOrg).isEqualTo(receiverSetupData.single().orgName) + assertThat(routedReport.receivingOrgSvc).isEqualTo(receiverSetupData.single().name) // check for no queue message verify(exactly = 0) { @@ -543,6 +545,8 @@ class FHIRReceiverFilterIntegrationTests : Logging { assertThat(routedReport.schemaTopic).isEqualTo(Topic.FULL_ELR) assertThat(routedReport.bodyFormat).isEqualTo("FHIR") assertThat(routedReport.itemCount).isZero() + assertThat(routedReport.receivingOrg).isEqualTo(receiverSetupData.single().orgName) + assertThat(routedReport.receivingOrgSvc).isEqualTo(receiverSetupData.single().name) // check for no queue message verify(exactly = 0) { @@ -731,6 +735,8 @@ class FHIRReceiverFilterIntegrationTests : Logging { assertThat(routedReport.schemaTopic).isEqualTo(Topic.FULL_ELR) assertThat(routedReport.bodyFormat).isEqualTo("FHIR") assertThat(routedReport.itemCount).isZero() + assertThat(routedReport.receivingOrg).isEqualTo(receiverSetupData.single().orgName) + assertThat(routedReport.receivingOrgSvc).isEqualTo(receiverSetupData.single().name) // check queue message verify(exactly = 0) { @@ -836,6 +842,8 @@ class FHIRReceiverFilterIntegrationTests : Logging { assertThat(routedReport.schemaTopic).isEqualTo(Topic.FULL_ELR) assertThat(routedReport.bodyFormat).isEqualTo("FHIR") assertThat(routedReport.itemCount).isZero() + assertThat(routedReport.receivingOrg).isEqualTo(receiverSetupData.single().orgName) + assertThat(routedReport.receivingOrgSvc).isEqualTo(receiverSetupData.single().name) // check filter logging val actionLogRecords = DSL.using(txn) @@ -1115,6 +1123,8 @@ class FHIRReceiverFilterIntegrationTests : Logging { assertThat(routedReport.schemaTopic).isEqualTo(Topic.FULL_ELR) assertThat(routedReport.bodyFormat).isEqualTo("FHIR") assertThat(routedReport.itemCount).isZero() + assertThat(routedReport.receivingOrg).isEqualTo(receiverSetupData.single().orgName) + assertThat(routedReport.receivingOrgSvc).isEqualTo(receiverSetupData.single().name) // check filter logging val actionLogRecords = DSL.using(txn) diff --git a/prime-router/src/test/kotlin/fhirengine/translation/hl7/FhirTransformerTests.kt b/prime-router/src/test/kotlin/fhirengine/translation/hl7/FhirTransformerTests.kt index 8577dd4abd5..1829329a702 100644 --- a/prime-router/src/test/kotlin/fhirengine/translation/hl7/FhirTransformerTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/translation/hl7/FhirTransformerTests.kt @@ -2,6 +2,7 @@ package gov.cdc.prime.router.fhirengine.translation.hl7 import assertk.assertFailure import assertk.assertThat +import assertk.assertions.contains import assertk.assertions.containsOnly import assertk.assertions.hasSize import assertk.assertions.isEmpty @@ -45,6 +46,7 @@ import org.hl7.fhir.r4.model.Observation import org.hl7.fhir.r4.model.Patient import org.hl7.fhir.r4.model.ServiceRequest import org.hl7.fhir.r4.model.StringType +import org.junit.jupiter.api.assertThrows import java.io.File import java.text.SimpleDateFormat import java.util.Date @@ -1240,6 +1242,92 @@ class FhirTransformerTests { assertThat(patient.name).hasSize(2) } + @Test + fun `test accessing by index while setting the actual bundle property`() { + val bundle = Bundle() + bundle.id = "abc123" + val patient = Patient() + val name = HumanName() + name.given = mutableListOf(StringType("foo"), StringType("bar")) + patient.name = mutableListOf(name) + patient.id = "def456" + val patientEntry = bundle.addEntry() + patientEntry.fullUrl = patient.id + patientEntry.resource = patient + + val updateFirstGivenNameSchemaElement = FhirTransformSchemaElement( + "update-first-given-name", + resource = "Bundle.entry.resource.ofType(Patient).name", + bundleProperty = "%resource.given[0]", + value = listOf("''") + ) + val schema = FhirTransformSchema(elements = mutableListOf(updateFirstGivenNameSchemaElement)) + + val transformer = FhirTransformer(schema) + val exception = assertThrows { + transformer.process(bundle) + } + assertThat(exception.message) + .contains("Schema is attempting to set a value for a particular index which is not allowed") + } + + @Test + fun `test deidentify human name`() { + val bundle = Bundle() + bundle.id = "abc123" + val patient = Patient() + val name = HumanName() + name.given = mutableListOf(StringType("foo"), StringType("bar")) + name.family = "family" + patient.name = mutableListOf(name) + patient.id = "def456" + val patientEntry = bundle.addEntry() + patientEntry.fullUrl = patient.id + patientEntry.resource = patient + + val updateFirstGivenNameSchemaElement = FhirTransformSchemaElement( + "update-first-given-name", + resource = "Bundle.entry.resource.ofType(Patient)", + bundleProperty = "%resource.name", + function = "deidentifyHumanName()", + ) + val schema = FhirTransformSchema(elements = mutableListOf(updateFirstGivenNameSchemaElement)) + + val transformer = FhirTransformer(schema) + transformer.process(bundle) + assertThat(name.given).isEmpty() + assertThat(name.family).isNull() + } + + @Test + fun `test deidentify human name with a value`() { + val bundle = Bundle() + bundle.id = "abc123" + val patient = Patient() + val name = HumanName() + name.given = mutableListOf(StringType("foo"), StringType("bar")) + name.family = "family" + patient.name = mutableListOf(name) + patient.id = "def456" + val patientEntry = bundle.addEntry() + patientEntry.fullUrl = patient.id + patientEntry.resource = patient + + val updateFirstGivenNameSchemaElement = FhirTransformSchemaElement( + "update-first-given-name", + resource = "Bundle.entry.resource.ofType(Patient)", + bundleProperty = "%resource.name", + function = "deidentifyHumanName('deidentified')", + ) + val schema = FhirTransformSchema(elements = mutableListOf(updateFirstGivenNameSchemaElement)) + + val transformer = FhirTransformer(schema) + transformer.process(bundle) + assertThat(name.given).transform { it -> it.map { st -> st.value } } + .containsOnly("deidentified", "deidentified") + assertThat(name.family).isEqualTo("deidentified") + } + @Test @Suppress("ktlint:standard:max-line-length") fun `test move Observation to ServiceRequest note`() { diff --git a/prime-router/src/test/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctionsTests.kt b/prime-router/src/test/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctionsTests.kt index e53c36c51e4..b81f4143a72 100644 --- a/prime-router/src/test/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctionsTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/translation/hl7/utils/CustomFHIRFunctionsTests.kt @@ -2,6 +2,7 @@ package gov.cdc.prime.router.fhirengine.translation.hl7.utils import assertk.assertFailure import assertk.assertThat +import assertk.assertions.containsOnly import assertk.assertions.doesNotHaveClass import assertk.assertions.hasClass import assertk.assertions.isEmpty @@ -15,6 +16,7 @@ import gov.cdc.prime.router.fhirengine.translation.hl7.SchemaException import org.hl7.fhir.r4.model.Base import org.hl7.fhir.r4.model.BaseDateTimeType import org.hl7.fhir.r4.model.DateTimeType +import org.hl7.fhir.r4.model.HumanName import org.hl7.fhir.r4.model.InstantType import org.hl7.fhir.r4.model.IntegerType import org.hl7.fhir.r4.model.MessageHeader @@ -503,4 +505,29 @@ class CustomFHIRFunctionsTests { ) }.hasClass(SchemaException::class.java) } + + @Test + fun `test deidentifies a human name`() { + val name = HumanName() + name.given = mutableListOf(StringType("foo"), StringType("bar")) + name.family = "family" + + CustomFHIRFunctions.deidentifyHumanName(mutableListOf(name), mutableListOf()) + + assertThat(name.given).isEmpty() + assertThat(name.family).isNull() + + val name2 = HumanName() + name2.given = mutableListOf(StringType("foo"), StringType("bar")) + name2.family = "family" + + CustomFHIRFunctions.deidentifyHumanName( + mutableListOf(name2), + mutableListOf(mutableListOf(StringType("baz"))) + ) + + assertThat(name2.given).transform { it -> it.map { st -> st.value } } + .containsOnly("baz", "baz") + assertThat(name2.family).isEqualTo("baz") + } } \ No newline at end of file diff --git a/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt b/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt index 1493ec5daf6..4df2eccea16 100644 --- a/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt +++ b/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt @@ -38,6 +38,37 @@ import org.testcontainers.junit.jupiter.Testcontainers @ExtendWith(ReportStreamTestDatabaseSetupExtension::class) class SubmissionFunctionIntegrationTests { + @Test + fun `it should return a history for a received report`() { + val submittedReport = reportGraph { + topic(Topic.FULL_ELR) + format(MimeFormat.HL7) + sender(UniversalPipelineTestUtils.hl7Sender) + + submission { + action(TaskAction.receive) + reportGraphNode { + action(TaskAction.convert) + } + } + }.generate(ReportStreamTestDatabaseContainer.testDatabaseAccess) + + val httpRequestMessage = MockHttpRequestMessage() + + val func = setupSubmissionFunction() + + val history = func + .getReportDetailedHistory(httpRequestMessage, submittedReport.node.reportId.toString()) + assertThat(history).isNotNull() + val historyNode = JacksonMapperUtilities.defaultMapper.readTree(history.body.toString()) + assertThat( + historyNode.get("overallStatus").asText() + ).isEqualTo(DetailedSubmissionHistory.Status.RECEIVED.toString()) + assertThat(historyNode.get("destinations").size()).isEqualTo(0) + assertThat(historyNode.get("errors").size()).isEqualTo(0) + assertThat(historyNode.get("warnings").size()).isEqualTo(0) + } + @Test fun `it should return a history for partially delivered submission`() { val submittedReport = reportGraph { @@ -201,6 +232,7 @@ class SubmissionFunctionIntegrationTests { log(ActionLog(InvalidParamMessage("log"), type = ActionLogLevel.warning)) reportGraphNode { action(TaskAction.destination_filter) + nextAction(TaskAction.none) } } } @@ -237,6 +269,7 @@ class SubmissionFunctionIntegrationTests { log(ActionLog(InvalidParamMessage("log"), type = ActionLogLevel.warning)) reportGraphNode { action(TaskAction.route) + nextAction(TaskAction.none) } } }