diff --git a/.environment/gitleaks/gitleaks-config.toml b/.environment/gitleaks/gitleaks-config.toml index 883ce714aff..552a31cfa0e 100644 --- a/.environment/gitleaks/gitleaks-config.toml +++ b/.environment/gitleaks/gitleaks-config.toml @@ -3,45 +3,36 @@ title = "PRIME ReportStream Gitleaks Configuration" # Global allowlist [allowlist] description = "Allow-list for files and paths" - files = [ - '(.*?)(bin|doc|gif|iml|jar|jp(e)?g|pdf|png|xlsx)$', - '^\.?gitleaks-config.toml$', - '^\.?gitleaks.report.json$', - '^package-lock\.json$', - 'cleanslate.sh.log', - 'yarn\.lock$', - ] paths = [ - '.environment/sftp-conf', - '.environment/soap_service/', - '.github/scripts/stale_items_report/', - '.idea/', - '.terraform/providers/', - 'frontend/dist', - 'frontend/node_modules/', - 'frontend/src/assets', - 'frontend-react/build/', - 'frontend-react/node_modules/', - 'frontend-react/src/components/ReportStreamHeader.tsx', - 'prime-router/.gradle/', - 'prime-router/.vault/env/', - 'prime-router/build/', - 'prime-router/build.gradle.kts', - 'prime-router/docs/dependency-graph-full/dependency-graph-full.txt', - 'prime-router/docs/schema_documentation/', - 'prime-router/docs/design/design/auth/auth-design.md', - 'prime-router/docs/getting_started.md', - 'prime-router/frontend/src/assets/fonts', - 'prime-router/frontend/src/assets/img', - 'prime-router/frontend/src/assets/pdf', - 'prime-router/frontend/src/assets/webfonts', - 'prime-router/src/main/kotlin/cli/tests/TestKeys.kt', - 'prime-router/src/test/csv_test_files/input/', - 'prime-router/src/test/kotlin/credentials/CredentialTests', - 'prime-router/src/test/', - 'prime-router/src/main/resources/metadata', - '.environment/gitleaks/gitleaks-config.toml', - 'exp/as2/keystore_steps.md', + # package manager files + 'package-lock\.json$', + 'yarn\.lock$', + # ide + '\.idea\/', + # misc + '(.*?)(bin|doc|gif|iml|jar|jp(e)?g|pdf|png|xlsx)$', + # devops + '\.terraform\/providers\/', + '^\.environment\/gitleaks\/gitleaks-config\.toml$', + '^\.environment\/sftp-conf\/', + '^\.environment\/soap_service\/', + '^\.github\/scripts\/stale_items_report\/', + # backend + '^prime-router\/\.gradle\/', + '^prime-router\/.vault\/env\/', + '^prime-router\/build\/', + '^prime-router\/build\.gradle\.kts', + '^prime-router\/docs\/dependency-graph-full/dependency-graph-full\.txt', + '^prime-router\/docs\/schema_documentation/', + '^prime-router\/docs\/design/design/auth/auth-design\.md', + '^prime-router\/docs\/getting_started\.md', + '^prime-router\/src\/main\/kotlin\/cli\/tests\/TestKeys\.kt', + '^prime-router\/src\/test\/csv_test_files\/input\/', + '^prime-router\/src\/test\/kotlin\/credentials\/CredentialTests', + '^prime-router\/src\/test\/', + '^prime-router\/src\/main\/resources\/metadata', + # frontend + '^frontend-react\/public\/assets\/', ] [[rules]] diff --git a/.environment/gitleaks/run-gitleaks.sh b/.environment/gitleaks/run-gitleaks.sh index 2c9db1866ed..0db19703c71 100755 --- a/.environment/gitleaks/run-gitleaks.sh +++ b/.environment/gitleaks/run-gitleaks.sh @@ -230,7 +230,8 @@ esac if [[ ${RC?} != 0 ]]; then error "(return code=${RC?}) Your code may contain secrets, consult the output above and/or one of the following files for more details:" error " - ${REPO_ROOT?}/${REPORT_JSON?}" - error " - ${REPO_ROOT?}/${LOGFILE?}" + # no log file currently, check the output of whatever ran this + # error " - ${REPO_ROOT?}/${LOGFILE?}" fi exit ${RC?} diff --git a/.github/ISSUE_TEMPLATE/platform-epic-template.md b/.github/ISSUE_TEMPLATE/platform-epic-template.md new file mode 100644 index 00000000000..f6bd412152e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/platform-epic-template.md @@ -0,0 +1,29 @@ +--- +name: Platform Epic +about: Platform team's Epic template +title: Platform Epic Title +labels: platform +assignees: '' + +--- + +## Outcome/Objective + +## Description + +## Product Requirement(s) + +## Use Case(s) + +## Dependencies + +## Acceptance criteria + +## Technical Requirement(s) + \ No newline at end of file diff --git a/frontend-react/.eslintrc.cjs b/frontend-react/.eslintrc.cjs deleted file mode 100644 index 36139517a94..00000000000 --- a/frontend-react/.eslintrc.cjs +++ /dev/null @@ -1,149 +0,0 @@ -module.exports = { - root: true, - env: { browser: true, es2020: true }, - extends: [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended-type-checked", - "plugin:@typescript-eslint/stylistic-type-checked", - "plugin:react-hooks/recommended", - "plugin:react/recommended", - "plugin:react/jsx-runtime", - "plugin:jsx-a11y/recommended", - "plugin:import/recommended", - "plugin:import/typescript", - "prettier", - ], - ignorePatterns: [ - "build", - ".eslintrc.cjs", - "vite.config.ts", - "lint-staged.config.js", - "scripts", - "playwright.config.ts", - "jest.config.ts", - "coverage", - "storybook-static", - "e2e-data", - ], - parser: "@typescript-eslint/parser", - parserOptions: { - ecmaFeatures: { - jsx: true, - }, - project: true, - tsconfigRootDir: __dirname, - }, - plugins: ["react-refresh", "@typescript-eslint", "react-hooks", "react", "jsx-a11y", "import"], - settings: { - react: { - version: "detect", - }, - "import/resolver": { - typescript: true, - node: true, - }, - }, - overrides: [ - /* Vitest */ - { - files: ["./src/**/__tests__/**/*.[jt]s?(x)", "./src/**/?(*.)+(spec|test).[jt]s?(x)"], - extends: [ - "plugin:testing-library/react", - "plugin:vitest/legacy-recommended", - "plugin:jest-dom/recommended", - ], - rules: { - /* Temporarily changed to warnings or disabled pending future work */ - "testing-library/no-node-access": ["warn"], - "vitest/no-mocks-import": ["warn"], - "vitest/expect-expect": ["warn"], - - /* Tweaks for plugin conflicts */ - "@typescript-eslint/unbound-method": "off", - - /* Custom project rules */ - "testing-library/no-await-sync-events": ["error", { eventModules: ["fire-event"] }], - "testing-library/no-render-in-lifecycle": "error", - "testing-library/prefer-screen-queries": "warn", - "testing-library/no-unnecessary-act": "warn", - "testing-library/no-await-sync-queries": "warn", - }, - }, - /* Storybook */ - { - files: ["./src/**/?(*.)+(stories).[jt]s?(x)"], - extends: ["plugin:storybook/recommended"], - }, - /* Playwright */ - { - files: ["./e2e/**/?(*.)+(spec|test).[jt]s"], - extends: ["plugin:playwright/recommended"], - rules: { - // TODO: investigate these for reconsideration or per-module ignoring - "playwright/no-conditional-in-test": ["off"], - "playwright/no-force-option": ["off"], - "playwright/expect-expect": ["off"], - }, - }, - ], - rules: { - /* Temporarily changed to warnings or disabled pending future work */ - "jsx-a11y/no-autofocus": ["warn"], - "react-refresh/only-export-components": ["off", { allowConstantExport: true }], - - // Requires extensive updates to types in code, however SHOULD BE ENABLED EVENTUALLY - "react/prop-types": ["warn"], - "@typescript-eslint/no-unsafe-call": ["off"], - "@typescript-eslint/no-unsafe-member-access": ["off"], - "@typescript-eslint/no-unsafe-return": ["off"], - "@typescript-eslint/no-unsafe-argument": ["off"], - "@typescript-eslint/no-unsafe-assignment": ["off"], - - /* Tweaks for plugin conflicts */ - "import/named": ["off"], - "import/namespace": ["off"], - "import/default": ["off"], - "import/no-named-as-default-member": ["off"], - "import/no-unresolved": ["off"], - indent: ["off"], - "@typescript-eslint/indent": ["off"], - "require-await": "off", - - /* Custom project rules */ - "no-console": ["error", { allow: ["warn", "error", "info", "trace"] }], - "@typescript-eslint/no-explicit-any": ["off"], - "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": [ - "error", - { - vars: "all", - varsIgnorePattern: "^_", - args: "after-used", - argsIgnorePattern: "^_", - caughtErrors: "all", - caughtErrorsIgnorePattern: "^_", - }, - ], - "import/order": [ - 1, - { - groups: ["external", "builtin", "internal", "sibling", "parent", "index"], - pathGroups: [ - { pattern: "components", group: "internal" }, - { pattern: "common", group: "internal" }, - { pattern: "routes/**", group: "internal" }, - { - pattern: "assets/**", - group: "internal", - position: "after", - }, - ], - pathGroupsExcludedImportTypes: ["internal"], - alphabetize: { order: "asc", caseInsensitive: true }, - }, - ], - "sort-imports": ["error", { ignoreCase: true, ignoreDeclarationSort: true }], - "@typescript-eslint/prefer-nullish-coalescing": ["error"], - "@typescript-eslint/no-empty-object-type": ["error", { allowInterfaces: "always" }], - }, -}; diff --git a/frontend-react/docs/okta.md b/frontend-react/docs/okta.md new file mode 100644 index 00000000000..db98dcc567b --- /dev/null +++ b/frontend-react/docs/okta.md @@ -0,0 +1,13 @@ +# Okta-side configuration + +Our frontend is configured to identify as the "Web" application. + +## Dev-side configuration + +Our use of okta in frontend is configured by the following environment variables whose values can be found in the application listing within Okta: +-VITE_OKTA_CLIENTID +-VITE_OKTA_URL + +These variables can be assigned locally for local development (.env.*.local) or by github actions (using values in secrets storage either in github itself or azure). + +We use Okta's [Embedded Sign-In Widget for React](https://developer.okta.com/docs/guides/sign-in-to-spa-embedded-widget/react/main/), which includes other Okta-related libraries for react, to handle okta workflows. \ No newline at end of file diff --git a/frontend-react/e2e/spec/all/BasePage.spec.ts b/frontend-react/e2e/spec/all/BasePage.spec.ts index c0a688c2508..f384ab0ec7c 100644 --- a/frontend-react/e2e/spec/all/BasePage.spec.ts +++ b/frontend-react/e2e/spec/all/BasePage.spec.ts @@ -6,6 +6,7 @@ export interface MockPageFixtures { } class MockPage extends BasePage { + // eslint-disable-next-line @typescript-eslint/no-explicit-any data?: any; constructor(testArgs: BasePageTestArgs) { @@ -20,6 +21,7 @@ class MockPage extends BasePage { [ "fake", async (res) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment this.data = await res.json(); }, ], @@ -30,9 +32,7 @@ class MockPage extends BasePage { if (this.isMocked) { this.addMockRouteHandlers([["/fake", { json: { foo: "bar" } }]]); } else { - const [url, handler] = this.createRouteHandlers([ - ["/fake", { json: { bar: "foo" } }], - ])[0]; + const [url, handler] = this.createRouteHandlers([["/fake", { json: { bar: "foo" } }]])[0]; this.routeHandlers.set(url, handler); } diff --git a/frontend-react/e2e/test.ts b/frontend-react/e2e/test.ts index 00f822e12ac..ad9f65d3356 100644 --- a/frontend-react/e2e/test.ts +++ b/frontend-react/e2e/test.ts @@ -37,27 +37,15 @@ const isCI = Boolean(process.env.CI); const frontendWarningsPath = join(e2eDataPath, "frontend-warnings.json"); const isMockDisabled = Boolean(process.env.MOCK_DISABLED); -function createLogins( - loginTypes: T, -): { - [K in T extends readonly (infer U)[] ? U : never]: { - username: string; - password: string; - totpCode: string; - path: string; - }; -} { +function createLogins(loginTypes: T): Record { const logins = Object.fromEntries( loginTypes.map((type) => { const username = process.env[`TEST_${type.toUpperCase()}_USERNAME`]; const password = process.env[`TEST_${type.toUpperCase()}_PASSWORD`]; - const totpCode = - process.env[`TEST_${type.toUpperCase()}_TOTP_CODE`]; + const totpCode = process.env[`TEST_${type.toUpperCase()}_TOTP_CODE`]; - if (!username) - throw new TypeError(`Missing username for login type: ${type}`); - if (!password) - throw new TypeError(`Missing password for login type: ${type}`); + if (!username) throw new TypeError(`Missing username for login type: ${type}`); + if (!password) throw new TypeError(`Missing password for login type: ${type}`); return [ type, @@ -70,7 +58,7 @@ function createLogins( ]; }), ); - return logins as any; + return logins as Record; } export const logins = createLogins(["admin", "receiver", "sender"]); @@ -94,9 +82,4 @@ export const test = base.extend({ isFrontendWarningsLog: isCI, }); -export type TestArgs

= Pick< - PlaywrightAllTestArgs, - P -> & - CustomFixtures; - +export type TestArgs

= Pick & CustomFixtures; diff --git a/frontend-react/eslint.config.js b/frontend-react/eslint.config.js new file mode 100644 index 00000000000..005177499d6 --- /dev/null +++ b/frontend-react/eslint.config.js @@ -0,0 +1,220 @@ +import { fixupPluginRules } from "@eslint/compat"; +import eslint from "@eslint/js"; +import configPrettier from "eslint-config-prettier"; +import _import from "eslint-plugin-import"; +import jestDom from "eslint-plugin-jest-dom"; +import jsxA11y from "eslint-plugin-jsx-a11y"; +import playwright from "eslint-plugin-playwright"; +import react from "eslint-plugin-react"; +import reactHooks from "eslint-plugin-react-hooks"; +import reactRefresh from "eslint-plugin-react-refresh"; +import storybook from "eslint-plugin-storybook"; +import testingLibrary from "eslint-plugin-testing-library"; +import vitest from "eslint-plugin-vitest"; +import tseslint from "typescript-eslint"; + +const ignoredFiles = [ + "build/**/*", + "vite.config.ts", + "lint-staged.config.js", + "scripts/**/*", + "playwright.config.ts", + "jest.config.ts", + "coverage/**/*", + "storybook-static/**/*", + "e2e-data/**/*", + ".storybook/**/*", +]; + +const defaultConfig = { + extends: [ + eslint.configs.recommended, + ...tseslint.configs.recommendedTypeChecked, + ...tseslint.configs.stylisticTypeChecked, + jsxA11y.flatConfigs.recommended, + react.configs.flat?.recommended, + react.configs.flat?.["jsx-runtime"], + _import.flatConfigs.recommended, + _import.flatConfigs.react, + _import.flatConfigs.typescript, + configPrettier, + ], + plugins: { + "react-hooks": fixupPluginRules(reactHooks), + "react-refresh": reactRefresh, + }, + ignores: ignoredFiles, + languageOptions: { + parserOptions: { + projectService: true, + tsconfigRootDir: import.meta.dirname, + ecmaFeatures: { + jsx: true, + }, + }, + }, + settings: { + react: { + version: "detect", + }, + + "import/resolver": { + typescript: true, + node: true, + }, + }, + rules: { + "jsx-a11y/no-autofocus": ["warn"], + + "react-refresh/only-export-components": [ + "off", + { + allowConstantExport: true, + }, + ], + + "react/prop-types": ["warn"], + "@typescript-eslint/no-unsafe-call": ["off"], + "@typescript-eslint/no-unsafe-member-access": ["off"], + "@typescript-eslint/no-unsafe-return": ["off"], + "@typescript-eslint/no-unsafe-argument": ["off"], + "@typescript-eslint/no-unsafe-assignment": ["off"], + "import/named": ["off"], + "import/namespace": ["off"], + "import/default": ["off"], + "import/no-named-as-default-member": ["off"], + "import/no-unresolved": ["off"], + indent: ["off"], + "@typescript-eslint/indent": ["off"], + "require-await": "off", + + "no-console": [ + "error", + { + allow: ["warn", "error", "info", "trace"], + }, + ], + + "@typescript-eslint/no-explicit-any": ["off"], + "no-unused-vars": "off", + + "@typescript-eslint/no-unused-vars": [ + "error", + { + vars: "all", + varsIgnorePattern: "^_", + args: "after-used", + argsIgnorePattern: "^_", + caughtErrors: "all", + caughtErrorsIgnorePattern: "^_", + }, + ], + + "import/order": [ + 1, + { + groups: ["external", "builtin", "internal", "sibling", "parent", "index"], + + pathGroups: [ + { + pattern: "components", + group: "internal", + }, + { + pattern: "common", + group: "internal", + }, + { + pattern: "routes/**", + group: "internal", + }, + { + pattern: "assets/**", + group: "internal", + position: "after", + }, + ], + + pathGroupsExcludedImportTypes: ["internal"], + + alphabetize: { + order: "asc", + caseInsensitive: true, + }, + }, + ], + + "sort-imports": [ + "error", + { + ignoreCase: true, + ignoreDeclarationSort: true, + }, + ], + + "@typescript-eslint/prefer-nullish-coalescing": ["error"], + + "@typescript-eslint/no-empty-object-type": [ + "error", + { + allowInterfaces: "always", + }, + ], + + /* React-Hooks Recommended */ + "react-hooks/rules-of-hooks": "error", + "react-hooks/exhaustive-deps": "warn", + }, +}; + +export default tseslint.config( + defaultConfig, + /* Vitest */ + { + ...defaultConfig, + files: ["src/**/__tests__/**/*.[jt]s?(x)", "src/**/?(*.)+(spec|test).[jt]s?(x)"], + extends: [ + ...defaultConfig.extends, + testingLibrary.configs["flat/react"], + vitest.configs.recommended, + jestDom.configs["flat/recommended"], + ], + rules: { + ...defaultConfig.rules, + /* Temporarily changed to warnings or disabled pending future work */ + "testing-library/no-node-access": ["warn"], + "vitest/no-mocks-import": ["warn"], + "vitest/expect-expect": ["warn"], + + /* Tweaks for plugin conflicts */ + "@typescript-eslint/unbound-method": "off", + + /* Custom project rules */ + "testing-library/no-await-sync-events": ["error", { eventModules: ["fire-event"] }], + "testing-library/no-render-in-lifecycle": "error", + "testing-library/prefer-screen-queries": "warn", + "testing-library/no-unnecessary-act": "warn", + "testing-library/no-await-sync-queries": "warn", + }, + }, + /* Storybook */ + { + ...defaultConfig, + files: ["src/**/?(*.)+(stories).[jt]s?(x)"], + extends: [...defaultConfig.extends, ...storybook.configs["flat/recommended"]], + }, + /* Playwright */ + { + ...defaultConfig, + files: ["e2e/**/?(*.)+(spec|test).[jt]s"], + extends: [...defaultConfig.extends, playwright.configs["flat/recommended"]], + rules: { + // TODO: investigate these for reconsideration or per-module ignoring + "playwright/no-conditional-in-test": ["off"], + "playwright/no-force-option": ["off"], + "playwright/expect-expect": ["off"], + + "react-hooks/rules-of-hooks": "off", + }, + }, +); diff --git a/frontend-react/package.json b/frontend-react/package.json index a1a5e5ea824..fad46aa994e 100644 --- a/frontend-react/package.json +++ b/frontend-react/package.json @@ -110,6 +110,8 @@ ] }, "devDependencies": { + "@eslint/compat": "^1.2.2", + "@eslint/js": "^9.13.0", "@mdx-js/react": "^3.1.0", "@mdx-js/rollup": "^3.1.0", "@playwright/test": "^1.48.1", @@ -134,6 +136,7 @@ "@types/dompurify": "^3.0.5", "@types/dotenv-flow": "^3.3.3", "@types/downloadjs": "^1.4.6", + "@types/eslint__js": "^8.42.3", "@types/github-slugger": "^1.3.0", "@types/html-to-text": "^9.0.4", "@types/lodash": "^4.17.12", @@ -144,8 +147,6 @@ "@types/react-router-dom": "^5.3.3", "@types/react-scroll-sync": "^0.9.0", "@types/sanitize-html": "^2.13.0", - "@typescript-eslint/eslint-plugin": "^8.10.0", - "@typescript-eslint/parser": "^8.10.0", "@vitejs/plugin-react": "^4.3.3", "@vitest/coverage-istanbul": "^2.1.3", "@vitest/ui": "^2.1.3", @@ -155,25 +156,26 @@ "chromatic": "^11.12.6", "cross-env": "^7.0.3", "dotenv-flow": "^4.1.0", - "eslint": "8.57", + "eslint": "9.13.0", "eslint-config-prettier": "^9.1.0", - "eslint-import-resolver-typescript": "^3.6.1", - "eslint-plugin-import": "^2.29.1", + "eslint-import-resolver-typescript": "^3.6.3", + "eslint-plugin-import": "^2.31.0", "eslint-plugin-jest-dom": "^5.4.0", - "eslint-plugin-jsx-a11y": "^6.9.0", - "eslint-plugin-playwright": "^1.8.1", - "eslint-plugin-react": "^7.34.3", - "eslint-plugin-react-hooks": "^4.6.2", - "eslint-plugin-react-refresh": "^0.4.7", + "eslint-plugin-jsx-a11y": "^6.10.2", + "eslint-plugin-playwright": "^2.0.0", + "eslint-plugin-react": "^7.37.2", + "eslint-plugin-react-hooks": "^5.0.0", + "eslint-plugin-react-refresh": "^0.4.14", "eslint-plugin-storybook": "^0.10.1", "eslint-plugin-testing-library": "^6.4.0", "eslint-plugin-vitest": "^0.5.4", + "globals": "^15.11.0", "husky": "^9.1.6", "jsdom": "^25.0.1", "lint-staged": "^15.2.10", "mockdate": "^3.0.5", "msw": "^2.4.11", - "msw-storybook-addon": "beta", + "msw-storybook-addon": "^2.0.3", "npm-run-all": "^4.1.5", "otpauth": "^9.3.4", "patch-package": "^8.0.0", @@ -189,8 +191,9 @@ "ts-node": "^10.9.2", "tslib": "^2.8.0", "typescript": "^5.6.3", + "typescript-eslint": "^8.12.2", "undici": "^6.20.1", - "vite": "^5.4.9", + "vite": "^5.4.10", "vite-plugin-checker": "^0.8.0", "vite-plugin-svgr": "^4.2.0", "vitest": "^2.1.3" diff --git a/frontend-react/src/__mockServers__/LookupTableMockServer.ts b/frontend-react/src/__mockServers__/LookupTableMockServer.ts index 105012a865d..3bba6a45e5c 100644 --- a/frontend-react/src/__mockServers__/LookupTableMockServer.ts +++ b/frontend-react/src/__mockServers__/LookupTableMockServer.ts @@ -1,11 +1,7 @@ import { http, HttpResponse } from "msw"; import { setupServer } from "msw/node"; -import { - ApiValueSet, - LookupTable, - lookupTablesEndpoints, -} from "../config/endpoints/lookupTables"; +import { ApiValueSet, LookupTable, lookupTablesEndpoints } from "../config/endpoints/lookupTables"; const tableListUrl = lookupTablesEndpoints.getTableList.toDynamicUrl(); const tableDataUrl = lookupTablesEndpoints.getTableData.toDynamicUrl({ @@ -45,8 +41,8 @@ const lookupTables: LookupTable[] = [ const lookupTableData: ApiValueSet[] = [1, 2, 3].map((_i) => ({ name: "sender_automation_value_set", - created_by: "", // eslint-disable-line camelcase - created_at: "", // eslint-disable-line camelcase + created_by: "", + created_at: "", system: "LOCAL", reference: "unused", referenceURL: "https://unused", diff --git a/frontend-react/src/components/FileHandlers/FileHandler.tsx b/frontend-react/src/components/FileHandlers/FileHandler.tsx index a3123ededf7..4c82d635cb6 100644 --- a/frontend-react/src/components/FileHandlers/FileHandler.tsx +++ b/frontend-react/src/components/FileHandlers/FileHandler.tsx @@ -10,14 +10,11 @@ import { WatersResponse } from "../../config/endpoints/waters"; import site from "../../content/site.json"; import { showToast } from "../../contexts/Toast"; import useOrganizationSettings from "../../hooks/api/organizations/UseOrganizationSettings/UseOrganizationSettings"; -import useFileHandler, { - FileHandlerActionType, - FileHandlerState, -} from "../../hooks/UseFileHandler/UseFileHandler"; +import useFileHandler, { FileHandlerActionType, FileHandlerState } from "../../hooks/UseFileHandler/UseFileHandler"; import { SchemaOption } from "../../hooks/UseSenderSchemaOptions/UseSenderSchemaOptions"; import Alert from "../../shared/Alert/Alert"; import Spinner from "../Spinner"; -import { USExtLink, USLink } from "../USLink"; +import { USExtLink } from "../USLink"; export interface FileHandlerStepProps extends FileHandlerState { isValid?: boolean; @@ -27,8 +24,7 @@ export interface FileHandlerStepProps extends FileHandlerState { } function mapStateToOrderedSteps(state: FileHandlerState) { - const { selectedSchemaOption, file, errors, warnings, overallStatus } = - state; + const { selectedSchemaOption, file, errors, warnings, overallStatus } = state; return [ { @@ -42,9 +38,7 @@ function mapStateToOrderedSteps(state: FileHandlerState) { { Component: FileHandlerErrorsWarningsStep, isValid: false, - shouldSkip: Boolean( - overallStatus && errors.length === 0 && warnings.length === 0, - ), + shouldSkip: Boolean(overallStatus && errors.length === 0 && warnings.length === 0), }, { Component: FileHandlerSuccessStep, @@ -56,15 +50,9 @@ function mapStateToOrderedSteps(state: FileHandlerState) { export default function FileHandler() { const { state, dispatch } = useFileHandler(); const { fileName, localError } = state; - const orderedSteps = mapStateToOrderedSteps(state).filter( - (step) => !step.shouldSkip, - ); + const orderedSteps = mapStateToOrderedSteps(state).filter((step) => !step.shouldSkip); const [currentStepIndex, setCurrentStepIndex] = useState(0); - const { - Component: StepComponent, - isValid, - shouldSkip, - } = orderedSteps[currentStepIndex]; + const { Component: StepComponent, isValid, shouldSkip } = orderedSteps[currentStepIndex]; useEffect(() => { if (localError) { @@ -142,31 +130,19 @@ export default function FileHandler() { name="description" content="Check that public health entities can receive your data through ReportStream by validating your file format." /> - - + +

ReportStream File Validator

- {organization?.description && ( -

- {organization.description} -

- )} + {organization?.description &&

{organization.description}

} {fileName && (
-

- File name -

+

File name

{fileName}

)} @@ -192,12 +168,8 @@ export default function FileHandler() { ); @@ -205,9 +177,7 @@ export default function FileHandler() { return ( ); case FileHandlerSuccessStep: @@ -219,20 +189,13 @@ export default function FileHandler() { {StepComponent !== FileHandlerSuccessStep && ( - Reference{" "} - - the data model - {" "} - for the information needed to validate your file - successfully. Pay special attention to which fields - are required and common mistakes. + Reference the data model for the information needed to validate your file successfully. Pay + special attention to which fields are required and common mistakes. )}

Questions or feedback? Please email{" "} - - {site.orgs.RS.email} - + {site.orgs.RS.email}

diff --git a/frontend-react/src/content/about/roadmap.mdx b/frontend-react/src/content/about/roadmap.mdx index c5749c1e094..f913f89536b 100644 --- a/frontend-react/src/content/about/roadmap.mdx +++ b/frontend-react/src/content/about/roadmap.mdx @@ -17,23 +17,27 @@ import site from "../../content/site.json" -

Last updated: October 07, 2024

+

Last updated: October 29, 2024

-## Objectives for FY24 -Learn about our goals and how we are measuring success for this fiscal year (FY), which runs from October 2023 through September 2024. +## Objectives for FY25 +Learn about our goals and how we are measuring success for this fiscal year (FY), which runs from October 2024 through September 2025.
### Support public health entities with time-sensitive, reportable condition electronic laboratory reporting (ELR) data and broader response readiness. * 100% of the 64 ELC-funded state, tribal, local, or territorial public health agencies (STLTs) are connected to ReportStream. -* At least 15 ELC-funded STLTs receive live data for two or more reportable conditions from ReportStream via ELR. -* Complete connection updates for all STLTs currently receiving COVID data so they are able to receive more conditions. -* Begin updating the connection of organizations sending COVID data so they are able to send additional conditions through ReportStream. -* Identify one or more new health care organizations, labs, or testing facilities for collaboration on their data reporting needs. +* Connect at least 5 new healthcare organizations to ReportStream for sending all of their reportable conditions. +* Route lab data for at least 15 reportable conditions. +* Complete connection updates for all healthcare organizations and STLTs currently receiving COVID data so they are able to send or receive data for more conditions. +* Build basic functionality to allow ReportStream to ingest CSV via an API for conditions beyond COVID. -### Pilot faster, simpler electronic test orders and results (ETOR) data transfer with federal and state public health labs. -* Complete successful test delivery of ELR from CDC’s infectious disease labs to STLTs. -* Support pilots for newborn screening ETOR with one or more hospital systems and state public health labs. +### Facilitate simple, efficient reporting for additional core public health data sources and public health systems. +* Route hospitalization data from at least one healthcare organization to the CDC National Syndromic Surveillance Program (NSSP). +* Route hospitalization data from at least one healthcare organization to one STLT. +* Connect ReportStream to one federal healthcare system outside of the CDC. +* Participate in CDC One Health activities and/or CDC Laboratory Response Network. +* Connect ReportStream to three third-party standards/services. +* Meet the needs of senders and STLTs by routing data in accordance with the HL7 LRI/LOI spec. ## Roadmap
@@ -42,11 +46,11 @@ Learn about our goals and how we are measuring success for this fiscal year (FY) isDivided={false} sections={[ { - title: "Enable STLTs to use ReportStream for conditions beyond COVID and mpox.", + title: "Strengthen response readiness by enabling healthcare organizations and STLTs to use ReportStream for conditions beyond COVID and mpox.", tag: "recently-completed", body: (

- Arkansas, Florida, Guam, Idaho, Illinois, Louisiana, Maine, Massachusetts, Minnesota, and Oklahoma are able to receive flu and RSV data from ReportStream. + Alabama, Maine, Missouri, Montana and South Dakota are able to receive flu and RSV data from ReportStream.

), }, @@ -54,85 +58,85 @@ Learn about our goals and how we are measuring success for this fiscal year (FY) tag: "working-on-now", body: (

- Onboard or update connections for Alabama, Connecticut, Delaware, Indiana, Iowa, Kentucky, Michigan, Mississippi, Missouri, Montana, New Hampshire, New Jersey, New Mexico, Ohio, Pennsylvania, Puerto Rico, Republic of Marshall Islands, South Dakota, Tennessee, Vermont, Virginia, Washington, Washington D.C., and Wyoming. + Onboard or update current connections for Delaware, New Jersey, Ohio, and Pennsylvania.

), }, { tag: "next", body: ( -

Support remaining STLTs to connect or update their connections to be able to receive additional conditions.

+

Onboard or update connections for American Samoa, Connecticut, Indiana, Iowa, Kentucky, Michigan, Mississippi, New Hampshire, New Mexico, Puerto Rico, Republic of Marshall Islands, Tennessee, Utah, Vermont, Virginia, Washington, Washington D.C., and Wyoming.

), }, + , { - title:

Partner with SimpleReport to help nontraditional and lower-resourced testing sites and facilities report test results and send data to STLTs and the CDC.

, - tag: "recently-completed", + tag: "next", body: ( -

Contacted nontraditional and lower-resourced HIV senders in Texas as part of targeted health equity improvement work.

+

Onboard 3 senders to ReportStream sending all of their ELR reportable conditions.

), }, { + title:

Route live hospitalization data from a healthcare system to a STLT and the CDC National Syndromic Surveillance Program (NSSP).

, tag: "recently-completed", body: ( -

- Alaska, Arizona, California, Colorado, Florida, Hawaii, Idaho, Illinois, Louisiana, Massachusetts, Minnesota, Nevada, New York, and Rhode Island are receiving flu A/B or RSV data. -

+

Design and implement an SFTP bucket to allow for submissions to meet submission needs of NSSP.

), }, { tag: "working-on-now", body: ( -

Transmit HIV test results to Los Angeles County and Texas.

+

Align on the technical requirements needed to route admission, discharge, and transfer (ADT) data from a hospital system to CDC NSSP with the Oklahoma State Department of Health.

), }, { - tag: "working-on-now", + tag: "next", body: ( -

Add support for hepatitis C and other STI reporting through SimpleReport.

+

Connect a hospital system and CDC NSSP to ReportStream to enable routing of ADT data.

), }, { - tag: "next", + title:

Connect ReportStream to one federal healthcare system outside of the CDC.

, + tag: "working-on-now", body: ( -

Add support for more conditions reporting through SimpleReport.

+

Route test results from MakeMyTestCount.org to one STLT in production.

), }, { - title:

Help at-home test takers and test manufacturers easily report results to STLTs and CDC.

, - tag: "recently-completed", + tag: "next", body: ( -

Route test results from MakeMyTestCount.org to one STLT in test environment.

+

Route test results from MakeMyTestCount.org to three more STLTs in production.

), }, { + title:

Participate in the public health community by partnering with other CDC organizations and engaging with third-party health IT standards/services.

, tag: "working-on-now", body: ( -

Route test results from MakeMyTestCount.org to one STLT in production.

+

Conduct discovery research to understand TEFCA’s live and upcoming public health use cases.

), }, { tag: "next", body: ( -

Route test results from another OTC testing (point-of-care) sender to one STLT in production.

+

Develop opportunity brief to identify next steps for ReportStream to participate in TEFCA.

), }, { - title: "Facilitate electronic laboratory reporting (ELR) for the CDC infectious disease labs and transmit to STLTs. ", + title:

Meet the needs of healthcare organizations and STLTs by routing data in accordance with the HL7 LRI/LOI spec.

, tag: "recently-completed", body: ( -

Complete final internal review of test result data transfer before moving to production.

+

Aligned with partners and user research on which part of the LRI/LOI spec will be most immediately useful.

), }, { - tag: "next", + tag: "working-on-now", body: ( -

Execute go-live plan with state public health agencies receiving test results.

+

Implement Basic Synchronous Accept ACKs in Submission API per LRI/LOI spec for the ETOR use case.

), }, { tag: "next", body: ( -

Develop product requirements for handling test orders from STLTs to the CDC infectious disease labs.

+

Develop executive summary of LRI and LOI specs and implications for ReportStream. Define high risk areas that should be investigated.

), }, ]} diff --git a/frontend-react/src/content/about/security.mdx b/frontend-react/src/content/about/security.mdx index 1cf78bfd5b2..64e32ce03df 100644 --- a/frontend-react/src/content/about/security.mdx +++ b/frontend-react/src/content/about/security.mdx @@ -111,7 +111,7 @@ ReportStream is committed to handling data securely and reliably. From federal r headingLevel:"h3", title: "What data does ReportStream send?", content: (<> -

ReportStream always sends synthetic, non-PII data in our standard data schema during testing.

+

ReportStream always sends synthetic, non-PII data in our standard data schema during testing.

For public health entities receiving data, we'll send patient data to your test environment for approval before going live with launch.

) } diff --git a/frontend-react/src/content/support/index.mdx b/frontend-react/src/content/support/index.mdx index 2c26c9d1c5c..cf7b9734cba 100644 --- a/frontend-react/src/content/support/index.mdx +++ b/frontend-react/src/content/support/index.mdx @@ -243,8 +243,7 @@ import site from "../../content/site.json"; title: "How do I map to the LIVD table?", content: (<>

- LOINC In Vitro Diagnostic (LIVD) mapping provides several data elements - {" "} you’ll need for your ReportStream test file. Including: + LOINC In Vitro Diagnostic (LIVD) mapping provides several data elements you’ll need for your ReportStream test file. Including:

    @@ -259,7 +258,6 @@ import site from "../../content/site.json"; When you know the devices you are reporting for, you can use the LIVD mapping table to identify the device identifier code. The code is in column F, labeled Test Performed LOINC Code. - Then, you can use that code as described in our data model.

    ) }, diff --git a/frontend-react/src/hooks/filters/filters.fixtures.ts b/frontend-react/src/hooks/filters/filters.fixtures.ts index 819aed2a5fc..bb4458c71ec 100644 --- a/frontend-react/src/hooks/filters/filters.fixtures.ts +++ b/frontend-react/src/hooks/filters/filters.fixtures.ts @@ -14,7 +14,7 @@ export const cursorManagerFixture: CursorManager = { }, hasPrev: false, hasNext: false, - // eslint-disable-next-line no-console + update: () => void 0, }; diff --git a/frontend-react/src/layouts/Main/MainLayout.test.tsx b/frontend-react/src/layouts/Main/MainLayout.test.tsx index a4b62dc3f00..e07e79949cf 100644 --- a/frontend-react/src/layouts/Main/MainLayout.test.tsx +++ b/frontend-react/src/layouts/Main/MainLayout.test.tsx @@ -5,7 +5,7 @@ import { renderApp } from "../../utils/CustomRenderUtils"; function ErroringComponent() { throw new Error("Test"); - // eslint-disable-next-line no-unreachable + return <>; } diff --git a/frontend-react/src/network/Middleware.ts b/frontend-react/src/network/Middleware.ts index 88dcd0ca498..19a5bf5307a 100644 --- a/frontend-react/src/network/Middleware.ts +++ b/frontend-react/src/network/Middleware.ts @@ -1,4 +1,3 @@ -/* eslint-disable react-hooks/rules-of-hooks */ import axios, { AxiosRequestConfig } from "axios"; import { Fetcher, Middleware, SuspenseQueryHook } from "react-query-kit"; @@ -6,18 +5,14 @@ import { RSEndpoint } from "../config/endpoints"; import useSessionContext from "../contexts/Session/useSessionContext"; import useAppInsightsContext from "../hooks/UseAppInsightsContext/UseAppInsightsContext"; -export type AuthMiddleware = Middleware< - SuspenseQueryHook ->; +export type AuthMiddleware = Middleware>; /** * react-query middleware that prepares the fetch configuration (dynamic url, auth etc.) * from an expected RSEndpoint variable. Will disable the query if it cannot pass on * a fetchConfig (either built here or given). */ -export const authMiddleware: Middleware< - SuspenseQueryHook -> = (useQueryNext) => { +export const authMiddleware: Middleware> = (useQueryNext) => { return (options, qc) => { if (!options.variables?.endpoint) throw new Error("Endpoint not found"); const { properties } = useAppInsightsContext(); @@ -25,9 +20,7 @@ export const authMiddleware: Middleware< const authHeaders = { "x-ms-session-id": properties.context.getSessionId(), "authentication-type": "okta", - authorization: `Bearer ${ - authState?.accessToken?.accessToken ?? "" - }`, + authorization: `Bearer ${authState?.accessToken?.accessToken ?? ""}`, organization: `${activeMembership?.parsedName ?? ""}`, }; const headers = { @@ -39,7 +32,7 @@ export const authMiddleware: Middleware< headers, }); const fetchConfig = - options.variables?.fetchConfig ?? axiosConfig + (options.variables?.fetchConfig ?? axiosConfig) ? { ...options.variables?.fetchConfig, ...axiosConfig, @@ -66,9 +59,7 @@ export type AuthFetch = Fetcher; /** * Calls fetch with the provided fetch config from variables. */ -export const authFetch: Fetcher = async ({ - fetchConfig, -}) => { +export const authFetch: Fetcher = async ({ fetchConfig }) => { if (!fetchConfig) throw new Error("Fetch config not found"); if (fetchConfig.enabled === false) return Promise.resolve(null); diff --git a/frontend-react/src/pages/admin/AdminRevHistory.test.tsx b/frontend-react/src/pages/admin/AdminRevHistory.test.tsx index 30fe9cc7d5b..d84153cf47d 100644 --- a/frontend-react/src/pages/admin/AdminRevHistory.test.tsx +++ b/frontend-react/src/pages/admin/AdminRevHistory.test.tsx @@ -67,11 +67,10 @@ describe("AdminRevHistory", () => { // a REAL test would need Cypress to click revisions in the top two accordion lists // and verify the diffs are rendering the diffs correctly - // eslint-disable-next-line react/jsx-pascal-case renderApp(<_exportForTesting.AdminRevHistory />); // useful: https://testing-library.com/docs/queries/about/ // we expect 2x because of the right and left list layout - // eslint-disable-next-line no-restricted-globals + expect(screen.getAllByText(/local@test.com/).length).toBe(2); // click an item in each list and make sure the diff loads. (click parent row) diff --git a/frontend-react/src/pages/deliveries/daily-data/TableReportsData.tsx b/frontend-react/src/pages/deliveries/daily-data/TableReportsData.tsx index a1060cff013..b467e57553e 100644 --- a/frontend-react/src/pages/deliveries/daily-data/TableReportsData.tsx +++ b/frontend-react/src/pages/deliveries/daily-data/TableReportsData.tsx @@ -21,25 +21,14 @@ function TableReportsData(props: Props) { {props.reports.map((report, idx) => ( - + {report.reportId} + {format(parseISO(report.sent.toString()), "yyyy-MM-dd HH:mm")} - {format( - parseISO(report.sent.toString()), - "yyyy-MM-dd HH:mm", - )} - - - {/* eslint-disable-next-line import/no-named-as-default-member */} - {format( - parseISO(report.expires.toString()), - "yyyy-MM-dd HH:mm", - )} + {} + {format(parseISO(report.expires.toString()), "yyyy-MM-dd HH:mm")} {report.total} diff --git a/frontend-react/src/pages/deliveries/details/DeliveryInfo.tsx b/frontend-react/src/pages/deliveries/details/DeliveryInfo.tsx index 20dc0062fc7..0dece562b52 100644 --- a/frontend-react/src/pages/deliveries/details/DeliveryInfo.tsx +++ b/frontend-react/src/pages/deliveries/details/DeliveryInfo.tsx @@ -18,37 +18,21 @@ function DeliveryInfo(props: Props) {
    -

    - Report type -

    +

    Report type

    {report.fileType}

    -

    - Available to download -

    +

    Available to download

    - {/* eslint-disable-next-line import/no-named-as-default-member */} - {format( - parseISO(report.batchReadyAt), - "eeee, LLL dd, yyyy HH:mm", - )} + {} + {format(parseISO(report.batchReadyAt), "eeee, LLL dd, yyyy HH:mm")}

    -

    - Total tests reported -

    +

    Total tests reported

    +

    {report.reportItemCount}

    +

    Download expires

    - {report.reportItemCount} -

    -

    - Download expires -

    -

    - {/* eslint-disable-next-line import/no-named-as-default-member */} - {format( - parseISO(report.expires), - "eeee, LLL dd, yyyy HH:mm", - )} + {} + {format(parseISO(report.expires), "eeee, LLL dd, yyyy HH:mm")}

    diff --git a/frontend-react/src/unsupportedBrowser.ts b/frontend-react/src/unsupportedBrowser.ts index 8369f9e1aa8..a0b673d2b23 100644 --- a/frontend-react/src/unsupportedBrowser.ts +++ b/frontend-react/src/unsupportedBrowser.ts @@ -3,7 +3,6 @@ import { minimum } from "./browsers.json"; const minUseragent = new RegExp(minimum.useragent); if (!minUseragent.test(navigator.userAgent)) { - // eslint-disable-next-line no-restricted-globals location.assign("/unsupported-browser.html"); } diff --git a/frontend-react/src/utils/DateTimeUtils.ts b/frontend-react/src/utils/DateTimeUtils.ts index 4e732a261bc..d5c149a77c1 100644 --- a/frontend-react/src/utils/DateTimeUtils.ts +++ b/frontend-react/src/utils/DateTimeUtils.ts @@ -36,9 +36,8 @@ export const generateDateTitles = (dateTimeString?: string) => { }; export function isDateExpired(dateTimeString: string | number) { - // eslint-disable-next-line import/no-named-as-default-member const now = new Date(); - // eslint-disable-next-line import/no-named-as-default-member + const dateToCompare = typeof dateTimeString === "string" ? !/^\d+$/.test(dateTimeString) @@ -67,12 +66,8 @@ export function formatDateWithoutSeconds(d: string) { * Rewrote to just use Date to save cpu * */ export const dateShortFormat = (d: Date) => { - const dayOfWeek = - ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][d.getDay()] || ""; - return ( - `${dayOfWeek}, ` + - `${d.getMonth() + 1}/${d.getDate()}/${d.getFullYear()}` - ); + const dayOfWeek = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][d.getDay()] || ""; + return `${dayOfWeek}, ` + `${d.getMonth() + 1}/${d.getDate()}/${d.getFullYear()}`; }; /** @@ -81,10 +76,7 @@ export const dateShortFormat = (d: Date) => { * @param dateNewer Date * @param dateOlder Date */ -export const durationFormatShort = ( - dateNewer: Date, - dateOlder: Date, -): string => { +export const durationFormatShort = (dateNewer: Date, dateOlder: Date): string => { const msDiff = dateNewer.getTime() - dateOlder.getTime(); const hrs = Math.floor(msDiff / (60 * 60 * 1000)).toString(); const mins = Math.floor((msDiff / (60 * 1000)) % 60).toString(); diff --git a/frontend-react/src/utils/DiffCompare/diff.ts b/frontend-react/src/utils/DiffCompare/diff.ts index 810b59a09f1..857407bcef8 100644 --- a/frontend-react/src/utils/DiffCompare/diff.ts +++ b/frontend-react/src/utils/DiffCompare/diff.ts @@ -1,5 +1,3 @@ -/* eslint-disable camelcase */ - /** * The algorithm implemented here is based on "An O(NP) Sequence Comparison Algorithm" * by described by Sun Wu, Udi Manber and Gene Myers @@ -150,24 +148,12 @@ export const Diff = (a_: string, b_: string) => { do { ++p; for (let k = -p; k <= delta - 1; ++k) { - fp[k + offset] = snake( - k, - (fp[k - 1 + offset] || -1) + 1, - fp[k + 1 + offset] || -1, - ); + fp[k + offset] = snake(k, (fp[k - 1 + offset] || -1) + 1, fp[k + 1 + offset] || -1); } for (let k = delta + p; k >= delta + 1; --k) { - fp[k + offset] = snake( - k, - (fp[k - 1 + offset] || -1) + 1, - fp[k + 1 + offset] || -1, - ); + fp[k + offset] = snake(k, (fp[k - 1 + offset] || -1) + 1, fp[k + 1 + offset] || -1); } - fp[delta + offset] = snake( - delta, - (fp[delta - 1 + offset] || -1) + 1, - fp[delta + 1 + offset] || -1, - ); + fp[delta + offset] = snake(delta, (fp[delta - 1 + offset] || -1) + 1, fp[delta + 1 + offset] || -1); } while ((fp[delta + offset] || -1) !== blen); editdistance = delta + 2 * p; diff --git a/frontend-react/src/utils/LazyRouteMarkdown.test.tsx b/frontend-react/src/utils/LazyRouteMarkdown.test.tsx index bcca67c5aa3..8163a25edae 100644 --- a/frontend-react/src/utils/LazyRouteMarkdown.test.tsx +++ b/frontend-react/src/utils/LazyRouteMarkdown.test.tsx @@ -15,10 +15,7 @@ vi.mock("react-helmet-async", () => { describe("lazyRouteMarkdown", () => { test("works with react-router", async () => { - const Component = lazy( - // eslint-disable-next-line import/no-unresolved - lazyRouteMarkdown(() => Promise.resolve(md)), - ); + const Component = lazy(lazyRouteMarkdown(() => Promise.resolve(md))); const router = createMemoryRouter([ { path: "/", diff --git a/frontend-react/tsconfig.json b/frontend-react/tsconfig.json index 4d587b83318..ee074878df5 100644 --- a/frontend-react/tsconfig.json +++ b/frontend-react/tsconfig.json @@ -22,6 +22,6 @@ "@trussworks/react-uswds/lib/*": ["./node_modules/@trussworks/react-uswds/lib/*"] } }, - "include": ["./src", "./e2e", "./__mocks__", "playwright.config.ts"], + "include": ["./src", "./e2e", "./__mocks__", "playwright.config.ts", "eslint.config.js"], "references": [{ "path": "./tsconfig.node.json" }] } diff --git a/frontend-react/vite.config.ts b/frontend-react/vite.config.ts index c4b32818d54..c790d99d27e 100644 --- a/frontend-react/vite.config.ts +++ b/frontend-react/vite.config.ts @@ -84,6 +84,7 @@ export default defineConfig(async ({ mode }) => { typescript: true, eslint: { lintCommand: 'eslint "./src/**/*[!.test][!.stories].{ts,tsx}"', + useFlatConfig: true, }, }), ], diff --git a/frontend-react/yarn.lock b/frontend-react/yarn.lock index 9c0111280a8..36326ff0312 100644 --- a/frontend-react/yarn.lock +++ b/frontend-react/yarn.lock @@ -673,45 +673,97 @@ __metadata: languageName: node linkType: hard -"@eslint-community/regexpp@npm:^4.10.0, @eslint-community/regexpp@npm:^4.6.1": - version: 4.10.0 - resolution: "@eslint-community/regexpp@npm:4.10.0" - checksum: 2a6e345429ea8382aaaf3a61f865cae16ed44d31ca917910033c02dc00d505d939f10b81e079fa14d43b51499c640138e153b7e40743c4c094d9df97d4e56f7b +"@eslint-community/regexpp@npm:^4.10.0, @eslint-community/regexpp@npm:^4.11.0": + version: 4.12.1 + resolution: "@eslint-community/regexpp@npm:4.12.1" + checksum: 0d628680e204bc316d545b4993d3658427ca404ae646ce541fcc65306b8c712c340e5e573e30fb9f85f4855c0c5f6dca9868931f2fcced06417fbe1a0c6cd2d6 languageName: node linkType: hard -"@eslint/eslintrc@npm:^2.1.4": - version: 2.1.4 - resolution: "@eslint/eslintrc@npm:2.1.4" +"@eslint/compat@npm:^1.2.2": + version: 1.2.2 + resolution: "@eslint/compat@npm:1.2.2" + peerDependencies: + eslint: ^9.10.0 + peerDependenciesMeta: + eslint: + optional: true + checksum: 02708de14b32870f44b4fbb78d1bf9e7fb8741a3038bcaea91239a161a0884b676b4c9b5a2346a145d76981710427f5a2d56f65e5bc0579bd288286e88f43ee2 + languageName: node + linkType: hard + +"@eslint/config-array@npm:^0.18.0": + version: 0.18.0 + resolution: "@eslint/config-array@npm:0.18.0" + dependencies: + "@eslint/object-schema": ^2.1.4 + debug: ^4.3.1 + minimatch: ^3.1.2 + checksum: 5ff748e1788745bfb3160c3b3151d62a7c054e336e9fe8069e86cfa6106f3abbd59b24f1253122268295f98c66803e9a7b23d7f947a8c00f62d2060cc44bc7fc + languageName: node + linkType: hard + +"@eslint/core@npm:^0.7.0": + version: 0.7.0 + resolution: "@eslint/core@npm:0.7.0" + checksum: 91d4aa2805f356fb0bba693411deab91590472666e22c9c03304ba03b288b74403a5e120db16d0926ea94281e15563a8d4d519cd1e565d514e2d5015a84b8575 + languageName: node + linkType: hard + +"@eslint/eslintrc@npm:^3.1.0": + version: 3.1.0 + resolution: "@eslint/eslintrc@npm:3.1.0" dependencies: ajv: ^6.12.4 debug: ^4.3.2 - espree: ^9.6.0 - globals: ^13.19.0 + espree: ^10.0.1 + globals: ^14.0.0 ignore: ^5.2.0 import-fresh: ^3.2.1 js-yaml: ^4.1.0 minimatch: ^3.1.2 strip-json-comments: ^3.1.1 - checksum: 10957c7592b20ca0089262d8c2a8accbad14b4f6507e35416c32ee6b4dbf9cad67dfb77096bbd405405e9ada2b107f3797fe94362e1c55e0b09d6e90dd149127 + checksum: b0a9bbd98c8b9e0f4d975b042ff9b874dde722b20834ea2ff46551c3de740d4f10f56c449b790ef34d7f82147cbddfc22b004a43cc885dbc2664bb134766b5e4 + languageName: node + linkType: hard + +"@eslint/js@npm:9.13.0, @eslint/js@npm:^9.13.0": + version: 9.13.0 + resolution: "@eslint/js@npm:9.13.0" + checksum: ad5dd72aa75bd8d5bd3c1ffe68cf748aed7edef5fcf97193eb52af35dbb89a1999f526a0e2c169ef5572afbbbbb5f37d6fd0af2991d9ccdc29f753da5cc0f532 languageName: node linkType: hard -"@eslint/js@npm:8.57.0": - version: 8.57.0 - resolution: "@eslint/js@npm:8.57.0" - checksum: 315dc65b0e9893e2bff139bddace7ea601ad77ed47b4550e73da8c9c2d2766c7a575c3cddf17ef85b8fd6a36ff34f91729d0dcca56e73ca887c10df91a41b0bb +"@eslint/object-schema@npm:^2.1.4": + version: 2.1.4 + resolution: "@eslint/object-schema@npm:2.1.4" + checksum: 5a03094115bcdab7991dbbc5d17a9713f394cebb4b44d3eaf990d7487b9b8e1877b817997334ab40be52e299a0384595c6f6ba91b389901e5e1d21efda779271 languageName: node linkType: hard -"@humanwhocodes/config-array@npm:^0.11.14": - version: 0.11.14 - resolution: "@humanwhocodes/config-array@npm:0.11.14" +"@eslint/plugin-kit@npm:^0.2.0": + version: 0.2.2 + resolution: "@eslint/plugin-kit@npm:0.2.2" dependencies: - "@humanwhocodes/object-schema": ^2.0.2 - debug: ^4.3.1 - minimatch: ^3.0.5 - checksum: 861ccce9eaea5de19546653bccf75bf09fe878bc39c3aab00aeee2d2a0e654516adad38dd1098aab5e3af0145bbcbf3f309bdf4d964f8dab9dcd5834ae4c02f2 + levn: ^0.4.1 + checksum: 08935d81f59f8b2ccc6df1e2517684d6cb9911390e210dacd861be60a000224b0b2f5aa9364ff78e4b14152d1d777aa621f587479aae07d0670b2e14a5a18ef6 + languageName: node + linkType: hard + +"@humanfs/core@npm:^0.19.1": + version: 0.19.1 + resolution: "@humanfs/core@npm:0.19.1" + checksum: 611e0545146f55ddfdd5c20239cfb7911f9d0e28258787c4fc1a1f6214250830c9367aaaeace0096ed90b6739bee1e9c52ad5ba8adaf74ab8b449119303babfe + languageName: node + linkType: hard + +"@humanfs/node@npm:^0.16.5": + version: 0.16.6 + resolution: "@humanfs/node@npm:0.16.6" + dependencies: + "@humanfs/core": ^0.19.1 + "@humanwhocodes/retry": ^0.3.0 + checksum: f9cb52bb235f8b9c6fcff43a7e500669a38f8d6ce26593404a9b56365a1644e0ed60c720dc65ff6a696b1f85f3563ab055bb554ec8674f2559085ba840e47710 languageName: node linkType: hard @@ -722,10 +774,10 @@ __metadata: languageName: node linkType: hard -"@humanwhocodes/object-schema@npm:^2.0.2": - version: 2.0.3 - resolution: "@humanwhocodes/object-schema@npm:2.0.3" - checksum: d3b78f6c5831888c6ecc899df0d03bcc25d46f3ad26a11d7ea52944dc36a35ef543fad965322174238d677a43d5c694434f6607532cff7077062513ad7022631 +"@humanwhocodes/retry@npm:^0.3.0, @humanwhocodes/retry@npm:^0.3.1": + version: 0.3.1 + resolution: "@humanwhocodes/retry@npm:0.3.1" + checksum: 7e5517bb51dbea3e02ab6cacef59a8f4b0ca023fc4b0b8cbc40de0ad29f46edd50b897c6e7fba79366a0217e3f48e2da8975056f6c35cfe19d9cc48f1d03c1dd languageName: node linkType: hard @@ -1132,7 +1184,7 @@ __metadata: languageName: node linkType: hard -"@nodelib/fs.walk@npm:^1.2.3, @nodelib/fs.walk@npm:^1.2.8": +"@nodelib/fs.walk@npm:^1.2.3": version: 1.2.8 resolution: "@nodelib/fs.walk@npm:1.2.8" dependencies: @@ -1142,6 +1194,13 @@ __metadata: languageName: node linkType: hard +"@nolyfill/is-core-module@npm:1.0.39": + version: 1.0.39 + resolution: "@nolyfill/is-core-module@npm:1.0.39" + checksum: 0d6e098b871eca71d875651288e1f0fa770a63478b0b50479c99dc760c64175a56b5b04f58d5581bbcc6b552b8191ab415eada093d8df9597ab3423c8cac1815 + languageName: node + linkType: hard + "@npmcli/fs@npm:^3.1.0": version: 3.1.0 resolution: "@npmcli/fs@npm:3.1.0" @@ -1653,6 +1712,13 @@ __metadata: languageName: node linkType: hard +"@rtsao/scc@npm:^1.1.0": + version: 1.1.0 + resolution: "@rtsao/scc@npm:1.1.0" + checksum: 17d04adf404e04c1e61391ed97bca5117d4c2767a76ae3e879390d6dec7b317fcae68afbf9e98badee075d0b64fa60f287729c4942021b4d19cd01db77385c01 + languageName: node + linkType: hard + "@selderee/plugin-htmlparser2@npm:^0.11.0": version: 0.11.0 resolution: "@selderee/plugin-htmlparser2@npm:0.11.0" @@ -2622,6 +2688,15 @@ __metadata: languageName: node linkType: hard +"@types/eslint__js@npm:^8.42.3": + version: 8.42.3 + resolution: "@types/eslint__js@npm:8.42.3" + dependencies: + "@types/eslint": "*" + checksum: e31f19de642d35a664695d0cab873ce6de19b8a3506755835b91f8a49a8c41099dcace449df49f1a486de6fa6565d21ceb1fa33be6004fc7adef9226e5d256a1 + languageName: node + linkType: hard + "@types/estree-jsx@npm:^1.0.0": version: 1.0.0 resolution: "@types/estree-jsx@npm:1.0.0" @@ -2631,7 +2706,7 @@ __metadata: languageName: node linkType: hard -"@types/estree@npm:*, @types/estree@npm:1.0.6, @types/estree@npm:^1.0.0": +"@types/estree@npm:*, @types/estree@npm:1.0.6, @types/estree@npm:^1.0.0, @types/estree@npm:^1.0.6": version: 1.0.6 resolution: "@types/estree@npm:1.0.6" checksum: 8825d6e729e16445d9a1dd2fb1db2edc5ed400799064cd4d028150701031af012ba30d6d03fe9df40f4d7a437d0de6d2b256020152b7b09bde9f2e420afdffd9 @@ -2743,7 +2818,7 @@ __metadata: languageName: node linkType: hard -"@types/json-schema@npm:*, @types/json-schema@npm:^7.0.9": +"@types/json-schema@npm:*, @types/json-schema@npm:^7.0.15, @types/json-schema@npm:^7.0.9": version: 7.0.15 resolution: "@types/json-schema@npm:7.0.15" checksum: 97ed0cb44d4070aecea772b7b2e2ed971e10c81ec87dd4ecc160322ffa55ff330dace1793489540e3e318d90942064bb697cc0f8989391797792d919737b3b98 @@ -3032,15 +3107,15 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/eslint-plugin@npm:^8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/eslint-plugin@npm:8.10.0" +"@typescript-eslint/eslint-plugin@npm:8.12.2": + version: 8.12.2 + resolution: "@typescript-eslint/eslint-plugin@npm:8.12.2" dependencies: "@eslint-community/regexpp": ^4.10.0 - "@typescript-eslint/scope-manager": 8.10.0 - "@typescript-eslint/type-utils": 8.10.0 - "@typescript-eslint/utils": 8.10.0 - "@typescript-eslint/visitor-keys": 8.10.0 + "@typescript-eslint/scope-manager": 8.12.2 + "@typescript-eslint/type-utils": 8.12.2 + "@typescript-eslint/utils": 8.12.2 + "@typescript-eslint/visitor-keys": 8.12.2 graphemer: ^1.4.0 ignore: ^5.3.1 natural-compare: ^1.4.0 @@ -3051,25 +3126,25 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: 2bb311eb9a882d530fc94f790f3e1f4745cd4e3523fd8d62ee0ed14d65c4230dc0c797c490c3421c1456fd71349e9bfa146c0b78f63860b75aae6e2a32a6c27c + checksum: a1707704d91cd525ece0cf5a978f17cb309bb8918d65ded349e18b0aa364f585555d018a365cb0ab9450f273912fc07fae5600f34294e637151b244ba4485bc2 languageName: node linkType: hard -"@typescript-eslint/parser@npm:^8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/parser@npm:8.10.0" +"@typescript-eslint/parser@npm:8.12.2": + version: 8.12.2 + resolution: "@typescript-eslint/parser@npm:8.12.2" dependencies: - "@typescript-eslint/scope-manager": 8.10.0 - "@typescript-eslint/types": 8.10.0 - "@typescript-eslint/typescript-estree": 8.10.0 - "@typescript-eslint/visitor-keys": 8.10.0 + "@typescript-eslint/scope-manager": 8.12.2 + "@typescript-eslint/types": 8.12.2 + "@typescript-eslint/typescript-estree": 8.12.2 + "@typescript-eslint/visitor-keys": 8.12.2 debug: ^4.3.4 peerDependencies: eslint: ^8.57.0 || ^9.0.0 peerDependenciesMeta: typescript: optional: true - checksum: 2e38f34d9d044e251450116cc081a8f84ba13183e9c3e1dda919ddc00eebe634a37d4dfd785998f259b64cdd770e863ecc6c5cf7c8f422baf3d2bc2a0f9241cf + checksum: 201f3e4b6073547726e447455b630c04816b0611346c1b9522493c47596d906c8edaf37d43e0d0e121e2965b374d9547c351e1fa0e125bceb37063e0fa806065 languageName: node linkType: hard @@ -3093,38 +3168,28 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/scope-manager@npm:8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/scope-manager@npm:8.10.0" - dependencies: - "@typescript-eslint/types": 8.10.0 - "@typescript-eslint/visitor-keys": 8.10.0 - checksum: 3df8df342e227b80514dcc9151774dea9a71bc649204f702d5b4a1b76a54b4814c5d5a970a6a9213462dd4df0d42342796fab35549e8663d4c0e5d84bd902bba - languageName: node - linkType: hard - -"@typescript-eslint/scope-manager@npm:8.11.0": - version: 8.11.0 - resolution: "@typescript-eslint/scope-manager@npm:8.11.0" +"@typescript-eslint/scope-manager@npm:8.12.2": + version: 8.12.2 + resolution: "@typescript-eslint/scope-manager@npm:8.12.2" dependencies: - "@typescript-eslint/types": 8.11.0 - "@typescript-eslint/visitor-keys": 8.11.0 - checksum: f36212ac1df6a2ed0953beda6bf66e57fd56fcc1c4b4d21149f3451ae621f63aa7ccb92aa1281615250264fdd22e56a163a5d11c5c772c857741ac0e25533325 + "@typescript-eslint/types": 8.12.2 + "@typescript-eslint/visitor-keys": 8.12.2 + checksum: dd960238f1cf0f24e6c16525f0cbdb6cf65bfc3cfe650f376ecda2583c378c2e3a7eb4c2d57e04e009626d009018226b722a670ca283086c2a6cc1931c2268d8 languageName: node linkType: hard -"@typescript-eslint/type-utils@npm:8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/type-utils@npm:8.10.0" +"@typescript-eslint/type-utils@npm:8.12.2": + version: 8.12.2 + resolution: "@typescript-eslint/type-utils@npm:8.12.2" dependencies: - "@typescript-eslint/typescript-estree": 8.10.0 - "@typescript-eslint/utils": 8.10.0 + "@typescript-eslint/typescript-estree": 8.12.2 + "@typescript-eslint/utils": 8.12.2 debug: ^4.3.4 ts-api-utils: ^1.3.0 peerDependenciesMeta: typescript: optional: true - checksum: 8b0cec8cff1926a08c2bd675b24b2ccff36e59a8d9169eed38343f70c4e3bba18796fc39f30a9307ded3f345881aded80dbd6dc1d78b9ae76cff04fbe8708788 + checksum: a8f540d84674c4919d6f038848add5b4d41ef39cdf572734a13b75f0f797b00d45903b179dc7c25f7ae7690f9dbaf115e5bda596d9e439b1a0a8d7f9d799260e languageName: node linkType: hard @@ -3142,17 +3207,10 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/types@npm:8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/types@npm:8.10.0" - checksum: 3839fd43b0f21b432a9f6090a39d5b2254ee48c1eecf14f8f66bea0cbaba9f2f33a7fc78aea37dfe8841442332d0a8f99cc65cd2d01ca43db99550d30d6f7fe8 - languageName: node - linkType: hard - -"@typescript-eslint/types@npm:8.11.0": - version: 8.11.0 - resolution: "@typescript-eslint/types@npm:8.11.0" - checksum: 2958f3b5b30d3a876aad79df15662e6c23fe3d0c7750c473f27adc725b2a20f303e914db785c64200bc4092c3489648407792e2bd89eccf3f7aaa4495be81681 +"@typescript-eslint/types@npm:8.12.2": + version: 8.12.2 + resolution: "@typescript-eslint/types@npm:8.12.2" + checksum: b0f7effdac842428b15d76710295a8b4f1fe1ff14e40fbb10c8f571c11fd517d75d76decbecf90412bc5eabce0cd4ac0acf53d6b0d8ba2bdde86ab3b627bdac2 languageName: node linkType: hard @@ -3193,12 +3251,12 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/typescript-estree@npm:8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/typescript-estree@npm:8.10.0" +"@typescript-eslint/typescript-estree@npm:8.12.2": + version: 8.12.2 + resolution: "@typescript-eslint/typescript-estree@npm:8.12.2" dependencies: - "@typescript-eslint/types": 8.10.0 - "@typescript-eslint/visitor-keys": 8.10.0 + "@typescript-eslint/types": 8.12.2 + "@typescript-eslint/visitor-keys": 8.12.2 debug: ^4.3.4 fast-glob: ^3.3.2 is-glob: ^4.0.3 @@ -3208,40 +3266,21 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: 3fc774f51d0a891a5e09bc77f5544b6aa268abec9c01cd9ec831f92dde9c9d61a5c818ca2800c124fb5d61d40ce7ac34740b347c21ba3493e756c052084afd65 + checksum: 923d297ba891cbaf4f00618db2313123238657b179f56a5d42d02a4e6433c513f73a9dd9aa62cd2c5b9fb2c5912a59319eb0a14ef2403792e15757142722309a languageName: node linkType: hard -"@typescript-eslint/typescript-estree@npm:8.11.0": - version: 8.11.0 - resolution: "@typescript-eslint/typescript-estree@npm:8.11.0" - dependencies: - "@typescript-eslint/types": 8.11.0 - "@typescript-eslint/visitor-keys": 8.11.0 - debug: ^4.3.4 - fast-glob: ^3.3.2 - is-glob: ^4.0.3 - minimatch: ^9.0.4 - semver: ^7.6.0 - ts-api-utils: ^1.3.0 - peerDependenciesMeta: - typescript: - optional: true - checksum: 03ae4740d4ff19ebc3ea68ac3be1a0265b4abe6348fdc48123e20d6f9206baaa70209e65c9fa4a91930da7d3952c55099a307014284c9b596b12f72bce741817 - languageName: node - linkType: hard - -"@typescript-eslint/utils@npm:8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/utils@npm:8.10.0" +"@typescript-eslint/utils@npm:8.12.2, @typescript-eslint/utils@npm:^8.8.1": + version: 8.12.2 + resolution: "@typescript-eslint/utils@npm:8.12.2" dependencies: "@eslint-community/eslint-utils": ^4.4.0 - "@typescript-eslint/scope-manager": 8.10.0 - "@typescript-eslint/types": 8.10.0 - "@typescript-eslint/typescript-estree": 8.10.0 + "@typescript-eslint/scope-manager": 8.12.2 + "@typescript-eslint/types": 8.12.2 + "@typescript-eslint/typescript-estree": 8.12.2 peerDependencies: eslint: ^8.57.0 || ^9.0.0 - checksum: db67603baacba9cccbbc625801a44e5320bc558be846646ff9962818c64a9ab07edcfdcad98b15a3f8954d3e398e3a41f085c1ec458f7169a1ce7b3674032d59 + checksum: 7ae4ef40d0961642fc31644c47e05f751369b47f3d9f5ea4e6c6eaa09d534efc6a2ea89f12368eed7dc8b32a7378e533f84379f70f2acd85418815f63b249b18 languageName: node linkType: hard @@ -3277,20 +3316,6 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/utils@npm:^8.8.1": - version: 8.11.0 - resolution: "@typescript-eslint/utils@npm:8.11.0" - dependencies: - "@eslint-community/eslint-utils": ^4.4.0 - "@typescript-eslint/scope-manager": 8.11.0 - "@typescript-eslint/types": 8.11.0 - "@typescript-eslint/typescript-estree": 8.11.0 - peerDependencies: - eslint: ^8.57.0 || ^9.0.0 - checksum: 0a6286fb6c6aaf497bcd5657e4f8167f29c32bb913e4feab3822c504f537ac30975d626dff442cc691e040384ad197313b5685d79296fc8a42ed6c827dcb52fc - languageName: node - linkType: hard - "@typescript-eslint/visitor-keys@npm:5.62.0": version: 5.62.0 resolution: "@typescript-eslint/visitor-keys@npm:5.62.0" @@ -3311,27 +3336,17 @@ __metadata: languageName: node linkType: hard -"@typescript-eslint/visitor-keys@npm:8.10.0": - version: 8.10.0 - resolution: "@typescript-eslint/visitor-keys@npm:8.10.0" - dependencies: - "@typescript-eslint/types": 8.10.0 - eslint-visitor-keys: ^3.4.3 - checksum: 0b3060a036dd3b6acacc32b1d81b3ada1ac5523cc2d16a369ecffd3ab5b389cd98802b248bf65ee8a266a166125a9e38acd7e917d4dd26044bdf2c805537b7e3 - languageName: node - linkType: hard - -"@typescript-eslint/visitor-keys@npm:8.11.0": - version: 8.11.0 - resolution: "@typescript-eslint/visitor-keys@npm:8.11.0" +"@typescript-eslint/visitor-keys@npm:8.12.2": + version: 8.12.2 + resolution: "@typescript-eslint/visitor-keys@npm:8.12.2" dependencies: - "@typescript-eslint/types": 8.11.0 + "@typescript-eslint/types": 8.12.2 eslint-visitor-keys: ^3.4.3 - checksum: 29057642bf63994646bd8c5b4baa704ae8b1ff094daa6254a6a92e9fbd252086e219b2b7e8050a131da58cd16cc4dee20bb9fc142bc0d3f22f92af2b59b5444e + checksum: 97b919a0f0982e16a46ed568ae195906ec4aed7db358308d2311e9829ceb7f521e4a2017b3bdedad264ee61fdf08d3d12ada7d5622f13b20ac324118fe5b8447 languageName: node linkType: hard -"@ungap/structured-clone@npm:^1.0.0, @ungap/structured-clone@npm:^1.2.0": +"@ungap/structured-clone@npm:^1.0.0": version: 1.2.0 resolution: "@ungap/structured-clone@npm:1.2.0" checksum: 4f656b7b4672f2ce6e272f2427d8b0824ed11546a601d8d5412b9d7704e83db38a8d9f402ecdf2b9063fc164af842ad0ec4a55819f621ed7e7ea4d1efcc74524 @@ -3589,12 +3604,12 @@ __metadata: languageName: node linkType: hard -"acorn@npm:^8.0.0, acorn@npm:^8.4.1, acorn@npm:^8.9.0": - version: 8.11.3 - resolution: "acorn@npm:8.11.3" +"acorn@npm:^8.0.0, acorn@npm:^8.14.0, acorn@npm:^8.4.1, acorn@npm:^8.9.0": + version: 8.14.0 + resolution: "acorn@npm:8.14.0" bin: acorn: bin/acorn - checksum: 76d8e7d559512566b43ab4aadc374f11f563f0a9e21626dd59cb2888444e9445923ae9f3699972767f18af61df89cd89f5eaaf772d1327b055b45cb829b4a88c + checksum: 8755074ba55fff94e84e81c72f1013c2d9c78e973c31231c8ae505a5f966859baf654bddd75046bffd73ce816b149298977fff5077a3033dedba0ae2aad152d4 languageName: node linkType: hard @@ -3761,7 +3776,7 @@ __metadata: languageName: node linkType: hard -"aria-query@npm:5.1.3, aria-query@npm:~5.1.3": +"aria-query@npm:5.1.3": version: 5.1.3 resolution: "aria-query@npm:5.1.3" dependencies: @@ -3770,7 +3785,7 @@ __metadata: languageName: node linkType: hard -"aria-query@npm:5.3.0, aria-query@npm:^5.0.0": +"aria-query@npm:5.3.0": version: 5.3.0 resolution: "aria-query@npm:5.3.0" dependencies: @@ -3779,6 +3794,13 @@ __metadata: languageName: node linkType: hard +"aria-query@npm:^5.0.0, aria-query@npm:^5.3.2": + version: 5.3.2 + resolution: "aria-query@npm:5.3.2" + checksum: d971175c85c10df0f6d14adfe6f1292409196114ab3c62f238e208b53103686f46cc70695a4f775b73bc65f6a09b6a092fd963c4f3a5a7d690c8fc5094925717 + languageName: node + linkType: hard + "array-buffer-byte-length@npm:^1.0.1": version: 1.0.1 resolution: "array-buffer-byte-length@npm:1.0.1" @@ -3796,7 +3818,7 @@ __metadata: languageName: node linkType: hard -"array-includes@npm:^3.1.6, array-includes@npm:^3.1.7, array-includes@npm:^3.1.8": +"array-includes@npm:^3.1.6, array-includes@npm:^3.1.8": version: 3.1.8 resolution: "array-includes@npm:3.1.8" dependencies: @@ -3817,19 +3839,6 @@ __metadata: languageName: node linkType: hard -"array.prototype.filter@npm:^1.0.3": - version: 1.0.3 - resolution: "array.prototype.filter@npm:1.0.3" - dependencies: - call-bind: ^1.0.2 - define-properties: ^1.2.0 - es-abstract: ^1.22.1 - es-array-method-boxes-properly: ^1.0.0 - is-string: ^1.0.7 - checksum: 5443cde6ad64596649e5751252b1b2f5242b41052980c2fb2506ba485e3ffd7607e8f6f2f1aefa0cb1cfb9b8623b2b2be103579cb367a161a3426400619b6e73 - languageName: node - linkType: hard - "array.prototype.findlast@npm:^1.2.5": version: 1.2.5 resolution: "array.prototype.findlast@npm:1.2.5" @@ -3844,16 +3853,17 @@ __metadata: languageName: node linkType: hard -"array.prototype.findlastindex@npm:^1.2.3": - version: 1.2.4 - resolution: "array.prototype.findlastindex@npm:1.2.4" +"array.prototype.findlastindex@npm:^1.2.5": + version: 1.2.5 + resolution: "array.prototype.findlastindex@npm:1.2.5" dependencies: - call-bind: ^1.0.5 + call-bind: ^1.0.7 define-properties: ^1.2.1 - es-abstract: ^1.22.3 + es-abstract: ^1.23.2 es-errors: ^1.3.0 + es-object-atoms: ^1.0.0 es-shim-unscopables: ^1.0.2 - checksum: cc8dce27a06dddf6d9c40a15d4c573f96ac5ca3583f89f8d8cd7d7ffdb96a71d819890a5bdb211f221bda8fafa0d97d1d8cbb5460a5cbec1fff57ae80b8abc31 + checksum: 2c81cff2a75deb95bf1ed89b6f5f2bfbfb882211e3b7cc59c3d6b87df774cd9d6b36949a8ae39ac476e092c1d4a4905f5ee11a86a456abb10f35f8211ae4e710 languageName: node linkType: hard @@ -3881,18 +3891,6 @@ __metadata: languageName: node linkType: hard -"array.prototype.toreversed@npm:^1.1.2": - version: 1.1.2 - resolution: "array.prototype.toreversed@npm:1.1.2" - dependencies: - call-bind: ^1.0.2 - define-properties: ^1.2.0 - es-abstract: ^1.22.1 - es-shim-unscopables: ^1.0.0 - checksum: 58598193426282155297bedf950dc8d464624a0d81659822fb73124286688644cb7e0e4927a07f3ab2daaeb6617b647736cc3a5e6ca7ade5bb8e573b284e6240 - languageName: node - linkType: hard - "array.prototype.tosorted@npm:^1.1.4": version: 1.1.4 resolution: "array.prototype.tosorted@npm:1.1.4" @@ -4015,10 +4013,10 @@ __metadata: languageName: node linkType: hard -"axe-core@npm:^4.2.0, axe-core@npm:^4.9.1": - version: 4.9.1 - resolution: "axe-core@npm:4.9.1" - checksum: 41d9227871781f96c2952e2a777fca73624959dd0e98864f6d82806a77602f82b4fc490852082a7e524d8cd864e50d8b4d9931819b4a150112981d8c932110c5 +"axe-core@npm:^4.10.0, axe-core@npm:^4.2.0": + version: 4.10.2 + resolution: "axe-core@npm:4.10.2" + checksum: 2b9b1c93ea73ea9f206604e4e17bd771d2d835f077bde54517d73028b8865c69b209460e73d5b109968cbdb39ab3d28943efa5695189bd79e16421ce1706719e languageName: node linkType: hard @@ -4033,12 +4031,10 @@ __metadata: languageName: node linkType: hard -"axobject-query@npm:~3.1.1": - version: 3.1.1 - resolution: "axobject-query@npm:3.1.1" - dependencies: - deep-equal: ^2.0.5 - checksum: c12a5da10dc7bab75e1cda9b6a3b5fcf10eba426ddf1a17b71ef65a434ed707ede7d1c4f013ba1609e970bc8c0cddac01365080d376204314e9b294719acd8a5 +"axobject-query@npm:^4.1.0": + version: 4.1.0 + resolution: "axobject-query@npm:4.1.0" + checksum: 7d1e87bf0aa7ae7a76cd39ab627b7c48fda3dc40181303d9adce4ba1d5b5ce73b5e5403ee6626ec8e91090448c887294d6144e24b6741a976f5be9347e3ae1df languageName: node linkType: hard @@ -5242,13 +5238,13 @@ __metadata: languageName: node linkType: hard -"enhanced-resolve@npm:^5.12.0": - version: 5.15.0 - resolution: "enhanced-resolve@npm:5.15.0" +"enhanced-resolve@npm:^5.15.0": + version: 5.17.1 + resolution: "enhanced-resolve@npm:5.17.1" dependencies: graceful-fs: ^4.2.4 tapable: ^2.2.0 - checksum: fbd8cdc9263be71cc737aa8a7d6c57b43d6aa38f6cc75dde6fcd3598a130cc465f979d2f4d01bb3bf475acb43817749c79f8eef9be048683602ca91ab52e4f11 + checksum: 4bc38cf1cea96456f97503db7280394177d1bc46f8f87c267297d04f795ac5efa81e48115a2f5b6273c781027b5b6bfc5f62b54df629e4d25fa7001a86624f59 languageName: node linkType: hard @@ -5343,13 +5339,6 @@ __metadata: languageName: node linkType: hard -"es-array-method-boxes-properly@npm:^1.0.0": - version: 1.0.0 - resolution: "es-array-method-boxes-properly@npm:1.0.0" - checksum: 2537fcd1cecf187083890bc6f5236d3a26bf39237433587e5bf63392e88faae929dbba78ff0120681a3f6f81c23fe3816122982c160d63b38c95c830b633b826 - languageName: node - linkType: hard - "es-define-property@npm:^1.0.0": version: 1.0.0 resolution: "es-define-property@npm:1.0.0" @@ -5359,7 +5348,7 @@ __metadata: languageName: node linkType: hard -"es-errors@npm:^1.0.0, es-errors@npm:^1.2.1, es-errors@npm:^1.3.0": +"es-errors@npm:^1.2.1, es-errors@npm:^1.3.0": version: 1.3.0 resolution: "es-errors@npm:1.3.0" checksum: ec1414527a0ccacd7f15f4a3bc66e215f04f595ba23ca75cdae0927af099b5ec865f9f4d33e9d7e86f512f252876ac77d4281a7871531a50678132429b1271b5 @@ -5383,9 +5372,9 @@ __metadata: languageName: node linkType: hard -"es-iterator-helpers@npm:^1.0.19": - version: 1.0.19 - resolution: "es-iterator-helpers@npm:1.0.19" +"es-iterator-helpers@npm:^1.1.0": + version: 1.1.0 + resolution: "es-iterator-helpers@npm:1.1.0" dependencies: call-bind: ^1.0.7 define-properties: ^1.2.1 @@ -5394,14 +5383,14 @@ __metadata: es-set-tostringtag: ^2.0.3 function-bind: ^1.1.2 get-intrinsic: ^1.2.4 - globalthis: ^1.0.3 + globalthis: ^1.0.4 has-property-descriptors: ^1.0.2 has-proto: ^1.0.3 has-symbols: ^1.0.3 internal-slot: ^1.0.7 - iterator.prototype: ^1.1.2 + iterator.prototype: ^1.1.3 safe-array-concat: ^1.1.2 - checksum: 7ae112b88359fbaf4b9d7d1d1358ae57c5138768c57ba3a8fb930393662653b0512bfd7917c15890d1471577fb012fee8b73b4465e59b331739e6ee94f961683 + checksum: 4ba3a32ab7ba05b85f0ae30604feeb8ffd801fe762e9df9577bd220a96b9eaa2e90af8e6bdc498e523051f293955e2f7d2bddd34de71e1428a1b8ff3fd961016 languageName: node linkType: hard @@ -5701,60 +5690,69 @@ __metadata: languageName: node linkType: hard -"eslint-import-resolver-typescript@npm:^3.6.1": - version: 3.6.1 - resolution: "eslint-import-resolver-typescript@npm:3.6.1" +"eslint-import-resolver-typescript@npm:^3.6.3": + version: 3.6.3 + resolution: "eslint-import-resolver-typescript@npm:3.6.3" dependencies: - debug: ^4.3.4 - enhanced-resolve: ^5.12.0 - eslint-module-utils: ^2.7.4 - fast-glob: ^3.3.1 - get-tsconfig: ^4.5.0 - is-core-module: ^2.11.0 + "@nolyfill/is-core-module": 1.0.39 + debug: ^4.3.5 + enhanced-resolve: ^5.15.0 + eslint-module-utils: ^2.8.1 + fast-glob: ^3.3.2 + get-tsconfig: ^4.7.5 + is-bun-module: ^1.0.2 is-glob: ^4.0.3 peerDependencies: eslint: "*" eslint-plugin-import: "*" - checksum: 454fa0646533050fb57f13d27daf8c71f51b0bb9156d6a461290ccb8576d892209fcc6702a89553f3f5ea8e5b407395ca2e5de169a952c953685f1f7c46b4496 + eslint-plugin-import-x: "*" + peerDependenciesMeta: + eslint-plugin-import: + optional: true + eslint-plugin-import-x: + optional: true + checksum: 1ed0cab4f3852de1b14ea6978e76c27694b253a289c2030a35847ba8ab6ac4258d513877f83ea7bc265f746d570240a6348b11d77cc9cd77589749ad86a32234 languageName: node linkType: hard -"eslint-module-utils@npm:^2.7.4, eslint-module-utils@npm:^2.8.0": - version: 2.8.0 - resolution: "eslint-module-utils@npm:2.8.0" +"eslint-module-utils@npm:^2.12.0, eslint-module-utils@npm:^2.8.1": + version: 2.12.0 + resolution: "eslint-module-utils@npm:2.12.0" dependencies: debug: ^3.2.7 peerDependenciesMeta: eslint: optional: true - checksum: 74c6dfea7641ebcfe174be61168541a11a14aa8d72e515f5f09af55cd0d0862686104b0524aa4b8e0ce66418a44aa38a94d2588743db5fd07a6b49ffd16921d2 + checksum: be3ac52e0971c6f46daeb1a7e760e45c7c45f820c8cc211799f85f10f04ccbf7afc17039165d56cb2da7f7ca9cec2b3a777013cddf0b976784b37eb9efa24180 languageName: node linkType: hard -"eslint-plugin-import@npm:^2.29.1": - version: 2.29.1 - resolution: "eslint-plugin-import@npm:2.29.1" +"eslint-plugin-import@npm:^2.31.0": + version: 2.31.0 + resolution: "eslint-plugin-import@npm:2.31.0" dependencies: - array-includes: ^3.1.7 - array.prototype.findlastindex: ^1.2.3 + "@rtsao/scc": ^1.1.0 + array-includes: ^3.1.8 + array.prototype.findlastindex: ^1.2.5 array.prototype.flat: ^1.3.2 array.prototype.flatmap: ^1.3.2 debug: ^3.2.7 doctrine: ^2.1.0 eslint-import-resolver-node: ^0.3.9 - eslint-module-utils: ^2.8.0 - hasown: ^2.0.0 - is-core-module: ^2.13.1 + eslint-module-utils: ^2.12.0 + hasown: ^2.0.2 + is-core-module: ^2.15.1 is-glob: ^4.0.3 minimatch: ^3.1.2 - object.fromentries: ^2.0.7 - object.groupby: ^1.0.1 - object.values: ^1.1.7 + object.fromentries: ^2.0.8 + object.groupby: ^1.0.3 + object.values: ^1.2.0 semver: ^6.3.1 + string.prototype.trimend: ^1.0.8 tsconfig-paths: ^3.15.0 peerDependencies: - eslint: ^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 - checksum: e65159aef808136d26d029b71c8c6e4cb5c628e65e5de77f1eb4c13a379315ae55c9c3afa847f43f4ff9df7e54515c77ffc6489c6a6f81f7dd7359267577468c + eslint: ^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8 || ^9 + checksum: b1d2ac268b3582ff1af2a72a2c476eae4d250c100f2e335b6e102036e4a35efa530b80ec578dfc36761fabb34a635b9bf5ab071abe9d4404a4bb054fdf22d415 languageName: node linkType: hard @@ -5774,90 +5772,85 @@ __metadata: languageName: node linkType: hard -"eslint-plugin-jsx-a11y@npm:^6.9.0": - version: 6.9.0 - resolution: "eslint-plugin-jsx-a11y@npm:6.9.0" +"eslint-plugin-jsx-a11y@npm:^6.10.2": + version: 6.10.2 + resolution: "eslint-plugin-jsx-a11y@npm:6.10.2" dependencies: - aria-query: ~5.1.3 + aria-query: ^5.3.2 array-includes: ^3.1.8 array.prototype.flatmap: ^1.3.2 ast-types-flow: ^0.0.8 - axe-core: ^4.9.1 - axobject-query: ~3.1.1 + axe-core: ^4.10.0 + axobject-query: ^4.1.0 damerau-levenshtein: ^1.0.8 emoji-regex: ^9.2.2 - es-iterator-helpers: ^1.0.19 hasown: ^2.0.2 jsx-ast-utils: ^3.3.5 language-tags: ^1.0.9 minimatch: ^3.1.2 object.fromentries: ^2.0.8 safe-regex-test: ^1.0.3 - string.prototype.includes: ^2.0.0 + string.prototype.includes: ^2.0.1 peerDependencies: - eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 - checksum: 122cbd22bbd8c3e4a37f386ec183ada63a4ecfa7af7d40cd8a110777ac5ad5ff542f60644596a9e2582ed138a1cc6d96c5d5ca934105e29d5245d6c951ebc3ef + eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9 + checksum: 0cc861398fa26ada61ed5703eef5b335495fcb96253263dcd5e399488ff019a2636372021baacc040e3560d1a34bfcd5d5ad9f1754f44cd0509c956f7df94050 languageName: node linkType: hard -"eslint-plugin-playwright@npm:^1.8.1": - version: 1.8.1 - resolution: "eslint-plugin-playwright@npm:1.8.1" +"eslint-plugin-playwright@npm:^2.0.0": + version: 2.0.0 + resolution: "eslint-plugin-playwright@npm:2.0.0" dependencies: globals: ^13.23.0 peerDependencies: eslint: ">=8.40.0" - eslint-plugin-jest: ">=25" - peerDependenciesMeta: - eslint-plugin-jest: - optional: true - checksum: a3c4c0f9ef56702350f20fae291a1f07d0a66d0959e91515c86da578fec589a088aff3105c5ca348e102f9c8b7b9f9ee51ef65c822b190c4c63ccf1086393e75 + checksum: 3621107bafe84801a09f664cb0744772ebfd0c235ad2cab7110b80122d1448088a08f6ad6727b5f2658a525bc0be0bde5b97e9a399de4edb5e90fd8600cef476 languageName: node linkType: hard -"eslint-plugin-react-hooks@npm:^4.6.2": - version: 4.6.2 - resolution: "eslint-plugin-react-hooks@npm:4.6.2" +"eslint-plugin-react-hooks@npm:^5.0.0": + version: 5.0.0 + resolution: "eslint-plugin-react-hooks@npm:5.0.0" peerDependencies: - eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 - checksum: 395c433610f59577cfcf3f2e42bcb130436c8a0b3777ac64f441d88c5275f4fcfc89094cedab270f2822daf29af1079151a7a6579a8e9ea8cee66540ba0384c4 + eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 + checksum: eddd514a8796e8f805aa0c712d5fe6120fa6db778e3ad2949459b208f8a4bed6a48c152edfa9613f137c7527b00b42d489b5f94363d01d3a509e1f31630674dd languageName: node linkType: hard -"eslint-plugin-react-refresh@npm:^0.4.7": - version: 0.4.7 - resolution: "eslint-plugin-react-refresh@npm:0.4.7" +"eslint-plugin-react-refresh@npm:^0.4.14": + version: 0.4.14 + resolution: "eslint-plugin-react-refresh@npm:0.4.14" peerDependencies: eslint: ">=7" - checksum: b2fe14d4ed158b6380ffd9831a5ebed4c79828ea806536d5db0aa8370f8a3878b198d77fc7da18bfd862cd9eb19ed4472cc9977f727f81679f80dcb48f8a3861 + checksum: 55a7fea190454f33edff6a86739767af21bee8797cd6132466cf2b69a9c75aefd76b03c29b39d0e8ea826af314bde1583d7046f21088afa7fac511f8af4eb714 languageName: node linkType: hard -"eslint-plugin-react@npm:^7.34.3": - version: 7.34.3 - resolution: "eslint-plugin-react@npm:7.34.3" +"eslint-plugin-react@npm:^7.37.2": + version: 7.37.2 + resolution: "eslint-plugin-react@npm:7.37.2" dependencies: array-includes: ^3.1.8 array.prototype.findlast: ^1.2.5 array.prototype.flatmap: ^1.3.2 - array.prototype.toreversed: ^1.1.2 array.prototype.tosorted: ^1.1.4 doctrine: ^2.1.0 - es-iterator-helpers: ^1.0.19 + es-iterator-helpers: ^1.1.0 estraverse: ^5.3.0 + hasown: ^2.0.2 jsx-ast-utils: ^2.4.1 || ^3.0.0 minimatch: ^3.1.2 object.entries: ^1.1.8 object.fromentries: ^2.0.8 - object.hasown: ^1.1.4 object.values: ^1.2.0 prop-types: ^15.8.1 resolve: ^2.0.0-next.5 semver: ^6.3.1 string.prototype.matchall: ^4.0.11 + string.prototype.repeat: ^1.0.0 peerDependencies: - eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 - checksum: 1a519b9792ab9392a5157f2543ce98ab1218c62f4a31c4c3ceb5dd3e7997def4aa07ab39f7276af0fe116ef002db29d97216a15b7aa3b200e55b641cf77d6292 + eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 || ^9.7 + checksum: 7f5203afee7fbe3702b27fdd2b9a3c0ccbbb47d0672f58311b9d8a08dea819c9da4a87c15e8bd508f2562f327a9d29ee8bd9cd189bf758d8dc903de5648b0bfa languageName: node linkType: hard @@ -5912,79 +5905,88 @@ __metadata: languageName: node linkType: hard -"eslint-scope@npm:^7.2.2": - version: 7.2.2 - resolution: "eslint-scope@npm:7.2.2" +"eslint-scope@npm:^8.1.0": + version: 8.2.0 + resolution: "eslint-scope@npm:8.2.0" dependencies: esrecurse: ^4.3.0 estraverse: ^5.2.0 - checksum: ec97dbf5fb04b94e8f4c5a91a7f0a6dd3c55e46bfc7bbcd0e3138c3a76977570e02ed89a1810c778dcd72072ff0e9621ba1379b4babe53921d71e2e4486fda3e + checksum: 750eff4672ca2bf274ec0d1bbeae08aadd53c1907d5c6aff5564d8e047a5f49afa8ae6eee333cab637fd3ebcab2141659d8f2f040f6fdc982b0f61f8bf03136f languageName: node linkType: hard -"eslint-visitor-keys@npm:^3.3.0, eslint-visitor-keys@npm:^3.4.1, eslint-visitor-keys@npm:^3.4.3": +"eslint-visitor-keys@npm:^3.3.0, eslint-visitor-keys@npm:^3.4.3": version: 3.4.3 resolution: "eslint-visitor-keys@npm:3.4.3" checksum: 36e9ef87fca698b6fd7ca5ca35d7b2b6eeaaf106572e2f7fd31c12d3bfdaccdb587bba6d3621067e5aece31c8c3a348b93922ab8f7b2cbc6aaab5e1d89040c60 languageName: node linkType: hard -"eslint@npm:8.57": - version: 8.57.0 - resolution: "eslint@npm:8.57.0" +"eslint-visitor-keys@npm:^4.1.0, eslint-visitor-keys@npm:^4.2.0": + version: 4.2.0 + resolution: "eslint-visitor-keys@npm:4.2.0" + checksum: 779c604672b570bb4da84cef32f6abb085ac78379779c1122d7879eade8bb38ae715645324597cf23232d03cef06032c9844d25c73625bc282a5bfd30247e5b5 + languageName: node + linkType: hard + +"eslint@npm:9.13.0": + version: 9.13.0 + resolution: "eslint@npm:9.13.0" dependencies: "@eslint-community/eslint-utils": ^4.2.0 - "@eslint-community/regexpp": ^4.6.1 - "@eslint/eslintrc": ^2.1.4 - "@eslint/js": 8.57.0 - "@humanwhocodes/config-array": ^0.11.14 + "@eslint-community/regexpp": ^4.11.0 + "@eslint/config-array": ^0.18.0 + "@eslint/core": ^0.7.0 + "@eslint/eslintrc": ^3.1.0 + "@eslint/js": 9.13.0 + "@eslint/plugin-kit": ^0.2.0 + "@humanfs/node": ^0.16.5 "@humanwhocodes/module-importer": ^1.0.1 - "@nodelib/fs.walk": ^1.2.8 - "@ungap/structured-clone": ^1.2.0 + "@humanwhocodes/retry": ^0.3.1 + "@types/estree": ^1.0.6 + "@types/json-schema": ^7.0.15 ajv: ^6.12.4 chalk: ^4.0.0 cross-spawn: ^7.0.2 debug: ^4.3.2 - doctrine: ^3.0.0 escape-string-regexp: ^4.0.0 - eslint-scope: ^7.2.2 - eslint-visitor-keys: ^3.4.3 - espree: ^9.6.1 - esquery: ^1.4.2 + eslint-scope: ^8.1.0 + eslint-visitor-keys: ^4.1.0 + espree: ^10.2.0 + esquery: ^1.5.0 esutils: ^2.0.2 fast-deep-equal: ^3.1.3 - file-entry-cache: ^6.0.1 + file-entry-cache: ^8.0.0 find-up: ^5.0.0 glob-parent: ^6.0.2 - globals: ^13.19.0 - graphemer: ^1.4.0 ignore: ^5.2.0 imurmurhash: ^0.1.4 is-glob: ^4.0.0 - is-path-inside: ^3.0.3 - js-yaml: ^4.1.0 json-stable-stringify-without-jsonify: ^1.0.1 - levn: ^0.4.1 lodash.merge: ^4.6.2 minimatch: ^3.1.2 natural-compare: ^1.4.0 optionator: ^0.9.3 - strip-ansi: ^6.0.1 text-table: ^0.2.0 + peerDependencies: + jiti: "*" + peerDependenciesMeta: + jiti: + optional: true bin: eslint: bin/eslint.js - checksum: 3a48d7ff85ab420a8447e9810d8087aea5b1df9ef68c9151732b478de698389ee656fd895635b5f2871c89ee5a2652b3f343d11e9db6f8486880374ebc74a2d9 + checksum: 99e878d6883864b8361bfaf2a2304f1e133347ac19976c79e1430623cd311cb38253bbd122100788082eded947693cce5c7e67dfd2b5173e6f05edb92dcb2206 languageName: node linkType: hard -"espree@npm:^9.6.0, espree@npm:^9.6.1": - version: 9.6.1 - resolution: "espree@npm:9.6.1" +"espree@npm:^10.0.1, espree@npm:^10.2.0": + version: 10.3.0 + resolution: "espree@npm:10.3.0" dependencies: - acorn: ^8.9.0 + acorn: ^8.14.0 acorn-jsx: ^5.3.2 - eslint-visitor-keys: ^3.4.1 - checksum: eb8c149c7a2a77b3f33a5af80c10875c3abd65450f60b8af6db1bfcfa8f101e21c1e56a561c6dc13b848e18148d43469e7cd208506238554fb5395a9ea5a1ab9 + eslint-visitor-keys: ^4.2.0 + checksum: 63e8030ff5a98cea7f8b3e3a1487c998665e28d674af08b9b3100ed991670eb3cbb0e308c4548c79e03762753838fbe530c783f17309450d6b47a889fee72bef languageName: node linkType: hard @@ -5998,12 +6000,12 @@ __metadata: languageName: node linkType: hard -"esquery@npm:^1.4.2": - version: 1.5.0 - resolution: "esquery@npm:1.5.0" +"esquery@npm:^1.5.0": + version: 1.6.0 + resolution: "esquery@npm:1.6.0" dependencies: estraverse: ^5.1.0 - checksum: aefb0d2596c230118656cd4ec7532d447333a410a48834d80ea648b1e7b5c9bc9ed8b5e33a89cb04e487b60d622f44cf5713bf4abed7c97343edefdc84a35900 + checksum: 08ec4fe446d9ab27186da274d979558557fbdbbd10968fa9758552482720c54152a5640e08b9009e5a30706b66aba510692054d4129d32d0e12e05bbc0b96fb2 languageName: node linkType: hard @@ -6233,7 +6235,7 @@ __metadata: languageName: node linkType: hard -"fast-glob@npm:^3.2.7, fast-glob@npm:^3.2.9, fast-glob@npm:^3.3.1, fast-glob@npm:^3.3.2": +"fast-glob@npm:^3.2.7, fast-glob@npm:^3.2.9, fast-glob@npm:^3.3.2": version: 3.3.2 resolution: "fast-glob@npm:3.3.2" dependencies: @@ -6313,12 +6315,12 @@ __metadata: languageName: node linkType: hard -"file-entry-cache@npm:^6.0.1": - version: 6.0.1 - resolution: "file-entry-cache@npm:6.0.1" +"file-entry-cache@npm:^8.0.0": + version: 8.0.0 + resolution: "file-entry-cache@npm:8.0.0" dependencies: - flat-cache: ^3.0.4 - checksum: f49701feaa6314c8127c3c2f6173cfefff17612f5ed2daaafc6da13b5c91fd43e3b2a58fd0d63f9f94478a501b167615931e7200e31485e320f74a33885a9c74 + flat-cache: ^4.0.0 + checksum: f67802d3334809048c69b3d458f672e1b6d26daefda701761c81f203b80149c35dea04d78ea4238969dd617678e530876722a0634c43031a0957f10cc3ed190f languageName: node linkType: hard @@ -6386,14 +6388,13 @@ __metadata: languageName: node linkType: hard -"flat-cache@npm:^3.0.4": - version: 3.2.0 - resolution: "flat-cache@npm:3.2.0" +"flat-cache@npm:^4.0.0": + version: 4.0.1 + resolution: "flat-cache@npm:4.0.1" dependencies: flatted: ^3.2.9 - keyv: ^4.5.3 - rimraf: ^3.0.2 - checksum: e7e0f59801e288b54bee5cb9681e9ee21ee28ef309f886b312c9d08415b79fc0f24ac842f84356ce80f47d6a53de62197ce0e6e148dc42d5db005992e2a756ec + keyv: ^4.5.4 + checksum: 899fc86bf6df093547d76e7bfaeb900824b869d7d457d02e9b8aae24836f0a99fbad79328cfd6415ee8908f180699bf259dc7614f793447cb14f707caf5996f6 languageName: node linkType: hard @@ -6692,7 +6693,7 @@ __metadata: languageName: node linkType: hard -"get-tsconfig@npm:^4.5.0, get-tsconfig@npm:^4.7.5": +"get-tsconfig@npm:^4.7.5": version: 4.7.5 resolution: "get-tsconfig@npm:4.7.5" dependencies: @@ -6772,7 +6773,7 @@ __metadata: languageName: node linkType: hard -"globals@npm:^13.19.0, globals@npm:^13.23.0": +"globals@npm:^13.23.0": version: 13.24.0 resolution: "globals@npm:13.24.0" dependencies: @@ -6781,12 +6782,27 @@ __metadata: languageName: node linkType: hard -"globalthis@npm:^1.0.3": - version: 1.0.3 - resolution: "globalthis@npm:1.0.3" +"globals@npm:^14.0.0": + version: 14.0.0 + resolution: "globals@npm:14.0.0" + checksum: 534b8216736a5425737f59f6e6a5c7f386254560c9f41d24a9227d60ee3ad4a9e82c5b85def0e212e9d92162f83a92544be4c7fd4c902cb913736c10e08237ac + languageName: node + linkType: hard + +"globals@npm:^15.11.0": + version: 15.11.0 + resolution: "globals@npm:15.11.0" + checksum: ef32d5ef987f3d4b47fc2e389a0b235f6a46f605160c4e405722fd7b576106ca407cb867e66fd1e0fc43b631800e2e2e71847f37691026d813f96f40339da702 + languageName: node + linkType: hard + +"globalthis@npm:^1.0.3, globalthis@npm:^1.0.4": + version: 1.0.4 + resolution: "globalthis@npm:1.0.4" dependencies: - define-properties: ^1.1.3 - checksum: fbd7d760dc464c886d0196166d92e5ffb4c84d0730846d6621a39fbbc068aeeb9c8d1421ad330e94b7bca4bb4ea092f5f21f3d36077812af5d098b4dc006c998 + define-properties: ^1.2.1 + gopd: ^1.0.1 + checksum: 39ad667ad9f01476474633a1834a70842041f70a55571e8dcef5fb957980a92da5022db5430fca8aecc5d47704ae30618c0bc877a579c70710c904e9ef06108a languageName: node linkType: hard @@ -7479,6 +7495,15 @@ __metadata: languageName: node linkType: hard +"is-bun-module@npm:^1.0.2": + version: 1.2.1 + resolution: "is-bun-module@npm:1.2.1" + dependencies: + semver: ^7.6.3 + checksum: 1c2cbcf1a76991add1b640d2d7fe09848e8697a76f96e1289dff44133a48c97f5dc601d4a66d3f3a86217a77178d72d33d10d0c9e14194e58e70ec8df3eae41a + languageName: node + linkType: hard + "is-callable@npm:^1.1.3, is-callable@npm:^1.1.4, is-callable@npm:^1.2.7": version: 1.2.7 resolution: "is-callable@npm:1.2.7" @@ -7486,12 +7511,12 @@ __metadata: languageName: node linkType: hard -"is-core-module@npm:^2.11.0, is-core-module@npm:^2.13.0, is-core-module@npm:^2.13.1": - version: 2.13.1 - resolution: "is-core-module@npm:2.13.1" +"is-core-module@npm:^2.13.0, is-core-module@npm:^2.15.1": + version: 2.15.1 + resolution: "is-core-module@npm:2.15.1" dependencies: - hasown: ^2.0.0 - checksum: 256559ee8a9488af90e4bad16f5583c6d59e92f0742e9e8bb4331e758521ee86b810b93bae44f390766ffbc518a0488b18d9dab7da9a5ff997d499efc9403f7c + hasown: ^2.0.2 + checksum: df134c168115690724b62018c37b2f5bba0d5745fa16960b329c5a00883a8bea6a5632fdb1e3efcce237c201826ba09f93197b7cd95577ea56b0df335be23633 languageName: node linkType: hard @@ -7637,13 +7662,6 @@ __metadata: languageName: node linkType: hard -"is-path-inside@npm:^3.0.3": - version: 3.0.3 - resolution: "is-path-inside@npm:3.0.3" - checksum: abd50f06186a052b349c15e55b182326f1936c89a78bf6c8f2b707412517c097ce04bc49a0ca221787bc44e1049f51f09a2ffb63d22899051988d3a618ba13e9 - languageName: node - linkType: hard - "is-plain-obj@npm:^3.0.0": version: 3.0.0 resolution: "is-plain-obj@npm:3.0.0" @@ -7842,16 +7860,16 @@ __metadata: languageName: node linkType: hard -"iterator.prototype@npm:^1.1.2": - version: 1.1.2 - resolution: "iterator.prototype@npm:1.1.2" +"iterator.prototype@npm:^1.1.3": + version: 1.1.3 + resolution: "iterator.prototype@npm:1.1.3" dependencies: define-properties: ^1.2.1 get-intrinsic: ^1.2.1 has-symbols: ^1.0.3 reflect.getprototypeof: ^1.0.4 set-function-name: ^2.0.1 - checksum: d8a507e2ccdc2ce762e8a1d3f4438c5669160ac72b88b648e59a688eec6bc4e64b22338e74000518418d9e693faf2a092d2af21b9ec7dbf7763b037a54701168 + checksum: 7d2a1f8bcbba7b76f72e956faaf7b25405f4de54430c9d099992e6fb9d571717c3044604e8cdfb8e624cb881337d648030ee8b1541d544af8b338835e3f47ebe languageName: node linkType: hard @@ -8060,7 +8078,7 @@ __metadata: languageName: node linkType: hard -"keyv@npm:^4.5.3": +"keyv@npm:^4.5.4": version: 4.5.4 resolution: "keyv@npm:4.5.4" dependencies: @@ -9013,7 +9031,7 @@ __metadata: languageName: node linkType: hard -"minimatch@npm:^3.0.4, minimatch@npm:^3.0.5, minimatch@npm:^3.1.2": +"minimatch@npm:^3.0.4, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2" dependencies: @@ -9175,14 +9193,14 @@ __metadata: languageName: node linkType: hard -"msw-storybook-addon@npm:beta": - version: 2.0.0-beta.2 - resolution: "msw-storybook-addon@npm:2.0.0-beta.2" +"msw-storybook-addon@npm:^2.0.3": + version: 2.0.3 + resolution: "msw-storybook-addon@npm:2.0.3" dependencies: is-node-process: ^1.0.1 peerDependencies: msw: ^2.0.0 - checksum: 3d66f15bcc7fca12c256150d589d7c023fd0ea7720b0fd723113a0f1dbd472c935a96eb204e62e3b900c45b91ed4e2390e71f8a5937d628d14d84e4e97bea9c7 + checksum: 13b607849aadf1f8f67c2f7447b27acbea2582363c76a9b7fe98ecab3ee5a101a078009fb1c93ac63e5dc52a723c38a8e0761e88d0494299e69965abca12e16a languageName: node linkType: hard @@ -9481,7 +9499,7 @@ __metadata: languageName: node linkType: hard -"object.fromentries@npm:^2.0.7, object.fromentries@npm:^2.0.8": +"object.fromentries@npm:^2.0.8": version: 2.0.8 resolution: "object.fromentries@npm:2.0.8" dependencies: @@ -9493,31 +9511,18 @@ __metadata: languageName: node linkType: hard -"object.groupby@npm:^1.0.1": - version: 1.0.2 - resolution: "object.groupby@npm:1.0.2" - dependencies: - array.prototype.filter: ^1.0.3 - call-bind: ^1.0.5 - define-properties: ^1.2.1 - es-abstract: ^1.22.3 - es-errors: ^1.0.0 - checksum: 5f95c2a3a5f60a1a8c05fdd71455110bd3d5e6af0350a20b133d8cd70f9c3385d5c7fceb6a17b940c3c61752d9c202d10d5e2eb5ce73b89002656a87e7bf767a - languageName: node - linkType: hard - -"object.hasown@npm:^1.1.4": - version: 1.1.4 - resolution: "object.hasown@npm:1.1.4" +"object.groupby@npm:^1.0.3": + version: 1.0.3 + resolution: "object.groupby@npm:1.0.3" dependencies: + call-bind: ^1.0.7 define-properties: ^1.2.1 es-abstract: ^1.23.2 - es-object-atoms: ^1.0.0 - checksum: bc46eb5ca22106fcd07aab1411508c2c68b7565fe8fb272f166fb9bf203972e8b5c86a5a4b2c86204beead0626a7a4119d32cefbaf7c5dd57b400bf9e6363cb6 + checksum: 0d30693ca3ace29720bffd20b3130451dca7a56c612e1926c0a1a15e4306061d84410bdb1456be2656c5aca53c81b7a3661eceaa362db1bba6669c2c9b6d1982 languageName: node linkType: hard -"object.values@npm:^1.1.6, object.values@npm:^1.1.7, object.values@npm:^1.2.0": +"object.values@npm:^1.1.6, object.values@npm:^1.2.0": version: 1.2.0 resolution: "object.values@npm:1.2.0" dependencies: @@ -10327,6 +10332,8 @@ __metadata: version: 0.0.0-use.local resolution: "react-frontend@workspace:." dependencies: + "@eslint/compat": ^1.2.2 + "@eslint/js": ^9.13.0 "@mdx-js/react": ^3.1.0 "@mdx-js/rollup": ^3.1.0 "@microsoft/applicationinsights-react-js": ^17.3.3 @@ -10359,6 +10366,7 @@ __metadata: "@types/dompurify": ^3.0.5 "@types/dotenv-flow": ^3.3.3 "@types/downloadjs": ^1.4.6 + "@types/eslint__js": ^8.42.3 "@types/github-slugger": ^1.3.0 "@types/html-to-text": ^9.0.4 "@types/lodash": ^4.17.12 @@ -10369,8 +10377,6 @@ __metadata: "@types/react-router-dom": ^5.3.3 "@types/react-scroll-sync": ^0.9.0 "@types/sanitize-html": ^2.13.0 - "@typescript-eslint/eslint-plugin": ^8.10.0 - "@typescript-eslint/parser": ^8.10.0 "@uswds/uswds": 3.7.1 "@vitejs/plugin-react": ^4.3.3 "@vitest/coverage-istanbul": ^2.1.3 @@ -10387,21 +10393,22 @@ __metadata: dompurify: ^3.1.7 dotenv-flow: ^4.1.0 downloadjs: ^1.4.7 - eslint: 8.57 + eslint: 9.13.0 eslint-config-prettier: ^9.1.0 - eslint-import-resolver-typescript: ^3.6.1 - eslint-plugin-import: ^2.29.1 + eslint-import-resolver-typescript: ^3.6.3 + eslint-plugin-import: ^2.31.0 eslint-plugin-jest-dom: ^5.4.0 - eslint-plugin-jsx-a11y: ^6.9.0 - eslint-plugin-playwright: ^1.8.1 - eslint-plugin-react: ^7.34.3 - eslint-plugin-react-hooks: ^4.6.2 - eslint-plugin-react-refresh: ^0.4.7 + eslint-plugin-jsx-a11y: ^6.10.2 + eslint-plugin-playwright: ^2.0.0 + eslint-plugin-react: ^7.37.2 + eslint-plugin-react-hooks: ^5.0.0 + eslint-plugin-react-refresh: ^0.4.14 eslint-plugin-storybook: ^0.10.1 eslint-plugin-testing-library: ^6.4.0 eslint-plugin-vitest: ^0.5.4 export-to-csv-fix-source-map: ^0.2.1 focus-trap-react: ^10.3.0 + globals: ^15.11.0 history: ^5.3.0 html-to-text: ^9.0.5 husky: ^9.1.6 @@ -10410,7 +10417,7 @@ __metadata: lodash: ^4.17.21 mockdate: ^3.0.5 msw: ^2.4.11 - msw-storybook-addon: beta + msw-storybook-addon: ^2.0.3 npm-run-all: ^4.1.5 otpauth: ^9.3.4 p-limit: ^6.1.0 @@ -10443,10 +10450,11 @@ __metadata: tslib: ^2.8.0 tsx: ^4.19.1 typescript: ^5.6.3 + typescript-eslint: ^8.12.2 undici: ^6.20.1 use-deep-compare-effect: ^1.8.1 uuid: ^10.0.0 - vite: ^5.4.9 + vite: ^5.4.10 vite-plugin-checker: ^0.8.0 vite-plugin-svgr: ^4.2.0 vitest: ^2.1.3 @@ -11233,7 +11241,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.2": +"semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.2, semver@npm:^7.6.3": version: 7.6.3 resolution: "semver@npm:7.6.3" bin: @@ -11670,13 +11678,14 @@ __metadata: languageName: node linkType: hard -"string.prototype.includes@npm:^2.0.0": - version: 2.0.0 - resolution: "string.prototype.includes@npm:2.0.0" +"string.prototype.includes@npm:^2.0.1": + version: 2.0.1 + resolution: "string.prototype.includes@npm:2.0.1" dependencies: - define-properties: ^1.1.3 - es-abstract: ^1.17.5 - checksum: cf413e7f603b0414b65fdf1e7e3670ba85fd992b31c7eadfbdd9a484b86d265f0260431e7558cdb44a318dcadd1da8442b7bb8193b9ddd0aea3c376d2a559859 + call-bind: ^1.0.7 + define-properties: ^1.2.1 + es-abstract: ^1.23.3 + checksum: ed4b7058b092f30d41c4df1e3e805eeea92479d2c7a886aa30f42ae32fde8924a10cc99cccc99c29b8e18c48216608a0fe6bf887f8b4aadf9559096a758f313a languageName: node linkType: hard @@ -11711,6 +11720,16 @@ __metadata: languageName: node linkType: hard +"string.prototype.repeat@npm:^1.0.0": + version: 1.0.0 + resolution: "string.prototype.repeat@npm:1.0.0" + dependencies: + define-properties: ^1.1.3 + es-abstract: ^1.17.5 + checksum: 95dfc514ed7f328d80a066dabbfbbb1615c3e51490351085409db2eb7cbfed7ea29fdadaf277647fbf9f4a1e10e6dd9e95e78c0fd2c4e6bb6723ea6e59401004 + languageName: node + linkType: hard + "string.prototype.trim@npm:^1.2.9": version: 1.2.9 resolution: "string.prototype.trim@npm:1.2.9" @@ -12356,6 +12375,20 @@ __metadata: languageName: node linkType: hard +"typescript-eslint@npm:^8.12.2": + version: 8.12.2 + resolution: "typescript-eslint@npm:8.12.2" + dependencies: + "@typescript-eslint/eslint-plugin": 8.12.2 + "@typescript-eslint/parser": 8.12.2 + "@typescript-eslint/utils": 8.12.2 + peerDependenciesMeta: + typescript: + optional: true + checksum: 55412f46f2ce3be317107778a9b3f60d1ccc8b3b6f600eefc077d20109890801d684e995eace8ed96a9f8ca824acb2d31c853c2754715f18edb38caf51a76270 + languageName: node + linkType: hard + "typescript@npm:^5.6.3": version: 5.6.3 resolution: "typescript@npm:5.6.3" @@ -12879,9 +12912,9 @@ __metadata: languageName: node linkType: hard -"vite@npm:^5.0.0, vite@npm:^5.4.9": - version: 5.4.9 - resolution: "vite@npm:5.4.9" +"vite@npm:^5.0.0, vite@npm:^5.4.10": + version: 5.4.10 + resolution: "vite@npm:5.4.10" dependencies: esbuild: ^0.21.3 fsevents: ~2.3.3 @@ -12918,7 +12951,7 @@ __metadata: optional: true bin: vite: bin/vite.js - checksum: d3229e0618ece284af0478ec09c474a7a70ac369920716afdb6ebed8e320fd17a17c60afddba0d436698fe4837474cccd057c3e7d8270281b57506b78c5fbb8c + checksum: 4db3b8ca3eddbc312d0a95f505d16656e74c6dfa68d3b5eb54b6d6b0f7be1df348d469c43dc69db27dadc06b802f029d654da48f392324efd665ef2c0ca9ba9e languageName: node linkType: hard diff --git a/prime-router/docs/getting-started/swagger.md b/prime-router/docs/getting-started/swagger.md index 8194d47b68e..0829fbcbd25 100644 --- a/prime-router/docs/getting-started/swagger.md +++ b/prime-router/docs/getting-started/swagger.md @@ -62,15 +62,15 @@ Starting from the Okta section in the `Authorize` menu 1. Login to OKTA as an administrator and click the "Admin" button in the top-right 2. In the left pane, navigate Applications -> applications -3. In the resulting right pane, select the instance to be configured (e.g. `Simple Report (localdev)`) +3. In the resulting right pane, select the instance to be configured (e.g. `Swagger`) 4. Viewing the details of the instance, you will see a `client_id` and `client_secret` (be sure to mask these values) 5. Under General Settings, ensure the "Authorization Code" flow is checked 6. Under Login, ensure this value appears in the sign-in redirect URI list: `http://127.0.0.1:10000/devstoreaccount1/apidocs/oauth2-redirect.html` 7. Ensure the application instance is associated with your OKTA account. Select assignment at the top of the page and ensure your username is selected. -8. You need to associate "Simple Report (localdev)" with you - your OKTA account (your email/password/MFA) - To do so, click the assignment on top of the page and you will see all the users : Joe Smith, Jane Doe etc., select your user name, and you will be tied to the app - Simple Report (localdev) +8. You need to associate "Swagger" with you - your OKTA account (your email/password/MFA) + To do so, click the assignment on top of the page and you will see all the users : Joe Smith, Jane Doe etc., select your user name, and you will be tied to the app - Swagger #### Server-to-server diff --git a/prime-router/docs/okta/admin-management.md b/prime-router/docs/okta/admin-management.md new file mode 100644 index 00000000000..f79ef9ee6fb --- /dev/null +++ b/prime-router/docs/okta/admin-management.md @@ -0,0 +1,39 @@ +# Admin Management + +More details about the organization within okta can be found in [this doc](https://cdc.sharepoint.com/:p:/r/teams/ReportStream/_layouts/15/Doc.aspx?sourcedoc=%7B313111b2-502c-4f60-ac8c-bbcf3c9b1dab%7D&action=edit&wdPreviousSession=a28aeb1e-02b3-b6be-49ab-cafb30120e6f) + +Okta admin potential responsibility areas are: +- App registry management +- User/group management +- Security configuration management +- Log checking + +ReportStream's Okta has the following specialized admin roles for team members: +- Owners +- Support Team +- Onboarding Engineers +- Front-end Engineers +- Tech Leads + + +## App registry management + +The app registry page can be found by the following side-navigation: Applications > Applications. + +All reportstream-developed programs with authentication elements should be configured towards an application listed on this page. + + +## User/Group management + +Accessible via the side-navigation: Directory > People or Directory > Groups + +## Security configuration management + +The policies are enforced in the following order (accessed through "Security" in side-navigation): +- Global Session Policy +- Authentication Policy +- Password Policy (from side-navigation: Security > Authenticators > Click Actions for the "Password" table line > Edit) + +## Log checking + +The global log can be accessed from side-navigation: Reports > System Log. They can also be filtered by user by going to the user's management page (side-navigation: Directory > People) and clicking "View Logs". \ No newline at end of file diff --git a/prime-router/docs/onboarding-users/receivers.md b/prime-router/docs/onboarding-users/receivers.md index 5ff677f3d70..1d141e11d14 100644 --- a/prime-router/docs/onboarding-users/receivers.md +++ b/prime-router/docs/onboarding-users/receivers.md @@ -176,7 +176,7 @@ output here: `/prime-router/build/sftp` ### 5. Create access to the Download site -* If the organization has elected for download access, set up an Okta account. +* If the organization has elected for download access, [set up an Okta account](./okta-account-creation.md). * If you are testing in Test, obviously you'll need to set up access to that download site. ### 6. Validation in Prod diff --git a/prime-router/settings/STLTs/AL/al-phl.yml b/prime-router/settings/STLTs/AL/al-phl.yml deleted file mode 100644 index 500a2f1ab89..00000000000 --- a/prime-router/settings/STLTs/AL/al-phl.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Alabama Public Health Lab settings for staging -# -# To load the settings to staging, run: -# ./prime login --env staging -# ./prime multiple-settings set --env staging --input ./settings/STLTs/AL/al-phl.yml -# -# To add the sender key in staging: -# ./prime organization addkey --env staging --public-key /path/to/public/key.pem --scope "al-phl.*.report" --orgName al-phl --kid al-phl.etor-nbs-results --doit -# -# To submit an order to al-phl, in staging: -# Note: replace the TOKEN with the auth JWT and the path to the FHIR message to send -# curl -H 'Authorization: Bearer TOKEN' -H 'Client: flexion.etor-service-sender' -H 'Content-Type: application/fhir+ndjson' --data-binary '@/path/to/oml.fhir' 'https://staging.prime.cdc.gov/api/waters' -# -# To submit a result from al-phl, in staging: -# curl -H 'Authorization: Bearer TOKEN' -H 'Client: al-phl.etor-nbs-results' -H 'Content-Type: application/hl7-v2' --data-binary '@/path/to/oru.hl7' 'https://staging.prime.cdc.gov/api/waters' ---- -- name: "al-phl" - description: "Alabama Public Health Lab" - jurisdiction: "STATE" - stateCode: "AL" - senders: - - name: "etor-nbs-results" - organizationName: "al-phl" - topic: "etor-ti" - customerStatus: "active" - format: "HL7" - receivers: - - name: "etor-nbs-orders" - organizationName: "al-phl" - topic: "etor-ti" - customerStatus: "active" - jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21'" # OML_O21 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI - - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.2,HD.3').value = '2.16.840.1.114222.4.3.26.1.1'" # receiver routing filter (MSH-6.2) - qualityFilter: - - "true" - timing: - operation: "MERGE" - numberPerDay: 1440 - initialTime: "00:00" - timeZone: "EASTERN" - maxReportCount: 100 - translation: - type: "HL7" - schemaName: "classpath:/metadata/hl7_mapping/receivers/Flexion/etor-oml-receiver-transform.yml" - useBatchHeaders: false - receivingApplicationName: "AL-PHL" - receivingFacilityName: "AL" - transport: - host: "ph.state.al.us" - port: "22" - filePath: "./" - credentialName: null - type: "SFTP" \ No newline at end of file diff --git a/prime-router/settings/STLTs/CA/ca-phl.yml b/prime-router/settings/STLTs/CA/ca-phl.yml index 36bc6e6ac4a..30e3f2c28d6 100644 --- a/prime-router/settings/STLTs/CA/ca-phl.yml +++ b/prime-router/settings/STLTs/CA/ca-phl.yml @@ -20,6 +20,6 @@ organizationName: "ca-phl" topic: "etor-ti" customerStatus: "active" - schemaName: classpath:/metadata/fhir_transforms/senders/Flexion/cdph-nbs-sender-transform.yml + schemaName: "classpath:/metadata/fhir_transforms/senders/Flexion/cdph-nbs-sender-transform.yml" format: "HL7" receivers: [] diff --git a/prime-router/settings/STLTs/CA/ucsd.yml b/prime-router/settings/STLTs/CA/ucsd.yml index 2d5afa6e39d..bacfc9b602b 100644 --- a/prime-router/settings/STLTs/CA/ucsd.yml +++ b/prime-router/settings/STLTs/CA/ucsd.yml @@ -25,11 +25,11 @@ receivingApplicationName: "EPIC-INNERCONNECT" receivingFacilityName: "CA" jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI - - "Bundle.entry.resource.ofType(MessageHeader).sender.resolve().identifier.where(value = 'CDPH').exists()" # sender routing filter (MSH-4.1) - - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.1').value in ('R797' | 'R508')" # receiver routing filter (MSH-6.1) - - "Bundle.identifier.value.contains('AUTOMATEDTEST-').not()" # exclude flexion automated test messages (MSH-10) + - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://terminology.hl7.org/CodeSystem/v2-0103').code in ('T' | 'P')" # partner processing ids (MSH-11) + - "Bundle.entry.resource.ofType(MessageHeader).sender.resolve().identifier.where(value = 'CDPH').exists()" # sender routing filter (MSH-4.1) + - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.1').value in ('R797' | 'R508')" # receiver routing filter (MSH-6.1) qualityFilter: - "true" timing: diff --git a/prime-router/settings/STLTs/Flexion/flexion.yml b/prime-router/settings/STLTs/Flexion/flexion.yml index ffaf0ebaf79..f736c5ff2eb 100644 --- a/prime-router/settings/STLTs/Flexion/flexion.yml +++ b/prime-router/settings/STLTs/Flexion/flexion.yml @@ -8,9 +8,10 @@ # ./prime multiple-settings set --env staging --input ./settings/STLTs/Flexion/flexion.yml # # To add the sender keys in staging: -# ./prime organization addkey --env staging --public-key /path/to/public/key.pem --scope "flexion.*.report" --orgName flexion --kid flexion.simulated-hospital --doit # ./prime organization addkey --env staging --public-key /path/to/public/key.pem --scope "flexion.*.report" --orgName flexion --kid flexion.etor-service-sender --doit +# ./prime organization addkey --env staging --public-key /path/to/public/key.pem --scope "flexion.*.report" --orgName flexion --kid flexion.simulated-hospital --doit # ./prime organization addkey --env staging --public-key /path/to/public/key.pem --scope "flexion.*.report" --orgName flexion --kid flexion.simulated-lab --doit +# ./prime organization addkey --env staging --public-key /path/to/public/key.pem --scope "flexion.*.report" --orgName flexion --kid flexion.simulated-sender --doit # # To submit an order or result to flexion.simulated-lab or flexion.simulated-hospital, in staging: # Note: replace the TOKEN with the auth JWT and the path to the FHIR message to send @@ -46,12 +47,13 @@ topic: "etor-ti" customerStatus: "active" format: "HL7" - # Sender used for automated tests in staging - - name: "automated-staging-test-sender" + # Sender used for local tests and automated tests in staging + - name: "simulated-sender" organizationName: "flexion" topic: "etor-ti" customerStatus: "active" format: "HL7" + schemaName: "classpath:/metadata/fhir_transforms/senders/Flexion/automated-testing-etor.yml" receivers: # ETOR Service Receiver for orders - Routes ORM_O01 and OML_O21 FHIR orders to TI orders endpoint - name: "etor-service-receiver-orders" @@ -59,8 +61,8 @@ topic: "etor-ti" customerStatus: "active" jurisdictionalFilter: - - "(Bundle.entry.resource.ofType(MessageHeader).event.code = 'O01') or (Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21')" # ORM_O01 or OML_O21 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.exists(system = 'http://localcodes.org/ETOR').not()" # required to avoid looping issue + - "(Bundle.entry.resource.ofType(MessageHeader).event.code = 'O01') or (Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21')" # ORM_O01 or OML_O21 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.exists(system = 'http://localcodes.org/ETOR').not()" # required to avoid looping issue qualityFilter: - "true" timing: @@ -97,8 +99,8 @@ topic: "etor-ti" customerStatus: "active" jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.exists(system = 'http://localcodes.org/ETOR').not()" # required to avoid looping issue + - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.exists(system = 'http://localcodes.org/ETOR').not()" # required to avoid looping issue qualityFilter: - "true" timing: @@ -133,9 +135,9 @@ topic: "etor-ti" customerStatus: "active" jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue - - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.2,HD.3').value = 'simulated-hospital-id'" # receiver routing filter (MSH-6.2) + - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://terminology.hl7.org/CodeSystem/v2-0103').code = 'D'" # internal test processing id (MSH-11) qualityFilter: - "true" timing: @@ -147,21 +149,15 @@ schemaName: "classpath:/metadata/hl7_mapping/ORU_R01/ORU_R01-base.yml" useTestProcessingMode: false useBatchHeaders: false - transport: - type: "SFTP" - host: "172.17.6.20" # use "sftp" if running locally - port: 22 - filePath: "./upload" - credentialName: null # use "DEFAULT-SFTP" if running locally # Simulated State Public Health Lab Receiver: converts OML_O21 from FHIR to HL7 and routes the message to the SFTP folder - name: "simulated-lab" organizationName: "flexion" topic: "etor-ti" customerStatus: "active" jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21'" # OML_O21 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue - - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.2,HD.3').value = 'simulated-lab-id'" # receiver routing filter (MSH-6.2) + - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21'" # OML_O21 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://terminology.hl7.org/CodeSystem/v2-0103').code = 'D'" # internal test processing id (MSH-11) qualityFilter: - "true" timing: @@ -173,20 +169,14 @@ schemaName: "classpath:/metadata/hl7_mapping/receivers/Flexion/etor-oml-receiver-transform.yml" useTestProcessingMode: false useBatchHeaders: false - transport: - type: "SFTP" - host: "172.17.6.20" # use "sftp" if running locally - port: 22 - filePath: "./upload" - credentialName: null # use "DEFAULT-SFTP" if running locally - name: "automated-staging-test-receiver-orders" organizationName: "flexion" topic: "etor-ti" customerStatus: "active" jurisdictionalFilter: - - "(Bundle.entry.resource.ofType(MessageHeader).event.code = 'O01') or (Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21')" # ORM_O01 or OML_O21 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue - - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.2,HD.3').value = 'automated-staging-test-receiver-id'" # receiver routing filter (MSH-6.2) + - "(Bundle.entry.resource.ofType(MessageHeader).event.code = 'O01') or (Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21')" # ORM_O01 or OML_O21 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://terminology.hl7.org/CodeSystem/v2-0103').code = 'N'" # automated test processing id (MSH-11) qualityFilter: - "true" timing: @@ -207,10 +197,9 @@ topic: "etor-ti" customerStatus: "active" jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue - - "(Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.2,HD.3').value = 'automated-staging-test-receiver-id') - or (Bundle.identifier.value.contains('AUTOMATEDTEST-'))" # receiver routing filter (MSH-6.2 or MSH-10) + - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://terminology.hl7.org/CodeSystem/v2-0103').code = 'N'" # automated test processing id (MSH-11) qualityFilter: - "true" timing: diff --git a/prime-router/settings/STLTs/LA/la-ochsner.yml b/prime-router/settings/STLTs/LA/la-ochsner.yml index 0e4f8c2efd6..f013303ed67 100644 --- a/prime-router/settings/STLTs/LA/la-ochsner.yml +++ b/prime-router/settings/STLTs/LA/la-ochsner.yml @@ -34,9 +34,10 @@ schemaName: "classpath:/metadata/hl7_mapping/ORU_R01/ORU_R01-base.yml" useBatchHeaders: false jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI - - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.2,HD.3').value = '1.2.840.114350.1.13.286.2.7.2.695071'" # receiver routing filter (MSH-6.2) + - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://terminology.hl7.org/CodeSystem/v2-0103').code in ('T' | 'P')" # partner processing ids (MSH-11) + - "Bundle.entry.resource.ofType(MessageHeader).destination.extension.where(url = 'https://reportstream.cdc.gov/fhir/StructureDefinition/universal-id').value = '1.2.840.114350.1.13.286.2.7.2.695071'" # receiver routing filter (MSH-5.2) qualityFilter: - "true" timing: diff --git a/prime-router/settings/STLTs/LA/la-phl.yml b/prime-router/settings/STLTs/LA/la-phl.yml index 006e500f903..17b8e1e5373 100644 --- a/prime-router/settings/STLTs/LA/la-phl.yml +++ b/prime-router/settings/STLTs/LA/la-phl.yml @@ -30,9 +30,10 @@ topic: "etor-ti" customerStatus: "active" jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21'" # OML_O21 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI - - "Bundle.entry.resource.ofType(MessageHeader).destination.extension.where(url = 'https://reportstream.cdc.gov/fhir/StructureDefinition/universal-id').value = '2.16.840.1.114222.4.3.26.3.2'" # receiver routing filter (MSH-5.2) + - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'O21'" # OML_O21 + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI + - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://terminology.hl7.org/CodeSystem/v2-0103').code in ('T' | 'P')" # partner processing ids (MSH-11) + - "Bundle.entry.resource.ofType(MessageHeader).destination.extension.where(url = 'https://reportstream.cdc.gov/fhir/StructureDefinition/universal-id').value = '2.16.840.1.114222.4.3.26.3.2'" # receiver routing filter (MSH-5.2) qualityFilter: - "true" timing: diff --git a/prime-router/settings/STLTs/Oracle/oracle-rln.yml b/prime-router/settings/STLTs/Oracle/oracle-rln.yml deleted file mode 100644 index 4d4608b8848..00000000000 --- a/prime-router/settings/STLTs/Oracle/oracle-rln.yml +++ /dev/null @@ -1,60 +0,0 @@ -# Oracle RLN settings in staging -# -# Oracle RLN is defined at the Federal level and will route the message to its final destination -# Partners using Oracle RLN are: -# - Baptist in Alabama -# -# To load the settings in staging, run: -# ./prime login --env staging -# ./prime multiple-settings set --env staging --input ./settings/STLTs/Oracle/oracle-rln.yml -# -# To add the sender key in staging: -# ./prime organization addkey --env staging --public-key /path/to/public/key.pem --scope "oracle-rln.*.report" --orgName oracle-rln --kid oracle-rln.etor-nbs-orders --doit -# -# To submit a result to oracle-rln, in staging: -# Note: replace the TOKEN with the auth JWT and the path to the FHIR message to send -# curl -H 'Authorization: Bearer TOKEN' -H 'Client: flexion.etor-service-sender' -H 'Content-Type: application/fhir+ndjson' --data-binary '@/path/to/oru.fhir' 'https://staging.prime.cdc.gov/api/waters' -# -# To submit an order from la-ochsner, in staging: -# curl -H 'Authorization: Bearer TOKEN' -H 'Client: oracle-rln.etor-nbs-orders' -H 'Content-Type: application/hl7-v2' --data-binary '@/path/to/order.hl7' 'https://staging.prime.cdc.gov/api/waters' ---- -- name: "oracle-rln" - description: "Oracle RLN" - jurisdiction: "FEDERAL" - senders: - - name: "etor-nbs-orders" - organizationName: "oracle-rln" - topic: "etor-ti" - customerStatus: "active" - format: "HL7" - receivers: - - name: "etor-nbs-results" - organizationName: "oracle-rln" - topic: "etor-ti" - customerStatus: "active" - translation: - type: "HL7" - schemaName: "classpath:/metadata/hl7_mapping/ORU_R01/ORU_R01-base.yml" - useBatchHeaders: false - jurisdictionalFilter: - - "Bundle.entry.resource.ofType(MessageHeader).event.code = 'R01'" # ORU_R01 - - "Bundle.entry.resource.ofType(MessageHeader).meta.tag.where(system = 'http://localcodes.org/ETOR').code = 'ETOR'" # required to avoid looping issue with TI - - "Bundle.entry.resource.ofType(MessageHeader).destination.receiver.resolve().identifier.where(extension.value = 'HD.2,HD.3').value in ('2.16.840.1.113883.3.1898' | '2.16.840.1.113883.3.1899')" # receiver routing filter (MSH-6.2) - qualityFilter: - - "true" - timing: - operation: "MERGE" - numberPerDay: 1440 - initialTime: "00:00" - timeZone: "EASTERN" - maxReportCount: 100 - description: "" - transport: - type: "REST" - reportUrl: "https://spaces.erxhubdevcert.cerner.com/etor" - authType: "apiKey" - authTokenUrl: "" - tlsKeystore: null - headers: - Content-Type: "text/plain" - shared-api-key: "From Vault" diff --git a/prime-router/src/main/kotlin/Receiver.kt b/prime-router/src/main/kotlin/Receiver.kt index ad7b2ed0908..5d0e74d3fe6 100644 --- a/prime-router/src/main/kotlin/Receiver.kt +++ b/prime-router/src/main/kotlin/Receiver.kt @@ -159,6 +159,9 @@ open class Receiver( @get:JsonIgnore val displayName: String get() = externalName ?: name + @get:JsonIgnore + val transportType: TransportType get() = transport ?: NullTransportType() + /** * Defines how batching of sending should proceed. Allows flexibility of * frequency and transmission time on daily basis, but not complete flexibility. diff --git a/prime-router/src/main/kotlin/TransportType.kt b/prime-router/src/main/kotlin/TransportType.kt index 88a1fdfc0bd..2950cc95f84 100644 --- a/prime-router/src/main/kotlin/TransportType.kt +++ b/prime-router/src/main/kotlin/TransportType.kt @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo JsonSubTypes.Type(GAENTransportType::class, name = "GAEN"), JsonSubTypes.Type(RESTTransportType::class, name = "REST") ) -abstract class TransportType(val type: String) +sealed class TransportType(val type: String) data class SFTPTransportType @JsonCreator constructor( diff --git a/prime-router/src/main/kotlin/azure/SendFunction.kt b/prime-router/src/main/kotlin/azure/SendFunction.kt index 1225af6da02..b1081c1a389 100644 --- a/prime-router/src/main/kotlin/azure/SendFunction.kt +++ b/prime-router/src/main/kotlin/azure/SendFunction.kt @@ -8,6 +8,7 @@ import com.microsoft.azure.functions.annotation.StorageAccount import gov.cdc.prime.router.AS2TransportType import gov.cdc.prime.router.BlobStoreTransportType import gov.cdc.prime.router.CustomerStatus +import gov.cdc.prime.router.EmailTransportType import gov.cdc.prime.router.GAENTransportType import gov.cdc.prime.router.NullTransportType import gov.cdc.prime.router.RESTTransportType @@ -25,7 +26,6 @@ import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService import gov.cdc.prime.router.transport.ITransport -import gov.cdc.prime.router.transport.NullTransport import gov.cdc.prime.router.transport.RetryToken import org.apache.logging.log4j.kotlin.Logging import java.time.OffsetDateTime @@ -100,30 +100,26 @@ class SendFunction( receiverStatus = receiver.customerStatus val inputReportId = header.reportFile.reportId actionHistory.trackExistingInputReport(inputReportId) - val serviceName = receiver.fullName val nextRetryItems = mutableListOf() val externalFileName = Report.formExternalFilename(header, workflowEngine.reportService) val sentReportId = UUID.randomUUID() // each sent report gets its own UUID - if (receiver.transport == null) { - actionHistory.setActionType(TaskAction.send_warning) - actionHistory.trackActionResult("Not sending $inputReportId to $serviceName: No transports defined") - } else { - val retryItems = retryToken?.items - val nextRetry = getTransport(receiver.transport)?.send( - receiver.transport, - header, - sentReportId, - externalFileName, - retryItems, - context, - actionHistory, - reportEventService, - workflowEngine.reportService - ) - if (nextRetry != null) { - nextRetryItems += nextRetry - } + val retryItems = retryToken?.items + val transport = getTransport(receiver.transportType) + val nextRetry = transport.send( + receiver.transportType, + header, + sentReportId, + externalFileName, + retryItems, + context, + actionHistory, + reportEventService, + workflowEngine.reportService + ) + if (nextRetry != null) { + nextRetryItems += nextRetry } + logger.info("For $inputReportId: finished send(). Checking to see if a retry is needed.") handleRetry( nextRetryItems, @@ -154,7 +150,9 @@ class SendFunction( } } - private fun getTransport(transportType: TransportType): ITransport? { + private fun getTransport(transportType: TransportType): ITransport { + // IntelliJ complains about this when, but there's a ticket in for it https://youtrack.jetbrains.com/issue/KTIJ-21016 + // It should still compile, unless a TransportType was added without adding it here return when (transportType) { is SFTPTransportType -> workflowEngine.sftpTransport is BlobStoreTransportType -> workflowEngine.blobStoreTransport @@ -162,8 +160,8 @@ class SendFunction( is SoapTransportType -> workflowEngine.soapTransport is GAENTransportType -> workflowEngine.gaenTransport is RESTTransportType -> workflowEngine.restTransport - is NullTransportType -> NullTransport() - else -> null + is NullTransportType -> workflowEngine.nullTransport + is EmailTransportType -> workflowEngine.emailTransport } } diff --git a/prime-router/src/main/kotlin/azure/WorkflowEngine.kt b/prime-router/src/main/kotlin/azure/WorkflowEngine.kt index 7f79e289a97..240602e6855 100644 --- a/prime-router/src/main/kotlin/azure/WorkflowEngine.kt +++ b/prime-router/src/main/kotlin/azure/WorkflowEngine.kt @@ -31,7 +31,9 @@ import gov.cdc.prime.router.serializers.Hl7Serializer import gov.cdc.prime.router.serializers.ReadResult import gov.cdc.prime.router.transport.AS2Transport import gov.cdc.prime.router.transport.BlobStoreTransport +import gov.cdc.prime.router.transport.EmailTransport import gov.cdc.prime.router.transport.GAENTransport +import gov.cdc.prime.router.transport.NullTransport import gov.cdc.prime.router.transport.RESTTransport import gov.cdc.prime.router.transport.RetryItems import gov.cdc.prime.router.transport.RetryToken @@ -71,6 +73,8 @@ class WorkflowEngine( val soapTransport: SoapTransport = SoapTransport(), val gaenTransport: GAENTransport = GAENTransport(), val restTransport: RESTTransport = RESTTransport(), + val nullTransport: NullTransport = NullTransport(), + val emailTransport: EmailTransport = EmailTransport(), ) : BaseEngine(queue) { /** diff --git a/prime-router/src/main/kotlin/azure/observability/event/ReportStreamEventService.kt b/prime-router/src/main/kotlin/azure/observability/event/ReportStreamEventService.kt index 42509ce0748..36ae8a908f7 100644 --- a/prime-router/src/main/kotlin/azure/observability/event/ReportStreamEventService.kt +++ b/prime-router/src/main/kotlin/azure/observability/event/ReportStreamEventService.kt @@ -1,7 +1,6 @@ package gov.cdc.prime.router.azure.observability.event import gov.cdc.prime.router.Report -import gov.cdc.prime.router.ReportId import gov.cdc.prime.router.Topic import gov.cdc.prime.router.azure.DatabaseAccess import gov.cdc.prime.router.azure.db.enums.TaskAction @@ -22,18 +21,28 @@ import java.util.UUID */ interface IReportStreamEventService { + /** + * Sends any events that have been queued up if the client specified sending them can be deferred. + * + * This is useful in contexts where the events should only be sent after all the business logic has + * executed and the DB transaction has been committed. + */ + fun sendQueuedEvents() + /** * Creates a report event from an [Report] * * @param eventName the business event value from [ReportStreamEventName] * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendReportEvent( eventName: ReportStreamEventName, childReport: Report, pipelineStepName: TaskAction, + shouldQueue: Boolean = false, initializer: ReportStreamReportEventBuilder.() -> Unit, ) @@ -43,12 +52,14 @@ interface IReportStreamEventService { * @param eventName the business event value from [ReportStreamEventName] * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendReportEvent( eventName: ReportStreamEventName, childReport: ReportFile, pipelineStepName: TaskAction, + shouldQueue: Boolean = false, initializer: ReportStreamReportEventBuilder.() -> Unit, ) @@ -59,6 +70,7 @@ interface IReportStreamEventService { * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event * @param error the error description + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendReportProcessingError( @@ -66,6 +78,7 @@ interface IReportStreamEventService { childReport: ReportFile, pipelineStepName: TaskAction, error: String, + shouldQueue: Boolean = false, initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, ) @@ -76,6 +89,7 @@ interface IReportStreamEventService { * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event * @param error the error description + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendReportProcessingError( @@ -83,25 +97,7 @@ interface IReportStreamEventService { childReport: Report, pipelineStepName: TaskAction, error: String, - initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, - ) - - /** - * Creates a general processing error event. This is not associated with a report or item. - * - * @param eventName the business event value from [ReportStreamEventName] - * @param pipelineStepName the pipeline step that is emitting the event - * @param error the error description - * @param submissionId the report id for the incoming report - * @param bodyUrl the blob url for the incoming report - * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] - */ - fun sendSubmissionProcessingError( - eventName: ReportStreamEventName, - pipelineStepName: TaskAction, - error: String, - submissionId: ReportId, - bodyUrl: String, + shouldQueue: Boolean = false, initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, ) @@ -111,12 +107,14 @@ interface IReportStreamEventService { * @param eventName the business event value from [ReportStreamEventName] * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendItemEvent( eventName: ReportStreamEventName, childReport: Report, pipelineStepName: TaskAction, + shouldQueue: Boolean = false, initializer: ReportStreamItemEventBuilder.() -> Unit, ) @@ -126,12 +124,14 @@ interface IReportStreamEventService { * @param eventName the business event value from [ReportStreamEventName] * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendItemEvent( eventName: ReportStreamEventName, childReport: ReportFile, pipelineStepName: TaskAction, + shouldQueue: Boolean = false, initializer: ReportStreamItemEventBuilder.() -> Unit, ) @@ -142,6 +142,7 @@ interface IReportStreamEventService { * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event * @param error the error description + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendItemProcessingError( @@ -149,6 +150,7 @@ interface IReportStreamEventService { childReport: ReportFile, pipelineStepName: TaskAction, error: String, + shouldQueue: Boolean = false, initializer: ReportStreamItemProcessingErrorEventBuilder.() -> Unit, ) @@ -159,6 +161,7 @@ interface IReportStreamEventService { * @param childReport the report that is getting emitted from the pipeline step * @param pipelineStepName the pipeline step that is emitting the event * @param error the error description + * @param shouldQueue whether to send the event immediately or defer it to be sent later * @param initializer additional data to initialize the creation of the event. See [AbstractReportStreamEventBuilder] */ fun sendItemProcessingError( @@ -166,6 +169,7 @@ interface IReportStreamEventService { childReport: Report, pipelineStepName: TaskAction, error: String, + shouldQueue: Boolean = false, initializer: ReportStreamItemProcessingErrorEventBuilder.() -> Unit, ) @@ -216,13 +220,23 @@ class ReportStreamEventService( private val reportService: ReportService, ) : IReportStreamEventService { + private val builtEvents = mutableListOf>() + + override fun sendQueuedEvents() { + builtEvents.forEach { + it.send() + } + builtEvents.clear() + } + override fun sendReportEvent( eventName: ReportStreamEventName, childReport: Report, pipelineStepName: TaskAction, + shouldQueue: Boolean, initializer: ReportStreamReportEventBuilder.() -> Unit, ) { - ReportStreamReportEventBuilder( + val builder = ReportStreamReportEventBuilder( this, azureEventService, eventName, @@ -232,16 +246,22 @@ class ReportStreamEventService( pipelineStepName ).apply( initializer - ).send() + ) + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun sendReportEvent( eventName: ReportStreamEventName, childReport: ReportFile, pipelineStepName: TaskAction, + shouldQueue: Boolean, initializer: ReportStreamReportEventBuilder.() -> Unit, ) { - ReportStreamReportEventBuilder( + val builder = ReportStreamReportEventBuilder( this, azureEventService, eventName, @@ -251,7 +271,13 @@ class ReportStreamEventService( pipelineStepName ).apply( initializer - ).send() + ) + + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun sendReportProcessingError( @@ -259,9 +285,10 @@ class ReportStreamEventService( childReport: ReportFile, pipelineStepName: TaskAction, error: String, + shouldQueue: Boolean, initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, ) { - ReportStreamReportProcessingErrorEventBuilder( + val builder = ReportStreamReportProcessingErrorEventBuilder( this, azureEventService, eventName, @@ -272,7 +299,13 @@ class ReportStreamEventService( error ).apply( initializer - ).send() + ) + + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun sendReportProcessingError( @@ -280,9 +313,10 @@ class ReportStreamEventService( childReport: Report, pipelineStepName: TaskAction, error: String, + shouldQueue: Boolean, initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, ) { - ReportStreamReportProcessingErrorEventBuilder( + val builder = ReportStreamReportProcessingErrorEventBuilder( this, azureEventService, eventName, @@ -293,38 +327,23 @@ class ReportStreamEventService( error ).apply( initializer - ).send() - } + ) - override fun sendSubmissionProcessingError( - eventName: ReportStreamEventName, - pipelineStepName: TaskAction, - error: String, - submissionId: ReportId, - bodyUrl: String, - initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, - ) { - ReportStreamReportProcessingErrorEventBuilder( - this, - azureEventService, - eventName, - submissionId, - bodyUrl, - theTopic = null, - pipelineStepName, - error - ).apply( - initializer - ).send() + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun sendItemEvent( eventName: ReportStreamEventName, childReport: Report, pipelineStepName: TaskAction, + shouldQueue: Boolean, initializer: ReportStreamItemEventBuilder.() -> Unit, ) { - ReportStreamItemEventBuilder( + val builder = ReportStreamItemEventBuilder( this, azureEventService, eventName, @@ -332,16 +351,23 @@ class ReportStreamEventService( childReport.bodyURL, childReport.schema.topic, pipelineStepName - ).apply(initializer).send() + ).apply(initializer) + + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun sendItemEvent( eventName: ReportStreamEventName, childReport: ReportFile, pipelineStepName: TaskAction, + shouldQueue: Boolean, initializer: ReportStreamItemEventBuilder.() -> Unit, ) { - ReportStreamItemEventBuilder( + val builder = ReportStreamItemEventBuilder( this, azureEventService, eventName, @@ -349,7 +375,13 @@ class ReportStreamEventService( childReport.bodyUrl, childReport.schemaTopic, pipelineStepName - ).apply(initializer).send() + ).apply(initializer) + + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun sendItemProcessingError( @@ -357,9 +389,10 @@ class ReportStreamEventService( childReport: ReportFile, pipelineStepName: TaskAction, error: String, + shouldQueue: Boolean, initializer: ReportStreamItemProcessingErrorEventBuilder.() -> Unit, ) { - ReportStreamItemProcessingErrorEventBuilder( + val builder = ReportStreamItemProcessingErrorEventBuilder( this, azureEventService, eventName, @@ -368,7 +401,13 @@ class ReportStreamEventService( childReport.schemaTopic, pipelineStepName, error - ).apply(initializer).send() + ).apply(initializer) + + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun sendItemProcessingError( @@ -376,9 +415,10 @@ class ReportStreamEventService( childReport: Report, pipelineStepName: TaskAction, error: String, + shouldQueue: Boolean, initializer: ReportStreamItemProcessingErrorEventBuilder.() -> Unit, ) { - ReportStreamItemProcessingErrorEventBuilder( + val builder = ReportStreamItemProcessingErrorEventBuilder( this, azureEventService, eventName, @@ -387,7 +427,13 @@ class ReportStreamEventService( childReport.schema.topic, pipelineStepName, error - ).apply(initializer).send() + ).apply(initializer) + + if (shouldQueue) { + builtEvents.add(builder) + } else { + builder.send() + } } override fun getReportEventData( @@ -398,13 +444,10 @@ class ReportStreamEventService( topic: Topic?, ): ReportEventData { val submittedReportIds = if (parentReportId != null) { - val rootReports = reportService.getRootReports(parentReportId) - rootReports.ifEmpty { - listOf(dbAccess.fetchReportFile(parentReportId)) - } + reportService.getRootReports(parentReportId) } else { emptyList() - }.map { it.reportId } + }.map { it.reportId }.ifEmpty { if (parentReportId != null) listOf(parentReportId) else emptyList() } return ReportEventData( childReportId, @@ -425,8 +468,10 @@ class ReportStreamEventService( trackingId: String?, ): ItemEventData { val submittedIndex = reportService.getRootItemIndex(parentReportId, parentItemIndex) ?: parentItemIndex + val rootReport = - reportService.getRootReports(parentReportId).firstOrNull() ?: dbAccess.fetchReportFile(parentReportId) + reportService.getRootReports(parentReportId).firstOrNull() ?: dbAccess.fetchReportFile(parentReportId) + return ItemEventData( childItemIndex, parentItemIndex, diff --git a/prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt b/prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt index 8cdfd6bc150..694601cace9 100644 --- a/prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt +++ b/prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt @@ -21,10 +21,22 @@ import gov.cdc.prime.router.Hl7Configuration import gov.cdc.prime.router.Metadata import gov.cdc.prime.router.MimeFormat import gov.cdc.prime.router.Receiver +import gov.cdc.prime.router.Report import gov.cdc.prime.router.ReportStreamFilter +import gov.cdc.prime.router.Topic import gov.cdc.prime.router.azure.BlobAccess import gov.cdc.prime.router.azure.ConditionStamper import gov.cdc.prime.router.azure.LookupTableConditionMapper +import gov.cdc.prime.router.azure.db.enums.TaskAction +import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile +import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService +import gov.cdc.prime.router.azure.observability.event.ItemEventData +import gov.cdc.prime.router.azure.observability.event.ReportEventData +import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName +import gov.cdc.prime.router.azure.observability.event.ReportStreamItemEventBuilder +import gov.cdc.prime.router.azure.observability.event.ReportStreamItemProcessingErrorEventBuilder +import gov.cdc.prime.router.azure.observability.event.ReportStreamReportEventBuilder +import gov.cdc.prime.router.azure.observability.event.ReportStreamReportProcessingErrorEventBuilder import gov.cdc.prime.router.cli.CommandUtilities.Companion.abort import gov.cdc.prime.router.cli.helpers.HL7DiffHelper import gov.cdc.prime.router.common.Environment @@ -161,96 +173,20 @@ class ProcessFhirCommands : CliktCommand( if (contents.isBlank()) throw CliktError("File ${inputFile.absolutePath} is empty.") // Check on the extension of the file for supported operations val inputFileType = inputFile.extension.uppercase() - val receiver = if (!isCli) { - getReceiver(environment, receiverName, orgName, GetMultipleSettings(), isCli) - } else { - null - } + val receiver = getReceiver(environment, receiverName, orgName, GetMultipleSettings(), isCli) - // If there is a receiver, check the filters - var bundle = FhirTranscoder.decode(contents) - if (receiver != null) { - val reportStreamFilters = mutableListOf>() - reportStreamFilters.add(Pair("Jurisdictional Filter", receiver.jurisdictionalFilter)) - reportStreamFilters.add(Pair("Quality Filter", receiver.qualityFilter)) - reportStreamFilters.add(Pair("Routing Filter", receiver.routingFilter)) - reportStreamFilters.add(Pair("Processing Mode Filter", receiver.processingModeFilter)) - - val validationErrors = mutableListOf() - reportStreamFilters.forEach { reportStreamFilter -> - reportStreamFilter.second.forEach { filter -> - val validation = OrganizationValidation.validateFilter(filter) - if (!validation) { - validationErrors.add( - "Filter of type ${reportStreamFilter.first} is not valid. " + - "Value: '$filter'" - ) - } else { - val result = FhirPathUtils.evaluate( - CustomContext( - bundle, - bundle, - mutableMapOf(), - CustomFhirPathFunctions() - ), - bundle, - bundle, - filter - ) - if (result.isEmpty() || - (result[0].isBooleanPrimitive && result[0].primitiveValue() == "false") - ) { - return MessageOrBundle( - filterErrors = - mutableListOf("Filter '$filter' filtered out everything, nothing to return."), - filtersPassed = false - ) - } - } - } - } - - if (validationErrors.isNotEmpty()) { - throw CliktError(validationErrors.joinToString("\n")) - } - - receiver.conditionFilter.forEach { conditionFilter -> - val validation = OrganizationValidation.validateFilter(conditionFilter) - if (!validation) { - return MessageOrBundle( - filterErrors = - mutableListOf("Condition filter '$conditionFilter' is not valid."), - filtersPassed = false - ) - } - } - } - - var messageOrBundle = MessageOrBundle() + val messageOrBundle = MessageOrBundle() when { // HL7 to FHIR conversion inputFileType == "HL7" && ( - (isCli && outputFormat == MimeFormat.FHIR.toString()) || + (outputFormat == MimeFormat.FHIR.toString()) || (receiver != null && receiver.format == MimeFormat.FHIR) ) -> { val fhirMessage = convertHl7ToFhir(contents, receiver).first - val enrichmentSchemaInfo = applyEnrichmentSchemas(fhirMessage, isCli) - setEnrichmentSchemaFields(messageOrBundle, enrichmentSchemaInfo) - - if (receiver != null && receiver.enrichmentSchemaNames.isNotEmpty()) { - receiver.enrichmentSchemaNames.forEach { currentSchema -> - val transfromer = FhirTransformer(currentSchema) - val returnedBundle = - transfromer.process(messageOrBundle.bundle!!) - setEnrichmentSchemaFields( - messageOrBundle, - transfromer.warnings, - transfromer.errors, - returnedBundle - ) - } - } - return handleSenderAndReceiverTransforms(messageOrBundle, senderSchema, isCli) + messageOrBundle.bundle = fhirMessage + handleSendAndReceiverFhirEnrichments(messageOrBundle, receiver, senderSchema, isCli) + + return messageOrBundle } // FHIR to HL7 conversion @@ -258,37 +194,17 @@ class ProcessFhirCommands : CliktCommand( (isCli && outputFormat == MimeFormat.HL7.toString()) || (receiver != null && (receiver.format == MimeFormat.HL7 || receiver.format == MimeFormat.HL7_BATCH)) ) -> { - if (receiver == null) { - return convertFhirToHl7( - jsonString = contents, - senderSchema = senderSchema, - isCli = isCli - ) - } - - bundle = FhirTranscoder.decode(contents) - messageOrBundle.bundle = bundle - if (receiver.enrichmentSchemaNames.isNotEmpty()) { - receiver.enrichmentSchemaNames.forEach { currentSchema -> - val transformer = FhirTransformer(currentSchema) - val returnedBundle = - transformer.process(bundle) - setEnrichmentSchemaFields( - messageOrBundle, - transformer.warnings, - transformer.errors, - returnedBundle - ) - } - } + messageOrBundle.bundle = FhirTranscoder.decode(contents) + handleSendAndReceiverFhirEnrichments(messageOrBundle, receiver, senderSchema, isCli) + + convertFhirToHl7( + (receiver?.translation ?: defaultHL7Configuration) as Hl7Configuration, + receiver, + isCli, + messageOrBundle + ) - messageOrBundle = convertFhirToHl7( - FhirTranscoder.encode(messageOrBundle.bundle!!), - receiver.translation as Hl7Configuration, - receiver, - senderSchema, - isCli - ) + return messageOrBundle } // FHIR to FHIR conversion @@ -296,7 +212,10 @@ class ProcessFhirCommands : CliktCommand( (isCli && outputFormat == MimeFormat.FHIR.toString()) || (receiver != null && receiver.format == MimeFormat.FHIR) ) -> { - return convertFhirToFhir(FhirTranscoder.encode(bundle), receiver, senderSchema, isCli) + messageOrBundle.bundle = FhirTranscoder.decode(contents) + handleSendAndReceiverFhirEnrichments(messageOrBundle, receiver, senderSchema, isCli) + + return messageOrBundle } // HL7 to FHIR to HL7 conversion @@ -308,47 +227,126 @@ class ProcessFhirCommands : CliktCommand( ) ) -> { val (bundle2, inputMessage) = convertHl7ToFhir(contents, receiver) - val output = convertFhirToHl7( - jsonString = FhirTranscoder.encode(bundle2), - senderSchema = senderSchema, - isCli = isCli + + messageOrBundle.bundle = bundle2 + handleSendAndReceiverFhirEnrichments(messageOrBundle, receiver, senderSchema, isCli) + + convertFhirToHl7( + (receiver?.translation ?: defaultHL7Configuration) as Hl7Configuration, + receiver, + isCli, + messageOrBundle ) if (diffHl7Output != null && isCli) { - val differences = hl7DiffHelper.diffHl7(output.message!!, inputMessage) + val differences = hl7DiffHelper.diffHl7(messageOrBundle.message!!, inputMessage) echo("-------diff output") echo("There were ${differences.size} differences between the input and output") differences.forEach { echo(it.toString()) } } - return output + return messageOrBundle } else -> throw CliktError("File extension ${inputFile.extension} is not supported.") } - return messageOrBundle } - private fun setEnrichmentSchemaFields( + private fun handleSendAndReceiverFhirEnrichments( messageOrBundle: MessageOrBundle, - enrichmentSchemaFields: FhirTransformer.BundleWithMessages, - ): MessageOrBundle { - messageOrBundle.enrichmentSchemaWarnings.addAll(enrichmentSchemaFields.warnings) - messageOrBundle.enrichmentSchemaErrors.addAll(enrichmentSchemaFields.errors) - messageOrBundle.enrichmentSchemaPassed = enrichmentSchemaFields.errors.isEmpty() - messageOrBundle.bundle = enrichmentSchemaFields.bundle - return messageOrBundle + receiver: Receiver?, + senderSchema: String?, + isCli: Boolean, + ) { + stampObservations(messageOrBundle, receiver) + + val senderSchemaName = when { + senderSchema != null -> senderSchema + senderSchemaParam != null -> senderSchemaParam + else -> null + } + + handleSenderTransforms(messageOrBundle, senderSchemaName) + + evaluateReceiverFilters(receiver, messageOrBundle, isCli) + + val receiverEnrichmentSchemaNames = when { + receiver != null && receiver.enrichmentSchemaNames.isNotEmpty() -> { + receiver.enrichmentSchemaNames.joinToString(",") + } + enrichmentSchemaNames != null -> enrichmentSchemaNames + else -> null + } + + handleReceiverFhirEnrichments(messageOrBundle, receiverEnrichmentSchemaNames) } - private fun setEnrichmentSchemaFields( - messageOrBundle: MessageOrBundle, - warnings: MutableList, - errors: MutableList, - bundle: Bundle, - ): MessageOrBundle { - messageOrBundle.enrichmentSchemaWarnings.addAll(warnings) - messageOrBundle.enrichmentSchemaErrors.addAll(errors) - messageOrBundle.enrichmentSchemaPassed = errors.isEmpty() - messageOrBundle.bundle = bundle - return messageOrBundle + fun handleReceiverFhirEnrichments(messageOrBundle: MessageOrBundle, schemaNames: String?) { + if (!schemaNames.isNullOrEmpty()) { + schemaNames.split(",").forEach { currentEnrichmentSchemaName -> + val transformer = FhirTransformer( + currentEnrichmentSchemaName, + errors = messageOrBundle.enrichmentSchemaErrors, + warnings = messageOrBundle.enrichmentSchemaWarnings + ) + val output = transformer.process( + messageOrBundle.bundle!! + ) + + messageOrBundle.bundle = output + } + messageOrBundle.enrichmentSchemaPassed = messageOrBundle.enrichmentSchemaErrors.isEmpty() + } + } + + private fun evaluateReceiverFilters(receiver: Receiver?, messageOrBundle: MessageOrBundle, isCli: Boolean) { + if (receiver != null && messageOrBundle.bundle != null) { + val reportStreamFilters = mutableListOf>() + reportStreamFilters.add(Pair("Jurisdictional Filter", receiver.jurisdictionalFilter)) + reportStreamFilters.add(Pair("Quality Filter", receiver.qualityFilter)) + reportStreamFilters.add(Pair("Routing Filter", receiver.routingFilter)) + reportStreamFilters.add(Pair("Processing Mode Filter", receiver.processingModeFilter)) + + reportStreamFilters.forEach { reportStreamFilter -> + reportStreamFilter.second.forEach { filter -> + val validation = OrganizationValidation.validateFilter(filter) + if (!validation) { + messageOrBundle.filterErrors.add( + "Filter of type ${reportStreamFilter.first} is not valid. " + + "Value: '$filter'" + ) + } else { + val result = FhirPathUtils.evaluate( + CustomContext( + messageOrBundle.bundle!!, + messageOrBundle.bundle!!, + mutableMapOf(), + CustomFhirPathFunctions() + ), + messageOrBundle.bundle!!, + messageOrBundle.bundle!!, + filter + ) + if (result.isEmpty() || + (result[0].isBooleanPrimitive && result[0].primitiveValue() == "false") + ) { + messageOrBundle.filterErrors.add( + "Filter '$filter' filtered out everything, nothing to return." + ) + } + } + } + } + + receiver.conditionFilter.forEach { conditionFilter -> + val validation = OrganizationValidation.validateFilter(conditionFilter) + if (!validation) { + messageOrBundle.filterErrors.add("Condition filter '$conditionFilter' is not valid.") + } + } + + if (isCli && messageOrBundle.filterErrors.isNotEmpty()) { + throw CliktError(messageOrBundle.filterErrors.joinToString("\n")) + } + } } abstract class MessageOrBundleParent( @@ -388,12 +386,16 @@ class ProcessFhirCommands : CliktCommand( // this is just for logging so it is fine to just make it up UUID.randomUUID().toString() } - val result = FHIRReceiverFilter().evaluateObservationConditionFilters( - receiver, - bundle, - ActionLogger(), - trackingId - ) + // TODO: https://github.com/CDCgov/prime-reportstream/issues/16407 + val result = + FHIRReceiverFilter( + reportStreamEventService = NoopReportStreamEventService() + ).evaluateObservationConditionFilters( + receiver, + bundle, + ActionLogger(), + trackingId + ) if (result is ReceiverFilterEvaluationResult.Success) { return result.bundle } else { @@ -458,147 +460,52 @@ class ProcessFhirCommands : CliktCommand( * @return an HL7 message */ private fun convertFhirToHl7( - jsonString: String, hl7Configuration: Hl7Configuration = defaultHL7Configuration, receiver: Receiver? = null, - senderSchema: String?, isCli: Boolean, - ): MessageOrBundle { - val fhirMessage = FhirTranscoder.decode(jsonString) - val enrichmentSchemaMessages = applyEnrichmentSchemas(fhirMessage, isCli) - val errors: MutableList = mutableListOf() - val warnings: MutableList = mutableListOf() - return when { - (isCli && receiverSchema == null) && (receiver == null || (isCli && receiver.schemaName.isBlank())) -> - // Receiver schema required because if it's coming out as HL7, it would be getting any transform info - // for that from a receiver schema. - throw CliktError("You must specify a receiver schema using --receiver-schema.") - - isCli && receiverSchema != null -> { - val senderTransformMessages = applySenderTransforms(enrichmentSchemaMessages.bundle, senderSchema) - val stamper = ConditionStamper(LookupTableConditionMapper(Metadata.getInstance())) - senderTransformMessages.bundle.getObservations().forEach { observation -> - stamper.stampObservation(observation) - } - if (receiver != null) { - senderTransformMessages.bundle = applyConditionFilter(receiver, senderTransformMessages.bundle) - } + messageOrBundle: MessageOrBundle, + ) { + if ((isCli && receiverSchema == null) && (receiver == null || receiver.schemaName.isBlank())) { + throw CliktError("You must specify a receiver schema using --receiver-schema.") + } - val message = FhirToHl7Converter( - receiverSchema!!, - BlobAccess.BlobContainerMetadata.build("metadata", Environment.get().storageEnvVar), - context = FhirToHl7Context( - CustomFhirPathFunctions(), - config = HL7TranslationConfig( - hl7Configuration, - receiver - ), - translationFunctions = CustomTranslationFunctions(), - ), - warnings = warnings, - errors = errors - ).process(senderTransformMessages.bundle) - val messageOrBundle = MessageOrBundle() - messageOrBundle.senderTransformPassed = senderTransformMessages.errors.isEmpty() - messageOrBundle.senderTransformWarnings = senderTransformMessages.warnings - messageOrBundle.senderTransformErrors = senderTransformMessages.errors - messageOrBundle.receiverTransformPassed = errors.isEmpty() - messageOrBundle.receiverTransformErrors = errors - messageOrBundle.receiverTransformWarnings = warnings - messageOrBundle.message = message - messageOrBundle - } - receiver != null && receiver.schemaName.isNotBlank() -> { - val senderTransformMessages = applySenderTransforms(fhirMessage, senderSchema) - val bundle = applyConditionFilter(receiver, senderTransformMessages.bundle) - val message = FhirToHl7Converter( - receiver.schemaName, - BlobAccess.BlobContainerMetadata.build("metadata", Environment.get().storageEnvVar), - context = FhirToHl7Context( - CustomFhirPathFunctions(), - config = HL7TranslationConfig( - hl7Configuration, - receiver - ), - translationFunctions = CustomTranslationFunctions(), + val receiverTransformSchemaName = when { + receiver != null && receiver.schemaName.isNotEmpty() -> receiver.enrichmentSchemaNames.joinToString(",") + receiverSchema != null -> receiverSchema + else -> null + } + + if (receiverTransformSchemaName != null) { + val message = FhirToHl7Converter( + receiverSchema!!, + BlobAccess.BlobContainerMetadata.build("metadata", Environment.get().storageEnvVar), + context = FhirToHl7Context( + CustomFhirPathFunctions(), + config = HL7TranslationConfig( + hl7Configuration, + receiver ), - warnings = warnings, - errors = errors - ).process(bundle) - val messageOrBundle = MessageOrBundle() - messageOrBundle.senderTransformPassed = senderTransformMessages.errors.isEmpty() - messageOrBundle.senderTransformWarnings = senderTransformMessages.warnings - messageOrBundle.senderTransformErrors = senderTransformMessages.errors - messageOrBundle.receiverTransformPassed = errors.isEmpty() - messageOrBundle.receiverTransformErrors = errors - messageOrBundle.receiverTransformWarnings = warnings - messageOrBundle.message = message - messageOrBundle - } - else -> { - if (isCli) { - throw CliktError("Error state reached when trying to apply the transforms.") - } else { - MessageOrBundle( - senderTransformErrors = - mutableListOf("Error state reached when trying to apply the transforms."), - receiverTransformErrors = mutableListOf( - "Error state reached when trying to apply the transforms." - ) - ) - } - } + translationFunctions = CustomTranslationFunctions(), + ), + warnings = messageOrBundle.receiverTransformWarnings, + errors = messageOrBundle.receiverTransformErrors + ).process(messageOrBundle.bundle!!) + messageOrBundle.message = message + messageOrBundle.receiverTransformPassed = messageOrBundle.receiverTransformErrors.isEmpty() } } - /** - * convert an FHIR message to FHIR message - */ - private fun convertFhirToFhir( - jsonString: String, + private fun stampObservations( + messageOrBundle: MessageOrBundle, receiver: Receiver?, - senderSchema: String?, - isCli: Boolean, - ): MessageOrBundle { - var fhirMessage = FhirTranscoder.decode(jsonString) + ) { val stamper = ConditionStamper(LookupTableConditionMapper(Metadata.getInstance())) - fhirMessage.getObservations().forEach { observation -> + messageOrBundle.bundle?.getObservations()?.forEach { observation -> stamper.stampObservation(observation) } - - val messageOrBundle = MessageOrBundle() if (receiver != null) { - fhirMessage = applyConditionFilter(receiver, fhirMessage) - if (receiver.enrichmentSchemaNames.isNotEmpty()) { - receiver.enrichmentSchemaNames.forEach { currentSchema -> - val transformer = FhirTransformer(currentSchema) - val bundle = transformer.process(fhirMessage) - setEnrichmentSchemaFields( - messageOrBundle, - transformer.warnings, - transformer.errors, - bundle - ) - } - } - } - setEnrichmentSchemaFields(messageOrBundle, applyEnrichmentSchemas(fhirMessage, isCli)) - if (( - (isCli && receiverSchema == null) || - (!isCli && (receiver == null || receiver.schemaName.isBlank())) - ) && senderSchema == null - ) { - // Must have at least one schema or else why are you doing this - throw CliktError("You must specify a schema.") - } else { - handleSenderAndReceiverTransforms( - messageOrBundle = messageOrBundle, - senderSchema = senderSchema, - isCli = isCli - ) + messageOrBundle.bundle = applyConditionFilter(receiver, messageOrBundle.bundle!!) } - - return messageOrBundle } /** @@ -650,82 +557,16 @@ class ProcessFhirCommands : CliktCommand( * @throws CliktError if senderSchema is present, but unable to be read. * @return If senderSchema is present, apply it, otherwise just return the input bundle. */ - private fun applySenderTransforms(bundle: Bundle, senderSchema: String?): FhirTransformer.BundleWithMessages { - return when { - senderSchema != null -> { - val transformer = FhirTransformer(senderSchema) - val returnedBundle = transformer.process(bundle) - FhirTransformer.BundleWithMessages(returnedBundle, transformer.warnings, transformer.errors) - } - - else -> FhirTransformer.BundleWithMessages(bundle = bundle, mutableListOf(), mutableListOf()) - } - } - - /** - * @throws CliktError if receiverSchema is present, but unable to be read. - * @throws CliktError if enrichmentSchemaName is present, but unable to be read. - * @return If receiverSchema is present, apply it, otherwise just return the input bundle. - */ - private fun applyReceiverEnrichmentAndTransforms(bundle: Bundle, isCli: Boolean): MessageOrBundle { - val messageOrBundle = MessageOrBundle() - setEnrichmentSchemaFields(messageOrBundle, applyEnrichmentSchemas(bundle, isCli)) - - if (isCli && receiverSchema != null) { - val transformer = FhirTransformer(receiverSchema!!) + private fun handleSenderTransforms(messageOrBundle: MessageOrBundle, senderSchema: String?) { + if (senderSchema != null) { + val transformer = FhirTransformer( + senderSchema, + errors = messageOrBundle.senderTransformErrors, + warnings = messageOrBundle.senderTransformWarnings + ) val returnedBundle = transformer.process(messageOrBundle.bundle!!) - messageOrBundle.receiverTransformWarnings.addAll(transformer.warnings) - messageOrBundle.receiverTransformErrors.addAll(transformer.errors) - messageOrBundle.receiverTransformPassed = transformer.errors.isEmpty() messageOrBundle.bundle = returnedBundle } - - return messageOrBundle - } - - /** - * Applies the enrichment schema to the bundle. - */ - private fun applyEnrichmentSchemas(bundle: Bundle, isCli: Boolean): FhirTransformer.BundleWithMessages { - var enrichedbundle = bundle - val warnings = mutableListOf() - val errors = mutableListOf() - if (isCli && !enrichmentSchemaNames.isNullOrEmpty()) { - enrichmentSchemaNames!!.split(",").forEach { currentEnrichmentSchemaName -> - val transformer = FhirTransformer(currentEnrichmentSchemaName) - val returnedBundle = transformer.process( - enrichedbundle - ) - errors.addAll(transformer.errors) - warnings.addAll(transformer.warnings) - enrichedbundle = returnedBundle - } - } - return FhirTransformer.BundleWithMessages(enrichedbundle, warnings, errors) - } - - /** - * Apply both sender and receiver schemas if present. - * @return the FHIR bundle after having sender and/or receiver schemas applied to it. - */ - private fun handleSenderAndReceiverTransforms( - messageOrBundle: MessageOrBundle, - senderSchema: String?, - isCli: Boolean, - ): MessageOrBundle { - val senderTransformInfo = applySenderTransforms(messageOrBundle.bundle!!, senderSchema) - val receiverTransformInfo = applyReceiverEnrichmentAndTransforms(senderTransformInfo.bundle, isCli) - messageOrBundle.bundle = receiverTransformInfo.bundle - messageOrBundle.senderTransformWarnings.addAll(senderTransformInfo.warnings) - messageOrBundle.senderTransformErrors.addAll(senderTransformInfo.errors) - messageOrBundle.senderTransformPassed = senderTransformInfo.errors.isEmpty() - messageOrBundle.receiverTransformErrors.addAll(receiverTransformInfo.receiverTransformErrors) - messageOrBundle.receiverTransformWarnings.addAll(receiverTransformInfo.receiverTransformWarnings) - messageOrBundle.receiverTransformPassed = receiverTransformInfo.receiverTransformPassed && - messageOrBundle.receiverTransformPassed - messageOrBundle.enrichmentSchemaPassed - - return messageOrBundle } /** @@ -1023,4 +864,116 @@ class FhirPathCommand : CliktCommand( stringValue.append("\n}\n") return stringValue.toString() } +} + +// This exists only because ProcessFhirCommands instantiates a FHIRReceiverFilter to access a function that likely could be +// made static +// TODO: https://github.com/CDCgov/prime-reportstream/issues/16407 +class NoopReportStreamEventService : IReportStreamEventService { + override fun sendQueuedEvents() { + throw NotImplementedError() + } + + override fun sendReportEvent( + eventName: ReportStreamEventName, + childReport: Report, + pipelineStepName: TaskAction, + shouldQueue: Boolean, + initializer: ReportStreamReportEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun sendReportEvent( + eventName: ReportStreamEventName, + childReport: ReportFile, + pipelineStepName: TaskAction, + shouldQueue: Boolean, + initializer: ReportStreamReportEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun sendReportProcessingError( + eventName: ReportStreamEventName, + childReport: ReportFile, + pipelineStepName: TaskAction, + error: String, + shouldQueue: Boolean, + initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun sendReportProcessingError( + eventName: ReportStreamEventName, + childReport: Report, + pipelineStepName: TaskAction, + error: String, + shouldQueue: Boolean, + initializer: ReportStreamReportProcessingErrorEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun sendItemEvent( + eventName: ReportStreamEventName, + childReport: Report, + pipelineStepName: TaskAction, + shouldQueue: Boolean, + initializer: ReportStreamItemEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun sendItemEvent( + eventName: ReportStreamEventName, + childReport: ReportFile, + pipelineStepName: TaskAction, + shouldQueue: Boolean, + initializer: ReportStreamItemEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun sendItemProcessingError( + eventName: ReportStreamEventName, + childReport: ReportFile, + pipelineStepName: TaskAction, + error: String, + shouldQueue: Boolean, + initializer: ReportStreamItemProcessingErrorEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun sendItemProcessingError( + eventName: ReportStreamEventName, + childReport: Report, + pipelineStepName: TaskAction, + error: String, + shouldQueue: Boolean, + initializer: ReportStreamItemProcessingErrorEventBuilder.() -> Unit, + ) { + throw NotImplementedError() + } + + override fun getReportEventData( + childReportId: UUID, + childBodyUrl: String, + parentReportId: UUID?, + pipelineStepName: TaskAction, + topic: Topic?, + ): ReportEventData { + throw NotImplementedError() + } + + override fun getItemEventData( + childItemIndex: Int, + parentReportId: UUID, + parentItemIndex: Int, + trackingId: String?, + ): ItemEventData { + throw NotImplementedError() + } } \ No newline at end of file diff --git a/prime-router/src/main/kotlin/fhirengine/azure/FHIRFunctions.kt b/prime-router/src/main/kotlin/fhirengine/azure/FHIRFunctions.kt index 0f7a31b560e..cd3e59e2d3d 100644 --- a/prime-router/src/main/kotlin/fhirengine/azure/FHIRFunctions.kt +++ b/prime-router/src/main/kotlin/fhirengine/azure/FHIRFunctions.kt @@ -5,25 +5,32 @@ import com.microsoft.azure.functions.annotation.FunctionName import com.microsoft.azure.functions.annotation.QueueTrigger import com.microsoft.azure.functions.annotation.StorageAccount import gov.cdc.prime.reportstream.shared.QueueMessage +import gov.cdc.prime.reportstream.shared.Submission import gov.cdc.prime.router.ActionLogger import gov.cdc.prime.router.azure.ActionHistory import gov.cdc.prime.router.azure.DataAccessTransaction import gov.cdc.prime.router.azure.DatabaseAccess import gov.cdc.prime.router.azure.QueueAccess +import gov.cdc.prime.router.azure.SubmissionTableService import gov.cdc.prime.router.azure.WorkflowEngine import gov.cdc.prime.router.azure.db.enums.TaskAction +import gov.cdc.prime.router.azure.observability.event.AzureEventService +import gov.cdc.prime.router.azure.observability.event.AzureEventServiceImpl import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties +import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService import gov.cdc.prime.router.common.BaseEngine import gov.cdc.prime.router.fhirengine.engine.FHIRConverter import gov.cdc.prime.router.fhirengine.engine.FHIRDestinationFilter import gov.cdc.prime.router.fhirengine.engine.FHIREngine -import gov.cdc.prime.router.fhirengine.engine.FHIRReceiver import gov.cdc.prime.router.fhirengine.engine.FHIRReceiverFilter import gov.cdc.prime.router.fhirengine.engine.FHIRTranslator -import gov.cdc.prime.router.fhirengine.engine.FhirReceiveQueueMessage +import gov.cdc.prime.router.fhirengine.engine.FhirConvertSubmissionQueueMessage import gov.cdc.prime.router.fhirengine.engine.PrimeRouterQueueMessage import gov.cdc.prime.router.fhirengine.engine.ReportPipelineMessage +import gov.cdc.prime.router.fhirengine.engine.SubmissionSenderNotFound +import gov.cdc.prime.router.history.db.ReportGraph +import gov.cdc.prime.router.report.ReportService import org.apache.commons.lang3.StringUtils import org.apache.logging.log4j.kotlin.Logging import org.jooq.exception.DataAccessException @@ -33,23 +40,41 @@ class FHIRFunctions( private val actionLogger: ActionLogger = ActionLogger(), private val databaseAccess: DatabaseAccess = BaseEngine.databaseAccessSingleton, private val queueAccess: QueueAccess = QueueAccess, + private val submissionTableService: SubmissionTableService = SubmissionTableService.getInstance(), + val reportService: ReportService = ReportService(ReportGraph(databaseAccess), databaseAccess), + val azureEventService: AzureEventService = AzureEventServiceImpl(), + val reportStreamEventService: ReportStreamEventService = + ReportStreamEventService(databaseAccess, azureEventService, reportService), ) : Logging { /** * An azure function for ingesting and recording submissions */ - @FunctionName("receive-fhir") + @FunctionName("convert-from-submissions-fhir") @StorageAccount("AzureWebJobsStorage") - fun receive( - @QueueTrigger(name = "message", queueName = QueueMessage.elrReceiveQueueName) + fun convertFromSubmissions( + @QueueTrigger(name = "message", queueName = QueueMessage.elrSubmissionConvertQueueName) message: String, // Number of times this message has been dequeued @BindingName("DequeueCount") dequeueCount: Int = 1, ) { logger.info( - "message consumed from elr-fhir-receive queue" + "message consumed from ${QueueMessage.elrSubmissionConvertQueueName} queue" ) - process(message, dequeueCount, FHIRReceiver(), ActionHistory(TaskAction.receive)) + process( + message, + dequeueCount, + FHIRConverter(reportStreamEventService = reportStreamEventService), + ActionHistory(TaskAction.convert) + ) + val messageContent = readMessage("convert", message, dequeueCount) + val tableEntity = Submission( + messageContent.reportId.toString(), + "Accepted", + messageContent.blobURL, + actionLogger.errors.takeIf { it.isNotEmpty() }?.map { it.detail.message }?.toString() + ) + submissionTableService.insertSubmission(tableEntity) } /** @@ -63,7 +88,12 @@ class FHIRFunctions( // Number of times this message has been dequeued @BindingName("DequeueCount") dequeueCount: Int = 1, ) { - process(message, dequeueCount, FHIRConverter(), ActionHistory(TaskAction.convert)) + process( + message, + dequeueCount, + FHIRConverter(reportStreamEventService = reportStreamEventService), + ActionHistory(TaskAction.convert) + ) } /** @@ -77,7 +107,12 @@ class FHIRFunctions( // Number of times this message has been dequeued @BindingName("DequeueCount") dequeueCount: Int = 1, ) { - process(message, dequeueCount, FHIRDestinationFilter(), ActionHistory(TaskAction.destination_filter)) + process( + message, + dequeueCount, + FHIRDestinationFilter(reportStreamEventService = reportStreamEventService), + ActionHistory(TaskAction.destination_filter) + ) } /** @@ -91,7 +126,12 @@ class FHIRFunctions( // Number of times this message has been dequeued @BindingName("DequeueCount") dequeueCount: Int = 1, ) { - process(message, dequeueCount, FHIRReceiverFilter(), ActionHistory(TaskAction.receiver_filter)) + process( + message, + dequeueCount, + FHIRReceiverFilter(reportStreamEventService = reportStreamEventService), + ActionHistory(TaskAction.receiver_filter) + ) } /** @@ -105,7 +145,12 @@ class FHIRFunctions( // Number of times this message has been dequeued @BindingName("DequeueCount") dequeueCount: Int = 1, ) { - process(message, dequeueCount, FHIRTranslator(), ActionHistory(TaskAction.translate)) + process( + message, + dequeueCount, + FHIRTranslator(reportStreamEventService = reportStreamEventService), + ActionHistory(TaskAction.translate) + ) } /** @@ -149,13 +194,27 @@ class FHIRFunctions( recordResults(message, actionHistory, txn) results } - + reportStreamEventService.sendQueuedEvents() return newMessages } catch (ex: DataAccessException) { // This is the one exception type that we currently will allow for retrying as there are occasional // DB connectivity issues that are resolved without intervention logger.error(ex) throw ex + } catch (ex: SubmissionSenderNotFound) { + // This is a specific error case that can occur while handling a report via the new Submission service + // In a situation that the sender is not found there is not enough information to record a report event + // and we want a poison queue message to be immediately added so that the configuration can be fixed + logger.error(ex) + val tableEntity = Submission( + ex.reportId.toString(), + "Rejected", + ex.blobURL, + actionLogger.errors.takeIf { it.isNotEmpty() }?.map { it.detail.message }?.toString() + ) + submissionTableService.insertSubmission(tableEntity) + queueAccess.sendMessage("${messageContent.messageQueueName}-poison", message) + return emptyList() } catch (ex: Exception) { // We're catching anything else that occurs because the most likely cause is a code or configuration error // that will not be resolved if the message is automatically retried @@ -186,7 +245,7 @@ class FHIRFunctions( return when (val queueMessage = QueueMessage.deserialize(message)) { is QueueMessage.ReceiveQueueMessage -> { - FhirReceiveQueueMessage( + FhirConvertSubmissionQueueMessage( queueMessage.reportId, queueMessage.blobURL, queueMessage.digest, diff --git a/prime-router/src/main/kotlin/fhirengine/engine/CustomFhirPathFunctions.kt b/prime-router/src/main/kotlin/fhirengine/engine/CustomFhirPathFunctions.kt index 0083feea221..b0e19c12982 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/CustomFhirPathFunctions.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/CustomFhirPathFunctions.kt @@ -36,6 +36,7 @@ class CustomFhirPathFunctions : FhirPathFunctions { LivdTableLookup, GetFakeValueForElement, FIPSCountyLookup, + GetStateFromZipCode, ; companion object { @@ -84,6 +85,14 @@ class CustomFhirPathFunctions : FhirPathFunctions { ) } + CustomFhirPathFunctionNames.GetStateFromZipCode -> { + FunctionDetails( + "Looks up the states that match the given zip code", + 0, + 0 + ) + } + else -> null } } @@ -110,6 +119,9 @@ class CustomFhirPathFunctions : FhirPathFunctions { CustomFhirPathFunctionNames.FIPSCountyLookup -> { fipsCountyLookup(parameters) } + CustomFhirPathFunctionNames.GetStateFromZipCode -> { + getStateFromZipCode(focus) + } else -> error(IllegalStateException("Tried to execute invalid FHIR Path function $functionName")) } ) @@ -353,4 +365,24 @@ class CustomFhirPathFunctions : FhirPathFunctions { mutableListOf(StringType(parameters.first().first().primitiveValue())) } } + + /** + * Returns a comma-separated string of the states that + * match the zip code stored in the [focus] element. + * @return a mutable list containing the state abbreviations + */ + fun getStateFromZipCode( + focus: MutableList, + metadata: Metadata = Metadata.getInstance(), + ): MutableList { + val lookupTable = metadata.findLookupTable("zip-code-data") + var filters = lookupTable?.FilterBuilder() ?: error("Could not find table zip-code-data") + + val zipCode = focus[0].primitiveValue() + filters = filters.isEqualTo("zipcode", zipCode) + val result = filters.findAllUnique("state_abbr") + + val stateList = result.joinToString(",") + return mutableListOf(StringType(stateList)) + } } \ No newline at end of file diff --git a/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt b/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt index 9527ef4dc19..e5cd8908412 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt @@ -16,6 +16,7 @@ import gov.cdc.prime.reportstream.shared.QueueMessage import gov.cdc.prime.router.ActionLogDetail import gov.cdc.prime.router.ActionLogScope import gov.cdc.prime.router.ActionLogger +import gov.cdc.prime.router.ClientSource import gov.cdc.prime.router.ErrorCode import gov.cdc.prime.router.InvalidReportMessage import gov.cdc.prime.router.Metadata @@ -23,6 +24,7 @@ import gov.cdc.prime.router.MimeFormat import gov.cdc.prime.router.Options import gov.cdc.prime.router.Report import gov.cdc.prime.router.SettingsProvider +import gov.cdc.prime.router.Topic import gov.cdc.prime.router.UnmappableConditionMessage import gov.cdc.prime.router.azure.ActionHistory import gov.cdc.prime.router.azure.BlobAccess @@ -39,6 +41,7 @@ import gov.cdc.prime.router.azure.observability.context.MDCUtils import gov.cdc.prime.router.azure.observability.context.withLoggingContext import gov.cdc.prime.router.azure.observability.event.AzureEventService import gov.cdc.prime.router.azure.observability.event.AzureEventServiceImpl +import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties import gov.cdc.prime.router.fhirengine.translation.HL7toFhirTranslator @@ -57,6 +60,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils import org.hl7.fhir.r4.model.Bundle import org.jooq.Field import java.time.OffsetDateTime +import java.util.UUID import java.util.stream.Collectors import java.util.stream.Stream @@ -75,7 +79,8 @@ class FHIRConverter( blob: BlobAccess = BlobAccess(), azureEventService: AzureEventService = AzureEventServiceImpl(), reportService: ReportService = ReportService(), -) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService) { + reportStreamEventService: IReportStreamEventService, +) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService, reportStreamEventService) { override val finishedField: Field = Tables.TASK.PROCESSED_AT @@ -83,6 +88,112 @@ class FHIRConverter( override val taskAction: TaskAction = TaskAction.convert + /** + * This object serves the purpose of consolidating the information needed to process a report + * through the convert step regardless of whether it comes from a [FhirConvertQueueMessage] + * or [FhirConvertSubmissionQueueMessage] + * + * @param reportId the report ID + * @param topic the topic the sender published to + * @param schemaName the FHIR transform to apply + * @param blobURL the URL for the blob to convert + * @param blobDigest the digest of the blob contents + */ + data class FHIRConvertInput( + val reportId: UUID, + val topic: Topic, + val schemaName: String, + val blobURL: String, + val blobDigest: String, + val blobSubFolderName: String, + ) { + + companion object { + + private val clientIdHeader = "client_id" + + /** + * Converts a [FhirConvertQueueMessage] into the input to the convert processing + * + * @param message the queue message + * @param actionHistory action history for recording details on the input report + */ + fun fromFhirConvertQueueMessage( + message: FhirConvertQueueMessage, + actionHistory: ActionHistory, + ): FHIRConvertInput { + val reportId = message.reportId + val topic = message.topic + val schemaName = message.schemaName + val blobUrl = message.blobURL + val blobDigest = message.digest + val blobSubFolderName = message.blobSubFolderName + actionHistory.trackExistingInputReport(reportId) + return FHIRConvertInput( + reportId, + topic, + schemaName, + blobUrl, + blobDigest, + blobSubFolderName + ) + } + + /** + * Converts a [FhirConvertSubmissionQueueMessage] into the input to the convert processing + * + * @param message the queue message + * @param actionHistory action history for recording details on the input report + * @param settings [SettingsProvider] for looking up the sender + */ + fun fromFHIRConvertSubmissionQueueMessage( + message: FhirConvertSubmissionQueueMessage, + actionHistory: ActionHistory, + settings: SettingsProvider, + ): FHIRConvertInput { + val reportId = message.reportId + val blobUrl = message.blobURL + val blobDigest = message.digest + val blobSubFolderName = message.blobSubFolderName + + val clientId = message.headers[clientIdHeader] + val sender = clientId?.takeIf { it.isNotBlank() }?.let { settings.findSender(it) } + if (sender == null) { + throw SubmissionSenderNotFound(clientId ?: "", reportId, blobUrl) + } + val topic = sender.topic + val schemaName = sender.schemaName + + val format = Report.getFormatFromBlobURL(blobUrl) + val report = Report( + sender.format, + listOf(ClientSource(organization = sender.organizationName, client = sender.name)), + 1, + nextAction = TaskAction.convert, + topic = sender.topic, + id = reportId, + bodyURL = blobUrl + ) + // This tracking is required so that the external report (coming from the submission service) + // is properly recorded in the report file table with the correct sender + actionHistory.trackExternalInputReport( + report, + BlobAccess.BlobInfo(format, blobUrl, blobDigest.toByteArray()) + ) + actionHistory.trackActionSenderInfo(sender.fullName) + + return FHIRConvertInput( + reportId, + topic, + schemaName, + blobUrl, + blobDigest, + blobSubFolderName + ) + } + } + } + /** * Accepts a [message] in either HL7 or FHIR format * HL7 messages will be converted into FHIR. @@ -97,7 +208,21 @@ class FHIRConverter( actionHistory: ActionHistory, ): List = when (message) { is FhirConvertQueueMessage -> { - fhirEngineRunResults(message, message.schemaName, actionLogger, actionHistory) + val input = FHIRConvertInput.fromFhirConvertQueueMessage(message, actionHistory) + + fhirEngineRunResults( + input, + actionLogger, + actionHistory + ) + } + is FhirConvertSubmissionQueueMessage -> { + val input = FHIRConvertInput.fromFHIRConvertSubmissionQueueMessage(message, actionHistory, settings) + fhirEngineRunResults( + input, + actionLogger, + actionHistory + ) } else -> { throw RuntimeException( @@ -107,21 +232,19 @@ class FHIRConverter( } private fun fhirEngineRunResults( - queueMessage: FhirConvertQueueMessage, - schemaName: String, + input: FHIRConvertInput, actionLogger: ActionLogger, actionHistory: ActionHistory, ): List { val contextMap = mapOf( MDCUtils.MDCProperty.ACTION_NAME to actionHistory.action.actionName.name, - MDCUtils.MDCProperty.REPORT_ID to queueMessage.reportId, - MDCUtils.MDCProperty.TOPIC to queueMessage.topic, - MDCUtils.MDCProperty.BLOB_URL to queueMessage.blobURL + MDCUtils.MDCProperty.REPORT_ID to input.reportId, + MDCUtils.MDCProperty.TOPIC to input.topic, + MDCUtils.MDCProperty.BLOB_URL to input.blobURL ) withLoggingContext(contextMap) { - actionLogger.setReportId(queueMessage.reportId) - actionHistory.trackExistingInputReport(queueMessage.reportId) - val format = Report.getFormatFromBlobURL(queueMessage.blobURL) + actionLogger.setReportId(input.reportId) + val format = Report.getFormatFromBlobURL(input.blobURL) logger.info("Starting FHIR Convert step") // This line is a workaround for a defect in the hapi-fhir library @@ -135,7 +258,7 @@ class FHIRConverter( // TODO: https://github.com/CDCgov/prime-reportstream/issues/14287 FhirPathUtils - val processedItems = process(format, queueMessage, actionLogger) + val processedItems = process(format, input.blobURL, input.blobDigest, input.topic, actionLogger) // processedItems can be empty in three scenarios: // - the blob had no contents, i.e. an empty file was submitted @@ -146,7 +269,7 @@ class FHIRConverter( "Applied sender transform and routed" ) { val transformer = getTransformerFromSchema( - schemaName + input.schemaName ) maybeParallelize( @@ -162,10 +285,10 @@ class FHIRConverter( MimeFormat.FHIR, emptyList(), parentItemLineageData = listOf( - Report.ParentItemLineageData(queueMessage.reportId, itemIndex.toInt() + 1) + Report.ParentItemLineageData(input.reportId, itemIndex.toInt() + 1) ), metadata = this.metadata, - topic = queueMessage.topic, + topic = input.topic, nextAction = TaskAction.none ) val noneEvent = ProcessEvent( @@ -182,14 +305,15 @@ class FHIRConverter( report, TaskAction.convert, processedItem.validationError!!.message, + shouldQueue = true ) { - parentReportId(queueMessage.reportId) + parentReportId(input.reportId) parentItemIndex(itemIndex.toInt() + 1) params( mapOf( ReportStreamEventProperties.ITEM_FORMAT to format, ReportStreamEventProperties.VALIDATION_PROFILE - to queueMessage.topic.validator.validatorProfileName + to input.topic.validator.validatorProfileName ) ) } @@ -205,10 +329,10 @@ class FHIRConverter( MimeFormat.FHIR, emptyList(), parentItemLineageData = listOf( - Report.ParentItemLineageData(queueMessage.reportId, itemIndex.toInt() + 1) + Report.ParentItemLineageData(input.reportId, itemIndex.toInt() + 1) ), metadata = this.metadata, - topic = queueMessage.topic, + topic = input.topic, nextAction = TaskAction.destination_filter ) @@ -227,7 +351,7 @@ class FHIRConverter( MimeFormat.FHIR, bodyBytes, report.id.toString(), - queueMessage.blobSubFolderName, + input.blobSubFolderName, routeEvent.eventAction ) report.bodyURL = blobInfo.blobUrl @@ -248,9 +372,10 @@ class FHIRConverter( reportEventService.sendItemEvent( ReportStreamEventName.ITEM_ACCEPTED, report, - TaskAction.convert + TaskAction.convert, + shouldQueue = true ) { - parentReportId(queueMessage.reportId) + parentReportId(input.reportId) parentItemIndex(itemIndex.toInt() + 1) trackingId(bundle) params( @@ -270,8 +395,8 @@ class FHIRConverter( report.id, blobInfo.blobUrl, BlobUtils.digestToString(blobInfo.digest), - queueMessage.blobSubFolderName, - queueMessage.topic + input.blobSubFolderName, + input.topic ) ) } @@ -283,7 +408,7 @@ class FHIRConverter( emptyList(), 0, metadata = this.metadata, - topic = queueMessage.topic, + topic = input.topic, nextAction = TaskAction.none ) actionHistory.trackEmptyReport(report) @@ -293,7 +418,7 @@ class FHIRConverter( TaskAction.convert, "Submitted report was either empty or could not be parsed into HL7" ) { - parentReportId(queueMessage.reportId) + parentReportId(input.reportId) params( mapOf( ReportStreamEventProperties.ITEM_FORMAT to format @@ -325,12 +450,14 @@ class FHIRConverter( */ internal fun process( format: MimeFormat, - queueMessage: FhirConvertQueueMessage, + blobURL: String, + blobDigest: String, + topic: Topic, actionLogger: ActionLogger, routeReportWithInvalidItems: Boolean = true, ): List> { - val validator = queueMessage.topic.validator - val rawReport = BlobAccess.downloadBlob(queueMessage.blobURL, queueMessage.digest) + val validator = topic.validator + val rawReport = BlobAccess.downloadBlob(blobURL, blobDigest) return if (rawReport.isBlank()) { actionLogger.error(InvalidReportMessage("Provided raw data is empty.")) emptyList() @@ -344,7 +471,7 @@ class FHIRConverter( "format" to format.name ) ) { - getBundlesFromRawHL7(rawReport, validator, queueMessage.topic.hl7ParseConfiguration) + getBundlesFromRawHL7(rawReport, validator, topic.hl7ParseConfiguration) } } catch (ex: ParseFailureError) { actionLogger.error( @@ -628,4 +755,15 @@ class FHIRConverter( } else { null } +} + +/** + * Exception generated if the sender ID from a message generated by Submissions service cannot be found + * + * @param senderId the full name of the missing sender, will be empty string if no sender was on the message + * @param reportId the unique identifier for the report which can be located in the azure table + * @param blobURL the blob URL for the report + */ +class SubmissionSenderNotFound(senderId: String, val reportId: UUID, val blobURL: String) : RuntimeException() { + override val message = "No sender was found for: $senderId" } \ No newline at end of file diff --git a/prime-router/src/main/kotlin/fhirengine/engine/FHIRDestinationFilter.kt b/prime-router/src/main/kotlin/fhirengine/engine/FHIRDestinationFilter.kt index 9f92a2b85b4..5be29b4626e 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/FHIRDestinationFilter.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/FHIRDestinationFilter.kt @@ -26,6 +26,7 @@ import gov.cdc.prime.router.azure.observability.context.MDCUtils import gov.cdc.prime.router.azure.observability.context.withLoggingContext import gov.cdc.prime.router.azure.observability.event.AzureEventService import gov.cdc.prime.router.azure.observability.event.AzureEventServiceImpl +import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties import gov.cdc.prime.router.fhirengine.translation.hl7.utils.CustomContext @@ -50,7 +51,8 @@ class FHIRDestinationFilter( blob: BlobAccess = BlobAccess(), azureEventService: AzureEventService = AzureEventServiceImpl(), reportService: ReportService = ReportService(), -) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService) { + reportStreamEventService: IReportStreamEventService, +) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService, reportStreamEventService) { override val finishedField: Field = Tables.TASK.DESTINATION_FILTERED_AT override val engineType: String = "DestinationFilter" diff --git a/prime-router/src/main/kotlin/fhirengine/engine/FHIREngine.kt b/prime-router/src/main/kotlin/fhirengine/engine/FHIREngine.kt index dd0d051c889..d32ad4269c7 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/FHIREngine.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/FHIREngine.kt @@ -11,6 +11,7 @@ import gov.cdc.prime.router.azure.BlobAccess import gov.cdc.prime.router.azure.DataAccessTransaction import gov.cdc.prime.router.azure.DatabaseAccess import gov.cdc.prime.router.azure.Event +import gov.cdc.prime.router.azure.SubmissionTableService import gov.cdc.prime.router.azure.db.enums.TaskAction import gov.cdc.prime.router.azure.observability.event.AzureEventService import gov.cdc.prime.router.azure.observability.event.AzureEventServiceImpl @@ -39,11 +40,7 @@ abstract class FHIREngine( val blob: BlobAccess = BlobAccess(), val azureEventService: AzureEventService = AzureEventServiceImpl(), val reportService: ReportService = ReportService(ReportGraph(db), db), - val reportEventService: IReportStreamEventService = ReportStreamEventService( - db, - azureEventService, - reportService - ), + val reportEventService: IReportStreamEventService, ) : BaseEngine() { /** @@ -65,6 +62,7 @@ abstract class FHIREngine( var azureEventService: AzureEventService? = null, var reportService: ReportService? = null, var reportEventService: IReportStreamEventService? = null, + var submissionTableService: SubmissionTableService? = null, ) { /** * Set the metadata instance. @@ -110,6 +108,10 @@ abstract class FHIREngine( this.reportEventService = reportEventService } + fun submissionTableService(submissionTableService: SubmissionTableService) = apply { + this.submissionTableService = submissionTableService + } + /** * Build the fhir engine instance. * @return the fhir engine instance @@ -123,21 +125,18 @@ abstract class FHIREngine( // create the correct FHIREngine type for the action being taken return when (taskAction) { - TaskAction.receive -> FHIRReceiver( - metadata ?: Metadata.getInstance(), - settingsProvider!!, - databaseAccess ?: databaseAccessSingleton, - blobAccess ?: BlobAccess(), - azureEventService ?: AzureEventServiceImpl(), - reportService ?: ReportService(), - ) TaskAction.process -> FHIRConverter( metadata ?: Metadata.getInstance(), settingsProvider!!, databaseAccess ?: databaseAccessSingleton, blobAccess ?: BlobAccess(), azureEventService ?: AzureEventServiceImpl(), - reportService ?: ReportService() + reportService ?: ReportService(), + ReportStreamEventService( + databaseAccess ?: databaseAccessSingleton, + azureEventService ?: AzureEventServiceImpl(), + reportService ?: ReportService() + ) ) TaskAction.destination_filter -> FHIRDestinationFilter( metadata ?: Metadata.getInstance(), @@ -145,7 +144,12 @@ abstract class FHIREngine( databaseAccess ?: databaseAccessSingleton, blobAccess ?: BlobAccess(), azureEventService ?: AzureEventServiceImpl(), - reportService ?: ReportService() + reportService ?: ReportService(), + ReportStreamEventService( + databaseAccess ?: databaseAccessSingleton, + azureEventService ?: AzureEventServiceImpl(), + reportService ?: ReportService() + ) ) TaskAction.receiver_filter -> FHIRReceiverFilter( metadata ?: Metadata.getInstance(), @@ -154,13 +158,24 @@ abstract class FHIREngine( blobAccess ?: BlobAccess(), azureEventService ?: AzureEventServiceImpl(), reportService ?: ReportService(), + ReportStreamEventService( + databaseAccess ?: databaseAccessSingleton, + azureEventService ?: AzureEventServiceImpl(), + reportService ?: ReportService() + ) ) TaskAction.translate -> FHIRTranslator( metadata ?: Metadata.getInstance(), settingsProvider!!, databaseAccess ?: databaseAccessSingleton, blobAccess ?: BlobAccess(), - azureEventService ?: AzureEventServiceImpl() + azureEventService ?: AzureEventServiceImpl(), + reportService ?: ReportService(), + ReportStreamEventService( + databaseAccess ?: databaseAccessSingleton, + azureEventService ?: AzureEventServiceImpl(), + reportService ?: ReportService() + ) ) else -> throw NotImplementedError("Invalid action type for FHIR engine") } diff --git a/prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiver.kt b/prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiver.kt deleted file mode 100644 index 9bea5e891ae..00000000000 --- a/prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiver.kt +++ /dev/null @@ -1,379 +0,0 @@ -package gov.cdc.prime.router.fhirengine.engine - -import ca.uhn.hl7v2.model.Message -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.prime.reportstream.shared.QueueMessage -import gov.cdc.prime.reportstream.shared.Submission -import gov.cdc.prime.router.ActionLogger -import gov.cdc.prime.router.ClientSource -import gov.cdc.prime.router.CustomerStatus -import gov.cdc.prime.router.InvalidParamMessage -import gov.cdc.prime.router.InvalidReportMessage -import gov.cdc.prime.router.Metadata -import gov.cdc.prime.router.MimeFormat -import gov.cdc.prime.router.Options -import gov.cdc.prime.router.Report -import gov.cdc.prime.router.Sender -import gov.cdc.prime.router.SettingsProvider -import gov.cdc.prime.router.azure.ActionHistory -import gov.cdc.prime.router.azure.BlobAccess -import gov.cdc.prime.router.azure.DatabaseAccess -import gov.cdc.prime.router.azure.Event -import gov.cdc.prime.router.azure.ProcessEvent -import gov.cdc.prime.router.azure.SubmissionTableService -import gov.cdc.prime.router.azure.db.Tables -import gov.cdc.prime.router.azure.db.enums.TaskAction -import gov.cdc.prime.router.azure.observability.context.MDCUtils -import gov.cdc.prime.router.azure.observability.context.withLoggingContext -import gov.cdc.prime.router.azure.observability.event.AzureEventService -import gov.cdc.prime.router.azure.observability.event.AzureEventServiceImpl -import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName -import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties -import gov.cdc.prime.router.common.AzureHttpUtils.getSenderIP -import gov.cdc.prime.router.fhirengine.utils.FhirTranscoder -import gov.cdc.prime.router.fhirengine.utils.HL7Reader -import gov.cdc.prime.router.report.ReportService -import org.jooq.Field -import java.time.OffsetDateTime - -/** - * FHIRReceiver is responsible for processing messages from the elr-fhir-receive azure queue - * and storing them for the next step in the pipeline. - * - * @param metadata Mockable metadata instance. - * @param settings Mockable settings provider. - * @param db Mockable database access. - * @param blob Mockable blob storage access. - * @param azureEventService Service for handling Azure events. - * @param reportService Service for handling report-related operations. - * @param submissionTableService Service for inserting to the submission azure storage table. - */ -class FHIRReceiver( - metadata: Metadata = Metadata.getInstance(), - settings: SettingsProvider = this.settingsProviderSingleton, - db: DatabaseAccess = this.databaseAccessSingleton, - blob: BlobAccess = BlobAccess(), - azureEventService: AzureEventService = AzureEventServiceImpl(), - reportService: ReportService = ReportService(), - val submissionTableService: SubmissionTableService = SubmissionTableService.getInstance(), -) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService) { - - override val finishedField: Field = Tables.TASK.PROCESSED_AT - - override val engineType: String = "Receive" - override val taskAction: TaskAction = TaskAction.receive - - private val clientIdHeader = "client_id" - private val contentTypeHeader = "content-type" - - /** - * Processes a message of type [QueueMessage]. This message can be in either HL7 or FHIR format and will be placed - * on a queue for further processing. - * - * @param message The incoming message to be logged and processed. - * @param actionLogger Logger to track actions and errors. - * @param actionHistory Tracks the history of actions performed. - * @return A list of results from the FHIR engine run. - */ - override fun doWork( - message: T, - actionLogger: ActionLogger, - actionHistory: ActionHistory, - ): List = when (message) { - is FhirReceiveQueueMessage -> processFhirReceiveQueueMessage(message, actionLogger, actionHistory) - else -> throw RuntimeException("Message was not a FhirReceive and cannot be processed: $message") - } - - /** - * Processes the FHIR receive queue message. - * - * @param queueMessage The queue message containing details about the report. - * @param actionLogger The logger used to track actions and errors. - * @param actionHistory The action history related to receiving the report. - * @return A list of FHIR engine run results. - */ - private fun processFhirReceiveQueueMessage( - queueMessage: FhirReceiveQueueMessage, - actionLogger: ActionLogger, - actionHistory: ActionHistory, - ): List { - val contextMap = createLoggingContextMap(queueMessage, actionHistory) - // Use the logging context for tracing - withLoggingContext(contextMap) { - actionLogger.setReportId(queueMessage.reportId) - val sender = getSender(queueMessage, actionLogger, actionHistory) ?: return emptyList() - - // Process the message if no errors occurred - return handleSuccessfulProcessing(queueMessage, sender, actionLogger, actionHistory) - } - } - - /** - * Creates the logging context map. - * - * @param queueMessage The queue message containing details about the report. - * @param actionHistory The action history related to receiving the report. - * @return The logging context map. - */ - private fun createLoggingContextMap( - queueMessage: FhirReceiveQueueMessage, - actionHistory: ActionHistory, - ): Map = mapOf( - MDCUtils.MDCProperty.ACTION_NAME to actionHistory.action.actionName.name, - MDCUtils.MDCProperty.REPORT_ID to queueMessage.reportId, - MDCUtils.MDCProperty.BLOB_URL to queueMessage.blobURL, - ) - - /** - * Retrieves the sender based on the queue message and logs any relevant errors. - * - * @param queueMessage The queue message containing details about the report. - * @param actionLogger The logger used to track actions and errors. - * @param actionHistory The action history related to receiving the report. - * @return The sender, or null if the sender was not found or is inactive. - */ - private fun getSender( - queueMessage: FhirReceiveQueueMessage, - actionLogger: ActionLogger, - actionHistory: ActionHistory, - ): Sender? { - val clientId = queueMessage.headers[clientIdHeader] - val sender = clientId?.takeIf { it.isNotBlank() }?.let { settings.findSender(it) } - - actionHistory.trackActionParams(queueMessage.headers.toString()) - - // Handle case where sender is not found - return if (sender == null) { - // Send an error event - reportEventService.sendSubmissionProcessingError( - ReportStreamEventName.REPORT_NOT_RECEIVABLE, - TaskAction.receive, - "Sender is not found in matching client id: ${queueMessage.headers[clientIdHeader]}.", - queueMessage.reportId, - queueMessage.blobURL - ) { - params( - actionLogger.errors.associateBy { ReportStreamEventProperties.PROCESSING_ERROR } - .plus( - mapOf( - ReportStreamEventProperties.REQUEST_PARAMETERS to queueMessage.headers.toString(), - ) - ) - ) - } - - // Insert the rejection into the submission table - val submission = - Submission( - queueMessage.reportId.toString(), "Rejected", - queueMessage.blobURL, - "Sender not found matching client_id: ${queueMessage.headers[clientIdHeader]}" - ) - submissionTableService.insertSubmission(submission) - null - } else { - // Handle case where sender is inactive - if (sender.customerStatus == CustomerStatus.INACTIVE) { - // Track the action result and log the error - actionHistory.trackActionResult(HttpStatus.NOT_ACCEPTABLE) - actionLogger.error( - InvalidParamMessage("Sender has customer status INACTIVE: " + queueMessage.headers[clientIdHeader]) - ) - } - - // Track sender information - actionHistory.trackActionSenderInfo(sender.fullName, queueMessage.headers["payloadname"]) - actionHistory.trackActionResult(HttpStatus.CREATED) - sender - } - } - - /** - * Handles successful processing of the queue message. - * - * @param queueMessage The queue message containing details about the report. - * @param sender The sender information. - * @param actionHistory The action history related to receiving the report. - * @return A list of FHIR engine run results. - */ - private fun handleSuccessfulProcessing( - queueMessage: FhirReceiveQueueMessage, - sender: Sender, - actionLogger: ActionLogger, - actionHistory: ActionHistory, - ): List { - // Get content from blob storage and create report - val report = validateSubmissionMessage(sender, actionLogger, queueMessage) ?: return emptyList() - - // Determine the mime format of the message - val mimeFormat = - MimeFormat.valueOfFromMimeType( - queueMessage.headers[contentTypeHeader]?.substringBefore(';') ?: "" - ) - - val blobInfo = BlobAccess.BlobInfo( - mimeFormat, - queueMessage.blobURL, - queueMessage.digest.toByteArray() - ) - - actionHistory.trackExternalInputReport( - report, - blobInfo - ) - - // Send an event indicating the report was received - reportEventService.sendReportEvent( - eventName = ReportStreamEventName.REPORT_RECEIVED, - childReport = report, - pipelineStepName = TaskAction.receive - ) { - params( - listOfNotNull( - ReportStreamEventProperties.REQUEST_PARAMETERS to queueMessage.headers.toString(), - ReportStreamEventProperties.SENDER_NAME to sender.fullName, - ReportStreamEventProperties.FILE_LENGTH to queueMessage.headers["content-length"].toString(), - getSenderIP(queueMessage.headers)?.let { ReportStreamEventProperties.SENDER_IP to it }, - ReportStreamEventProperties.ITEM_FORMAT to mimeFormat - ).toMap() - ) - } - - // Insert the acceptance into the submissions table - val tableEntity = Submission( - queueMessage.reportId.toString(), - "Accepted", - queueMessage.blobURL, - actionLogger.errors.takeIf { it.isNotEmpty() }?.map { it.detail.message }?.toString() - ) - submissionTableService.insertSubmission(tableEntity) - - return if (actionLogger.errors.isNotEmpty()) { - // Send an event indicating the report was received - reportEventService.sendReportProcessingError( - ReportStreamEventName.REPORT_NOT_PROCESSABLE, - report, - TaskAction.receive, - "Submitted report was either empty or could not be parsed." - ) { - params( - actionLogger.errors.associateBy { ReportStreamEventProperties.PROCESSING_ERROR } - .plus( - mapOf( - ReportStreamEventProperties.REQUEST_PARAMETERS to queueMessage.headers.toString(), - ) - ) - ) - } - emptyList() - } else { - // Create a route event - val routeEvent = ProcessEvent(Event.EventAction.CONVERT, report.id, Options.None, emptyMap(), emptyList()) - - // Return the result of the FHIR engine run - listOf( - FHIREngineRunResult( - routeEvent, - report, - queueMessage.blobURL, - FhirConvertQueueMessage( - report.id, - queueMessage.blobURL, - queueMessage.digest, - queueMessage.blobSubFolderName, - sender.topic, - sender.schemaName - ) - ) - ) - } - } - - private fun validateSubmissionMessage( - sender: Sender, - actionLogger: ActionLogger, - queueMessage: FhirReceiveQueueMessage, - ): Report? { - val rawReport = BlobAccess.downloadBlob(queueMessage.blobURL, queueMessage.digest) - return if (rawReport.isBlank()) { - actionLogger.error(InvalidReportMessage("Provided raw data is empty.")) - null - } else { - val report: Report - val sources = listOf(ClientSource(organization = sender.organizationName, client = sender.name)) - - when (sender.format) { - MimeFormat.HL7 -> { - val messages: List = HL7Reader(actionLogger).getMessages(rawReport) - val isBatch: Boolean = HL7Reader(actionLogger).isBatch(rawReport, messages.size) - // create a Report for this incoming HL7 message to use for tracking in the database - - report = Report( - if (isBatch) MimeFormat.HL7_BATCH else MimeFormat.HL7, - sources, - messages.size, - metadata = metadata, - nextAction = TaskAction.convert, - topic = sender.topic, - id = queueMessage.reportId, - bodyURL = queueMessage.blobURL - ) - - // check for valid message type - messages.forEachIndexed { idx, element -> - MessageType.validateMessageType(element, actionLogger, idx + 1) - } - } - - MimeFormat.FHIR -> { - val bundles = FhirTranscoder.getBundles(rawReport, actionLogger) - report = Report( - MimeFormat.FHIR, - sources, - bundles.size, - metadata = metadata, - nextAction = TaskAction.convert, - topic = sender.topic, - id = queueMessage.reportId, - bodyURL = queueMessage.blobURL - ) - } - - else -> { - actionLogger.error(InvalidReportMessage("Unsupported sender format: ${sender.format}")) - reportEventService.sendSubmissionProcessingError( - ReportStreamEventName.REPORT_NOT_PROCESSABLE, - TaskAction.receive, - "Unsupported sender format ${sender.format}.", - queueMessage.reportId, - queueMessage.blobURL - ) { - params( - actionLogger.errors.associateBy { ReportStreamEventProperties.PROCESSING_ERROR } - .plus( - mapOf( - ReportStreamEventProperties.REQUEST_PARAMETERS to queueMessage.headers.toString(), - ReportStreamEventProperties.SENDER_NAME to sender.fullName, - ReportStreamEventProperties.FILE_LENGTH to queueMessage.headers["content-length"] - .toString(), - ReportStreamEventProperties.SENDER_IP to (getSenderIP(queueMessage.headers) ?: ""), - ReportStreamEventProperties.ITEM_FORMAT to sender.format - ) - ) - ) - } - // Insert the acceptance into the submissions table - val tableEntity = Submission( - queueMessage.reportId.toString(), - "Rejected", - queueMessage.blobURL, - actionLogger.errors.takeIf { it.isNotEmpty() }?.map { it.detail.message }?.toString() - ) - submissionTableService.insertSubmission(tableEntity) - throw IllegalStateException("Unsupported sender format: ${sender.format}") - } - } - report - } - } -} \ No newline at end of file diff --git a/prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiverFilter.kt b/prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiverFilter.kt index aa11ba34f13..4f9a0998923 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiverFilter.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiverFilter.kt @@ -31,6 +31,7 @@ import gov.cdc.prime.router.azure.observability.context.withLoggingContext import gov.cdc.prime.router.azure.observability.event.AzureEventService import gov.cdc.prime.router.azure.observability.event.AzureEventServiceImpl import gov.cdc.prime.router.azure.observability.event.AzureEventUtils +import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties import gov.cdc.prime.router.codes @@ -64,7 +65,9 @@ class FHIRReceiverFilter( blob: BlobAccess = BlobAccess(), azureEventService: AzureEventService = AzureEventServiceImpl(), reportService: ReportService = ReportService(ReportGraph(db), db), -) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService) { + reportStreamEventService: IReportStreamEventService, +) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService, reportStreamEventService) { + override val finishedField: Field = Tables.TASK.RECEIVER_FILTERED_AT override val engineType: String = "ReceiverFilter" diff --git a/prime-router/src/main/kotlin/fhirengine/engine/FHIRTranslator.kt b/prime-router/src/main/kotlin/fhirengine/engine/FHIRTranslator.kt index 5d319720ddd..674ea533330 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/FHIRTranslator.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/FHIRTranslator.kt @@ -25,6 +25,7 @@ import gov.cdc.prime.router.azure.observability.context.MDCUtils import gov.cdc.prime.router.azure.observability.context.withLoggingContext import gov.cdc.prime.router.azure.observability.event.AzureEventService import gov.cdc.prime.router.azure.observability.event.AzureEventServiceImpl +import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService import gov.cdc.prime.router.common.Environment import gov.cdc.prime.router.fhirengine.config.HL7TranslationConfig import gov.cdc.prime.router.fhirengine.translation.hl7.FhirToHl7Context @@ -53,7 +54,8 @@ class FHIRTranslator( blob: BlobAccess = BlobAccess(), azureEventService: AzureEventService = AzureEventServiceImpl(), reportService: ReportService = ReportService(), -) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService) { + reportStreamEventService: IReportStreamEventService, +) : FHIREngine(metadata, settings, db, blob, azureEventService, reportService, reportStreamEventService) { /** * Accepts a [FhirTranslateQueueMessage] [message] and, based on its parameters, sends a report to the next pipeline * step containing either the first ancestor's blob or a new blob that has been translated per diff --git a/prime-router/src/main/kotlin/fhirengine/engine/PrimeRouterQueueMessage.kt b/prime-router/src/main/kotlin/fhirengine/engine/PrimeRouterQueueMessage.kt index 6350a676a33..f547b24b975 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/PrimeRouterQueueMessage.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/PrimeRouterQueueMessage.kt @@ -37,14 +37,14 @@ abstract class ReportPipelineMessage : PrimeRouterQueueMessage() @JsonTypeName("receive") -data class FhirReceiveQueueMessage( +data class FhirConvertSubmissionQueueMessage( override val reportId: ReportId, override val blobURL: String, override val digest: String, override val blobSubFolderName: String, override val headers: Map = emptyMap(), ) : ReportPipelineMessage(), QueueMessage.ReceiveInformation { - override val messageQueueName = QueueMessage.Companion.elrReceiveQueueName + override val messageQueueName = QueueMessage.Companion.elrSubmissionConvertQueueName } @JsonTypeName("convert") diff --git a/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt b/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt index a206b0262c4..5d07fbf2e04 100644 --- a/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt +++ b/prime-router/src/main/kotlin/fhirengine/translation/hl7/FhirTransformer.kt @@ -45,10 +45,12 @@ class FhirTransformer( "metadata", Environment.get().storageEnvVar ), + errors: MutableList = mutableListOf(), + warnings: MutableList = mutableListOf(), ) : this( schemaRef = fhirTransformSchemaFromFile(schema, blobConnectionInfo), - mutableListOf(), - mutableListOf(), + errors = errors, + warnings = warnings ) /** diff --git a/prime-router/src/main/kotlin/transport/NullTransport.kt b/prime-router/src/main/kotlin/transport/NullTransport.kt index 2bb329c9acc..b57367e37e8 100644 --- a/prime-router/src/main/kotlin/transport/NullTransport.kt +++ b/prime-router/src/main/kotlin/transport/NullTransport.kt @@ -10,7 +10,7 @@ import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService import gov.cdc.prime.router.report.ReportService /** - * The Null transport is intended for testing and benchmarking purposes. + * The Null transport is used for testing and benchmarking purposes or when a transport is not configured for a receiver. */ class NullTransport : ITransport { override fun send( @@ -26,7 +26,7 @@ class NullTransport : ITransport { ): RetryItems? { if (header.content == null) error("No content for report ${header.reportFile.reportId}") val receiver = header.receiver ?: error("No receiver defined for report ${header.reportFile.reportId}") - val msg = "Sending to Null Transport" + val msg = "Sending to Null Transport. File can be downloaded by Receiver until it expires." actionHistory.trackActionResult(msg) actionHistory.trackSentReport( receiver, diff --git a/prime-router/src/main/resources/metadata/fhir_transforms/senders/Flexion/automated-testing-etor.yml b/prime-router/src/main/resources/metadata/fhir_transforms/senders/Flexion/automated-testing-etor.yml new file mode 100644 index 00000000000..fd762c40254 --- /dev/null +++ b/prime-router/src/main/resources/metadata/fhir_transforms/senders/Flexion/automated-testing-etor.yml @@ -0,0 +1,15 @@ +# $schema: ../../../../../../../metadata/json_schema/fhir/fhir-to-fhir-transform.json +# Sender transform for ETOR simulated-sender automated testing. Will be used only for automated tests in staging environment to apply any sender transforms for ETOR-NBS senders. +constants: + cdphSender: 'Bundle.entry.resource.ofType(MessageHeader).source.name = "SISGDSP"' + +elements: + + # REGEX removes leading zeroes from left of decimal point unless there is only a single zero + - name: remove-leading-zeros-from-nm-data-type + resource: 'Bundle.entry.resource.ofType(Observation).value.ofType(Quantity)' + condition: > + %resource.value.toString().matches("^0\\d+(\\.\\d+)?$") and + %cdphSender + bundleProperty: '%resource.value' + value: [ '%resource.value.toString().replaceMatches("^0+(\\d*|0)(\\.\\d+)?$", "$1$2")' ] \ No newline at end of file diff --git a/prime-router/src/main/resources/metadata/fhir_transforms/senders/MMTC/mmtc-sender-transform.yml b/prime-router/src/main/resources/metadata/fhir_transforms/senders/MMTC/mmtc-sender-transform.yml new file mode 100644 index 00000000000..60110a9f807 --- /dev/null +++ b/prime-router/src/main/resources/metadata/fhir_transforms/senders/MMTC/mmtc-sender-transform.yml @@ -0,0 +1,12 @@ +extends: classpath:/metadata/fhir_transforms/senders/original-pipeline-transforms.yml + +elements: + - name: patient-state-from-zip-code + resource: "Bundle.entry.resource.ofType(Patient).address" + bundleProperty: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/state-from-zip-code").value[x]' + value: [ '%resource.postalCode.getStateFromZipCode()' ] + + - name: ordering-facility-state-from-zip-code + resource: "Bundle.entry.resource.ofType(ServiceRequest).requester.resolve().organization.resolve().address" + bundleProperty: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/state-from-zip-code").value[x]' + value: [ '%resource.postalCode.getStateFromZipCode()' ] diff --git a/prime-router/src/test/kotlin/ReceiverTests.kt b/prime-router/src/test/kotlin/ReceiverTests.kt index 0a566d70613..084dc7059d2 100644 --- a/prime-router/src/test/kotlin/ReceiverTests.kt +++ b/prime-router/src/test/kotlin/ReceiverTests.kt @@ -267,4 +267,10 @@ internal class ReceiverTests { ) assertThat(receiver.schemaName).isEqualTo("CO") } + + @Test + fun `test receiver type with null transport`() { + val receiver = Receiver("elr", "IGNORE", Topic.COVID_19, CustomerStatus.INACTIVE, translatorConfig) + assertThat(receiver.transportType.type).isEqualTo("NULL") + } } \ No newline at end of file diff --git a/prime-router/src/test/kotlin/azure/ActionHistoryTests.kt b/prime-router/src/test/kotlin/azure/ActionHistoryTests.kt index c6b6c5389a6..b6f26ae29b0 100644 --- a/prime-router/src/test/kotlin/azure/ActionHistoryTests.kt +++ b/prime-router/src/test/kotlin/azure/ActionHistoryTests.kt @@ -376,9 +376,9 @@ class ActionHistoryTests { "" ) every { - mockReportEventService.sendReportEvent(any(), any(), any(), any()) + mockReportEventService.sendReportEvent(any(), any(), any(), any(), any()) } returns Unit - every { mockReportEventService.sendItemEvent(any(), any(), any(), any()) } returns Unit + every { mockReportEventService.sendItemEvent(any(), any(), any(), any(), any()) } returns Unit mockkObject(Report) mockkObject(FhirTranscoder) every { FhirTranscoder.decode(any(), any()) } returns mockk() @@ -433,8 +433,8 @@ class ActionHistoryTests { assertThat(reportFile.itemCount).isEqualTo(15) assertThat(actionHistory1.action.externalName).isEqualTo("filename1") verify(exactly = 1) { - mockReportEventService.sendReportEvent(any(), any(), any(), any()) - mockReportEventService.sendItemEvent(any(), any(), any(), any()) + mockReportEventService.sendReportEvent(any(), any(), any(), any(), any()) + mockReportEventService.sendItemEvent(any(), any(), any(), any(), any()) } // not allowed to track the same report twice. assertFailure { @@ -516,9 +516,9 @@ class ActionHistoryTests { every { anyConstructed().generateDigest(any()) } returns mockk() val header = mockk() every { - mockReportEventService.sendReportEvent(any(), any(), any(), any()) + mockReportEventService.sendReportEvent(any(), any(), any(), any(), any()) } returns Unit - every { mockReportEventService.sendItemEvent(any(), any(), any(), any()) } returns Unit + every { mockReportEventService.sendItemEvent(any(), any(), any(), any(), any()) } returns Unit val inReportFile = mockk() every { header.reportFile } returns inReportFile every { header.content } returns "".toByteArray() @@ -571,8 +571,8 @@ class ActionHistoryTests { assertThat(actionHistory2.reportsOut[uuid]?.schemaName) .isEqualTo("STED/NESTED/STLTs/REALLY_LONG_STATE_NAME/REALLY_LONG_STATE_NAME") verify(exactly = 2) { - mockReportEventService.sendReportEvent(any(), any(), any(), any()) - mockReportEventService.sendItemEvent(any(), any(), any(), any()) + mockReportEventService.sendReportEvent(any(), any(), any(), any(), any()) + mockReportEventService.sendItemEvent(any(), any(), any(), any(), any()) } } @@ -784,9 +784,9 @@ class ActionHistoryTests { every { anyConstructed().generateDigest(any()) } returns mockk() val header = mockk() every { - mockReportEventService.sendReportEvent(any(), any(), any(), any()) + mockReportEventService.sendReportEvent(any(), any(), any(), any(), any()) } returns Unit - every { mockReportEventService.sendItemEvent(any(), any(), any(), any()) } returns Unit + every { mockReportEventService.sendItemEvent(any(), any(), any(), any(), any()) } returns Unit val inReportFile = mockk() every { header.reportFile } returns inReportFile every { header.content } returns "".toByteArray() @@ -837,8 +837,8 @@ class ActionHistoryTests { assertContains(blobUrls[0], org.receivers[0].fullName) assertContains(blobUrls[1], org.receivers[1].fullName) verify(exactly = 2) { - mockReportEventService.sendReportEvent(any(), any(), any(), any()) - mockReportEventService.sendItemEvent(any(), any(), any(), any()) + mockReportEventService.sendReportEvent(any(), any(), any(), any(), any()) + mockReportEventService.sendItemEvent(any(), any(), any(), any(), any()) } } diff --git a/prime-router/src/test/kotlin/azure/FHIRFunctionsTests.kt b/prime-router/src/test/kotlin/azure/FHIRFunctionsTests.kt index 1d3454c1824..6ccd26f0852 100644 --- a/prime-router/src/test/kotlin/azure/FHIRFunctionsTests.kt +++ b/prime-router/src/test/kotlin/azure/FHIRFunctionsTests.kt @@ -66,7 +66,11 @@ class FHIRFunctionsTests { .databaseAccess(accessSpy) .build() every { accessSpy.fetchReportFile(any()) } returns mockk(relaxed = true) - return FHIRFunctions(workflowEngine, databaseAccess = accessSpy) + return FHIRFunctions( + workflowEngine, + databaseAccess = accessSpy, + submissionTableService = mockk() + ) } @Test @@ -76,7 +80,14 @@ class FHIRFunctionsTests { val mockReportEventService = mockk(relaxed = true) val init = slot Unit>() every { - mockReportEventService.sendReportProcessingError(any(), any(), any(), any(), capture(init)) + mockReportEventService.sendReportProcessingError( + any(), + any(), + any(), + any(), + any(), + capture(init) + ) } returns Unit val mockFHIRConverter = mockk(relaxed = true) every { mockFHIRConverter.run(any(), any(), any(), any()) } throws RuntimeException("Error") @@ -94,6 +105,7 @@ class FHIRFunctionsTests { any(), TaskAction.convert, "Error", + any(), init.captured ) } diff --git a/prime-router/src/test/kotlin/azure/SendFunctionTests.kt b/prime-router/src/test/kotlin/azure/SendFunctionTests.kt index 9c6695494aa..57c6eb9a0cf 100644 --- a/prime-router/src/test/kotlin/azure/SendFunctionTests.kt +++ b/prime-router/src/test/kotlin/azure/SendFunctionTests.kt @@ -14,6 +14,7 @@ import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile import gov.cdc.prime.router.azure.db.tables.pojos.Task import gov.cdc.prime.router.azure.observability.event.InMemoryAzureEventService import gov.cdc.prime.router.report.ReportService +import gov.cdc.prime.router.transport.NullTransport import gov.cdc.prime.router.transport.RetryToken import gov.cdc.prime.router.transport.SftpTransport import gov.cdc.prime.router.unittest.UnitTestUtils @@ -23,6 +24,7 @@ import io.mockk.mockk import io.mockk.mockkClass import io.mockk.mockkConstructor import io.mockk.mockkObject +import io.mockk.unmockkAll import io.mockk.verify import org.jooq.Configuration import org.junit.jupiter.api.AfterEach @@ -38,6 +40,7 @@ class SendFunctionTests { val logger = mockkClass(Logger::class) val workflowEngine = mockkClass(WorkflowEngine::class) val sftpTransport = mockkClass(SftpTransport::class) + val nullTransport = mockkClass(NullTransport::class) val reportId = UUID.randomUUID() val task = Task( reportId, @@ -84,11 +87,12 @@ class SendFunctionTests { every { workflowEngine.settings }.returns(settings) every { workflowEngine.readBody(any()) }.returns("body".toByteArray()) every { workflowEngine.sftpTransport }.returns(sftpTransport) + every { workflowEngine.nullTransport }.returns(nullTransport) every { workflowEngine.azureEventService }.returns(InMemoryAzureEventService()) every { workflowEngine.reportService }.returns(mockk(relaxed = true)) } - fun makeHeader(): WorkflowEngine.Header { + fun makeIgnoreDotCSVHeader(): WorkflowEngine.Header { return WorkflowEngine.Header( task, reportFile, null, @@ -99,9 +103,21 @@ class SendFunctionTests { ) } + fun makeIgnoreDotHL7NullHeader(): WorkflowEngine.Header { + return WorkflowEngine.Header( + task, reportFile, + null, + settings.findOrganization("ignore"), + settings.findReceiver("ignore.HL7_NULL"), + metadata.findSchema("covid-19"), "hello".toByteArray(), + true + ) + } + @AfterEach fun reset() { clearAllMocks() + unmockkAll() } @Test @@ -113,10 +129,40 @@ class SendFunctionTests { every { workflowEngine.handleReportEvent(any(), any()) }.answers { val block = secondArg() as (header: WorkflowEngine.Header, retryToken: RetryToken?, txn: Configuration?) -> ReportEvent - val header = makeHeader() + val header = makeIgnoreDotCSVHeader() + nextEvent = block(header, null, null) + } + every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any()) }.returns(null) + every { workflowEngine.recordAction(any()) }.returns(Unit) + every { workflowEngine.azureEventService.trackEvent(any()) }.returns(Unit) + every { workflowEngine.reportService.getRootReports(any()) } returns reportList + every { workflowEngine.db } returns mockk() + mockkObject(Report.Companion) + every { Report.formExternalFilename(any(), any(), any(), any(), any(), any(), any()) } returns "" + + // Invoke + val event = ReportEvent(Event.EventAction.SEND, reportId, false) + SendFunction(workflowEngine).run(event.toQueueMessage(), context) + + // Verify + assertThat(nextEvent).isNotNull() + assertThat(nextEvent!!.eventAction).isEqualTo(Event.EventAction.NONE) + assertThat(nextEvent!!.retryToken).isNull() + } + + @Test + fun `Test with null transport`() { + var nextEvent: ReportEvent? = null + val reportList = listOf(reportFile) + setupLogger() + setupWorkflow() + every { workflowEngine.handleReportEvent(any(), any()) }.answers { + val block = secondArg() as + (header: WorkflowEngine.Header, retryToken: RetryToken?, txn: Configuration?) -> ReportEvent + val header = makeIgnoreDotHL7NullHeader() nextEvent = block(header, null, null) } - every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any(),) }.returns(null) + every { nullTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any()) }.returns(null) every { workflowEngine.recordAction(any()) }.returns(Unit) every { workflowEngine.azureEventService.trackEvent(any()) }.returns(Unit) every { workflowEngine.reportService.getRootReports(any()) } returns reportList @@ -129,6 +175,8 @@ class SendFunctionTests { SendFunction(workflowEngine).run(event.toQueueMessage(), context) // Verify + verify { nullTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any()) } + verify { workflowEngine.recordAction(match { it.action.actionName == TaskAction.send }) } assertThat(nextEvent).isNotNull() assertThat(nextEvent!!.eventAction).isEqualTo(Event.EventAction.NONE) assertThat(nextEvent!!.retryToken).isNull() @@ -144,11 +192,11 @@ class SendFunctionTests { every { workflowEngine.handleReportEvent(any(), any()) }.answers { val block = secondArg() as (header: WorkflowEngine.Header, retryToken: RetryToken?, txn: Configuration?) -> ReportEvent - val header = makeHeader() + val header = makeIgnoreDotCSVHeader() nextEvent = block(header, null, null) } setupWorkflow() - every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any(),) } + every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any()) } .returns(RetryToken.allItems) every { workflowEngine.recordAction(any()) }.returns(Unit) every { workflowEngine.db } returns mockk() @@ -175,13 +223,13 @@ class SendFunctionTests { val block = secondArg() as (header: WorkflowEngine.Header, retryToken: RetryToken?, txn: Configuration?) -> ReportEvent - val header = makeHeader() + val header = makeIgnoreDotCSVHeader() nextEvent = block( header, RetryToken(2, RetryToken.allItems), null ) } setupWorkflow() - every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any(),) } + every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any()) } .returns(RetryToken.allItems) every { workflowEngine.recordAction(any()) }.returns(Unit) every { workflowEngine.db } returns mockk() @@ -211,14 +259,14 @@ class SendFunctionTests { every { workflowEngine.handleReportEvent(any(), any()) }.answers { val block = secondArg() as (header: WorkflowEngine.Header, retryToken: RetryToken?, txn: Configuration?) -> ReportEvent - val header = makeHeader() + val header = makeIgnoreDotCSVHeader() // Should be high enough retry count that the next action should have an error nextEvent = block( header, RetryToken(100, RetryToken.allItems), null ) } setupWorkflow() - every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any(),) } + every { sftpTransport.send(any(), any(), any(), any(), any(), any(), any(), any(), any()) } .returns(RetryToken.allItems) every { workflowEngine.recordAction(any()) }.returns(Unit) every { workflowEngine.db } returns mockk(relaxed = true) diff --git a/prime-router/src/test/kotlin/azure/observability/event/ReportEventServiceTest.kt b/prime-router/src/test/kotlin/azure/observability/event/ReportEventServiceTest.kt index 2604cffdee4..00360b9600b 100644 --- a/prime-router/src/test/kotlin/azure/observability/event/ReportEventServiceTest.kt +++ b/prime-router/src/test/kotlin/azure/observability/event/ReportEventServiceTest.kt @@ -202,7 +202,7 @@ class ReportEventServiceTest { val data = reportEventService.getItemEventData( 1, - translateNode.node.reportId, + translateNode.node.reportId, 1, "" ) diff --git a/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt b/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt index f158a08993b..45f03b9514e 100644 --- a/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt +++ b/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt @@ -19,6 +19,7 @@ import gov.cdc.prime.router.azure.BlobAccess import gov.cdc.prime.router.azure.DataAccessTransaction import gov.cdc.prime.router.azure.Event import gov.cdc.prime.router.azure.ProcessEvent +import gov.cdc.prime.router.azure.SubmissionTableService import gov.cdc.prime.router.azure.WorkflowEngine import gov.cdc.prime.router.azure.db.Tables import gov.cdc.prime.router.azure.db.enums.TaskAction @@ -30,6 +31,7 @@ import gov.cdc.prime.router.db.ReportStreamTestDatabaseContainer import gov.cdc.prime.router.fhirengine.azure.FHIRFunctions import gov.cdc.prime.router.metadata.LookupTable import gov.cdc.prime.router.unittest.UnitTestUtils +import io.mockk.mockk import org.jooq.impl.DSL import org.testcontainers.containers.GenericContainer import java.io.File @@ -282,6 +284,7 @@ object UniversalPipelineTestUtils { txn: DataAccessTransaction, expectedItems: Int? = null, expectedReports: Int = 1, + parentIsRoot: Boolean = false, ): List { val itemLineages = DSL .using(txn) @@ -297,6 +300,8 @@ object UniversalPipelineTestUtils { // itemCount is on the report created by the test. It will not be null. if (parent.itemCount > 1) { assertThat(itemLineages.map { it.parentIndex }).isEqualTo((1..expectedItems).toList()) + } else if (parentIsRoot) { + assertThat(itemLineages.map { it.parentIndex }).isEqualTo((1..expectedItems).toList()) } else { assertThat(itemLineages.map { it.parentIndex }).isEqualTo(MutableList(expectedItems) { 1 }) } @@ -399,7 +404,11 @@ object UniversalPipelineTestUtils { .settingsProvider(settings) .databaseAccess(ReportStreamTestDatabaseContainer.testDatabaseAccess) .build() - return FHIRFunctions(workflowEngine, databaseAccess = ReportStreamTestDatabaseContainer.testDatabaseAccess) + return FHIRFunctions( + workflowEngine, + databaseAccess = ReportStreamTestDatabaseContainer.testDatabaseAccess, + submissionTableService = mockk() + ) } fun getBlobContainerMetadata(azuriteContainer: GenericContainer<*>): BlobAccess.BlobContainerMetadata { diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt index bd483b02079..a5a553442a5 100644 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt @@ -9,6 +9,8 @@ import assertk.assertions.isEqualToIgnoringGivenProperties import assertk.assertions.matchesPredicate import gov.cdc.prime.reportstream.shared.BlobUtils import gov.cdc.prime.reportstream.shared.QueueMessage +import gov.cdc.prime.router.ClientSource +import gov.cdc.prime.router.CustomerStatus import gov.cdc.prime.router.FileSettings import gov.cdc.prime.router.Metadata import gov.cdc.prime.router.MimeFormat @@ -16,12 +18,14 @@ import gov.cdc.prime.router.Options import gov.cdc.prime.router.Report import gov.cdc.prime.router.Sender import gov.cdc.prime.router.Topic +import gov.cdc.prime.router.UniversalPipelineSender import gov.cdc.prime.router.azure.ActionHistory import gov.cdc.prime.router.azure.BlobAccess import gov.cdc.prime.router.azure.DatabaseLookupTableAccess import gov.cdc.prime.router.azure.Event import gov.cdc.prime.router.azure.ProcessEvent import gov.cdc.prime.router.azure.QueueAccess +import gov.cdc.prime.router.azure.SubmissionTableService import gov.cdc.prime.router.azure.WorkflowEngine import gov.cdc.prime.router.azure.db.Tables import gov.cdc.prime.router.azure.db.enums.ActionLogType @@ -35,6 +39,7 @@ import gov.cdc.prime.router.azure.observability.event.ItemEventData import gov.cdc.prime.router.azure.observability.event.ReportEventData import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties +import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService import gov.cdc.prime.router.azure.observability.event.ReportStreamItemEvent import gov.cdc.prime.router.cli.tests.CompareData import gov.cdc.prime.router.common.TestcontainersUtils @@ -67,11 +72,14 @@ import gov.cdc.prime.router.fhirengine.engine.FHIRConverter import gov.cdc.prime.router.fhirengine.engine.FhirDestinationFilterQueueMessage import gov.cdc.prime.router.fhirengine.utils.FhirTranscoder import gov.cdc.prime.router.history.DetailedActionLog +import gov.cdc.prime.router.history.db.ReportGraph import gov.cdc.prime.router.metadata.LookupTable import gov.cdc.prime.router.metadata.ObservationMappingConstants +import gov.cdc.prime.router.report.ReportService import gov.cdc.prime.router.unittest.UnitTestUtils import gov.cdc.prime.router.version.Version import io.mockk.every +import io.mockk.mockk import io.mockk.mockkConstructor import io.mockk.mockkObject import io.mockk.unmockkAll @@ -80,6 +88,7 @@ import org.jooq.impl.DSL import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.ExtendWith import org.testcontainers.junit.jupiter.Container import org.testcontainers.junit.jupiter.Testcontainers @@ -87,6 +96,7 @@ import tech.tablesaw.api.StringColumn import tech.tablesaw.api.Table import java.nio.charset.Charset import java.time.OffsetDateTime +import java.util.UUID @Testcontainers @ExtendWith(ReportStreamTestDatabaseSetupExtension::class) @@ -101,6 +111,14 @@ class FHIRConverterIntegrationTests { ) val azureEventService = InMemoryAzureEventService() + val mockSubmissionTableService = mockk() + val reportStreamEventService = ReportStreamEventService( + ReportStreamTestDatabaseContainer.testDatabaseAccess, azureEventService, + ReportService( + ReportGraph(ReportStreamTestDatabaseContainer.testDatabaseAccess), + ReportStreamTestDatabaseContainer.testDatabaseAccess + ) + ) private fun createFHIRFunctionsInstance(): FHIRFunctions { val settings = FileSettings().loadOrganizations(universalPipelineOrganization) @@ -114,7 +132,14 @@ class FHIRConverterIntegrationTests { .settingsProvider(settings) .databaseAccess(ReportStreamTestDatabaseContainer.testDatabaseAccess) .build() - return FHIRFunctions(workflowEngine, databaseAccess = ReportStreamTestDatabaseContainer.testDatabaseAccess) + + return FHIRFunctions( + workflowEngine, + databaseAccess = ReportStreamTestDatabaseContainer.testDatabaseAccess, + submissionTableService = mockSubmissionTableService, + azureEventService = azureEventService, + reportStreamEventService = reportStreamEventService + ) } private fun createFHIRConverter(): FHIRConverter { @@ -128,19 +153,15 @@ class FHIRConverterIntegrationTests { settings, ReportStreamTestDatabaseContainer.testDatabaseAccess, azureEventService = azureEventService, + reportStreamEventService = reportStreamEventService ) } - private fun generateQueueMessage( + private fun generateFHIRConvertQueueMessage( report: Report, blobContents: String, sender: Sender, - headers: Map? = null, ): String { - val headersString = headers?.entries?.joinToString(separator = ",\n") { (key, value) -> - """"$key": "$value"""" - } ?: "" - return """ { "type": "convert", @@ -150,7 +171,29 @@ class FHIRConverterIntegrationTests { "blobSubFolderName": "${sender.fullName}", "topic": "${sender.topic.jsonVal}", "schemaName": "${sender.schemaName}" - ${if (headersString.isNotEmpty()) ",\n$headersString" else ""} + } + """.trimIndent() + } + + private fun generateFHIRConvertSubmissionQueueMessage( + report: Report, + blobContents: String, + sender: Sender, + ): String { + // TODO: something is wrong with the Jackson configuration as it should not require the type to parse this + val headers = mapOf("client_id" to sender.fullName) + val headersStringMap = headers.entries.joinToString(separator = ",\n") { (key, value) -> + """"$key": "$value"""" + } + val headersString = "[\"java.util.LinkedHashMap\",{$headersStringMap}]" + return """ + { + "type": "receive-fhir", + "reportId": "${report.id}", + "blobURL": "${report.bodyURL}", + "digest": "${BlobUtils.digestToString(BlobUtils.sha256Digest(blobContents.toByteArray()))}", + "blobSubFolderName": "${sender.fullName}", + "headers":$headersString } """.trimIndent() } @@ -164,6 +207,9 @@ class FHIRConverterIntegrationTests { mockkObject(BlobAccess.BlobContainerMetadata) every { BlobAccess.BlobContainerMetadata.build(any(), any()) } returns getBlobContainerMetadata() mockkConstructor(DatabaseLookupTableAccess::class) + every { mockSubmissionTableService.insertSubmission(any()) } returns Unit + mockkObject(Metadata) + every { Metadata.getInstance() } returns UnitTestUtils.simpleMetadata } @AfterEach @@ -239,6 +285,235 @@ class FHIRConverterIntegrationTests { } } + @Test + fun `should add a message to the poison queue if the sender is not found and not do any work`() { + val receivedReportContents = + listOf(cleanHL7Record, invalidHL7Record, unparseableHL7Record, badEncodingHL7Record) + .joinToString("\n") + val receiveBlobUrl = BlobAccess.uploadBlob( + "receive/happy-path.hl7", + receivedReportContents.toByteArray(), + getBlobContainerMetadata() + ) + + val receiveReport = Report( + hl7SenderWithNoTransform.format, + listOf( + ClientSource( + organization = hl7SenderWithNoTransform.organizationName, + client = hl7SenderWithNoTransform.name + ) + ), + 1, + metadata = UnitTestUtils.simpleMetadata, + nextAction = TaskAction.convert, + topic = hl7SenderWithNoTransform.topic, + id = UUID.randomUUID(), + bodyURL = receiveBlobUrl + ) + val missingSender = UniversalPipelineSender( + "foo", + "phd", + MimeFormat.HL7, + CustomerStatus.ACTIVE, + topic = Topic.FULL_ELR, + ) + val queueMessage = + generateFHIRConvertSubmissionQueueMessage(receiveReport, receivedReportContents, missingSender) + val fhirFunctions = createFHIRFunctionsInstance() + + fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) + ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> + assertThrows { + ReportStreamTestDatabaseContainer.testDatabaseAccess.fetchReportFile(receiveReport.id, txn = txn) + } + val processedReports = fetchChildReports( + receiveReport, txn, 0, 0, parentIsRoot = true + ) + assertThat(processedReports).hasSize(0) + verify(exactly = 1) { + QueueAccess.sendMessage( + "${QueueMessage.elrSubmissionConvertQueueName}-poison", + queueMessage + + ) + } + } + } + + @Test + fun `should successfully process a FhirConvertSubmissionQueueMessage`() { + val receivedReportContents = + listOf(cleanHL7Record, invalidHL7Record, unparseableHL7Record, badEncodingHL7Record) + .joinToString("\n") + val receiveBlobUrl = BlobAccess.uploadBlob( + "receive/happy-path.hl7", + receivedReportContents.toByteArray(), + getBlobContainerMetadata() + ) + + val receiveReport = Report( + hl7SenderWithNoTransform.format, + listOf( + ClientSource( + organization = hl7SenderWithNoTransform.organizationName, + client = hl7SenderWithNoTransform.name + ) + ), + 1, + metadata = UnitTestUtils.simpleMetadata, + nextAction = TaskAction.convert, + topic = hl7SenderWithNoTransform.topic, + id = UUID.randomUUID(), + bodyURL = receiveBlobUrl + ) + val queueMessage = + generateFHIRConvertSubmissionQueueMessage(receiveReport, receivedReportContents, hl7SenderWithNoTransform) + val fhirFunctions = createFHIRFunctionsInstance() + + fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) + + ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> + val externalReportRecord = + ReportStreamTestDatabaseContainer.testDatabaseAccess.fetchReportFile(receiveReport.id, txn = txn) + assertThat(externalReportRecord.sendingOrg).isEqualTo(hl7SenderWithNoTransform.organizationName) + assertThat(externalReportRecord.sendingOrgClient).isEqualTo(hl7SenderWithNoTransform.name) + val (routedReports, unroutedReports) = fetchChildReports( + receiveReport, txn, 4, 4, parentIsRoot = true + ).partition { it.nextAction != TaskAction.none } + assertThat(routedReports).hasSize(2) + routedReports.forEach { + assertThat(it.nextAction).isEqualTo(TaskAction.destination_filter) + assertThat(it.receivingOrg).isEqualTo(null) + assertThat(it.receivingOrgSvc).isEqualTo(null) + assertThat(it.schemaName).isEqualTo("None") + assertThat(it.schemaTopic).isEqualTo(Topic.FULL_ELR) + assertThat(it.bodyFormat).isEqualTo("FHIR") + } + assertThat(unroutedReports).hasSize(2) + unroutedReports.forEach { + assertThat(it.nextAction).isEqualTo(TaskAction.none) + assertThat(it.receivingOrg).isEqualTo(null) + assertThat(it.receivingOrgSvc).isEqualTo(null) + assertThat(it.schemaName).isEqualTo("None") + assertThat(it.schemaTopic).isEqualTo(Topic.FULL_ELR) + assertThat(it.bodyFormat).isEqualTo("FHIR") + } + // Verify that the expected FHIR bundles were uploaded + val reportAndBundles = + routedReports.map { + Pair( + it, + BlobAccess.downloadBlobAsByteArray(it.bodyUrl, getBlobContainerMetadata()) + ) + } + + assertThat(reportAndBundles).transform { pairs -> pairs.map { it.second } }.each { + it.matchesPredicate { bytes -> + val invalidHL7Result = CompareData().compare( + cleanHL7RecordConverted.byteInputStream(), + bytes.inputStream(), + MimeFormat.FHIR, + null + ) + invalidHL7Result.passed + + val cleanHL7Result = CompareData().compare( + invalidHL7RecordConverted.byteInputStream(), + bytes.inputStream(), + MimeFormat.FHIR, + null + ) + invalidHL7Result.passed || cleanHL7Result.passed + } + } + + val expectedQueueMessages = reportAndBundles.map { (report, fhirBundle) -> + FhirDestinationFilterQueueMessage( + report.reportId, + report.bodyUrl, + BlobUtils.digestToString(BlobUtils.sha256Digest(fhirBundle)), + hl7SenderWithNoTransform.fullName, + hl7SenderWithNoTransform.topic + ) + }.map { it.serialize() } + + verify(exactly = 2) { + QueueAccess.sendMessage( + QueueMessage.elrDestinationFilterQueueName, + match { expectedQueueMessages.contains(it) } + ) + } + + val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()).from(Tables.ACTION_LOG) + .where(Tables.ACTION_LOG.REPORT_ID.eq(receiveReport.id)) + .and(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) + .fetchInto( + DetailedActionLog::class.java + ) + + assertThat(actionLogs).hasSize(2) + @Suppress("ktlint:standard:max-line-length") + assertThat(actionLogs).transform { logs -> logs.map { it.detail.message } } + .containsOnly( + "Item 3 in the report was not parseable. Reason: exception while parsing HL7: Determine encoding for message. The following is the first 50 chars of the message for reference, although this may not be where the issue is: MSH^~\\&|CDC PRIME - Atlanta, Georgia (Dekalb)^2.16", + "Item 4 in the report was not parseable. Reason: exception while parsing HL7: Invalid or incomplete encoding characters - MSH-2 is ^~\\&#!" + ) + assertThat(actionLogs).transform { + it.map { log -> + log.trackingId + } + }.containsOnly( + "", + "" + ) + + assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.ITEM_ACCEPTED]!!).hasSize(2) + val event = + azureEventService + .reportStreamEvents[ReportStreamEventName.ITEM_ACCEPTED]!!.last() as ReportStreamItemEvent + assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( + ReportEventData( + routedReports[1].reportId, + receiveReport.id, + listOf(receiveReport.id), + Topic.FULL_ELR, + routedReports[1].bodyUrl, + TaskAction.convert, + OffsetDateTime.now(), + Version.commitId + ), + ReportEventData::timestamp + ) + assertThat(event.itemEventData).isEqualToIgnoringGivenProperties( + ItemEventData( + 1, + 2, + 2, + "371784", + "phd.hl7-elr-no-transform" + ) + ) + assertThat(event.params).isEqualTo( + mapOf( + ReportStreamEventProperties.ITEM_FORMAT to MimeFormat.HL7, + ReportStreamEventProperties.BUNDLE_DIGEST to BundleDigestLabResult( + observationSummaries = AzureEventUtils + .getObservationSummaries( + FhirTranscoder.decode( + reportAndBundles[1].second.toString(Charset.defaultCharset()) + ) + ), + patientState = listOf("TX"), + orderingFacilityState = listOf("FL"), + performerState = emptyList(), + eventType = "ORU^R01^ORU_R01" + ) + ) + ) + } + } + @Test fun `should successfully convert HL7 messages`() { val receivedReportContents = @@ -251,7 +526,8 @@ class FHIRConverterIntegrationTests { ) val receiveReport = setupConvertStep(MimeFormat.HL7, hl7SenderWithNoTransform, receiveBlobUrl, 4) - val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, hl7SenderWithNoTransform) + val queueMessage = + generateFHIRConvertQueueMessage(receiveReport, receivedReportContents, hl7SenderWithNoTransform) val fhirFunctions = createFHIRFunctionsInstance() fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) @@ -362,7 +638,7 @@ class FHIRConverterIntegrationTests { OffsetDateTime.now(), Version.commitId ), - ReportEventData::timestamp + ReportEventData::timestamp ) assertThat(event.itemEventData).isEqualToIgnoringGivenProperties( ItemEventData( @@ -432,7 +708,7 @@ class FHIRConverterIntegrationTests { MimeFormat.FHIR, fhirSenderWithNoTransform, receiveBlobUrl, 4 ) - val queueMessage = generateQueueMessage( + val queueMessage = generateFHIRConvertQueueMessage( receiveReport, receivedReportContents, fhirSenderWithNoTransform ) @@ -535,7 +811,7 @@ class FHIRConverterIntegrationTests { OffsetDateTime.now(), Version.commitId ), - ReportEventData::timestamp + ReportEventData::timestamp ) assertThat(event.itemEventData).isEqualToIgnoringGivenProperties( ItemEventData( @@ -578,7 +854,7 @@ class FHIRConverterIntegrationTests { ) val receiveReport = setupConvertStep(MimeFormat.HL7, senderWithValidation, receiveBlobUrl, 2) - val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, senderWithValidation) + val queueMessage = generateFHIRConvertQueueMessage(receiveReport, receivedReportContents, senderWithValidation) val fhirFunctions = createFHIRFunctionsInstance() fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) @@ -671,7 +947,7 @@ class FHIRConverterIntegrationTests { OffsetDateTime.now(), Version.commitId ), - ReportEventData::timestamp + ReportEventData::timestamp ) assertThat(event.itemEventData).isEqualToIgnoringGivenProperties( ItemEventData( @@ -689,7 +965,7 @@ class FHIRConverterIntegrationTests { ReportStreamEventProperties.VALIDATION_PROFILE to Topic.MARS_OTC_ELR.validator.validatorProfileName, @Suppress("ktlint:standard:max-line-length") ReportStreamEventProperties.PROCESSING_ERROR - to "Item 2 in the report was not valid. Reason: HL7 was not valid at MSH[1]-21[1].3 for validator: RADx MARS" + to "Item 2 in the report was not valid. Reason: HL7 was not valid at MSH[1]-21[1].3 for validator: RADx MARS" ) ) } @@ -706,7 +982,7 @@ class FHIRConverterIntegrationTests { ) val receiveReport = setupConvertStep(MimeFormat.HL7, hl7Sender, receiveBlobUrl, 2) - val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, hl7Sender) + val queueMessage = generateFHIRConvertQueueMessage(receiveReport, receivedReportContents, hl7Sender) val fhirFunctions = createFHIRFunctionsInstance() fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) @@ -787,7 +1063,7 @@ class FHIRConverterIntegrationTests { ) val receiveReport = setupConvertStep(MimeFormat.HL7, hl7Sender, receiveBlobUrl, 1) - val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, hl7Sender) + val queueMessage = generateFHIRConvertQueueMessage(receiveReport, receivedReportContents, hl7Sender) val fhirFunctions = createFHIRFunctionsInstance() fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) @@ -816,7 +1092,7 @@ class FHIRConverterIntegrationTests { ) val receiveReport = setupConvertStep(MimeFormat.HL7, hl7Sender, receiveBlobUrl, 1) - val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, hl7Sender) + val queueMessage = generateFHIRConvertQueueMessage(receiveReport, receivedReportContents, hl7Sender) val fhirFunctions = createFHIRFunctionsInstance() fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) @@ -845,7 +1121,7 @@ class FHIRConverterIntegrationTests { ) val receiveReport = setupConvertStep(MimeFormat.HL7, hl7Sender, receiveBlobUrl, 1) - val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, hl7Sender) + val queueMessage = generateFHIRConvertQueueMessage(receiveReport, receivedReportContents, hl7Sender) val fhirFunctions = createFHIRFunctionsInstance() fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt index 8e2f420d888..2c8f314fbe6 100644 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt @@ -33,6 +33,7 @@ import gov.cdc.prime.router.azure.observability.event.ItemEventData import gov.cdc.prime.router.azure.observability.event.ReportEventData import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties +import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService import gov.cdc.prime.router.azure.observability.event.ReportStreamItemEvent import gov.cdc.prime.router.common.TestcontainersUtils import gov.cdc.prime.router.common.UniversalPipelineTestUtils @@ -120,7 +121,14 @@ class FHIRDestinationFilterIntegrationTests : Logging { settings, db = ReportStreamTestDatabaseContainer.testDatabaseAccess, reportService = ReportService(ReportGraph(ReportStreamTestDatabaseContainer.testDatabaseAccess)), - azureEventService = azureEventService + azureEventService = azureEventService, + reportStreamEventService = ReportStreamEventService( + ReportStreamTestDatabaseContainer.testDatabaseAccess, azureEventService, + ReportService( + ReportGraph(ReportStreamTestDatabaseContainer.testDatabaseAccess), + ReportStreamTestDatabaseContainer.testDatabaseAccess + ) + ) ) } diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt index eae5b63fe2c..4acfecbf826 100644 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverFilterIntegrationTests.kt @@ -39,6 +39,7 @@ import gov.cdc.prime.router.azure.observability.event.ItemEventData import gov.cdc.prime.router.azure.observability.event.ReportEventData import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties +import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService import gov.cdc.prime.router.azure.observability.event.ReportStreamItemEvent import gov.cdc.prime.router.common.TestcontainersUtils import gov.cdc.prime.router.common.UniversalPipelineTestUtils @@ -214,7 +215,14 @@ class FHIRReceiverFilterIntegrationTests : Logging { settings, db = ReportStreamTestDatabaseContainer.testDatabaseAccess, reportService = ReportService(ReportGraph(ReportStreamTestDatabaseContainer.testDatabaseAccess)), - azureEventService = azureEventService + azureEventService = azureEventService, + reportStreamEventService = ReportStreamEventService( + ReportStreamTestDatabaseContainer.testDatabaseAccess, azureEventService, + ReportService( + ReportGraph(ReportStreamTestDatabaseContainer.testDatabaseAccess), + ReportStreamTestDatabaseContainer.testDatabaseAccess + ) + ) ) } diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverIntegrationTests.kt deleted file mode 100644 index 525e53905d2..00000000000 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRReceiverIntegrationTests.kt +++ /dev/null @@ -1,881 +0,0 @@ -package gov.cdc.prime.router.fhirengine.azure - -import assertk.assertThat -import assertk.assertions.hasSize -import assertk.assertions.isEmpty -import assertk.assertions.isEqualTo -import assertk.assertions.isEqualToIgnoringGivenProperties -import assertk.assertions.isNull -import gov.cdc.prime.reportstream.shared.BlobUtils -import gov.cdc.prime.router.FileSettings -import gov.cdc.prime.router.MimeFormat -import gov.cdc.prime.router.Sender -import gov.cdc.prime.router.Topic -import gov.cdc.prime.router.azure.ActionHistory -import gov.cdc.prime.router.azure.BlobAccess -import gov.cdc.prime.router.azure.QueueAccess -import gov.cdc.prime.router.azure.SubmissionTableService -import gov.cdc.prime.router.azure.TableAccess -import gov.cdc.prime.router.azure.WorkflowEngine -import gov.cdc.prime.router.azure.db.Tables -import gov.cdc.prime.router.azure.db.enums.ActionLogType -import gov.cdc.prime.router.azure.db.enums.TaskAction -import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile -import gov.cdc.prime.router.azure.observability.event.InMemoryAzureEventService -import gov.cdc.prime.router.azure.observability.event.ReportEventData -import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName -import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties -import gov.cdc.prime.router.azure.observability.event.ReportStreamReportEvent -import gov.cdc.prime.router.common.TestcontainersUtils -import gov.cdc.prime.router.common.UniversalPipelineTestUtils.csvSenderWithNoTransform -import gov.cdc.prime.router.common.UniversalPipelineTestUtils.fhirSenderWithNoTransform -import gov.cdc.prime.router.common.UniversalPipelineTestUtils.fhirSenderWithNoTransformInactive -import gov.cdc.prime.router.common.UniversalPipelineTestUtils.hl7SenderWithNoTransform -import gov.cdc.prime.router.common.UniversalPipelineTestUtils.universalPipelineOrganization -import gov.cdc.prime.router.common.cleanHL7Record -import gov.cdc.prime.router.common.invalidMalformedFHIRRecord -import gov.cdc.prime.router.common.unparseableHL7Record -import gov.cdc.prime.router.common.validFHIRRecord1 -import gov.cdc.prime.router.db.ReportStreamTestDatabaseContainer -import gov.cdc.prime.router.db.ReportStreamTestDatabaseSetupExtension -import gov.cdc.prime.router.fhirengine.engine.FHIRReceiver -import gov.cdc.prime.router.history.DetailedActionLog -import gov.cdc.prime.router.history.DetailedReport -import gov.cdc.prime.router.unittest.UnitTestUtils -import gov.cdc.prime.router.version.Version -import io.mockk.clearAllMocks -import io.mockk.every -import io.mockk.mockkObject -import io.mockk.unmockkAll -import io.mockk.verify -import org.jooq.impl.DSL -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import org.testcontainers.junit.jupiter.Container -import org.testcontainers.junit.jupiter.Testcontainers -import java.time.OffsetDateTime -import java.util.UUID -import kotlin.test.assertNotNull - -@Testcontainers -@ExtendWith(ReportStreamTestDatabaseSetupExtension::class) -class FHIRReceiverIntegrationTests { - - @Container - val azuriteContainer = TestcontainersUtils.createAzuriteContainer( - customImageName = "azurite_fhirreceiverintegration", - customEnv = mapOf( - "AZURITE_ACCOUNTS" to "devstoreaccount1:keydevstoreaccount1" - ) - ) - - private val azureEventService = InMemoryAzureEventService() - private lateinit var submissionTableService: SubmissionTableService - - private fun createFHIRFunctionsInstance(): FHIRFunctions { - val settings = FileSettings().loadOrganizations(universalPipelineOrganization) - val metadata = UnitTestUtils.simpleMetadata - val workflowEngine = WorkflowEngine - .Builder() - .metadata(metadata) - .settingsProvider(settings) - .databaseAccess(ReportStreamTestDatabaseContainer.testDatabaseAccess) - .build() - return FHIRFunctions(workflowEngine, databaseAccess = ReportStreamTestDatabaseContainer.testDatabaseAccess) - } - - private fun createFHIRReceiver(): FHIRReceiver { - val settings = FileSettings().loadOrganizations(universalPipelineOrganization) - val metadata = UnitTestUtils.simpleMetadata - return FHIRReceiver( - metadata, - settings, - ReportStreamTestDatabaseContainer.testDatabaseAccess, - azureEventService = azureEventService, - submissionTableService = submissionTableService - ) - } - - private fun generateReceiveQueueMessage( - reportId: String, - blobURL: String, - blobContents: String, - sender: Sender, - headers: Map, - ): String { - val headersStringMap = headers.entries.joinToString(separator = ",\n") { (key, value) -> - """"$key": "$value"""" - } - val headersString = "[\"java.util.LinkedHashMap\",{$headersStringMap}]" - - return """{"type":"receive-fhir","blobURL":"$blobURL", - "digest":"${BlobUtils.digestToString(BlobUtils.sha256Digest(blobContents.toByteArray()))}", - "blobSubFolderName":"${sender.fullName}","reportId":"$reportId","headers":$headersString} - """.trimIndent() - } - - @BeforeEach - fun beforeEach() { - clearAllMocks() - mockkObject(QueueAccess) - every { QueueAccess.sendMessage(any(), any()) } returns "" - mockkObject(BlobAccess) - every { BlobAccess getProperty "defaultBlobMetadata" } returns getBlobContainerMetadata() - mockkObject(BlobAccess.BlobContainerMetadata) - every { BlobAccess.BlobContainerMetadata.build(any(), any()) } returns getBlobContainerMetadata() - - mockkObject(TableAccess) - every { TableAccess.getConnectionString() } returns getConnString() - - submissionTableService = SubmissionTableService.getInstance() - submissionTableService.reset() - } - - @AfterEach - fun afterEach() { - unmockkAll() - } - - private fun getBlobContainerMetadata(): BlobAccess.BlobContainerMetadata = BlobAccess.BlobContainerMetadata( - "container1", - getConnString() - ) - - private fun getConnString(): String { - @Suppress("ktlint:standard:max-line-length") - return """DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=keydevstoreaccount1;BlobEndpoint=http://${azuriteContainer.host}:${azuriteContainer.getMappedPort(10000)}/devstoreaccount1;QueueEndpoint=http://${azuriteContainer.host}:${azuriteContainer.getMappedPort(10001)}/devstoreaccount1;TableEndpoint=http://${azuriteContainer.host}:${azuriteContainer.getMappedPort(10002)}/devstoreaccount1;""" - } - - @Test - fun `should handle inactive sender gracefully`() { - val receivedReportContents = - listOf(validFHIRRecord1) - .joinToString("\n") - val receiveBlobUrl = BlobAccess.uploadBlob( - "receive/happy-path.fhir", - receivedReportContents.toByteArray(), - getBlobContainerMetadata() - ) - - val reportId = UUID.randomUUID() - val headers = mapOf( - "content-type" to "application/fhir+ndjson;test", - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to fhirSenderWithNoTransformInactive.fullName, - "content-length" to "100" - ) - - val receiveQueueMessage = generateReceiveQueueMessage( - reportId.toString(), - receiveBlobUrl, - receivedReportContents, - fhirSenderWithNoTransformInactive, - headers = headers - ) - - val fhirFunctions = createFHIRFunctionsInstance() - - fhirFunctions.process( - receiveQueueMessage, - 1, - createFHIRReceiver(), - ActionHistory(TaskAction.receive) - ) - - ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - - val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()) - .from(Tables.ACTION_LOG) - .where(Tables.ACTION_LOG.REPORT_ID.eq(reportId)) - .and(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) - .fetchInto(DetailedActionLog::class.java) - - assertThat(actionLogs.first()).transform { it.detail.message } - .isEqualTo("Sender has customer status INACTIVE: phd.fhir-elr-no-transform-inactive") - - val reportFile = DSL.using(txn).select(Tables.REPORT_FILE.asterisk()) - .from(Tables.REPORT_FILE) - .where(Tables.REPORT_FILE.REPORT_ID.eq(reportId)) - .fetchInto(DetailedReport::class.java) - - assertThat(actionLogs.count()).isEqualTo(1) - assertThat(reportFile.count()).isEqualTo(1) - } - - verify(exactly = 0) { - QueueAccess.sendMessage(any(), any()) - } - - val tableRow = submissionTableService.getSubmission(reportId.toString(), "Accepted") - - assertNotNull(tableRow) - assertThat(tableRow.detail).isEqualTo( - "[Sender has customer status INACTIVE: phd.fhir-elr-no-transform-inactive]" - ) - assertThat(tableRow.bodyURL).isEqualTo(receiveBlobUrl) - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!).hasSize(1) - val event = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!.last() as ReportStreamReportEvent - assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - Topic.FULL_ELR, - receiveBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(event.params).isEqualTo( - mapOf( - ReportStreamEventProperties.ITEM_FORMAT to MimeFormat.FHIR, - ReportStreamEventProperties.SENDER_NAME to fhirSenderWithNoTransformInactive.fullName, - ReportStreamEventProperties.FILE_LENGTH to headers["content-length"], - ReportStreamEventProperties.SENDER_IP to headers["x-azure-clientip"], - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString() - ) - ) - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_NOT_PROCESSABLE]!!).hasSize(1) - val notProcessableEvent = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_NOT_PROCESSABLE]!!.last() as ReportStreamReportEvent - assertThat(notProcessableEvent.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - Topic.FULL_ELR, - receiveBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(notProcessableEvent.params).isEqualTo( - mapOf( - ReportStreamEventProperties.PROCESSING_ERROR to - "Submitted report was either empty or could not be parsed.", - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString() - ) - ) - } - - @Test - fun `should handle sender not found gracefully`() { - val submissionMessageContents = validFHIRRecord1 - val submissionBlobUrl = "http://anyblob.com" - - val reportId = UUID.randomUUID() - val headers = mapOf( - "content-type" to "application/fhir+ndjson;test", - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to "unknown_sender", - "content-length" to "100" - ) - - val receiveQueueMessage = generateReceiveQueueMessage( - reportId.toString(), - submissionBlobUrl, - submissionMessageContents, - fhirSenderWithNoTransformInactive, - headers = headers - ) - - val fhirFunctions = createFHIRFunctionsInstance() - - fhirFunctions.process( - receiveQueueMessage, - 1, - createFHIRReceiver(), - ActionHistory(TaskAction.receive) - ) - - ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()) - .from(Tables.ACTION_LOG) - .where(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) - .fetchInto(DetailedActionLog::class.java) - - assertThat(actionLogs).isEmpty() - - val reportFile = DSL.using(txn).select(Tables.REPORT_FILE.asterisk()) - .from(Tables.REPORT_FILE) - .where(Tables.REPORT_FILE.REPORT_ID.eq(reportId)) - .fetchInto(DetailedReport::class.java) - - assertThat(reportFile).isEmpty() - } - - verify(exactly = 0) { - QueueAccess.sendMessage(any(), any()) - } - - val tableRow = submissionTableService.getSubmission( - reportId.toString(), - "Rejected" - ) - - assertNotNull(tableRow) - assertThat(tableRow.detail).isEqualTo("Sender not found matching client_id: unknown_sender") - assertThat(tableRow.bodyURL).isEqualTo(submissionBlobUrl) - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_NOT_RECEIVABLE]!!).hasSize(1) - val event = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_NOT_RECEIVABLE]!!.last() as ReportStreamReportEvent - assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - null, - submissionBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(event.params).isEqualTo( - mapOf( - ReportStreamEventProperties.PROCESSING_ERROR to - "Sender is not found in matching client id: unknown_sender.", - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString() - ) - ) - } - - @Test - fun `should successfully process valid FHIR message`() { - val receivedReportContents = - listOf(validFHIRRecord1) - .joinToString("\n") - val receiveBlobUrl = BlobAccess.uploadBlob( - "receive/happy-path.fhir", - receivedReportContents.toByteArray(), - getBlobContainerMetadata() - ) - - val reportId = UUID.randomUUID() - val headers = mapOf( - "content-type" to "application/fhir+ndjson;test", - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to fhirSenderWithNoTransform.fullName, - "content-length" to "100" - ) - - val receiveQueueMessage = generateReceiveQueueMessage( - reportId.toString(), - receiveBlobUrl, - receivedReportContents, - fhirSenderWithNoTransform, - headers = headers - ) - - val fhirFunctions = createFHIRFunctionsInstance() - - fhirFunctions.process( - receiveQueueMessage, - 1, - createFHIRReceiver(), - ActionHistory(TaskAction.receive) - ) - - ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()) - .from(Tables.ACTION_LOG) - .where(Tables.ACTION_LOG.REPORT_ID.eq(reportId)) - .and(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) - .fetchInto(DetailedActionLog::class.java) - - assertThat(actionLogs).isEmpty() - - val reportFile = DSL.using(txn).select(Tables.REPORT_FILE.asterisk()) - .from(Tables.REPORT_FILE) - .where(Tables.REPORT_FILE.REPORT_ID.eq(reportId)) - .fetchInto(ReportFile::class.java) - - assertThat(reportFile).hasSize(1) - reportFile.first().apply { - assertThat(nextAction).isEqualTo(TaskAction.convert) - assertThat(receivingOrg).isEqualTo(null) - assertThat(receivingOrgSvc).isEqualTo(null) - assertThat(schemaName).isEqualTo("None") - assertThat(schemaTopic).isEqualTo(Topic.FULL_ELR) - assertThat(bodyFormat).isEqualTo("FHIR") - assertThat(sendingOrg).isEqualTo("phd") - assertThat(sendingOrgClient).isEqualTo("fhir-elr-no-transform") - } - } - - verify(exactly = 1) { - QueueAccess.sendMessage(any(), any()) - } - - val tableRow = submissionTableService.getSubmission( - reportId.toString(), - "Accepted" - ) - - assertNotNull(tableRow) - assertThat(tableRow.bodyURL).isEqualTo(receiveBlobUrl) - assertThat(tableRow.detail).isNull() - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!).hasSize(1) - val event = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!.last() as ReportStreamReportEvent - assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - Topic.FULL_ELR, - receiveBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(event.params).isEqualTo( - mapOf( - ReportStreamEventProperties.ITEM_FORMAT to MimeFormat.FHIR, - ReportStreamEventProperties.SENDER_NAME to fhirSenderWithNoTransform.fullName, - ReportStreamEventProperties.FILE_LENGTH to headers["content-length"], - ReportStreamEventProperties.SENDER_IP to headers["x-azure-clientip"], - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString() - ) - ) - } - - @Test - fun `should successfully process valid HL7 message`() { - val receivedReportContents = - listOf(cleanHL7Record) - .joinToString("\n") - val receiveBlobUrl = BlobAccess.uploadBlob( - "receive/happy-path.hl7", - receivedReportContents.toByteArray(), - getBlobContainerMetadata() - ) - - val reportId = UUID.randomUUID() - val headers = mapOf( - "content-type" to "application/hl7-v2;test", - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to hl7SenderWithNoTransform.fullName, - "content-length" to "100" - ) - val receiveQueueMessage = generateReceiveQueueMessage( - reportId.toString(), - receiveBlobUrl, - receivedReportContents, - hl7SenderWithNoTransform, - headers - ) - - val fhirFunctions = createFHIRFunctionsInstance() - - fhirFunctions.process( - receiveQueueMessage, - 1, - createFHIRReceiver(), - ActionHistory(TaskAction.receive) - ) - - ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()) - .from(Tables.ACTION_LOG) - .where(Tables.ACTION_LOG.REPORT_ID.eq(reportId)) - .and(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) - .fetchInto(DetailedActionLog::class.java) - - assertThat(actionLogs).isEmpty() - - val reportFile = DSL.using(txn).select(Tables.REPORT_FILE.asterisk()) - .from(Tables.REPORT_FILE) - .where(Tables.REPORT_FILE.REPORT_ID.eq(reportId)) - .fetchInto(ReportFile::class.java) - - assertThat(reportFile).hasSize(1) - reportFile.first().apply { - assertThat(nextAction).isEqualTo(TaskAction.convert) - assertThat(receivingOrg).isEqualTo(null) - assertThat(receivingOrgSvc).isEqualTo(null) - assertThat(schemaName).isEqualTo("None") - assertThat(schemaTopic).isEqualTo(Topic.FULL_ELR) - assertThat(bodyFormat).isEqualTo("HL7") - assertThat(sendingOrg).isEqualTo("phd") - assertThat(sendingOrgClient).isEqualTo("hl7-elr-no-transform") - } - } - - verify(exactly = 1) { - QueueAccess.sendMessage(any(), any()) - } - - val tableRow = submissionTableService.getSubmission( - reportId.toString(), - "Accepted" - ) - - assertNotNull(tableRow) - assertThat(tableRow.bodyURL).isEqualTo(receiveBlobUrl) - assertThat(tableRow.detail).isNull() - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!).hasSize(1) - val event = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!.last() as ReportStreamReportEvent - assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - Topic.FULL_ELR, - receiveBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(event.params).isEqualTo( - mapOf( - ReportStreamEventProperties.ITEM_FORMAT to MimeFormat.HL7, - ReportStreamEventProperties.SENDER_NAME to hl7SenderWithNoTransform.fullName, - ReportStreamEventProperties.FILE_LENGTH to headers["content-length"], - ReportStreamEventProperties.SENDER_IP to headers["x-azure-clientip"], - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString() - ) - ) - } - - @Test - fun `test process invalid FHIR message`() { - val invalidReceivedReportContents = - listOf(invalidMalformedFHIRRecord) - .joinToString("\n") - val receiveBlobUrl = BlobAccess.uploadBlob( - "receive/fail-path.fhir", - invalidReceivedReportContents.toByteArray(), - getBlobContainerMetadata() - ) - - val reportId = UUID.randomUUID() - val headers = mapOf( - "content-type" to "application/fhir+ndjson;test", - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to fhirSenderWithNoTransform.fullName, - "content-length" to "100" - ) - - val receiveQueueMessage = generateReceiveQueueMessage( - reportId.toString(), - receiveBlobUrl, - invalidReceivedReportContents, - fhirSenderWithNoTransform, - headers = headers - ) - - val fhirFunctions = createFHIRFunctionsInstance() - - fhirFunctions.process( - receiveQueueMessage, - 1, - createFHIRReceiver(), - ActionHistory(TaskAction.receive) - ) - - ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()) - .from(Tables.ACTION_LOG) - .where(Tables.ACTION_LOG.REPORT_ID.eq(reportId)) - .and(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) - .fetchInto(DetailedActionLog::class.java) - - assertThat(actionLogs.count()).isEqualTo(1) - assertThat(actionLogs.first().detail.message).isEqualTo("1: Unable to parse FHIR data.") - - val reportFile = DSL.using(txn).select(Tables.REPORT_FILE.asterisk()) - .from(Tables.REPORT_FILE) - .where(Tables.REPORT_FILE.REPORT_ID.eq(reportId)) - .fetchInto(ReportFile::class.java) - - assertThat(reportFile).hasSize(1) - reportFile.first().apply { - assertThat(nextAction).isEqualTo(TaskAction.convert) - assertThat(receivingOrg).isEqualTo(null) - assertThat(receivingOrgSvc).isEqualTo(null) - assertThat(schemaName).isEqualTo("None") - assertThat(schemaTopic).isEqualTo(Topic.FULL_ELR) - assertThat(bodyFormat).isEqualTo("FHIR") - assertThat(sendingOrg).isEqualTo("phd") - assertThat(sendingOrgClient).isEqualTo("fhir-elr-no-transform") - } - } - - verify(exactly = 0) { - QueueAccess.sendMessage(any(), any()) - } - - val tableRow = submissionTableService.getSubmission( - reportId.toString(), - "Accepted" - ) - - assertNotNull(tableRow) - assertThat(tableRow.bodyURL).isEqualTo(receiveBlobUrl) - assertThat(tableRow.detail).isEqualTo("[1: Unable to parse FHIR data.]") - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!).hasSize(1) - val event = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!.last() as ReportStreamReportEvent - assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - Topic.FULL_ELR, - receiveBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(event.params).isEqualTo( - mapOf( - ReportStreamEventProperties.ITEM_FORMAT to MimeFormat.FHIR, - ReportStreamEventProperties.SENDER_NAME to fhirSenderWithNoTransform.fullName, - ReportStreamEventProperties.FILE_LENGTH to headers["content-length"], - ReportStreamEventProperties.SENDER_IP to headers["x-azure-clientip"], - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString() - ) - ) - } - - @Test - fun `test process invalid HL7 message`() { - val invalidReceivedReportContents = - listOf(unparseableHL7Record) - .joinToString("\n") - val receiveBlobUrl = BlobAccess.uploadBlob( - "receive/fail-path.hl7", - invalidReceivedReportContents.toByteArray(), - getBlobContainerMetadata() - ) - - val reportId = UUID.randomUUID() - val headers = mapOf( - "content-type" to "application/hl7-v2;test", - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to hl7SenderWithNoTransform.fullName, - "content-length" to "100" - ) - - val receiveQueueMessage = generateReceiveQueueMessage( - reportId.toString(), - receiveBlobUrl, - invalidReceivedReportContents, - hl7SenderWithNoTransform, - headers = headers - ) - - val fhirFunctions = createFHIRFunctionsInstance() - - fhirFunctions.process( - receiveQueueMessage, - 1, - createFHIRReceiver(), - ActionHistory(TaskAction.receive) - ) - - ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()) - .from(Tables.ACTION_LOG) - .where(Tables.ACTION_LOG.REPORT_ID.eq(reportId)) - .and(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) - .fetchInto(DetailedActionLog::class.java) - - assertThat(actionLogs.count()).isEqualTo(2) - - val reportFile = DSL.using(txn).select(Tables.REPORT_FILE.asterisk()) - .from(Tables.REPORT_FILE) - .where(Tables.REPORT_FILE.REPORT_ID.eq(reportId)) - .fetchInto(ReportFile::class.java) - - assertThat(reportFile).hasSize(1) - reportFile.first().apply { - assertThat(nextAction).isEqualTo(TaskAction.convert) - assertThat(receivingOrg).isEqualTo(null) - assertThat(receivingOrgSvc).isEqualTo(null) - assertThat(schemaName).isEqualTo("None") - assertThat(schemaTopic).isEqualTo(Topic.FULL_ELR) - assertThat(bodyFormat).isEqualTo("HL7") - assertThat(sendingOrg).isEqualTo("phd") - assertThat(sendingOrgClient).isEqualTo("hl7-elr-no-transform") - } - } - - verify(exactly = 0) { - QueueAccess.sendMessage(any(), any()) - } - - val tableRow = submissionTableService.getSubmission( - reportId.toString(), - "Accepted" - ) - - assertNotNull(tableRow) - assertThat(tableRow.bodyURL).isEqualTo(receiveBlobUrl) - assertThat(tableRow.detail).isEqualTo("[Failed to parse message, Failed to parse message]") - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!).hasSize(1) - val event = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_RECEIVED]!!.last() as ReportStreamReportEvent - assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - Topic.FULL_ELR, - receiveBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(event.params).isEqualTo( - mapOf( - ReportStreamEventProperties.ITEM_FORMAT to MimeFormat.HL7, - ReportStreamEventProperties.SENDER_NAME to hl7SenderWithNoTransform.fullName, - ReportStreamEventProperties.FILE_LENGTH to headers["content-length"], - ReportStreamEventProperties.SENDER_IP to headers["x-azure-clientip"], - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString() - ) - ) - } - - @Test - fun `test process CSV message`() { - val invalidReceivedReportContents = - listOf(unparseableHL7Record) - .joinToString("\n") - val receiveBlobUrl = BlobAccess.uploadBlob( - "receive/fail-path.hl7", - invalidReceivedReportContents.toByteArray(), - getBlobContainerMetadata() - ) - - val reportId = UUID.randomUUID() - val headers = mapOf( - "content-type" to "application/hl7-v2;test", - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to csvSenderWithNoTransform.fullName, - "content-length" to "100" - ) - - val receiveQueueMessage = generateReceiveQueueMessage( - reportId.toString(), - receiveBlobUrl, - invalidReceivedReportContents, - csvSenderWithNoTransform, - headers = headers - ) - - val fhirFunctions = createFHIRFunctionsInstance() - - var exception: Exception? = null - try { - fhirFunctions.process( - receiveQueueMessage, - 1, - createFHIRReceiver(), - ActionHistory(TaskAction.receive) - ) - } catch (e: Exception) { - exception = e - } - - assertThat(exception!!.javaClass.name).isEqualTo("java.lang.IllegalStateException") - - ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val actionLogs = DSL.using(txn).select(Tables.ACTION_LOG.asterisk()) - .from(Tables.ACTION_LOG) - .where(Tables.ACTION_LOG.REPORT_ID.eq(reportId)) - .and(Tables.ACTION_LOG.TYPE.eq(ActionLogType.error)) - .fetchInto(DetailedActionLog::class.java) - - assertThat(actionLogs.count()).isEqualTo(0) - - val reportFile = DSL.using(txn).select(Tables.REPORT_FILE.asterisk()) - .from(Tables.REPORT_FILE) - .where(Tables.REPORT_FILE.REPORT_ID.eq(reportId)) - .fetchInto(ReportFile::class.java) - - assertThat(reportFile).isEmpty() - } - - verify(exactly = 0) { - QueueAccess.sendMessage(any(), any()) - } - - val tableRow = submissionTableService.getSubmission( - reportId.toString(), - "Rejected" - ) - - assertNotNull(tableRow) - assertThat(tableRow.bodyURL).isEqualTo(receiveBlobUrl) - assertThat(tableRow.detail).isEqualTo("[Unsupported sender format: CSV]") - - assertThat(azureEventService.reportStreamEvents[ReportStreamEventName.REPORT_NOT_PROCESSABLE]!!).hasSize(1) - val event = - azureEventService - .reportStreamEvents[ReportStreamEventName.REPORT_NOT_PROCESSABLE]!!.last() as ReportStreamReportEvent - assertThat(event.reportEventData).isEqualToIgnoringGivenProperties( - ReportEventData( - reportId, - null, - emptyList(), - null, - receiveBlobUrl, - TaskAction.receive, - OffsetDateTime.now(), - Version.commitId - ), - ReportEventData::timestamp - ) - assertThat(event.params).isEqualTo( - mapOf( - ReportStreamEventProperties.ITEM_FORMAT to MimeFormat.CSV, - ReportStreamEventProperties.SENDER_NAME to csvSenderWithNoTransform.fullName, - ReportStreamEventProperties.FILE_LENGTH to headers["content-length"], - ReportStreamEventProperties.SENDER_IP to headers["x-azure-clientip"], - ReportStreamEventProperties.REQUEST_PARAMETERS to headers.toString(), - ReportStreamEventProperties.PROCESSING_ERROR to "Unsupported sender format CSV." - ) - ) - } -} \ No newline at end of file diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRTranslatorIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRTranslatorIntegrationTests.kt index c45b9fe3e28..df375e85ed3 100644 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRTranslatorIntegrationTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/azure/FHIRTranslatorIntegrationTests.kt @@ -22,6 +22,7 @@ import gov.cdc.prime.router.azure.db.enums.TaskAction import gov.cdc.prime.router.azure.db.tables.Task import gov.cdc.prime.router.azure.observability.event.AzureEventService import gov.cdc.prime.router.azure.observability.event.InMemoryAzureEventService +import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService import gov.cdc.prime.router.cli.tests.CompareData import gov.cdc.prime.router.common.TestcontainersUtils import gov.cdc.prime.router.common.UniversalPipelineTestUtils @@ -102,7 +103,14 @@ class FHIRTranslatorIntegrationTests : Logging { metadata, settings, reportService = ReportService(ReportGraph(ReportStreamTestDatabaseContainer.testDatabaseAccess)), - azureEventService = azureEventService + azureEventService = azureEventService, + reportStreamEventService = ReportStreamEventService( + ReportStreamTestDatabaseContainer.testDatabaseAccess, azureEventService, + ReportService( + ReportGraph(ReportStreamTestDatabaseContainer.testDatabaseAccess), + ReportStreamTestDatabaseContainer.testDatabaseAccess + ) + ) ) } diff --git a/prime-router/src/test/kotlin/fhirengine/engine/CustomFhirPathFunctionTest.kt b/prime-router/src/test/kotlin/fhirengine/engine/CustomFhirPathFunctionTest.kt index 95f92e40785..7080940cfc9 100644 --- a/prime-router/src/test/kotlin/fhirengine/engine/CustomFhirPathFunctionTest.kt +++ b/prime-router/src/test/kotlin/fhirengine/engine/CustomFhirPathFunctionTest.kt @@ -2,6 +2,7 @@ package gov.cdc.prime.router.fhirengine.engine import assertk.assertFailure import assertk.assertThat +import assertk.assertions.contains import assertk.assertions.isEqualTo import assertk.assertions.isNotNull import assertk.assertions.isNull @@ -323,4 +324,85 @@ class CustomFhirPathFunctionTest { (result[0] as StringType).value ).isEqualTo("Shasta") } + + @Test + fun `test getting state from zip code - one state`() { + val testTable = Table.create( + "zip-code-data", + StringColumn.create("state_fips", "40", "48", "6"), + StringColumn.create("state", "Oklahoma", "Texas", "California"), + StringColumn.create("state_abbr", "OK", "TX", "CA"), + StringColumn.create("zipcode", "73949", "73949", "92356"), + StringColumn.create("county", "Texas", "Sherman", "San Bernardino"), + StringColumn.create("city", "Texhoma", "", "Lucerne valley") + + ) + val testLookupTable = LookupTable(name = "zip-code-data", table = testTable) + + mockkConstructor(Metadata::class) + every { anyConstructed().findLookupTable("zip-code-data") } returns testLookupTable + mockkObject(Metadata) + every { Metadata.getInstance() } returns UnitTestUtils.simpleMetadata + + val result = CustomFhirPathFunctions().getStateFromZipCode(mutableListOf(StringType("92356"))) + + assertThat( + (result[0] as StringType).value + ).isEqualTo("CA") + } + + @Test + fun `test getting state from zip code - multiple states`() { + val testTable = Table.create( + "zip-code-data", + StringColumn.create("state_fips", "40", "48", "6"), + StringColumn.create("state", "Oklahoma", "Texas", "California"), + StringColumn.create("state_abbr", "OK", "TX", "CA"), + StringColumn.create("zipcode", "73949", "73949", "92356"), + StringColumn.create("county", "Texas", "Sherman", "San Bernardino"), + StringColumn.create("city", "Texhoma", "", "Lucerne valley") + + ) + val testLookupTable = LookupTable(name = "zip-code-data", table = testTable) + + mockkConstructor(Metadata::class) + every { anyConstructed().findLookupTable("zip-code-data") } returns testLookupTable + mockkObject(Metadata) + every { Metadata.getInstance() } returns UnitTestUtils.simpleMetadata + + val result = CustomFhirPathFunctions().getStateFromZipCode(mutableListOf(StringType("73949"))) + + assertThat( + (result[0] as StringType).value + ).contains("OK") + assertThat( + (result[0] as StringType).value + ).contains("TX") + } + + @Test + fun `test getting state from zip code - no matching state found`() { + val testTable = Table.create( + "zip-code-data", + StringColumn.create("state_fips", "40", "48", "6"), + StringColumn.create("state", "Oklahoma", "Texas", "California"), + StringColumn.create("state_abbr", "OK", "TX", "CA"), + StringColumn.create("zipcode", "73949", "73949", "92356"), + StringColumn.create("county", "Texas", "Sherman", "San Bernardino"), + StringColumn.create("city", "Texhoma", "", "Lucerne valley") + + ) + val testLookupTable = LookupTable(name = "zip-code-data", table = testTable) + + mockkConstructor(Metadata::class) + every { anyConstructed().findLookupTable("zip-code-data") } returns testLookupTable + mockkObject(Metadata) + every { Metadata.getInstance() } returns UnitTestUtils.simpleMetadata + + val result = CustomFhirPathFunctions().getStateFromZipCode(mutableListOf(StringType("79902"))) + + assertThat( + (result[0] as StringType).value + ).isEqualTo("") + } } \ No newline at end of file diff --git a/prime-router/src/test/kotlin/fhirengine/engine/FHIRReceiverTests.kt b/prime-router/src/test/kotlin/fhirengine/engine/FHIRReceiverTests.kt deleted file mode 100644 index 8edc912f5f1..00000000000 --- a/prime-router/src/test/kotlin/fhirengine/engine/FHIRReceiverTests.kt +++ /dev/null @@ -1,273 +0,0 @@ -package gov.cdc.prime.router.fhirengine.engine - -import assertk.assertThat -import assertk.assertions.hasSize -import assertk.assertions.isEqualTo -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.prime.reportstream.shared.Submission -import gov.cdc.prime.router.ActionLog -import gov.cdc.prime.router.ActionLogDetail -import gov.cdc.prime.router.ActionLogger -import gov.cdc.prime.router.CovidSender -import gov.cdc.prime.router.CustomerStatus -import gov.cdc.prime.router.DeepOrganization -import gov.cdc.prime.router.FileSettings -import gov.cdc.prime.router.InvalidParamMessage -import gov.cdc.prime.router.Metadata -import gov.cdc.prime.router.MimeFormat -import gov.cdc.prime.router.Organization -import gov.cdc.prime.router.Receiver -import gov.cdc.prime.router.Schema -import gov.cdc.prime.router.SettingsProvider -import gov.cdc.prime.router.Topic -import gov.cdc.prime.router.azure.ActionHistory -import gov.cdc.prime.router.azure.BlobAccess -import gov.cdc.prime.router.azure.DatabaseAccess -import gov.cdc.prime.router.azure.SubmissionTableService -import gov.cdc.prime.router.azure.db.enums.TaskAction -import gov.cdc.prime.router.azure.db.tables.pojos.Action -import gov.cdc.prime.router.common.cleanHL7Record -import gov.cdc.prime.router.report.ReportService -import io.mockk.clearAllMocks -import io.mockk.every -import io.mockk.mockk -import io.mockk.mockkClass -import io.mockk.mockkObject -import io.mockk.spyk -import io.mockk.unmockkAll -import io.mockk.verify -import org.jooq.tools.jdbc.MockConnection -import org.jooq.tools.jdbc.MockDataProvider -import org.jooq.tools.jdbc.MockResult -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import java.util.UUID -import kotlin.test.Test - -class FHIRReceiverTest { - - // Common mock objects and setup - val dataProvider = MockDataProvider { emptyArray() } - val connection = MockConnection(dataProvider) - val accessSpy = spyk(DatabaseAccess(connection)) - val blobMock = mockkClass(BlobAccess::class) - val reportService: ReportService = mockk() - private val submissionTableService: SubmissionTableService = mockk() - - val oneOrganization = DeepOrganization( - "co-phd", - "test", - Organization.Jurisdiction.FEDERAL, - receivers = listOf(Receiver("elr", "co-phd", Topic.TEST, CustomerStatus.INACTIVE, "one")) - ) - val settings = FileSettings().loadOrganizations(oneOrganization) - val one = Schema(name = "None", topic = Topic.FULL_ELR, elements = emptyList()) - val metadata = Metadata(schema = one) - - private fun makeFhirReceiver(metadata: Metadata, settings: SettingsProvider): FHIRReceiver { - return FHIRReceiver( - metadata, - settings, - accessSpy, - blobMock, - reportService = reportService, - submissionTableService = submissionTableService - ) - -// FHIREngine.Builder().metadata(metadata).settingsProvider(settings).databaseAccess(accessSpy) -// .reportService(reportService).blobAccess(blobMock).build(taskAction) - } - - @BeforeEach - fun reset() { - clearAllMocks() - } - - @AfterEach - fun tearDown() { - unmockkAll() - } - - data class FHIRTestSetup( - val engine: FHIRReceiver, - val actionLogger: ActionLogger, - val actionHistory: ActionHistory, - val message: FhirReceiveQueueMessage, - ) - - private fun setupMocksForProcessingTest( - clientId: String, - contentType: String, - customerStatus: CustomerStatus, - hasErrors: Boolean, - reportID: UUID = UUID.randomUUID(), - - ): FHIRTestSetup { - mockkObject(BlobAccess) - val actionHistory = mockk() - val actionLogger = mockk() - val sender = CovidSender( - "Test Sender", - "test", - MimeFormat.HL7, - schemaName = "one", - customerStatus = customerStatus - ) - - val engine = spyk(makeFhirReceiver(metadata, settings)) - val message = mockk(relaxed = true) - val action = Action() - action.actionName = TaskAction.receive - - val headers = mapOf( - "x-azure-clientip" to "0.0.0.0", - "payloadname" to "test_message", - "client_id" to clientId, - "content-type" to contentType - ) - - every { message.headers } returns headers - every { message.reportId } returns reportID - every { actionLogger.hasErrors() } returns hasErrors - every { actionLogger.setReportId(any()) } returns actionLogger - every { actionLogger.error(any()) } returns Unit - every { engine.settings.findSender(any()) } returns sender - every { actionHistory.trackActionResult(any()) } returns Unit - every { actionHistory.trackActionParams(any()) } returns Unit - every { actionHistory.trackActionSenderInfo(any(), any()) } returns Unit - every { actionHistory.trackExternalInputReport(any(), any()) } returns Unit - every { actionHistory.trackLogs(any>()) } returns Unit - every { submissionTableService.insertSubmission(any()) } returns Unit - every { actionHistory.action } returns action - every { BlobAccess.downloadBlob(any(), any()) }.returns(cleanHL7Record) - - return FHIRTestSetup(engine, actionLogger, actionHistory, message) - } - - @Test - fun `test handle sender not found`() { - val fhirSetup = - setupMocksForProcessingTest( - "unknown_client_id", - "application/hl7-v2;test", - CustomerStatus.ACTIVE, - true - ) - val engine = fhirSetup.engine - val queueMessage = fhirSetup.message - val actionLogger = ActionLogger() - val actionHistory = fhirSetup.actionHistory - - every { engine.settings.findSender(any()) } returns null - - accessSpy.transact { txn -> - engine.run(queueMessage, actionLogger, actionHistory, txn) - } - - assertThat(actionLogger.errors).hasSize(0) - - val reportId = queueMessage.reportId.toString() - val blobURL = queueMessage.blobURL - verify(exactly = 1) { - Submission( - reportId, - "Rejected", - blobURL, - "Sender not found matching client_id: unknown_client_id" - ) - submissionTableService.insertSubmission(any()) - } - } - - @Test - fun `test handle inactive sender`() { - val fhirSetup = - setupMocksForProcessingTest( - "known_client_id", - "application/hl7-v2;test", - CustomerStatus.INACTIVE, - true - ) - val engine = fhirSetup.engine - val queueMessage = fhirSetup.message - val actionLogger = ActionLogger() - val actionHistory = fhirSetup.actionHistory - - accessSpy.transact { txn -> - engine.run(queueMessage, actionLogger, actionHistory, txn) - } - - assertThat(actionLogger.errors).hasSize(1) - - assertThat( - actionLogger.errors[0].equals( - actionLogger.errors[0].equals( - InvalidParamMessage("Sender has customer status INACTIVE: unknown_client_id") - ) - ) - ) - - verify(exactly = 1) { - submissionTableService.insertSubmission(any()) - actionHistory.trackActionResult(HttpStatus.NOT_ACCEPTABLE) - actionHistory.trackActionSenderInfo("test.Test Sender", "test_message") - } - } - - @Test - fun `test successful processing`() { - val reportID = UUID.randomUUID() - val fhirSetup = - setupMocksForProcessingTest( - "known_client_id", - "application/hl7-v2;test", - CustomerStatus.ACTIVE, - false, - reportID - ) - val engine = fhirSetup.engine - val queueMessage = fhirSetup.message - val actionLogger = fhirSetup.actionLogger - val actionHistory = fhirSetup.actionHistory - every { actionLogger.errors } returns emptyList() - - accessSpy.transact { txn -> - engine.run(queueMessage, actionLogger, actionHistory, txn) - } - - verify(exactly = 1) { - actionHistory.trackActionResult(HttpStatus.CREATED) - actionHistory.trackActionSenderInfo("test.Test Sender", "test_message") - actionHistory.trackExternalInputReport(any(), any()) - submissionTableService.insertSubmission(any()) - } - } - - @Test - fun `test invalid MIME type`() { - val fhirSetup = - setupMocksForProcessingTest( - "known_client_id", - "invalid/mime-type", - CustomerStatus.ACTIVE, - true - ) - val engine = fhirSetup.engine - val queueMessage = fhirSetup.message - val actionLogger = ActionLogger() - val actionHistory = fhirSetup.actionHistory - - var exception: Exception? = null - try { - accessSpy.transact { txn -> - engine.run(queueMessage, actionLogger, actionHistory, txn) - } - } catch (e: Exception) { - exception = e - } - - assertThat(exception!!.javaClass.name).isEqualTo("java.lang.IllegalArgumentException") - assertThat(actionLogger.errors).hasSize(1) - assertThat(actionLogger.errors[0].detail.message).isEqualTo("Unexpected MIME type invalid/mime-type.") - } -} \ No newline at end of file diff --git a/prime-router/src/test/kotlin/fhirengine/engine/FhirConverterTests.kt b/prime-router/src/test/kotlin/fhirengine/engine/FhirConverterTests.kt index 6b48d2e7260..33731c0e5af 100644 --- a/prime-router/src/test/kotlin/fhirengine/engine/FhirConverterTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/engine/FhirConverterTests.kt @@ -28,6 +28,7 @@ import gov.cdc.prime.router.Topic import gov.cdc.prime.router.azure.ActionHistory import gov.cdc.prime.router.azure.BlobAccess import gov.cdc.prime.router.azure.DatabaseAccess +import gov.cdc.prime.router.azure.SubmissionTableService import gov.cdc.prime.router.azure.db.enums.TaskAction import gov.cdc.prime.router.azure.db.tables.pojos.Action import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile @@ -84,6 +85,7 @@ class FhirConverterTests { val connection = MockConnection(dataProvider) val accessSpy = spyk(DatabaseAccess(connection)) val blobMock = mockkClass(BlobAccess::class) + val mockSubmissionTableService = mockk() val reportService: ReportService = mockk() val oneOrganization = DeepOrganization( "co-phd", @@ -137,7 +139,8 @@ class FhirConverterTests { private fun makeFhirEngine(metadata: Metadata, settings: SettingsProvider, taskAction: TaskAction): FHIREngine { return FHIREngine.Builder().metadata(metadata).settingsProvider(settings).databaseAccess(accessSpy) - .reportService(reportService).blobAccess(blobMock).build(taskAction) + .reportService(reportService).blobAccess(blobMock) + .submissionTableService(mockSubmissionTableService).build(taskAction) } @BeforeEach @@ -559,10 +562,10 @@ class FhirConverterTests { mockkObject(BlobAccess) val engine = spyk(makeFhirEngine(metadata, settings, TaskAction.process) as FHIRConverter) val actionLogger = ActionLogger() - val mockMessage = mockk(relaxed = true) - every { mockMessage.topic } returns Topic.FULL_ELR every { BlobAccess.downloadBlob(any(), any()) } returns "" - val bundles = engine.process(MimeFormat.FHIR, mockMessage, actionLogger) + val bundles = engine.process( + MimeFormat.FHIR, "", "", Topic.FULL_ELR, actionLogger + ) assertThat(bundles).isEmpty() assertThat(actionLogger.errors.map { it.detail.message }).contains("Provided raw data is empty.") } @@ -584,7 +587,9 @@ class FhirConverterTests { every { mockMessage.topic } returns Topic.FULL_ELR every { mockMessage.reportId } returns UUID.randomUUID() every { BlobAccess.downloadBlob(any(), any()) } returns simpleHL7 - val bundles = engine.process(MimeFormat.HL7, mockMessage, actionLogger) + val bundles = engine.process( + MimeFormat.HL7, "", "", Topic.FULL_ELR, actionLogger + ) assertThat(bundles).isEmpty() assertThat( actionLogger.errors.map { @@ -598,11 +603,10 @@ class FhirConverterTests { mockkObject(BlobAccess) val engine = spyk(makeFhirEngine(metadata, settings, TaskAction.process) as FHIRConverter) val actionLogger = ActionLogger() - val mockMessage = mockk(relaxed = true) - every { mockMessage.topic } returns Topic.FULL_ELR - every { mockMessage.reportId } returns UUID.randomUUID() every { BlobAccess.downloadBlob(any(), any()) } returns "test,1,2" - val bundles = engine.process(MimeFormat.CSV, mockMessage, actionLogger) + val bundles = engine.process( + MimeFormat.CSV, "", "", Topic.FULL_ELR, actionLogger + ) assertThat(bundles).isEmpty() assertThat(actionLogger.errors.map { it.detail.message }) .contains("Received unsupported report format: CSV") @@ -613,11 +617,8 @@ class FhirConverterTests { mockkObject(BlobAccess) val engine = spyk(makeFhirEngine(metadata, settings, TaskAction.process) as FHIRConverter) val actionLogger = ActionLogger() - val mockMessage = mockk(relaxed = true) - every { mockMessage.topic } returns Topic.FULL_ELR - every { mockMessage.reportId } returns UUID.randomUUID() every { BlobAccess.downloadBlob(any(), any()) } returns "{\"id\":}" - val processedItems = engine.process(MimeFormat.FHIR, mockMessage, actionLogger) + val processedItems = engine.process(MimeFormat.FHIR, "", "", Topic.FULL_ELR, actionLogger) assertThat(processedItems).hasSize(1) assertThat(processedItems.first().bundle).isNull() assertThat(actionLogger.errors.map { it.detail.message }).contains( @@ -647,7 +648,9 @@ class FhirConverterTests { every { mockMessage.topic } returns Topic.FULL_ELR every { mockMessage.reportId } returns UUID.randomUUID() every { BlobAccess.downloadBlob(any(), any()) } returns "{\"id\":\"1\", \"resourceType\":\"Bundle\"}" - val processedItems = engine.process(MimeFormat.FHIR, mockMessage, actionLogger) + val processedItems = engine.process( + MimeFormat.FHIR, "", "", Topic.FULL_ELR, actionLogger + ) assertThat(processedItems).hasSize(1) assertThat(processedItems.first().bundle).isNull() assertThat(actionLogger.errors.map { it.detail.message }).contains( @@ -666,7 +669,7 @@ class FhirConverterTests { every { BlobAccess.downloadBlob(any(), any()) } returns unparseableHL7 - val processedItems = engine.process(MimeFormat.HL7, mockMessage, actionLogger) + val processedItems = engine.process(MimeFormat.HL7, "", "", Topic.FULL_ELR, actionLogger) assertThat(processedItems).hasSize(1) assertThat(processedItems.first().bundle).isNull() assertThat( @@ -701,7 +704,7 @@ class FhirConverterTests { every { BlobAccess.downloadBlob(any(), any()) } returns simpleHL7 - val processedItems = engine.process(MimeFormat.HL7, mockMessage, actionLogger) + val processedItems = engine.process(MimeFormat.HL7, "", "", Topic.FULL_ELR, actionLogger) assertThat(processedItems).hasSize(1) assertThat(processedItems.first().bundle).isNull() @Suppress("ktlint:standard:max-line-length") @@ -730,7 +733,7 @@ class FhirConverterTests { every { BlobAccess.downloadBlob(any(), any()) } returns simpleHL7 - val processedItems = engine.process(MimeFormat.HL7, mockMessage, actionLogger) + val processedItems = engine.process(MimeFormat.HL7, "", "", Topic.FULL_ELR, actionLogger) assertThat(processedItems).hasSize(1) assertThat(processedItems.first().bundle).isNull() assertThat( @@ -756,14 +759,14 @@ class FhirConverterTests { } returns """{\"id\":} {"id":"1", "resourceType":"Bundle"} """.trimMargin() - val processedItems = engine.process(MimeFormat.FHIR, mockMessage, actionLogger) + val processedItems = engine.process(MimeFormat.FHIR, "", "", Topic.FULL_ELR, actionLogger) assertThat(processedItems).hasSize(2) assertThat(actionLogger.errors.map { it.detail.message }).contains( @Suppress("ktlint:standard:max-line-length") "Item 1 in the report was not parseable. Reason: exception while parsing FHIR: HAPI-1861: Failed to parse JSON encoded FHIR content: Unexpected character ('\\' (code 92)): was expecting double-quote to start field name\n at [line: 1, column: 2]" ) - val bundles2 = engine.process(MimeFormat.FHIR, mockMessage, actionLogger, false) + val bundles2 = engine.process(MimeFormat.FHIR, "", "", Topic.FULL_ELR, actionLogger, false) assertThat(bundles2).hasSize(0) assertThat(actionLogger.errors.map { it.detail.message }).contains( @Suppress("ktlint:standard:max-line-length") @@ -783,7 +786,7 @@ class FhirConverterTests { every { BlobAccess.downloadBlob(any(), any()) } returns simpleHL7 - val bundles = engine.process(MimeFormat.HL7, mockMessage, actionLogger) + val bundles = engine.process(MimeFormat.HL7, "", "", Topic.FULL_ELR, actionLogger) assertThat(bundles).hasSize(1) assertThat(actionLogger.errors).isEmpty() } @@ -803,7 +806,7 @@ class FhirConverterTests { every { BlobAccess.downloadBlob(any(), any()) } returns simpleHL7 + "\n" + simpleHL7 + "\n" + simpleHL7 - val bundles = engine.process(MimeFormat.HL7, mockMessage, actionLogger) + val bundles = engine.process(MimeFormat.HL7, "", "", Topic.FULL_ELR, actionLogger) assertThat(bundles).hasSize(3) assertThat(actionLogger.errors).isEmpty() @@ -834,7 +837,7 @@ class FhirConverterTests { every { BlobAccess.downloadBlob(any(), any()) } returns simpleHL7 - val bundles = engine.process(MimeFormat.HL7, mockMessage, actionLogger) + val bundles = engine.process(MimeFormat.HL7, "", "", Topic.FULL_ELR, actionLogger) assertThat(bundles).hasSize(1) assertThat(actionLogger.errors).isEmpty() } diff --git a/prime-router/src/test/kotlin/fhirengine/engine/FhirDestinationFilterTests.kt b/prime-router/src/test/kotlin/fhirengine/engine/FhirDestinationFilterTests.kt index 964308c2ed6..8dd47bbaf17 100644 --- a/prime-router/src/test/kotlin/fhirengine/engine/FhirDestinationFilterTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/engine/FhirDestinationFilterTests.kt @@ -236,13 +236,14 @@ class FhirDestinationFilterTests { ) private fun makeFhirEngine(metadata: Metadata, settings: SettingsProvider): FHIREngine { - val rootReport = mockk() + val rootReport = mockk(relaxed = true) every { rootReport.reportId } returns submittedId every { rootReport.sendingOrg } returns "sendingOrg" every { rootReport.sendingOrgClient } returns "sendingOrgClient" every { reportServiceMock.getRootReport(any()) } returns rootReport every { reportServiceMock.getRootReports(any()) } returns listOf(rootReport) every { reportServiceMock.getRootItemIndex(any(), any()) } returns 1 + every { accessSpy.fetchReportFile(any()) } returns rootReport return FHIREngine.Builder() .metadata(metadata) diff --git a/shared/src/main/kotlin/gov/cdc/prime/reportstream/shared/QueueMessage.kt b/shared/src/main/kotlin/gov/cdc/prime/reportstream/shared/QueueMessage.kt index f10cd20d744..06146f7bf86 100644 --- a/shared/src/main/kotlin/gov/cdc/prime/reportstream/shared/QueueMessage.kt +++ b/shared/src/main/kotlin/gov/cdc/prime/reportstream/shared/QueueMessage.kt @@ -68,7 +68,7 @@ interface QueueMessage { /** * Constant for receive queue on UP */ - const val elrReceiveQueueName = "elr-fhir-receive" + const val elrSubmissionConvertQueueName = "elr-fhir-convert-submission" /** * Constant for convert queue on UP @@ -152,7 +152,7 @@ interface QueueMessage { ) : QueueMessage, ReportInformation, ReceiveInformation { - override val messageQueueName = elrReceiveQueueName + override val messageQueueName = elrSubmissionConvertQueueName } /** diff --git a/submissions/src/main/resources/application.yml b/submissions/src/main/resources/application.yml index c75d070b3fa..b4794b2d1fa 100644 --- a/submissions/src/main/resources/application.yml +++ b/submissions/src/main/resources/application.yml @@ -13,7 +13,7 @@ azure: storage: connection-string: ${AZURE_STORAGE_CONNECTION_STRING:DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;} container-name: ${AZURE_STORAGE_CONTAINER_NAME:reports} - queue-name: ${AZURE_STORAGE_QUEUE_NAME:elr-fhir-receive} + queue-name: ${AZURE_STORAGE_QUEUE_NAME:elr-fhir-convert-submission} table-name: ${AZURE_STORAGE_TABLE_NAME:submission} allowed: