Merge branch 'master' into platform/jamie/auth-poc

CDCgov · Sep 6, 2024 · 1663c1a · 1663c1a
2 parents a9cb906 + 7f93cf9
commit 1663c1a
Show file tree

Hide file tree

Showing 274 changed files with 4,046 additions and 26,446 deletions.
diff --git a/.environment/frontend/run-frontend.sh b/.environment/frontend/run-frontend.sh
@@ -19,13 +19,12 @@ function modified_check() {
 cd "$DIR/../../frontend-react";
 FRONTEND_DIR=$(pwd);
 
-if [ ! -f "$HOOK_FILE" ]; then
-    echo "$FRONTEND_DIR/$HOOK_FILE does not exist. Please make sure you run yarn first."
-    exit 1
-fi
-
 modified_check
 if [[ ${isModified} == 1 ]]; then
+    if [ ! -f "$HOOK_FILE" ]; then
+        echo "$FRONTEND_DIR/$HOOK_FILE does not exist. Please make sure you run yarn first."
+        exit 1
+    fi
     "./$HOOK_FILE"
 fi
 RC=$?

diff --git a/.environment/gitleaks/gitleaks-config.toml b/.environment/gitleaks/gitleaks-config.toml
@@ -203,7 +203,10 @@ title = "PRIME ReportStream Gitleaks Configuration"
             'authority\", \"extension\"',                                                  # FHIR extension URL also shows up in normal FHIR test data
             'ApiKeyCredential\(\"flexion\"',
             'authType: \"two-legged\"',
+            '\"apiKey\"',
+            'api-key\" to \"oracle123\"',
             'Authorization-Type: \"username/password\"',
+            'cdctiautomated_sa'
         ]
         paths = [
             '.terraform/modules/',

diff --git a/.github/ISSUE_TEMPLATE/up-receiver-migration-validate-test-data.md b/.github/ISSUE_TEMPLATE/up-receiver-migration-validate-test-data.md
@@ -2,10 +2,11 @@
 name: UP Migrating Receiver - Validate Test Data
 about: This is the fourth step in migrating a STLT to the UP 
 title: "[name of STLT] - UP Migration - Validate Test Data"
-labels: onboarding-ops, receiver
+labels: onboarding-ops, receiver, engineering, needs-refinement
 assignees: ''
 
 ---
+[STLT] = name of STLT
 
 ### User Story:
 As a developer, I want the [STLT] to validate test messages in staging, so that they can receive data in the format they expect.
@@ -27,4 +28,16 @@ As a developer, I want the [STLT] to validate test messages in staging, so that
 
 ### Acceptance Criteria 
 - [ ] Created and sent data using Simple Report
-- [ ] Confirmed with STLT that data passes validation 
+- [ ] Confirmed with STLT that data passes validation
+- [ ] When finished, tag Mikaelah, Chris, and Glenn in a slack post stating:
+> [STLT] "Testing in Staging" is complete and will be moving on to "Monitor in Production"
+>
+> Connection Type: [REST, SFTP, SOAP, etc]
+>
+> Conditions tested: [List of Conditions]
+>
+> Positives set up to receive: [List of Conditions]
+>
+> Negatives set up to receive: [List of Conditions]
+>
+> Format receiving: [HL7 or FHIR]
diff --git a/.github/actions/az-cost/action.yml b/.github/actions/az-cost/action.yml
@@ -25,7 +25,7 @@ runs:
     - name: Install azure-cost-cli
       shell: bash
       run: |
-        dotnet new tool-manifest
+        dotnet new tool-manifest --force
         dotnet tool install azure-cost-cli
 
     - name: Run azure-cost-cli for daily totals
@@ -38,6 +38,16 @@ runs:
         dotnet azure-cost accumulatedCost -g ${{ inputs.rg }} -t custom --from $(date --date='${{ inputs.days_ago }} days ago' '+%m/%d/%Y') --to $(date --date='1 days ago' '+%m/%d/%Y') -o csv >> $GITHUB_OUTPUT
         echo "$EOF" >> $GITHUB_OUTPUT
 
+    - name: Run azure-cost-cli for daily totals by resource
+      if: inputs.total_format == 'dailyByResource'
+      id: run_azure_cost_daily_by_resource
+      shell: bash
+      run: |
+        EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
+        echo "RESULT<<$EOF" >> $GITHUB_OUTPUT
+        dotnet azure-cost dailyCosts -g ${{ inputs.rg }} -t custom --from $(date --date='${{ inputs.days_ago }} days ago' '+%m/%d/%Y') --to $(date --date='1 days ago' '+%m/%d/%Y') -o csv --exclude-meter-details --dimension ConsumedService >> $GITHUB_OUTPUT
+        echo "$EOF" >> $GITHUB_OUTPUT
+
     - name: Run azure-cost-cli for total
       if: inputs.total_format == 'summary'
       id: run_azure_cost_summary
@@ -52,5 +62,5 @@ runs:
       run: |
         EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
         echo "RESULT<<$EOF" >> $GITHUB_OUTPUT
-        echo -e "${{ steps.run_azure_cost_daily.outputs.RESULT }}${{ steps.run_azure_cost_summary.outputs.RESULT }}" >> $GITHUB_OUTPUT
+        echo -e "${{ steps.run_azure_cost_daily.outputs.RESULT }}${{ steps.run_azure_cost_daily_by_resource.outputs.RESULT }}${{ steps.run_azure_cost_summary.outputs.RESULT }}" >> $GITHUB_OUTPUT
         echo "$EOF" >> $GITHUB_OUTPUT
diff --git a/.github/actions/build-backend/action.yml b/.github/actions/build-backend/action.yml
@@ -100,7 +100,7 @@ runs:
         shell: bash
 
     - name: Publish Unit Test Results
-      uses: EnricoMi/publish-unit-test-result-action/linux@567cc7f8dcea3eba5da355f6ebc95663310d8a07
+      uses: EnricoMi/publish-unit-test-result-action/linux@82082dac68ad6a19d980f8ce817e108b9f496c2a
       if: >
         always() &&
         github.event.sender.login != 'dependabot[bot]' &&
@@ -151,7 +151,7 @@ runs:
       shell: bash
 
     - name: Publish Integration Test Results
-      uses: EnricoMi/publish-unit-test-result-action/linux@567cc7f8dcea3eba5da355f6ebc95663310d8a07
+      uses: EnricoMi/publish-unit-test-result-action/linux@82082dac68ad6a19d980f8ce817e108b9f496c2a
       if: >
         always() &&
         github.event.sender.login != 'dependabot[bot]' &&
@@ -194,7 +194,7 @@ runs:
 
     - name: Upload Artifact
       if: inputs.upload-build == 'true'
-      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
         name: prime-router-build-${{ inputs.version }}
         path: prime-router/prime-router-build.tar.gz

diff --git a/.github/actions/build-frontend/action.yml b/.github/actions/build-frontend/action.yml
@@ -113,7 +113,7 @@ runs:
       shell: bash
 
     - name: Store E2E Results
-      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       if: always() && ${{ !startsWith(inputs.env-name, 'trialfrontend') }}
       with:
         name: e2e-data
@@ -182,7 +182,7 @@ runs:
       run: tar -czf static-website-react.tar.gz build
 
     - name: Upload frontend artifact
-      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
         name: static-website-react-${{ inputs.version }}
         path: frontend-react/static-website-react.tar.gz

diff --git a/.github/actions/build-submissions/action.yml b/.github/actions/build-submissions/action.yml
@@ -39,7 +39,7 @@ runs:
         distribution: "temurin"
         cache: "gradle"
 
-    - uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582
+    - uses: gradle/actions/setup-gradle@16bf8bc8fe830fa669c3c9f914d3eb147c629707
 
     - name: Lint
       if: inputs.run-integration-tests == 'true'

diff --git a/.github/actions/sonarcloud/action.yml b/.github/actions/sonarcloud/action.yml
@@ -20,8 +20,8 @@ runs:
         SONAR_TOKEN: ${{ inputs.sonar-token }}
       with:
         args: >
-          -Dsonar.coverage.exclusions=prime-router/src/test/**,prime-router/src/testIntegration/**,prime-router/src/main/kotlin/cli/tests/**,frontend-react/**/__mocks__/**,frontend-react/**/mocks/**,frontend-react/**/*.test.*
-          -Dsonar.cpd.exclusions=frontend-react/**/*.test.*,prime-router/src/test/**,prime-router/src/testIntegration/**,prime-router/src/main/kotlin/cli/tests/**
+          -Dsonar.coverage.exclusions=prime-router/src/test/**,prime-router/src/testIntegration/**,prime-router/src/main/kotlin/cli/tests/**,frontend-react/**/__mocks__/**,frontend-react/**/mocks/**,frontend-react/**/*.test.*,submissions/src/test/**
+          -Dsonar.cpd.exclusions=frontend-react/**/*.test.*,prime-router/src/test/**,prime-router/src/testIntegration/**,prime-router/src/main/kotlin/cli/tests/**,submissions/src/test/**
           -Dsonar.sources=frontend-react/src,prime-router/src,submissions/src,shared/src
           -Dsonar.projectKey=CDCgov_prime-data-hub
           -Dsonar.organization=cdcgov

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -115,6 +115,10 @@ updates:
     directory: "/.github/actions/az-cost"
     schedule:
       interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/.github/actions/build-submissions"
+    schedule:
+      interval: "daily"
 
   # Frontend
   - package-ecosystem: "npm"

diff --git a/.github/workflows/build_hub.yml b/.github/workflows/build_hub.yml
@@ -80,8 +80,8 @@ jobs:
           ./gradlew :prime-router:primeCLI --args='validate-yaml --type Organizations --file settings/organizations.yml --dir settings/STLTs'
           ./gradlew :prime-router:primeCLI --args='validate-yaml --type FhirToFhirTransform --dir src/main/resources/metadata/fhir_transforms'
           ./gradlew :prime-router:primeCLI --args='validate-yaml --type FhirToHL7Mapping --dir src/main/resources/metadata/hl7_mapping'
-          ./gradlew :prime-router:primeCLI --args='validate-yaml --type HL7ToFhirMappingMessageTemplate --dir metadata/HL7/catchall/hl7/message metadata/HL7/v251-elr/hl7/message'
-          ./gradlew :prime-router:primeCLI --args='validate-yaml --type HL7ToFhirMappingResourceTemplate --dir metadata/HL7 --exclude-dir metadata/HL7/catchall/hl7/message metadata/HL7/v251-elr/hl7/message metadata/HL7/v251-elr/hl7/codesystem metadata/HL7/catchall/hl7/codesystem --exclude-file metadata/HL7/catchall/fhir/resourcemapping.yml metadata/HL7/v251-elr/fhir/resourcemapping.yml metadata/HL7/catchall/hl7/resource/Common.yml metadata/HL7/v251-elr/hl7/resource/Common.yml'
+          ./gradlew :prime-router:primeCLI --args='validate-yaml --type HL7ToFhirMappingMessageTemplate --dir metadata/HL7/catchall/hl7/message'
+          ./gradlew :prime-router:primeCLI --args='validate-yaml --type HL7ToFhirMappingResourceTemplate --dir metadata/HL7 --exclude-dir metadata/HL7/catchall/hl7/message metadata/HL7/catchall/hl7/codesystem --exclude-file metadata/HL7/catchall/fhir/resourcemapping.yml metadata/HL7/catchall/hl7/resource/Common.yml'
 
       - name: Check for Uncommited Schema Docs
         id: check_changes

diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml
@@ -4,33 +4,57 @@ on:
   schedule:
     # The workflow runs every day at 3PM
     - cron: "0 19 * * *"  #UTC-5
-
 jobs:
   export_cost_data:
     name: Export Azure Cost Data to Storage
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - rg: "prime-data-hub-staging"
+            env: "staging"
+            pwd_key: "POSTGRESQL_STAGING_PWD"
+            db: "pdhstaging-pgsql"
+          - rg: "prime-data-hub-prod"
+            env: "prod"
+            pwd_key: "POSTGRESQL_PROD_PWD"
+            db: "pdhprod-pgsql"
     runs-on: ubuntu-latest
     steps:
       - name: "Check out changes"
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       - name: Connect to VPN and login to Azure
         uses: ./.github/actions/vpn-azure
         with:
-          env-name: prod
+          env-name: ${{ matrix.env }}
           sp-creds: ${{ secrets.SERVICE_PRINCIPAL_CREDS }}
           tf-auth: false
       - name: Run Az Cost CLI
         id: az-cost
         uses: ./.github/actions/az-cost
         with:
-          rg: prime-data-hub-prod
+          rg: ${{ matrix.rg }}
           days_ago: 180
           total_format: daily
       - name: Get Az Cost Result
         run: |
           echo "${{ steps.az-cost.outputs.result }}" > costs_raw.csv
           touch costs.csv
-          sed 's/\r//' costs_raw.csv > costs.csv
-          cat ./costs.csv
+          sed 's/\r//' costs_raw.csv > costs_total.csv
+          cat ./costs_total.csv
+      - name: Run Az Cost CLI By Resource
+        id: az-cost-by-resource
+        uses: ./.github/actions/az-cost
+        with:
+          rg: ${{ matrix.rg }}
+          days_ago: 180
+          total_format: dailyByResource
+      - name: Get Az Cost Result By Resource
+        run: |
+          echo "${{ steps.az-cost-by-resource.outputs.result }}" > costs_by_res_raw.csv
+          touch costs_by_res.csv
+          sed 's/\r//' costs_by_res_raw.csv > costs_by_res.csv
+          cat ./costs_by_res.csv
       - name: Get runner ip
         id: runner_ip
         uses: ./.github/actions/runner-ip
@@ -39,15 +63,15 @@ jobs:
         id: add-db-access
         shell: bash
         run: |
-          az postgres server update -g prime-data-hub-prod -n pdhprod-pgsql --public-network-access "Enabled"
+          az postgres server update -g ${{ matrix.rg }} -n ${{ matrix.db }} --public-network-access "Enabled"
           sleep 10;
-          az postgres server firewall-rule create -g prime-data-hub-prod -s pdhprod-pgsql -n github_actions_runner \
+          az postgres server firewall-rule create -g ${{ matrix.rg }} -s ${{ matrix.db }} -n github_actions_runner \
             --start-ip-address ${{ steps.runner_ip.outputs.ip-address }} --end-ip-address ${{ steps.runner_ip.outputs.ip-address }}
       - name: Store cost data in DB
         uses: ./.github/actions/db-query
         with:
-          pass: ${{ secrets['POSTGRESQL_PROD_PWD'] }}
-          host: pdhprod-pgsql
+          pass: ${{ secrets[matrix.pwd_key] }}
+          host: ${{ matrix.db }}
           port: 5432
           user: prime
           database: prime_data_hub
@@ -60,12 +84,24 @@ jobs:
               Currency   varchar(3)
             );
             TRUNCATE Azure_Costs;
-            \copy Azure_Costs (Date, Cost, CostUsd, Currency) FROM './costs.csv' WITH DELIMITER ',' CSV HEADER;
+            \copy Azure_Costs (Date, Cost, CostUsd, Currency) FROM './costs_total.csv' WITH DELIMITER ',' CSV HEADER;
+
+            CREATE TABLE IF NOT EXISTS Azure_Resource_Costs (
+              Date       date,
+              Name       varchar(128),
+              Cost       NUMERIC(13, 8),
+              CostUsd    NUMERIC(13, 8),
+              Currency   varchar(3),
+              Tags       varchar(256)
+            );
+            TRUNCATE Azure_Resource_Costs;
+            \copy Azure_Resource_Costs (Date, Name, Cost, CostUsd, Currency, Tags) FROM './costs_by_res.csv' WITH DELIMITER ',' CSV HEADER;
+
       # Remove runner db access
       - name: Remove DB access
         if: always() && steps.add-db-access.outcome == 'success'
         shell: bash
         run: |
-          az postgres server firewall-rule delete -g prime-data-hub-prod -s pdhprod-pgsql  -n github_actions_runner --yes
+          az postgres server firewall-rule delete -g ${{ matrix.rg }} -s ${{ matrix.db }}  -n github_actions_runner --yes
           sleep 10;
-          az postgres server update -g prime-data-hub-prod -n pdhprod-pgsql --public-network-access "Disabled"
+          az postgres server update -g ${{ matrix.rg }} -n ${{ matrix.db }} --public-network-access "Disabled"
diff --git a/.github/workflows/frontend_ci.yml b/.github/workflows/frontend_ci.yml
@@ -114,7 +114,7 @@ jobs:
         run: yarn run test:e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
       - name: Upload blob report to GitHub Actions Artifacts
         if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
           name: frontend-e2e-data--shard-${{ matrix.shardIndex }}
           path: frontend-react/e2e-data
@@ -158,7 +158,7 @@ jobs:
           cache-dependency-path: frontend-react/yarn.lock
 
       - name: Download blob reports from GitHub Actions Artifacts
-        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
         with:
           path: frontend-react/e2e-data
           pattern: frontend-e2e-data--shard-*
@@ -170,7 +170,7 @@ jobs:
       - name: Merge folders
         run: mv ./playwright-report/* ./e2e-data/report && rmdir ./playwright-report && rm ./e2e-data/report/*.zip && rm -rf ./e2e-data/report/resources
       - name: Upload final e2e-data
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
           name: frontend-e2e-data--attempt-${{ github.run_attempt }}
           path: frontend-react/e2e-data

diff --git a/.github/workflows/release_to_github.yml b/.github/workflows/release_to_github.yml
@@ -83,13 +83,13 @@ jobs:
           fetch-depth: 0
       - name: Download router artifact
         if: ${{ needs.wf_vars.outputs.enable_builds == 'true' }}
-        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
         with:
           name: prime-router-build-${{ needs.wf_vars.outputs.version }}
           path: archives/
       - name: Download frontend artifact
         if: ${{ needs.wf_vars.outputs.enable_builds == 'true' }}
-        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
         with:
           name: static-website-react-${{ needs.wf_vars.outputs.version }}
           path: archives/

diff --git a/.github/workflows/restore_databases.yml b/.github/workflows/restore_databases.yml
@@ -311,7 +311,7 @@ jobs:
         if: |
           needs.pre_job.outputs.sink_env_name != 'prod' &&
           (${{ matrix.db }} == 'prime_data_hub' || ${{ matrix.db }} == 'prime_data_hub_candidate')
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
           name: clean_output_${{ matrix.db }}
           path: clean_output_${{ matrix.db }}.html

diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -52,7 +52,7 @@ jobs:
 
       - name: "Get changed files with yaml"
         id: changed-files-yaml
-        uses: tj-actions/changed-files@40853de9f8ce2d6cfdc73c1b96f14e22ba44aec4
+        uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
         with:
           files_yaml: |
             frontend:

diff --git a/.github/workflows/start_frontend_smoke.yml b/.github/workflows/start_frontend_smoke.yml
@@ -41,7 +41,7 @@ jobs:
           echo "::endgroup::"
       - name: "Store E2E Results"
         if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
          name: smoke-data
          path: frontend-react/e2e-data/

diff --git a/.github/workflows/validate_terraform.yml b/.github/workflows/validate_terraform.yml
@@ -48,7 +48,7 @@ jobs:
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
 
       - name: Run Checkov action
-        uses: bridgecrewio/checkov-action@1b813e8f72afe2b6263a6ea10c873707e21ebe44
+        uses: bridgecrewio/checkov-action@c9421864e014ef6b8acfa35d0bf3c7e52c13ab10
         with:
           directory: operations/app/terraform
           skip_check: CKV_AZURE_139,CKV_AZURE_137,CKV_AZURE_103,CKV_AZURE_104,CKV_AZURE_102,CKV_AZURE_130,CKV_AZURE_121,CKV_AZURE_67,CKV_AZURE_56,CKV_AZURE_17,CKV_AZURE_63,CKV_AZURE_18,CKV_AZURE_88,CKV_AZURE_65,CKV_AZURE_13,CKV_AZURE_66,CKV_AZURE_33,CKV_AZURE_35,CKV_AZURE_36,CKV_AZURE_98,CKV2_AZURE_1,CKV2_AZURE_15,CKV2_AZURE_21,CKV_AZURE_213,CKV_AZURE_59,CKV2_AZURE_33,CKV2_AZURE_32,CKV2_AZURE_28,CKV_AZURE_206,CKV_AZURE_42,CKV_AZURE_110,CKV_AZURE_109,CKV_AZURE_166,CKV2_AZURE_38,CKV2_AZURE_40,CKV2_AZURE_41,CKV_AZURE_235
diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,7 @@ hs_err_pid*
 **/build/**
 **/log/**
 **/dependency-reduced-pom.xml
+**/version.json
 
 # Frontend build output
 **/node_modules/**