feat: use changed-files instead of jitterbit for deployment workflow [DE-231]

.github/workflows/deploy_cta_dbt_models.yaml

@@ -1,4 +1,13 @@
-name: Deploy Google Composer DAGs
+name: Deploy dbt to Google Composer
+
+# BEWARE DUPLICATE FILES IN YOUR DEV DEPLOYS:
+# After you deploy a branch to dev, the workflow will always be comparing your branch to main -
+# so if (for example) you create a file and then delete it after triggering the dev deploy,
+# the workflow will deploy the file when you create it, but it WILL NOT subsequently delete it.
+# This is because the run triggered by the commit deleting the file will compare `main` to your branch,
+# neither of which contain the file you added and deleted, so no action will be taken.
+# Since your earlier commit deployed the file, it will continue to linger in dev.
+# (This quirk does not apply to prod deploys since the workflow only runs once, when the PR is merged.)
 
 on:
     workflow_call:
@@ -17,83 +26,186 @@ env:
 
 jobs:
   deploy:
-    name: Deploy Google Composer DAGs
+    name: Deploy dbt to Google Composer
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     permissions:
       contents: 'read'
       id-token: 'write'
     steps:
+
       #############
       ### setup ###
       #############
 
       # Checkout repo, this needs to happen first
       - name: Checkout
         uses: actions/checkout@v3
+
       # Keyless Auth!
       - id: 'auth'
         name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v0
         with:
           workload_identity_provider: ${{ env.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ env.GOOGLE_SERVICE_ACCOUNT }}
-      # Set up gcloud pointing to our airbyte-prod project
+
+      # Set up gcloud pointing to CTA project
       - name: Set up Cloud SDK
         uses: 'google-github-actions/setup-gcloud@v0'
         with:
           project_id: ${{ env.GOOGLE_PROJECT_ID }}
 
-      ##########################
-      ### deploy DAG changes ###
-      ##########################
+      ###########################
+      ### deploy file changes ###
+      ###########################
 
-      # Get list of files in last commit
-      - id: 'files'
-        name: Get list of changed files
-        uses: jitterbit/get-changed-files@v1
-        continue-on-error: true
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v35
         with:
-          format: 'csv'
+          include_all_old_new_renamed_files: "true"
+
+      - name: List all new/modified/deleted/renamed files
+        run: |
+
+          echo "NEW:"
+          for file in ${{ steps.changed-files.outputs.added_files }}; do
+            if [[ ${#file} > 0 ]]; then
+              echo "$file was added."
+            else
+              echo "No files added."
+            fi
+          done
+
+          echo "MODIFIED:"
+          for file in ${{ steps.changed-files.outputs.modified_files }}; do
+            if [[ ${#file} > 0 ]]; then
+              echo "$file was modified."
+            else
+              echo "No files modified."
+            fi
+          done
+
+          echo "RENAMED:"
+          for file in ${{ steps.changed-files.outputs.all_old_new_renamed_files }}; do
+            if [[ ${#file} > 0 ]]; then
+              export old_name="$(echo "$file" | cut -d ',' -f 1)"
+              export new_name="$(echo "$file" | cut -d ',' -f 2)"
+              echo "$old_name was renamed to $new_name."
+            else
+              echo "No files renamed."
+            fi
+          done
+
+          echo "DELETED:"
+          for file in ${{ steps.changed-files.outputs.deleted_files }}; do
+            if [[ ${#file} > 0 ]]; then
+              echo "$file was deleted."
+            else
+              echo "No files deleted."
+            fi
+          done
 
-      # Deploy each dbt file that was added/modified
-      - name: Deploy added or modified dbt models
-        id: 'add_modify_dbt'
+      # Deploy new, modified, or renamed files (new file name)
+      - name: Deploy new files
+        id: 'deploy_new'
         run: |
-          mapfile -d ',' -t added_modified_files < <(printf '%s,' '${{ steps.files.outputs.added_modified }}')
-          mapfile -d ',' -t renamed_files < <(printf '%s,' '${{ steps.files.outputs.renamed }}')
-          added_modified_renamed_files+=( "${added_modified_files[@]}" "${renamed_files[@]}" )
-          for file in "${added_modified_renamed_files[@]}"; do
-            # Search for the filename
-            if [[ $(find dbt-cta -name $(basename ${file})) ]] && [[ ${file} == *.sql ]]; then
-              # Get full path of DAG file
-              file_name=$(realpath ${file});
-              path_to_file=$(dirname ${file});
-              gcloud composer environments storage dags import \
-                --project=$GOOGLE_PROJECT_ID \
-                --location=us-central1 \
+          for file in ${{ steps.changed-files.outputs.added_files }}; do
+            if [[ ${#file} > 0 ]]; then
+              # Search for the filename
+              if [[ $(find dbt-cta -name $(basename ${file})) ]] && [[ ${file} == *.sql || ${file} == *.yml || ${file} == *.yaml ]]; then
+                # Get full path of file
+                file_name=$(realpath ${file});
+                path_to_file=$(dirname ${file});
+                gcloud composer environments storage dags import \
+                  --project=$GOOGLE_PROJECT_ID \
+                  --location=us-central1 \
+                  --environment=$GOOGLE_COMPOSER_ENVIRONMENT \
+                  --destination=cta-dags/${path_to_file} \
+                  --source=${file_name}
+                echo "Deployed new file: $(basename ${file}))"
+              else
+                echo "No new files to deploy."
+              fi
+            fi
+          done
+
+      - name: Deploy modified files
+        id: 'deploy_modified'
+        run: |
+          for file in ${{ steps.changed-files.outputs.modified_files }}; do
+            if [[ ${#file} > 0 ]]; then
+              # Search for the filename
+              if [[ $(find dbt-cta -name $(basename ${file})) ]] && [[ ${file} == *.sql || ${file} == *.yml || ${file} == *.yaml ]]; then
+                # Get full path of file
+                file_name=$(realpath ${file});
+                path_to_file=$(dirname ${file});
+                gcloud composer environments storage dags import \
+                  --project=$GOOGLE_PROJECT_ID \
+                  --location=us-central1 \
+                  --environment=$GOOGLE_COMPOSER_ENVIRONMENT \
+                  --destination=cta-dags/${path_to_file} \
+                  --source=${file_name}
+                echo "Deployed modified file: $(basename ${file}))"
+              else
+                echo "No modified files to deploy."
+              fi
+            fi
+          done
+
+      - name: Deploy renamed files
+        id: 'deploy_renamed'
+        run: |
+          for file in ${{ steps.changed-files.outputs.renamed_files }}; do
+            if [[ ${#file} > 0 ]]; then
+              # Search for the filename
+              if [[ $(find dbt-cta -name $(basename ${file})) ]] && [[ ${file} == *.sql || ${file} == *.yml || ${file} == *.yaml ]]; then
+                # Get full path of file
+                file_name=$(realpath ${file});
+                path_to_file=$(dirname ${file});
+                gcloud composer environments storage dags import \
+                  --project=$GOOGLE_PROJECT_ID \
+                  --location=us-central1 \
+                  --environment=$GOOGLE_COMPOSER_ENVIRONMENT \
+                  --destination=cta-dags/${path_to_file} \
+                  --source=${file_name}
+                echo "Deployed renamed file: $(basename ${file}))"
+              else
+                echo "No renamed files to deploy."
+              fi
+            fi
+          done
+
+      # Delete any files that were removed or renamed (old file name)
+      - name: Delete renamed files
+        id: 'delete_renamed'
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_old_new_renamed_files }}; do
+            export old_name="$(echo "$file" | cut -d ',' -f 1)"
+            export new_name="$(echo "$file" | cut -d ',' -f 2)"
+            if [[ ${#file} > 0 ]]; then
+              gcloud composer environments storage dags delete cta-dags/${old_name} \
                 --environment=$GOOGLE_COMPOSER_ENVIRONMENT \
-                --destination=cta-dags/${path_to_file} \
-                --source=${file_name}
-              echo "Deployed file: $(basename ${file}))"
-            else
-              echo "Skipping $(basename ${file})) (no new dbt to deploy)."
+                --project=$GOOGLE_PROJECT_ID \
+                --location=us-central1
+              echo "Deleted renamed file: ${old_name} (renamed to ${new_name})."
+              else
+                echo "No renamed files to delete."
             fi
           done
 
-      # Delete any dbt that was removed
-      - name: Delete removed dbt
-        id: 'remove_dbt'
+      - name: Delete removed files
+        id: 'delete_removed'
         run: |
-          mapfile -d ',' -t removed_files < <(printf '%s,' '${{ steps.files.outputs.removed }}')
-          for file in "${removed_files[@]}"; do
+          for file in ${{ steps.changed-files.outputs.deleted_files }}; do
             if [[ ${#file} > 0 ]]; then
               gcloud composer environments storage dags delete cta-dags/${file} \
                 --environment=$GOOGLE_COMPOSER_ENVIRONMENT \
                 --project=$GOOGLE_PROJECT_ID \
                 --location=us-central1
               echo "Deleted file: ${file}"
-            else
-              echo "No dbt to remove for file removed: ${file}"
+              else
+                echo "No files to delete"
             fi
           done