feature/bull_mq_integration

.github/actions/argocd-action/action.yml

@@ -10,13 +10,16 @@ inputs:
   argocd_server:
     description: "ArgoCD Server"
     required: true
+  tg_svc_key:
+    description: "Twingate Key"
+    required: true
 runs:
   using: composite
   steps:
     - name: Setup Twingate
       uses: twingate/github-action@v1
       with:
-        service-key: ${{ secrets.TWINGATE_SERVICE_KEY_SECRET_NAME }}
+        service-key: ${{ inputs.tg_svc_key }}
 
     - name: Obtain ArgoCD JWT Token
       id: get_token
@@ -26,7 +29,7 @@ runs:
         ARGOCD_PASSWORD: ${{ inputs.argocd_password }}
         ARGOCD_SERVER: ${{ inputs.argocd_server }}
       run: |
-        TOKEN=$(curl -s -X POST "${ARGOCD_SERVER}/api/v1/session" \
+        TOKEN=$(curl -k --insecure -s -X POST "${ARGOCD_SERVER}/api/v1/session" \
           -d '{"username": "'"${ARGOCD_USERNAME}"'", "password": "'"${ARGOCD_PASSWORD}"'"}' \
           -H "Content-Type: application/json" | jq -r '.token')
         echo "ARGOCD_TOKEN=$TOKEN" >> $GITHUB_ENV

.github/workflows/db-ops.yaml

@@ -0,0 +1,179 @@
+name: New Database Operations
+
+on:
+  repository_dispatch:
+    types: [run-test-migration]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository_owner }}/workflows-service
+  SHORT_HASH: ${{ github.event.client_payload.environment == 'prod' && vars.PROD_WF_SHORT_SHA || github.event.client_payload.environment == 'sb' && vars.SB_WF_SHORT_SHA || vars.DEV_WF_SHORT_SHA }}
+  MIGRATION_ENV: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.environment }}
+  MIGRATION_REF: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref }}
+
+
+jobs:
+  build-and-push-ee-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    outputs: 
+      SUBMODULE_SHORT_HASH: ${{ steps.lastcommit.outputs.shorthash }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4 
+
+      - name: Checkout wf-data-migration
+        uses: actions/checkout@v4
+        with:
+          repository: ballerine-io/wf-data-migration
+          token: ${{ secrets.SUBMODULES_TOKEN }}
+          ref: ${{ env.MIGRATION_REF }}
+          fetch-depth: 1
+          path: services/workflows-service/prisma/data-migrations
+
+      - name: Get Latest Commit ID
+        id: lastcommit
+        uses: nmbgeek/github-action-get-latest-commit@main
+        with:
+          owner: ${{ github.repository_owner }}
+          token: ${{ secrets.SUBMODULES_TOKEN }}
+          repo: wf-data-migration
+          branch: ${{ env.MIGRATION_REF }}
+
+      - name: Cache Docker layers
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-docker-${{ hashFiles('**/Dockerfile') }}
+          restore-keys: |
+            ${{ runner.os }}-docker-${{ hashFiles('**/Dockerfile') }}
+            ${{ runner.os }}-docker-
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
+
+      - name: Log in to the container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for ee Docker images
+        id: eemeta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}-ee
+          tags: |
+            type=raw,value=${{ env.MIGRATION_ENV }}
+            type=raw,value=${{ env.SHORT_HASH }}-${{ steps.lastcommit.outputs.shorthash }}-${{ env.MIGRATION_ENV }}
+            type=raw,value=latest,enable=${{ env.MIGRATION_ENV == 'prod' }}
+            type=sha,format=short
+
+      - name: Build and push ee Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: services/workflows-service
+          file: services/workflows-service/Dockerfile.ee
+          platforms: linux/amd64
+          push: true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          tags: ${{ steps.eemeta.outputs.tags }}
+          build-args: |
+            "BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/workflows-service:${{ env.SHORT_HASH }}-${{ env.MIGRATION_ENV }}"
+
+  update-helm-chart:
+    runs-on: ubuntu-latest
+    needs: build-and-push-ee-image
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Checkout cloud-infra-config repository
+        uses: actions/checkout@v4
+        with:
+          repository: ballerine-io/cloud-infra-config
+          token: ${{ secrets.GIT_TOKEN }}
+          ref: main
+          fetch-depth: 1
+          sparse-checkout: |
+            kubernetes/helm/wf-service
+          sparse-checkout-cone-mode: true
+      - name: Check if values yaml file exists 
+        id: update_helm_check
+        shell: bash
+        run: |
+          if [ -f "kubernetes/helm/wf-service/${{ env.MIGRATION_ENV }}-custom-values.yaml" ]; then
+            echo "file_name=${{ env.MIGRATION_ENV }}-custom-values.yaml" >> "$GITHUB_OUTPUT"
+            echo ${{ env.SHORT_HASH }}-${{ needs.build-and-push-ee-image.outputs.SUBMODULE_SHORT_HASH }}
+          else
+            echo "file_name=dev-custom-values.yaml" >> "$GITHUB_OUTPUT"
+            echo ${{ env.SHORT_HASH }}-${{ needs.build-and-push-ee-image.outputs.SUBMODULE_SHORT_HASH }}
+          fi
+
+      - name: Update workflow-service image version in the HelmChart
+        uses: fjogeleit/yaml-update-action@main
+        with:
+          repository: ballerine-io/cloud-infra-config
+          branch: main
+          commitChange: true
+          message: "Update ${{ env.MIGRATION_ENV }} wf-service image Version to ${{ env.SHORT_HASH }}-${{ needs.build-and-push-ee-image.outputs.SUBMODULE_SHORT_HASH }} - (Commit hash: ${{ github.sha }})"
+          token: ${{ secrets.GIT_TOKEN }}
+          changes: |
+            {
+              "kubernetes/helm/wf-service/${{steps.update_helm_check.outputs.file_name}}": {
+                "dbMigrate.image.tag": "${{ env.SHORT_HASH }}-${{ needs.build-and-push-ee-image.outputs.SUBMODULE_SHORT_HASH }}-${{ env.MIGRATION_ENV }}",
+                "dataSync.image.tag": "${{ env.SHORT_HASH }}-${{ needs.build-and-push-ee-image.outputs.SUBMODULE_SHORT_HASH }}-${{ env.MIGRATION_ENV }}"
+              }
+            }
+
+  sync-argo-app:
+    needs: update-helm-chart
+    if: ${{ needs.update-helm-chart.result == 'success' }}
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.client_payload.environment }}
+    env:
+      stage: ${{ github.event.client_payload.environment }}
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Run ArgoCD Action
+        id: argocd_api
+        uses: ./.github/actions/argocd-action
+        with:
+          argocd_username: ${{ secrets.ARGOCD_USERNAME }}
+          argocd_password: ${{ secrets.ARGOCD_PASSWORD }}
+          argocd_server: ${{ secrets.ARGOCD_SERVER }}
+          tg_svc_key: ${{ secrets.TWINGATE_SERVICE_KEY_SECRET_NAME }}
+
+  send-to-slack:
+    runs-on: ubuntu-latest
+    needs: [update-helm-chart,build-and-push-ee-image]
+    if: ${{ needs.update-helm-chart.result == 'success' }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Send alert to Slack channel
+        id: slack
+        uses: slackapi/[email protected]
+        with:
+          channel-id: '${{ secrets.ARGO_SLACK_CHANNEL_ID }}'
+          slack-message: "Wf-service Migrations in ${{ env.MIGRATION_ENV }} with tag: ${{ env.SHORT_HASH }}-${{ needs.build-and-push-ee-image.outputs.SUBMODULE_SHORT_HASH }}-${{ env.MIGRATION_ENV }} and build result: ${{ job.status }}. successfully updated the wf-service migration jobs helm values for ${{ env.MIGRATION_ENV }}."
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.ARGO_SLACK_BOT_TOKEN }}