docker-unified.yml

name: Docker Build, Scan, Test
on:
  push:
    branches:
      - master
    paths-ignore:
      - "docs/**"
      - "**.md"
  pull_request:
    branches:
      - master
    paths-ignore:
      - "docs/**"
      - "**.md"
  release:
    types: [published, edited]

concurrency:
  # Using `github.run_id` (unique val) instead of `github.ref` here
  # because we don't want to cancel this workflow on master only for PRs
  #   as that makes reproducing issues easier
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

env:
  DATAHUB_GMS_IMAGE: "linkedin/datahub-gms"
  DATAHUB_FRONTEND_IMAGE: "linkedin/datahub-frontend-react"
  DATAHUB_MAE_CONSUMER_IMAGE: "linkedin/datahub-mae-consumer"
  DATAHUB_MCE_CONSUMER_IMAGE: "linkedin/datahub-mce-consumer"
  DATAHUB_KAFKA_SETUP_IMAGE: "linkedin/datahub-kafka-setup"
  DATAHUB_ELASTIC_SETUP_IMAGE: "linkedin/datahub-elasticsearch-setup"
  DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup"
  DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade"

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      unique_tag: ${{ steps.tag.outputs.unique_tag }}
      publish: ${{ steps.publish.outputs.publish }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Compute Tag
        id: tag
        run: |
          echo "GITHUB_REF: $GITHUB_REF"
          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
          UNIQUE_TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
          echo "tag=$TAG" >> $GITHUB_OUTPUT
          echo "unique_tag=$UNIQUE_TAG" >> $GITHUB_OUTPUT
      - name: Check whether publishing enabled
        id: publish
        env:
          ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
        run: |
          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT

  gms_build:
    name: Build and Push DataHub GMS Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_GMS_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_GMS_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-gms/Dockerfile
          platforms: linux/amd64,linux/arm64
  gms_scan:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: "[Monitoring] Scan GMS images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, gms_build]
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'template'
          template: '@/contrib/sarif.tpl'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          ignore-unfixed: true
          vuln-type: "os,library"          
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'


  mae_consumer_build:
    name: Build and Push DataHub MAE Consumer Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-mae-consumer/Dockerfile
          platforms: linux/amd64,linux/arm64
  mae_consumer_scan:
    name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, mae_consumer_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'template'
          template: '@/contrib/sarif.tpl'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'


  mce_consumer_build:
    name: Build and Push DataHub MCE Consumer Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-mce-consumer/Dockerfile
          platforms: linux/amd64,linux/arm64
  mce_consumer_scan:
    name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, mce_consumer_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'template'
          template: '@/contrib/sarif.tpl'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'


  datahub_upgrade_build:
    name: Build and Push DataHub Upgrade Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_UPGRADE_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_UPGRADE_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-upgrade/Dockerfile
          platforms: linux/amd64,linux/arm64
  datahub_upgrade_scan:
    name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, datahub_upgrade_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'template'
          template: '@/contrib/sarif.tpl'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'

  frontend_build:
    name: Build and Push DataHub Frontend Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_FRONTEND_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_FRONTEND_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-frontend/Dockerfile
          platforms: linux/amd64,linux/arm64
  frontend_scan:
    name: "[Monitoring] Scan Frontend images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, frontend_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'template'
          template: '@/contrib/sarif.tpl'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'

  kafka_setup_build:
    name: Build and Push DataHub Kafka Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: ./docker/kafka-setup
          file: ./docker/kafka-setup/Dockerfile
          platforms: linux/amd64,linux/arm64

  mysql_setup_build:
    name: Build and Push DataHub MySQL Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/mysql-setup/Dockerfile
          platforms: linux/amd64,linux/arm64

  elasticsearch_setup_build:
    name: Build and Push DataHub Elasticsearch Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/elasticsearch-setup/Dockerfile
          platforms: linux/amd64,linux/arm64

  smoke_test:
    name: Run Smoke Tests
    runs-on: ubuntu-latest
    needs:
      [
        setup,
        gms_build,
        frontend_build,
        kafka_setup_build,
        mysql_setup_build,
        elasticsearch_setup_build,
        mae_consumer_build,
        mce_consumer_build,
      ]
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
      - name: Set up JDK 11
        uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: 11
      - uses: actions/setup-python@v4
        with:
          python-version: "3.7"
      - name: Install dependencies
        run: ./metadata-ingestion/scripts/install_deps.sh
      - name: Build datahub cli
        run: |
          ./gradlew :metadata-ingestion:install
      - name: Download GMS image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Frontend image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Kafka Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Mysql Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Elastic Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download MCE Consumer image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download MCE Consumer image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Smoke test
        env:
          DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
        run: |
          echo "$DATAHUB_VERSION"
          ./smoke-test/smoke.sh
      - name: store logs
        if: failure()
        run: |
          docker ps -a
          docker logs datahub-gms >& gms.log
      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: docker logs
          path: "*.log"
      - name: Upload screenshots
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: cypress-snapshots
          path: smoke-test/tests/cypress/cypress/screenshots/
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: Test Results (smoke tests)
          path: |
            **/build/reports/tests/test/**
            **/build/test-results/test/**
            **/junit.*.xml
      - name: Slack failure notification
        if: failure() && github.event_name == 'push'
        uses: kpritam/slack-job-status-action@v1
        with:
          job-status: ${{ job.status }}
          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel: github-activities