containerize-agent.yaml

name: containerize-agent

on:
  push:
    branches:
      - main

jobs:
  containerize-agent:
    runs-on: ubuntu-latest
    permissions:
      id-token: write

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Read python version from .python-version
        id: read-python-version
        run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ steps.read-python-version.outputs.PYTHON_VERSION }}
          cache-dependency-path: poetry.lock

      - name: Load cached Poetry installation
        id: cached-poetry
        uses: actions/cache@v4
        with:
          path: ~/.local # the path depends on the OS
          key: poetry-0 # increment to reset cache

      - name: Install Poetry
        if: steps.cached-poetry.outputs.cache-hit != 'true'
        uses: snok/install-poetry@v1

      - name: Install dependencies
        run: poetry install --with dev,sql --sync

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Configure ECR AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-east-1
          role-to-assume: arn:aws:iam::258143015559:role/github-amazonec2containerregistrypoweruser

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2

      # Cannot multi arch build AND load
      # https://github.com/docker/buildx/issues/59
      - name: Build arm64, No Load, No Push
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/arm64
          push: false
          load: false
          tags: |
            greatexpectations/agent:latest
            greatexpectations/agent:stable

      # https://docs.docker.com/build/ci/github-actions/test-before-push/
      - name: Build amd64, With Load, No Push
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64
          push: false
          load: true
          tags: |
            greatexpectations/agent:latest
            greatexpectations/agent:stable

      - name: Smoke Test the Image
        run: docker run --rm greatexpectations/agent:stable poetry run gx-agent -h

      # Uses local image built in previous step
      - name: Test New amd64 Agent Image
        run: |
          echo "STARTING db and mq"
          docker compose up db mq -d
          echo "STARTING db-provisioner"
          docker compose run db-provisioner
          echo "STARTING db-seeder"
          docker compose run db-seeder
          echo "STARTING mq-wait"
          docker compose run mq-wait
          echo "STARTING mercury-service-api"
          docker compose up mercury-service-api -d
          echo "STARTING mercury-service-api-v1"
          docker compose up mercury-service-api-v1 -d
          echo "STARTING nginx"
          docker compose up nginx -d
          echo "STARTING gx-agent"
          docker compose up gx-agent -d
          echo "RUNNING TESTS"
          poetry run pytest -m "agentjobs"
        env:
          GX_CLOUD_BASE_URL: "http://localhost:5000"
          GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }}
          GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }}
          AUTH0_API_AUDIENCE: ${{ secrets.AUTH0_API_AUDIENCE }}
          AUTH0_MERCURY_API_CLIENT_ID: ${{ secrets.AUTH0_MERCURY_API_CLIENT_ID }}
          AUTH0_MERCURY_API_CLIENT_SECRET: ${{ secrets.AUTH0_MERCURY_API_CLIENT_SECRET }}
          AUTH0_DOMAIN: ${{ secrets.AUTH0_DOMAIN }}
          GE_USAGE_STATISTICS_URL: ${{ secrets.GE_USAGE_STATISTICS_URL }}

      - name: Get version from pyproject.toml
        id: get_version
        run: echo "POETRY_VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT

      - name: Get is_dev from pyproject.toml
        id: get_is_dev
        run: echo "IS_DEV=$(poetry run invoke is-dev-release)" >> $GITHUB_OUTPUT

      - name: Get is_pre_release from pyproject.toml
        id: get_is_pre_release
        run: echo "IS_PRERELEASE=$(poetry run invoke is-pre-release)" >> $GITHUB_OUTPUT

      - name: Get is_release from pyproject.toml
        id: get_is_release
        run: echo "IS_RELEASE=$(poetry run invoke is-release)" >> $GITHUB_OUTPUT

      # Uses the version from pyproject.toml for the tag
      - name: Push Pre-Release to Docker Hub and Internal ECR
        if: ${{ steps.get_is_dev.outputs.IS_DEV == 'True'}}
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            greatexpectations/agent:dev
            greatexpectations/agent:${{ steps.get_version.outputs.POETRY_VERSION }}
            258143015559.dkr.ecr.us-east-1.amazonaws.com/gx/agent:${{ steps.get_version.outputs.POETRY_VERSION }}

      - name: Push Release to Docker Hub and Internal ECR
        if: ${{ steps.get_is_release.outputs.IS_RELEASE == 'True' }}
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            greatexpectations/agent:dev
            greatexpectations/agent:${{ steps.get_version.outputs.POETRY_VERSION }}
            greatexpectations/agent:latest
            greatexpectations/agent:stable
            258143015559.dkr.ecr.us-east-1.amazonaws.com/gx/agent:${{ steps.get_version.outputs.POETRY_VERSION }}

      - name: Show logs, if failure
        if: failure()
        run: docker compose logs

      # NOTE: can't use coveralls here without refactoring workflows

      # upload coverage report to codecov
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v4.1.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: great-expectations/cloud
          # use the same flags as the test markers
          flags: agentjobs