Allow none for batch parameters for "latest batch" (#561) #581
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: containerize-agent | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| containerize-agent: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Read python version from .python-version | |
| id: read-python-version | |
| run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ steps.read-python-version.outputs.PYTHON_VERSION }} | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-0 # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install --with dev,sql --sync | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Configure ECR AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::258143015559:role/github-amazonec2containerregistrypoweruser | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| # Cannot multi arch build AND load | |
| # https://github.com/docker/buildx/issues/59 | |
| - name: Build arm64, No Load, No Push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/arm64 | |
| push: false | |
| load: false | |
| tags: | | |
| greatexpectations/agent:latest | |
| greatexpectations/agent:stable | |
| # https://docs.docker.com/build/ci/github-actions/test-before-push/ | |
| - name: Build amd64, With Load, No Push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: false | |
| load: true | |
| tags: | | |
| greatexpectations/agent:latest | |
| greatexpectations/agent:stable | |
| - name: Smoke Test the Image | |
| run: docker run --rm greatexpectations/agent:stable poetry run gx-agent -h | |
| # Uses local image built in previous step | |
| - name: Test New amd64 Agent Image | |
| run: | | |
| echo "STARTING db and mq" | |
| docker compose up db mq -d | |
| echo "STARTING db-provisioner" | |
| docker compose run db-provisioner | |
| echo "STARTING db-seeder" | |
| docker compose run db-seeder | |
| echo "STARTING mq-wait" | |
| docker compose run mq-wait | |
| echo "STARTING mercury-service-api" | |
| docker compose up mercury-service-api -d | |
| echo "STARTING mercury-service-api-v1" | |
| docker compose up mercury-service-api-v1 -d | |
| echo "STARTING nginx" | |
| docker compose up nginx -d | |
| echo "STARTING gx-agent" | |
| docker compose up gx-agent -d | |
| echo "RUNNING TESTS" | |
| poetry run pytest -m "agentjobs" | |
| env: | |
| GX_CLOUD_BASE_URL: "http://localhost:5000" | |
| GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }} | |
| GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }} | |
| AUTH0_API_AUDIENCE: ${{ secrets.AUTH0_API_AUDIENCE }} | |
| AUTH0_MERCURY_API_CLIENT_ID: ${{ secrets.AUTH0_MERCURY_API_CLIENT_ID }} | |
| AUTH0_MERCURY_API_CLIENT_SECRET: ${{ secrets.AUTH0_MERCURY_API_CLIENT_SECRET }} | |
| AUTH0_DOMAIN: ${{ secrets.AUTH0_DOMAIN }} | |
| GE_USAGE_STATISTICS_URL: ${{ secrets.GE_USAGE_STATISTICS_URL }} | |
| - name: Get version from pyproject.toml | |
| id: get_version | |
| run: echo "POETRY_VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT | |
| - name: Get is_dev from pyproject.toml | |
| id: get_is_dev | |
| run: echo "IS_DEV=$(poetry run invoke is-dev-release)" >> $GITHUB_OUTPUT | |
| - name: Get is_pre_release from pyproject.toml | |
| id: get_is_pre_release | |
| run: echo "IS_PRERELEASE=$(poetry run invoke is-pre-release)" >> $GITHUB_OUTPUT | |
| - name: Get is_release from pyproject.toml | |
| id: get_is_release | |
| run: echo "IS_RELEASE=$(poetry run invoke is-release)" >> $GITHUB_OUTPUT | |
| # Uses the version from pyproject.toml for the tag | |
| - name: Push Pre-Release to Docker Hub and Internal ECR | |
| if: ${{ steps.get_is_dev.outputs.IS_DEV == 'True'}} | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| greatexpectations/agent:dev | |
| greatexpectations/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| 258143015559.dkr.ecr.us-east-1.amazonaws.com/gx/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| - name: Push Release to Docker Hub and Internal ECR | |
| if: ${{ steps.get_is_release.outputs.IS_RELEASE == 'True' }} | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| greatexpectations/agent:dev | |
| greatexpectations/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| greatexpectations/agent:latest | |
| greatexpectations/agent:stable | |
| 258143015559.dkr.ecr.us-east-1.amazonaws.com/gx/agent:${{ steps.get_version.outputs.POETRY_VERSION }} | |
| - name: Show logs, if failure | |
| if: failure() | |
| run: docker compose logs | |
| # NOTE: can't use coveralls here without refactoring workflows | |
| # upload coverage report to codecov | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| slug: great-expectations/cloud | |
| # use the same flags as the test markers | |
| flags: agentjobs |