ci.yaml

name: ci

on:
  pull_request:
  schedule:
    - cron: "0 */2 * * *"

jobs:
  static-analysis:
    strategy:
      matrix:
        task: ["lint", "fmt", "type-check", "python-build", "docker"]
      fail-fast: false
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Read python version from .python-version
        id: python-version
        run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ steps.python-version.outputs.PYTHON_VERSION }}
          cache-dependency-path: poetry.lock

      - name: Load cached Poetry installation
        id: cached-poetry
        uses: actions/cache@v4
        with:
          path: ~/.local # the path depends on the OS
          key: poetry-${{ steps.python-version.outputs.PYTHON_VERSION }} # increment to reset cache

      - name: Install Poetry
        if: steps.cached-poetry.outputs.cache-hit != 'true'
        uses: snok/install-poetry@v1

      - name: Install dependencies
        run: poetry install --sync

      - run: poetry run invoke ${{ matrix.task }} --check

  tests:
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache-dependency-path: poetry.lock

      - name: Load cached Poetry installation
        id: cached-poetry
        uses: actions/cache@v4
        with:
          path: ~/.local # the path depends on the OS
          key: poetry-${{ matrix.python-version  }} # increment to reset cache

      - name: Install Poetry
        if: steps.cached-poetry.outputs.cache-hit != 'true'
        uses: snok/install-poetry@v1

      - name: Install dependencies
        run: poetry install --sync

      - name: Run tests
        run: poetry run pytest -m "not agentjobs and not integration" --cov-report xml --junitxml=junit.xml -ra
        env:
          # GX Agent
          GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }}
          GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }}
          GX_CLOUD_BASE_URL: "http://localhost:5000"
          # GX Cloud
          AUTH0_DOMAIN: ${{secrets.AUTH0_DOMAIN}}
          AUTH0_API_AUDIENCE: ${{secrets.AUTH0_API_AUDIENCE}}
          AUTH0_MERCURY_API_CLIENT_ID: ${{secrets.AUTH0_MERCURY_API_CLIENT_ID}}
          AUTH0_MERCURY_API_CLIENT_SECRET: ${{secrets.AUTH0_MERCURY_API_CLIENT_SECRET}}
          LOGGING_LEVEL: DEBUG
          ENVIRONMENT: local
          USE_MOCK_CONFIG: true

      # upload coverage report to codecov
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v4.1.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: great-expectations/cloud
          flags: ${{ matrix.python-version }}

      # upload test results to codecov
      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/test-results-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.python-version }}

  integration-tests:
    strategy:
      matrix:
        python-version: ["3.10"]
    permissions:
      id-token: write
      contents: read
      actions: read
    runs-on: ubuntu-latest

    env:
      # GX Agent
      GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }}
      GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }}
      GX_CLOUD_BASE_URL: "http://localhost:5000"
      # GX Cloud
      AUTH0_DOMAIN: ${{secrets.AUTH0_DOMAIN}}
      AUTH0_API_AUDIENCE: ${{secrets.AUTH0_API_AUDIENCE}}
      AUTH0_MERCURY_API_CLIENT_ID: ${{secrets.AUTH0_MERCURY_API_CLIENT_ID}}
      AUTH0_MERCURY_API_CLIENT_SECRET: ${{secrets.AUTH0_MERCURY_API_CLIENT_SECRET}}
      LOGGING_LEVEL: DEBUG
      ENVIRONMENT: local

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache-dependency-path: poetry.lock

      - name: Load cached Poetry installation
        id: cached-poetry
        uses: actions/cache@v4
        with:
          path: ~/.local # the path depends on the OS
          key: poetry-${{ matrix.python-version  }} # increment to reset cache

      - name: Install Poetry
        if: steps.cached-poetry.outputs.cache-hit != 'true'
        uses: snok/install-poetry@v1

      - name: Install dependencies
        run: poetry install --sync

      - name: Configure ECR AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-east-1
          role-to-assume: arn:aws:iam::258143015559:role/github-amazonec2containerregistryreadonly

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2

      - name: Start services
        run: |
          docker compose up db mq -d
          docker compose run db-provisioner
          docker compose run db-seeder
          docker compose run mq-wait
          docker compose up mercury-service-api -d
          docker compose up mercury-service-api-v1 -d
          docker compose up nginx -d

      - name: Show logs for debugging
        if: failure()
        run: |
          docker compose -f docker-compose.yml logs

      - name: Run tests
        run: poetry run pytest -vv -m integration -W default::great_expectations.datasource.fluent.GxInvalidDatasourceWarning --cov-report xml --junitxml=junit.xml -ra

      # upload coverage report to codecov
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v4.1.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: great-expectations/cloud
          flags: integration

      # upload test results to codecov
      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/test-results-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: integration

  test-docker-build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
          cache-dependency-path: poetry.lock

      - name: Load cached Poetry installation
        id: cached-poetry
        uses: actions/cache@v4
        with:
          path: ~/.local # the path depends on the OS
          key: poetry-3.10 # increment to reset cache

      - name: Install Poetry
        if: steps.cached-poetry.outputs.cache-hit != 'true'
        uses: snok/install-poetry@v1

      - name: Install dependencies
        run: poetry install --sync --only dev

      - run: poetry run invoke docker --no-pty

  check-if-agent-changed:
    runs-on: ubuntu-latest
    if: ${{ github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'no version bump') }}
    outputs:
      agent_changed: ${{ steps.filter.outputs.agent }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            agent:
              - 'great_expectations_cloud/agent/**'
              - 'poetry.lock'
              - 'pyproject.toml'

  check-version-is-bumped:
    # This job checks that the version in the PR is different from the version in main if the agent has changed.
    needs: check-if-agent-changed
    runs-on: ubuntu-latest
    if: ${{ needs.check-if-agent-changed.outputs.agent_changed == 'true' && !contains(github.event.pull_request.labels.*.name, 'no version bump') && github.event_name != 'schedule' }}
    outputs:
      is_version_bumped: ${{ steps.is_version_bumped.outputs.IS_VERSION_BUMPED }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        # Read the python version from pyproject.toml

        with:
          python-version:
          cache-dependency-path: poetry.lock

      - name: Load cached Poetry installation
        id: cached-poetry
        uses: actions/cache@v4
        with:
          path: ~/.local # the path depends on the OS
          key: poetry-3.8 # increment to reset cache

      - name: Install Poetry
        if: steps.cached-poetry.outputs.cache-hit != 'true'
        uses: snok/install-poetry@v1

      - name: Get version from pyproject.toml for PR
        id: pr_version
        run: echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT

      - name: Checkout `main` branch
        uses: actions/checkout@v4
        with:
          ref: main

      - name: Get version from pyproject.toml for main
        id: main_version
        run: echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT

      - name: Get is_version_bumped
        id: is_version_bumped
        run: |
          if [ "${{ steps.pr_version.outputs.VERSION }}" == "${{ steps.main_version.outputs.VERSION }}" ]; then
            echo "IS_VERSION_BUMPED=false" >> $GITHUB_OUTPUT
          else
            echo "IS_VERSION_BUMPED=true" >> $GITHUB_OUTPUT
          fi

  bump-pre-release-version:
    permissions:
      contents: write # Allow us to write to the repository
    needs: check-version-is-bumped
    runs-on: ubuntu-latest
    if: ${{ needs.check-version-is-bumped.outputs.is_version_bumped == 'false' && !contains(github.event.pull_request.labels.*.name, 'no version bump') && !contains(github.event.pull_request.labels.*.name, 'dependencies') && github.event_name != 'schedule' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.8"
          cache-dependency-path: poetry.lock

      - name: Load cached Poetry installation
        id: cached-poetry
        uses: actions/cache@v4
        with:
          path: ~/.local # the path depends on the OS
          key: poetry-3.8 # increment to reset cache

      - name: Install Poetry
        if: steps.cached-poetry.outputs.cache-hit != 'true'
        uses: snok/install-poetry@v1

      - name: Install dependencies
        run: poetry install --sync # Needed for calling invoke

      - name: Checkout PR
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh pr checkout ${{ github.event.pull_request.number }}

      - name: Bump pre-release version
        run: poetry run invoke pre-release

      - name: Push changes to PR
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          git config --global user.email "team@greatexpectations.io"
          git config --global user.name "Great Expectations"
          git add .
          git commit -m "Pre-release version bump"
          git push