ci #9103
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| pull_request: | |
| schedule: | |
| - cron: "0 */2 * * *" | |
| jobs: | |
| static-analysis: | |
| strategy: | |
| matrix: | |
| task: ["lint", "fmt", "type-check", "python-build", "docker"] | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Read python version from .python-version | |
| id: python-version | |
| run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ steps.python-version.outputs.PYTHON_VERSION }} | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-${{ steps.python-version.outputs.PYTHON_VERSION }} # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install --sync | |
| - run: poetry run invoke ${{ matrix.task }} --check | |
| tests: | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-${{ matrix.python-version }} # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install --sync | |
| - name: Run tests | |
| run: poetry run pytest -m "not agentjobs and not integration" --cov-report xml --junitxml=junit.xml -ra | |
| env: | |
| # GX Agent | |
| GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }} | |
| GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }} | |
| GX_CLOUD_BASE_URL: "http://localhost:5000" | |
| # GX Cloud | |
| AUTH0_DOMAIN: ${{secrets.AUTH0_DOMAIN}} | |
| AUTH0_API_AUDIENCE: ${{secrets.AUTH0_API_AUDIENCE}} | |
| AUTH0_MERCURY_API_CLIENT_ID: ${{secrets.AUTH0_MERCURY_API_CLIENT_ID}} | |
| AUTH0_MERCURY_API_CLIENT_SECRET: ${{secrets.AUTH0_MERCURY_API_CLIENT_SECRET}} | |
| LOGGING_LEVEL: DEBUG | |
| ENVIRONMENT: local | |
| USE_MOCK_CONFIG: true | |
| # upload coverage report to codecov | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| slug: great-expectations/cloud | |
| flags: ${{ matrix.python-version }} | |
| # upload test results to codecov | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| flags: ${{ matrix.python-version }} | |
| integration-tests: | |
| strategy: | |
| matrix: | |
| python-version: ["3.10"] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| actions: read | |
| runs-on: ubuntu-latest | |
| env: | |
| # GX Agent | |
| GX_CLOUD_ACCESS_TOKEN: ${{ secrets.GX_CLOUD_ACCESS_TOKEN }} | |
| GX_CLOUD_ORGANIZATION_ID: ${{ secrets.GX_CLOUD_ORGANIZATION_ID }} | |
| GX_CLOUD_BASE_URL: "http://localhost:5000" | |
| # GX Cloud | |
| AUTH0_DOMAIN: ${{secrets.AUTH0_DOMAIN}} | |
| AUTH0_API_AUDIENCE: ${{secrets.AUTH0_API_AUDIENCE}} | |
| AUTH0_MERCURY_API_CLIENT_ID: ${{secrets.AUTH0_MERCURY_API_CLIENT_ID}} | |
| AUTH0_MERCURY_API_CLIENT_SECRET: ${{secrets.AUTH0_MERCURY_API_CLIENT_SECRET}} | |
| LOGGING_LEVEL: DEBUG | |
| ENVIRONMENT: local | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-${{ matrix.python-version }} # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install --sync | |
| - name: Configure ECR AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::258143015559:role/github-amazonec2containerregistryreadonly | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Start services | |
| run: | | |
| docker compose up db mq -d | |
| docker compose run db-provisioner | |
| docker compose run db-seeder | |
| docker compose run mq-wait | |
| docker compose up mercury-service-api -d | |
| docker compose up mercury-service-api-v1 -d | |
| docker compose up nginx -d | |
| - name: Show logs for debugging | |
| if: failure() | |
| run: | | |
| docker compose -f docker-compose.yml logs | |
| - name: Run tests | |
| run: poetry run pytest -vv -m integration -W default::great_expectations.datasource.fluent.GxInvalidDatasourceWarning --cov-report xml --junitxml=junit.xml -ra | |
| # upload coverage report to codecov | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| slug: great-expectations/cloud | |
| flags: integration | |
| # upload test results to codecov | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| flags: integration | |
| test-docker-build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-3.10 # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install --sync --only dev | |
| - run: poetry run invoke docker --no-pty | |
| check-if-agent-changed: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'no version bump') }} | |
| outputs: | |
| agent_changed: ${{ steps.filter.outputs.agent }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| agent: | |
| - 'great_expectations_cloud/agent/**' | |
| - 'poetry.lock' | |
| - 'pyproject.toml' | |
| check-version-is-bumped: | |
| # This job checks that the version in the PR is different from the version in main if the agent has changed. | |
| needs: check-if-agent-changed | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.check-if-agent-changed.outputs.agent_changed == 'true' && !contains(github.event.pull_request.labels.*.name, 'no version bump') && github.event_name != 'schedule' }} | |
| outputs: | |
| is_version_bumped: ${{ steps.is_version_bumped.outputs.IS_VERSION_BUMPED }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| # Read the python version from pyproject.toml | |
| with: | |
| python-version: | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-3.8 # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Get version from pyproject.toml for PR | |
| id: pr_version | |
| run: echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT | |
| - name: Checkout `main` branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - name: Get version from pyproject.toml for main | |
| id: main_version | |
| run: echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT | |
| - name: Get is_version_bumped | |
| id: is_version_bumped | |
| run: | | |
| if [ "${{ steps.pr_version.outputs.VERSION }}" == "${{ steps.main_version.outputs.VERSION }}" ]; then | |
| echo "IS_VERSION_BUMPED=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "IS_VERSION_BUMPED=true" >> $GITHUB_OUTPUT | |
| fi | |
| bump-pre-release-version: | |
| permissions: | |
| contents: write # Allow us to write to the repository | |
| needs: check-version-is-bumped | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.check-version-is-bumped.outputs.is_version_bumped == 'false' && !contains(github.event.pull_request.labels.*.name, 'no version bump') && !contains(github.event.pull_request.labels.*.name, 'dependencies') && github.event_name != 'schedule' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| cache-dependency-path: poetry.lock | |
| - name: Load cached Poetry installation | |
| id: cached-poetry | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.local # the path depends on the OS | |
| key: poetry-3.8 # increment to reset cache | |
| - name: Install Poetry | |
| if: steps.cached-poetry.outputs.cache-hit != 'true' | |
| uses: snok/install-poetry@v1 | |
| - name: Install dependencies | |
| run: poetry install --sync # Needed for calling invoke | |
| - name: Checkout PR | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh pr checkout ${{ github.event.pull_request.number }} | |
| - name: Bump pre-release version | |
| run: poetry run invoke pre-release | |
| - name: Push changes to PR | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "Great Expectations" | |
| git add . | |
| git commit -m "Pre-release version bump" | |
| git push |