Fix dbt and IMDb models #45
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Pipeline for PRs to main | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| env: | |
| DBT_PROFILES_DIR: ./ | |
| POSTGRES_HOST: ${{ secrets.POSTGRES_HOST }} | |
| POSTGRES_PROD_USER: ${{ secrets.POSTGRES_PROD_USER }} | |
| POSTGRES_PROD_PW: ${{ secrets.POSTGRES_PROD_PW }} | |
| POSTGRES_CI_SCHEMA: "${{ format('{0}_{1}', 'ci_pr', github.event.number) }}" | |
| jobs: | |
| lint-models: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: "actions/checkout@v2" | |
| with: | |
| fetch-depth: 0 | |
| - uses: "actions/setup-python@v2" | |
| with: | |
| python-version: "3.12" | |
| - name: Install requirements | |
| run: pip install -q -r requirements.txt | |
| - name: Get changed files | |
| id: get_changed_files | |
| run: | | |
| git fetch origin main | |
| main_sha=$(git rev-parse origin/main) | |
| echo "changed_files=$(git diff --diff-filter=d --no-commit-id --name-only -r $main_sha... | grep "\.sql$" | tr '\n' ' ')" >> $GITHUB_OUTPUT | |
| - name: Lint dbt models | |
| if: steps.get_changed_files.outputs.changed_files != '' | |
| run: | | |
| echo "Linting:" ${{ steps.get_changed_files.outputs.changed_files }} | |
| sqlfluff lint ${{ steps.get_changed_files.outputs.changed_files }} --ignore templating | |
| slim-ci: | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| steps: | |
| - uses: "actions/checkout@v2" | |
| - uses: "actions/setup-python@v2" | |
| with: | |
| python-version: "3.12" | |
| - name: Install requirements | |
| run: pip install -q -r requirements.txt | |
| - name: Get dependencies | |
| run: dbt deps | |
| - name: Grab manifest.json from S3 | |
| run: | | |
| aws s3 cp ${{ secrets.AWS_BUCKET_URL }} ./manifest.json | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: eu-central-1 | |
| - name: Run dbt build (Slim CI) | |
| run: dbt build --select state:modified+1 --exclude duckdb --defer --state ./ --target ci |