Adding optimization rewrite pass to utilize server with information about masked columns #2645
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR Testing | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| py310: | |
| description: "Python 3.10" | |
| type: boolean | |
| default: true | |
| py311: | |
| description: "Python 3.11" | |
| type: boolean | |
| default: true | |
| py312: | |
| description: "Python 3.12" | |
| type: boolean | |
| default: true | |
| run-all: | |
| description: "Run All Tests" | |
| type: boolean | |
| required: false | |
| default: false | |
| run-python: | |
| description: "Run Main PyDough Tests" | |
| type: boolean | |
| required: false | |
| default: true | |
| run-sf: | |
| description: "Run Snowflake Tests" | |
| type: boolean | |
| required: false | |
| default: false | |
| run-mysql: | |
| description: "Run MySQL Tests" | |
| type: boolean | |
| required: false | |
| default: false | |
| run-postgres: | |
| description: "Run Postgres Tests" | |
| type: boolean | |
| required: false | |
| default: false | |
| run-sf_masked: | |
| description: "Run Snowflake Masked Tests" | |
| type: boolean | |
| required: false | |
| default: false | |
| run-s3: | |
| description: "Run S3 Datasets Tests" | |
| type: boolean | |
| required: false | |
| default: false | |
| # Limit CI to cancel previous runs in the same PR | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| get-msg: | |
| name: Get Commit Message | |
| runs-on: ubuntu-latest | |
| outputs: | |
| commitMsg: ${{ steps.get_msg.outputs.commitMsg }} | |
| steps: | |
| # Fetch the branch for the history only | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| sparse-checkout: . | |
| - name: Get Commit Message | |
| id: get_msg | |
| run: | | |
| set -xe pipefail | |
| echo "commitMsg=$(git log -1 --pretty=format:'%s')" >> $GITHUB_OUTPUT | |
| # Get Python versions from inputs if workflow_dispatch is used | |
| # Otherwise, use all versions for pull_request events. | |
| get-py-ver-matrix: | |
| name: Get Python Version Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| # Get the matrix output produced by the step below (set-matrix). | |
| # This will be used in the run tests job. | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| # Capture the Python versions selected via dispatch as a JSON array output | |
| # For manual runs (workflow_dispatch), it uses the selected | |
| # inputs to determine which versions to run. | |
| # For other events like pull_request, it defaults to all versions. | |
| - name: Set selected Python versions | |
| id: set-matrix | |
| run: | | |
| versions=() | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| if [ "${{ inputs.py310 }}" == "true" ]; then | |
| versions+=("\"3.10\"") | |
| fi | |
| if [ "${{ inputs.py311 }}" == "true" ]; then | |
| versions+=("\"3.11\"") | |
| fi | |
| if [ "${{ inputs.py312 }}" == "true" ]; then | |
| versions+=("\"3.12\"") | |
| fi | |
| else | |
| # For pull_request and other events, use all versions by default | |
| versions=( "\"3.10\"" "\"3.11\"" "\"3.12\"" ) | |
| fi | |
| # Join the array elements with commas and wrap in brackets to make valid JSON | |
| joined=$(IFS=, ; echo "[${versions[*]}]") | |
| # Output to GitHub Actions expected format | |
| echo "matrix=$joined" >> $GITHUB_OUTPUT | |
| run-python-tests: | |
| name: Main Python Tests | |
| needs: [get-msg, get-py-ver-matrix] | |
| # https://docs.github.com/en/actions/learn-github-actions/expressions#contains | |
| # contains is case-insensitive | |
| if: | | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run ci]')) || | |
| (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-python)) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ${{ github.event_name == 'workflow_dispatch' | |
| && fromJSON(needs.get-py-ver-matrix.outputs.matrix) | |
| || fromJSON('["3.10", "3.11", "3.12"]') }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| # Install a specific version of uv. | |
| version: "0.4.23" | |
| - name: Download TPCH DB | |
| run: ./demos/setup_tpch.sh ./tpch.db | |
| - name: Run Ruff | |
| run: uv run ruff check . | |
| - name: Run Tests | |
| run: uv run pytest tests/ -m "not (snowflake or mysql or postgres or sf_masked or s3)" -rs | |
| run-defog-daily-update: | |
| name: Run DEFOG Daily Update | |
| needs: [get-msg] | |
| if: | | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run sf]')) || | |
| (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-sf)) | |
| uses: ./.github/workflows/defog_daily_update.yml | |
| secrets: | |
| SF_USERNAME: ${{ secrets.SF_USERNAME }} | |
| SF_PASSWORD: ${{ secrets.SF_PASSWORD }} | |
| SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }} | |
| run-sf-tests: | |
| name: Snowflake Tests | |
| needs: [run-defog-daily-update, get-msg, get-py-ver-matrix] | |
| if: | | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run sf]')) || | |
| (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-sf)) | |
| uses: ./.github/workflows/sf_testing.yml | |
| secrets: | |
| SF_USERNAME: ${{ secrets.SF_USERNAME }} | |
| SF_PASSWORD: ${{ secrets.SF_PASSWORD }} | |
| SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }} | |
| with: | |
| python-versions: ${{ github.event_name == 'workflow_dispatch' | |
| && needs.get-py-ver-matrix.outputs.matrix | |
| || '["3.10", "3.11", "3.12"]' }} | |
| run-mysql-tests: | |
| name: MySQL Tests | |
| needs: [get-msg, get-py-ver-matrix] | |
| if: | | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run mysql]')) || | |
| (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-mysql)) | |
| uses: ./.github/workflows/mysql_testing.yml | |
| secrets: | |
| MYSQL_USERNAME: ${{ secrets.MYSQL_USERNAME }} | |
| MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }} | |
| with: | |
| python-versions: ${{ github.event_name == 'workflow_dispatch' | |
| && needs.get-py-ver-matrix.outputs.matrix | |
| || '["3.10", "3.11", "3.12"]' }} | |
| run-postgres-tests: | |
| name: Postgres Tests | |
| needs: [get-msg, get-py-ver-matrix] | |
| if: | | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run postgres]')) || | |
| (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-postgres)) | |
| uses: ./.github/workflows/postgres_testing.yml | |
| secrets: | |
| POSTGRES_USER: ${{ secrets.POSTGRES_USER }} | |
| POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} | |
| with: | |
| python-versions: ${{ github.event_name == 'workflow_dispatch' | |
| && needs.get-py-ver-matrix.outputs.matrix | |
| || '["3.10", "3.11", "3.12"]' }} | |
| run-sf-masked-tests: | |
| name: Masked Snowflake Tests | |
| needs: [get-msg, get-py-ver-matrix] | |
| if: | | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run sf_masked]')) || | |
| (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-sf_masked)) | |
| uses: ./.github/workflows/sf_masked_testing.yml | |
| secrets: | |
| SF_FULL_USERNAME: ${{ secrets.SF_FULL_USERNAME }} | |
| SF_FULL_PASSWORD: ${{ secrets.SF_FULL_PASSWORD }} | |
| SF_PARTIAL_USERNAME: ${{ secrets.SF_PARTIAL_USERNAME }} | |
| SF_PARTIAL_PASSWORD: ${{ secrets.SF_PARTIAL_PASSWORD }} | |
| SF_NONE_USERNAME: ${{ secrets.SF_NONE_USERNAME }} | |
| SF_NONE_PASSWORD: ${{ secrets.SF_NONE_PASSWORD }} | |
| SF_MASKED_ACCOUNT: ${{ secrets.SF_MASKED_ACCOUNT }} | |
| with: | |
| python-versions: ${{ github.event_name == 'workflow_dispatch' | |
| && needs.get-py-ver-matrix.outputs.matrix | |
| || '["3.10", "3.11", "3.12"]' }} | |
| run-s3-tests: | |
| name: S3 datasets Tests | |
| needs: [get-msg, get-py-ver-matrix] | |
| if: | | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) || | |
| (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run s3]')) || | |
| (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-s3)) | |
| uses: ./.github/workflows/s3_testing.yml | |
| secrets: | |
| READ_LLM_FIXTURES_ROLE: ${{ secrets.READ_LLM_FIXTURES_ROLE }} | |
| with: | |
| python-versions: ${{ github.event_name == 'workflow_dispatch' | |
| && needs.get-py-ver-matrix.outputs.matrix | |
| || '["3.10", "3.11", "3.12"]' }} |