Adding custom & s3 tests infrastructure

.github/workflows/custom_testing.yml

@@ -0,0 +1,105 @@
+name: Run Custom Tests All dialects
+
+on:
+  workflow_call:
+    inputs:
+      python-versions:
+        description: "JSON string of Python versions"
+        type: string
+        required: true
+    secrets:
+      READ_LLM_FIXTURES_ROLE:
+        required: true
+      SF_USERNAME:
+        required: true
+      SF_PASSWORD:
+        required: true
+      SF_ACCOUNT:
+        required: true
+      MYSQL_USERNAME:
+        required: true
+      MYSQL_PASSWORD:
+        required: true
+      POSTGRES_USER:
+        required: true
+      POSTGRES_PASSWORD:
+        required: true
+
+jobs:
+  custom-tests:
+    name: Custom Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        python-version: ${{ fromJSON(inputs.python-versions) }}
+
+    # Define services here to run Docker containers alongside your job
+    services:
+      mysql:
+        image: bodoai1/pydough-mysql-tpch:latest
+        env:
+          # Set environment variables for MySQL container
+          MYSQL_ROOT_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
+          MYSQL_DATABASE: tpch
+        ports:
+          - 3306:3306
+
+      postgres:
+        image: bodoai1/pydough-postgres-tpch:latest
+        env:
+          # Set environment variables for Postgres container
+          POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          POSTGRES_DB: "pydough_test"
+        ports:
+          - 5432:5432
+    env:
+      # MYSQL env
+      MYSQL_USERNAME: ${{ secrets.MYSQL_USERNAME }}
+      MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
+      MYSQL_DATABASE: tpch
+      MYSQL_HOST: 127.0.0.1
+      # POSTGRES env
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+      POSTGRES_DB: pydough_test
+      POSTGRES_HOST: 127.0.0.1
+      # SNOWFLAKE env
+      SF_USERNAME: ${{ secrets.SF_USERNAME }}
+      SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
+      SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }}
+
+    steps:
+      - name: Configure AWS Credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.READ_LLM_FIXTURES_ROLE }}
+          aws-region: us-east-2
+
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.23"
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install dependencies
+        run: uv pip install -e ".[boto3, snowflake, mysql, postgres]"
+
+      - name: Confirm all connectors are installed
+        run: uv run python -c "import boto3; import mysql.connector; import snowflake.connector; import psycopg2; print('All connectors installed')"
+
+      - name: Run Custom Tests for all dialects
+        run: uv run pytest -m custom tests/ -rs

.github/workflows/pr_testing.yml

@@ -46,6 +46,11 @@ on:
         type: boolean
         required: false
         default: false
+      run-custom:
+        description: "Run Custom Datasets Tests"
+        type: boolean
+        required: false
+        default: false
 
 # Limit CI to cancel previous runs in the same PR
 concurrency:
@@ -108,8 +113,6 @@ jobs:
           # Output to GitHub Actions expected format
           echo "matrix=$joined" >> $GITHUB_OUTPUT
 
-
-
   run-python-tests:
     name: Main Python Tests
     needs: [get-msg, get-py-ver-matrix]
@@ -148,7 +151,7 @@ jobs:
         run: uv run ruff check .
 
       - name: Run Tests
-        run: uv run pytest tests/ -m "not (snowflake or mysql or postgres or sf_masked)" -rs
+        run: uv run pytest tests/ -m "not (snowflake or mysql or postgres or sf_masked or custom)" -rs
 
   run-defog-daily-update:
     name: Run DEFOG Daily Update
@@ -232,3 +235,25 @@ jobs:
       python-versions: ${{ github.event_name == 'workflow_dispatch'
                       && needs.get-py-ver-matrix.outputs.matrix
                       || '["3.10", "3.11", "3.12"]' }}
+
+  run-custom-tests:
+    name: Custom datasets Tests
+    needs: [get-msg, get-py-ver-matrix]
+    if: |
+      (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) ||
+      (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run custom]')) ||
+      (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-custom))
+    uses: ./.github/workflows/custom_testing.yml
+    secrets:
+      READ_LLM_FIXTURES_ROLE: ${{ secrets.READ_LLM_FIXTURES_ROLE }}
+      SF_USERNAME: ${{ secrets.SF_USERNAME }}
+      SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
+      SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }}
+      MYSQL_USERNAME: ${{ secrets.MYSQL_USERNAME }}
+      MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+    with:
+      python-versions: ${{ github.event_name == 'workflow_dispatch'
+                      && needs.get-py-ver-matrix.outputs.matrix
+                      || '["3.10", "3.11", "3.12"]' }}

pyproject.toml

@@ -43,6 +43,7 @@ snowflake = ["snowflake-connector-python[pandas]"]
 mysql = ["mysql-connector-python"]
 postgres = ["psycopg2-binary"]
 server = ["fastapi", "httpx", "uvicorn"]
+boto3 = ["boto3"]
 
 [build-system]
 requires = ["hatchling", "hatch-vcs"]

pytest.ini

@@ -6,3 +6,4 @@ markers =
     postgres: marks tests that require PostgresSQL credentials
     server: marks tests that require api mock server
     sf_masked: marks tests that require Snowflake Masked credentials
+    custom: marks tests that require custom datasets from s3