Adding custom & s3 tests infrastructure

.github/workflows/pr_testing.yml

@@ -46,6 +46,11 @@ on:
         type: boolean
         required: false
         default: false
+      run-s3:
+        description: "Run S3 Datasets Tests"
+        type: boolean
+        required: false
+        default: false
 
 # Limit CI to cancel previous runs in the same PR
 concurrency:
@@ -108,8 +113,6 @@ jobs:
           # Output to GitHub Actions expected format
           echo "matrix=$joined" >> $GITHUB_OUTPUT
 
-
-
   run-python-tests:
     name: Main Python Tests
     needs: [get-msg, get-py-ver-matrix]
@@ -148,7 +151,7 @@ jobs:
         run: uv run ruff check .
 
       - name: Run Tests
-        run: uv run pytest tests/ -m "not (snowflake or mysql or postgres or sf_masked)" -rs
+        run: uv run pytest tests/ -m "not (snowflake or mysql or postgres or sf_masked or s3)" -rs
 
   run-defog-daily-update:
     name: Run DEFOG Daily Update
@@ -232,3 +235,18 @@ jobs:
       python-versions: ${{ github.event_name == 'workflow_dispatch'
                       && needs.get-py-ver-matrix.outputs.matrix
                       || '["3.10", "3.11", "3.12"]' }}
+
+  run-s3-tests:
+    name: S3 datasets Tests
+    needs: [get-msg, get-py-ver-matrix]
+    if: |
+      (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run all]')) ||
+      (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run s3]')) ||
+      (github.event_name == 'workflow_dispatch' && (inputs.run-all || inputs.run-s3))
+    uses: ./.github/workflows/s3_testing.yml
+    secrets:
+      READ_LLM_FIXTURES_ROLE: ${{ secrets.READ_LLM_FIXTURES_ROLE }}
+    with:
+      python-versions: ${{ github.event_name == 'workflow_dispatch'
+                      && needs.get-py-ver-matrix.outputs.matrix
+                      || '["3.10", "3.11", "3.12"]' }}

.github/workflows/s3_testing.yml

@@ -0,0 +1,55 @@
+name: Run S3 Tests
+
+on:
+  workflow_call:
+    inputs:
+      python-versions:
+        description: "JSON string of Python versions"
+        type: string
+        required: true
+    secrets:
+      READ_LLM_FIXTURES_ROLE:
+        required: true
+
+jobs:
+  s3-tests:
+    name: S3 Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        python-version: ${{ fromJSON(inputs.python-versions) }}
+
+    steps:
+      - name: Configure AWS Credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.READ_LLM_FIXTURES_ROLE }}
+          aws-region: us-east-2
+
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.23"
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install dependencies
+        run: uv pip install -e ".[boto3]"
+
+      - name: Confirm all connectors are installed
+        run: uv run python -c "import boto3; print('Boto3 installed')"
+
+      - name: Run S3 Tests
+        run: uv run pytest -m s3 tests/ -rs

pyproject.toml

@@ -32,6 +32,7 @@ dev-dependencies = [
     "pytest",
     "ruff==0.6.7",
     "pytest-repeat",
+    "boto3",
     "pydough[snowflake]",
     "pydough[mysql]",
     "pydough[postgres]",

pytest.ini

@@ -6,3 +6,4 @@ markers =
     postgres: marks tests that require PostgresSQL credentials
     server: marks tests that require api mock server
     sf_masked: marks tests that require Snowflake Masked credentials
+    s3: marks tests that require custom datasets from s3