From 7fabd2a701e2cd4d045ee000aec8bc4b160e204f Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Thu, 11 Sep 2025 22:45:40 -0400 Subject: [PATCH 01/24] Add external PR testing with maintainer approval controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split rust-ci tests: basic tests for external PRs, full tests for trusted PRs - External PRs run unit tests only (--lib) without external service secrets - Internal PRs and labeled external PRs get full integration testing - Added label-based approval: maintainers can add 'test-external-services' label - Added manual dispatch workflow for on-demand external PR testing - Reusable workflow design prevents code duplication - Maintains security while enabling external contributions 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/rust-ci.yaml | 21 ++++++++++++++++++--- .github/workflows/test-external-pr.yaml | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test-external-pr.yaml diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 3d9b82ff0..a0dd637c6 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -11,6 +11,13 @@ on: schedule: # three times a day to run the integration tests that take a long time - cron: '33 3,10,15 * * *' + workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string + default: '' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -47,7 +54,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} - name: Stand up docker services if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' @@ -87,8 +94,17 @@ jobs: # workspaces: "rust -> target" key: ${{ env.RUST_CHANNEL }} - - name: Run tests with Docker services + - name: Run basic tests if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' + run: | + just compile-tests "--locked" + just test --lib + + - name: Run integration tests with external services + if: | + (matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm') && + (github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'test-external-services')) env: RUST_LOG: trace R2_BUCKET: ${{ secrets.R2_BUCKET }} @@ -107,7 +123,6 @@ jobs: TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} run: | - just compile-tests "--locked" just test - name: Run doc tests diff --git a/.github/workflows/test-external-pr.yaml b/.github/workflows/test-external-pr.yaml new file mode 100644 index 000000000..f36095ebb --- /dev/null +++ b/.github/workflows/test-external-pr.yaml @@ -0,0 +1,17 @@ +name: Test External PR with Secrets + +on: + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to test' + required: true + type: number + +jobs: + test: + name: Test PR #${{ github.event.inputs.pr_number }} + uses: ./.github/workflows/rust-ci.yaml + with: + ref: refs/pull/${{ github.event.inputs.pr_number }}/merge + secrets: inherit From fb01ebfa089f643819085a15760a907a9738da54 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 00:46:49 -0400 Subject: [PATCH 02/24] Implement GitHub deployment protection for external PR testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split rust-ci into separate jobs: rust-safe (immediate) + rust-external-services (approval required) - rust-safe: runs unit tests, MinIO/Azurite integration tests, doc tests, examples across all platforms - rust-external-services: requires 'external-services' environment approval for AWS/R2/Tigris tests - External PRs get comprehensive feedback immediately while secrets remain protected - Maintainers can review code changes before approving external service tests - Uses GitHub's built-in deployment protection with audit trail - Optimized external tests to only run AWS/R2/Tigris (excludes redundant MinIO tests) - Maintains backward compatibility with workflow_call for reusability - Removed separate test-external-pr.yaml (functionality now integrated) 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/rust-ci.yaml | 125 +++++++++++++++++++----- .github/workflows/test-external-pr.yaml | 17 ---- 2 files changed, 99 insertions(+), 43 deletions(-) delete mode 100644 .github/workflows/test-external-pr.yaml diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index a0dd637c6..9d5f9ad1a 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -33,17 +33,13 @@ env: RUST_CHANNEL: '1.89.0' jobs: - rust: - name: Rust CI + rust-safe: + name: Rust CI (Safe Tests) timeout-minutes: 20 runs-on: ${{ matrix.os }} defaults: run: working-directory: ./ - #permissions: - #contents: read - #actions: read - #pull-requests: read strategy: fail-fast: false @@ -94,17 +90,80 @@ jobs: # workspaces: "rust -> target" key: ${{ env.RUST_CHANNEL }} - - name: Run basic tests + - name: Run safe tests if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' run: | just compile-tests "--locked" - just test --lib + just test # Runs all tests except #[ignore] ones (safe: unit tests + MinIO/Azurite) - - name: Run integration tests with external services - if: | - (matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm') && - (github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'test-external-services')) + - name: Run doc tests + if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' + run: | + just doctest + + - name: Run examples + if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' + run: | + just run-all-examples + + - name: Run unit tests only + if: matrix.os != 'ubuntu-latest' && matrix.os != 'ubuntu-24.04-arm' + run: | + cargo test --lib + + rust-external-services: + name: Rust CI (External Services) + timeout-minutes: 20 + runs-on: ubuntu-latest + environment: + name: external-services + url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + defaults: + run: + working-directory: ./ + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} + + - name: Stand up docker services + run: | + docker compose up -d + + - name: Wait for containers to be ready + run: | + for _ in {1..30}; do + if curl --silent --fail http://localhost:9000/minio/health/live; then + break + fi + sleep 1 + done + docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 + for _ in {1..60}; do + if curl --silent --fail "http://localhost:10000/devstoreaccount1/testcontainer?sv=2023-01-03&ss=btqf&srt=sco&spr=https%2Chttp&st=2025-01-06T14%3A53%3A30Z&se=2035-01-07T14%3A53%3A00Z&sp=rwdftlacup&sig=jclETGilOzONYp4Y0iK9SpVRLGyehaS5lg5booJ9VYA%3D&restype=container"; then + break + fi + sleep 1 + done + + - name: Install Just + run: sudo snap install --edge --classic just + + - name: Install Rust toolchain + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy + rustup default ${{ env.RUST_CHANNEL }} + + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + # workspaces: "rust -> target" + key: ${{ env.RUST_CHANNEL }} + + - name: Run external service tests env: RUST_LOG: trace R2_BUCKET: ${{ secrets.R2_BUCKET }} @@ -123,25 +182,39 @@ jobs: TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} run: | - just test + # Run only the external service tests (AWS, R2, Tigris) - excludes MinIO tests + cargo test --all --all-targets -- --ignored "_in_aws" + cargo test --all --all-targets -- --ignored "_in_r2" + cargo test --all --all-targets -- --ignored "_in_tigris" + + rust-cron-integration: + name: Rust CI (Cron Integration Tests) + timeout-minutes: 30 + runs-on: ubuntu-latest + if: github.event_name == 'cron' + environment: + name: external-services + url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + defaults: + run: + working-directory: ./ - - name: Run doc tests - if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' - run: | - just doctest + steps: + - name: Checkout repository + uses: actions/checkout@v5 - - name: Run examples - if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' + - name: Install Rust toolchain run: | - just run-all-examples + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy + rustup default ${{ env.RUST_CHANNEL }} - - name: Run unit tests only - if: matrix.os != 'ubuntu-latest' && matrix.os != 'ubuntu-24.04-arm' - run: | - cargo test --lib + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: ${{ env.RUST_CHANNEL }} - name: Run integration tests against object stores - if: github.event_name == 'cron' env: R2_BUCKET: ${{ secrets.R2_BUCKET }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} diff --git a/.github/workflows/test-external-pr.yaml b/.github/workflows/test-external-pr.yaml deleted file mode 100644 index f36095ebb..000000000 --- a/.github/workflows/test-external-pr.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: Test External PR with Secrets - -on: - workflow_dispatch: - inputs: - pr_number: - description: 'PR number to test' - required: true - type: number - -jobs: - test: - name: Test PR #${{ github.event.inputs.pr_number }} - uses: ./.github/workflows/rust-ci.yaml - with: - ref: refs/pull/${{ github.event.inputs.pr_number }}/merge - secrets: inherit From 30995fd143df7b7b18d704480e8faeaa911dde08 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 10:24:03 -0400 Subject: [PATCH 03/24] Combine external services and cron integration jobs for better CI efficiency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Merge rust-external-services and rust-cron-integration into single job - For manual approval (PR/push): Run targeted external service tests (AWS, R2, Tigris) - For cron schedule: Run comprehensive integration tests (all ignored tests) - Remove duplicate job infrastructure and unnecessary Docker setup - Maintain same deployment protection for both execution modes 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/rust-ci.yaml | 86 ++++++---------------------------- 1 file changed, 14 insertions(+), 72 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 9d5f9ad1a..ec0d53a12 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -113,8 +113,10 @@ jobs: rust-external-services: name: Rust CI (External Services) - timeout-minutes: 20 + timeout-minutes: 30 runs-on: ubuntu-latest + # Run for cron events or when deployment protection is triggered + if: github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push' environment: name: external-services url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -128,26 +130,6 @@ jobs: with: ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} - - name: Stand up docker services - run: | - docker compose up -d - - - name: Wait for containers to be ready - run: | - for _ in {1..30}; do - if curl --silent --fail http://localhost:9000/minio/health/live; then - break - fi - sleep 1 - done - docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 - for _ in {1..60}; do - if curl --silent --fail "http://localhost:10000/devstoreaccount1/testcontainer?sv=2023-01-03&ss=btqf&srt=sco&spr=https%2Chttp&st=2025-01-06T14%3A53%3A30Z&se=2035-01-07T14%3A53%3A00Z&sp=rwdftlacup&sig=jclETGilOzONYp4Y0iK9SpVRLGyehaS5lg5booJ9VYA%3D&restype=container"; then - break - fi - sleep 1 - done - - name: Install Just run: sudo snap install --edge --classic just @@ -182,54 +164,14 @@ jobs: TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} run: | - # Run only the external service tests (AWS, R2, Tigris) - excludes MinIO tests - cargo test --all --all-targets -- --ignored "_in_aws" - cargo test --all --all-targets -- --ignored "_in_r2" - cargo test --all --all-targets -- --ignored "_in_tigris" - - rust-cron-integration: - name: Rust CI (Cron Integration Tests) - timeout-minutes: 30 - runs-on: ubuntu-latest - if: github.event_name == 'cron' - environment: - name: external-services - url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - defaults: - run: - working-directory: ./ - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - - - name: Install Rust toolchain - run: | - rustup update --no-self-update ${{ env.RUST_CHANNEL }} - rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy - rustup default ${{ env.RUST_CHANNEL }} - - - name: Cache Dependencies - uses: Swatinem/rust-cache@v2 - with: - key: ${{ env.RUST_CHANNEL }} - - - name: Run integration tests against object stores - env: - R2_BUCKET: ${{ secrets.R2_BUCKET }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} - TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} - TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - - run: | - cargo test --all --all-targets -- --ignored + if [[ "${{ github.event_name }}" == "cron" ]]; then + # Cron job: run all ignored tests (comprehensive integration testing) + echo "Running comprehensive integration tests for cron job" + cargo test --all --all-targets -- --ignored + else + # Manual approval: run only specific external service tests (AWS, R2, Tigris) + echo "Running targeted external service tests" + cargo test --all --all-targets "_in_aws" -- --ignored + cargo test --all --all-targets "_in_r2" -- --ignored + cargo test --all --all-targets "_in_tigris" -- --ignored + fi From f6bbb6d546dd9b99b204c15c9ae90cd44149ff64 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 10:31:01 -0400 Subject: [PATCH 04/24] Use existing ci-with-secrets environment for deployment protection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update environment name from external-services to ci-with-secrets - Leverages existing GitHub environment configuration - Maintains deployment protection for external service tests 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/rust-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index ec0d53a12..cb34f7500 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -118,7 +118,7 @@ jobs: # Run for cron events or when deployment protection is triggered if: github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push' environment: - name: external-services + name: ci-with-secrets url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} defaults: run: From 04ddc522be91a5ee8398f7ffe1975b23c5b69f4c Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 10:46:06 -0400 Subject: [PATCH 05/24] Implement simple environment-based external PR protection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single rust-external-services job with conditional environment protection: - External PRs: Require manual approval via ci-with-secrets environment - Internal PRs & cron: Run automatically without environment protection - Combines external services and cron integration into unified job - Maintains test separation: safe tests (MinIO/Azurite) vs external services 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/external-service-tests.yml | 127 +++++++++++++++++++ .github/workflows/rust-ci.yaml | 12 +- 2 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/external-service-tests.yml diff --git a/.github/workflows/external-service-tests.yml b/.github/workflows/external-service-tests.yml new file mode 100644 index 000000000..9562c9f9c --- /dev/null +++ b/.github/workflows/external-service-tests.yml @@ -0,0 +1,127 @@ +name: External Service Tests + +on: + workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string + default: '' + needs_approval: + description: 'Whether this run needs manual approval' + required: false + type: boolean + default: false + +jobs: + run-tests-auto: + name: External Service Tests (Auto) + timeout-minutes: 30 + runs-on: ubuntu-latest + if: ${{ !inputs.needs_approval }} + defaults: + run: + working-directory: ./ + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} + + - name: Install Just + run: sudo snap install --edge --classic just + + - name: Install Rust toolchain + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy + rustup default ${{ env.RUST_CHANNEL }} + + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: ${{ env.RUST_CHANNEL }} + + - name: Run external service tests + env: + R2_BUCKET: ${{ secrets.R2_BUCKET }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} + TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} + TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} + TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} + run: | + if [[ "${{ github.event_name }}" == "cron" ]]; then + echo "Running comprehensive integration tests for cron job" + cargo test --all --all-targets -- --ignored + else + echo "Running targeted external service tests" + cargo test --all --all-targets "_in_aws" -- --ignored + cargo test --all --all-targets "_in_r2" -- --ignored + cargo test --all --all-targets "_in_tigris" -- --ignored + fi + + run-tests-approval: + name: External Service Tests (Approval Required) + timeout-minutes: 30 + runs-on: ubuntu-latest + if: ${{ inputs.needs_approval }} + environment: + name: ci-with-secrets + url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + defaults: + run: + working-directory: ./ + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} + + - name: Install Just + run: sudo snap install --edge --classic just + + - name: Install Rust toolchain + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy + rustup default ${{ env.RUST_CHANNEL }} + + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: ${{ env.RUST_CHANNEL }} + + - name: Run external service tests + env: + R2_BUCKET: ${{ secrets.R2_BUCKET }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} + TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} + TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} + TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} + run: | + if [[ "${{ github.event_name }}" == "cron" ]]; then + echo "Running comprehensive integration tests for cron job" + cargo test --all --all-targets -- --ignored + else + echo "Running targeted external service tests" + cargo test --all --all-targets "_in_aws" -- --ignored + cargo test --all --all-targets "_in_r2" -- --ignored + cargo test --all --all-targets "_in_tigris" -- --ignored + fi diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index cb34f7500..88acae089 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -115,11 +115,9 @@ jobs: name: Rust CI (External Services) timeout-minutes: 30 runs-on: ubuntu-latest - # Run for cron events or when deployment protection is triggered if: github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push' - environment: - name: ci-with-secrets - url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Apply environment protection only for external PRs + environment: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ci-with-secrets' || null }} defaults: run: working-directory: ./ @@ -142,7 +140,6 @@ jobs: - name: Cache Dependencies uses: Swatinem/rust-cache@v2 with: - # workspaces: "rust -> target" key: ${{ env.RUST_CHANNEL }} - name: Run external service tests @@ -152,24 +149,19 @@ jobs: R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} AWS_REGION: ${{ secrets.AWS_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - run: | if [[ "${{ github.event_name }}" == "cron" ]]; then - # Cron job: run all ignored tests (comprehensive integration testing) echo "Running comprehensive integration tests for cron job" cargo test --all --all-targets -- --ignored else - # Manual approval: run only specific external service tests (AWS, R2, Tigris) echo "Running targeted external service tests" cargo test --all --all-targets "_in_aws" -- --ignored cargo test --all --all-targets "_in_r2" -- --ignored From 0f2bd92ca5d14719a92f3a5e2b62368f2a4291dd Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 10:47:42 -0400 Subject: [PATCH 06/24] Remove unused external-service-tests.yml workflow file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clean up leftover file from shared workflow approach. Using single job with conditional environment protection instead. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/external-service-tests.yml | 127 ------------------- 1 file changed, 127 deletions(-) delete mode 100644 .github/workflows/external-service-tests.yml diff --git a/.github/workflows/external-service-tests.yml b/.github/workflows/external-service-tests.yml deleted file mode 100644 index 9562c9f9c..000000000 --- a/.github/workflows/external-service-tests.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: External Service Tests - -on: - workflow_call: - inputs: - ref: - description: 'Git ref to checkout' - required: false - type: string - default: '' - needs_approval: - description: 'Whether this run needs manual approval' - required: false - type: boolean - default: false - -jobs: - run-tests-auto: - name: External Service Tests (Auto) - timeout-minutes: 30 - runs-on: ubuntu-latest - if: ${{ !inputs.needs_approval }} - defaults: - run: - working-directory: ./ - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} - - - name: Install Just - run: sudo snap install --edge --classic just - - - name: Install Rust toolchain - run: | - rustup update --no-self-update ${{ env.RUST_CHANNEL }} - rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy - rustup default ${{ env.RUST_CHANNEL }} - - - name: Cache Dependencies - uses: Swatinem/rust-cache@v2 - with: - key: ${{ env.RUST_CHANNEL }} - - - name: Run external service tests - env: - R2_BUCKET: ${{ secrets.R2_BUCKET }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} - TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} - TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - run: | - if [[ "${{ github.event_name }}" == "cron" ]]; then - echo "Running comprehensive integration tests for cron job" - cargo test --all --all-targets -- --ignored - else - echo "Running targeted external service tests" - cargo test --all --all-targets "_in_aws" -- --ignored - cargo test --all --all-targets "_in_r2" -- --ignored - cargo test --all --all-targets "_in_tigris" -- --ignored - fi - - run-tests-approval: - name: External Service Tests (Approval Required) - timeout-minutes: 30 - runs-on: ubuntu-latest - if: ${{ inputs.needs_approval }} - environment: - name: ci-with-secrets - url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - defaults: - run: - working-directory: ./ - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} - - - name: Install Just - run: sudo snap install --edge --classic just - - - name: Install Rust toolchain - run: | - rustup update --no-self-update ${{ env.RUST_CHANNEL }} - rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy - rustup default ${{ env.RUST_CHANNEL }} - - - name: Cache Dependencies - uses: Swatinem/rust-cache@v2 - with: - key: ${{ env.RUST_CHANNEL }} - - - name: Run external service tests - env: - R2_BUCKET: ${{ secrets.R2_BUCKET }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} - TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} - TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - run: | - if [[ "${{ github.event_name }}" == "cron" ]]; then - echo "Running comprehensive integration tests for cron job" - cargo test --all --all-targets -- --ignored - else - echo "Running targeted external service tests" - cargo test --all --all-targets "_in_aws" -- --ignored - cargo test --all --all-targets "_in_r2" -- --ignored - cargo test --all --all-targets "_in_tigris" -- --ignored - fi From 2e815473123e4a35f0ecfd97c8b7c4b07d51918d Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 10:49:12 -0400 Subject: [PATCH 07/24] Fix checkout ref for regular workflow jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove inputs.ref which only applies to reusable workflows. Use standard github.event.pull_request.head.sha || github.sha instead. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/rust-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 88acae089..d7c4eb79c 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -50,7 +50,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 with: - ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Stand up docker services if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' @@ -126,7 +126,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 with: - ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Install Just run: sudo snap install --edge --classic just From 52df7f74c3f1c0f8cc917f378dec37d63f47e5b4 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 10:50:57 -0400 Subject: [PATCH 08/24] Clean up leftover reusable workflow artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove unused workflow_call section and inputs - Remove 'labeled' trigger that was for abandoned label-based bypass - Keep clean regular workflow without reusable workflow leftovers 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/rust-ci.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index d7c4eb79c..c1f6d06fd 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -4,20 +4,13 @@ name: Rust CI on: pull_request: - types: [opened, reopened, synchronize, labeled] + types: [opened, reopened, synchronize] push: branches: - main schedule: # three times a day to run the integration tests that take a long time - cron: '33 3,10,15 * * *' - workflow_call: - inputs: - ref: - description: 'Git ref to checkout' - required: false - type: string - default: '' concurrency: group: ${{ github.workflow }}-${{ github.ref }} From b846164acd6e066d4ab75e2081580a14938fd854 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 11:02:15 -0400 Subject: [PATCH 09/24] test reapproval --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1dff24e64..8f5188fb4 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ > **Icechunk 1.0 is released!** Better API, more performance and stability --- +small test change Icechunk is an open-source (Apache 2.0), transactional storage engine for tensor / ND-array data designed for use on cloud object storage. Icechunk works together with **[Zarr](https://zarr.dev/)**, augmenting the Zarr core data model with features that enhance performance, collaboration, and safety in a cloud-computing context. From 31c12d741622a802299aee065d9afdf8cb68a6ca Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 12 Sep 2025 14:21:25 -0400 Subject: [PATCH 10/24] change to trigger approval --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8f5188fb4..1dff24e64 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ > **Icechunk 1.0 is released!** Better API, more performance and stability --- -small test change Icechunk is an open-source (Apache 2.0), transactional storage engine for tensor / ND-array data designed for use on cloud object storage. Icechunk works together with **[Zarr](https://zarr.dev/)**, augmenting the Zarr core data model with features that enhance performance, collaboration, and safety in a cloud-computing context. From 224be0f79f7b6e69c21c6b11d0d8c8f7dea36259 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 14:06:10 -0400 Subject: [PATCH 11/24] use vars instead of secrets in new env --- .github/workflows/rust-ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index c1f6d06fd..b38282f74 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -138,16 +138,16 @@ jobs: - name: Run external service tests env: RUST_LOG: trace - R2_BUCKET: ${{ secrets.R2_BUCKET }} + R2_BUCKET: ${{ vars.R2_BUCKET }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_BUCKET: ${{ vars.AWS_BUCKET }} + AWS_REGION: ${{ vars.AWS_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} + TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} + TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} run: | From cca33a191cef27a158ba67a37a90401ac625574e Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 14:27:27 -0400 Subject: [PATCH 12/24] spelling --- .github/workflows/rust-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index b38282f74..bd5531a1d 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -142,8 +142,8 @@ jobs: R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ vars.AWS_BUCKET }} - AWS_REGION: ${{ vars.AWS_REGION }} + AWS_BUCKET: ${{ vars.S3_BUCKET }} + AWS_REGION: ${{ vars.S3_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} From 93a1e726b7356bd1947ea20c720757a5951ed03f Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 14:48:40 -0400 Subject: [PATCH 13/24] debug --- .github/workflows/rust-ci.yaml | 37 ++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index bd5531a1d..b27e34a72 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -135,6 +135,35 @@ jobs: with: key: ${{ env.RUST_CHANNEL }} + - name: Check environment variables + env: + R2_BUCKET: ${{ vars.R2_BUCKET }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + AWS_BUCKET: ${{ vars.S3_BUCKET }} + AWS_REGION: ${{ vars.S3_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} + TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} + TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} + TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} + run: | + echo "=== Environment Variables Status ===" + echo "R2_BUCKET: ${R2_BUCKET:+SET} ${R2_BUCKET:-UNSET}" + echo "R2_ACCESS_KEY_ID: ${R2_ACCESS_KEY_ID:+SET} ${R2_ACCESS_KEY_ID:-UNSET}" + echo "R2_SECRET_ACCESS_KEY: ${R2_SECRET_ACCESS_KEY:+SET} ${R2_SECRET_ACCESS_KEY:-UNSET}" + echo "R2_ACCOUNT_ID: ${R2_ACCOUNT_ID:+SET} ${R2_ACCOUNT_ID:-UNSET}" + echo "AWS_BUCKET: ${AWS_BUCKET:+SET} ${AWS_BUCKET:-UNSET}" + echo "AWS_REGION: ${AWS_REGION:+SET} ${AWS_REGION:-UNSET}" + echo "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:+SET} ${AWS_ACCESS_KEY_ID:-UNSET}" + echo "AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:+SET} ${AWS_SECRET_ACCESS_KEY:-UNSET}" + echo "TIGRIS_BUCKET: ${TIGRIS_BUCKET:+SET} ${TIGRIS_BUCKET:-UNSET}" + echo "TIGRIS_REGION: ${TIGRIS_REGION:+SET} ${TIGRIS_REGION:-UNSET}" + echo "TIGRIS_ACCESS_KEY_ID: ${TIGRIS_ACCESS_KEY_ID:+SET} ${TIGRIS_ACCESS_KEY_ID:-UNSET}" + echo "TIGRIS_SECRET_ACCESS_KEY: ${TIGRIS_SECRET_ACCESS_KEY:+SET} ${TIGRIS_SECRET_ACCESS_KEY:-UNSET}" + - name: Run external service tests env: RUST_LOG: trace @@ -153,10 +182,10 @@ jobs: run: | if [[ "${{ github.event_name }}" == "cron" ]]; then echo "Running comprehensive integration tests for cron job" - cargo test --all --all-targets -- --ignored + cargo test --package icechunk -- --ignored else echo "Running targeted external service tests" - cargo test --all --all-targets "_in_aws" -- --ignored - cargo test --all --all-targets "_in_r2" -- --ignored - cargo test --all --all-targets "_in_tigris" -- --ignored + cargo test --package icechunk "_in_aws" -- --ignored + cargo test --package icechunk "_in_r2" -- --ignored + cargo test --package icechunk "_in_tigris" -- --ignored fi From 6c4efd72daeb192843df9c303c63db090531ef00 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 14:54:05 -0400 Subject: [PATCH 14/24] more debug --- .github/workflows/rust-ci.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index b27e34a72..e834e5778 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -135,6 +135,15 @@ jobs: with: key: ${{ env.RUST_CHANNEL }} + - name: Debug PR context + run: | + echo "=== PR Context Debug ===" + echo "Event name: ${{ github.event_name }}" + echo "Head repo: ${{ github.event.pull_request.head.repo.full_name }}" + echo "Base repo: ${{ github.repository }}" + echo "Is external PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}" + echo "Environment should be: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ci-with-secrets' || 'null' }}" + - name: Check environment variables env: R2_BUCKET: ${{ vars.R2_BUCKET }} From bbbf369f200adff738c9c63955a10fff54b69910 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 14:59:04 -0400 Subject: [PATCH 15/24] debug --- .github/workflows/rust-ci.yaml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index e834e5778..d8ca3355e 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -143,9 +143,12 @@ jobs: echo "Base repo: ${{ github.repository }}" echo "Is external PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}" echo "Environment should be: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ci-with-secrets' || 'null' }}" + echo "Current environment: ${GITHUB_ENVIRONMENT:-none}" + echo "Job URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - name: Check environment variables env: + DEBUG_VAR: "DEBUG" R2_BUCKET: ${{ vars.R2_BUCKET }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} @@ -160,18 +163,19 @@ jobs: TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} run: | echo "=== Environment Variables Status ===" - echo "R2_BUCKET: ${R2_BUCKET:+SET} ${R2_BUCKET:-UNSET}" - echo "R2_ACCESS_KEY_ID: ${R2_ACCESS_KEY_ID:+SET} ${R2_ACCESS_KEY_ID:-UNSET}" - echo "R2_SECRET_ACCESS_KEY: ${R2_SECRET_ACCESS_KEY:+SET} ${R2_SECRET_ACCESS_KEY:-UNSET}" - echo "R2_ACCOUNT_ID: ${R2_ACCOUNT_ID:+SET} ${R2_ACCOUNT_ID:-UNSET}" - echo "AWS_BUCKET: ${AWS_BUCKET:+SET} ${AWS_BUCKET:-UNSET}" - echo "AWS_REGION: ${AWS_REGION:+SET} ${AWS_REGION:-UNSET}" - echo "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:+SET} ${AWS_ACCESS_KEY_ID:-UNSET}" - echo "AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:+SET} ${AWS_SECRET_ACCESS_KEY:-UNSET}" - echo "TIGRIS_BUCKET: ${TIGRIS_BUCKET:+SET} ${TIGRIS_BUCKET:-UNSET}" - echo "TIGRIS_REGION: ${TIGRIS_REGION:+SET} ${TIGRIS_REGION:-UNSET}" - echo "TIGRIS_ACCESS_KEY_ID: ${TIGRIS_ACCESS_KEY_ID:+SET} ${TIGRIS_ACCESS_KEY_ID:-UNSET}" - echo "TIGRIS_SECRET_ACCESS_KEY: ${TIGRIS_SECRET_ACCESS_KEY:+SET} ${TIGRIS_SECRET_ACCESS_KEY:-UNSET}" + echo "DEBUG_VAR: '${DEBUG_VAR}' - using \${VAR:+SET}: '${DEBUG_VAR:+SET}' - using \${VAR:-UNSET}: '${DEBUG_VAR:-UNSET}'" + echo "R2_BUCKET: '${R2_BUCKET}' - Status: ${R2_BUCKET:+SET}${R2_BUCKET:-UNSET}" + echo "R2_ACCESS_KEY_ID: Status: ${R2_ACCESS_KEY_ID:+SET}${R2_ACCESS_KEY_ID:-UNSET}" + echo "R2_SECRET_ACCESS_KEY: Status: ${R2_SECRET_ACCESS_KEY:+SET}${R2_SECRET_ACCESS_KEY:-UNSET}" + echo "R2_ACCOUNT_ID: Status: ${R2_ACCOUNT_ID:+SET}${R2_ACCOUNT_ID:-UNSET}" + echo "AWS_BUCKET: '${AWS_BUCKET}' - Status: ${AWS_BUCKET:+SET}${AWS_BUCKET:-UNSET}" + echo "AWS_REGION: '${AWS_REGION}' - Status: ${AWS_REGION:+SET}${AWS_REGION:-UNSET}" + echo "AWS_ACCESS_KEY_ID: Status: ${AWS_ACCESS_KEY_ID:+SET}${AWS_ACCESS_KEY_ID:-UNSET}" + echo "AWS_SECRET_ACCESS_KEY: Status: ${AWS_SECRET_ACCESS_KEY:+SET}${AWS_SECRET_ACCESS_KEY:-UNSET}" + echo "TIGRIS_BUCKET: '${TIGRIS_BUCKET}' - Status: ${TIGRIS_BUCKET:+SET}${TIGRIS_BUCKET:-UNSET}" + echo "TIGRIS_REGION: '${TIGRIS_REGION}' - Status: ${TIGRIS_REGION:+SET}${TIGRIS_REGION:-UNSET}" + echo "TIGRIS_ACCESS_KEY_ID: Status: ${TIGRIS_ACCESS_KEY_ID:+SET}${TIGRIS_ACCESS_KEY_ID:-UNSET}" + echo "TIGRIS_SECRET_ACCESS_KEY: Status: ${TIGRIS_SECRET_ACCESS_KEY:+SET}${TIGRIS_SECRET_ACCESS_KEY:-UNSET}" - name: Run external service tests env: From 38eb02c0befcabdc931241e0923f5fd5774ad6d0 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 15:03:03 -0400 Subject: [PATCH 16/24] debug env --- .github/workflows/rust-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index d8ca3355e..df8b6c93e 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -110,7 +110,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push' # Apply environment protection only for external PRs - environment: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ci-with-secrets' || null }} + environment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ci-with-secrets' }} defaults: run: working-directory: ./ From b68324638964879693321a6892185adfebe02a74 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 17:17:40 -0400 Subject: [PATCH 17/24] debug --- .github/workflows/rust-ci.yaml | 66 +++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index df8b6c93e..2efb0cae1 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -104,13 +104,67 @@ jobs: run: | cargo test --lib - rust-external-services: - name: Rust CI (External Services) + rust-external-services-internal: + name: Rust CI (External Services - Internal) timeout-minutes: 30 runs-on: ubuntu-latest - if: github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push' - # Apply environment protection only for external PRs - environment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ci-with-secrets' }} + if: (github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) + defaults: + run: + working-directory: ./ + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Install Just + run: sudo snap install --edge --classic just + + - name: Install Rust toolchain + run: | + rustup update --no-self-update ${{ env.RUST_CHANNEL }} + rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy + rustup default ${{ env.RUST_CHANNEL }} + + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: ${{ env.RUST_CHANNEL }} + + - name: Run external service tests (Internal) + env: + RUST_LOG: trace + R2_BUCKET: ${{ vars.R2_BUCKET }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + AWS_BUCKET: ${{ vars.S3_BUCKET }} + AWS_REGION: ${{ vars.S3_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} + TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} + TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} + TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} + run: | + if [[ "${{ github.event_name }}" == "cron" ]]; then + echo "Running comprehensive integration tests for cron job" + cargo test --package icechunk -- --ignored + else + echo "Running targeted external service tests" + cargo test --package icechunk "_in_aws" -- --ignored + cargo test --package icechunk "_in_r2" -- --ignored + cargo test --package icechunk "_in_tigris" -- --ignored + fi + + rust-external-services-external: + name: Rust CI (External Services - External PR) + timeout-minutes: 30 + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + environment: ci-with-secrets defaults: run: working-directory: ./ @@ -143,7 +197,9 @@ jobs: echo "Base repo: ${{ github.repository }}" echo "Is external PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}" echo "Environment should be: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ci-with-secrets' || 'null' }}" + echo "Actual environment expression: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ci-with-secrets' }}" echo "Current environment: ${GITHUB_ENVIRONMENT:-none}" + echo "Workflow SHA: ${{ github.sha }}" echo "Job URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - name: Check environment variables From 890bb0a3e8d51cf1fd51a452ab4f156426f97cd2 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 17:45:12 -0400 Subject: [PATCH 18/24] more debug --- .github/workflows/rust-ci.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 2efb0cae1..2343f7ffa 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -189,6 +189,13 @@ jobs: with: key: ${{ env.RUST_CHANNEL }} + - name: Debug External PR Environment + run: | + echo "=== External PR Job Debug ===" + echo "Job: rust-external-services-external" + echo "Current environment: ${GITHUB_ENVIRONMENT:-none}" + echo "This should be 'ci-with-secrets'" + - name: Debug PR context run: | echo "=== PR Context Debug ===" From 7641cf30c2ffa0799a35f4bc052264eb255230fe Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Fri, 26 Sep 2025 18:36:19 -0400 Subject: [PATCH 19/24] debug --- .github/workflows/external-pr-tests.yaml | 100 +++++++++++++++++++++ .github/workflows/rust-ci.yaml | 108 +---------------------- 2 files changed, 102 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/external-pr-tests.yaml diff --git a/.github/workflows/external-pr-tests.yaml b/.github/workflows/external-pr-tests.yaml new file mode 100644 index 000000000..4fa132a4e --- /dev/null +++ b/.github/workflows/external-pr-tests.yaml @@ -0,0 +1,100 @@ +name: External PR Tests (with secrets) + +on: + workflow_run: + workflows: ["Rust CI"] + types: + - completed + branches: + - main + +jobs: + external-pr-tests: + name: External PR Tests + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name != github.repository + environment: ci-with-secrets + defaults: + run: + working-directory: ./ + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ github.event.workflow_run.head_sha }} + + - name: Install Just + run: sudo snap install --edge --classic just + + - name: Install Rust toolchain + run: | + rustup update --no-self-update 1.89.0 + rustup component add --toolchain 1.89.0 rustfmt rust-src clippy + rustup default 1.89.0 + + - name: Cache Dependencies + uses: Swatinem/rust-cache@v2 + with: + key: 1.89.0 + + - name: Debug environment + run: | + echo "=== External PR Workflow Run Debug ===" + echo "Workflow run event: ${{ github.event.workflow_run.event }}" + echo "Head repo: ${{ github.event.workflow_run.head_repository.full_name }}" + echo "Base repo: ${{ github.repository }}" + echo "Head SHA: ${{ github.event.workflow_run.head_sha }}" + echo "Current environment: ${GITHUB_ENVIRONMENT:-none}" + + - name: Check environment variables + env: + DEBUG_VAR: "DEBUG" + R2_BUCKET: ${{ vars.R2_BUCKET }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + AWS_BUCKET: ${{ vars.S3_BUCKET }} + AWS_REGION: ${{ vars.S3_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} + TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} + TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} + TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} + run: | + echo "=== Environment Variables Status ===" + echo "DEBUG_VAR: '${DEBUG_VAR}' - Status: ${DEBUG_VAR:+SET}${DEBUG_VAR:-UNSET}" + echo "R2_BUCKET: '${R2_BUCKET}' - Status: ${R2_BUCKET:+SET}${R2_BUCKET:-UNSET}" + echo "R2_ACCESS_KEY_ID: Status: ${R2_ACCESS_KEY_ID:+SET}${R2_ACCESS_KEY_ID:-UNSET}" + echo "R2_SECRET_ACCESS_KEY: Status: ${R2_SECRET_ACCESS_KEY:+SET}${R2_SECRET_ACCESS_KEY:-UNSET}" + echo "R2_ACCOUNT_ID: Status: ${R2_ACCOUNT_ID:+SET}${R2_ACCOUNT_ID:-UNSET}" + echo "AWS_BUCKET: '${AWS_BUCKET}' - Status: ${AWS_BUCKET:+SET}${AWS_BUCKET:-UNSET}" + echo "AWS_REGION: '${AWS_REGION}' - Status: ${AWS_REGION:+SET}${AWS_REGION:-UNSET}" + echo "AWS_ACCESS_KEY_ID: Status: ${AWS_ACCESS_KEY_ID:+SET}${AWS_ACCESS_KEY_ID:-UNSET}" + echo "AWS_SECRET_ACCESS_KEY: Status: ${AWS_SECRET_ACCESS_KEY:+SET}${AWS_SECRET_ACCESS_KEY:-UNSET}" + echo "TIGRIS_BUCKET: '${TIGRIS_BUCKET}' - Status: ${TIGRIS_BUCKET:+SET}${TIGRIS_BUCKET:-UNSET}" + echo "TIGRIS_REGION: '${TIGRIS_REGION}' - Status: ${TIGRIS_REGION:+SET}${TIGRIS_REGION:-UNSET}" + echo "TIGRIS_ACCESS_KEY_ID: Status: ${TIGRIS_ACCESS_KEY_ID:+SET}${TIGRIS_ACCESS_KEY_ID:-UNSET}" + echo "TIGRIS_SECRET_ACCESS_KEY: Status: ${TIGRIS_SECRET_ACCESS_KEY:+SET}${TIGRIS_SECRET_ACCESS_KEY:-UNSET}" + + - name: Run external service tests + env: + RUST_LOG: trace + R2_BUCKET: ${{ vars.R2_BUCKET }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} + AWS_BUCKET: ${{ vars.S3_BUCKET }} + AWS_REGION: ${{ vars.S3_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} + TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} + TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} + TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} + run: | + echo "Running external service tests with secrets access" + cargo test --package icechunk "_in_aws" -- --ignored + cargo test --package icechunk "_in_r2" -- --ignored + cargo test --package icechunk "_in_tigris" -- --ignored diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 2343f7ffa..4ddb462f8 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -159,109 +159,5 @@ jobs: cargo test --package icechunk "_in_tigris" -- --ignored fi - rust-external-services-external: - name: Rust CI (External Services - External PR) - timeout-minutes: 30 - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository - environment: ci-with-secrets - defaults: - run: - working-directory: ./ - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Install Just - run: sudo snap install --edge --classic just - - - name: Install Rust toolchain - run: | - rustup update --no-self-update ${{ env.RUST_CHANNEL }} - rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy - rustup default ${{ env.RUST_CHANNEL }} - - - name: Cache Dependencies - uses: Swatinem/rust-cache@v2 - with: - key: ${{ env.RUST_CHANNEL }} - - - name: Debug External PR Environment - run: | - echo "=== External PR Job Debug ===" - echo "Job: rust-external-services-external" - echo "Current environment: ${GITHUB_ENVIRONMENT:-none}" - echo "This should be 'ci-with-secrets'" - - - name: Debug PR context - run: | - echo "=== PR Context Debug ===" - echo "Event name: ${{ github.event_name }}" - echo "Head repo: ${{ github.event.pull_request.head.repo.full_name }}" - echo "Base repo: ${{ github.repository }}" - echo "Is external PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}" - echo "Environment should be: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ci-with-secrets' || 'null' }}" - echo "Actual environment expression: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ci-with-secrets' }}" - echo "Current environment: ${GITHUB_ENVIRONMENT:-none}" - echo "Workflow SHA: ${{ github.sha }}" - echo "Job URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - - - name: Check environment variables - env: - DEBUG_VAR: "DEBUG" - R2_BUCKET: ${{ vars.R2_BUCKET }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ vars.S3_BUCKET }} - AWS_REGION: ${{ vars.S3_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} - TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} - TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - run: | - echo "=== Environment Variables Status ===" - echo "DEBUG_VAR: '${DEBUG_VAR}' - using \${VAR:+SET}: '${DEBUG_VAR:+SET}' - using \${VAR:-UNSET}: '${DEBUG_VAR:-UNSET}'" - echo "R2_BUCKET: '${R2_BUCKET}' - Status: ${R2_BUCKET:+SET}${R2_BUCKET:-UNSET}" - echo "R2_ACCESS_KEY_ID: Status: ${R2_ACCESS_KEY_ID:+SET}${R2_ACCESS_KEY_ID:-UNSET}" - echo "R2_SECRET_ACCESS_KEY: Status: ${R2_SECRET_ACCESS_KEY:+SET}${R2_SECRET_ACCESS_KEY:-UNSET}" - echo "R2_ACCOUNT_ID: Status: ${R2_ACCOUNT_ID:+SET}${R2_ACCOUNT_ID:-UNSET}" - echo "AWS_BUCKET: '${AWS_BUCKET}' - Status: ${AWS_BUCKET:+SET}${AWS_BUCKET:-UNSET}" - echo "AWS_REGION: '${AWS_REGION}' - Status: ${AWS_REGION:+SET}${AWS_REGION:-UNSET}" - echo "AWS_ACCESS_KEY_ID: Status: ${AWS_ACCESS_KEY_ID:+SET}${AWS_ACCESS_KEY_ID:-UNSET}" - echo "AWS_SECRET_ACCESS_KEY: Status: ${AWS_SECRET_ACCESS_KEY:+SET}${AWS_SECRET_ACCESS_KEY:-UNSET}" - echo "TIGRIS_BUCKET: '${TIGRIS_BUCKET}' - Status: ${TIGRIS_BUCKET:+SET}${TIGRIS_BUCKET:-UNSET}" - echo "TIGRIS_REGION: '${TIGRIS_REGION}' - Status: ${TIGRIS_REGION:+SET}${TIGRIS_REGION:-UNSET}" - echo "TIGRIS_ACCESS_KEY_ID: Status: ${TIGRIS_ACCESS_KEY_ID:+SET}${TIGRIS_ACCESS_KEY_ID:-UNSET}" - echo "TIGRIS_SECRET_ACCESS_KEY: Status: ${TIGRIS_SECRET_ACCESS_KEY:+SET}${TIGRIS_SECRET_ACCESS_KEY:-UNSET}" - - - name: Run external service tests - env: - RUST_LOG: trace - R2_BUCKET: ${{ vars.R2_BUCKET }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ vars.S3_BUCKET }} - AWS_REGION: ${{ vars.S3_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} - TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} - TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - run: | - if [[ "${{ github.event_name }}" == "cron" ]]; then - echo "Running comprehensive integration tests for cron job" - cargo test --package icechunk -- --ignored - else - echo "Running targeted external service tests" - cargo test --package icechunk "_in_aws" -- --ignored - cargo test --package icechunk "_in_r2" -- --ignored - cargo test --package icechunk "_in_tigris" -- --ignored - fi + # External PRs will trigger the "External PR Tests" workflow via workflow_run + # This provides a secure way to run tests with secrets for external PRs From 121778eeef334a6bf00696bffd87576c38ca4cfd Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Tue, 30 Sep 2025 22:08:33 -0400 Subject: [PATCH 20/24] cleanup --- .github/workflows/external-pr-tests.yaml | 100 ----------------------- .github/workflows/rust-ci.yaml | 10 +-- 2 files changed, 4 insertions(+), 106 deletions(-) delete mode 100644 .github/workflows/external-pr-tests.yaml diff --git a/.github/workflows/external-pr-tests.yaml b/.github/workflows/external-pr-tests.yaml deleted file mode 100644 index 4fa132a4e..000000000 --- a/.github/workflows/external-pr-tests.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: External PR Tests (with secrets) - -on: - workflow_run: - workflows: ["Rust CI"] - types: - - completed - branches: - - main - -jobs: - external-pr-tests: - name: External PR Tests - runs-on: ubuntu-latest - if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name != github.repository - environment: ci-with-secrets - defaults: - run: - working-directory: ./ - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - ref: ${{ github.event.workflow_run.head_sha }} - - - name: Install Just - run: sudo snap install --edge --classic just - - - name: Install Rust toolchain - run: | - rustup update --no-self-update 1.89.0 - rustup component add --toolchain 1.89.0 rustfmt rust-src clippy - rustup default 1.89.0 - - - name: Cache Dependencies - uses: Swatinem/rust-cache@v2 - with: - key: 1.89.0 - - - name: Debug environment - run: | - echo "=== External PR Workflow Run Debug ===" - echo "Workflow run event: ${{ github.event.workflow_run.event }}" - echo "Head repo: ${{ github.event.workflow_run.head_repository.full_name }}" - echo "Base repo: ${{ github.repository }}" - echo "Head SHA: ${{ github.event.workflow_run.head_sha }}" - echo "Current environment: ${GITHUB_ENVIRONMENT:-none}" - - - name: Check environment variables - env: - DEBUG_VAR: "DEBUG" - R2_BUCKET: ${{ vars.R2_BUCKET }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ vars.S3_BUCKET }} - AWS_REGION: ${{ vars.S3_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} - TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} - TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - run: | - echo "=== Environment Variables Status ===" - echo "DEBUG_VAR: '${DEBUG_VAR}' - Status: ${DEBUG_VAR:+SET}${DEBUG_VAR:-UNSET}" - echo "R2_BUCKET: '${R2_BUCKET}' - Status: ${R2_BUCKET:+SET}${R2_BUCKET:-UNSET}" - echo "R2_ACCESS_KEY_ID: Status: ${R2_ACCESS_KEY_ID:+SET}${R2_ACCESS_KEY_ID:-UNSET}" - echo "R2_SECRET_ACCESS_KEY: Status: ${R2_SECRET_ACCESS_KEY:+SET}${R2_SECRET_ACCESS_KEY:-UNSET}" - echo "R2_ACCOUNT_ID: Status: ${R2_ACCOUNT_ID:+SET}${R2_ACCOUNT_ID:-UNSET}" - echo "AWS_BUCKET: '${AWS_BUCKET}' - Status: ${AWS_BUCKET:+SET}${AWS_BUCKET:-UNSET}" - echo "AWS_REGION: '${AWS_REGION}' - Status: ${AWS_REGION:+SET}${AWS_REGION:-UNSET}" - echo "AWS_ACCESS_KEY_ID: Status: ${AWS_ACCESS_KEY_ID:+SET}${AWS_ACCESS_KEY_ID:-UNSET}" - echo "AWS_SECRET_ACCESS_KEY: Status: ${AWS_SECRET_ACCESS_KEY:+SET}${AWS_SECRET_ACCESS_KEY:-UNSET}" - echo "TIGRIS_BUCKET: '${TIGRIS_BUCKET}' - Status: ${TIGRIS_BUCKET:+SET}${TIGRIS_BUCKET:-UNSET}" - echo "TIGRIS_REGION: '${TIGRIS_REGION}' - Status: ${TIGRIS_REGION:+SET}${TIGRIS_REGION:-UNSET}" - echo "TIGRIS_ACCESS_KEY_ID: Status: ${TIGRIS_ACCESS_KEY_ID:+SET}${TIGRIS_ACCESS_KEY_ID:-UNSET}" - echo "TIGRIS_SECRET_ACCESS_KEY: Status: ${TIGRIS_SECRET_ACCESS_KEY:+SET}${TIGRIS_SECRET_ACCESS_KEY:-UNSET}" - - - name: Run external service tests - env: - RUST_LOG: trace - R2_BUCKET: ${{ vars.R2_BUCKET }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - AWS_BUCKET: ${{ vars.S3_BUCKET }} - AWS_REGION: ${{ vars.S3_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TIGRIS_BUCKET: ${{ vars.TIGRIS_BUCKET }} - TIGRIS_REGION: ${{ vars.TIGRIS_REGION }} - TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} - TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} - run: | - echo "Running external service tests with secrets access" - cargo test --package icechunk "_in_aws" -- --ignored - cargo test --package icechunk "_in_r2" -- --ignored - cargo test --package icechunk "_in_tigris" -- --ignored diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 4ddb462f8..ca8d9c887 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -104,10 +104,11 @@ jobs: run: | cargo test --lib - rust-external-services-internal: - name: Rust CI (External Services - Internal) + rust-external-services: + name: Rust CI (External Services) timeout-minutes: 30 runs-on: ubuntu-latest + # Skip external service tests for external PRs (they can't access secrets anyway) if: (github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) defaults: run: @@ -133,7 +134,7 @@ jobs: with: key: ${{ env.RUST_CHANNEL }} - - name: Run external service tests (Internal) + - name: Run external service tests env: RUST_LOG: trace R2_BUCKET: ${{ vars.R2_BUCKET }} @@ -158,6 +159,3 @@ jobs: cargo test --package icechunk "_in_r2" -- --ignored cargo test --package icechunk "_in_tigris" -- --ignored fi - - # External PRs will trigger the "External PR Tests" workflow via workflow_run - # This provides a secure way to run tests with secrets for external PRs From 8c84fcf8cabeb87352ee109f0aa67df47b2be6df Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Tue, 30 Sep 2025 22:09:50 -0400 Subject: [PATCH 21/24] simplify --- .github/workflows/rust-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index ca8d9c887..867cfba33 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -109,7 +109,7 @@ jobs: timeout-minutes: 30 runs-on: ubuntu-latest # Skip external service tests for external PRs (they can't access secrets anyway) - if: (github.event_name == 'cron' || github.event_name == 'pull_request' || github.event_name == 'push') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository defaults: run: working-directory: ./ From f983dd956dca02d1a25b3576e3e7b98f2716c584 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Tue, 30 Sep 2025 22:54:49 -0400 Subject: [PATCH 22/24] debug --- .github/workflows/rust-ci.yaml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 867cfba33..ee97bd3a9 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -104,12 +104,26 @@ jobs: run: | cargo test --lib + debug-external-pr-check: + name: Debug External PR Check + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Debug PR info + run: | + echo "=== PR Debug Info ===" + echo "Event: ${{ github.event_name }}" + echo "Head repo: ${{ github.event.pull_request.head.repo.full_name }}" + echo "Base repo: ${{ github.repository }}" + echo "Are they equal: ${{ github.event.pull_request.head.repo.full_name == github.repository }}" + echo "Condition result: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository) }}" + rust-external-services: name: Rust CI (External Services) timeout-minutes: 30 runs-on: ubuntu-latest # Skip external service tests for external PRs (they can't access secrets anyway) - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + if: github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository) defaults: run: working-directory: ./ @@ -134,6 +148,14 @@ jobs: with: key: ${{ env.RUST_CHANNEL }} + - name: Debug job execution + run: | + echo "=== Debug Job Execution ===" + echo "Event: ${{ github.event_name }}" + echo "Head repo: ${{ github.event.pull_request.head.repo.full_name }}" + echo "Base repo: ${{ github.repository }}" + echo "Should run: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository) }}" + - name: Run external service tests env: RUST_LOG: trace From af6c16ab385c1f317137fbae5baf0b9440ba0655 Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Tue, 30 Sep 2025 22:54:53 -0400 Subject: [PATCH 23/24] debug --- .github/workflows/rust-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index ee97bd3a9..1a0d487c1 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -123,7 +123,7 @@ jobs: timeout-minutes: 30 runs-on: ubuntu-latest # Skip external service tests for external PRs (they can't access secrets anyway) - if: github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository) + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} defaults: run: working-directory: ./ From fcca8863271c1496970ad21e70dcf46d6eb6df1e Mon Sep 17 00:00:00 2001 From: Ian Hunt-Isaak Date: Tue, 30 Sep 2025 23:00:32 -0400 Subject: [PATCH 24/24] remove debug --- .github/workflows/rust-ci.yaml | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 1a0d487c1..0262dc626 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -10,7 +10,7 @@ on: - main schedule: # three times a day to run the integration tests that take a long time - - cron: '33 3,10,15 * * *' + - cron: "33 3,10,15 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,7 +23,7 @@ env: RUST_BACKTRACE: short RUSTFLAGS: "-D warnings -W unreachable-pub -W bare-trait-objects" RUSTUP_MAX_RETRIES: 10 - RUST_CHANNEL: '1.89.0' + RUST_CHANNEL: "1.89.0" jobs: rust-safe: @@ -104,20 +104,6 @@ jobs: run: | cargo test --lib - debug-external-pr-check: - name: Debug External PR Check - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - name: Debug PR info - run: | - echo "=== PR Debug Info ===" - echo "Event: ${{ github.event_name }}" - echo "Head repo: ${{ github.event.pull_request.head.repo.full_name }}" - echo "Base repo: ${{ github.repository }}" - echo "Are they equal: ${{ github.event.pull_request.head.repo.full_name == github.repository }}" - echo "Condition result: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository) }}" - rust-external-services: name: Rust CI (External Services) timeout-minutes: 30 @@ -148,17 +134,8 @@ jobs: with: key: ${{ env.RUST_CHANNEL }} - - name: Debug job execution - run: | - echo "=== Debug Job Execution ===" - echo "Event: ${{ github.event_name }}" - echo "Head repo: ${{ github.event.pull_request.head.repo.full_name }}" - echo "Base repo: ${{ github.repository }}" - echo "Should run: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository) }}" - - name: Run external service tests env: - RUST_LOG: trace R2_BUCKET: ${{ vars.R2_BUCKET }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}