tests.yml

name: Tests

on:
  push:
    branches:
      - "**"
    # A deploy pushes production-<sha>/<date> and v<date>.<n> to a commit that
    # already passed on main.
    tags-ignore:
      - "**"
  pull_request_target:

permissions:
  contents: read

concurrency:
  # One lane per main commit: a shared lane runs one commit at a time and lets
  # a newer push replace a pending run, so a burst of merges queued every suite
  # behind the last. A re-run of a commit stays in that commit's lane.
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request_target' && github.event.pull_request.number || github.ref }}${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && format('-{0}', github.sha) || '' }}
  cancel-in-progress: ${{ github.event_name == 'pull_request_target' || github.ref != 'refs/heads/main' }}

jobs:
  ci_gate:
    name: CI Gate${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && ' (internal PR noop)' || '' }}
    runs-on: ubicloud-standard-2
    environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'ci-protected' || '' }}
    # The one run-all-specs lookup of the run. The image build, run_scope and
    # test_relevant_runner read it: two lookups could disagree when the label lands
    # between them, and a skipped build would then skip the full suite while
    # ci/green turned green.
    permissions:
      pull-requests: read
    outputs:
      full: ${{ steps.scope.outputs.full }}
      base_sha: ${{ steps.base.outputs.sha }}
    steps:
      - id: scope
        env:
          GH_TOKEN: ${{ github.token }}
          SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
        run: |
          if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then
            echo "full=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          if [ "$GITHUB_EVENT_NAME" = "pull_request_target" ]; then
            LABELS=$(gh api "repos/$GITHUB_REPOSITORY/pulls/${{ github.event.pull_request.number }}" --jq '[.labels[].name] | join("\n")')
          else
            LABELS=$(gh api "repos/$GITHUB_REPOSITORY/commits/$SHA/pulls" --jq '[.[].labels[].name] | join("\n")' || true)
          fi
          if echo "$LABELS" | grep -qx 'run-all-specs'; then
            echo "full=true" >> "$GITHUB_OUTPUT"
          else
            echo "full=false" >> "$GITHUB_OUTPUT"
          fi
      # One base commit for every test_relevant_runner shard: each selects its own
      # specs, and Knapsack builds the queue from whichever shard's list comes first.
      - id: base
        if: steps.scope.outputs.full != 'true' && github.event_name == 'push'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          SHA=$(gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)
          [[ "$SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::could not resolve main's commit"; exit 1; }
          echo "sha=$SHA" >> "$GITHUB_OUTPUT"

  run_scope:
    # Decides whether this run gets the full Fast/Slow suite or the trimmed
    # test_relevant job. Full suite: every push to main (deploy gate), or a
    # PR carrying the run-all-specs label. Push events have no PR payload,
    # so the label is looked up via the commit's PRs.
    #
    # The label lookup itself runs in ci_gate, so the image build reads the same
    # answer. Reuse of a prior Fast/Slow run lives in reuse_scope, which is
    # main-push only.
    name: Run scope
    runs-on: ubicloud-standard-2
    needs: ci_gate
    if: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository }}
    outputs:
      full: ${{ steps.scope.outputs.full }}
    steps:
      - id: scope
        env:
          FULL: ${{ needs.ci_gate.outputs.full }}
        run: echo "full=$FULL" >> "$GITHUB_OUTPUT"

  reuse_scope:
    # Main-only. Asks whether this exact git tree already passed Fast+Slow on
    # a merged PR. Isolated from pull_request_target so a fork cannot put
    # bin/reuse-full-suite in front of a token that can read Actions.
    name: Reuse full suite
    runs-on: ubicloud-standard-2
    needs: ci_gate
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    permissions:
      pull-requests: read
      actions: read
      contents: read
    outputs:
      reuse: ${{ steps.reuse.outputs.reuse }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.sha }}
          persist-credentials: false
          sparse-checkout: |
            bin/reuse-full-suite
            .github/workflows/tests.yml
          sparse-checkout-cone-mode: false
      - id: reuse
        env:
          GH_TOKEN: ${{ github.token }}
          SHA: ${{ github.sha }}
        run: |
          echo "reuse=false" >> "$GITHUB_OUTPUT"
          REPO="$GITHUB_REPOSITORY" bin/reuse-full-suite "$SHA" || true

  lint_js:
    name: Lint JS/TS
    runs-on: ubicloud-standard-4
    needs: ci_gate
    if: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository }}
    steps:
      # Full history without file contents: bin/vitest-ci diffs the branch against
      # its merge base with main.
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          persist-credentials: false
          fetch-depth: 0
          filter: blob:none
      - uses: actions/setup-node@v4
        with:
          node-version-file: package.json
          cache: npm
      # Saved only by main pushes: a fork PR's pull_request_target run writes to the
      # base branch's cache scope and must not be able to plant node_modules there.
      - uses: actions/cache/restore@v4
        id: node_modules
        with:
          path: node_modules
          key: node-modules-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'package-lock.json', '.npmrc', 'patches/**') }}
      - run: npm ci
        if: steps.node_modules.outputs.cache-hit != 'true'
      - uses: actions/cache/save@v4
        if: steps.node_modules.outputs.cache-hit != 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          path: node_modules
          key: ${{ steps.node_modules.outputs.cache-primary-key }}
      - run: npm run lint-fast
      # Branch pushes run the tests related to their changes; main and fork PRs
      # run the whole suite.
      - name: Run vitest
        run: |
          if [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" != "refs/heads/main" ]]; then
            ruby bin/vitest-ci origin/main
          else
            npm test
          fi
        env:
          LANG: en_US.UTF-8
          LC_ALL: en_US.UTF-8

  typecheck:
    name: Typecheck TS
    runs-on: ubicloud-standard-4
    needs: ci_gate
    if: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          persist-credentials: false
      - uses: ruby/setup-ruby@v1
        with:
          bundler-cache: true
      - uses: actions/setup-node@v4
        with:
          node-version-file: package.json
          cache: npm
      - uses: actions/cache/restore@v4
        id: node_modules
        with:
          path: node_modules
          key: node-modules-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'package-lock.json', '.npmrc', 'patches/**') }}
      - run: npm ci
        if: steps.node_modules.outputs.cache-hit != 'true'
      - uses: actions/cache/save@v4
        if: steps.node_modules.outputs.cache-hit != 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
        with:
          path: node_modules
          key: ${{ steps.node_modules.outputs.cache-primary-key }}
      # Builds and saves the test assets that test_relevant_runner restores. It runs
      # on main, on the same runner type, because the build job's Vite cache lives
      # with another runner provider. The key hashes that cache's inputs, with this
      # file as the recipe, before js:export writes into app/javascript.
      - name: Restore the runner test assets
        id: test_assets
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        uses: actions/cache/restore@v4
        with:
          path: |
            public/vite-test
            tmp/cache/vite
            public/js/gumroad-bundle.js
            public/js/gumroad-embed-bundle.js
            public/pages-tailwind.css
            public/pages-tailwind-manifest.json
            public/assets/pages
            app/javascript/stylesheets/pages_tailwind.generated.html
          key: >-
            runner-test-assets-v1-${{ runner.os }}-${{ hashFiles(
              'app/javascript/**', 'app/views/**',
              'config/routes.rb', 'config/routes/**', 'config/domain.rb',
              'config/application.rb', 'config/boot.rb', 'config/environment.rb',
              'config/environments/**', 'config/redis.rb',
              'lib/catch_bad_request_errors.rb', 'lib/utilities/global_config.rb',
              'config/initializers/js-routes.rb', 'config/vite.json', 'config/vite/**',
              'config/currencies.json', 'lib/json_schemas/**', 'lib/tasks/**', 'Rakefile',
              'public/**', 'vendor/assets/**', 'patches/**', 'scripts/build_pages_tailwind.mjs',
              'docker/base/**', 'docker/web/Dockerfile.test', '.github/workflows/tests.yml',
              'package.json', 'package-lock.json', '.npmrc', 'tsconfig.json',
              'vite.config.ts', 'vite.config.widget.ts', 'postcss.config.js',
              'Gemfile.lock', '.ruby-version') }}
      # tsc needs the gitignored js:export output (app/javascript/utils/routes.*
      # and app/javascript/json_schemas/*). The export boots Rails off the
      # committed .env.test but tolerates every backing service being down, so
      # no MySQL/Redis/etc. containers are needed here.
      - run: bundle exec rails js:export
        env:
          RAILS_ENV: test
      - run: npm run typecheck
      # Downloads the files test_relevant_runner installs (bin/ci-test-packages), on
      # the same runner type, so its shards download nothing.
      - name: Restore the test packages
        id: test_packages
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        uses: actions/cache/restore@v4
        with:
          path: ~/test-packages
          key: test-packages-v1-${{ runner.os }}-${{ hashFiles('bin/ci-test-packages') }}
      # Best-effort: the shards download whatever the cache lacks, so a failed
      # download must not fail main's checks.
      - name: Download the test packages
        id: download_test_packages
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.test_packages.outputs.cache-hit != 'true'
        continue-on-error: true
        run: bin/ci-test-packages download ~/test-packages
      - uses: actions/cache/save@v4
        if: steps.download_test_packages.outcome == 'success'
        with:
          path: ~/test-packages
          key: ${{ steps.test_packages.outputs.cache-primary-key }}
      - name: Build the runner test assets
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.test_assets.outputs.cache-hit != 'true'
        # pages_tailwind:build first, as in test_relevant_runner.
        run: bundle exec rake pages_tailwind:build assets:precompile
        env:
          RAILS_ENV: test
          NODE_ENV: test
          VITE_RUBY_SKIP_ASSETS_PRECOMPILE_INSTALL: true
      - uses: actions/cache/save@v4
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.test_assets.outputs.cache-hit != 'true'
        with:
          path: |
            public/vite-test
            tmp/cache/vite
            public/js/gumroad-bundle.js
            public/js/gumroad-embed-bundle.js
            public/pages-tailwind.css
            public/pages-tailwind-manifest.json
            public/assets/pages
            app/javascript/stylesheets/pages_tailwind.generated.html
          key: ${{ steps.test_assets.outputs.cache-primary-key }}

  lint_ruby:
    name: Lint Ruby
    runs-on: ubicloud-standard-4
    needs: ci_gate
    if: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          persist-credentials: false
      - uses: ruby/setup-ruby@v1
        with:
          bundler-cache: true
      - run: bundle exec rubocop --format github

  migration_versions:
    name: Migration versions
    runs-on: ubicloud-standard-2
    needs: ci_gate
    if: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          # A duplicate version is only visible against the base branch's
          # migration set, so the base ref has to be in the clone.
          fetch-depth: 0
          persist-credentials: false
      # Filenames only, no bundle: this must fail before the shards spend an
      # hour to reach the same `db:prepare` abort.
      - name: Check migration versions against the base branch
        env:
          BASE_REF: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.ref || 'main' }}
        run: ruby bin/check-migration-versions "origin/$BASE_REF" HEAD

      - name: Self-test the checker
        run: ruby spec/bin/check_migration_versions_test.rb

      # Hosted here, not in relevant_specs: run-all-specs skips that job, and
      # every PR that edits the selector must carry the label, so a self-test
      # there would never run on the PRs that change what it tests. This job
      # runs on every PR and needs no bundle, same as the self-test.
      - name: Self-test the branch-specs selector
        run: ruby spec/bin/branch_specs_test.rb

      - name: Self-test the main spec-failure classifier
        run: ruby spec/bin/classify_main_spec_failure_test.rb

      - name: Self-test the timeout re-run classifier
        run: ruby spec/bin/classify_hung_checkout_test.rb

      - name: Self-test the deploy-before-main-suite gate
        run: ruby spec/bin/deploy_before_main_suite_gate_test.rb

      - name: Self-test the Buildkite unblock script
        run: ruby spec/bin/unblock_buildkite_deploy_test.rb

      # Hosted here so a reuse-only PR cannot ship on the user_spec.rb floor.
      - name: Self-test reuse-full-suite
        run: bash script/test-reuse-full-suite

      - name: Self-test the revert decision for red main deploys
        run: ruby spec/bin/revert_red_main_decision_test.rb

      - name: Self-test the vitest selection for branches
        run: ruby spec/bin/vitest_ci_test.rb

      - name: Self-test the main asset cache shadow
        run: bash .buildkite/scripts/main_asset_cache_shadow_test.sh

      - name: Self-test the web pre-scale step
        run: bash .buildkite/scripts/prescale_web_clusters_test.sh

  # Branch pushes build the test image in the Prebuild test image workflow, after
  # this one, so ci/green does not wait for it. A branch run builds it here only
  # when its own full-suite jobs use it.
  build:
    name: Build images${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && ' (internal PR noop)' || '' }}
    needs: ci_gate
    if: |
      (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository) &&
      (github.event_name != 'push' || github.ref == 'refs/heads/main' || needs.ci_gate.outputs.full == 'true')
    uses: ./.github/workflows/build-test-image.yml
    secrets: inherit

  test_minitest:
    name: Test Minitest ${{ matrix.ci_node_index }}${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && ' (internal PR noop)' || '' }}
    needs: ci_gate
    if: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository }}
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        ci_node_total: [8]
        ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7]

    env:
      RAILS_ENV: test
      BUNDLE_GEMS__CONTRIBSYS__COM: ${{ secrets.BUNDLE_GEMS__CONTRIBSYS__COM }}

    steps:
      # Before checkout, so no branch or fork code runs while the Docker Hub login
      # exists. GitHub service containers cannot pass mysqld flags, and these skip
      # the disk syncs the tests do not need; the default collation stays. MySQL
      # and Elasticsearch finish starting while the steps below run.
      - name: Start services
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
        run: |
          for attempt in 1 2 3; do
            echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin && break
            [ "$attempt" = 3 ] && exit 1
            sleep 10
          done
          docker run -d --name mysql -p 3306:3306 --tmpfs /var/lib/mysql:size=1G \
            -e MYSQL_ROOT_PASSWORD=password -e MYSQL_DATABASE=gumroad_test mysql:8.4.11 \
            --innodb-flush-log-at-trx-commit=0 --innodb-doublewrite=0 --sync-binlog=0 \
            --innodb-flush-method=nosync --skip-log-bin &
          docker run -d --name redis -p 6379:6379 redis:7 &
          docker run -d --name stripe_mock -p 12111:12111 -p 12112:12112 stripemock/stripe-mock:latest &
          docker run -d --name dynamodb -p 8002:8000 amazon/dynamodb-local:3.3.1 &
          docker run -d --name elasticsearch -p 9200:9200 -e discovery.type=single-node \
            -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" elasticsearch:7.9.3 &
          docker run -d --name minio -p 9000:9000 -p 9001:9001 \
            -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin \
            cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1 server /data --console-address ":9001" &
          status=0
          for job in $(jobs -p); do wait "$job" || status=1; done
          docker logout
          exit "$status"

      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          persist-credentials: false

      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          bundler-cache: true

      # Over TCP: during its first start, MySQL answers on the socket before it
      # listens on the port.
      - name: Wait for MySQL
        run: |
          for _ in $(seq 1 60); do
            docker exec mysql mysql --protocol=tcp -h127.0.0.1 -uroot -ppassword -e "SELECT 1" > /dev/null 2>&1 && exit 0
            sleep 2
          done
          docker logs mysql | tail -20
          echo "MySQL did not become ready"
          exit 1

      - name: Prepare test database and assets
        run: bundle exec rake db:prepare

      - name: Set up MinIO buckets
        run: |
          for i in $(seq 1 30); do curl -fsS http://localhost:9000/minio/health/ready >/dev/null 2>&1 && break; sleep 2; done
          curl -fsS http://localhost:9000/minio/health/ready || { echo "MinIO did not become ready"; exit 1; }
          # Provision the buckets the app's :test storage service expects (see
          # config/storage.yml). This mirrors the `createbuckets` service in
          # docker/docker-compose-test-and-ci.yml so the two CI lanes behave the same:
          # mc runs from the same digest-pinned image, which bundles it (an unpinned :latest
          # can change CLI behavior and break this job with no code change), and both buckets get the
          # public read policy and versioning the compose stack sets. The :test service is
          # declared `public: true`, so without `anonymous set public` any future test that
          # fetches blob.url over plain HTTP would 403 here while passing locally.
          #
          # `mc anonymous set` and `mc version enable` each take exactly ONE bucket — passing
          # two makes mc print its usage text and exit 1 — so they run per bucket in a loop.
          # (`mc mb` does accept several.)
          docker run --rm --network host --entrypoint /bin/sh cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1 -c "
            set -e
            mc alias set local http://localhost:9000 minioadmin minioadmin
            mc mb --ignore-existing local/gumroad-specs local/gumroad-invoices
            for bucket in gumroad-specs gumroad-invoices; do
              mc anonymous set public local/\$bucket
              mc version enable local/\$bucket
            done"

      - name: Wait for Elasticsearch
        run: |
          for i in $(seq 1 30); do curl -fsS http://localhost:9200/_cluster/health >/dev/null 2>&1 && break; sleep 2; done
          curl -fsS http://localhost:9200/_cluster/health || { echo "Elasticsearch did not become ready"; exit 1; }

      - name: Run Minitest
        run: bundle exec rails test
        env:
          RUBY_YJIT_ENABLE: 1
          CI: true
          MINITEST_SHARD: ${{ matrix.ci_node_index }}/${{ matrix.ci_node_total }}

  test_fast:
    name: Test Fast ${{ matrix.ci_node_index }}${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && ' (internal PR noop)' || '' }}
    env:
      COMPOSE_PROJECT_NAME: web_${{ github.run_id }}_${{ github.run_attempt }}_fast_${{ matrix.ci_node_index }}
    runs-on: ubicloud-standard-4
    needs: [build, run_scope, reuse_scope]
    # Full suite: main pushes (deploy gate) and PRs labeled run-all-specs,
    # unless this main commit is the same tree as a PR that already passed Fast+Slow.
    # reuse_scope is skipped off main — always() so that skip does not cancel
    # a run-all-specs PR. Empty reuse output is not 'true', so Fast still runs.
    if: |
      always() &&
      !cancelled() &&
      needs.build.result == 'success' &&
      needs.run_scope.result == 'success' &&
      needs.run_scope.outputs.full == 'true' &&
      needs.reuse_scope.outputs.reuse != 'true'
    strategy:
      # Not on main: Knapsack hands a cancelled shard's unrun tests to nobody, so
      # a re-run of a fail-fast cancel passes with 0 examples. Main runs must
      # finish every shard for rerun-main-spec-failure.yml to retry safely.
      fail-fast: ${{ github.event_name != 'push' || github.ref != 'refs/heads/main' }}
      matrix:
        ci_node_total: [18]
        ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        timeout-minutes: 5
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          persist-credentials: false
      - name: Login to Docker Hub
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 120
          max_attempts: 3
          retry_wait_seconds: 10
          command: echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Start services and pull test image
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_wait_seconds: 5
          command: |
            docker pull --quiet $WEB_REPO:${{ needs.build.outputs.test_image_tag }} &
            PULL_PID=$!
            docker compose -f docker/docker-compose-test-and-ci.yml up -d
            COMPOSE_EXIT=$?
            wait $PULL_PID
            PULL_EXIT=$?
            exit $((COMPOSE_EXIT + PULL_EXIT))
        env:
          COMPOSE_HTTP_TIMEOUT: "300"
          WEB_REPO: antiwork/gumroad-web
      - name: Wait for services and prepare test database
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_wait_seconds: 10
          command: |
            IMG=$WEB_REPO:${{ needs.build.outputs.test_image_tag }}
            NET=${{ env.COMPOSE_PROJECT_NAME }}_default
            # Restore the baked test database dump (seconds) instead of
            # re-running db:prepare via Rails; ES/minio waits run in parallel.
            (
              docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh db_test 3306 \
                && docker/ci/restore_test_db.sh $NET $IMG
            ) &
            DB_PID=$!
            docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh elasticsearch 9200 &
            ES_PID=$!
            docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh minio 9000 &
            MINIO_PID=$!
            wait $DB_PID || exit 1
            wait $ES_PID || exit 1
            wait $MINIO_PID || exit 1
        env:
          WEB_REPO: antiwork/gumroad-web

      - name: Run tests
        timeout-minutes: 15
        run: |
          mkdir -p ./test-logs
          chmod 777 ./test-logs
          docker run --rm --entrypoint="" --network ${{ env.COMPOSE_PROJECT_NAME }}_default \
            -v "$(pwd)/test-logs:/app/log" \
            -e RUBY_YJIT_ENABLE \
            -e KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC \
            -e KNAPSACK_PRO_CI_NODE_TOTAL \
            -e KNAPSACK_PRO_CI_NODE_INDEX \
            -e KNAPSACK_PRO_LOG_LEVEL \
            -e KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES \
            -e KNAPSACK_PRO_TEST_FILE_PATTERN \
            -e KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN \
            -e KNAPSACK_PRO_COMMIT_HASH \
            -e KNAPSACK_PRO_BRANCH \
            -e KNAPSACK_PRO_CI_NODE_BUILD_ID \
            -e KNAPSACK_PRO_CI_NODE_RETRY_COUNT \
            -e KNAPSACK_PRO_PROJECT_DIR \
            -e KNAPSACK_PRO_FIXED_QUEUE_SPLIT \
            -e CI \
            -e IN_DOCKER \
            $WEB_REPO:${{ needs.build.outputs.test_image_tag }} \
            /usr/local/bin/gosu app bundle exec rake "knapsack_pro:queue:rspec[--format RSpec::Github::Formatter --tag ~skip --format progress]"
        env:
          RUBY_YJIT_ENABLE: 1
          KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC_FAST }}
          KNAPSACK_PRO_CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
          KNAPSACK_PRO_CI_NODE_INDEX: ${{ matrix.ci_node_index }}
          KNAPSACK_PRO_LOG_LEVEL: info
          KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES: true
          KNAPSACK_PRO_TEST_FILE_PATTERN: "spec/**/*_spec.rb"
          KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN: "spec/requests/**/*_spec.rb"
          KNAPSACK_PRO_COMMIT_HASH: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          KNAPSACK_PRO_BRANCH: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref_name }}
          KNAPSACK_PRO_CI_NODE_BUILD_ID: ${{ github.run_id }}
          KNAPSACK_PRO_CI_NODE_RETRY_COUNT: ${{ github.run_attempt }}
          KNAPSACK_PRO_PROJECT_DIR: /app
          KNAPSACK_PRO_FIXED_QUEUE_SPLIT: true
          CI: true
          IN_DOCKER: true
          WEB_REPO: antiwork/gumroad-web
      - name: Archive flaky specs log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: flaky-specs-fast-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: ./test-logs/flaky_specs.log
          retention-days: 14
          if-no-files-found: ignore
      - name: Clean up
        if: always()
        run: |
          docker compose -f docker/docker-compose-test-and-ci.yml down -v 2>/dev/null || true
          docker rm -f $(docker ps -aq -f name=${{ env.COMPOSE_PROJECT_NAME }}) 2>/dev/null || true
          docker volume rm $(docker volume ls -q -f name=${{ env.COMPOSE_PROJECT_NAME }}) 2>/dev/null || true
          docker network rm ${{ env.COMPOSE_PROJECT_NAME }}_default 2>/dev/null || true

  test_slow:
    name: Test Slow ${{ matrix.ci_node_index }}${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && ' (internal PR noop)' || '' }}
    env:
      COMPOSE_PROJECT_NAME: web_${{ github.run_id }}_${{ github.run_attempt }}_slow_${{ matrix.ci_node_index }}
      WEB_REPO: antiwork/gumroad-web
    runs-on: ubicloud-standard-4
    needs: [build, run_scope, reuse_scope]
    # Full suite: main pushes (deploy gate) and PRs labeled run-all-specs,
    # unless this main commit is the same tree as a PR that already passed Fast+Slow.
    # reuse_scope is skipped off main — always() so that skip does not cancel
    # a run-all-specs PR. Empty reuse output is not 'true', so Slow still runs.
    if: |
      always() &&
      !cancelled() &&
      needs.build.result == 'success' &&
      needs.run_scope.result == 'success' &&
      needs.run_scope.outputs.full == 'true' &&
      needs.reuse_scope.outputs.reuse != 'true'
    strategy:
      fail-fast: false
      matrix:
        ci_node_total: [50]
        ci_node_index:
          [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            26,
            27,
            28,
            29,
            30,
            31,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            39,
            40,
            41,
            42,
            43,
            44,
            45,
            46,
            47,
            48,
            49,
          ]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        timeout-minutes: 5
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          persist-credentials: false
      - name: Login to Docker Hub
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 120
          max_attempts: 3
          retry_wait_seconds: 10
          command: echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Start services and pull test image
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_wait_seconds: 5
          command: |
            docker pull --quiet $WEB_REPO:${{ needs.build.outputs.test_image_tag }} &
            PULL_PID=$!
            docker compose -f docker/docker-compose-test-and-ci.yml up -d
            COMPOSE_EXIT=$?
            wait $PULL_PID
            PULL_EXIT=$?
            exit $((COMPOSE_EXIT + PULL_EXIT))
        env:
          COMPOSE_HTTP_TIMEOUT: "300"
          WEB_REPO: antiwork/gumroad-web
      - name: Wait for services and prepare test database
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_wait_seconds: 10
          command: |
            IMG=$WEB_REPO:${{ needs.build.outputs.test_image_tag }}
            NET=${{ env.COMPOSE_PROJECT_NAME }}_default
            # Restore the baked test database dump (seconds) instead of
            # re-running db:prepare via Rails; ES/minio waits run in parallel.
            (
              docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh db_test 3306 \
                && docker/ci/restore_test_db.sh $NET $IMG
            ) &
            DB_PID=$!
            docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh elasticsearch 9200 &
            ES_PID=$!
            docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh minio 9000 &
            MINIO_PID=$!
            wait $DB_PID || exit 1
            wait $ES_PID || exit 1
            wait $MINIO_PID || exit 1
        env:
          WEB_REPO: antiwork/gumroad-web

      - name: Run tests
        timeout-minutes: 25
        run: |
          mkdir -p ./capybara-artifacts
          chmod 777 ./capybara-artifacts
          mkdir -p ./test-logs
          chmod 777 ./test-logs
          docker run --rm --entrypoint="" --network ${{ env.COMPOSE_PROJECT_NAME }}_default \
            -v "$(pwd)/capybara-artifacts:/app/tmp/capybara" \
            -v "$(pwd)/test-logs:/app/log" \
            -e RUBY_YJIT_ENABLE \
            -e KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC \
            -e KNAPSACK_PRO_CI_NODE_TOTAL \
            -e KNAPSACK_PRO_CI_NODE_INDEX \
            -e KNAPSACK_PRO_LOG_LEVEL \
            -e KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES \
            -e KNAPSACK_PRO_TEST_FILE_PATTERN \
            -e KNAPSACK_PRO_COMMIT_HASH \
            -e KNAPSACK_PRO_BRANCH \
            -e KNAPSACK_PRO_CI_NODE_BUILD_ID \
            -e KNAPSACK_PRO_CI_NODE_RETRY_COUNT \
            -e KNAPSACK_PRO_PROJECT_DIR \
            -e KNAPSACK_PRO_FIXED_QUEUE_SPLIT \
            -e CI \
            -e IN_DOCKER \
            $WEB_REPO:${{ needs.build.outputs.test_image_tag }} \
            /usr/local/bin/gosu app bundle exec rake "knapsack_pro:queue:rspec[--format RSpec::Github::Formatter --tag ~skip --format progress]"
        env:
          RUBY_YJIT_ENABLE: 1
          KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC_FAST }}
          KNAPSACK_PRO_CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
          KNAPSACK_PRO_CI_NODE_INDEX: ${{ matrix.ci_node_index }}
          KNAPSACK_PRO_LOG_LEVEL: info
          KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES: true
          KNAPSACK_PRO_TEST_FILE_PATTERN: "spec/requests/**/*_spec.rb"
          KNAPSACK_PRO_COMMIT_HASH: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          KNAPSACK_PRO_BRANCH: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref_name }}
          KNAPSACK_PRO_CI_NODE_BUILD_ID: ${{ github.run_id }}
          KNAPSACK_PRO_CI_NODE_RETRY_COUNT: ${{ github.run_attempt }}
          KNAPSACK_PRO_PROJECT_DIR: /app
          KNAPSACK_PRO_FIXED_QUEUE_SPLIT: true
          CI: true
          IN_DOCKER: true
          WEB_REPO: antiwork/gumroad-web
      - name: Archive capybara artifacts
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: capybara-screenshots-slow-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: ./capybara-artifacts/*.png
          retention-days: 7
          if-no-files-found: ignore
      - name: Archive test logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: test-logs-slow-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: ./test-logs/test.log
          retention-days: 7
          if-no-files-found: ignore
      - name: Archive flaky specs log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: flaky-specs-slow-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: ./test-logs/flaky_specs.log
          retention-days: 14
          if-no-files-found: ignore
      - name: Clean up
        if: always()
        run: |
          docker compose -f docker/docker-compose-test-and-ci.yml down -v 2>/dev/null || true
          docker rm -f $(docker ps -aq -f name=${{ env.COMPOSE_PROJECT_NAME }}) 2>/dev/null || true
          docker volume rm $(docker volume ls -q -f name=${{ env.COMPOSE_PROJECT_NAME }}) 2>/dev/null || true
          docker network rm ${{ env.COMPOSE_PROJECT_NAME }}_default 2>/dev/null || true

  relevant_specs:
    # Runs the fork-controlled bin/branch-specs in a job with NO secrets and
    # no Docker login, so untrusted code can never sit in front of a
    # credential-bearing step (a $GITHUB_PATH write here can't reach
    # test_relevant — PATH edits don't cross jobs). The list crosses over as
    # a plain data output. Fork PRs only: test_relevant_runner selects its own.
    name: Compute relevant specs
    runs-on: ubicloud-standard-2
    needs: run_scope
    if: ${{ needs.run_scope.outputs.full != 'true' && github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository }}
    outputs:
      files_csv: ${{ steps.specs.outputs.files_csv }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        timeout-minutes: 5
        with:
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          # bin/branch-specs diffs against origin/main, so the base has to be
          # in the clone.
          fetch-depth: 0
          persist-credentials: false
      - name: Compute relevant specs
        id: specs
        run: |
          # Best-effort: checkout (fetch-depth 0) already has origin/main, and this
          # anonymous fetch (persist-credentials off) intermittently gets a
          # credential prompt from GitHub, which would fail the job for nothing.
          git fetch --no-tags --quiet origin main || echo "origin/main refresh failed; using the ref from checkout"
          set +e
          SPECS=$(ruby bin/branch-specs)
          STATUS=$?
          set -e
          # Exit 3 = the selector refused to certify a trimmed run (dependency/
          # config/migration/factory change, mapping gap, or oversized
          # selection). The full suite is required: add the run-all-specs label.
          if [ "$STATUS" -eq 3 ]; then
            echo "::error::bin/branch-specs escalated — this diff needs the full suite. Add the run-all-specs label to this PR and re-run."
            exit 1
          elif [ "$STATUS" -ne 0 ]; then
            echo "bin/branch-specs failed with status $STATUS"
            exit "$STATUS"
          fi
          COUNT=$(echo "$SPECS" | grep -c . || true)
          # An empty list here means the diff touched nothing rspec-affecting
          # (selector escalates on app-code diffs that map to nothing).
          if [ "$COUNT" -eq 0 ]; then
            echo "No rspec-affecting changes; running the schema smoke spec as a floor."
            SPECS="spec/models/user_spec.rb"
            COUNT=1
          fi
          echo "Running $COUNT spec file(s):"
          echo "$SPECS" | sed 's/^/  /'
          echo "files_csv=$(echo "$SPECS" | paste -sd, -)" >> "$GITHUB_OUTPUT"

  test_relevant:
    name: Test Relevant ${{ matrix.ci_node_index }}${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository && ' (internal PR noop)' || '' }}
    env:
      COMPOSE_PROJECT_NAME: web_${{ github.run_id }}_${{ github.run_attempt }}_relevant_${{ matrix.ci_node_index }}
      WEB_REPO: antiwork/gumroad-web
    runs-on: ubicloud-standard-4
    needs: [build, run_scope, relevant_specs]
    # Fork PR counterpart of test_fast/test_slow: the same docker setup and
    # Knapsack queue distribution, restricted to the specs relevant_specs
    # mapped from the diff plus the money-path smoke set. Main pushes and
    # run-all-specs PRs run the full suite instead. Branch pushes in this
    # repository use test_relevant_runner below.
    if: needs.run_scope.outputs.full != 'true' && github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        # Twelve dynamically queued shards let selections of up to
        # MAX_SELECTED_FILES drain within the existing 25-minute timeout.
        ci_node_total: [12]
        ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
    steps:
      - name: Check out trusted workflow files
        uses: actions/checkout@v4
        timeout-minutes: 5
        with:
          # This checkout feeds docker compose after Docker Hub auth, so fork PRs
          # must use the trusted base revision. The image under test still comes
          # from the fork build via needs.build.outputs.test_image_tag.
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.sha }}
          persist-credentials: false
      - name: Check out fork fixtures
        # The base checkout above pins Compose/workflow files, but its
        # spec/support/fixtures is base-revision data, not what the fork
        # image under test was built against. Fetch the fork's fixture
        # files into a side directory (no execution of fork code) and
        # overlay them onto the trusted checkout before Compose runs.
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v4
        timeout-minutes: 5
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          persist-credentials: false
          path: fork-head-fixtures
          sparse-checkout: |
            spec/support/fixtures
          sparse-checkout-cone-mode: false
      - name: Overlay fork fixtures onto the trusted checkout
        if: github.event_name == 'pull_request_target'
        run: |
          rm -rf spec/support/fixtures
          mkdir -p spec/support
          cp -a fork-head-fixtures/spec/support/fixtures spec/support/fixtures
          rm -rf fork-head-fixtures
      - name: Login to Docker Hub
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 120
          max_attempts: 3
          retry_wait_seconds: 10
          command: echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Start services and pull test image
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_wait_seconds: 5
          command: |
            docker pull --quiet $WEB_REPO:${{ needs.build.outputs.test_image_tag }} &
            PULL_PID=$!
            docker compose -f docker/docker-compose-test-and-ci.yml up -d
            COMPOSE_EXIT=$?
            wait $PULL_PID
            PULL_EXIT=$?
            exit $((COMPOSE_EXIT + PULL_EXIT))
        env:
          COMPOSE_HTTP_TIMEOUT: "300"
      - name: Wait for services and prepare test database
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_wait_seconds: 10
          command: |
            IMG=$WEB_REPO:${{ needs.build.outputs.test_image_tag }}
            NET=${{ env.COMPOSE_PROJECT_NAME }}_default
            (
              docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh db_test 3306 \
                && docker/ci/restore_test_db.sh $NET $IMG
            ) &
            DB_PID=$!
            docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh elasticsearch 9200 &
            ES_PID=$!
            docker run --rm --entrypoint="" --network $NET $IMG docker/ci/wait_on_connection.sh minio 9000 &
            MINIO_PID=$!
            wait $DB_PID || exit 1
            wait $ES_PID || exit 1
            wait $MINIO_PID || exit 1
      - name: Run tests
        timeout-minutes: 25
        run: |
          mkdir -p ./capybara-artifacts ./test-logs
          chmod 777 ./capybara-artifacts ./test-logs
          docker run --rm --entrypoint="" --network ${{ env.COMPOSE_PROJECT_NAME }}_default \
            -v "$(pwd)/capybara-artifacts:/app/tmp/capybara" \
            -v "$(pwd)/test-logs:/app/log" \
            -e RUBY_YJIT_ENABLE \
            -e KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC \
            -e KNAPSACK_PRO_CI_NODE_TOTAL \
            -e KNAPSACK_PRO_CI_NODE_INDEX \
            -e KNAPSACK_PRO_LOG_LEVEL \
            -e KNAPSACK_PRO_TEST_FILE_LIST \
            -e KNAPSACK_PRO_COMMIT_HASH \
            -e KNAPSACK_PRO_BRANCH \
            -e KNAPSACK_PRO_CI_NODE_BUILD_ID \
            -e KNAPSACK_PRO_CI_NODE_RETRY_COUNT \
            -e KNAPSACK_PRO_PROJECT_DIR \
            -e KNAPSACK_PRO_FIXED_QUEUE_SPLIT \
            -e CI \
            -e IN_DOCKER \
            $WEB_REPO:${{ needs.build.outputs.test_image_tag }} \
            /usr/local/bin/gosu app bundle exec rake "knapsack_pro:queue:rspec[--format RSpec::Github::Formatter --tag ~skip --format progress]"
        env:
          RUBY_YJIT_ENABLE: 1
          KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC_FAST }}
          KNAPSACK_PRO_CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
          KNAPSACK_PRO_CI_NODE_INDEX: ${{ matrix.ci_node_index }}
          KNAPSACK_PRO_LOG_LEVEL: info
          KNAPSACK_PRO_TEST_FILE_LIST: ${{ needs.relevant_specs.outputs.files_csv }}
          KNAPSACK_PRO_COMMIT_HASH: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          KNAPSACK_PRO_BRANCH: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref_name }}
          KNAPSACK_PRO_CI_NODE_BUILD_ID: ${{ github.run_id }}
          KNAPSACK_PRO_CI_NODE_RETRY_COUNT: ${{ github.run_attempt }}
          KNAPSACK_PRO_PROJECT_DIR: /app
          KNAPSACK_PRO_FIXED_QUEUE_SPLIT: true
          CI: true
          IN_DOCKER: true
      - name: Archive capybara artifacts
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: capybara-screenshots-relevant-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: ./capybara-artifacts/*.png
          retention-days: 7
          if-no-files-found: ignore
      - name: Archive test logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: test-logs-relevant-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: ./test-logs/test.log
          retention-days: 7
          if-no-files-found: ignore
      - name: Clean up
        if: always()
        run: |
          docker compose -f docker/docker-compose-test-and-ci.yml down -v 2>/dev/null || true
          docker rm -f $(docker ps -aq -f name=${{ env.COMPOSE_PROJECT_NAME }}) 2>/dev/null || true
          docker volume rm $(docker volume ls -q -f name=${{ env.COMPOSE_PROJECT_NAME }}) 2>/dev/null || true
          docker network rm ${{ env.COMPOSE_PROJECT_NAME }}_default 2>/dev/null || true

  test_relevant_runner:
    name: Test Relevant ${{ matrix.ci_node_index }}
    runs-on: ubicloud-standard-4
    # ci_gate holds the run-all-specs answer and the base commit, so the shards
    # start without waiting for run_scope.
    needs: ci_gate
    # Branch pushes in this repository run the relevant specs on the runner, so
    # they skip the test image build and pull. Fork PRs keep the image job above:
    # this job runs the branch's code next to the Knapsack token. For the same
    # reason each shard selects its specs itself instead of waiting for
    # relevant_specs, which keeps fork code away from secrets.
    if: needs.ci_gate.outputs.full != 'true' && github.event_name == 'push'
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ci_node_total: [12]
        ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
    env:
      COMPOSE_FILE: docker/docker-compose-test-and-ci.yml:docker/docker-compose-ci-runner.yml
      COMPOSE_PROJECT_NAME: relevant_${{ matrix.ci_node_index }}
      RAILS_ENV: test
      # No BUNDLE_GEMS__CONTRIBSYS__COM: sidekiq-pro is optional (config/initializers/sidekiq.rb),
      # and leaving it out reuses the gem cache that the lint jobs save.
      # From docker/web/Dockerfile.test, which sets them for the asset build too.
      NODE_ENV: test
      RAILS_LOG_LEVEL: info
      # Also selects the headless Chrome drivers in spec/support/capybara_driver.rb.
      IN_DOCKER: true
      # The image's locale and time zone: Chrome formats dates from both.
      LANG: C.UTF-8
      TZ: UTC
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        timeout-minutes: 5
        with:
          persist-credentials: false
          # bin/branch-specs diffs against main, so the base has to be in the clone.
          fetch-depth: 0
      # Before the setup below, so a diff that needs the full suite fails at once.
      - name: Select relevant specs
        id: specs
        run: bin/ci-relevant-specs "$BASE_SHA"
        env:
          BASE_SHA: ${{ needs.ci_gate.outputs.base_sha }}
      - name: Login to Docker Hub
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 120
          max_attempts: 3
          retry_wait_seconds: 10
          command: echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
      # In the background, so the image pulls and the MinIO health check overlap
      # with the setup steps below. "Prepare test database" waits for it.
      - name: Start services
        run: (docker compose up -d > /tmp/compose.log 2>&1; echo $? > /tmp/compose.status) &
      # Saved by the typecheck job on main, so apt downloads nothing on a hit.
      - name: Restore the test packages
        uses: actions/cache/restore@v4
        with:
          path: ~/test-packages
          key: test-packages-v1-${{ runner.os }}-${{ hashFiles('bin/ci-test-packages') }}
      - name: Install system packages
        run: bin/ci-test-packages install ~/test-packages
      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          bundler-cache: true
      - uses: actions/setup-node@v4
        with:
          node-version-file: package.json
          cache: npm
      # Shares lint_js's cache, which only main pushes save.
      - uses: actions/cache/restore@v4
        id: node_modules
        with:
          path: node_modules
          key: node-modules-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'package-lock.json', '.npmrc', 'patches/**') }}
      - run: npm ci
        if: steps.node_modules.outputs.cache-hit != 'true'
      # Saved by the typecheck job on main, or by shard 0 of an earlier push to
      # this branch.
      - name: Restore the test assets
        id: test_assets
        uses: actions/cache/restore@v4
        with:
          path: |
            public/vite-test
            tmp/cache/vite
            public/js/gumroad-bundle.js
            public/js/gumroad-embed-bundle.js
            public/pages-tailwind.css
            public/pages-tailwind-manifest.json
            public/assets/pages
            app/javascript/stylesheets/pages_tailwind.generated.html
          key: >-
            runner-test-assets-v1-${{ runner.os }}-${{ hashFiles(
              'app/javascript/**', 'app/views/**',
              'config/routes.rb', 'config/routes/**', 'config/domain.rb',
              'config/application.rb', 'config/boot.rb', 'config/environment.rb',
              'config/environments/**', 'config/redis.rb',
              'lib/catch_bad_request_errors.rb', 'lib/utilities/global_config.rb',
              'config/initializers/js-routes.rb', 'config/vite.json', 'config/vite/**',
              'config/currencies.json', 'lib/json_schemas/**', 'lib/tasks/**', 'Rakefile',
              'public/**', 'vendor/assets/**', 'patches/**', 'scripts/build_pages_tailwind.mjs',
              'docker/base/**', 'docker/web/Dockerfile.test', '.github/workflows/tests.yml',
              'package.json', 'package-lock.json', '.npmrc', 'tsconfig.json',
              'vite.config.ts', 'vite.config.widget.ts', 'postcss.config.js',
              'Gemfile.lock', '.ruby-version') }}
      - name: Prepare test database and assets
        run: |
          for _ in $(seq 1 90); do [ -f /tmp/compose.status ] && break; sleep 2; done
          cat /tmp/compose.log
          [ "$(cat /tmp/compose.status 2>/dev/null)" = 0 ] || { echo "docker compose up did not finish"; exit 1; }
          # Over TCP: during its first start, MySQL answers on the socket before it
          # listens on the port.
          ready=false
          for _ in $(seq 1 60); do
            if docker compose exec -T db_test mysql --protocol=tcp -h127.0.0.1 -uroot -ppassword -e "SELECT 1" > /dev/null 2>&1 \
              && curl -fsS http://localhost:9200/_cluster/health > /dev/null 2>&1 \
              && curl -fsS http://localhost:9000/minio/health/ready > /dev/null 2>&1; then
              ready=true
              break
            fi
            sleep 2
          done
          [ "$ready" = true ] || { echo "MySQL, Elasticsearch or MinIO did not become ready"; exit 1; }
          # The same preparation the build job bakes into the test image (the
          # database, the Vite test build, the widget bundles and the pages CSS), in
          # one Rails boot as docker/web/compile_assets.sh does. npm ci already ran
          # above. pages_tailwind:build goes first because it writes into
          # app/javascript, which Vite's digest covers; assets:precompile then skips it.
          if [ "$ASSETS_RESTORED" = true ]; then
            bundle exec rake js:export db:setup
          else
            VITE_RUBY_SKIP_ASSETS_PRECOMPILE_INSTALL=true bundle exec rake js:export db:setup pages_tailwind:build assets:precompile
          fi
          # docker wait prints the exit code instead of returning it.
          copy_status=$(docker wait "$(docker compose ps -aq copyfixturefiles)")
          [ "$copy_status" = 0 ] || { docker compose logs copyfixturefiles; exit 1; }
        env:
          ASSETS_RESTORED: ${{ steps.test_assets.outputs.cache-hit }}
      - uses: actions/cache/save@v4
        if: steps.test_assets.outputs.cache-hit != 'true' && matrix.ci_node_index == 0
        with:
          path: |
            public/vite-test
            tmp/cache/vite
            public/js/gumroad-bundle.js
            public/js/gumroad-embed-bundle.js
            public/pages-tailwind.css
            public/pages-tailwind-manifest.json
            public/assets/pages
            app/javascript/stylesheets/pages_tailwind.generated.html
          key: ${{ steps.test_assets.outputs.cache-primary-key }}
      - name: Run tests
        timeout-minutes: 25
        run: bundle exec rake "knapsack_pro:queue:rspec[--format RSpec::Github::Formatter --tag ~skip --format progress]"
        env:
          RUBY_YJIT_ENABLE: 1
          KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC_FAST }}
          KNAPSACK_PRO_CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
          KNAPSACK_PRO_CI_NODE_INDEX: ${{ matrix.ci_node_index }}
          KNAPSACK_PRO_LOG_LEVEL: info
          KNAPSACK_PRO_TEST_FILE_LIST: ${{ steps.specs.outputs.files_csv }}
          KNAPSACK_PRO_COMMIT_HASH: ${{ github.sha }}
          KNAPSACK_PRO_BRANCH: ${{ github.ref_name }}
          KNAPSACK_PRO_CI_NODE_BUILD_ID: ${{ github.run_id }}
          # Knapsack 10 keys the queue on this. It would read GITHUB_RUN_ID, which every
          # runner job in this run shares.
          KNAPSACK_PRO_TEST_QUEUE_ID: ${{ github.run_id }}-relevant
          KNAPSACK_PRO_CI_NODE_RETRY_COUNT: ${{ github.run_attempt }}
          KNAPSACK_PRO_FIXED_QUEUE_SPLIT: true
          CI: true
      - name: Archive capybara artifacts
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: capybara-screenshots-relevant-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: tmp/capybara/*.png
          retention-days: 7
          if-no-files-found: ignore
      - name: Archive test logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: test-logs-relevant-${{ matrix.ci_node_index }}-attempt-${{ github.run_attempt }}
          path: log/test.log
          retention-days: 7
          if-no-files-found: ignore

  unblock_deployment_from_buildkite:
    name: Unblock deployment from Buildkite
    runs-on: ubicloud-standard-2
    needs: [run_scope, reuse_scope, test_fast, test_slow, test_minitest]
    # Skipped Fast/Slow (reuse) must not block deploy. A needed job that is
    # skipped fails the default success gate, so always() is required or
    # this job never runs on the reuse path.
    if: |
      always() &&
      !cancelled() &&
      github.event_name == 'push' && github.ref == 'refs/heads/main' &&
      needs.test_minitest.result == 'success' &&
      needs.reuse_scope.result == 'success' &&
      (
        needs.reuse_scope.outputs.reuse == 'true' ||
        (needs.test_fast.result == 'success' && needs.test_slow.result == 'success')
      )
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          sparse-checkout: |
            bin/unblock-buildkite-deploy
          sparse-checkout-cone-mode: false

      # The gate appears only after the asset compile passes (#7316). Poll every 20 s, and cap
      # the step at 30 minutes: a shorter window leaves the deploy for a person to unblock, and
      # slow attempts could otherwise stretch 90 attempts to about an hour.
      - name: Unblock corresponding Buildkite build
        timeout-minutes: 30
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 20
          retry_wait_seconds: 20
          max_attempts: 90
          warning_on_retry: false
          command: |
            if [ "$UNBLOCK_DEPLOYMENT" != "true" ]; then
              echo "UNBLOCK_DEPLOYMENT_FROM_BUILDKITE is not set to 'true'. Skipping unblock."
              exit 0
            fi

            bin/unblock-buildkite-deploy
        env:
          UNBLOCK_DEPLOYMENT: ${{ secrets.UNBLOCK_DEPLOYMENT_FROM_BUILDKITE }}
          BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
          GITHUB_TOKEN: ${{ github.token }}
          COMMIT_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
          ORG_SLUG: gumroad-inc
          PIPELINE_SLUG: ${{ secrets.BUILDKITE_PIPELINE_SLUG }}