Diagnostics: record what holds a shard open after its last test

.github/workflows/tests.yml

@@ -159,9 +159,98 @@ jobs:
           sqlite3 --version
           bats --version
 
+      # Diagnostics for the shards that report every test ok and then sit
+      # silent until the job's cap. The sampler writes to a file and never to
+      # this step's stdout: a process holding that pipe is one of the things
+      # under suspicion, so the instrument must not be able to cause what it is
+      # measuring. It closes fds 3 and 4 for the same reason, and gives itself a
+      # hard lifetime so it cannot outlive the job it is watching.
+      - name: Start hang sampler
+        if: needs.changes.outputs.docs_only != 'true'
+        continue-on-error: true
+        run: |
+          set +e
+          cat > "$RUNNER_TEMP/sampler.sh" <<'SAMPLER'
+          #!/usr/bin/env bash
+          out="$RUNNER_TEMP/hang-samples.txt"
+          : > "$out"
+          for i in $(seq 1 46); do
+            [ -f "$RUNNER_TEMP/bats-done" ] && { echo "== bats finished, sampler stopping at sample $i" >> "$out"; break; }
+            {
+              echo "===== sample $i  $(date -u +%H:%M:%S)"
+              ps -ef 2>/dev/null | grep -vE '\[' | tail -n +2
+              echo "----- surviving node/python and what they hold:"
+              for pid in $(ps -eo pid=,comm= 2>/dev/null | awk '$2 ~ /node|python/ {print $1}'); do
+                echo "--- pid $pid: $(ps -o command= -p "$pid" 2>/dev/null | cut -c1-160)"
+                lsof -p "$pid" 2>/dev/null \
+                  | awk 'NR == 1 || $4 ~ /^[0-9]+[rwu]?$/ { n++; if (n <= 14) print }'
+              done
+              echo "----- bats temp dirs left behind:"
+              ls -la "${TMPDIR:-/tmp}" 2>/dev/null \
+                | awk 'tolower($0) ~ /bats/ { n++; if (n <= 20) print }'
+            } >> "$out" 2>&1
+            sleep 30
+          done
+          SAMPLER
+          chmod +x "$RUNNER_TEMP/sampler.sh"
+          nohup "$RUNNER_TEMP/sampler.sh" >/dev/null 2>&1 3>&- 4>&- &
+          echo "sampler pid $!"
+
       - name: Run bats suite (this shard)
         if: needs.changes.outputs.docs_only != 'true'
-        run: xargs bats --print-output-on-failure < shard-files.txt
+        run: |
+          set +e
+          xargs bats --print-output-on-failure < shard-files.txt
+          status=$?
+          : > "$RUNNER_TEMP/bats-done"
+          exit $status
+
+      # always() so this also runs when the job is cancelled at its cap, which
+      # is the only case that matters here.
+      # continue-on-error and a best-effort body: this step exists to describe a
+      # failure, and a probe that returns non-zero -- lsof on a pid that has just
+      # exited, a grep that matches nothing -- must never be what turns a green
+      # shard red. The runner's default shell is `bash -e`, so -e is dropped
+      # explicitly and every probe is allowed to fail. `head` is avoided in
+      # pipelines for the same reason: it closes the pipe early and SIGPIPEs
+      # whatever was writing.
+      - name: Hang forensics
+        if: always() && needs.changes.outputs.docs_only != 'true'
+        continue-on-error: true
+        run: |
+          set +e
+          set +o pipefail 2>/dev/null || true
+          echo "##### live snapshot at $(date -u +%H:%M:%S 2>/dev/null)"
+          ps -ef 2>/dev/null | awk 'NR > 1 && NR <= 61'
+          echo "##### anything still holding a pipe or fifo:"
+          pids="$(ps -eo pid=,comm= 2>/dev/null | awk '$2 ~ /node|python|bats|bash|sleep/ {print $1}')"
+          for pid in $pids; do
+            held="$(lsof -p "$pid" 2>/dev/null | awk '$5 == "PIPE" || $5 == "FIFO"' | awk 'NR <= 4')"
+            if [ -n "$held" ]; then
+              echo "--- pid $pid $(ps -o command= -p "$pid" 2>/dev/null | cut -c1-120)"
+              echo "$held"
+            fi
+          done
+          echo "##### bats temp remnants:"
+          ls -la "${TMPDIR:-/tmp}" 2>/dev/null \
+            | awk 'tolower($0) ~ /bats/ { n++; if (n <= 20) print }'
+          echo "##### sampler timeline:"
+          if [ -s "$RUNNER_TEMP/hang-samples.txt" ]; then
+            cat "$RUNNER_TEMP/hang-samples.txt" 2>/dev/null
+          else
+            echo "(no samples)"
+          fi
+          exit 0
+
+      - name: Upload hang samples
+        if: always() && needs.changes.outputs.docs_only != 'true'
+        continue-on-error: true
+        uses: actions/upload-artifact@v4
+        with:
+          name: hang-samples-${{ matrix.os }}-${{ matrix.shard }}
+          path: ${{ runner.temp }}/hang-samples.txt
+          if-no-files-found: ignore
+          retention-days: 3
 
       - name: Record which files this shard ran
         if: needs.changes.outputs.docs_only != 'true'