Merge branch 'main' into claude/roadmap-thread-reopen-w80epb

Author: jonathanKingston

.github/actions/setup/action.yml

@@ -72,7 +72,7 @@ runs:
     # node_modules cache hits next time. So this layer is pure waste on the hit path
     # and redundant on the miss path.
     - uses: actions/setup-node@v6
-      with: { node-version: '22' }
+      with: { node-version-file: '.nvmrc' }
     # ripgrep backs the app's code-search services at runtime; the unit suite
     # and the e2e app both exercise them. GitHub-hosted runners have passwordless
     # sudo + apt, so install it there. Self-hosted runners typically have neither

.github/workflows/ci.yml

@@ -201,24 +201,15 @@ jobs:
         run: |
           git fetch --no-tags --depth=1 origin "$BASE_SHA" 2>/dev/null || true
           node scripts/check-screenshots.mts --plan --base "$BASE_SHA"
-      # e2e shard fan-out is runner-dependent. The GitHub-hosted nightly keeps 8
-      # shards to stay under the 2-core/7GB OOM ceiling (see the e2e job comment).
-      # PR/push runs on the self-hosted Linux Docker `copse-e2e` fleet (see #422),
-      # where 8 shards just means paying the (mostly-cached but non-trivial, and
-      # occasionally cold) dep restore + Electron setup up to 8× across the fleet.
-      # Fewer, larger shards there cut that repeated setup without duplicating any
-      # spec: wdio `--shard current/total` and the `subset` round-robin both
-      # partition the specs disjointly, so a lower total just makes each slice
-      # bigger. Tune the self-hosted count below to your fleet size.
+      # Keep every full e2e run at 8 shards. Both runner classes have a tight
+      # memory ceiling: GitHub-hosted is 7 GB and each self-hosted `copse-e2e`
+      # container is capped at 6 GB. Packing the suite into 3 shards makes each
+      # container launch too many sequential Electron sessions; accumulated
+      # Electron/gortex processes then OOM-crash unrelated late-running specs.
+      # `--shard` and the subset round-robin still partition specs disjointly.
       - id: shards
-        env:
-          EVENT: ${{ github.event_name }}
         run: |
-          if [ "$EVENT" = "schedule" ]; then
-            { echo "total=8"; echo "list=[1,2,3,4,5,6,7,8]"; } >> "$GITHUB_OUTPUT"
-          else
-            { echo "total=3"; echo "list=[1,2,3]"; } >> "$GITHUB_OUTPUT"
-          fi
+          { echo "total=8"; echo "list=[1,2,3,4,5,6,7,8]"; } >> "$GITHUB_OUTPUT"
 
   # Auto-fix the PR branch: run every autofix we have — ESLint `--fix` then
   # Prettier `--write` — and commit the result back onto the head branch. This
@@ -455,19 +446,12 @@ jobs:
     # slice of specs via wdio's `--shard current/total`. The shared `dist` artifact
     # from the `build` job is unpacked here rather than rebuilt per shard.
     #
-    # Shard count is runner-dependent (computed by the precheck `shards` step):
-    #   - hosted nightly: 8. Each shard runs its slice as one sequential wdio
-    #     process, relaunching Electron per spec. On the 2-core/7GB runner, once a
-    #     shard runs too many specs the accumulated electron/gortex processes
-    #     exhaust it and a later spec's Electron fails to boot or OOM-crashes the
-    #     runner ("shutdown signal"). #339's data showed 8 stays under that limit
-    #     even in the densest spec range.
-    #   - self-hosted PR/push: 3. These run on the Linux Docker `copse-e2e`
-    #     runners (see #422), which have the headroom 8 was protecting against, so
-    #     the only thing 8 buys there is paying dep restore + Electron setup up to
-    #     8× across the fleet. Fewer, larger shards cut that repeated (and
-    #     sometimes cold) setup — the dominant wall-clock cost — without
-    #     duplicating any spec. Tune to the `copse-e2e` runner count.
+    # Every full run uses 8 shards. Each shard runs its slice as one sequential
+    # wdio process, relaunching Electron per spec. Both the hosted runner (7 GB)
+    # and self-hosted container (6 GB cap) eventually exhaust memory when a shard
+    # launches too many Electron/gortex processes; a later unrelated spec then
+    # fails to boot or OOM-crashes ("shutdown signal"). #339's data showed that
+    # 8 shards stays under that limit even in the densest spec range.
     # The per-attempt `timeout` is 480s (step cap 26 min for 3 attempts) so attempt
     # 1 finishes instead of being SIGKILL'd mid-retry, letting attempt 2 relaunch.
     strategy:
@@ -506,9 +490,11 @@ jobs:
       # The Electron/Chromedriver e2e session occasionally fails to start on the
       # runner ("DevToolsActivePort file doesn't exist" / "POST /session" timeout).
       # The hang wedges the whole wdio run, so in-process retries don't recover —
-      # only a fresh xvfb+Electron launch does. Retry the entire shard, and wrap
-      # each attempt in `timeout` so a wedged session is killed and retried
-      # (a plain hang would otherwise never exit and stall the job).
+      # only a fresh xvfb+Electron launch does. Retry the entire shard, cleaning
+      # orphaned session processes before each attempt and when the step exits.
+      # The runners are one job per PID-isolated container, so this cannot touch
+      # another job. Wrap each attempt in `timeout` so a plain hang cannot stall
+      # the job indefinitely.
       # Oracle gate (see the precheck job's plan step): `full` shards the whole
       # suite; `subset` distributes only the planned specs round-robin across the
       # same 8 shards (so no shard exceeds the per-shard memory budget the count
@@ -550,7 +536,25 @@ jobs:
           # fall back to no inner timeout (the step `timeout-minutes` still caps
           # a wedged run) so the command can't die with "timeout: not found".
           TIMEOUT="$(command -v timeout || command -v gtimeout || true)"
+
+          cleanup_e2e_processes() {
+            # Bracketed patterns avoid matching this shell's own command line.
+            # A failed deleteSession can orphan any of these; self-hosted runner
+            # containers are reused, so clean before attempt 1 as well as retries.
+            pkill -TERM -f '[e]lectron/dist/electron' 2>/dev/null || true
+            pkill -TERM -f '[e]lectron-chromedriver/bin/chromedriver' 2>/dev/null || true
+            pkill -TERM -f '[X]vfb' 2>/dev/null || true
+            pkill -TERM -f '[g]ortex' 2>/dev/null || true
+            sleep 1
+            pkill -KILL -f '[e]lectron/dist/electron' 2>/dev/null || true
+            pkill -KILL -f '[e]lectron-chromedriver/bin/chromedriver' 2>/dev/null || true
+            pkill -KILL -f '[X]vfb' 2>/dev/null || true
+            pkill -KILL -f '[g]ortex' 2>/dev/null || true
+          }
+
+          trap cleanup_e2e_processes EXIT
           for attempt in 1 2 3; do
+            cleanup_e2e_processes
             echo "::group::e2e shard ${{ matrix.shard }} attempt $attempt"
             if ${TIMEOUT:+$TIMEOUT -k 15 480} npm run test:e2e:ci -- $SHARD_ARGS $SPEC_ARGS; then
               echo "::endgroup::"
@@ -965,15 +969,33 @@ jobs:
   # have their own explicit safe-tier result below: they may pass only after
   # `precheck` actually ran on a GitHub-hosted runner, never merely because all
   # code-executing jobs were skipped.
+  #
+  # Concurrency supersession: top-level `cancel-in-progress` cancels in-flight
+  # jobs when a newer push/sync arrives on the same ref. Those cancelled jobs
+  # must not paint the superseded SHA's `CI Passed` red — tip CI is the gate
+  # that matters. We detect supersession by comparing this run's head SHA to
+  # the live tip; genuine cancels of the tip (manual cancel, runner death with
+  # no tip move) still fail. Sibling-matrix cancels after a real shard failure
+  # still fail because `ANY_FAILURE` is true first.
   ci-passed:
     name: CI Passed
     if: always()
     needs: [precheck, check, bench, build, e2e]
     # Tiny aggregate gate — keep on hosted so a saturated self-hosted check
     # fleet cannot leave a green pipeline without its required status check.
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
     steps:
       - name: Check upstream job results
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPOSITORY: ${{ github.repository }}
+          EVENT_NAME: ${{ github.event_name }}
+          REF_NAME: ${{ github.ref_name }}
+          RUN_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          PR_NUMBER: ${{ github.event.pull_request.number || '' }}
         run: |
           FORK_PR=${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
           if $FORK_PR; then
@@ -987,7 +1009,7 @@ jobs:
           fi
           MODE="${{ needs.precheck.outputs.mode }}"
           ANY_FAILURE=${{ contains(needs.*.result, 'failure') }}
-          ANY_BAD=${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+          ANY_CANCELLED=${{ contains(needs.*.result, 'cancelled') }}
           CORE_OK=${{ needs.precheck.result == 'success' }}
           # Skip-mode runs (screenshots-only pushes, e.g. the e2e screenshot
           # bot's own commits) intentionally do not execute check/build/e2e;
@@ -1003,8 +1025,32 @@ jobs:
             echo "All CI jobs passed (heavy jobs intentionally skipped: mode=skip)."
             exit 0
           fi
-          if $ANY_BAD; then
-            echo "One or more CI jobs failed or were cancelled:"
+          if $ANY_FAILURE; then
+            echo "One or more CI jobs failed:"
+            echo '${{ toJSON(needs) }}'
+            exit 1
+          fi
+          if $ANY_CANCELLED; then
+            # Fail closed if we cannot resolve tip — a tip cancel must stay red.
+            if [ -n "$PR_NUMBER" ]; then
+              TIP_SHA=$(gh api "repos/${REPOSITORY}/pulls/${PR_NUMBER}" --jq .head.sha) || {
+                echo "Could not resolve PR tip SHA; treating cancel as failure."
+                echo '${{ toJSON(needs) }}'
+                exit 1
+              }
+            else
+              TIP_SHA=$(gh api "repos/${REPOSITORY}/commits/${REF_NAME}" --jq .sha) || {
+                echo "Could not resolve branch tip SHA; treating cancel as failure."
+                echo '${{ toJSON(needs) }}'
+                exit 1
+              }
+            fi
+            if [ -n "$TIP_SHA" ] && [ "$TIP_SHA" != "$RUN_HEAD_SHA" ]; then
+              echo "Run superseded by newer tip ${TIP_SHA} (this SHA=${RUN_HEAD_SHA}); cancelled jobs are concurrency noise."
+              echo '${{ toJSON(needs) }}'
+              exit 0
+            fi
+            echo "One or more CI jobs were cancelled while still at tip (${RUN_HEAD_SHA}):"
             echo '${{ toJSON(needs) }}'
             exit 1
           fi

.github/workflows/release-mac.yml

@@ -31,7 +31,24 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  # Keep a deliberately uncached install in the release path. The shared setup
+  # action restores a complete node_modules tree when possible, which is useful
+  # for packaging but can hide an undeclared/transitive production dependency.
+  # This gate exercises the lockfile from scratch with install scripts disabled,
+  # then proves the production bundle is self-contained before packaging starts.
+  verify-clean-release-build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7.0.0
+      - uses: actions/setup-node@v6
+        with: { node-version-file: '.nvmrc' }
+      - name: Verify clean release build
+        run: npm ci --ignore-scripts && npm run build:release
+
   build-mac:
+    needs: verify-clean-release-build
     # Apple-Silicon GitHub-hosted runner; electron-builder downloads the x64
     # Electron to cross-build the Intel artifacts in the same run.
     runs-on: macos-14

AGENTS.md

@@ -195,6 +195,12 @@ decisions) and `permission-gate.test.ts` (gate wiring + MCP decisions).
 - `npm test` runs Node's test runner over `src/**/*.test.ts` (esbuild-bundled into `dist-test/`).
 - `npm run test:e2e` is WebdriverIO + `@wdio/electron-service` and needs a display. It passes under
   `npm run test:e2e` on this headless VM (WDIO auto-starts Xvfb on Linux).
+- **Don't block the machine on e2e while iterating.** When a remote e2e host is available
+  (`.tmp/remote-e2e/host.json` exists, or one can be provisioned), prefer
+  `npm run e2e:remote -- run --detach` — it snapshots the working tree (uncommitted changes
+  included), runs the oracle-selected specs on a cloud container, and leaves results in
+  `.tmp/remote-e2e/runs/<run-id>/`; finish with `e2e:remote -- wait <run-id>` and keep working in
+  between. See [`ci-runners/README.md`](ci-runners/README.md#remote-e2e-dev-hosts-npm-run-e2eremote).
 
 For _which tier a test belongs in_ — favour unit/component tests, reserve e2e for broad validation
 and real-runtime checks (sizing/rendering, Monaco, terminal, webview, main IPC) — read

CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog and release notes
+
+The canonical changelog is the set of
+[GitHub Releases](https://github.com/copse-dev/agent-pane/releases). Published
+release notes are owned and maintained with the GitHub Release; this file records
+the release-note process and the current unreleased summary rather than copying
+every published entry.
+
+## Unreleased
+
+- Added general-availability security, support, privacy/data-flow, release, and
+  recovery documentation.
+
+## Release-note process
+
+For every release:
+
+1. Draft the GitHub Release from the matching `v<version>` tag.
+2. Turn merged changes into user-facing notes, grouped into features, fixes,
+   security/privacy changes, and developer changes as applicable.
+3. State the supported OS and architectures, known issues, data migrations, and
+   recovery implications. Copse supports forward fixes only; do not recommend a
+   downgrade.
+4. Link issues or pull requests that provide important detail without exposing
+   confidential security-report information.
+5. Review the notes with the artifacts, then publish them as part of the GitHub
+   Release.
+6. Reset the `Unreleased` section here after publication. Do not mirror the
+   published notes into a second historical list in this file.
+
+The complete shipping procedure is in
+[docs/release-checklist.md](docs/release-checklist.md).

Makefile

@@ -67,9 +67,30 @@ BUILD_STAMP := $(STAMP_DIR)/build.stamp
 
 # Source that, when changed, should trigger a rebuild of `dist/`. Evaluated when
 # the Makefile is parsed; new files are picked up on the next `make` invocation.
-BUILD_SRC := $(shell find src packages scripts -type f 2>/dev/null) \
+# `assets` is included because scripts/build.mts copies it into dist/ wholesale.
+BUILD_SRC_DIRS := src packages scripts assets
+BUILD_SRC := $(shell find $(BUILD_SRC_DIRS) -type f 2>/dev/null) \
              package.json tsconfig.json tsconfig.node.json tsconfig.web.json
 
+# find-based prerequisites only notice files that still exist. A branch switch
+# that merely deletes or renames a module leaves every surviving file older
+# than the stamp, so make declares dist/ "up to date" while the bundle still
+# contains the deleted code. Fingerprint the file *list* into a stamp that is
+# refreshed (at parse time) whenever the list changes; ordinary mtime logic
+# then forces the rebuild.
+SRC_LIST_STAMP := $(STAMP_DIR)/src-list.stamp
+SRC_LIST_SUM := $(shell find $(BUILD_SRC_DIRS) -type f 2>/dev/null | sort | cksum | cut -d' ' -f1)
+ifneq ($(SRC_LIST_SUM),$(shell cat $(SRC_LIST_STAMP) 2>/dev/null))
+$(shell mkdir -p $(STAMP_DIR) && echo '$(SRC_LIST_SUM)' > $(SRC_LIST_STAMP))
+endif
+
+# The build stamp only proves a build *ran* — not that dist/ still holds its
+# output. `npm run dev` writes watch-mode bundles (a partial set, dev flags)
+# into the same dist/, and a hand-deleted dist/ leaves the stamp behind. If the
+# main bundle is missing or newer than the stamp, dist/ was touched outside
+# make: drop the stamp so the next build re-syncs it.
+$(shell if [ -f $(BUILD_STAMP) ] && { [ ! -f dist/main/index.js ] || [ dist/main/index.js -nt $(BUILD_STAMP) ]; }; then rm -f $(BUILD_STAMP); fi)
+
 # ----------------------------------------------------------------------------
 # Help
 # ----------------------------------------------------------------------------
@@ -248,12 +269,32 @@ check-node:
 # terminal fails to launch. The flag is scoped to this one invocation, so it
 # doesn't touch your global config. See the "Hardened npm profiles" section of
 # the README.
+#
+# On macOS, `npm ci` after a lockfile change (e.g. switching branches) can die
+# with ENOTEMPTY/EBUSY/EPERM while pruning stale packages — Spotlight, Finder,
+# or an editor briefly holds a file inside a directory npm is rmdir-ing. The
+# tree it leaves behind is half-pruned, so every rerun fails the same way until
+# node_modules is removed by hand. Detect that failure class from npm's output,
+# wipe node_modules, and retry once; any other failure (or a second one) still
+# aborts loudly.
+NPM_CI := npm ci --ignore-scripts=false
+
 .PHONY: deps
 deps: $(DEPS_STAMP)
 
 $(DEPS_STAMP): package-lock.json package.json | $(STAMP_DIR) check-node
 	@echo "==> Dependencies out of date — running 'npm ci' (scripts forced on)…"
-	@$(USE_NVM); npm ci --ignore-scripts=false
+	@$(USE_NVM); \
+	log="$(STAMP_DIR)/npm-ci.log"; \
+	if ! $(NPM_CI) 2>&1 | tee "$$log"; then \
+	  if grep -qE 'code (ENOTEMPTY|EBUSY|EPERM)' "$$log"; then \
+	    echo "==> npm ci hit a filesystem race pruning node_modules — wiping it and retrying…"; \
+	    rm -rf node_modules; \
+	    $(NPM_CI); \
+	  else \
+	    exit 1; \
+	  fi; \
+	fi
 	touch $(DEPS_STAMP)
 
 # --- build ------------------------------------------------------------------
@@ -262,7 +303,7 @@ $(DEPS_STAMP): package-lock.json package.json | $(STAMP_DIR) check-node
 .PHONY: build
 build: $(BUILD_STAMP)
 
-$(BUILD_STAMP): $(DEPS_STAMP) $(BUILD_SRC) | $(STAMP_DIR)
+$(BUILD_STAMP): $(DEPS_STAMP) $(BUILD_SRC) $(SRC_LIST_STAMP) | $(STAMP_DIR)
 	@echo "==> Source changed — clearing dist/ and rebuilding…"
 	rm -rf dist
 	@$(USE_NVM); npm run build
@@ -279,4 +320,4 @@ run: build
 .PHONY: clean
 clean:
 	@echo "==> Removing dist/ and build/deps stamps…"
-	rm -rf dist $(DEPS_STAMP) $(BUILD_STAMP)
+	rm -rf dist $(DEPS_STAMP) $(BUILD_STAMP) $(SRC_LIST_STAMP)