Strip down comments

Author: expede

pnpm-workspace.yaml

@@ -1,7 +1,5 @@
-# pnpm settings (pnpm ≥10.30 reads these from here, NOT from the
-# package.json "pnpm" field — overrides silently moved during the
-# 2026-06-12 git-dependency migration after we noticed the old location
-# was being ignored).
+# pnpm settings. They must live here: pnpm ≥10.3x silently ignores the
+# package.json "pnpm" field.
 
 # ADR-021: force a single @automerge/automerge so the CJS and ESM module
 # graphs share one Wasm instance. Two copies = two Wasm heaps = documents

src/commands.ts

@@ -190,14 +190,10 @@ async function setupCommandContext(
  * Safely shutdown a repository with proper error handling
  */
 async function safeRepoShutdown(repo: Repo): Promise<void> {
-  // The historical 3s pre-shutdown grace sleep is gone: since the
-  // feed-macrotasks shutdown quiesce pass (ADR-023/ADR-024),
-  // `repo.shutdown()` itself awaits in-flight sync rounds and runs a
-  // final bounded round for un-broadcast commits — deterministically
-  // doing what the sleep only hoped for. Verified 2026-06-12: online
-  // init + clone byte-identical and the two-repo suite green with no
-  // grace. PUSHWORK_SYNC_GRACE_MS remains as an escape hatch for
-  // debugging delivery issues against slow/flaky relays.
+  // `repo.shutdown()` quiesces outbound sync itself (awaits in-flight
+  // rounds, then a final bounded round for un-broadcast commits), so no
+  // grace sleep is needed. PUSHWORK_SYNC_GRACE_MS remains as an escape
+  // hatch for debugging delivery against slow or flaky relays.
   const graceMs = Number(process.env.PUSHWORK_SYNC_GRACE_MS ?? 0);
   if (Number.isFinite(graceMs) && graceMs > 0) {
     await new Promise((resolve) => setTimeout(resolve, graceMs));

src/core/change-detection.ts

@@ -55,7 +55,15 @@ export class ChangeDetector {
 	}
 
 	/**
-	 * Detect all changes between local filesystem and snapshot
+	 * Detect all changes between local filesystem and snapshot.
+	 *
+	 * Shard-mode (PUSHWORK_PARALLEL_INGEST=2) params:
+	 * - `freshPaths`: paths just pushed by worker-owned repos. Their snapshot
+	 *   heads are already current; materializing their docs on the main
+	 *   thread would reinstate the serial cost the workers exist to avoid,
+	 *   so both local and remote detection skip them for this run.
+	 * - `deferRemoteContent`: emit URL-only changes for new remote files
+	 *   (no doc materialization); shard-pull workers fetch them.
 	 */
 	async detectChanges(
 		snapshot: SyncSnapshot,
@@ -110,9 +118,7 @@ export class ChangeDetector {
 		await Promise.all(
 			Array.from(currentFiles.entries()).map(
 				async ([relativePath, fileInfo]) => {
-					// Paths just written by this run with known-current heads
-					// (shared-nothing worker ingest). Reading their docs here
-					// would materialize them on the main thread for no signal.
+					// Worker-pushed this run; see detectChanges docs.
 					if (freshPaths?.has(relativePath)) return
 
 					const snapshotEntry = snapshot.files.get(relativePath)
@@ -290,10 +296,8 @@ export class ChangeDetector {
 		await Promise.all(
 			Array.from(snapshot.files.entries()).map(
 				async ([relativePath, snapshotEntry]) => {
-					// Paths just pushed via shared-nothing worker repos: heads
-					// in the snapshot are current, and materializing the doc
-					// here would defeat the experiment. Concurrent remote
-					// edits are deferred to the next sync.
+					// Worker-pushed this run; see detectChanges docs. Genuinely
+					// concurrent remote edits surface on the next sync.
 					if (freshPaths?.has(relativePath)) return
 
 					// Find the file's current entry in the remote directory hierarchy
@@ -461,12 +465,10 @@ export class ChangeDetector {
 						// This is a remote file not in our snapshot
 						const localContent = await this.getLocalContent(entryPath)
 
-						// EXPERIMENT (shared-nothing clone): when deferring,
-						// don't materialize the file doc here — emit a
-						// deferred change carrying only the entry URL, to be
-						// fetched + written by a shard-pull worker. Only the
-						// clean clone case (no local copy) is deferrable;
-						// local/remote coexistence needs content to compare.
+						// Deferred fetch (shard mode): emit a URL-only change for
+						// a shard-pull worker. Only the clean case (no local
+						// copy) is deferrable — coexistence needs content here
+						// to compare.
 						if (deferRemoteContent && localContent === null) {
 							changes.push({
 								path: entryPath,

src/core/sync-engine.ts

@@ -144,10 +144,9 @@ export class SyncEngine {
 	// Map from path to handle for leaf-first sync ordering
 	// Path depth determines sync order (deepest first)
 	private handlesByPath: Map<string, DocHandle<unknown>> = new Map()
-	// Paths pushed by shard-ingest workers this run (shared-nothing
-	// experiment). These docs were created, persisted, and uploaded in
-	// worker-owned repos; the main thread must NOT materialize them —
-	// detect:post and the snapshot head-update loop skip these paths.
+	// Paths handled by worker-owned repos this run (shard mode). The main
+	// thread must NOT materialize these docs — detect:post and the snapshot
+	// head-update loop skip them; their reported heads are already current.
 	private workerPushedPaths: Set<string> = new Set()
 	private config: DirectoryConfig
 
@@ -549,9 +548,7 @@ export class SyncEngine {
 					undefined,
 					currentFiles,
 					undefined,
-					// Shared-nothing experiment: don't materialize new remote
-					// file docs during discovery; shard-pull workers fetch them.
-					parallelIngestMode() === "shard"
+					parallelIngestMode() === "shard" // defer remote fetches to workers
 				)
 			)
 
@@ -709,24 +706,16 @@ export class SyncEngine {
 				debug(`sync: excluding ${deletedPaths.size} deleted paths from re-detection`)
 			}
 			debug("sync: re-detecting changes after network sync")
-			// Shared-nothing experiment: paths pushed via worker-owned repos
-			// must not be re-checked here — that would repo.find (and thus
-			// materialize) every doc on the main thread, reinstating the
-			// serial cost the workers exist to avoid. Their snapshot heads
-			// were written from worker reports moments ago; genuinely
-			// concurrent remote edits get picked up on the next sync.
 			const freshChanges = await profileAsync("detect:post", () =>
 				this.changeDetector.detectChanges(
 					snapshot,
 					deletedPaths,
 					currentFiles,
+					// Skip worker-pushed paths (see workerPushedPaths).
 					this.workerPushedPaths.size > 0
 						? this.workerPushedPaths
 						: undefined,
-					// Shared-nothing experiment: emit deferred (URL-only)
-					// changes for new remote files; pull routes them to
-					// shard-pull workers.
-					parallelIngestMode() === "shard"
+					parallelIngestMode() === "shard" // defer remote fetches to workers
 				)
 			)
 			const freshRemoteChanges = freshChanges.filter(
@@ -759,9 +748,7 @@ export class SyncEngine {
 			// can't find the entries to splice out).
 			await profileAsync("snapshot:head-update", async () => {
 			for (const [filePath, snapshotEntry] of snapshot.files.entries()) {
-				// Shared-nothing experiment: worker-pushed docs aren't (and
-				// must not be) materialized on the main thread; their heads
-				// were recorded from the worker reports and are current.
+				// Worker-pushed heads are already current (see workerPushedPaths).
 				if (this.workerPushedPaths.has(filePath)) continue
 				try {
 					const handle = await this.repo.find(getPlainUrl(snapshotEntry.url))
@@ -883,16 +870,11 @@ export class SyncEngine {
 		debug(`push: ${localChanges.length} local changes (${newFiles.length} new, ${modifiedFiles.length} modified, ${deletedFiles.length} deleted)`)
 		out.update(`Pushing ${localChanges.length} local changes (${newFiles.length} new, ${modifiedFiles.length} modified, ${deletedFiles.length} deleted)`)
 
-		// EXPERIMENT (PUSHWORK_PARALLEL_INGEST=2, "shard"): build + persist +
-		// upload the docs for NEW files in worker-owned repos (darn-style
-		// parallel leaf phase), then proceed with the normal serial directory
-		// phase below. Workers own full repos (own Wasm/storage-writes/
-		// socket); the main thread never materializes the docs, only records
-		// {url, heads}. Files whose worker failed aren't in the map and fall
-		// back to main-thread creation in the per-file loop.
-		// (The "import" variant — workers shipping doc bytes for main-thread
-		// repo.import — was measured wall-neutral and removed; see
-		// .ignore/PARALLEL_INGEST_EXPERIMENT.md.)
+		// Shard mode (PUSHWORK_PARALLEL_INGEST=2): NEW files are built,
+		// persisted, and uploaded by worker-owned repos; the serial directory
+		// phase below stitches the reported {url, heads} in. Files whose
+		// worker failed are absent from the map and fall back to main-thread
+		// creation. See ingestNewFilesInShardedRepos.
 		const ingestMode = newFiles.length > 1 ? parallelIngestMode() : null
 		let shardResults = new Map<string, {url: AutomergeUrl; heads: UrlHeads}>()
 		if (ingestMode === "shard") {
@@ -1168,10 +1150,8 @@ export class SyncEngine {
 				c.changeType === ChangeType.BOTH_CHANGED
 		)
 
-		// EXPERIMENT (shared-nothing clone): deferred changes carry only
-		// an entry URL — fetch + materialize + write them in shard-pull
-		// workers. Failed paths fall back to the normal serial loop by
-		// re-fetching content on the main thread.
+		// Deferred changes (shard mode) carry only an entry URL; shard-pull
+		// workers fetch + write them. Failures fall back to the serial loop.
 		const deferred = remoteChanges.filter(c => c.deferredFetch)
 		const immediate = remoteChanges.filter(c => !c.deferredFetch)
 		let fallbackPaths: Set<string> | undefined
@@ -1230,14 +1210,13 @@ export class SyncEngine {
 	}
 
 	/**
-	 * EXPERIMENT (shared-nothing clone): pull deferred remote files in
-	 * worker-owned repos. Workers download/load + materialize the docs
-	 * with their own Wasm instances, write the local files themselves,
-	 * and report `{url, heads, contentHash}`; the main thread records
-	 * snapshot entries without ever materializing the documents.
+	 * Shard-mode pull: fetch deferred remote files in worker-owned repos.
+	 * Workers materialize the docs with their own Wasm instances, write the
+	 * local files themselves, and report `{url, heads, contentHash}`; the
+	 * main thread records snapshot entries without materializing anything.
 	 *
-	 * Returns the set of paths the workers could NOT handle (caller runs
-	 * the main-thread fallback for those).
+	 * Returns the paths the workers could not handle (caller runs the
+	 * main-thread fallback for those).
 	 */
 	private async pullRemoteFilesInShardedRepos(
 		deferred: DetectedChange[],
@@ -1507,18 +1486,18 @@ export class SyncEngine {
 	}
 
 	/**
-	 * EXPERIMENT (shared-nothing): ingest new files in worker-owned repos.
+	 * Shard-mode push: ingest new files in worker-owned repos.
 	 *
-	 * Workers build, persist (shared storage directory, disjoint doc
-	 * keys), and upload their shard with their OWN Repo + Wasm + socket,
-	 * reporting only `{url, heads}`. The main thread never materializes
-	 * these docs; the per-file loop stitches the URLs into directory
-	 * entries and the snapshot.
+	 * Workers build, persist (shared storage directory, disjoint doc keys),
+	 * and upload their shard with their own Repo + Wasm + socket, reporting
+	 * only `{url, heads}`. The main thread never materializes these docs;
+	 * the per-file loop stitches the URLs into directory entries and the
+	 * snapshot.
 	 *
 	 * Paths that failed or did not converge to the server are absent from
 	 * the returned map, so the per-file loop falls back to main-thread
-	 * `createRemoteFile` for them (the worker's copy becomes an
-	 * unreferenced orphan in storage, same class as deleted-file orphans).
+	 * `createRemoteFile` (the worker's copy becomes an unreferenced storage
+	 * orphan, same class as deleted-file orphans).
 	 */
 	private async ingestNewFilesInShardedRepos(
 		newFiles: DetectedChange[],

src/types/documents.ts

@@ -89,11 +89,9 @@ export interface DetectedChange {
 	/** New remote URL when the remote document was replaced (artifact URL change) */
 	remoteUrl?: AutomergeUrl
 	/**
-	 * EXPERIMENT (shared-nothing clone): the remote document was NOT
-	 * materialized during discovery — `remoteContent`/`remoteHead` are
-	 * unset and `remoteUrl` carries the directory entry URL. These
-	 * changes are routed to shard-pull workers instead of the normal
-	 * pull loop.
+	 * Shard mode: the remote doc was not materialized during discovery —
+	 * `remoteContent`/`remoteHead` are unset, `remoteUrl` carries the entry
+	 * URL, and the change is routed to a shard-pull worker.
 	 */
 	deferredFetch?: boolean
 }

src/utils/fs.ts

@@ -241,9 +241,6 @@ export async function movePath(
 	await fs.rename(sourcePath, destPath)
 }
 
-// (calculateContentHash removed 2026-06-12: it was an unused duplicate of
-// utils/content.ts's contentHash — use that.)
-
 /**
  * Get MIME type for file
  */

src/utils/ingest-pool.ts

@@ -1,15 +1,12 @@
 /**
- * Worker pools for the shared-nothing parallel-ingest experiment
+ * Worker pools for shared-nothing parallel ingest
  * (PUSHWORK_PARALLEL_INGEST=2 / "shard"): workers own full Repos (own
  * Wasm, own storage writes, own socket) and report only {relPath, url,
- * heads}; the main thread never materializes the docs.
+ * heads}; the main thread never materializes the docs. Background and
+ * measurements: .ignore/PARALLEL_INGEST_EXPERIMENT.md.
  *
  * Worker scripts run as compiled CommonJS, so a build (`npm run build`)
  * must exist even when the engine itself runs from src via tsx.
- *
- * (A second variant — "import" mode, workers shipping doc bytes for the
- * main thread to repo.import — was measured wall-neutral and deleted
- * 2026-06-12; see .ignore/PARALLEL_INGEST_EXPERIMENT.md.)
  */
 
 import * as fs from "node:fs"
@@ -30,10 +27,7 @@ export interface IngestTask {
 
 export type ParallelIngestMode = "shard" | null
 
-/**
- * PUSHWORK_PARALLEL_INGEST=2 (or "shard") enables shared-nothing shard
- * ingest. ("1"/"import" was the refuted repo.import variant, removed.)
- */
+/** PUSHWORK_PARALLEL_INGEST=2 (or "shard") enables shard ingest. */
 export function parallelIngestMode(): ParallelIngestMode {
 	switch (process.env.PUSHWORK_PARALLEL_INGEST) {
 		case "2":
@@ -91,9 +85,7 @@ export async function runShardIngest(
 	protocol: SyncProtocol,
 	tasks: IngestTask[],
 	workerCount = ingestWorkerCount(),
-	// Test seam: inject a stub worker script (e.g. one that exits without
-	// reporting) to exercise the pool's failure paths without Wasm.
-	workerScript?: string
+	workerScript?: string // test seam: stub script exercising failure paths
 ): Promise<ShardRunOutcome> {
 	const script = workerScript ?? resolveShardWorkerScript()
 	const count = Math.min(workerCount, tasks.length)
@@ -241,10 +233,7 @@ export async function runShardPull(
 					resolve()
 				})
 				worker.on("exit", () => {
-					// Exit-before-report is a failure regardless of exit code: a
-					// worker that process.exit(0)s without posting still lost its
-					// shard. (fail() is a no-op once the report has arrived, which
-					// covers the eager-terminate exit after a successful report.)
+					// Exit-before-report = failure (see runShardIngest).
 					fail()
 					resolve()
 				})