diff --git a/src/ingest-pool.ts b/src/ingest-pool.ts new file mode 100644 index 0000000..8d4dfd7 --- /dev/null +++ b/src/ingest-pool.ts @@ -0,0 +1,237 @@ +/** + * Worker pools for shared-nothing parallel ingest / clone + * (PUSHWORK_PARALLEL_INGEST=shard). + * + * Each worker owns a full Repo (own Wasm, own storage writes, own socket) and + * reports only URLs/heads (push) or which files it wrote (clone); the main + * thread never materializes the file documents. New files / leaves are split + * round-robin across a bounded pool, and a per-worker failure falls back to + * main-thread handling rather than failing the whole operation. + * + * The worker scripts run as compiled CommonJS sitting next to this module + * (dist/workers/...), so a build must exist even when the engine runs from + * source via tsx. + */ +import * as os from "os"; +import * as path from "path"; +import { Worker } from "worker_threads"; +import { + parseAutomergeUrl, + stringifyAutomergeUrl, + type AutomergeUrl, + type UrlHeads, +} from "@automerge/automerge-repo"; +import type { Backend } from "./config.js"; +import { log } from "./log.js"; +import { isInArtifactDir } from "./shapes/index.js"; +import type { + ShardIngestData, + ShardIngestReport, + ShardIngestTask, +} from "./workers/shard-ingest-worker.js"; +import type { + ShardCloneData, + ShardCloneReport, +} from "./workers/shard-clone-worker.js"; + +const dlog = log("ingest-pool"); + +// Higher worker counts exhaust file descriptors during the online upload +// storm (EMFILE). Overridable for experiments via PUSHWORK_WORKERS. +const SHARD_WORKER_CAP = Math.max( + 1, + Math.floor(Number(process.env.PUSHWORK_WORKERS) || 8), +); + +// Below this many items, spinning up workers (each ~1s of Wasm + Repo init) +// doesn't pay for itself versus the main-thread path. +const SHARD_MIN_ITEMS = 8; + +// Auto-dispatch thresholds for ingest: shard automatically only when the CRDT +// build is heavy enough to beat per-worker overhead. Tiny files are +// overhead-bound (a wash), so a small-file tree stays on the main thread even +// past the item floor. +const AUTO_AVG_BYTES = 8 * 1024; +const AUTO_TOTAL_BYTES = 4 * 1024 * 1024; + +type ExplicitMode = "on" | "off" | null; + +// `PUSHWORK_PARALLEL_INGEST=shard`/`2` forces the pools on; `off`/`0` forces +// them off; anything else leaves the adaptive policy in charge. +function explicitMode(): ExplicitMode { + const v = process.env.PUSHWORK_PARALLEL_INGEST; + if (v === "shard" || v === "2") return "on"; + if (v === "off" || v === "0") return "off"; + return null; +} + +/** + * Whether to shard an ingest of `files`. Explicit on/off always wins; otherwise + * auto-enable only for CPU-bound trees (large average file or large total + * bytes), since the per-char op-tree build is what the workers parallelize. + */ +export function shouldShardIngest(files: Map): boolean { + if (files.size < SHARD_MIN_ITEMS) return false; + const mode = explicitMode(); + if (mode !== null) return mode === "on"; + let total = 0; + for (const bytes of files.values()) total += bytes.length; + return total >= AUTO_TOTAL_BYTES || total / files.size >= AUTO_AVG_BYTES; +} + +/** + * Whether to shard a clone/materialize of `leafCount` files. Parallel + * download/read wins broadly (the serial path pays a per-file `repo.find`), so + * auto-enable past the floor; explicit `off`/`0` still declines. + */ +export function shouldShardClone(leafCount: number): boolean { + if (leafCount < SHARD_MIN_ITEMS) return false; + return explicitMode() !== "off"; +} + +function workerCount(itemCount: number): number { + const cores = Math.max(1, (os.cpus()?.length ?? 1) - 1); + return Math.max(1, Math.min(SHARD_WORKER_CAP, cores, itemCount)); +} + +// Worker scripts are compiled next to this module: dist/ingest-pool.js + +// dist/workers/.js (or dist-bench/src/... under the bench build). +function workerPath(name: string): string { + return path.join(__dirname, "workers", name); +} + +function partition(items: T[], buckets: number): T[][] { + const out: T[][] = Array.from({ length: buckets }, () => []); + items.forEach((item, i) => out[i % buckets].push(item)); + return out; +} + +function pinFromHeads(url: AutomergeUrl, heads: string[]): AutomergeUrl { + const { documentId } = parseAutomergeUrl(url); + return stringifyAutomergeUrl({ documentId, heads: heads as UrlHeads }); +} + +// Spawn a worker, resolve with its single report message, and terminate it +// eagerly on report (the report is sent strictly after the worker's storage +// flush / shutdown, so nothing of value remains — and a leftover sync timer +// would otherwise keep the worker's event loop alive long after shutdown). +function runWorker( + scriptPath: string, + workerData: TData, +): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(scriptPath, { workerData }); + let settled = false; + worker.once("message", (msg: TReport) => { + settled = true; + void worker.terminate(); + resolve(msg); + }); + worker.once("error", (err) => { + settled = true; + reject(err); + }); + worker.once("exit", (code) => { + if (!settled) reject(new Error(`worker exited with code ${code}`)); + }); + }); +} + +export type ShardIngestOpts = { + root: string; + backend: Backend; + online: boolean; + files: Map; + artifactDirs: readonly string[]; +}; + +/** + * Create file documents for `files` across the worker pool. Returns the URLs + * the workers created (artifact paths pinned to their reported heads) keyed by + * posix path, plus the paths a worker could not create (caller materializes + * those on the main thread). + */ +export async function shardIngest( + opts: ShardIngestOpts, +): Promise<{ created: Map; failed: string[] }> { + const tasks: ShardIngestTask[] = Array.from(opts.files, ([relPath, bytes]) => ({ + relPath, + bytes, + isArtifact: isInArtifactDir(relPath, opts.artifactDirs), + })); + const n = workerCount(tasks.length); + dlog("shardIngest files=%d workers=%d online=%s", tasks.length, n, opts.online); + + const reports = await Promise.all( + partition(tasks, n).map((shard) => + runWorker( + workerPath("shard-ingest-worker.js"), + { root: opts.root, backend: opts.backend, online: opts.online, tasks: shard }, + ).catch((err): ShardIngestReport => { + dlog("ingest worker failed, falling back to main: %s", err); + return { + results: shard.map((t) => ({ + relPath: t.relPath, + ok: false, + error: String(err), + })), + }; + }), + ), + ); + + const created = new Map(); + const failed: string[] = []; + for (const report of reports) { + for (const r of report.results) { + if (r.ok) { + created.set(r.relPath, r.isArtifact ? pinFromHeads(r.url, r.heads) : r.url); + } else { + failed.push(r.relPath); + } + } + } + dlog("shardIngest created=%d failed=%d", created.size, failed.length); + return { created, failed }; +} + +export type ShardCloneOpts = { + root: string; + backend: Backend; + online: boolean; + leaves: Map; +}; + +/** + * Write the `leaves` (posix path → file-doc URL) to disk across the worker + * pool. Returns the paths workers wrote and the paths they could not (caller + * materializes those on the main thread). + */ +export async function shardClone( + opts: ShardCloneOpts, +): Promise<{ written: Set; failed: string[] }> { + const entries = Array.from(opts.leaves) as [string, AutomergeUrl][]; + const n = workerCount(entries.length); + dlog("shardClone leaves=%d workers=%d online=%s", entries.length, n, opts.online); + + const reports = await Promise.all( + partition(entries, n).map((shard) => + runWorker( + workerPath("shard-clone-worker.js"), + { root: opts.root, backend: opts.backend, online: opts.online, leaves: shard }, + ).catch((err): ShardCloneReport => { + dlog("clone worker failed, falling back to main: %s", err); + return { written: [], failed: shard.map(([relPath]) => relPath) }; + }), + ), + ); + + const written = new Set(); + const failed: string[] = []; + for (const report of reports) { + for (const p of report.written) written.add(p); + for (const p of report.failed) failed.push(p); + } + dlog("shardClone written=%d failed=%d", written.size, failed.length); + return { written, failed }; +} diff --git a/src/pushwork.ts b/src/pushwork.ts index ba28a3b..b7c5c07 100644 --- a/src/pushwork.ts +++ b/src/pushwork.ts @@ -19,6 +19,12 @@ import { } from "./config.js"; import { loadIgnore } from "./ignore.js"; import { byteEq, walkDir, writeFileAtomic } from "./fs-tree.js"; +import { + shardClone, + shardIngest, + shouldShardClone, + shouldShardIngest, +} from "./ingest-pool.js"; import { log } from "./log.js"; import { openRepo, waitForSync } from "./repo.js"; import { @@ -116,7 +122,9 @@ export async function init(opts: InitOpts): Promise { dlog("init walked %d files", fsFiles.size); const title = path.basename(root) || undefined; - const tree = await pushFiles(repo, fsFiles, undefined, artifactDirs); + const tree = shouldShardIngest(fsFiles) + ? await ingestSharded(repo, root, opts.backend, online, fsFiles, artifactDirs) + : await pushFiles(repo, fsFiles, undefined, artifactDirs); const folderUrl = await shape.encode({ repo, tree, title }); dlog("init encoded folder=%s title=%s", folderUrl, title); const folderHandle = await repo.find(folderUrl); @@ -196,7 +204,7 @@ export async function clone(opts: CloneOpts): Promise { }); const tree = await shape.decode({ repo, root: folderHandle }); - await materializeTree(repo, root, tree); + await materializeTree(repo, root, tree, { backend: opts.backend, online }); await writeConfig(root, { version: CONFIG_VERSION, @@ -832,6 +840,49 @@ async function pushFiles( return root; } +/** + * Like `pushFiles` for the all-new case (init), but builds the file documents + * across the shared-nothing worker pool: workers create + persist (+ upload) + * their shard and report `{path, url, heads}`; this thread only stitches the + * URLs into the tree (pinning artifacts from the reported heads) and falls + * back to main-thread creation for any path a worker could not handle. + */ +async function ingestSharded( + repo: Repo, + root: string, + backend: Backend, + online: boolean, + fsFiles: Map, + artifactDirs: readonly string[], +): Promise { + const { created, failed } = await shardIngest({ + root, + backend, + online, + files: fsFiles, + artifactDirs, + }); + + const tree = newDir(); + for (const [posixPath, url] of created) { + setFileAt(tree, posixPath.split("/").filter(Boolean), url); + } + + for (const posixPath of failed) { + const bytes = fsFiles.get(posixPath); + if (!bytes) continue; + const isArtifact = isInArtifactDir(posixPath, artifactDirs); + const handle = repo.create( + makeFileEntry(posixPath, bytes, isArtifact), + ); + const url = isArtifact ? pinUrl(handle) : handle.url; + setFileAt(tree, posixPath.split("/").filter(Boolean), url); + } + + dlog("ingestSharded created=%d main-fallback=%d", created.size, failed.length); + return tree; +} + /** * Re-pin every artifact leaf in the folder doc to its file doc's current * heads. Bare (non-artifact) URLs are left as-is since they already track @@ -885,9 +936,20 @@ async function materializeTree( repo: Repo, root: string, tree: VfsNode, + shardCtx?: { backend: Backend; online: boolean }, ): Promise { + const leaves = flattenLeaves(tree); + + // Clone path: fan the per-file download + write out to the worker pool. + // Only the caller that knows the working tree is freshly materialized + // (clone) passes shardCtx, so writing every leaf is correct here. + if (shardCtx && shouldShardClone(leaves.size)) { + await materializeSharded(repo, root, leaves, shardCtx); + return; + } + const desired = new Map(); - for (const [posixPath, fileUrl] of flattenLeaves(tree)) { + for (const [posixPath, fileUrl] of leaves) { const handle = await repo.find(fileUrl); desired.set(posixPath, contentToBytes(handle.doc().content)); } @@ -916,6 +978,52 @@ async function materializeTree( dlog("materialize done: %d written, %d removed", written, removed); } +async function materializeSharded( + repo: Repo, + root: string, + leaves: Map, + shardCtx: { backend: Backend; online: boolean }, +): Promise { + const { written, failed } = await shardClone({ + root, + backend: shardCtx.backend, + online: shardCtx.online, + leaves, + }); + + // Main-thread fallback for any leaf a worker could not write. + for (const posixPath of failed) { + const url = leaves.get(posixPath); + if (!url) continue; + const handle = await repo.find(url); + await writeFileAtomic( + path.join(root, fromPosix(posixPath)), + contentToBytes(handle.doc().content), + ); + } + + // Remove anything on disk the tree no longer references. + const ig = await loadIgnore(root); + const present = await walkDir(root, ig); + let removed = 0; + for (const posixPath of present.keys()) { + if (leaves.has(posixPath)) continue; + try { + await fs.unlink(path.join(root, fromPosix(posixPath))); + removed++; + } catch { + // already gone + } + await pruneEmptyDirs(root, path.dirname(fromPosix(posixPath))); + } + dlog( + "materialize (shard) written=%d main-fallback=%d removed=%d", + written.size, + failed.length, + removed, + ); +} + const fromPosix = (p: string) => p.split("/").join(path.sep); async function pruneEmptyDirs(root: string, relDir: string): Promise { diff --git a/src/workers/shard-clone-worker.ts b/src/workers/shard-clone-worker.ts new file mode 100644 index 0000000..d6c574c --- /dev/null +++ b/src/workers/shard-clone-worker.ts @@ -0,0 +1,58 @@ +/** + * Shared-nothing shard-clone worker (PUSHWORK_PARALLEL_INGEST=shard). + * + * Mirror of the ingest worker for the pull side. Each worker owns a full Repo + * (own Wasm, own storage view, own WebSocket online) and handles its shard of + * file leaves end-to-end: `repo.find` the document (download online / load + * from shared storage offline), read its content, and write the file to disk + * itself. It reports which relative paths it wrote; the main thread never + * materializes these documents. + * + * Runs as compiled CommonJS (dist/workers/...), spawned by ../ingest-pool. + */ +import * as path from "path"; +import { parentPort, workerData } from "worker_threads"; +import type { AutomergeUrl } from "@automerge/automerge-repo"; +import { storageDir, type Backend } from "../config.js"; +import { writeFileAtomic } from "../fs-tree.js"; +import { openRepo } from "../repo.js"; +import { findFileEntry } from "../shapes/index.js"; + +export type ShardCloneData = { + root: string; + backend: Backend; + online: boolean; + leaves: [relPath: string, url: AutomergeUrl][]; +}; + +export type ShardCloneReport = { written: string[]; failed: string[] }; + +const fromPosix = (p: string) => p.split("/").join(path.sep); + +async function run(): Promise { + const { root, backend, online, leaves } = workerData as ShardCloneData; + const repo = await openRepo(backend, storageDir(root), { offline: !online }); + + const written: string[] = []; + const failed: string[] = []; + try { + for (const [relPath, url] of leaves) { + try { + const { bytes } = await findFileEntry(repo, url); + await writeFileAtomic(path.join(root, fromPosix(relPath)), bytes); + written.push(relPath); + } catch { + failed.push(relPath); + } + } + } finally { + await repo.shutdown(); + } + + parentPort!.postMessage({ written, failed } satisfies ShardCloneReport); +} + +run().catch((error) => { + console.error("[pushwork:shard-clone] fatal:", error); + process.exit(1); +}); diff --git a/src/workers/shard-ingest-worker.ts b/src/workers/shard-ingest-worker.ts new file mode 100644 index 0000000..d55d8e0 --- /dev/null +++ b/src/workers/shard-ingest-worker.ts @@ -0,0 +1,97 @@ +/** + * Shared-nothing shard-ingest worker (PUSHWORK_PARALLEL_INGEST=shard). + * + * Each worker owns a full Repo — its own Wasm instance, its own + * NodeFSStorageAdapter writing the same `.pushwork/storage` directory + * (document keys are disjoint and writes are atomic-rename, so concurrent + * workers are safe), and, when online, its own WebSocket to the sync server. + * + * The worker creates its shard of new file documents end-to-end (build doc → + * persist to shared storage → upload), then reports only `{relPath, url, + * heads}` per file. The main thread never materializes these documents; it + * stitches the reported URLs into the folder document afterwards ("parallel + * leaves, serial directories"), so children reach the server before parents. + * + * Runs as compiled CommonJS (dist/workers/...), spawned by ../ingest-pool. + */ +import { parentPort, workerData } from "worker_threads"; +import type { AutomergeUrl, DocHandle } from "@automerge/automerge-repo"; +import { storageDir, type Backend } from "../config.js"; +import { openRepo, waitForSync } from "../repo.js"; +import { makeFileEntry, type UnixFileEntry } from "../shapes/index.js"; + +export type ShardIngestTask = { + relPath: string; + bytes: Uint8Array; + isArtifact: boolean; +}; + +export type ShardIngestData = { + root: string; + backend: Backend; + online: boolean; + tasks: ShardIngestTask[]; +}; + +export type ShardFileResult = + | { + relPath: string; + ok: true; + url: AutomergeUrl; + heads: string[]; + isArtifact: boolean; + } + | { relPath: string; ok: false; error: string }; + +export type ShardIngestReport = { results: ShardFileResult[] }; + +async function run(): Promise { + const { root, backend, online, tasks } = workerData as ShardIngestData; + const repo = await openRepo(backend, storageDir(root), { offline: !online }); + + const results: ShardFileResult[] = []; + let lastHandle: DocHandle | undefined; + try { + for (const task of tasks) { + try { + const entry = makeFileEntry(task.relPath, task.bytes, task.isArtifact); + const handle = repo.create(entry); + lastHandle = handle; + results.push({ + relPath: task.relPath, + ok: true, + url: handle.url, + heads: handle.heads() ?? [], + isArtifact: task.isArtifact, + }); + } catch (error) { + results.push({ + relPath: task.relPath, + ok: false, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + // Online: local head-stability is NOT a server-delivery guarantee, but + // it gives the repo a sync window (minMs floor) to connect and upload + // this shard's documents; the shutdown quiesce then finalizes delivery. + // Without this window the worker would shut down before its socket even + // connected, leaving its file docs undelivered (clone → "unavailable"). + if (online && lastHandle) { + await waitForSync(lastHandle, { minMs: 3000, idleMs: 1500, maxMs: 15000 }); + } + } finally { + // Flush storage (and, online, quiesce-deliver to the server) before + // reporting. The pool terminates the worker on the report message, so + // nothing of value may remain after this resolves. + await repo.shutdown(); + } + + parentPort!.postMessage({ results } satisfies ShardIngestReport); +} + +run().catch((error) => { + console.error("[pushwork:shard-ingest] fatal:", error); + process.exit(1); +}); diff --git a/test/integration/fixtures/shard-roundtrip.ts b/test/integration/fixtures/shard-roundtrip.ts new file mode 100644 index 0000000..6bab9d0 --- /dev/null +++ b/test/integration/fixtures/shard-roundtrip.ts @@ -0,0 +1,86 @@ +/** + * Offline shard round-trip used by shard.test.ts. + * + * Generates a multi-file tree (text + binary + nested dirs), `init`s it + * offline, copies the automerge storage into a fresh dir, `clone`s offline + * from that storage, and byte-compares source vs clone. Exercises both the + * shard-ingest and shard-clone worker pools when PUSHWORK_PARALLEL_INGEST is + * set. Exits non-zero (with a printed reason) on any mismatch. + * + * Run as a plain-node subprocess against the built dist (Node strips the types): + * the worker scripts must resolve as compiled CommonJS and the Subduction Wasm + * must load as a single consistent instance — the same reason the bench runs + * compiled rather than under tsx. + */ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { clone, init } from "../../../dist/index.js"; + +const N = Number(process.env.SHARD_FIXTURE_FILES) || 40; + +function gen(root: string, n: number): void { + for (let i = 0; i < n; i++) { + const dir = path.join(root, `d${Math.floor(i / 8)}`, `sub${i % 3}`); + fs.mkdirSync(dir, { recursive: true }); + const isBinary = i % 5 === 0; + const body: Buffer | string = isBinary + ? Buffer.from([0, 1, 2, 3, i & 0xff, 254, 255]) + : `file ${i} ` + "lorem ipsum dolor sit amet ".repeat(25) + "\n"; + fs.writeFileSync(path.join(dir, `f${i}.${isBinary ? "bin" : "txt"}`), body); + } +} + +function listFiles(root: string): Map { + const out = new Map(); + (function walk(dir: string, rel: string): void { + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + if (e.name === ".pushwork") continue; + const full = path.join(dir, e.name); + const r = rel ? rel + "/" + e.name : e.name; + if (e.isDirectory()) walk(full, r); + else out.set(r, fs.readFileSync(full)); + } + })(root, ""); + return out; +} + +async function main(): Promise { + const base = fs.mkdtempSync(path.join(os.tmpdir(), "shard-roundtrip-")); + const src = path.join(base, "src"); + const dst = path.join(base, "dst"); + fs.mkdirSync(src); + fs.mkdirSync(dst); + gen(src, N); + + const url = await init({ dir: src, backend: "subduction", shape: "vfs", online: false }); + fs.cpSync( + path.join(src, ".pushwork", "storage"), + path.join(dst, ".pushwork", "storage"), + { recursive: true }, + ); + await clone({ url, dir: dst, backend: "subduction", shape: "vfs", online: false }); + + const a = listFiles(src); + const b = listFiles(dst); + const problems: string[] = []; + if (a.size !== N) problems.push(`source has ${a.size} files, expected ${N}`); + if (a.size !== b.size) problems.push(`clone has ${b.size} files, expected ${a.size}`); + for (const [rel, bytesA] of a) { + const bytesB = b.get(rel); + if (!bytesB) problems.push(`missing in clone: ${rel}`); + else if (!bytesA.equals(bytesB)) problems.push(`content differs: ${rel}`); + } + + fs.rmSync(base, { recursive: true, force: true }); + if (problems.length > 0) { + console.error("shard round-trip FAILED:\n " + problems.join("\n ")); + process.exit(1); + } + console.log(`shard round-trip OK: ${a.size} files byte-identical`); +} + +main().catch((e) => { + console.error(e); + process.exit(2); +}); diff --git a/test/integration/shard.test.ts b/test/integration/shard.test.ts new file mode 100644 index 0000000..78b6f7a --- /dev/null +++ b/test/integration/shard.test.ts @@ -0,0 +1,43 @@ +/** + * Correctness guard for the shared-nothing shard worker pools + * (PUSHWORK_PARALLEL_INGEST=shard): an offline init → copy-storage → clone + * round-trip must be byte-identical to the source, exercising both the + * shard-ingest and shard-clone workers. + * + * The round-trip runs in a plain-node subprocess (fixtures/shard-roundtrip.ts, + * type-stripped by Node 24) against the built dist, because the worker scripts + * must be compiled CommonJS and the Subduction Wasm needs a single consistent + * module instance — the same reason the bench runs compiled rather than tsx. + */ +import * as path from "path"; +import { execFile } from "child_process"; +import { promisify } from "util"; + +const execFileP = promisify(execFile); +const REPO_ROOT = path.join(__dirname, "..", ".."); +const FIXTURE = path.join(__dirname, "fixtures", "shard-roundtrip.ts"); + +beforeAll(async () => { + await execFileP("pnpm", ["build"], { cwd: REPO_ROOT, timeout: 120_000 }); +}, 120_000); + +describe("shard parallel ingest/clone", () => { + it( + "round-trips a multi-file tree byte-identically (offline)", + async () => { + // Throws if the fixture exits non-zero (it asserts byte-identity and + // the expected file count internally). + const { stdout } = await execFileP( + "node", + ["--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", FIXTURE], + { + env: { ...process.env, PUSHWORK_PARALLEL_INGEST: "shard" }, + timeout: 60_000, + maxBuffer: 16 * 1024 * 1024, + }, + ); + expect(stdout).toContain("byte-identical"); + }, + 90_000, + ); +});