diff --git a/apps/browser-demos/lib/init/vfs-utils.ts b/apps/browser-demos/lib/init/vfs-utils.ts index b7f78e0b9..8adb13779 100644 --- a/apps/browser-demos/lib/init/vfs-utils.ts +++ b/apps/browser-demos/lib/init/vfs-utils.ts @@ -5,39 +5,10 @@ * and are used by demo build scripts that construct VFS images. */ import type { MemoryFileSystem } from "../../../../host/src/vfs/memory-fs"; - -const encoder = new TextEncoder(); - -/** - * Write a text file to the VFS. Opens with O_WRONLY|O_CREAT|O_TRUNC, - * writes the encoded content, and closes the fd. - */ -export function writeVfsFile( - fs: MemoryFileSystem, - path: string, - content: string, - mode = 0o644, -): void { - const data = encoder.encode(content); - const fd = fs.open(path, 0o1101, mode); // O_WRONLY | O_CREAT | O_TRUNC - fs.write(fd, data, 0, data.length); - fs.close(fd); -} - -/** - * Write a binary file to the VFS. Opens with O_WRONLY|O_CREAT|O_TRUNC, - * writes the raw bytes, and closes the fd. - */ -export function writeVfsBinary( - fs: MemoryFileSystem, - path: string, - data: Uint8Array, - mode = 0o755, -): void { - const fd = fs.open(path, 0o1101, mode); // O_WRONLY | O_CREAT | O_TRUNC - fs.write(fd, data, 0, data.length); - fs.close(fd); -} +export { + writeVfsBinary, + writeVfsFile, +} from "../../../../host/src/vfs/image-helpers"; /** * Create a directory, ignoring EEXIST errors. @@ -79,4 +50,3 @@ export function ensureDirRecursive( ensureDir(fs, current, mode); } } - diff --git a/docs/architecture.md b/docs/architecture.md index 67e369071..949923027 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -263,7 +263,26 @@ five-export fallback, while ABI 18 and later require role claims and reject stale call-graph artifacts. ABI 19 combines that contract with live main's concurrent/nested pthread continuation-buffer repair. Dlopen replay records both the parent's memory base and exact table base, including null gaps -left by failed loads. The supported +left by failed loads. TLS-bearing side modules additionally record their live, +positive `__tls_base`. A child restores the pointer-width-correct mutable +global without calling `__wasm_init_tls`, because copied memory already holds +the parent's live TLS bytes and reinitialization would reset C++ unwinder state +and application `thread_local` values. C++ exceptions and longjmp use one +canonical pointer-width tag identity across the main image and all side +modules; a main-exported tag wins over the host-created fallback. + +The dlopen replay list and its atomic pthread-fork lock live in a transient, +host-private control record. The same host build writes and reads that record +during one process lifetime; guest code and persisted artifacts never +interpret it. Changing that record's size is therefore not a guest ABI change, +while the public ABI snapshot/classifier remains authoritative. + +Pthread workers have separate Wasm instances, tables, and exception tags, none +of which can be structured-cloned from the process worker. `dlopen()` from a +pthread therefore fails normally with `dlerror()`. If the process has loaded a +side module, `fork()` from a pthread returns `ENOTSUP`; an atomic process lock +excludes a racing main-worker dlopen across the pthread's archive check, +unwind, memory copy, and parent rewind. The supported direct-main-to-side boundary and the remaining opaque cross-side callback limitation are specified in [fork-instrumentation.md](fork-instrumentation.md#fork-from-a-dlopened-side-module). @@ -596,6 +615,12 @@ const restored = MemoryFileSystem.fromImage(image, { maxByteLength: 1024 * 1024 The image must also have been built with a large enough filesystem maximum, for example `MemoryFileSystem.create(sab, 1024 * 1024 * 1024)`. `fromImage(..., { maxByteLength })` only controls the restored buffer's runtime growth ceiling; `statfs`/`df` and allocation remain capped by the image superblock maximum. +A consumer that must stage files larger than the image's recorded allocation +ceiling first calls `rebaseToNewFileSystem(requiredMaxBytes)`. Shared image +helpers never treat a partial file as complete: `writeVfsBinary` advances over +positive short writes and throws on zero/negative progress or an underlying +filesystem error, while still closing the descriptor. + Kandelo browser UI presets use this approach. Each image builder pre-populates a VFS with runtime files, directory structure, configs, and symlinks, then saves it as a `.vfs.zst` file (zstd-compressed; `saveImage()` compresses on write). At runtime, the UI fetches the file and `MemoryFileSystem.fromImage` decompresses transparently - restoring the image replaces thousands of individual file writes with a single buffer copy. The empty regions of the SharedFS allocator compress to almost nothing, so a 32 MB filesystem with a few MB of real content typically ships as a 1-3 MB download. There are two consumption patterns for VFS images, depending on whether the demo wants the kernel worker to fully own the filesystem: diff --git a/docs/binary-releases.md b/docs/binary-releases.md index aa6f26c13..e5b47b010 100644 --- a/docs/binary-releases.md +++ b/docs/binary-releases.md @@ -337,9 +337,11 @@ For each declared arch in the package's `arches = [...]` (default - `abi_versions` must contain the in-tree `ABI_VERSION`. - `cache_key_sha` must match the resolver's locally-computed cache-key sha (catches recipe drift). -6. Places `binaries/programs//.wasm` symlinks pointing - into the cache, so browser/Node demos can load by relative path - without re-fetching. +6. Places each program output under `binaries/programs//` using the + manifest's output layout, and places declared non-Wasm runtime files under + `binaries/programs///`. Both are symlinks into the + validated cache, so browser/Node image builders load the same bytes without + re-fetching. Local builds use the identical layout under `local-binaries/`. On any verification failure, the resolver logs a warning and falls through to a source build (the package's build script). This diff --git a/docs/fork-instrumentation.md b/docs/fork-instrumentation.md index cfb1137bb..674f0adfc 100644 --- a/docs/fork-instrumentation.md +++ b/docs/fork-instrumentation.md @@ -229,9 +229,23 @@ function pointers passed through main-module memory or the shared table cannot currently be attributed to their originating module; using such a pointer to create a side A -> side B -> fork path is unsupported and is not yet guaranteed to fail before control-flow corruption. A future module-activation protocol is -required to close that residual. Fork from a pthread into a dlopened side -module is also unsupported; pthread workers do not install the side-module -coordinator. +required to close that residual. + +Pthread workers do not own the process worker's side-module instances, table, +or exception-tag identities. `dlopen()` from a pthread consequently returns +NULL with a precise `dlerror()`. Once the process main worker has published a +dlopen archive entry, `fork()` from a pthread returns `ENOTSUP` without +creating a child. A host-private atomic lock prevents main-worker dlopen from +racing the pthread's archive check and is held through unwind, SYS_FORK/memory +copy, and parent rewind; the child clears its copied lock before replay. Fork +from a pthread remains supported while that process-wide archive is empty. + +For TLS-bearing side modules, each archive entry also preserves the live +positive `__tls_base`. Replay restores only that mutable global using the +process pointer type. It does not call `__wasm_init_tls`: the child memory copy +already contains live TLS, and reinitialization would overwrite C++ landing-pad +and application `thread_local` state. TLS-relative exports relocate from that +base, while `__tls_size` and `__tls_align` remain scalar constants. Every participating module still uses the fixed 16 KiB save-buffer limit described below. A dynamically allocated side buffer avoids overlap with the diff --git a/docs/package-management.md b/docs/package-management.md index 58eecc441..6a6e8a145 100644 --- a/docs/package-management.md +++ b/docs/package-management.md @@ -150,8 +150,8 @@ The split is load-bearing post the Required fields: ```toml -name = "zlib" # logical library name -version = "1.3.1" # upstream version +name = "zlib" # logical library name; one safe path component +version = "1.3.1" # upstream version; one safe path component depends_on = [] # ["zlib@1.3.1", ...] — exact versions, no ranges [source] @@ -176,8 +176,51 @@ script_path = "packages/registry/zlib/build-zlib.sh" libs = ["lib/libz.a"] # must exist post-build headers = ["include/zlib.h", "include/zconf.h"] pkgconfig = ["lib/pkgconfig/zlib.pc"] +files = ["share/runtime-data.bin"] # other runtime data ``` +Program packages use `[[outputs]]` for executable/side-module artifacts. A +non-Wasm file required at runtime is declared separately so it remains part of +the same reproducible archive and cache key: + +```toml +[[outputs]] +name = "php" +wasm = "php.wasm" + +[[runtime_files]] +artifact = "icu.dat" # relative to the package cache/archive +guest_path = "/usr/lib/php/icu.dat" # installation path in a VFS image +mode = 420 # optional decimal TOML; default 0644 +``` + +`[[runtime_files]]` is program-only. Artifact paths are normalized portable +relative paths; guest paths are normalized absolute POSIX paths; files, +ancestor paths, and resolver-mirror destinations may not collide. The resolver +requires regular non-symlink runtime files after fresh builds, cache hits, and +remote fetches. It mirrors them at +`{local-,}binaries/programs///` independently of the +number of `[[outputs]]` entries. Missing fetched files make the archive stale +and trigger source fallback (or a hard failure in fetch-only mode). + +Repo-side VFS/test builders query the authoritative path and mode with +`xtask build-deps runtime-file-metadata `; they must not +scan library caches or invent environment-only guest paths. Published VFS +images contain the installed bytes already, so this query is a build-tool +contract rather than a runtime host API. The structured metadata also lists +the package's complete mirror closure (every `[[outputs]]` artifact plus every +`[[runtime_files]]` file). Repo-side consumers resolve that set from one +complete provenance root: a partial local override may fall back wholesale to +a complete fetched package, but local, fetched, and installed-package tiers +are never combined. If artifacts exist but no tier has the complete accepted +closure, resolution fails loudly. + +Top-level keys are closed-schema: misspellings such as `[[runtime_file]]` +(singular) are rejected instead of silently dropping a runtime dependency. +Package names, versions, dependency names, and exact dependency-version tokens +must each be safe single filesystem components; `/`, `\`, NUL, `.` and `..` +spellings are rejected before cache, archive, or registry path construction. + `package.toml` **must NOT** carry `revision`, `[binary]`, `[build].repo_url`, or `[build].commit`. Those moved to `build.toml` during the binary-resolution-via-index-ledger migration; @@ -353,7 +396,7 @@ that doesn't respect them cannot be cached safely. | Variable | Meaning | |---|---| -| `WASM_POSIX_DEP_OUT_DIR` | Temp dir the script must install into. Layout matches `outputs.libs` / `outputs.headers` / `outputs.pkgconfig` relative paths. | +| `WASM_POSIX_DEP_OUT_DIR` | Temp dir the script must install into. Layout matches `outputs.libs` / `outputs.headers` / `outputs.pkgconfig` / `outputs.files` relative paths. | | `WASM_POSIX_DEP_NAME` | `name` from package.toml. | | `WASM_POSIX_DEP_VERSION` | `version` from package.toml. | | `WASM_POSIX_DEP_REVISION` | Effective package revision after `build.toml` is overlaid. | @@ -371,7 +414,7 @@ from source; it does not restrict normal SDK users compiling against a published sysroot/libc++ artifact. After the script exits 0, the resolver verifies every path in -`outputs.{libs,headers,pkgconfig}` exists under `$WASM_POSIX_DEP_OUT_DIR`. +`outputs.{libs,headers,pkgconfig,files}` exists under `$WASM_POSIX_DEP_OUT_DIR`. A missing output fails the build (and the temp dir is cleaned up, so a retry starts clean). diff --git a/docs/posix-status.md b/docs/posix-status.md index 720684f3b..0c67d9d83 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -615,7 +615,10 @@ These PHP needs are well-handled by the current kernel: - Memory: anonymous mmap, munmap, brk - Multi-process: fork (kernel syscall), exec (host-initiated), waitpid (kernel syscall) - Networking: AF_INET TCP (connect, bind, listen, accept, send, recv), getaddrinfo -- Dynamic linking: dlopen, dlsym, dlclose, dlerror (Wasm dylink) +- Dynamic linking: dlopen, dlsym, dlclose, dlerror (Wasm dylink on the process + worker). Pthread workers cannot share the process's Wasm table/tag graph, so + pthread `dlopen` fails and pthread `fork` after a process dlopen returns + `ENOTSUP`. - POSIX timers: partial `SIGEV_SIGNAL`/`SIGEV_NONE` timer_create, timer_settime, timer_gettime, timer_delete. The PHP package imports a cooperative Wasm host hook because native `SIGEV_THREAD_ID` delivery is unavailable; that package @@ -639,7 +642,10 @@ These require features fundamentally unavailable in the Wasm architecture: - **Wasm FP exceptions (110 math tests):** WebAssembly has no floating-point exception flags (`fenv.h`). All `fe*` math tests fail. `long double` variants pass because they use software fp128. - **No pthread_cancel:** Wasm has no async cancellation mechanism or cancel-point assembly. `pthread_create` works; `pthread_cancel` does not. -- **No dlopen/TLS:** `tls_get_new-dtv_dso` requires loading a shared library with TLS at runtime. +- **No musl DTV expansion for arbitrary DSOs:** `tls_get_new-dtv_dso` requires + native-style per-thread dynamic TLS-vector growth. Kandelo can replay the + fixed TLS reservation of a Wasm side module across process `fork`, but that + does not implement musl's general pthread DTV contract. ### Linker Requirements for Signal Handlers diff --git a/host/src/binary-resolver.ts b/host/src/binary-resolver.ts index 8eebfc5e0..f309f24f3 100644 --- a/host/src/binary-resolver.ts +++ b/host/src/binary-resolver.ts @@ -1,5 +1,6 @@ /** - * Resolve a binary (wasm, zip bundle, vfs image) from the repo's + * Resolve a packaged artifact (Wasm executable/side module, VFS image, + * archive, or declared runtime data file) from the repo's * `local-binaries/` or `binaries/` tree. * * Priority: @@ -81,13 +82,14 @@ function packageRoot(): string { } /** - * Resolve a binary relative to the binaries tree. + * Resolve an artifact relative to the binaries tree. * * Example paths: * `kernel.wasm` * `userspace.wasm` * `programs/vim.zip` (implicit wasm32 — see below) * `programs/git/git.wasm` (implicit wasm32) + * `programs/php/icu.dat` (implicit wasm32 runtime file) * `programs/wasm64/mariadb-vfs.vfs.zst` (explicit arch) * * Per-arch layout: `binaries/programs/` and `local-binaries/programs/` @@ -109,20 +111,64 @@ function applyDefaultArch(relPath: string): string { return `programs/wasm32/${tail}`; } -function packagedBinaryCandidates(relPath: string): string[] { - const root = packageRoot(); +function packagedBinaryCandidates( + relPath: string, + root = join(packageRoot(), "wasm"), +): string[] { const adjusted = applyDefaultArch(relPath); - const candidates = [join(root, "wasm", adjusted)]; + const candidates = [join(root, adjusted)]; if (relPath === "kernel.wasm") { - candidates.push(join(root, "wasm", "kandelo-kernel.wasm")); + candidates.push(join(root, "kandelo-kernel.wasm")); } else if (relPath === "userspace.wasm") { - candidates.push(join(root, "wasm", "wasm_posix_userspace.wasm")); + candidates.push(join(root, "wasm_posix_userspace.wasm")); } else if (relPath === "rootfs.vfs") { - candidates.push(join(root, "wasm", "rootfs.vfs")); + candidates.push(join(root, "rootfs.vfs")); } return candidates; } +interface BinaryCandidateTier { + label: string; + root: string; + candidatesFor(relPath: string): string[]; +} + +/** + * Ordered provenance roots used by both single-artifact and package-closure + * resolution. Keeping the grouping explicit lets a closure fall back as a + * unit without ever combining local, fetched, and installed-package bytes. + */ +function binaryCandidateTiers(): BinaryCandidateTier[] { + const tiers: BinaryCandidateTier[] = []; + try { + const repo = findRepoRoot(); + for (const [label, root] of [ + ["local-binaries", join(repo, "local-binaries")], + ["binaries", join(repo, "binaries")], + ] as const) { + tiers.push({ + label, + root, + candidatesFor(relPath: string): string[] { + return [join(root, applyDefaultArch(relPath))]; + }, + }); + } + } catch { + // Installed npm consumers do not carry a source repo root. + } + + const root = join(packageRoot(), "wasm"); + tiers.push({ + label: "installed package", + root, + candidatesFor(relPath: string): string[] { + return packagedBinaryCandidates(relPath, root); + }, + }); + return tiers; +} + let cachedForkInstrumentationDisabledOutputs: Set | null = null; interface ProgramOutputPolicy { @@ -282,22 +328,11 @@ export function resolveBinary(relPath: string): string { const adjusted = applyDefaultArch(relPath); const checked: string[] = []; const candidates: string[] = []; - try { - const repo = findRepoRoot(); - const local = join(repo, "local-binaries", adjusted); - checked.push(local); - candidates.push(local); - const fetched = join(repo, "binaries", adjusted); - checked.push(fetched); - candidates.push(fetched); - } catch { - // Installed npm consumers do not have a source repo root. Fall - // through to packaged assets below. - } - const packaged = packagedBinaryCandidates(relPath); - for (const candidate of packaged) { - checked.push(candidate); - candidates.push(candidate); + for (const tier of binaryCandidateTiers()) { + for (const candidate of tier.candidatesFor(relPath)) { + checked.push(candidate); + candidates.push(candidate); + } } const candidate = chooseBinaryCandidate(candidates, relPath); if (candidate) return candidate; @@ -320,6 +355,53 @@ export function tryResolveBinary(relPath: string): string | null { } } +/** + * Resolve a related artifact set from one complete provenance tier. + * + * A partial or policy-invalid local tier is skipped as a whole when a later + * tier is complete. If artifacts exist across the candidate roots but no + * single root contains an accepted complete set, this throws instead of + * silently composing a package from unrelated builds. It returns `null` only + * when none of the requested artifacts exists in any tier. + * + * Returned paths preserve `relPaths` order and are guaranteed to share the + * same local, fetched, or installed-package root. + */ +export function tryResolveBinarySet(relPaths: readonly string[]): string[] | null { + if (relPaths.length === 0) return []; + + let anyExisting = false; + const incomplete: string[] = []; + for (const tier of binaryCandidateTiers()) { + const selected: string[] = []; + const unavailable: string[] = []; + for (const relPath of relPaths) { + const candidates = tier.candidatesFor(relPath); + const existing = candidates.filter((candidate) => existsSync(candidate)); + anyExisting ||= existing.length > 0; + const candidate = chooseBinaryCandidate(candidates, relPath); + if (candidate) { + selected.push(candidate); + } else if (existing.length > 0) { + unavailable.push(`${relPath} (rejected by artifact policy)`); + } else { + unavailable.push(`${relPath} (missing)`); + } + } + if (unavailable.length === 0) return selected; + incomplete.push( + ` ${tier.label} (${tier.root}): ${unavailable.join(", ")}`, + ); + } + + if (!anyExisting) return null; + throw new Error( + "Package artifact closure is incomplete: no single provenance tier " + + "contains every accepted artifact, and tiers will not be mixed.\n" + + incomplete.join("\n"), + ); +} + /** Returns the absolute path of binaries/ whether or not it exists. */ export function binariesDir(): string { return join(findRepoRoot(), "binaries"); diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index cea749224..0f4fba8b3 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -1530,6 +1530,7 @@ async function handleClone( programBytes: processInfo.programBytes, programModule: threadModule, memory, + processChannelOffset: processInfo.channelOffset, channelOffset: alloc.channelOffset, fnPtr, argPtr, diff --git a/host/src/dylink.ts b/host/src/dylink.ts index 5904ed89e..26a99a4b0 100644 --- a/host/src/dylink.ts +++ b/host/src/dylink.ts @@ -223,6 +223,44 @@ export function parseDylinkSection(wasmBytes: Uint8Array): DylinkMetadata | null return metadata; } +/** + * Return function exports whose indices refer to module-defined functions. + * + * WebAssembly.Module.exports() exposes only names and kinds. Dynamic-linker + * self-import handling also needs the function index: an imported function can + * be re-exported under the same name, but that is not a local definition and + * must not receive a trampoline back to itself. + * + * `module` has already validated the binary before this helper is called, so + * section bounds and LEB encodings are known to be structurally valid. + */ +function readDefinedFunctionExports( + wasmBytes: Uint8Array, + importedFunctionCount: number, +): Set { + const result = new Set(); + const offset = { value: 8 }; + while (offset.value < wasmBytes.length) { + const sectionId = wasmBytes[offset.value++]; + const sectionSize = readVarUint(wasmBytes, offset); + const sectionEnd = offset.value + sectionSize; + if (sectionId !== 7) { + offset.value = sectionEnd; + continue; + } + + const exportCount = readVarUint(wasmBytes, offset); + for (let i = 0; i < exportCount; i++) { + const name = readString(wasmBytes, offset); + const kind = wasmBytes[offset.value++]; + const index = readVarUint(wasmBytes, offset); + if (kind === 0 && index >= importedFunctionCount) result.add(name); + } + break; + } + return result; +} + /** Align a value up to the given alignment (must be power of 2). */ function alignUp(value: number, align: number): number { return (value + align - 1) & ~(align - 1); @@ -246,6 +284,8 @@ export interface LoadedSharedLibrary { name: string; /** Fork save buffer for an instrumented side module importing env.fork. */ forkBufAddr?: number; + /** Thread-local-storage base captured from the parent instance. */ + tlsBase?: number; /** Whether this module can originate a coordinated env.fork unwind. */ forkCapable: boolean; /** Function/GOT.func imports used for conservative cross-side isolation. */ @@ -300,6 +340,11 @@ export interface DylinkReplayOptions { tableBase: number; /** Exact side-module save buffer copied from the fork parent. */ forkBufAddr?: number; + /** Exact mutable `__tls_base` value from the fork parent. The child memory + * already contains the parent's live TLS bytes, so replay restores only + * this instance-local global and deliberately does not call + * `__wasm_init_tls`, which would reset those bytes to the initial image. */ + tlsBase?: number; } /** @@ -330,6 +375,12 @@ export interface LoadSharedLibraryOptions { * is retained on this options object for subsequent loads. */ longjmpTag?: WebAssembly.Tag; + /** + * Process-owned C++ exception tag shared by every C++ side module. C++ + * exceptions crossing side-module calls require tag identity as well as a + * matching payload type, so this must not be allocated per dlopen. + */ + cppExceptionTag?: WebAssembly.Tag; /** Process pointer width, which also determines the __c_longjmp payload. */ ptrWidth?: 4 | 8; /** Immutable symbol names exported by the main module. */ @@ -363,6 +414,17 @@ export function createLongjmpTag(ptrWidth: 4 | 8): WebAssembly.Tag | undefined { : undefined; } +/** Create the process-owned C++ exception tag for the target pointer width. */ +export function createCppExceptionTag(ptrWidth: 4 | 8): WebAssembly.Tag | undefined { + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new TypeError(`invalid process pointer width ${String(ptrWidth)}`); + } + const Tag = tagConstructor(); + return Tag + ? new Tag({ parameters: [ptrWidth === 8 ? "i64" : "i32"] }) + : undefined; +} + /** Reject lookalike values before handing an exception-tag import to Wasm. */ export function requireLongjmpTag(value: unknown, context: string): WebAssembly.Tag { const Tag = tagConstructor(); @@ -375,6 +437,18 @@ export function requireLongjmpTag(value: unknown, context: string): WebAssembly. return value; } +/** Reject lookalike values before handing the C++ tag import to Wasm. */ +export function requireCppExceptionTag(value: unknown, context: string): WebAssembly.Tag { + const Tag = tagConstructor(); + if (!Tag) { + throw new Error(`${context}: this WebAssembly runtime does not support exception tags`); + } + if (!(value instanceof Tag)) { + throw new TypeError(`${context}: __cpp_exception must be an actual WebAssembly.Tag`); + } + return value; +} + function validateLongjmpConfiguration(options: LoadSharedLibraryOptions): void { const ptrWidth = options.ptrWidth ?? 4; if (ptrWidth !== 4 && ptrWidth !== 8) { @@ -394,6 +468,17 @@ function resolveLongjmpTag(options: LoadSharedLibraryOptions): WebAssembly.Tag { return options.longjmpTag; } +function resolveCppExceptionTag(options: LoadSharedLibraryOptions): WebAssembly.Tag { + validateLongjmpConfiguration(options); + if (options.cppExceptionTag !== undefined) { + return requireCppExceptionTag(options.cppExceptionTag, "dynamic linker"); + } + const ptrWidth = options.ptrWidth ?? 4; + const tag = createCppExceptionTag(ptrWidth); + options.cppExceptionTag = requireCppExceptionTag(tag, "dynamic linker"); + return options.cppExceptionTag; +} + const SIDE_DYNAMIC_LOOKUP_IMPORTS = new Set([ "__wasm_dlopen", "__wasm_dlsym", @@ -500,6 +585,26 @@ function instantiateSharedLibrary( const functionExports = new Set( moduleExports.filter((exp) => exp.kind === "function").map((exp) => exp.name), ); + const importedFunctionCount = moduleImports.filter((imp) => imp.kind === "function").length; + const definedFunctionExports = readDefinedFunctionExports( + wasmBytes, + importedFunctionCount, + ); + // wasm-ld can make an interposable C++ definition both an env import and a + // module export. The main process still wins when it supplies the symbol; + // otherwise route only this genuine self-definition back to the module. + // Do not manufacture trampolines for arbitrary unresolved imports: those + // remain instantiation errors instead of turning an ABI gap into a delayed + // failure on a possibly-unexecuted path. + const selfFunctionImports = new Set( + moduleImports + .filter((imp) => + imp.module === "env" + && imp.kind === "function" + && definedFunctionExports.has(imp.name) + ) + .map((imp) => imp.name), + ); const importsDynamicLookup = moduleImports.some((imp) => imp.module === "env" && imp.kind === "function" @@ -511,6 +616,14 @@ function instantiateSharedLibrary( && (imp.kind as string) === "tag" ); const longjmpTag = importsLongjmpTag ? resolveLongjmpTag(options) : undefined; + const importsCppExceptionTag = moduleImports.some((imp) => + imp.module === "env" + && imp.name === "__cpp_exception" + && (imp.kind as string) === "tag" + ); + const cppExceptionTag = importsCppExceptionTag + ? resolveCppExceptionTag(options) + : undefined; if (presentForkExports.length > 0 && !hasCompleteForkInstrumentation) { const missing = SIDE_MODULE_FORK_EXPORTS.filter((exportName) => @@ -763,19 +876,30 @@ function instantiateSharedLibrary( case "__table_base": return tableBaseGlobal; case "__stack_pointer": return options.stackPointer; case "__c_longjmp": return longjmpTag; + case "__cpp_exception": return cppExceptionTag; case "fork": if (importsFork) return sideModuleForkImport; break; } const sym = options.globalSymbols.get(prop); if (sym !== undefined) return sym; + if (selfFunctionImports.has(prop)) { + return (...args: unknown[]) => { + const fn = instance?.exports[prop]; + if (typeof fn !== "function") { + throw new Error(`${name}: self import env.${prop} is unavailable`); + } + return (fn as Function)(...args); + }; + } return undefined; }, has(_target, prop: string) { if (["memory", "__indirect_function_table", "__memory_base", - "__table_base", "__stack_pointer", "__c_longjmp"].includes(prop)) return true; + "__table_base", "__stack_pointer", "__c_longjmp", + "__cpp_exception"].includes(prop)) return true; if (prop === "fork" && importsFork) return true; - return options.globalSymbols.has(prop); + return options.globalSymbols.has(prop) || selfFunctionImports.has(prop); }, }), "GOT.mem": new Proxy({} as Record, { @@ -793,6 +917,103 @@ function instantiateSharedLibrary( // Instantiate synchronously after validating the side-module fork contract. instance = new WebAssembly.Instance(module, imports); + // A threaded wasm-ld side module initializes its mutable __tls_base from + // __memory_base in the start function. Fork-child memory already carries + // the parent's `__wasm_init_memory_flag == 2`, so the fresh child instance + // skips that initialization and otherwise leaves __tls_base at zero. + // Capture the live parent value and restore it during replay without + // calling __wasm_init_tls: the latter would overwrite copied, live TLS + // state (including the C++ unwinder's landing-pad context) with .tdata. + const tlsSizeExport = instance.exports.__tls_size; + const tlsSize = tlsSizeExport instanceof WebAssembly.Global + ? Number(tlsSizeExport.value) + : 0; + let tlsBase: number | undefined; + if (metadata.tlsExports.size > 0 && !(tlsSizeExport instanceof WebAssembly.Global)) { + throw new Error(`${name}: TLS exports require an exported __tls_size global`); + } + if (!Number.isSafeInteger(tlsSize) || tlsSize < 0) { + throw new Error(`${name}: invalid side-module TLS size ${String(tlsSize)}`); + } + if (tlsSize > 0) { + const tlsBaseExport = instance.exports.__tls_base; + const tlsAlignExport = instance.exports.__tls_align; + if (!(tlsBaseExport instanceof WebAssembly.Global)) { + throw new Error( + `${name}: TLS-bearing side modules must export mutable __tls_base for fork replay`, + ); + } + if (!(tlsAlignExport instanceof WebAssembly.Global)) { + throw new Error(`${name}: TLS-bearing side modules must export __tls_align`); + } + const tlsAlign = Number(tlsAlignExport.value); + if ( + !Number.isSafeInteger(tlsAlign) + || tlsAlign <= 0 + || (tlsAlign & (tlsAlign - 1)) !== 0 + ) { + throw new Error(`${name}: invalid side-module TLS alignment ${String(tlsAlign)}`); + } + + const initialRawTlsBase = tlsBaseExport.value; + const expectedTlsBaseType = (options.ptrWidth ?? 4) === 8 ? "bigint" : "number"; + if (typeof initialRawTlsBase !== expectedTlsBaseType) { + throw new Error( + `${name}: __tls_base type does not match the ${(options.ptrWidth ?? 4) * 8}-bit process pointer width`, + ); + } + try { + // A self-assignment is the only portable reflection available for + // distinguishing a mutable WebAssembly.Global from an immutable one. + tlsBaseExport.value = initialRawTlsBase; + } catch { + throw new Error(`${name}: exported __tls_base must be mutable for fork replay`); + } + if (replay) { + if (!Number.isSafeInteger(replay.tlsBase) || replay.tlsBase! <= 0) { + throw new Error(`${name}: fork replay is missing a valid side-module TLS base`); + } + try { + tlsBaseExport.value = typeof initialRawTlsBase === "bigint" + ? BigInt(replay.tlsBase!) + : replay.tlsBase!; + } catch { + throw new Error(`${name}: exported __tls_base must be mutable for fork replay`); + } + } + tlsBase = Number(tlsBaseExport.value); + // Address zero is reserved as the archive's explicit "no TLS" sentinel. + // A real TLS allocation cannot live there: the process memory allocator + // always returns a positive address and the null page must stay invalid. + if (!Number.isSafeInteger(tlsBase) || tlsBase <= 0) { + throw new Error(`${name}: invalid side-module TLS base ${String(tlsBase)}`); + } + if (tlsBase % tlsAlign !== 0) { + throw new Error( + `${name}: side-module TLS base 0x${tlsBase.toString(16)} is not aligned to ${tlsAlign}`, + ); + } + const tlsEnd = tlsBase + tlsSize; + const moduleMemoryEnd = memoryBase + metadata.memorySize; + if ( + !Number.isSafeInteger(tlsEnd) + || tlsBase < memoryBase + || tlsEnd > moduleMemoryEnd + ) { + throw new Error( + `${name}: TLS range 0x${tlsBase.toString(16)}..0x${tlsEnd.toString(16)} ` + + `escapes module reservation 0x${memoryBase.toString(16)}..0x${moduleMemoryEnd.toString(16)}`, + ); + } + if (tlsEnd > options.memory.buffer.byteLength) { + throw new Error( + `${name}: TLS range 0x${tlsBase.toString(16)}..0x${tlsEnd.toString(16)} exceeds memory`, + ); + } + } else if (replay?.tlsBase !== undefined) { + throw new Error(`${name}: fork replay supplied TLS state for a module without TLS`); + } + // Relocate exports: data address globals need memoryBase added const relocatedExports: Record = {}; for (const [exportName, exportValue] of Object.entries(instance.exports)) { @@ -801,9 +1022,23 @@ function instantiateSharedLibrary( (exportValue as any).value = (exportValue as any).value; relocatedExports[exportName] = exportValue; } catch { + // These are scalar ABI facts, not data addresses. + if (exportName === "__tls_size" || exportName === "__tls_align") { + relocatedExports[exportName] = exportValue; + continue; + } + const rawValue = exportValue.value; + const relocationBase = metadata.tlsExports.has(exportName) + ? tlsBase + : memoryBase; + if (relocationBase === undefined) { + throw new Error(`${name}: TLS export ${exportName} has no live TLS base`); + } relocatedExports[exportName] = new WebAssembly.Global( - { value: "i32", mutable: false }, - (exportValue as WebAssembly.Global).value + memoryBase, + { value: typeof rawValue === "bigint" ? "i64" : "i32", mutable: false }, + typeof rawValue === "bigint" + ? rawValue + BigInt(relocationBase) + : rawValue + relocationBase, ); } } else { @@ -864,6 +1099,7 @@ function instantiateSharedLibrary( metadata, name, forkBufAddr: sideForkBufAddr || undefined, + tlsBase, forkCapable: importsFork, functionImports, functionExports, diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index f91aa5881..c5515c60a 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -1370,6 +1370,7 @@ async function handleClone( programBytes: processInfo.programBytes, programModule: threadModule, memory, + processChannelOffset: processInfo.channelOffset, channelOffset: alloc.channelOffset, fnPtr, argPtr, diff --git a/host/src/vfs/image-helpers.ts b/host/src/vfs/image-helpers.ts index aa1ba8230..606c12798 100644 --- a/host/src/vfs/image-helpers.ts +++ b/host/src/vfs/image-helpers.ts @@ -28,8 +28,27 @@ export function writeVfsBinary( mode = 0o755, ): void { const fd = fs.open(path, O_WRONLY_CREAT_TRUNC, mode); - fs.write(fd, data, 0, data.length); - fs.close(fd); + try { + let offset = 0; + while (offset < data.length) { + const remaining = data.subarray(offset); + const written = fs.write(fd, remaining, offset, remaining.length); + if ( + !Number.isInteger(written) + || written <= 0 + || written > remaining.length + ) { + const detail = written < 0 + ? `write failed with error code ${written}` + : `write made invalid progress ` + + `(${written} of ${remaining.length} remaining bytes)`; + throw new Error(`Failed to stage complete VFS file ${path}: ${detail}`); + } + offset += written; + } + } finally { + fs.close(fd); + } } /** mkdir, swallowing EEXIST. */ diff --git a/host/src/worker-main.ts b/host/src/worker-main.ts index c38e94b49..4d63ce182 100644 --- a/host/src/worker-main.ts +++ b/host/src/worker-main.ts @@ -11,11 +11,13 @@ import type { WorkerToHostMessage, } from "./worker-protocol"; import { + createCppExceptionTag, createLongjmpTag, DynamicLinker, FORK_CAP_DYLINK_MAIN, forkInstrumentRoleAvailable, readForkInstrumentCapabilityClaim, + requireCppExceptionTag, requireLongjmpTag, type LoadedSharedLibrary, type SideModuleForkState, @@ -198,6 +200,40 @@ export interface DlopenSupport { beginSideModuleForkRewind: () => void; /** Reject a leaked active side-module identity on a normal main return. */ assertNoActiveSideModuleFork: () => void; + /** Clear a fork parent's copied archive lock in the child's private memory. */ + resetForkChildLock: () => void; +} + +/** + * Thread workers instantiate a separate Wasm module/table/tag graph, so they + * cannot safely load or invoke process side modules. Keep dlopen's ordinary C + * failure contract (NULL plus dlerror text) instead of letting a generic + * unresolved-import stub trap the pthread. + */ +function buildUnsupportedThreadDlopenImports( + memory: WebAssembly.Memory, +): Record { + const message = new TextEncoder().encode( + "dlopen is unsupported from pthread workers; load side modules on the process main worker", + ); + const n = (value: number | bigint): number => + typeof value === "bigint" ? Number(value) : value; + return { + __wasm_dlopen: (): number => 0, + __wasm_dlsym: (): number => 0, + __wasm_dlclose: (): number => -1, + __wasm_dlerror: (bufPtr: number | bigint, bufMax: number | bigint): number => { + const ptr = n(bufPtr); + const max = n(bufMax); + if (!Number.isSafeInteger(ptr) || !Number.isSafeInteger(max) || ptr < 0 || max <= 0) { + return 0; + } + const len = Math.min(message.length, max, memory.buffer.byteLength - ptr); + if (len <= 0) return 0; + new Uint8Array(memory.buffer, ptr, len).set(message.subarray(0, len)); + return len; + }, + }; } /** @@ -208,18 +244,22 @@ export interface DlopenSupport { * don't use dlopen. * * Each successful dlopen is also persisted into a per-process archive - * (linked list in linear memory, head pointer at a fixed slot below - * forkBufAddr) so the fork child can replay them via `replayDlopens`. + * (linked list in linear memory, with control slots below the main process + * channel's fork buffer) so the fork child can replay them via + * `replayDlopens`. The archive anchor is deliberately independent of the + * call-site rewind buffer: a fork issued by a pthread rewinds from that + * thread's buffer but still inherits the one process-wide dlopen archive. */ function buildDlopenImports( memory: WebAssembly.Memory, channelOffset: number, - forkBufAddr: number, + archiveControlAddr: number, getTable: () => WebAssembly.Table | undefined, getStackPointer: () => WebAssembly.Global | undefined, getInstance: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8, longjmpTag: WebAssembly.Tag | undefined, + cppExceptionTag: WebAssembly.Tag | undefined, mainHasDylinkForkRole: boolean, ): DlopenSupport { let linker: DynamicLinker | null = null; @@ -233,8 +273,12 @@ function buildDlopenImports( const sideForkOffset = ptrWidth === 8 ? DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64 : DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM32; - const headSlot = forkBufAddr - headOffset; - const activeSideForkSlot = forkBufAddr - sideForkOffset; + const lockOffset = ptrWidth === 8 + ? DLOPEN_LOCK_OFFSET_WASM64 + : DLOPEN_LOCK_OFFSET_WASM32; + const headSlot = archiveControlAddr - headOffset; + const activeSideForkSlot = archiveControlAddr - sideForkOffset; + const archiveLock = new Int32Array(memory.buffer, archiveControlAddr - lockOffset, 1); const entrySize = ptrWidth === 8 ? DLOPEN_ENTRY_SIZE_WASM64 : DLOPEN_ENTRY_SIZE_WASM32; const readPtr = (view: DataView, addr: number): number => @@ -243,7 +287,44 @@ function buildDlopenImports( if (ptrWidth === 8) view.setBigUint64(addr, BigInt(value), true); else view.setUint32(addr, value, true); }; + const readArchiveHead = (): number => ptrWidth === 8 + ? Number(Atomics.load(new BigUint64Array(memory.buffer, headSlot, 1), 0)) + : Atomics.load(new Uint32Array(memory.buffer, headSlot, 1), 0); + const writeArchiveHead = (value: number): void => { + if (ptrWidth === 8) { + Atomics.store(new BigUint64Array(memory.buffer, headSlot, 1), 0, BigInt(value)); + } else { + Atomics.store(new Uint32Array(memory.buffer, headSlot, 1), 0, value); + } + }; const linkerAllocations = new Map(); + let hostDlopenError: string | null = null; + let mainDlopenDepth = 0; + const acquireMainDlopenLock = (): boolean => { + if (mainDlopenDepth > 0) { + mainDlopenDepth++; + return true; + } + const owner = Atomics.compareExchange(archiveLock, 0, 0, 1); + if (owner !== 0) { + hostDlopenError = owner === 2 + ? "dlopen is temporarily unavailable while a pthread is forking" + : "dlopen is temporarily unavailable while another dlopen operation owns the process lock"; + return false; + } + mainDlopenDepth = 1; + return true; + }; + const releaseMainDlopenLock = (): void => { + if (mainDlopenDepth <= 0) { + throw new Error("dlopen process lock released without ownership"); + } + mainDlopenDepth--; + if (mainDlopenDepth === 0) { + Atomics.store(archiveLock, 0, 0); + Atomics.notify(archiveLock, 0); + } + }; // The kernel mmap allocator. Shared with the linker, but also used // directly by persistArchiveEntry to obtain blocks for the archive. @@ -319,6 +400,7 @@ function buildDlopenImports( const RESERVED = new Set([ "memory", "__indirect_function_table", "__memory_base", "__table_base", "__stack_pointer", "__c_longjmp", + "__cpp_exception", ]); const globalSymbols = new Map(); const inst = getInstance(); @@ -332,6 +414,19 @@ function buildDlopenImports( } const mainModuleSymbols = new Set(globalSymbols.keys()); + // A main-defined/exported tag is the process ABI authority. If the main + // image instead imports and re-exports the host tag, the identity is the + // same; if it has no export, retain the process-owned fallback created + // before main instantiation. Every side module must receive this one + // canonical identity for cross-module exception propagation. + const exportedLongjmpTag = inst?.exports.__c_longjmp; + const canonicalLongjmpTag = exportedLongjmpTag === undefined + ? longjmpTag + : requireLongjmpTag(exportedLongjmpTag, "main module export"); + const exportedCppExceptionTag = inst?.exports.__cpp_exception; + const canonicalCppExceptionTag = exportedCppExceptionTag === undefined + ? cppExceptionTag + : requireCppExceptionTag(exportedCppExceptionTag, "main module export"); const mainFork = inst?.exports.fork; const mainForkState = inst?.exports.wpk_fork_state; const sideModuleFork = mainHasDylinkForkRole @@ -386,7 +481,8 @@ function buildDlopenImports( globalSymbols, got: new Map(), loadedLibraries, - longjmpTag, + longjmpTag: canonicalLongjmpTag, + cppExceptionTag: canonicalCppExceptionTag, ptrWidth, mainModuleSymbols, sideModuleFork, @@ -409,6 +505,7 @@ function buildDlopenImports( memoryBase: number, tableBase: number, sideForkBufAddr: number, + tlsBase: number, ): void => { const nameBytes = encoder.encode(name); const nameLen = nameBytes.length; @@ -429,6 +526,7 @@ function buildDlopenImports( view.setBigUint64(entry + 40, BigInt(memoryBase), true); view.setBigUint64(entry + 48, BigInt(tableBase), true); view.setBigUint64(entry + 56, BigInt(sideForkBufAddr), true); + view.setBigUint64(entry + 64, BigInt(tlsBase), true); } else { view.setUint32(entry + 0, 0, true); view.setUint32(entry + 4, namePtr, true); @@ -438,15 +536,19 @@ function buildDlopenImports( view.setUint32(entry + 20, memoryBase, true); view.setUint32(entry + 24, tableBase, true); view.setUint32(entry + 28, sideForkBufAddr, true); + view.setUint32(entry + 32, tlsBase, true); } new Uint8Array(memory.buffer, namePtr, nameLen).set(nameBytes); new Uint8Array(memory.buffer, bytesPtr, bytes.length).set(bytes); // Append to tail (preserves insertion order). - const head = readPtr(view, headSlot); + const head = readArchiveHead(); if (head === 0) { - writePtr(view, headSlot, entry); + // Publish only after the complete entry and payload are visible. A + // pthread fork acquire-loads this word before deciding whether it can + // safely fork without access to the process side-module graph. + writeArchiveHead(entry); return; } let cursor = head; @@ -462,7 +564,7 @@ function buildDlopenImports( const replayDlopens = (): void => { const view = new DataView(memory.buffer); - let cursor = readPtr(view, headSlot); + let cursor = readArchiveHead(); if (cursor === 0) return; // Force linker creation: it's lazily built on the first C-side @@ -479,6 +581,7 @@ function buildDlopenImports( let memoryBase: number; let tableBase: number; let sideForkBufAddr: number; + let tlsBase: number; if (ptrWidth === 8) { next = Number(view.getBigUint64(cursor + 0, true)); namePtr = Number(view.getBigUint64(cursor + 8, true)); @@ -488,6 +591,7 @@ function buildDlopenImports( memoryBase = Number(view.getBigUint64(cursor + 40, true)); tableBase = Number(view.getBigUint64(cursor + 48, true)); sideForkBufAddr = Number(view.getBigUint64(cursor + 56, true)); + tlsBase = Number(view.getBigUint64(cursor + 64, true)); } else { next = view.getUint32(cursor + 0, true); namePtr = view.getUint32(cursor + 4, true); @@ -497,6 +601,7 @@ function buildDlopenImports( memoryBase = view.getUint32(cursor + 20, true); tableBase = view.getUint32(cursor + 24, true); sideForkBufAddr = view.getUint32(cursor + 28, true); + tlsBase = view.getUint32(cursor + 32, true); } // Copy name + bytes out of shared memory before passing to @@ -513,6 +618,7 @@ function buildDlopenImports( memoryBase, tableBase, forkBufAddr: sideForkBufAddr || undefined, + tlsBase: tlsBase === 0 ? undefined : tlsBase, }); if (handle === 0) { throw new Error(`dlopen(${name}): ${lk.dlerror() || "unknown"}`); @@ -523,6 +629,12 @@ function buildDlopenImports( throw new Error(`${name}: fork replay restored a mismatched save buffer`); } } + if (tlsBase !== 0) { + const loaded = loadedLibraries.get(name); + if (!loaded || loaded.tlsBase !== tlsBase) { + throw new Error(`${name}: fork replay restored a mismatched TLS base`); + } + } cursor = next; } @@ -598,9 +710,17 @@ function buildDlopenImports( } }; + const resetForkChildLock = (): void => { + Atomics.store(archiveLock, 0, 0); + Atomics.notify(archiveLock, 0); + }; + const imports: Record = { __wasm_dlopen: (bytesPtr: number, bytesLen: number, namePtr: number, nameLen: number): number => { + if (!acquireMainDlopenLock()) return 0; + hostDlopenError = null; + try { const bytes = new Uint8Array(memory.buffer, bytesPtr, bytesLen); // Copy bytes since memory.buffer may detach during Wasm instantiation const bytesCopy = new Uint8Array(bytes); @@ -627,9 +747,13 @@ function buildDlopenImports( loaded.memoryBase, loaded.tableBase, loaded.forkBufAddr ?? 0, + loaded.tlsBase ?? 0, ); } return handle; + } finally { + releaseMainDlopenLock(); + } }, __wasm_dlsym: (handle: number, namePtr: number, nameLen: number): number => { @@ -647,7 +771,8 @@ function buildDlopenImports( }, __wasm_dlerror: (bufPtr: number, bufMax: number): number => { - const err = getLinker().dlerror(); + const err = hostDlopenError ?? getLinker().dlerror(); + hostDlopenError = null; if (!err) return 0; const encoded = encoder.encode(err); const len = Math.min(encoded.length, bufMax); @@ -662,6 +787,7 @@ function buildDlopenImports( completeSideModuleForkUnwind, beginSideModuleForkRewind, assertNoActiveSideModuleFork, + resetForkChildLock, }; } @@ -677,6 +803,7 @@ function buildImportObject( getInstance?: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8 = 4, longjmpTag?: WebAssembly.Tag, + cppExceptionTag?: WebAssembly.Tag, postVmInterruptTimer?: ( timedOutPtr: number, vmInterruptPtr: number, @@ -711,6 +838,13 @@ function buildImportObject( ) as unknown as WebAssembly.ExportValue; } + if (moduleImports.some(i => i.module === "env" && i.name === "__cpp_exception" && (i.kind as string) === "tag")) { + envImports.__cpp_exception = requireCppExceptionTag( + cppExceptionTag, + "process module", + ) as unknown as WebAssembly.ExportValue; + } + // Add dlopen imports if provided if (dlopenImports) { Object.assign(envImports, dlopenImports); @@ -939,16 +1073,32 @@ function buildImportObject( /** Size of the fork save buffer used by wpk_fork_* instrumentation */ const FORK_BUF_SIZE = FORK_SAVE_BUFFER_SIZE; -// Slot below forkBufAddr that stores the head pointer of the dlopen -// archive linked list. Fork's memcpy carries the parent's archive into -// the child intact; the child walks it to replay each dlopen before -// wpk_fork rewind. +// Host-private control slots below the process main channel's fork buffer. +// Fork's memcpy carries the parent's dlopen archive into the child intact; +// the child walks it to replay each module before wpk_fork rewind. These are +// intentionally not relative to a pthread's rewind buffer. const DLOPEN_HEAD_OFFSET_WASM32 = 12; const DLOPEN_HEAD_OFFSET_WASM64 = 24; const DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM32 = 16; const DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64 = 32; -const DLOPEN_ENTRY_SIZE_WASM32 = 32; -const DLOPEN_ENTRY_SIZE_WASM64 = 64; +// Atomic host-private arbitration between process-main dlopen and pthread +// fork. The pthread holds value 2 from its pre-unwind archive check through +// memory-copy/SYS_FORK and parent rewind; process dlopen holds value 1 until +// the complete archive entry is published. A fork child clears its copied +// value before replay because its memory is already independent. +const DLOPEN_LOCK_OFFSET_WASM32 = 20; +const DLOPEN_LOCK_OFFSET_WASM64 = 40; +// Each entry also carries the side module's instance-local TLS base. Fork +// copies the TLS bytes in memory, but a new replay instance's mutable global +// must be restored explicitly. Zero is the explicit no-TLS sentinel; TLS +// allocations are required to have a positive base. +// +// This is a host-private, transient replay record: the same host build writes +// and reads it around one fork, and neither guest code nor persisted package +// artifacts interpret the layout. Enlarging it therefore does not alter the +// guest/kernel ABI. The ABI classifier/check still guards the public contract. +const DLOPEN_ENTRY_SIZE_WASM32 = 40; +const DLOPEN_ENTRY_SIZE_WASM64 = 72; const WPK_FORK_EXPORTS = [ "wpk_fork_unwind_begin", @@ -1119,6 +1269,7 @@ export async function centralizedWorkerMain( // --- SDK module path (existing) --- const processLongjmpTag = createLongjmpTag(ptrWidth); + const processCppExceptionTag = createCppExceptionTag(ptrWidth); let kernelExitStatus: number | null = null; const kernelImports = buildKernelImports( memory, @@ -1140,6 +1291,7 @@ export async function centralizedWorkerMain( // Fork state — captured by kernel_fork closure let forkResult = 0; const forkBufAddr = initData.forkBufAddr ?? channelOffset - FORK_BUF_SIZE; + const dlopenArchiveControlAddr = channelOffset - FORK_BUF_SIZE; if (hasForkInstrumentation) { // Override kernel_fork with fork-instrumentation-aware version. @@ -1170,16 +1322,17 @@ export async function centralizedWorkerMain( const dlopenSupport = buildDlopenImports( memory, channelOffset, - forkBufAddr, + dlopenArchiveControlAddr, () => processInstance?.exports.__indirect_function_table as WebAssembly.Table | undefined, () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + processCppExceptionTag, hasDylinkForkRole, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -1191,6 +1344,9 @@ export async function centralizedWorkerMain( }); const instance = await WebAssembly.instantiate(module, importObject); processInstance = instance; + if (initData.isForkChild) { + dlopenSupport.resetForkChildLock(); + } verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); // For the fork-parent case (initial launch, not a fork child), install @@ -1336,16 +1492,17 @@ export async function centralizedWorkerMain( const dlopenSupport = buildDlopenImports( memory, channelOffset, - forkBufAddr, + dlopenArchiveControlAddr, () => processInstance?.exports.__indirect_function_table as WebAssembly.Table | undefined, () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + processCppExceptionTag, false, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -1993,11 +2150,30 @@ export async function centralizedThreadWorkerMain( port: MessagePort, initData: CentralizedThreadInitMessage, ): Promise { - const { memory, channelOffset, pid, tid, fnPtr, argPtr, stackPtr, tlsPtr, ctidPtr } = initData; + const { + memory, + processChannelOffset, + channelOffset, + pid, + tid, + fnPtr, + argPtr, + stackPtr, + tlsPtr, + ctidPtr, + } = initData; const tlsOffset = initData.tlsOffset ?? initData.tlsAllocAddr; const ptrWidth = initData.ptrWidth ?? 4; let threadInstance: WebAssembly.Instance | undefined; + let processDlopenLock: Int32Array | undefined; + let pthreadForkLockHeld = false; + const releasePthreadForkLock = (): void => { + if (!pthreadForkLockHeld || !processDlopenLock) return; + Atomics.store(processDlopenLock, 0, 0); + Atomics.notify(processDlopenLock, 0); + pthreadForkLockHeld = false; + }; try { // Strip the start section AND neuter the constructor function body to prevent @@ -2014,6 +2190,37 @@ export async function centralizedThreadWorkerMain( const moduleExports = WebAssembly.Module.exports(module); const hasForkInstrumentation = hasCompleteForkInstrumentation(moduleExports, pid); const forkBufAddr = channelOffset - FORK_BUF_SIZE; + const processArchiveHeadOffset = ptrWidth === 8 + ? DLOPEN_HEAD_OFFSET_WASM64 + : DLOPEN_HEAD_OFFSET_WASM32; + const processArchiveHeadAddr = processChannelOffset + - FORK_BUF_SIZE + - processArchiveHeadOffset; + const processArchiveLockOffset = ptrWidth === 8 + ? DLOPEN_LOCK_OFFSET_WASM64 + : DLOPEN_LOCK_OFFSET_WASM32; + const processArchiveLockAddr = processChannelOffset + - FORK_BUF_SIZE + - processArchiveLockOffset; + if ( + !Number.isSafeInteger(processArchiveHeadAddr) + || processArchiveHeadAddr <= 0 + || processArchiveHeadAddr + ptrWidth > memory.buffer.byteLength + || !Number.isSafeInteger(processArchiveLockAddr) + || processArchiveLockAddr <= 0 + || processArchiveLockAddr + 4 > memory.buffer.byteLength + ) { + throw new Error( + `pid=${pid} tid=${tid}: invalid process dlopen archive anchor ` + + `${String(processArchiveHeadAddr)}`, + ); + } + processDlopenLock = new Int32Array(memory.buffer, processArchiveLockAddr, 1); + const processHasDlopenArchive = (): boolean => { + return ptrWidth === 8 + ? Atomics.load(new BigUint64Array(memory.buffer, processArchiveHeadAddr, 1), 0) !== 0n + : Atomics.load(new Uint32Array(memory.buffer, processArchiveHeadAddr, 1), 0) !== 0; + }; let forkResult = 0; let kernelThreadExitStatus: number | null = null; @@ -2034,14 +2241,35 @@ export async function centralizedThreadWorkerMain( const state = getState(); if (state === 2) { (threadInstance.exports.wpk_fork_rewind_end as () => void)(); + releasePthreadForkLock(); return forkResult; } - (threadInstance.exports.wpk_fork_unwind_begin as (addr: number) => void)(forkBufAddr); + // Side modules live in the process main worker's module/table/tag + // graph. A pthread worker cannot replay that graph into its own + // instance, so fork must fail before unwind once the process has ever + // loaded a side module. The head is read live from shared memory so a + // dlopen after pthread creation is still observed. + if (Atomics.compareExchange(processDlopenLock!, 0, 0, 2) !== 0) { + return -95; // ENOTSUP: process-main dlopen is active + } + pthreadForkLockHeld = true; + if (processHasDlopenArchive()) { + releasePthreadForkLock(); + return -95; // ENOTSUP: pthreads cannot replay process side modules + } + + try { + (threadInstance.exports.wpk_fork_unwind_begin as (addr: number) => void)(forkBufAddr); + } catch (error) { + releasePthreadForkLock(); + throw error; + } return 0; }; } else { kernelImports.kernel_fork = (): number => { + if (processHasDlopenArchive()) return -95; // ENOTSUP throw new Error( `pid=${pid} tid=${tid}: kernel_fork reached without complete ` + "wasm-fork-instrument exports. Rebuild the program with " + @@ -2050,8 +2278,10 @@ export async function centralizedThreadWorkerMain( }; } const threadLongjmpTag = createLongjmpTag(ptrWidth); - const importObject = buildImportObject(module, memory, kernelImports, channelOffset, undefined, - () => threadInstance, ptrWidth, threadLongjmpTag, + const threadCppExceptionTag = createCppExceptionTag(ptrWidth); + const threadDlopenImports = buildUnsupportedThreadDlopenImports(memory); + const importObject = buildImportObject(module, memory, kernelImports, channelOffset, threadDlopenImports, + () => threadInstance, ptrWidth, threadLongjmpTag, threadCppExceptionTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -2134,8 +2364,17 @@ export async function centralizedThreadWorkerMain( const forkState = getState(); if (forkState === 1) { unwindEnd(); + // Close the race where the process main worker dlopens after this + // pthread began unwinding but before it completed. Rewind locally + // with ENOTSUP and do not create a child. + if (processHasDlopenArchive()) { + forkResult = -95; + needsRewind = true; + continue; + } const childPid = sendForkSyscall(memory, channelOffset); if (childPid < 0) { + releasePthreadForkLock(); throw new Error(`Fork failed: errno=${-childPid}`); } forkResult = childPid; @@ -2192,6 +2431,7 @@ export async function centralizedThreadWorkerMain( tid, } satisfies WorkerToHostMessage); } catch (err) { + releasePthreadForkLock(); const message = err instanceof Error ? `${err.message}\n${err.stack ?? ""}` : String(err); diff --git a/host/src/worker-protocol.ts b/host/src/worker-protocol.ts index c837a91ab..2b80c6359 100644 --- a/host/src/worker-protocol.ts +++ b/host/src/worker-protocol.ts @@ -70,6 +70,9 @@ export interface CentralizedThreadInitMessage { programBytes: ArrayBuffer; programModule?: WebAssembly.Module; memory: WebAssembly.Memory; + /** Main process channel offset. The thread reads the process-wide dlopen + * archive head relative to this live shared-memory anchor before fork. */ + processChannelOffset: number; channelOffset: number; fnPtr: number; argPtr: number; diff --git a/host/test/binary-resolver.test.ts b/host/test/binary-resolver.test.ts index c0444c53b..ce77e9de3 100644 --- a/host/test/binary-resolver.test.ts +++ b/host/test/binary-resolver.test.ts @@ -7,6 +7,7 @@ import { binariesDir, localBinariesDir, resolveBinary, + tryResolveBinarySet, } from "../src/binary-resolver"; import { ABI_VERSION } from "../src/generated/abi"; import { @@ -104,7 +105,7 @@ async function vfsImage( return compressed ? new Uint8Array(zstdCompressSync(image)) : image; } -function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst"): string { +function fixtureClosureRelPaths(names: readonly string[]): string[] { const testRoot = "programs/wasm32/__binary_resolver_test__"; const dir = `${testRoot}/${randomUUID()}`; cleanupDirs.add(join(localBinariesDir(), dir)); @@ -115,7 +116,11 @@ function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst"): string { cleanupEmptyDirs.add(join(root, "programs")); cleanupEmptyDirs.add(root); } - return `${dir}/artifact${extension}`; + return names.map((name) => `${dir}/${name}`); +} + +function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst" | ".dat"): string { + return fixtureClosureRelPaths([`artifact${extension}`])[0]; } function candidatePath(root: string, relPath: string): string { @@ -190,4 +195,136 @@ describe("binary resolver artifact policy", () => { expect(resolveBinary(relPath)).toBe(fetchedPath); }); + + it("prefers a local declared runtime data file over the fetched candidate", () => { + const relPath = fixtureRelPath(".dat"); + const localPath = writeCandidate( + localBinariesDir(), + relPath, + new TextEncoder().encode("local-runtime"), + ); + writeCandidate( + binariesDir(), + relPath, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(resolveBinary(relPath)).toBe(localPath); + }); +}); + +describe("binary resolver package closures", () => { + it("returns a complete local closure from one provenance root", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + const wasmPath = writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const dataPath = writeCandidate( + localBinariesDir(), + dataRel, + new TextEncoder().encode("local-runtime"), + ); + writeCandidate( + binariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(tryResolveBinarySet([wasmRel, dataRel])).toEqual([wasmPath, dataPath]); + }); + + it("falls back wholesale from a partial local closure to complete fetched bytes", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const fetchedWasm = writeCandidate( + binariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const fetchedData = writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(tryResolveBinarySet([wasmRel, dataRel])).toEqual([ + fetchedWasm, + fetchedData, + ]); + }); + + it("rejects complementary partial tiers instead of mixing a closure", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(() => tryResolveBinarySet([wasmRel, dataRel])).toThrow( + /no single provenance tier.*tiers will not be mixed/s, + ); + }); + + it("falls back wholesale when a local closure member fails artifact policy", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION - 1), + ); + writeCandidate( + localBinariesDir(), + dataRel, + new TextEncoder().encode("local-runtime"), + ); + const fetchedWasm = writeCandidate( + binariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const fetchedData = writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(tryResolveBinarySet([wasmRel, dataRel])).toEqual([ + fetchedWasm, + fetchedData, + ]); + }); + + it("returns null only when no closure member exists in any tier", () => { + const relPaths = fixtureClosureRelPaths(["program.wasm", "runtime.dat"]); + expect(tryResolveBinarySet(relPaths)).toBeNull(); + }); }); diff --git a/host/test/centralized-test-helper.ts b/host/test/centralized-test-helper.ts index c0c808675..6fe9a32d3 100644 --- a/host/test/centralized-test-helper.ts +++ b/host/test/centralized-test-helper.ts @@ -140,6 +140,9 @@ export interface RunProgramOptions { captureForkCount?: boolean; /** Use the canonical rootfs image in worker-thread mode. Defaults to true. */ useDefaultRootfs?: boolean; + /** Exact VFS image for tests that stage package runtime files. Overrides + * `useDefaultRootfs`; omitted means the canonical image. */ + rootfsImage?: "default" | ArrayBuffer | Uint8Array; } export interface RunProgramResult { @@ -211,7 +214,8 @@ async function runInWorkerThread(options: RunProgramOptions): Promise { stdout += new TextDecoder().decode(data); @@ -316,6 +320,7 @@ async function runOnMainThread(options: RunProgramOptions): Promise(); const processPtrWidths = new Map(); const forkReplayContexts = new Map(); + let mainThreadForkCount: bigint | undefined; const pid = 100; @@ -525,6 +530,10 @@ async function runOnMainThread(options: RunProgramOptions): Promise { if (exitPid === pid) { + if (options.captureForkCount) { + mainThreadForkCount = kernelWorker.getForkCount(exitPid); + } kernelWorker.unregisterProcess(exitPid); processProgramBytes.delete(exitPid); processLayouts.delete(exitPid); @@ -694,5 +708,12 @@ async function runOnMainThread(options: RunProgramOptions): Promise { + const bytes = [...new TextEncoder().encode(exportName)]; + if (bytes.length >= 128) throw new Error("test TLS export name is too long"); + return [bytes.length, ...bytes, 1]; // WASM_DYLINK_FLAG_TLS + }); + const exportInfo = tlsExports.length > 0 + ? [3, 1 + exportInfoBody.length, tlsExports.length, ...exportInfoBody] + : []; + const payload = new Uint8Array(1 + dylinkName.length + 6 + exportInfo.length); payload[0] = dylinkName.length; payload.set(dylinkName, 1); payload.set([1, 4, memorySize, 0, tableSize, 0], 1 + dylinkName.length); + payload.set(exportInfo, 1 + dylinkName.length + 6); const section = new Uint8Array(2 + payload.length); section[0] = 0; section[1] = payload.length; @@ -202,6 +214,67 @@ describe.skipIf(typeof WebAssembly.Tag !== "function")("longjmp tag identity", ( }); }); +describe.skipIf(typeof WebAssembly.Tag !== "function")("C++ exception tag identity", () => { + const cases = [ + { ptrWidth: 4 as const, wasmType: "i32", value: 42 }, + { ptrWidth: 8 as const, wasmType: "i64", value: 42n }, + ]; + + it.each(cases)( + "shares one process-owned $wasmType tag across throwing and catching side modules", + ({ ptrWidth, wasmType, value }) => { + const throwerBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $cpp (import "env" "__cpp_exception") (param ${wasmType})) + (func (export "throw_cpp") (param $value ${wasmType}) + local.get $value + throw $cpp)) + `, `cpp-thrower-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const catcherBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "throw_cpp" (func $throw_cpp (param ${wasmType}))) + (tag $cpp (import "env" "__cpp_exception") (param ${wasmType})) + (func (export "catch_cpp") (param $value ${wasmType}) (result ${wasmType}) + (try (result ${wasmType}) + (do + local.get $value + call $throw_cpp + unreachable) + (catch $cpp)))) + `, `cpp-catcher-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.ptrWidth = ptrWidth; + options.cppExceptionTag = createCppExceptionTag(ptrWidth)!; + + loadSharedLibrarySync(`libcpp-thrower-${wasmType}.so`, throwerBytes, options); + const processTag = options.cppExceptionTag; + const catcher = loadSharedLibrarySync( + `libcpp-catcher-${wasmType}.so`, + catcherBytes, + options, + ); + + expect(options.cppExceptionTag).toBe(processTag); + expect((catcher.exports.catch_cpp as Function)(value)).toBe(value); + }, + ); + + it("rejects a lookalike process C++ tag before instantiation", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $cpp (import "env" "__cpp_exception") (param i32))) + `, "invalid-cpp-tag", undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.cppExceptionTag = {} as WebAssembly.Tag; + + expect(() => loadSharedLibrarySync("libinvalid-cpp-tag.so", wasmBytes, options)) + .toThrow(/__cpp_exception must be an actual WebAssembly\.Tag/); + }); +}); + describe.skipIf(!hasCompiler())("dylink.0 parser", () => { it("parses a simple shared library", () => { const wasmBytes = buildSharedLib( @@ -735,6 +808,185 @@ describe("dylink symbol interposition", () => { }); describe("dylink replay layout and rollback", () => { + it("restores copied live TLS without re-running side-module TLS initialization", () => { + const tlsSide = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__memory_base" (global $memory_base i32)) + (global $tls_base (export "__tls_base") (mut i32) (i32.const 0)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4)) + (global (export "__wasm_lpad_context") i32 (i32.const 8)) + (func $init_tls (export "__wasm_init_tls") (param $base i32) + local.get $base + global.set $tls_base + local.get $base + i32.const 42 + i32.store) + (func $start + global.get $memory_base + i32.load + i32.eqz + if + global.get $memory_base + i32.const 2 + i32.store + global.get $memory_base + i32.const 8 + i32.add + call $init_tls + end) + (start $start) + (func (export "get_tls") (result i32) + global.get $tls_base + i32.load) + (func (export "set_tls") (param $value i32) + global.get $tls_base + local.get $value + i32.store)) + `, "replay-live-tls", undefined, 0, 16, [], ["__wasm_lpad_context"]); + const parent = createSideForkLoadOptions(); + const loaded = loadSharedLibrarySync("libtls.so", tlsSide, parent); + expect(loaded.tlsBase).toBe(1032); + expect((loaded.exports.__wasm_lpad_context as WebAssembly.Global).value) + .toBe(loaded.tlsBase! + 8); + expect((loaded.exports.__tls_size as WebAssembly.Global).value).toBe(4); + expect((loaded.exports.__tls_align as WebAssembly.Global).value).toBe(4); + expect((loaded.exports.get_tls as Function)()).toBe(42); + (loaded.exports.set_tls as Function)(99); + + const child = createSideForkLoadOptions(); + new Uint8Array(child.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + const replayed = loadSharedLibrarySync("libtls.so", tlsSide, child, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: loaded.tlsBase, + }); + + expect(replayed.tlsBase).toBe(loaded.tlsBase); + expect((replayed.exports.__wasm_lpad_context as WebAssembly.Global).value) + .toBe(replayed.tlsBase! + 8); + expect((replayed.exports.get_tls as Function)()).toBe(99); + + const ambiguous = createSideForkLoadOptions(); + new Uint8Array(ambiguous.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + expect(() => loadSharedLibrarySync("libtls.so", tlsSide, ambiguous, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: 0, + })).toThrow(/missing a valid side-module TLS base/); + }); + + it("restores an i64 TLS-base global for the 64-bit pointer contract", () => { + const tlsSide = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__memory_base" (global $memory_base i32)) + (global $tls_base (export "__tls_base") (mut i64) (i64.const 0)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4)) + (func $init_tls (param $base i32) + local.get $base + i64.extend_i32_u + global.set $tls_base + local.get $base + i32.const 17 + i32.store) + (func $start + global.get $memory_base + i32.load + i32.eqz + if + global.get $memory_base + i32.const 2 + i32.store + global.get $memory_base + i32.const 8 + i32.add + call $init_tls + end) + (start $start) + (func (export "get_tls") (result i32) + global.get $tls_base + i32.wrap_i64 + i32.load) + (func (export "set_tls") (param $value i32) + global.get $tls_base + i32.wrap_i64 + local.get $value + i32.store)) + `, "replay-live-tls-i64", undefined, 0, 16); + const parent = createSideForkLoadOptions(); + parent.ptrWidth = 8; + const loaded = loadSharedLibrarySync("libtls64.so", tlsSide, parent); + (loaded.exports.set_tls as Function)(71); + + const child = createSideForkLoadOptions(); + child.ptrWidth = 8; + new Uint8Array(child.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + const replayed = loadSharedLibrarySync("libtls64.so", tlsSide, child, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: loaded.tlsBase, + }); + + expect(replayed.tlsBase).toBe(loaded.tlsBase); + expect((replayed.exports.__tls_base as WebAssembly.Global).value) + .toBe(BigInt(loaded.tlsBase!)); + expect((replayed.exports.get_tls as Function)()).toBe(71); + }); + + it("rejects incomplete, immutable, and out-of-reservation TLS state", () => { + const missingBase = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4))) + `, "tls-missing-base", undefined, 0, 16); + expect(() => loadSharedLibrarySync( + "libtls-missing-base.so", + missingBase, + createSideForkLoadOptions(), + )).toThrow(/must export mutable __tls_base/); + + const immutableBase = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (global (export "__tls_base") i32 (i32.const 1032)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4))) + `, "tls-immutable-base", undefined, 0, 16); + expect(() => loadSharedLibrarySync( + "libtls-immutable-base.so", + immutableBase, + createSideForkLoadOptions(), + )).toThrow(/must be mutable/); + + const live = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__memory_base" (global $memory_base i32)) + (global $tls_base (export "__tls_base") (mut i32) (i32.const 0)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4)) + (func $start + global.get $memory_base + i32.const 8 + i32.add + global.set $tls_base) + (start $start)) + `, "tls-outside-reservation", undefined, 0, 16); + const parent = createSideForkLoadOptions(); + const loaded = loadSharedLibrarySync("libtls-outside.so", live, parent); + const child = createSideForkLoadOptions(); + new Uint8Array(child.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + expect(() => loadSharedLibrarySync("libtls-outside.so", live, child, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: loaded.memoryBase + 16, + })).toThrow(/escapes module reservation/); + }); + it("pads replay to the exact parent table base and rejects overshoot", () => { const wasmBytes = buildDylinkWat(` (module @@ -912,3 +1164,117 @@ describe.skipIf(!hasCompiler())("DynamicLinker", () => { expect(h1).toBe(h2); }); }); + +function hasWat2Wasm(): boolean { + try { + execFileSync("wat2wasm", ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +// A minimal dylink.0 section (MEM_INFO: mem/table size + align all 0), which +// marks a module as a side module. wat2wasm emits custom sections last, but the +// loader requires dylink.0 first, so it is injected as raw bytes below rather +// than declared in the WAT. +const DYLINK_SECTION = new Uint8Array([ + 0x00, 0x0f, // custom section, size 15 + 0x08, 0x64, 0x79, 0x6c, 0x69, 0x6e, 0x6b, 0x2e, 0x30, // name "dylink.0" + 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, // MEM_INFO subsection +]); + +/** + * Assemble a hand-written side module. Using WAT (rather than a compiled C/C++ + * fixture) lets these tests reproduce the exact import shapes a real C++ side + * module produces — a self-defined symbol that is *also* an env import, and an + * env.__cpp_exception tag — which no self-contained C fixture can emit. + */ +function assembleSideModule(wat: string, name: string): Uint8Array { + const dir = join(tmpdir(), "wasm-dylink-test"); + mkdirSync(dir, { recursive: true }); + const watPath = join(dir, `${name}.wat`); + const wasmPath = join(dir, `${name}.wasm`); + writeFileSync(watPath, wat); + execFileSync("wat2wasm", ["--enable-exceptions", watPath, "-o", wasmPath], + { stdio: "pipe" }); + const raw = new Uint8Array(readFileSync(wasmPath)); + const out = new Uint8Array(8 + DYLINK_SECTION.length + (raw.length - 8)); + out.set(raw.subarray(0, 8), 0); // magic + version + out.set(DYLINK_SECTION, 8); // dylink.0 first + out.set(raw.subarray(8), 8 + DYLINK_SECTION.length); + return out; +} + +describe.skipIf(!hasWat2Wasm())("weak self-import handling", () => { + function createLoadOptions(): LoadSharedLibraryOptions { + return { + memory: new WebAssembly.Memory({ initial: 1, maximum: 100, shared: true }), + table: new WebAssembly.Table({ initial: 1, element: "anyfunc" }), + stackPointer: new WebAssembly.Global({ value: "i32", mutable: true }, 65536), + heapPointer: { value: 1024 }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }; + } + + it("routes an unresolved env import to the module's own export", () => { + // `self_fn` is both imported from env and exported: the shape wasm-ld emits + // for an interposable weak C++ symbol the module also defines. + const lib = loadSharedLibrarySync("self-import.so", assembleSideModule(` + (module + (import "env" "self_fn" (func $self_fn (result i32))) + (func (export "self_fn") (result i32) (i32.const 42)) + (func (export "call_self") (result i32) (call $self_fn))) + `, "self-import"), createLoadOptions()); + expect((lib.exports.call_self as Function)()).toBe(42); + }); + + it("rejects a genuinely absent env import during instantiation", () => { + // Only an import that the side module itself exports gets a trampoline. + // A real ABI gap must remain an eager load failure rather than hiding on + // an unexecuted path behind a delayed stub. + expect(() => loadSharedLibrarySync( + "missing-import.so", + assembleSideModule(` + (module + (import "env" "missing_fn" (func $missing (result i32))) + (func (export "call_missing") (result i32) (call $missing))) + `, "missing-import"), + createLoadOptions(), + )).toThrow(); + }); + + it("does not mistake an imported-function re-export for a definition", () => { + // A same-name export can point straight back at the imported function. + // Treating that shape as a definition would recurse forever through the + // host trampoline instead of rejecting the unresolved import at dlopen. + expect(() => loadSharedLibrarySync( + "reexported-import.so", + assembleSideModule(` + (module + (import "env" "reexported_fn" (func $reexported (result i32))) + (export "reexported_fn" (func $reexported)) + (func (export "call_reexported") (result i32) (call $reexported))) + `, "reexported-import"), + createLoadOptions(), + )).toThrow(); + }); + + it("provides the __cpp_exception tag to -fwasm-exceptions modules", () => { + // C++ side modules import this i32-payload tag. Exercise an actual + // throw/catch so both the supplied signature and engine behavior are + // covered, not merely WebAssembly.Instance's value-type check. + const lib = loadSharedLibrarySync("tag.so", assembleSideModule(` + (module + (import "env" "__cpp_exception" (tag $exc (param i32))) + (func (export "throw_and_catch") (result i32) + (try (result i32) + (do + (throw $exc (i32.const 42))) + (catch $exc)))) + `, "tag"), createLoadOptions()); + expect((lib.exports.throw_and_catch as Function)()).toBe(42); + }); +}); diff --git a/host/test/fork-dlopen-replay-e2e.test.ts b/host/test/fork-dlopen-replay-e2e.test.ts index 57f4cf17a..7a5b0375d 100644 --- a/host/test/fork-dlopen-replay-e2e.test.ts +++ b/host/test/fork-dlopen-replay-e2e.test.ts @@ -13,8 +13,8 @@ * This fixture is expected to FAIL until that fix lands. */ import { describe, it, expect, beforeAll } from "vitest"; -import { execSync } from "node:child_process"; -import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { execFileSync, execSync } from "node:child_process"; +import { readFileSync, writeFileSync, mkdirSync, existsSync, realpathSync } from "node:fs"; import { join, dirname } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; @@ -23,13 +23,25 @@ import { NodePlatformIO } from "../src/platform/node"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = join(__dirname, "../.."); -const SYSROOT = join(REPO_ROOT, "sysroot"); +const SYSROOT = process.env.KANDELO_TEST_SYSROOT ?? join(REPO_ROOT, "sysroot"); const GLUE_DIR = join(REPO_ROOT, "libc", "glue"); -const LLVM_BIN = process.env.LLVM_BIN || "/opt/homebrew/opt/llvm@21/bin"; -const CLANG = `${LLVM_BIN}/clang`; -const WASM_LD = process.env.LLVM_BIN - ? `${LLVM_BIN}/wasm-ld` - : "/opt/homebrew/bin/wasm-ld"; +const clangDriver = process.env.CLANG ?? "clang"; + +function llvmTool(name: "clang" | "clang++" | "wasm-ld"): string { + const override = name === "wasm-ld" ? process.env.WASM_LD : undefined; + if (override) return override; + try { + return execFileSync(clangDriver, [`-print-prog-name=${name}`], { + encoding: "utf8", + }).trim() || name; + } catch { + return name; + } +} + +const CLANG = llvmTool("clang"); +const CLANGXX = llvmTool("clang++"); +const WASM_LD = llvmTool("wasm-ld"); const FORK_INSTRUMENT = join(REPO_ROOT, "scripts", "run-wasm-fork-instrument.sh"); const hasSysroot = existsSync(join(SYSROOT, "lib", "libc.a")); @@ -38,6 +50,39 @@ const hasKernel = existsSync(join(REPO_ROOT, "binaries", "kernel.wasm")) || const BUILD_DIR = join(tmpdir(), "wasm-fork-dlopen-replay-e2e"); +function findLibcxxPrefix(): string | undefined { + const explicit = process.env.KANDELO_LIBCXX_PREFIX; + if ( + explicit + && existsSync(join(explicit, "lib", "libc++-pic.a")) + && existsSync(join(explicit, "lib", "libc++abi-pic.a")) + ) { + return explicit; + } + const sysrootArchive = join(SYSROOT, "lib", "libc++.a"); + if (!existsSync(sysrootArchive)) return undefined; + const prefix = dirname(dirname(realpathSync(sysrootArchive))); + return existsSync(join(prefix, "lib", "libc++-pic.a")) + && existsSync(join(prefix, "lib", "libc++abi-pic.a")) + ? prefix + : undefined; +} + +const libcxxPrefix = findLibcxxPrefix(); +const hasCppPrerequisites = hasSysroot && hasKernel && libcxxPrefix !== undefined; + +if (process.env.KANDELO_REQUIRE_CPP_DYLINK_FORK_E2E === "1" && !hasCppPrerequisites) { + throw new Error( + "C++ dlopen/fork e2e was required but kernel.wasm, sysroot/libc.a, or libcxx PIC archives are missing", + ); +} + +const CPP_RUNTIME_MAIN_EXPORTS = [ + "getenv", "fprintf", "fflush", "malloc", "strlen", "memcmp", "realloc", + "free", "fwrite", "vfprintf", "fputc", "abort", "memchr", "snprintf", + "aligned_alloc", "strcmp", "pthread_mutex_lock", "pthread_mutex_unlock", "calloc", +]; + /** Build a shared Wasm library (.so side module) from C source. */ function buildSharedLib(source: string, name: string): string { const srcPath = join(BUILD_DIR, `${name}.c`); @@ -58,8 +103,46 @@ function buildSharedLib(source: string, name: string): string { return soPath; } +/** Build a real C++ EH side module, including its TLS-bearing unwinder. */ +function buildCppSharedLib(source: string, name: string): string { + if (!libcxxPrefix) throw new Error("libcxx PIC prefix unavailable"); + const srcPath = join(BUILD_DIR, `${name}.cpp`); + const objPath = join(BUILD_DIR, `${name}.o`); + const soPath = join(BUILD_DIR, `${name}.so`); + writeFileSync(srcPath, source); + execFileSync(CLANGXX, [ + "--target=wasm32-unknown-unknown", + `--sysroot=${SYSROOT}`, + "-nostdlib", + "-fPIC", + "-O2", + "-fwasm-exceptions", + "-matomics", + "-mbulk-memory", + `-I${join(libcxxPrefix, "include", "c++", "v1")}`, + "-c", + srcPath, + "-o", + objPath, + ], { stdio: "pipe" }); + execFileSync(WASM_LD, [ + "--experimental-pic", + "--shared", + "--shared-memory", + "--export-all", + "--allow-undefined", + "--export=__tls_base", + "-o", + soPath, + objPath, + join(libcxxPrefix, "lib", "libc++-pic.a"), + join(libcxxPrefix, "lib", "libc++abi-pic.a"), + ], { stdio: "pipe" }); + return soPath; +} + /** Build a main program with dlopen + fork support. */ -function buildMainProgram(source: string, name: string): string { +function buildMainProgram(source: string, name: string, forceExports: string[] = []): string { const srcPath = join(BUILD_DIR, `${name}.c`); const wasmPath = join(BUILD_DIR, `${name}.wasm`); @@ -97,6 +180,8 @@ function buildMainProgram(source: string, name: string): string { "-Wl,--export=__tls_align", "-Wl,--export=__stack_pointer", "-Wl,--export=__wasm_thread_init", + ...(forceExports.length > 0 ? ["-Wl,--export-all"] : []), + ...forceExports.map((symbol) => `-Wl,-u,${symbol}`), ]; const allArgs = [...cflags, srcPath, ...linkFlags, "-o", wasmPath]; @@ -188,4 +273,134 @@ describe.skipIf(!hasSysroot || !hasKernel)("fork after dlopen end-to-end", () => expect(result.exitCode).toBe(0); expect(result.stdout).toContain("ok"); }); + + it("fails pthread dlopen and fork after process dlopen without creating a child", { timeout: 30_000 }, async () => { + const soPath = buildSharedLib( + `int pthread_boundary_fixture(void) { return 1; }`, + "libpthreadboundary", + ); + const wasmPath = buildMainProgram(` + #include + #include + #include + #include + #include + #include + + static const char *side_path; + static int thread_result; + + static void *run_thread(void *unused) { + (void)unused; + void *nested = dlopen(side_path, RTLD_NOW); + const char *error = dlerror(); + if (nested != NULL || error == NULL || strstr(error, "pthread workers") == NULL) { + thread_result = 1; + return NULL; + } + errno = 0; + pid_t child = fork(); + if (child != -1 || errno != ENOTSUP) { + thread_result = 2; + return NULL; + } + thread_result = 0; + return NULL; + } + + int main(int argc, char **argv) { + side_path = argv[1]; + void *side = dlopen(side_path, RTLD_NOW); + if (!side) { fprintf(stderr, "main dlopen: %s\\n", dlerror()); return 2; } + pthread_t thread; + if (pthread_create(&thread, NULL, run_thread, NULL) != 0) return 3; + if (pthread_join(thread, NULL) != 0) return 4; + if (thread_result != 0) return 10 + thread_result; + puts("pthread dylink boundary ok"); + return 0; + } + `, "test-pthread-dylink-boundary"); + + const result = await runCentralizedProgram({ + programPath: wasmPath, + argv: ["pthread-dylink-boundary", soPath], + timeout: 30_000, + io: io(), + captureForkCount: true, + }); + + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("pthread dylink boundary ok"); + expect(result.forkCount).toBe(0n); + }); + + it.skipIf(!hasCppPrerequisites)( + "child preserves side-module TLS for a real compiled C++ throw/catch", + { timeout: 30_000 }, + async () => { + const soPath = buildCppSharedLib(` + thread_local int cpp_tls_marker = 7; + extern "C" void cpp_set_tls_marker(int value) { cpp_tls_marker = value; } + extern "C" int cpp_get_tls_marker(void) { return cpp_tls_marker; } + extern "C" int cpp_throw_and_catch(int value) { + try { throw value; } + catch (int caught) { return caught; } + } + `, "libcppthrow"); + const sideModule = new WebAssembly.Module( + new Uint8Array(readFileSync(soPath)) as unknown as BufferSource, + ); + const sideImports = WebAssembly.Module.imports(sideModule); + const sideExports = WebAssembly.Module.exports(sideModule); + expect(sideImports.some((entry) => + entry.module === "env" + && entry.name === "__cpp_exception" + && (entry.kind as string) === "tag" + )).toBe(true); + expect(sideExports.map((entry) => entry.name)).toEqual( + expect.arrayContaining(["__tls_base", "__tls_size", "__wasm_init_tls"]), + ); + + const wasmPath = buildMainProgram(` + #include + #include + #include + #include + #include + typedef int (*cpp_throw_fn)(int); + typedef void (*cpp_set_marker_fn)(int); + typedef int (*cpp_get_marker_fn)(void); + int main(int argc, char **argv) { + void *lib = dlopen(argv[1], RTLD_NOW); + if (!lib) { fprintf(stderr, "dlopen: %s\\n", dlerror()); return 2; } + cpp_throw_fn run = (cpp_throw_fn)dlsym(lib, "cpp_throw_and_catch"); + cpp_set_marker_fn set_marker = (cpp_set_marker_fn)dlsym(lib, "cpp_set_tls_marker"); + cpp_get_marker_fn get_marker = (cpp_get_marker_fn)dlsym(lib, "cpp_get_tls_marker"); + if (!run || !set_marker || !get_marker || run(41) != 41) return 3; + set_marker(99); + if (get_marker() != 99) return 4; + pid_t pid = fork(); + if (pid == 0) _exit(get_marker() == 99 && run(42) == 42 ? 0 : 5); + if (pid < 0) return 6; + int status = 0; + if (waitpid(pid, &status, 0) != pid) return 7; + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 8; + puts("cpp throw after fork ok"); + return 0; + } + `, "test-cpp-throw-after-dlopen-fork", CPP_RUNTIME_MAIN_EXPORTS); + + const result = await runCentralizedProgram({ + programPath: wasmPath, + argv: ["cpp-throw-main", soPath], + timeout: 30_000, + io: io(), + }); + + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("cpp throw after fork ok"); + }, + ); }); diff --git a/host/test/package-runtime-file.test.ts b/host/test/package-runtime-file.test.ts new file mode 100644 index 000000000..47dc49007 --- /dev/null +++ b/host/test/package-runtime-file.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { + parsePackageRuntimeFileContract, + readPackageRuntimeFileContract, +} from "../../scripts/package-runtime-file"; +import { findRepoRoot } from "../src/binary-resolver"; + +function metadata(overrides: Record = {}): string { + return JSON.stringify({ + artifact: "icu.dat", + guest_path: "/usr/lib/php/icu.dat", + mode: 0o644, + mirror_path: "php/icu.dat", + closure_mirror_paths: ["php/php.wasm", "php/intl.so", "php/icu.dat"], + ...overrides, + }); +} + +describe("package runtime-file closure metadata", () => { + it("reports every declared PHP output and runtime file", () => { + const contract = readPackageRuntimeFileContract( + findRepoRoot(), + "php", + "icu.dat", + ); + expect(contract.closureMirrorPaths).toEqual([ + "php/php.wasm", + "php/php-fpm.wasm", + "php/opcache.so", + "php/curl.so", + "php/phar.so", + "php/zend_test.so", + "php/zip.so", + "php/intl.so", + "php/icu.dat", + ]); + }, 120_000); + + it("rejects duplicate or incomplete closure path metadata", () => { + expect(() => parsePackageRuntimeFileContract( + metadata({ + closure_mirror_paths: ["php/php.wasm", "php/icu.dat", "php/icu.dat"], + }), + "php", + "icu.dat", + )).toThrow(/invalid runtime-file metadata/); + + expect(() => parsePackageRuntimeFileContract( + metadata({ closure_mirror_paths: ["php/php.wasm", "php/intl.so"] }), + "php", + "icu.dat", + )).toThrow(/invalid runtime-file metadata/); + }); + + it("rejects closure mirror traversal before binary resolution", () => { + expect(() => parsePackageRuntimeFileContract( + metadata({ + closure_mirror_paths: ["php/php.wasm", "../outside", "php/icu.dat"], + }), + "php", + "icu.dat", + )).toThrow(/invalid runtime-file metadata/); + }); +}); diff --git a/host/test/vfs/image-helpers.test.ts b/host/test/vfs/image-helpers.test.ts new file mode 100644 index 000000000..547595e24 --- /dev/null +++ b/host/test/vfs/image-helpers.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFileSystem } from "../../src/vfs/memory-fs"; +import { writeVfsBinary, writeVfsFile } from "../../src/vfs/image-helpers"; +import { + writeVfsBinary as writeBrowserVfsBinary, + writeVfsFile as writeBrowserVfsFile, +} from "../../../apps/browser-demos/lib/init/vfs-utils"; + +const O_RDONLY = 0; + +function readFile(fs: MemoryFileSystem, path: string): Uint8Array { + const size = fs.stat(path).size; + const bytes = new Uint8Array(size); + const fd = fs.open(path, O_RDONLY, 0); + try { + const read = fs.read(fd, bytes, null, bytes.length); + if (read !== bytes.length) { + throw new Error(`short test read: ${read} of ${bytes.length}`); + } + } finally { + fs.close(fd); + } + return bytes; +} + +describe("VFS image write helpers", () => { + it("backs browser-demo writers with the shared strict helpers", () => { + expect(writeBrowserVfsBinary).toBe(writeVfsBinary); + expect(writeBrowserVfsFile).toBe(writeVfsFile); + }); + + it("stages every byte of a binary file", () => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + const data = new Uint8Array(256 * 1024); + for (let i = 0; i < data.length; i++) data[i] = i & 0xff; + + writeVfsBinary(fs, "/payload.bin", data, 0o640); + + expect(fs.stat("/payload.bin").size).toBe(data.length); + expect(readFile(fs, "/payload.bin")).toEqual(data); + }); + + it("reports terminal ENOSPC after preserving a positive partial write", () => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(128 * 1024)); + const data = new Uint8Array(1024 * 1024).fill(0xa5); + + expect(() => writeVfsBinary(fs, "/partial.bin", data)).toThrow(); + expect(fs.stat("/partial.bin").size).toBeGreaterThan(0); + expect(fs.stat("/partial.bin").size).toBeLessThan(data.length); + }); + + it("continues from the correct offset after a positive short write", () => { + const data = new Uint8Array([1, 2, 3, 4]); + const close = vi.fn(); + const write = vi.fn() + .mockReturnValueOnce(2) + .mockReturnValueOnce(2); + const fs = { + open: vi.fn(() => 7), + write, + close, + } as unknown as MemoryFileSystem; + + writeVfsBinary(fs, "/fixture.bin", data); + + expect(write).toHaveBeenCalledTimes(2); + expect(write.mock.calls[0][1]).toEqual(new Uint8Array([1, 2, 3, 4])); + expect(write.mock.calls[0].slice(2)).toEqual([0, 4]); + expect(write.mock.calls[1][1]).toEqual(new Uint8Array([3, 4])); + expect(write.mock.calls[1].slice(2)).toEqual([2, 2]); + expect(close).toHaveBeenCalledTimes(1); + expect(close).toHaveBeenCalledWith(7); + }); + + it("closes the descriptor for zero, negative, invalid, and thrown writes", () => { + const data = new Uint8Array([1, 2, 3, 4]); + const outcomes: Array = [ + 0, + -28, + 5, + new Error("ENOSPC"), + ]; + + for (const outcome of outcomes) { + const close = vi.fn(); + const fs = { + open: vi.fn(() => 7), + write: vi.fn(() => { + if (outcome instanceof Error) throw outcome; + return outcome; + }), + close, + } as unknown as MemoryFileSystem; + + expect(() => writeVfsBinary(fs, "/fixture.bin", data)).toThrow(); + expect(close).toHaveBeenCalledTimes(1); + expect(close).toHaveBeenCalledWith(7); + } + }); +}); diff --git a/images/vfs/scripts/build-php-test-vfs-image.ts b/images/vfs/scripts/build-php-test-vfs-image.ts index fa527c5ac..68f66178d 100644 --- a/images/vfs/scripts/build-php-test-vfs-image.ts +++ b/images/vfs/scripts/build-php-test-vfs-image.ts @@ -34,15 +34,17 @@ import { findRepoRoot, tryResolveBinary } from "../../../host/src/binary-resolve import { preparePhpTestFixtures } from "./php-test-fixtures"; import { ensureSourceExtract } from "./source-extract-helper"; import { saveImage, walkAndWrite } from "./vfs-image-helpers"; +import { resolvePackageRuntimeFile } from "../../../scripts/package-runtime-file"; const REPO_ROOT = findRepoRoot(); const PHP_FIXTURE_ROOT = join(REPO_ROOT, "tests/php-fixtures"); const LOCAL_PHP_SRC = join(REPO_ROOT, "packages/registry/php/php-src"); +const ICU_RUNTIME = resolvePackageRuntimeFile(REPO_ROOT, "php", "icu.dat"); const PHP_WASM = process.env.PHP_WASM - ?? tryResolveBinary("programs/php/php.wasm") + ?? ICU_RUNTIME?.closureHostPaths.get("php/php.wasm") ?? join(LOCAL_PHP_SRC, "sapi/cli/php"); const OPCACHE_SO = process.env.PHP_OPCACHE_SO - ?? tryResolveBinary("programs/php/opcache.so"); + ?? ICU_RUNTIME?.closureHostPaths.get("php/opcache.so"); const PHP_EXTENSION_DIRS = [ dirname(PHP_WASM), ...((process.env.PHP_EXTENSION_DIR ?? "") @@ -50,10 +52,16 @@ const PHP_EXTENSION_DIRS = [ .map((path) => path.trim()) .filter(Boolean)), ]; +const INTL_SO = ICU_RUNTIME?.closureHostPaths.get("php/intl.so") + ?? [...PHP_EXTENSION_DIRS] + .reverse() + .map((dir) => join(dir, "intl.so")) + .find((path) => existsSync(path)); const PHP_FPM_WASM = process.env.PHP_FPM_WASM - ?? tryResolveBinary("programs/php/php-fpm.wasm"); + ?? ICU_RUNTIME?.closureHostPaths.get("php/php-fpm.wasm"); const ROOTFS_VFS = process.env.ROOTFS_VFS - ?? tryResolveBinary("programs/rootfs/rootfs.vfs") + ?? tryResolveBinary("rootfs.vfs") + ?? tryResolveBinary("programs/rootfs.vfs") ?? join(REPO_ROOT, "host/wasm/rootfs.vfs"); const OUT_FILE = process.env.PHP_TEST_VFS_OUT ?? join(REPO_ROOT, "apps/browser-demos/public/php-test.vfs.zst"); @@ -116,6 +124,13 @@ function phpTestVfsFingerprint(sourceRoot: string): string { hashInputPath(hash, `extensions-${index}`, extensionDir); } hashInputPath(hash, "opcache", OPCACHE_SO); + hashInputPath(hash, "intl", INTL_SO); + hashInputPath(hash, "intl-icu-data", ICU_RUNTIME?.hostPath); + if (ICU_RUNTIME) { + hash.update( + `runtime-contract\0${ICU_RUNTIME.artifact}\0${ICU_RUNTIME.guestPath}\0${ICU_RUNTIME.mode}\0`, + ); + } return hash.digest("hex"); } @@ -283,6 +298,11 @@ async function main() { `rootfs.vfs not found at ${ROOTFS_VFS}. Build the rootfs package or set ROOTFS_VFS`, ); } + if (INTL_SO && !ICU_RUNTIME) { + throw new Error( + "PHP intl.so is present but the declared php:icu.dat runtime file is not materialized", + ); + } const phpSourceInput = resolvePhpSource(); if (!existsSync(phpSourceInput)) { throw new Error(`php-src not found at ${phpSourceInput}`); @@ -358,6 +378,21 @@ async function main() { new Uint8Array(readFileSync(OPCACHE_SO)), ); } + if (INTL_SO && ICU_RUNTIME) { + writeVfsBinary( + fs, + "/usr/lib/php/extensions/intl.so", + new Uint8Array(readFileSync(INTL_SO)), + 0o755, + ); + ensureDirRecursive(fs, dirname(ICU_RUNTIME.guestPath)); + writeVfsBinary( + fs, + ICU_RUNTIME.guestPath, + new Uint8Array(readFileSync(ICU_RUNTIME.hostPath)), + ICU_RUNTIME.mode, + ); + } const phptDirs = collectPhptDirs(phpSrc); const supportDirs = collectPhptSupportDirs(phpSrc, phptDirs); diff --git a/packages/registry/icu/build-icu.sh b/packages/registry/icu/build-icu.sh new file mode 100755 index 000000000..79ea14e6b --- /dev/null +++ b/packages/registry/icu/build-icu.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# +# Build ICU4C (libicuuc.a, libicui18n.a, libicudata.a stub + icu.dat) for +# wasm32-posix-kernel. +# +# ICU requires a TWO-STAGE build: +# +# Stage 1 (HOST): build ICU natively to produce the data-generation tools +# (genrb, pkgdata, icupkg, genccode, …) and the ICU common +# data. These run on the build machine. +# Stage 2 (CROSS): configure ICU for wasm32 with --with-cross-build pointing +# at the stage-1 build dir. The cross build reuses the host +# tools and host-generated data; it only compiles the C++ +# sources into wasm32 static libraries. +# +# Data is built in `archive` packaging mode, which emits the ICU common data as +# a standalone `icudtl.dat` file (NOT linked into libicudata.a — that +# becomes a stub). We install that file as `share/icu.dat`; PHP's intl side +# module loads it at runtime via udata_setCommonData() (the name `icu.dat` is +# deliberate and is NOT ICU's default-searched name). See +# packages/registry/php/build-php.sh for the intl side. +# +# Honors the dep-resolver build-script contract (see docs/package-management.md). +# When invoked via `cargo xtask build-deps resolve icu`, the resolver sets: +# WASM_POSIX_DEP_OUT_DIR # where to install +# WASM_POSIX_DEP_VERSION # upstream version (e.g. "74.2") +# WASM_POSIX_DEP_SOURCE_URL # tarball URL +# WASM_POSIX_DEP_SOURCE_SHA256 # expected sha256 of the tarball +# WASM_POSIX_DEP_LIBCXX_DIR # resolved libcxx prefix (direct dep) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +# shellcheck source=/dev/null +source "$REPO_ROOT/sdk/activate.sh" + +if ! command -v wasm32posix-cc &>/dev/null; then + echo "ERROR: wasm32posix-cc not found after sourcing sdk/activate.sh." >&2 + exit 1 +fi + +# --- Inputs from resolver, with ad-hoc fallbacks --- +ICU_VERSION="${WASM_POSIX_DEP_VERSION:-${ICU_VERSION:-74.2}}" +ICU_VER_UNDERSCORE="${ICU_VERSION//./_}" # 74.2 -> 74_2 +ICU_MAJOR="${ICU_VERSION%%.*}" # 74.2 -> 74 +TARGET_ARCH="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" +INSTALL_DIR="${WASM_POSIX_DEP_OUT_DIR:-$SCRIPT_DIR/icu-install}" +SOURCE_URL="${WASM_POSIX_DEP_SOURCE_URL:-https://github.com/unicode-org/icu/releases/download/release-${ICU_MAJOR}-${ICU_VERSION#*.}/icu4c-${ICU_VER_UNDERSCORE}-src.tgz}" +SOURCE_SHA256="${WASM_POSIX_DEP_SOURCE_SHA256:-68db082212a96d6f53e35d60f47d38b962e9f9d207a74cfac78029ae8ff5e08c}" + +SYSROOT="${WASM_POSIX_SYSROOT:-$REPO_ROOT/sysroot}" +export WASM_POSIX_SYSROOT="$SYSROOT" + +if [ "$TARGET_ARCH" != "wasm32" ]; then + echo "ERROR: ICU currently supports only wasm32, got $TARGET_ARCH" >&2 + exit 1 +fi + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/kandelo-icu.XXXXXX")" +cleanup() { + status=$? + trap - EXIT + if [ "${WASM_POSIX_KEEP_BUILD_DIR:-0}" = "1" ]; then + echo "==> Preserving ICU build directory: $WORK_DIR" >&2 + else + rm -rf "$WORK_DIR" + fi + exit "$status" +} +trap cleanup EXIT + +SRC_ROOT="$WORK_DIR/source" # contains icu/ (with source/) +ICU_SRC="$SRC_ROOT/icu/source" +HOST_BUILD="$WORK_DIR/host-build" # stage-1 native build (out-of-tree) + +# --- Resolve libcxx (ICU is C++), symlink into sysroot (mariadb pattern) --- +HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" +resolve_dep() { + (cd "$REPO_ROOT" && cargo run -p xtask --target "$HOST_TARGET" --quiet -- build-deps --arch "$TARGET_ARCH" resolve "$1") +} +LIBCXX_PREFIX="${WASM_POSIX_DEP_LIBCXX_DIR:-}" +if [ -z "$LIBCXX_PREFIX" ]; then + echo "==> Resolving libcxx via cargo xtask build-deps..." + LIBCXX_PREFIX="$(resolve_dep libcxx)" +fi +[ -f "$LIBCXX_PREFIX/lib/libc++.a" ] || { echo "ERROR: libcxx resolve missing libc++.a at $LIBCXX_PREFIX" >&2; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++abi.a" ] || { echo "ERROR: libcxx resolve missing libc++abi.a at $LIBCXX_PREFIX" >&2; exit 1; } +[ -d "$LIBCXX_PREFIX/include/c++/v1" ] || { echo "ERROR: libcxx resolve missing include/c++/v1 at $LIBCXX_PREFIX" >&2; exit 1; } + +echo "==> Linking libcxx into sysroot ($LIBCXX_PREFIX)..." +mkdir -p "$SYSROOT/lib" "$SYSROOT/include/c++" +ln -sf "$LIBCXX_PREFIX/lib/libc++.a" "$SYSROOT/lib/libc++.a" +ln -sf "$LIBCXX_PREFIX/lib/libc++abi.a" "$SYSROOT/lib/libc++abi.a" +rm -rf "$SYSROOT/include/c++/v1" +ln -sfn "$LIBCXX_PREFIX/include/c++/v1" "$SYSROOT/include/c++/v1" + +# --- Fetch + verify source --- +if [ ! -d "$ICU_SRC" ]; then + echo "==> Downloading ICU $ICU_VERSION..." + TARBALL="$WORK_DIR/icu4c-${ICU_VER_UNDERSCORE}-src.tgz" + curl --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors -fsSL "$SOURCE_URL" -o "$TARBALL" + echo "==> Verifying source sha256..." + echo "$SOURCE_SHA256 $TARBALL" | shasum -a 256 -c - + mkdir -p "$SRC_ROOT" + tar xzf "$TARBALL" -C "$SRC_ROOT" # extracts icu/ + rm "$TARBALL" +fi + +NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" + +run_logged() { + local label="$1" + shift + local log="$WORK_DIR/$label.log" + if "$@" >"$log" 2>&1; then + tail -20 "$log" + return 0 + fi + echo "ERROR: $label failed; final log follows:" >&2 + tail -200 "$log" >&2 + return 1 +} + +# ============================================================ +# Stage 1 — HOST build (native tools + data) +# ============================================================ +# Uses the host compiler (clang/clang++ from the dev shell), NOT the wasm +# wrappers. sdk/activate.sh only prepends SDK bin to PATH; it does not export +# CC/CXX, so an explicit host CC/CXX keeps this stage native. +# +# On Linux, statically fold the GNU C++/GCC runtime into the data tools: the Nix +# CI runner has no libstdc++.so.6 on its loader path, so a dynamically linked +# icupkg/pkgdata (run here by Stage 2's make) aborts at exec with "cannot open +# shared object file". macOS clang links a self-contained libc++ and rejects the +# flags. LDFLAGS set here is honored: runConfigureICU re-exports it to configure. +case "$(uname -s)" in + Linux) HOST_LDFLAGS="-static-libstdc++ -static-libgcc" ;; + *) HOST_LDFLAGS="" ;; +esac +if [ ! -x "$HOST_BUILD/bin/icupkg" ] && [ ! -x "$HOST_BUILD/bin/genccode" ]; then + echo "==> Stage 1: building ICU natively for host tools + data..." + rm -rf "$HOST_BUILD" + mkdir -p "$HOST_BUILD" + ( cd "$HOST_BUILD" + CC="${HOST_CC:-clang}" CXX="${HOST_CXX:-clang++}" \ + LDFLAGS="$HOST_LDFLAGS" \ + "$ICU_SRC/runConfigureICU" MacOSX \ + --enable-static --disable-shared \ + --disable-samples --disable-tests --disable-extras + run_logged host-make make -j"$NPROC" + ) +else + echo "==> Stage 1: reusing existing host build at $HOST_BUILD" +fi + +# ============================================================ +# Stage 2 — CROSS build (wasm32 static libs) +# ============================================================ +echo "==> Stage 2: cross-configuring ICU for wasm32..." +# In-tree cross build (wasm32posix-configure runs ./configure in CWD). +# Scrub any prior cross-build state in the source tree. +cd "$ICU_SRC" +make distclean 2>/dev/null || true + +# ICU maps the configure host triple to a config/mh- makefile +# fragment. Our SDK forces --host=wasm32-unknown-none, whose OS component +# ("none") ICU does not recognize, so it selects the stock config/mh-unknown — +# a stub that hard-errors "configure could not detect your platform" and aborts +# `make`. ICU's own remedy (printed in that error) is to supply mh-unknown from +# a known platform. We use mh-linux: this is a --disable-shared --enable-static +# build, so mh-linux's Linux shared-library rules are never exercised; only its +# generic compile rules apply, driven by our wasm CC/CXX. Idempotent overwrite, +# re-applied every run because a fresh source extraction resets it. +cp "$ICU_SRC/config/mh-linux" "$ICU_SRC/config/mh-unknown" + +# C++ flags: ICU 74 needs C++17. libc++ headers come from the sysroot symlink. +# LDFLAGS carries -lc++ -lc++abi so configure's C++ link probes resolve. +# -fPIC: ICU's static libs are absorbed into intl.so, a wasm SIDE MODULE linked +# with `-shared --experimental-pic`. wasm-ld requires EVERY input object to be +# position-independent; a non-PIC ICU object triggers "R_WASM_MEMORY_ADDR_SLEB +# cannot be used against symbol ...; recompile with -fPIC" at the intl.so link. +CXXFLAGS="-O2 -std=c++17 -fPIC" \ +CFLAGS="-O2 -fPIC" \ +LDFLAGS="-lc++ -lc++abi" \ +wasm32posix-configure \ + --with-cross-build="$HOST_BUILD" \ + --enable-static --disable-shared \ + --disable-tools --disable-tests --disable-samples --disable-extras \ + --disable-layoutex \ + --with-data-packaging=archive \ + --prefix="$INSTALL_DIR" + +echo "==> Stage 2: building wasm32 libraries..." +# ICU bakes ICUDATA_DIR into common/putil.ao when data packaging is `common`. +# The resolver install prefix is a random `.tmp-` directory, so leaving +# the generated default in place makes libicuuc.a differ on every clean build. +# The PHP extension stages icu.dat at this stable guest directory and calls +# udata_setCommonData() explicitly; use the same path for the fallback string. +run_logged wasm-make make -j"$NPROC" ICUDATA_DIR=/usr/lib/php + +echo "==> Installing to $INSTALL_DIR..." +rm -rf "$INSTALL_DIR" +run_logged wasm-install make install + +# ICU's generated pkg-config metadata records --prefix verbatim. Resolver +# builds install into a temporary directory and later rewrite that to the +# producer's canonical cache path, which still breaks after an archive is +# fetched or moved elsewhere. Keep the metadata package-relative instead: each +# file lives in lib/pkgconfig, so pcfiledir/../.. is the current ICU prefix in +# every cache, extracted archive, or local overlay. +for pc in icu-uc.pc icu-i18n.pc icu-io.pc; do + pc_path="$INSTALL_DIR/lib/pkgconfig/$pc" + [ -f "$pc_path" ] || { echo "ERROR: missing ICU pkg-config output $pc_path" >&2; exit 1; } + sed 's|^prefix = .*|prefix = ${pcfiledir}/../..|' "$pc_path" > "$pc_path.tmp" + mv "$pc_path.tmp" "$pc_path" +done + +# --- Stage the common data as icu.dat (see header) --- +DAT_SRC="$(find "$ICU_SRC/data" "$HOST_BUILD/data" -name "icudt${ICU_MAJOR}l.dat" 2>/dev/null | head -1 || true)" +if [ -z "$DAT_SRC" ]; then + echo "ERROR: could not locate icudt${ICU_MAJOR}l.dat after build" >&2 + exit 1 +fi +mkdir -p "$INSTALL_DIR/share" +cp "$DAT_SRC" "$INSTALL_DIR/share/icu.dat" +echo "==> staged $(basename "$DAT_SRC") -> $INSTALL_DIR/share/icu.dat ($(wc -c < "$INSTALL_DIR/share/icu.dat") bytes)" + +# --- Sanity: the static libs we promise (icuio included: PHP's PHP_SETUP_ICU +# requires the icu-io pkg-config module, so intl won't configure without it) --- +for lib in libicuuc.a libicui18n.a libicuio.a libicudata.a; do + [ -f "$INSTALL_DIR/lib/$lib" ] || { echo "ERROR: missing $INSTALL_DIR/lib/$lib" >&2; exit 1; } +done +echo "==> ICU build complete." +ls -lh "$INSTALL_DIR/lib/"*.a "$INSTALL_DIR/share/icu.dat" diff --git a/packages/registry/icu/build.toml b/packages/registry/icu/build.toml new file mode 100644 index 000000000..c2d176224 --- /dev/null +++ b/packages/registry/icu/build.toml @@ -0,0 +1,16 @@ +script_path = "packages/registry/icu/build-icu.sh" +repo_url = "https://github.com/Automattic/kandelo.git" +commit = "UNPUBLISHED" +# Revision 3: ICU static libs rebuilt with -fPIC so they can be absorbed into +# intl.so (a wasm side module built with -shared --experimental-pic, which +# requires all inputs to be position-independent). +# Revision 4: Stage-1 host tools statically link the GNU C++/GCC runtime on +# Linux so icupkg/pkgdata do not need libstdc++.so.6 on the Nix CI runner. +# Revision 5: compile ICU's default data directory as the stable guest path +# /usr/lib/php instead of the resolver's random temporary install prefix. +# Revision 6: make the three pkg-config prefixes relative to pcfiledir so a +# fetched or moved archive resolves its own headers and libraries. +revision = 6 + +[binary] +index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/icu/package.toml b/packages/registry/icu/package.toml new file mode 100644 index 000000000..f49bdd7e7 --- /dev/null +++ b/packages/registry/icu/package.toml @@ -0,0 +1,86 @@ +kind = "library" +name = "icu" +version = "74.2" +kernel_abi = 7 +# ICU4C is C++; it links libc++/libc++abi from the libcxx package. +depends_on = ["libcxx@21.1.7"] +# wasm32 only for now — the sole consumer is PHP's intl side module, +# which is a wasm32 build. Add wasm64 only if a wasm64 consumer appears. +arches = ["wasm32"] + +# ICU4C for wasm32, built as static libraries: +# lib/libicuuc.a (common) +# lib/libicui18n.a (internationalization) +# lib/libicuio.a (ICU stream I/O used by PHP's configure contract) +# lib/libicudata.a (stubdata — real data lives in the .dat, see below) +# include/unicode/*.h +# lib/pkgconfig/icu-{uc,i18n,io}.pc +# share/icu.dat (common data archive; renamed from icudt74l.dat per +# the intl side-module design — loaded at runtime via +# udata_setCommonData, NOT ICU's default name search) +# +# ICU requires a two-stage build: a HOST build (to generate the data and the +# genrb/pkgdata/icupkg tools) followed by a wasm32 cross build pointed at the +# host build via --with-cross-build. Data is built in `archive` packaging mode +# so the common data is emitted as a standalone icudt74l.dat instead of being +# linked into libicudata.a; we stage that file as `icu.dat`. + +[source] +url = "https://github.com/unicode-org/icu/releases/download/release-74-2/icu4c-74_2-src.tgz" +sha256 = "68db082212a96d6f53e35d60f47d38b962e9f9d207a74cfac78029ae8ff5e08c" + +[license] +spdx = "ICU" +url = "https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE" + +[build] +script_path = "packages/registry/icu/build-icu.sh" + +# Stage 1 compiles native ICU data tools; Stage 2 invokes GNU make around the +# SDK's wasm wrappers. Source fetching and direct (non-resolver) dependency +# fallback also need the declared network/Rust tools. Keep these probes aligned +# with the repository dev shell instead of accepting ambient host state. +[[host_tools]] +name = "clang" +version_constraint = ">=21.0" +probe = { args = ["--version"], version_regex = "clang version (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "clang++" +version_constraint = ">=21.0" +probe = { args = ["--version"], version_regex = "clang version (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "make" +version_constraint = ">=3.80" +probe = { args = ["--version"], version_regex = "GNU Make (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "curl" +version_constraint = ">=7.0" +probe = { args = ["--version"], version_regex = "curl (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "rustc" +version_constraint = ">=1.85" +probe = { args = ["--version"], version_regex = "rustc (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "cargo" +version_constraint = ">=1.85" +probe = { args = ["--version"], version_regex = "cargo (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[outputs] +libs = ["lib/libicuuc.a", "lib/libicui18n.a", "lib/libicuio.a", "lib/libicudata.a"] +headers = ["include/unicode"] +pkgconfig = ["lib/pkgconfig/icu-uc.pc", "lib/pkgconfig/icu-i18n.pc", "lib/pkgconfig/icu-io.pc"] +# ICU common data is a first-class declared runtime file, staged from +# icudtl.dat. Consumers read it from the resolved dependency directory at +# $WASM_POSIX_DEP_ICU_DIR/share/icu.dat and load it with udata_setCommonData(). +files = ["share/icu.dat"] diff --git a/packages/registry/libcxx/.gitignore b/packages/registry/libcxx/.gitignore index 201dd024e..d13a937fe 100644 --- a/packages/registry/libcxx/.gitignore +++ b/packages/registry/libcxx/.gitignore @@ -1,5 +1,7 @@ node_modules/ # LLVM source clone (per major version) — build script populates on demand. llvm-project-*/ +# Per-arch LLVM source symlink tree the build script recreates each run. +llvm-source-*/ # Per-arch build trees produced by the build script. build-*/ diff --git a/packages/registry/libcxx/build-libcxx.sh b/packages/registry/libcxx/build-libcxx.sh index 96396ed8f..22dd7420e 100755 --- a/packages/registry/libcxx/build-libcxx.sh +++ b/packages/registry/libcxx/build-libcxx.sh @@ -103,7 +103,10 @@ if [ ! -d "$NIX_LIBUNWIND_SOURCE/libunwind" ]; then fi BUILD_DIR="$SCRIPT_DIR/build-${ARCH}" -LLVM_SRC_DIR="$BUILD_DIR/llvm-source" +# Assembled source tree lives OUTSIDE the build dirs so both the default +# (static, non-PIC) build and the position-independent build below can share it +# without one build's `rm -rf` deleting the other's source. +LLVM_SRC_DIR="$SCRIPT_DIR/llvm-source-${ARCH}" # --- Verify prerequisites --- if [ ! -f "$SYSROOT/lib/libc.a" ]; then @@ -134,12 +137,19 @@ echo "==> Building libc++ and libc++abi for ${ARCH}..." # against the modern ABI; consumers linking against this libcxx # archive must also compile with `-wasm-use-legacy-eh=false` (the # SDK's `compileFlags` was updated in lock-step). -WASM_C_FLAGS="--target=${WASM_TARGET} -matomics -mbulk-memory -mexception-handling -mllvm -wasm-enable-sjlj -mllvm -wasm-use-legacy-eh=false -fexceptions -fno-trapping-math --sysroot=${SYSROOT} -O2 -DNDEBUG" +# +# LLVM records source/compilation paths in archive members. All assembled +# sources and both variant build directories live under REPO_ROOT, so one +# stable worktree mapping covers LLVM_SRC_DIR, BUILD_DIR, PIC_BUILD_DIR, the +# generated smoke source, and sysroot paths for both PIC and non-PIC builds. +# Without it, libc++abi.a and every side module absorbing it differ solely by +# the caller's checkout path. +REPRODUCIBLE_PREFIX_MAPS="-ffile-prefix-map=${REPO_ROOT}=/usr/src/kandelo -fdebug-prefix-map=${REPO_ROOT}=/usr/src/kandelo -fmacro-prefix-map=${REPO_ROOT}=/usr/src/kandelo" +WASM_C_FLAGS="--target=${WASM_TARGET} -matomics -mbulk-memory -mexception-handling -mllvm -wasm-enable-sjlj -mllvm -wasm-use-legacy-eh=false -fexceptions -fno-trapping-math --sysroot=${SYSROOT} -O2 -DNDEBUG ${REPRODUCIBLE_PREFIX_MAPS}" -# Always start with a fresh build tree so a cache-miss rebuild does -# not mix old + new cmake artifacts. -rm -rf "$BUILD_DIR" -mkdir -p "$BUILD_DIR" +# Start with a fresh source tree so a cache-miss rebuild does not mix old + new +# artifacts. (Each build dir is cleaned by build_libcxx_variant below.) +rm -rf "$LLVM_SRC_DIR" # Assemble the monorepo-shaped source tree expected by runtimes/CMakeLists.txt # from exact Nix source derivations. Nix's libcxx source carries runtimes/, @@ -160,63 +170,77 @@ for entry in "$NIX_LIBCXX_SOURCE/runtimes"/*; do done ln -s "$NIX_LIBUNWIND_SOURCE/libunwind" "$LLVM_SRC_DIR/libunwind" -cd "$BUILD_DIR" +NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" -cmake -G "Unix Makefiles" -S "$LLVM_SRC_DIR/runtimes" \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DCMAKE_SYSTEM_NAME=Generic \ - -DCMAKE_SYSTEM_PROCESSOR="${ARCH}" \ - -DCMAKE_C_COMPILER="$LLVM_CLANG" \ - -DCMAKE_CXX_COMPILER="$LLVM_CLANG" \ - -DCMAKE_AR="$LLVM_AR" \ - -DCMAKE_RANLIB="$LLVM_RANLIB" \ - -DCMAKE_NM="$LLVM_NM" \ - -DCMAKE_C_COMPILER_TARGET="${WASM_TARGET}" \ - -DCMAKE_CXX_COMPILER_TARGET="${WASM_TARGET}" \ - -DCMAKE_C_FLAGS="${WASM_C_FLAGS}" \ - -DCMAKE_CXX_FLAGS="${WASM_C_FLAGS}" \ - -DCMAKE_SYSROOT="${SYSROOT}" \ - -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ - \ - -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_ENABLE_STATIC=ON \ - -DLIBCXX_ENABLE_EXCEPTIONS=ON \ - -DLIBCXX_ENABLE_RTTI=ON \ - -DLIBCXX_HAS_MUSL_LIBC=ON \ - -DLIBCXX_HAS_PTHREAD_API=ON \ - -DLIBCXX_CXX_ABI=libcxxabi \ - -DLIBCXX_INCLUDE_BENCHMARKS=OFF \ - -DLIBCXX_INCLUDE_TESTS=OFF \ - -DLIBCXX_ENABLE_FILESYSTEM=ON \ - -DLIBCXX_ENABLE_MONOTONIC_CLOCK=ON \ - -DLIBCXX_ENABLE_RANDOM_DEVICE=OFF \ - -DLIBCXX_ENABLE_LOCALIZATION=ON \ - -DLIBCXX_ENABLE_WIDE_CHARACTERS=ON \ - -DLIBCXX_ENABLE_NEW_DELETE_DEFINITIONS=ON \ - \ - -DLIBCXXABI_ENABLE_SHARED=OFF \ - -DLIBCXXABI_ENABLE_STATIC=ON \ - -DLIBCXXABI_ENABLE_EXCEPTIONS=ON \ - -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ - -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ - -DLIBCXXABI_STATICALLY_LINK_UNWINDER_IN_STATIC_LIBRARY=ON \ - -DLIBCXXABI_ENABLE_THREADS=ON \ - -DLIBCXXABI_HAS_PTHREAD_API=ON \ - -DLIBCXXABI_INCLUDE_TESTS=OFF \ - \ - -DLIBUNWIND_ENABLE_SHARED=OFF \ - -DLIBUNWIND_ENABLE_STATIC=ON \ - -DLIBUNWIND_ENABLE_THREADS=ON \ - -DLIBUNWIND_USE_COMPILER_RT=OFF \ - -DLIBUNWIND_INCLUDE_TESTS=OFF \ - -DLIBUNWIND_HIDE_SYMBOLS=ON \ - \ - -DCMAKE_SIZEOF_VOID_P="${SIZEOF_VOID_P}" \ - 2>&1 | tail -20 +# Configure + build libc++/libc++abi/libunwind into with the given +# compile-flags string (plus any extra cmake args). Factored so the default +# static archives and the position-independent variant (below) share ONE cmake +# recipe and cannot drift apart. +build_libcxx_variant() { + local variant_build_dir="$1"; shift + local variant_c_flags="$1"; shift + rm -rf "$variant_build_dir" + mkdir -p "$variant_build_dir" + ( cd "$variant_build_dir" + cmake -G "Unix Makefiles" -S "$LLVM_SRC_DIR/runtimes" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DCMAKE_SYSTEM_NAME=Generic \ + -DCMAKE_SYSTEM_PROCESSOR="${ARCH}" \ + -DCMAKE_C_COMPILER="$LLVM_CLANG" \ + -DCMAKE_CXX_COMPILER="$LLVM_CLANG" \ + -DCMAKE_AR="$LLVM_AR" \ + -DCMAKE_RANLIB="$LLVM_RANLIB" \ + -DCMAKE_NM="$LLVM_NM" \ + -DCMAKE_C_COMPILER_TARGET="${WASM_TARGET}" \ + -DCMAKE_CXX_COMPILER_TARGET="${WASM_TARGET}" \ + -DCMAKE_C_FLAGS="${variant_c_flags}" \ + -DCMAKE_CXX_FLAGS="${variant_c_flags}" \ + -DCMAKE_SYSROOT="${SYSROOT}" \ + -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ + \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXX_ENABLE_EXCEPTIONS=ON \ + -DLIBCXX_ENABLE_RTTI=ON \ + -DLIBCXX_HAS_MUSL_LIBC=ON \ + -DLIBCXX_HAS_PTHREAD_API=ON \ + -DLIBCXX_CXX_ABI=libcxxabi \ + -DLIBCXX_INCLUDE_BENCHMARKS=OFF \ + -DLIBCXX_INCLUDE_TESTS=OFF \ + -DLIBCXX_ENABLE_FILESYSTEM=ON \ + -DLIBCXX_ENABLE_MONOTONIC_CLOCK=ON \ + -DLIBCXX_ENABLE_RANDOM_DEVICE=OFF \ + -DLIBCXX_ENABLE_LOCALIZATION=ON \ + -DLIBCXX_ENABLE_WIDE_CHARACTERS=ON \ + -DLIBCXX_ENABLE_NEW_DELETE_DEFINITIONS=ON \ + \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBCXXABI_ENABLE_EXCEPTIONS=ON \ + -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ + -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ + -DLIBCXXABI_STATICALLY_LINK_UNWINDER_IN_STATIC_LIBRARY=ON \ + -DLIBCXXABI_ENABLE_THREADS=ON \ + -DLIBCXXABI_HAS_PTHREAD_API=ON \ + -DLIBCXXABI_INCLUDE_TESTS=OFF \ + \ + -DLIBUNWIND_ENABLE_SHARED=OFF \ + -DLIBUNWIND_ENABLE_STATIC=ON \ + -DLIBUNWIND_ENABLE_THREADS=ON \ + -DLIBUNWIND_USE_COMPILER_RT=OFF \ + -DLIBUNWIND_INCLUDE_TESTS=OFF \ + -DLIBUNWIND_HIDE_SYMBOLS=ON \ + \ + -DCMAKE_SIZEOF_VOID_P="${SIZEOF_VOID_P}" \ + "$@" \ + 2>&1 | tail -20 -echo "==> Compiling (this may take a few minutes)..." -NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" -make -j"$NPROC" cxx cxxabi unwind 2>&1 | tail -10 + echo "==> Compiling (this may take a few minutes)..." + make -j"$NPROC" cxx cxxabi unwind 2>&1 | tail -10 ) +} + +echo "==> Building default (static, non-PIC) libc++/libc++abi for ${ARCH}..." +build_libcxx_variant "$BUILD_DIR" "${WASM_C_FLAGS}" # --- Install into the resolver's OUT_DIR --- echo "==> Installing to $INSTALL_DIR..." @@ -298,7 +322,32 @@ if ! "$LLVM_CLANG" ${WASM_C_FLAGS} \ fi echo "==> Header smoke compile passed." +# --- Position-independent variant for wasm side modules --- +# The default archives above are non-PIC, which is correct for the common case: +# static linking into a main wasm module (php.wasm, mariadb, ruby). But a wasm +# SIDE MODULE (built with `-shared --experimental-pic`, e.g. PHP's intl.so, which +# statically absorbs libc++/libc++abi) requires EVERY input object to be +# position-independent, or wasm-ld fails with "relocation R_WASM_MEMORY_ADDR_SLEB +# cannot be used against symbol ...; recompile with -fPIC". Emit a parallel PIC +# pair alongside the defaults. This is purely additive: libc++.a / libc++abi.a +# and the header set above are untouched, so existing static consumers are +# unaffected; only side-module consumers reach for the -pic archives. +echo "==> Building position-independent libc++/libc++abi (for wasm side modules)..." +PIC_BUILD_DIR="$SCRIPT_DIR/build-${ARCH}-pic" +build_libcxx_variant "$PIC_BUILD_DIR" "${WASM_C_FLAGS} -fPIC" -DCMAKE_POSITION_INDEPENDENT_CODE=ON + +LIBCXX_PIC_A=$(find "$PIC_BUILD_DIR" -name "libc++.a" -not -path "*/CMakeFiles/*" | head -1) +LIBCXXABI_PIC_A=$(find "$PIC_BUILD_DIR" -name "libc++abi.a" -not -path "*/CMakeFiles/*" | head -1) +if [ -z "$LIBCXX_PIC_A" ] || [ -z "$LIBCXXABI_PIC_A" ]; then + echo "ERROR: PIC libraries not found under $PIC_BUILD_DIR" >&2 + exit 1 +fi +cp "$LIBCXX_PIC_A" "$INSTALL_DIR/lib/libc++-pic.a" +cp "$LIBCXXABI_PIC_A" "$INSTALL_DIR/lib/libc++abi-pic.a" + echo "==> Done!" -echo " libc++.a: $(wc -c < "$INSTALL_DIR/lib/libc++.a" | tr -d ' ') bytes" -echo " libc++abi.a: $(wc -c < "$INSTALL_DIR/lib/libc++abi.a" | tr -d ' ') bytes" -echo " headers: $INSTALL_DIR/include/c++/v1/" +echo " libc++.a: $(wc -c < "$INSTALL_DIR/lib/libc++.a" | tr -d ' ') bytes" +echo " libc++abi.a: $(wc -c < "$INSTALL_DIR/lib/libc++abi.a" | tr -d ' ') bytes" +echo " libc++-pic.a: $(wc -c < "$INSTALL_DIR/lib/libc++-pic.a" | tr -d ' ') bytes" +echo " libc++abi-pic.a: $(wc -c < "$INSTALL_DIR/lib/libc++abi-pic.a" | tr -d ' ') bytes" +echo " headers: $INSTALL_DIR/include/c++/v1/" diff --git a/packages/registry/libcxx/build.toml b/packages/registry/libcxx/build.toml index 875768a9e..40b16e4ef 100644 --- a/packages/registry/libcxx/build.toml +++ b/packages/registry/libcxx/build.toml @@ -1,13 +1,19 @@ script_path = "packages/registry/libcxx/build-libcxx.sh" -repo_url = "https://github.com/brandonpayton/kandelo.git" -commit = "8c53383229fab78f97b098c3207a655159c03041" +repo_url = "https://github.com/Automattic/kandelo.git" +commit = "UNPUBLISHED" # Revision 4: libcxx + libcxxabi + libunwind rebuilt with # `-mllvm -wasm-use-legacy-eh=false` explicitly set. # Revision 5: libcxx builds only from flake.nix's exact Nix LLVM # source derivations, hard-fails on compiler/source version drift, and # installs headers from the build tree so the header set cannot drift # from the built library. -revision = 5 +# Revision 6: additionally emits position-independent libc++-pic.a / +# libc++abi-pic.a (a second -fPIC build) so wasm side modules like PHP's +# intl.so can statically absorb libc++. The non-PIC pair is unchanged. +# Revision 7: map the worktree (including the assembled LLVM source and both +# build variants) to a stable producer path so archives do not retain the +# caller's absolute checkout directory. +revision = 7 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/libcxx/package.toml b/packages/registry/libcxx/package.toml index 49d406570..9fcc01e8c 100644 --- a/packages/registry/libcxx/package.toml +++ b/packages/registry/libcxx/package.toml @@ -42,5 +42,11 @@ url = "https://github.com/llvm/llvm-project/blob/main/LICENSE.TXT" script_path = "packages/registry/libcxx/build-libcxx.sh" [outputs] -libs = ["lib/libc++.a", "lib/libc++abi.a"] +# libc++.a / libc++abi.a are the default non-PIC archives for static linking +# into a main wasm module (php.wasm, mariadb, ruby). libc++-pic.a / +# libc++abi-pic.a are the position-independent variants required by wasm SIDE +# MODULES (`-shared --experimental-pic`), e.g. PHP's intl.so, which statically +# absorbs libc++ and would otherwise hit a wasm-ld "recompile with -fPIC" error. +# Additive: existing consumers keep using the non-PIC pair unchanged. +libs = ["lib/libc++.a", "lib/libc++abi.a", "lib/libc++-pic.a", "lib/libc++abi-pic.a"] headers = ["include/c++/v1"] diff --git a/packages/registry/php/build-php.sh b/packages/registry/php/build-php.sh index 976a00724..9349334c3 100755 --- a/packages/registry/php/build-php.sh +++ b/packages/registry/php/build-php.sh @@ -75,6 +75,13 @@ LIBZIP_PREFIX="${WASM_POSIX_DEP_LIBZIP_DIR:-}" [ -z "$LIBZIP_PREFIX" ] && { echo "==> Resolving libzip..."; LIBZIP_PREFIX="$(resolve_dep libzip)"; } LIBCURL_PREFIX="${WASM_POSIX_DEP_LIBCURL_DIR:-}" [ -z "$LIBCURL_PREFIX" ] && { echo "==> Resolving libcurl..."; LIBCURL_PREFIX="$(resolve_dep libcurl)"; } +# ICU and libcxx back only the runtime-loadable intl side module. Keep them as +# explicit direct dependencies so the recipe never succeeds because a prior +# local build happened to leave C++ headers or ICU archives in the sysroot. +ICU_PREFIX="${WASM_POSIX_DEP_ICU_DIR:-}" +[ -z "$ICU_PREFIX" ] && { echo "==> Resolving icu..."; ICU_PREFIX="$(resolve_dep icu)"; } +LIBCXX_PREFIX="${WASM_POSIX_DEP_LIBCXX_DIR:-}" +[ -z "$LIBCXX_PREFIX" ] && { echo "==> Resolving libcxx..."; LIBCXX_PREFIX="$(resolve_dep libcxx)"; } [ -f "$ZLIB_PREFIX/lib/libz.a" ] || { echo "ERROR: zlib resolve missing libz.a"; exit 1; } [ -f "$SQLITE_PREFIX/lib/libsqlite3.a" ] || { echo "ERROR: sqlite resolve missing libsqlite3.a"; exit 1; } [ -f "$OPENSSL_PREFIX/lib/libssl.a" ] || { echo "ERROR: openssl resolve missing libssl.a"; exit 1; } @@ -82,6 +89,16 @@ LIBCURL_PREFIX="${WASM_POSIX_DEP_LIBCURL_DIR:-}" [ -f "$LIBICONV_PREFIX/lib/libiconv.a" ] || { echo "ERROR: GNU libiconv resolve missing libiconv.a"; exit 1; } [ -f "$LIBZIP_PREFIX/lib/libzip.a" ] || { echo "ERROR: libzip resolve missing libzip.a"; exit 1; } [ -f "$LIBCURL_PREFIX/lib/libcurl.a" ] || { echo "ERROR: libcurl resolve missing libcurl.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicuuc.a" ] || { echo "ERROR: icu resolve missing libicuuc.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicui18n.a" ] || { echo "ERROR: icu resolve missing libicui18n.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicuio.a" ] || { echo "ERROR: icu resolve missing libicuio.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicudata.a" ] || { echo "ERROR: icu resolve missing libicudata.a"; exit 1; } +[ -f "$ICU_PREFIX/share/icu.dat" ] || { echo "ERROR: icu resolve missing share/icu.dat"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++.a" ] || { echo "ERROR: libcxx resolve missing libc++.a"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++abi.a" ] || { echo "ERROR: libcxx resolve missing libc++abi.a"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++-pic.a" ] || { echo "ERROR: libcxx resolve missing libc++-pic.a"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++abi-pic.a" ] || { echo "ERROR: libcxx resolve missing libc++abi-pic.a"; exit 1; } +[ -d "$LIBCXX_PREFIX/include/c++/v1" ] || { echo "ERROR: libcxx resolve missing include/c++/v1"; exit 1; } echo "==> zlib at $ZLIB_PREFIX" echo "==> sqlite at $SQLITE_PREFIX" echo "==> openssl at $OPENSSL_PREFIX" @@ -89,10 +106,24 @@ echo "==> libxml2 at $LIBXML2_PREFIX" echo "==> GNU libiconv at $LIBICONV_PREFIX" echo "==> libzip at $LIBZIP_PREFIX" echo "==> libcurl at $LIBCURL_PREFIX" +echo "==> icu at $ICU_PREFIX" +echo "==> libcxx at $LIBCXX_PREFIX" + +# The SDK's C++ driver reads libc++ from the worktree-local sysroot. Point that +# declared toolchain location at the resolver-owned dependency, matching the +# existing MariaDB/SpiderMonkey package contract. The intl side-module link +# below names the PIC archives explicitly; these non-PIC names are only for +# configure probes and any main-module C++ checks. +mkdir -p "$SYSROOT/lib" "$SYSROOT/include/c++" +ln -sf "$LIBCXX_PREFIX/lib/libc++.a" "$SYSROOT/lib/libc++.a" +ln -sf "$LIBCXX_PREFIX/lib/libc++abi.a" "$SYSROOT/lib/libc++abi.a" +ln -sf "$LIBCXX_PREFIX/lib/libc++.a" "$SYSROOT/lib/libstdc++.a" +rm -rf "$SYSROOT/include/c++/v1" +ln -sfn "$LIBCXX_PREFIX/include/c++/v1" "$SYSROOT/include/c++/v1" # Compose PKG_CONFIG_PATH for all deps so wasm32posix-configure's # pkg-config probes can find them in the cache instead of the sysroot. -DEP_PKG_CONFIG_PATH="$ZLIB_PREFIX/lib/pkgconfig:$SQLITE_PREFIX/lib/pkgconfig:$OPENSSL_PREFIX/lib/pkgconfig:$LIBXML2_PREFIX/lib/pkgconfig:$LIBICONV_PREFIX/lib/pkgconfig:$LIBZIP_PREFIX/lib/pkgconfig:$LIBCURL_PREFIX/lib/pkgconfig" +DEP_PKG_CONFIG_PATH="$ZLIB_PREFIX/lib/pkgconfig:$SQLITE_PREFIX/lib/pkgconfig:$OPENSSL_PREFIX/lib/pkgconfig:$LIBXML2_PREFIX/lib/pkgconfig:$LIBICONV_PREFIX/lib/pkgconfig:$LIBZIP_PREFIX/lib/pkgconfig:$LIBCURL_PREFIX/lib/pkgconfig:$ICU_PREFIX/lib/pkgconfig" # Compose -I and -L flags for defense-in-depth (autoconf raw probes). DEP_CPPFLAGS="-I$ZLIB_PREFIX/include -I$SQLITE_PREFIX/include -I$OPENSSL_PREFIX/include -I$LIBXML2_PREFIX/include -I$LIBICONV_PREFIX/include -I$LIBZIP_PREFIX/include -I$LIBCURL_PREFIX/include" @@ -137,6 +168,8 @@ REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBXML2_PREFIX" /usr/src/kandel REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBICONV_PREFIX" /usr/src/kandelo-deps/libiconv)" REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBZIP_PREFIX" /usr/src/kandelo-deps/libzip)" REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBCURL_PREFIX" /usr/src/kandelo-deps/libcurl)" +REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$ICU_PREFIX" /usr/src/kandelo-deps/icu)" +REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBCXX_PREFIX" /usr/src/kandelo-deps/libcxx)" echo "==> Downloading PHP $PHP_VERSION..." TARBALL="$WORK_DIR/php.tar.gz" @@ -813,6 +846,11 @@ if [ ! -f Makefile ]; then # state. The second -u group is the measured subset that base PHP does not # otherwise pull into php.wasm; --export-all then exposes those symbols to # the side module. The post-build import-closure test guards this list. + # The following group forces libc symbols intl.so imports but base PHP + # never references (allocator, wide-char, math, and the pthread mutex/ + # cond/TLS that ICU's UMutex uses). They must resolve to php.wasm's own + # musl so intl.so shares one libc state — one allocator, one pthread key + # table; without -u they never enter php.wasm and intl.so fails to load. # # -Wl,-z,stack-size=4194304: 4 MB wasm stack. The default wasm-ld # stack is 64 KB, which sits ~100 KB above PHP's `alloc_globals` @@ -851,6 +889,13 @@ if [ ! -f Makefile ]; then -u inet_pton -u inet_ntop -u sched_yield -u alarm -u basename \ -u OCSP_basic_verify -u OCSP_cert_status_str -u OCSP_crl_reason_str \ -u OCSP_response_status_str -u SSL_alert_desc_string_long \ +-u aligned_alloc -u div -u modf -u round -u tanhf \ +-u swprintf -u wcstod -u wcstof -u wcstol -u wcstold \ +-u wcstoll -u wcstoul -u wcstoull -u wmemchr -u wmemcmp \ +-u pthread_cond_broadcast -u pthread_cond_destroy -u pthread_cond_signal \ +-u pthread_cond_timedwait -u pthread_cond_wait -u pthread_detach \ +-u pthread_getspecific -u pthread_key_create -u pthread_self \ +-u pthread_setspecific \ -Wl,-z,stack-size=4194304" \ ZLIB_CFLAGS="$ZLIB_CFLAGS_VALUE" \ ZLIB_LIBS="$ZLIB_LIBS_VALUE" \ @@ -883,6 +928,7 @@ if [ ! -f Makefile ]; then --enable-cli \ --enable-fpm \ --enable-opcache \ + --enable-intl=shared \ --enable-mbstring \ --disable-mbregex \ --enable-ctype \ @@ -1177,6 +1223,63 @@ wasm32posix-cc -shared -fPIC -o "$BIN_DIR/zip.so" \ "$LIBZIP_PREFIX/lib/libzip.a" echo "==> zip.so: $(wc -c < "$BIN_DIR/zip.so") bytes" +# Build intl as a shared .so, same libtool workaround as opcache: make compiles +# the PIC objects under ext/intl/**/.libs/ but the bundled libtool can't emit the +# final .so on this target, so we link it with `wasm32posix-cc -shared`. intl +# statically absorbs ICU and libc++/libc++abi so neither enters php.wasm; the ICU +# common data stays out of the .so as icu.dat (loaded by intl-icu-data-loader.c). +echo "==> Building intl.so (PHP extension)..." +# Ask PHP's generated Makefile for the shared extension's complete object list +# and build those targets directly. Invoking intl.la would deliberately reach +# the unsupported libtool side-module link, while `|| true` would also hide a +# genuine compile failure and could link a partial extension. +INTL_LO_TARGETS_RAW="$(make -s --no-print-directory -f Makefile -f - print-intl-objects <<'MAKE' +.PHONY: print-intl-objects +print-intl-objects: + @printf '%s\n' $(shared_objects_intl) +MAKE +)" +mapfile -t INTL_LO_TARGETS <<< "$INTL_LO_TARGETS_RAW" +[ "${#INTL_LO_TARGETS[@]}" -gt 0 ] || { + echo "ERROR: PHP Makefile did not declare shared_objects_intl" >&2 + exit 1 +} +make -j"$(sysctl -n hw.ncpu 2>/dev/null || nproc)" \ + EXTRA_CFLAGS="$EXTRA_INC_LIBXML" \ + "${INTL_LO_TARGETS[@]}" + +# Compile the icu.dat loader (PIC) that feeds ICU its common data at dlopen. +wasm32posix-cc -fPIC -O2 -c "$SCRIPT_DIR/intl-icu-data-loader.c" \ + -I"$ICU_PREFIX/include" -o ext/intl/kandelo_icu_data_loader.o + +# Collect every PIC object libtool produced for ext/intl (top dir + the +# collator/, dateformat/, formatter/, … subdirs each have their own .libs/). +mapfile -t INTL_OBJS < <(find ext/intl -path '*/.libs/*.o' | sort) +[ "${#INTL_OBJS[@]}" -gt 0 ] || { echo "ERROR: no ext/intl PIC objects found — did 'make ext/intl/intl.la' compile?" >&2; exit 1; } +echo "==> linking intl.so from ${#INTL_OBJS[@]} objects + ICU static libs + libc++" + +# wasm-ld resolves archive back-references without --start-group, so the ICU +# archives are listed in dependency order (i18n -> io -> uc -> data), then +# libc++/libc++abi. A -shared PIC module requires every input to be PIC, so the +# libc++ PIC variants are named explicitly to win over the non-PIC sysroot ones. +wasm32posix-cc -shared -fPIC -Wl,--export=__tls_base -o "$BIN_DIR/intl.so" \ + "${INTL_OBJS[@]}" \ + ext/intl/kandelo_icu_data_loader.o \ + "$ICU_PREFIX/lib/libicui18n.a" \ + "$ICU_PREFIX/lib/libicuio.a" \ + "$ICU_PREFIX/lib/libicuuc.a" \ + "$ICU_PREFIX/lib/libicudata.a" \ + "$LIBCXX_PREFIX/lib/libc++-pic.a" \ + "$LIBCXX_PREFIX/lib/libc++abi-pic.a" +echo "==> intl.so: $(wc -c < "$BIN_DIR/intl.so") bytes" + +# Make the program package's runtime closure self-contained. ICU remains the +# build dependency/source of truth; PHP's archive carries these exact resolved +# bytes so normal local/fetched materialization can stage the declared guest +# file without inspecting the library cache. +cp "$ICU_PREFIX/share/icu.dat" "$BIN_DIR/icu.dat" +chmod 0644 "$BIN_DIR/icu.dat" + # Copy to bin/ with .wasm extension (needed for Vite browser demos) cp sapi/cli/php "$BIN_DIR/php.wasm" cp sapi/fpm/php-fpm "$BIN_DIR/php-fpm.wasm" @@ -1222,4 +1325,6 @@ if [ -z "${WASM_POSIX_DEP_OUT_DIR:-}" ]; then install_local_binary php "$BIN_DIR/phar.so" install_local_binary php "$BIN_DIR/zend_test.so" install_local_binary php "$BIN_DIR/zip.so" + install_local_binary php "$BIN_DIR/intl.so" + install_local_runtime_file php "$BIN_DIR/icu.dat" fi diff --git a/packages/registry/php/build.toml b/packages/registry/php/build.toml index 699816789..a733f0df5 100644 --- a/packages/registry/php/build.toml +++ b/packages/registry/php/build.toml @@ -1,10 +1,13 @@ script_path = "packages/registry/php/build-php.sh" inputs = [ "packages/registry/php/build-php.sh", + "packages/registry/php/intl-icu-data-loader.c", ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 12 +# Batch 3 extension landings advance this recipe once per byte-changing layer: +# zip (11), curl (12), then intl/ICU (13). +revision = 13 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/php/intl-icu-data-loader.c b/packages/registry/php/intl-icu-data-loader.c new file mode 100644 index 000000000..1f88fb168 --- /dev/null +++ b/packages/registry/php/intl-icu-data-loader.c @@ -0,0 +1,106 @@ +/* + * Feeds ICU its common data at intl.so load time. + * + * ICU ships as the standalone file icu.dat (see packages/registry/icu), but + * ICU's automatic loader only looks for the conventional icudt.dat + * name and would never find icu.dat on its own. So instead of embedding the + * ~30 MB blob in the .so, we hand it to ICU via udata_setCommonData() from a + * constructor: the side-module loader runs __wasm_call_ctors before PHP calls + * intl's MINIT, so the data is in place before any ICU service touches it. + * + * A missing/unreadable icu.dat is non-fatal at load (intl.so may be present + * without any code using intl) but stays loud: we warn to stderr and let ICU + * fail with U_MISSING_RESOURCE_ERROR when a service actually needs data, rather + * than silently succeeding. The PHP package runtime-file contract installs the + * exact matching bytes at /usr/lib/php/icu.dat on every host. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define KANDELO_ICU_DAT_PATH "/usr/lib/php/icu.dat" + +static void kandelo_intl_load_icu_data(void) __attribute__((constructor)); + +static void kandelo_intl_load_icu_data(void) { + const char *path = KANDELO_ICU_DAT_PATH; + + int fd = open(path, O_RDONLY); + if (fd < 0) { + fprintf(stderr, + "[intl] ICU data not loaded: cannot open %s. " + "intl functions will fail with U_MISSING_RESOURCE_ERROR. " + "Rebuild/materialize the complete PHP package runtime closure.\n", + path); + return; + } + + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size <= 0) { + fprintf(stderr, "[intl] ICU data not loaded: cannot stat %s.\n", path); + close(fd); + return; + } + + /* + * ICU keeps this pointer for the life of the process, so the buffer must + * outlive this function and is deliberately never freed. A plain read (not + * mmap) sidesteps the VFS's emulated mmap and runs once per process. + */ + size_t size = (size_t) st.st_size; + void *buf = malloc(size); + if (buf == NULL) { + fprintf(stderr, "[intl] ICU data not loaded: OOM reading %s (%zu bytes).\n", + path, size); + close(fd); + return; + } + + size_t off = 0; + while (off < size) { + ssize_t n = read(fd, (char *) buf + off, size - off); + if (n < 0) { + fprintf(stderr, "[intl] ICU data not loaded: read error on %s.\n", path); + free(buf); + close(fd); + return; + } + if (n == 0) break; + off += (size_t) n; + } + close(fd); + + if (off != size) { + fprintf(stderr, "[intl] ICU data not loaded: short read on %s (%zu/%zu).\n", + path, off, size); + free(buf); + return; + } + + UErrorCode status = U_ZERO_ERROR; + udata_setCommonData(buf, &status); + if (U_FAILURE(status)) { + fprintf(stderr, + "[intl] udata_setCommonData(%s) failed: %s. " + "(Likely an ICU library/data version mismatch.)\n", + path, u_errorName(status)); + free(buf); + return; + } + + /* Force ICU to validate/initialize now so version skew surfaces at load. */ + status = U_ZERO_ERROR; + u_init(&status); + if (U_FAILURE(status)) { + fprintf(stderr, "[intl] u_init after loading %s failed: %s.\n", + path, u_errorName(status)); + } +} diff --git a/packages/registry/php/package.toml b/packages/registry/php/package.toml index 12da91d21..1a34dd52e 100644 --- a/packages/registry/php/package.toml +++ b/packages/registry/php/package.toml @@ -5,7 +5,7 @@ kernel_abi = 19 # The rebuilt executables import `__wasm_posix_vm_interrupt_after`, which is # part of the ABI-19 host contract. Older package archives must not satisfy this # recipe; rebuild the CLI, FPM, and side modules for this exact ABI epoch. -depends_on = ["zlib@1.3.1", "openssl@3.3.2", "sqlite@3.49.1", "libxml2@2.13.8", "libiconv@1.17", "libzip@1.11.4", "libcurl@8.11.1"] +depends_on = ["zlib@1.3.1", "openssl@3.3.2", "sqlite@3.49.1", "libxml2@2.13.8", "libiconv@1.17", "libzip@1.11.4", "libcurl@8.11.1", "icu@74.2", "libcxx@21.1.7"] # wasm32 only: build-php.sh invokes the wasm32 SDK explicitly. If a wasm64 PHP # build is introduced later, make the recipe architecture-aware and validate # every CLI, FPM, and extension output before advertising that architecture. @@ -76,3 +76,17 @@ wasm = "zend_test.so" [[outputs]] name = "zip" wasm = "zip.so" + +# intl is a normal, opt-in PHP extension. It statically absorbs the ICU and +# PIC libc++ archives. PHP copies the exact resolved ICU common-data bytes into +# its own declared runtime closure for VFS consumers. +[[outputs]] +name = "intl" +wasm = "intl.so" + +# Non-Wasm runtime closure for intl. The resolver/archive mirrors this at +# programs//php/icu.dat; VFS consumers install it at the declared guest +# path instead of scanning a library cache or injecting a host-only override. +[[runtime_files]] +artifact = "icu.dat" +guest_path = "/usr/lib/php/icu.dat" diff --git a/packages/registry/php/test/browser/php-browser.spec.ts b/packages/registry/php/test/browser/php-browser.spec.ts index 9aef3208f..f0e81012e 100644 --- a/packages/registry/php/test/browser/php-browser.spec.ts +++ b/packages/registry/php/test/browser/php-browser.spec.ts @@ -8,46 +8,48 @@ * The browser harness runs multiple PHP tests (inline, file-based, extensions) * and reports all results as JSON in the #results element. * - * Run: npx playwright test --config packages/registry/php/test/browser/playwright.config.ts + * Run the default suite: + * npx playwright test --config packages/registry/php/test/browser/playwright.config.ts + * Both cases resolve their complete PHP package closure through the normal + * local/published binary mirrors. */ import { test, expect } from "@playwright/test"; import { existsSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { resolvePackageRuntimeFile } from "../../../../../scripts/package-runtime-file"; +import { tryResolveBinary } from "../../../../../host/src/binary-resolver"; const __dirname = dirname(fileURLToPath(import.meta.url)); const repoRoot = join(__dirname, "../../../../.."); -const hasKernelWasm = [ - join(repoRoot, "local-binaries/kernel.wasm"), - join(repoRoot, "binaries/kernel.wasm"), -].some((candidate) => existsSync(candidate)); -const hasPhpWasm = [ - join(repoRoot, "local-binaries/programs/wasm32/php/php.wasm"), - join(repoRoot, "binaries/programs/wasm32/php/php.wasm"), - join(repoRoot, "packages/registry/php/php-src/sapi/cli/php"), -].some((candidate) => existsSync(candidate)); -const hasZipSo = [ - join(repoRoot, "local-binaries/programs/wasm32/php/zip.so"), - join(repoRoot, "binaries/programs/wasm32/php/zip.so"), - join(repoRoot, "packages/registry/php/bin/zip.so"), -].some((candidate) => existsSync(candidate)); -const hasCurlSo = [ - join(repoRoot, "local-binaries/programs/wasm32/php/curl.so"), - join(repoRoot, "binaries/programs/wasm32/php/curl.so"), - join(repoRoot, "packages/registry/php/bin/curl.so"), -].some((candidate) => existsSync(candidate)); -const hasRootfsVfs = existsSync(join(repoRoot, "host/wasm/rootfs.vfs")); +const phpIcuRuntime = resolvePackageRuntimeFile(repoRoot, "php", "icu.dat"); +const hasKernelWasm = tryResolveBinary("kernel.wasm") != null; +const hasPhpWasm = phpIcuRuntime?.closureHostPaths.has("php/php.wasm") === true + || existsSync(join(repoRoot, "packages/registry/php/php-src/sapi/cli/php")); +const hasZipSo = phpIcuRuntime?.closureHostPaths.has("php/zip.so") === true + || existsSync(join(repoRoot, "packages/registry/php/bin/zip.so")); +const hasCurlSo = phpIcuRuntime?.closureHostPaths.has("php/curl.so") === true + || existsSync(join(repoRoot, "packages/registry/php/bin/curl.so")); +const hasIntlSo = phpIcuRuntime?.closureHostPaths.has("php/intl.so") === true + || existsSync(join(repoRoot, "packages/registry/php/bin/intl.so")); +if (hasIntlSo && !phpIcuRuntime) { + throw new Error( + "PHP intl.so is present but the declared php:icu.dat runtime file is not materialized", + ); +} +const hasRootfsVfs = tryResolveBinary("rootfs.vfs") != null + || tryResolveBinary("programs/rootfs.vfs") != null; test.skip(!hasKernelWasm, "kernel.wasm is not built or fetched"); test.skip(!hasPhpWasm, "php.wasm is not built or fetched"); -test.skip(!hasZipSo, "zip.so is not built or fetched"); -test.skip(!hasCurlSo, "curl.so is not built or fetched"); test.skip(!hasRootfsVfs, "rootfs.vfs is not built"); test("PHP CLI runs in the browser (inline, file, session, SQLite, fileinfo, XML, OpenSSL, extensions)", async ({ page, }) => { + test.skip(!hasZipSo, "zip.so is not built or fetched"); + test.skip(!hasCurlSo, "curl.so is not built or fetched"); const runtimeErrors: string[] = []; page.on("console", (msg) => { if (msg.type() === "error") runtimeErrors.push(`console: ${msg.text()}`); @@ -138,3 +140,51 @@ test("PHP CLI runs in the browser (inline, file, session, SQLite, fileinfo, XML, }); expect(runtimeErrors).toEqual([]); }); + +test("PHP intl and ICU data survive browser side-module fork replay", async ({ page }) => { + test.skip(!hasIntlSo, "intl.so is not built or fetched"); + + const runtimeErrors: string[] = []; + page.on("console", (msg) => { + if (msg.type() === "error") runtimeErrors.push(`console: ${msg.text()}`); + }); + page.on("pageerror", (error) => { + runtimeErrors.push(`pageerror: ${error.message}`); + }); + page.on("requestfailed", (request) => { + runtimeErrors.push( + `requestfailed: ${request.url()} ${request.failure()?.errorText ?? "failed"}`, + ); + }); + page.on("response", (response) => { + if (response.status() >= 400) { + runtimeErrors.push(`response: ${response.status()} ${response.url()}`); + } + }); + + await page.goto("/?intl=1"); + await page.waitForFunction( + () => { + const status = document.getElementById("status"); + return status?.textContent === "done" || status?.textContent === "error"; + }, + { timeout: 120_000 }, + ); + + const status = await page.locator("#status").textContent(); + const stderr = await page.locator("#stderr").textContent(); + const resultsText = await page.locator("#results").textContent(); + const exitCode = await page.locator("#exit-code").textContent(); + if (status === "error" || exitCode !== "0") { + console.log("INTL STDERR:", stderr); + console.log("INTL RESULTS:", resultsText); + } + + expect(status).toBe("done"); + expect(exitCode).toBe("0"); + expect(stderr).toBe(""); + const results = JSON.parse(resultsText!); + expect(results.intlFork).toContain("child=French"); + expect(results.intlFork).toContain("parent=French:apple,banana,cherry"); + expect(runtimeErrors).toEqual([]); +}); diff --git a/packages/registry/php/test/browser/run-php.ts b/packages/registry/php/test/browser/run-php.ts index 0c69ab3e7..4e026ecc2 100644 --- a/packages/registry/php/test/browser/run-php.ts +++ b/packages/registry/php/test/browser/run-php.ts @@ -23,6 +23,9 @@ const statusEl = document.getElementById("status")!; const resultsEl = document.getElementById("results")!; const PHP_PATH = "/usr/local/bin/php"; +const PHP_BROWSER_VFS_MAX_BYTES = 256 * 1024 * 1024; +const O_RDONLY = 0; +const VERIFY_CHUNK_BYTES = 64 * 1024; interface TestResult { stdout: string; @@ -30,33 +33,78 @@ interface TestResult { exitCode: number; } +interface BinaryFixture { + bytes: ArrayBuffer; + mode: number; +} + +function assertVfsBinaryRoundTrip( + fs: MemoryFileSystem, + path: string, + expected: Uint8Array, +): void { + const size = fs.stat(path).size; + if (size !== expected.byteLength) { + throw new Error( + `staged VFS file ${path} has ${size} bytes; expected ${expected.byteLength}`, + ); + } + + const fd = fs.open(path, O_RDONLY, 0); + const chunk = new Uint8Array(Math.min(VERIFY_CHUNK_BYTES, expected.byteLength)); + let offset = 0; + try { + while (offset < expected.byteLength) { + const wanted = Math.min(chunk.byteLength, expected.byteLength - offset); + const read = fs.read(fd, chunk, null, wanted); + if (read <= 0) { + throw new Error( + `short VFS verification read for ${path}: ${offset} of ${expected.byteLength}`, + ); + } + for (let i = 0; i < read; i++) { + if (chunk[i] !== expected[offset + i]) { + throw new Error( + `staged VFS file ${path} differs at byte ${offset + i}`, + ); + } + } + offset += read; + } + } finally { + fs.close(fd); + } +} + async function runPhp( phpBytes: ArrayBuffer, kernelBytes: ArrayBuffer, rootfsBytes: ArrayBuffer, files: Record, - binaryFiles: Record, argv: string[], + binaryFiles: Record = {}, ): Promise { let stdout = ""; let stderr = ""; const decoder = new TextDecoder(); const memfs = MemoryFileSystem.fromImage(new Uint8Array(rootfsBytes), { - maxByteLength: 256 * 1024 * 1024, + maxByteLength: PHP_BROWSER_VFS_MAX_BYTES, }); for (const dir of ["/tmp", "/root", "/home", "/dev"]) ensureDir(memfs, dir); memfs.chmod("/tmp", 0o777); memfs.chmod("/root", 0o700); ensureDirRecursive(memfs, "/usr/local/bin"); - writeVfsBinary(memfs, PHP_PATH, new Uint8Array(phpBytes)); + const phpData = new Uint8Array(phpBytes); + writeVfsBinary(memfs, PHP_PATH, phpData); for (const [path, content] of Object.entries(files)) { writeVfsFile(memfs, path, content); } - for (const [path, content] of Object.entries(binaryFiles)) { - const separator = path.lastIndexOf("/"); - if (separator > 0) ensureDirRecursive(memfs, path.slice(0, separator)); - writeVfsBinary(memfs, path, new Uint8Array(content)); + for (const [path, fixture] of Object.entries(binaryFiles)) { + ensureDirRecursive(memfs, path.slice(0, path.lastIndexOf("/")) || "/"); + const data = new Uint8Array(fixture.bytes); + writeVfsBinary(memfs, path, data, fixture.mode); + assertVfsBinaryRoundTrip(memfs, path, data); } const vfsImage = await memfs.saveImage(); @@ -99,76 +147,130 @@ async function runPhp( async function main() { try { - const [kernelBytes, rootfsBytes, phpBytes, zipBytes, curlBytes] = await Promise.all([ + const [kernelBytes, rootfsBytes, phpBytes] = await Promise.all([ fetch(kernelWasmUrl).then((r) => r.arrayBuffer()), fetch(rootfsVfsUrl).then((r) => r.arrayBuffer()), fetch("/php-artifacts/php.wasm").then((r) => r.arrayBuffer()), + ]); + + // Run the dedicated intl/fork contract only for its Playwright case. The + // server exposes bytes and installation metadata from PHP's declared + // runtime closure; this browser path does not duplicate the ICU guest path. + if (new URL(window.location.href).searchParams.has("intl")) { + const [intlBytes, icuDataBytes, icuContract] = await Promise.all([ + fetch("/php-artifacts/intl.so").then((r) => r.arrayBuffer()), + fetch("/php-artifacts/icu.dat").then((r) => r.arrayBuffer()), + fetch("/php-runtime-files/icu.dat").then(async (response) => { + if (!response.ok) throw new Error(await response.text()); + return await response.json() as { + artifact: string; + guestPath: string; + mode: number; + }; + }), + ]); + const intlResult = await runPhp( + phpBytes, + kernelBytes, + rootfsBytes, + {}, + ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=intl.so", "-r", ` + $before = Locale::getDisplayLanguage("fr", "en"); + $pid = pcntl_fork(); + if ($pid < 0) { fwrite(STDERR, "fork-failed"); exit(20); } + if ($pid === 0) { + $child = Locale::getDisplayLanguage("fr", "en"); + echo "child=" . $child . "\\n"; + exit($child === "French" ? 0 : 21); + } + $status = 0; + $waited = pcntl_waitpid($pid, $status); + $c = new Collator("en_US"); + $a = ["banana", "apple", "cherry"]; + $c->sort($a); + echo "parent=" . $before . ":" . implode(",", $a) . "\\n"; + if ($waited !== $pid || !pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { + fwrite(STDERR, "child-status=" . $status); + exit(22); + } + `], + { + "/usr/lib/php/extensions/intl.so": { bytes: intlBytes, mode: 0o755 }, + [icuContract.guestPath]: { bytes: icuDataBytes, mode: icuContract.mode }, + }, + ); + stdoutEl.textContent = intlResult.stdout; + stderrEl.textContent = intlResult.stderr; + exitCodeEl.textContent = String(intlResult.exitCode); + resultsEl.textContent = JSON.stringify({ intlFork: intlResult.stdout.trim() }); + statusEl.textContent = "done"; + return; + } + + const [zipBytes, curlBytes] = await Promise.all([ fetch("/php-artifacts/zip.so").then((r) => r.arrayBuffer()), fetch("/php-artifacts/curl.so").then((r) => r.arrayBuffer()), ]); + const binaryFiles: Record = { + "/usr/lib/php/extensions/zip.so": { bytes: zipBytes, mode: 0o755 }, + "/usr/lib/php/extensions/curl.so": { bytes: curlBytes, mode: 0o755 }, + }; const files = { "/home/script.php": '', "/home/ext_test.php": ' mb_strlen("hello"), "ctype" => ctype_alpha("hello") ? "yes" : "no"]); ?>', }; - const binaryFiles = { - "/usr/lib/php/extensions/zip.so": zipBytes, - "/usr/lib/php/extensions/curl.so": curlBytes, - }; // Test 1: Hello World (inline) - const r1 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", 'echo "Hello World\n";']); + const r1 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", 'echo "Hello World\n";'], binaryFiles); // Test 2: File-based execution - const r2 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "/home/script.php"]); + const r2 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "/home/script.php"], binaryFiles); // Test 3: Extensions (mbstring + ctype) - const r3 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "/home/ext_test.php"]); + const r3 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "/home/ext_test.php"], binaryFiles); // Test 4: Session - const r4 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", 'session_start(); echo strlen(session_id()) > 0 ? "session-ok" : "fail";']); + const r4 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", 'session_start(); echo strlen(session_id()) > 0 ? "session-ok" : "fail";'], binaryFiles); // Test 5: SQLite3 in-memory - const r5 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$db=new SQLite3(":memory:");$db->exec("CREATE TABLE t(v TEXT)");$db->exec("INSERT INTO t VALUES(\'sqlite-ok\')");echo $db->querySingle("SELECT v FROM t");']); + const r5 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$db=new SQLite3(":memory:");$db->exec("CREATE TABLE t(v TEXT)");$db->exec("INSERT INTO t VALUES(\'sqlite-ok\')");echo $db->querySingle("SELECT v FROM t");'], binaryFiles); // Test 6: fileinfo - const r6 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$f=new finfo(FILEINFO_MIME_TYPE);echo $f->buffer("GIF89a");']); + const r6 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$f=new finfo(FILEINFO_MIME_TYPE);echo $f->buffer("GIF89a");'], binaryFiles); // Test 7: SimpleXML - const r7 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$x=new SimpleXMLElement("xml-ok");echo $x->i;']); + const r7 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$x=new SimpleXMLElement("xml-ok");echo $x->i;'], binaryFiles); // Test 8: rootfs OpenSSL defaults are present and key + CSR generation succeeds. - const r8 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$k=openssl_pkey_new();$c=$k?openssl_csr_new(["commonName"=>"kandelo.test"],$k):false;if(!$k||!$c){while($e=openssl_error_string()){fwrite(STDERR,$e."\\n");}exit(1);}echo "openssl-defaults-ok";']); + const r8 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$k=openssl_pkey_new();$c=$k?openssl_csr_new(["commonName"=>"kandelo.test"],$k):false;if(!$k||!$c){while($e=openssl_error_string()){fwrite(STDERR,$e."\\n");}exit(1);}echo "openssl-defaults-ok";'], binaryFiles); // Test 9: load the packaged zip side module from the kernel-owned VFS and // prove that a DEFLATE archive survives close/reopen in the browser host. - const r9 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, + const r9 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=zip.so", "-r", - '$p="/tmp/browser-zip-smoke.zip";$z=new ZipArchive;if($z->open($p,ZipArchive::CREATE|ZipArchive::OVERWRITE)!==true)exit(10);if(!$z->addFromString("hello.txt","browser-zip-ok"))exit(11);if(!$z->setCompressionName("hello.txt",ZipArchive::CM_DEFLATE))exit(12);if(!$z->close())exit(13);$r=new ZipArchive;if($r->open($p)!==true)exit(14);$s=$r->statName("hello.txt");if($s===false||$s["comp_method"]!==ZipArchive::CM_DEFLATE)exit(15);echo $r->getFromName("hello.txt");$r->close();']); + '$p="/tmp/browser-zip-smoke.zip";$z=new ZipArchive;if($z->open($p,ZipArchive::CREATE|ZipArchive::OVERWRITE)!==true)exit(10);if(!$z->addFromString("hello.txt","browser-zip-ok"))exit(11);if(!$z->setCompressionName("hello.txt",ZipArchive::CM_DEFLATE))exit(12);if(!$z->close())exit(13);$r=new ZipArchive;if($r->open($p)!==true)exit(14);$s=$r->statName("hello.txt");if($s===false||$s["comp_method"]!==ZipArchive::CM_DEFLATE)exit(15);echo $r->getFromName("hello.txt");$r->close();'], binaryFiles); // Test 10: load the packaged curl side module from the same browser VFS - // path and call into the linked libcurl implementation. The next test - // exercises browser TCP; this one isolates the PHP/dlopen side-module - // contract on the real browser host. - const r10 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, + // path and call into the linked libcurl implementation. + const r10 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=curl.so", "-r", - 'echo json_encode(["loaded"=>extension_loaded("curl"),"version"=>curl_version()["version"],"constant"=>defined("CURLOPT_URL"),"handle"=>is_object(curl_init())]);']); + 'echo json_encode(["loaded"=>extension_loaded("curl"),"version"=>curl_version()["version"],"constant"=>defined("CURLOPT_URL"),"handle"=>is_object(curl_init())]);'], binaryFiles); - // Test 11: serve one HTTP response on the kernel's loopback network and - // fetch it with libcurl from a fork child. Loading curl.so before fork - // also exercises browser-side dlopen replay in the child process. - const r11 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, + // Test 11: exercise browser TCP with libcurl from a fork child. Loading + // curl.so before fork also verifies browser-side dlopen replay. + const r11 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=curl.so", "-r", - '$server=stream_socket_server("tcp://127.0.0.1:0",$errno,$error);if($server===false){fwrite(STDERR,"$errno:$error");exit(10);}$address=stream_socket_get_name($server,false);$pid=pcntl_fork();if($pid<0){fwrite(STDERR,"fork failed");exit(11);}if($pid===0){fclose($server);$ch=curl_init("http://$address/probe");curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);curl_setopt($ch,CURLOPT_TIMEOUT,10);$body=curl_exec($ch);if($body===false){fwrite(STDERR,curl_error($ch));exit(12);}echo json_encode(["body"=>$body,"status"=>curl_getinfo($ch,CURLINFO_RESPONSE_CODE)]);exit(0);}$client=stream_socket_accept($server,10);if($client===false){fwrite(STDERR,"accept failed");exit(13);}$request="";while(!str_contains($request,"\\r\\n\\r\\n")){$chunk=fread($client,4096);if($chunk===false||$chunk===""){fwrite(STDERR,"request read failed");exit(14);}$request.=$chunk;}fwrite($client,"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 16\\r\\nConnection: close\\r\\n\\r\\nkandelo-curl-ok\\n");fclose($client);fclose($server);pcntl_waitpid($pid,$status);if(!pcntl_wifexited($status)||pcntl_wexitstatus($status)!==0)exit(15);']); + '$server=stream_socket_server("tcp://127.0.0.1:0",$errno,$error);if($server===false){fwrite(STDERR,"$errno:$error");exit(10);}$address=stream_socket_get_name($server,false);$pid=pcntl_fork();if($pid<0){fwrite(STDERR,"fork failed");exit(11);}if($pid===0){fclose($server);$ch=curl_init("http://$address/probe");curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);curl_setopt($ch,CURLOPT_TIMEOUT,10);$body=curl_exec($ch);if($body===false){fwrite(STDERR,curl_error($ch));exit(12);}echo json_encode(["body"=>$body,"status"=>curl_getinfo($ch,CURLINFO_RESPONSE_CODE)]);exit(0);}$client=stream_socket_accept($server,10);if($client===false){fwrite(STDERR,"accept failed");exit(13);}$request="";while(!str_contains($request,"\\r\\n\\r\\n")){$chunk=fread($client,4096);if($chunk===false||$chunk===""){fwrite(STDERR,"request read failed");exit(14);}$request.=$chunk;}fwrite($client,"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 16\\r\\nConnection: close\\r\\n\\r\\nkandelo-curl-ok\\n");fclose($client);fclose($server);pcntl_waitpid($pid,$status);if(!pcntl_wifexited($status)||pcntl_wexitstatus($status)!==0)exit(15);'], binaryFiles); const results = { hello: r1.stdout.trim(), diff --git a/packages/registry/php/test/browser/vite.config.ts b/packages/registry/php/test/browser/vite.config.ts index b7a649210..87f4aa688 100644 --- a/packages/registry/php/test/browser/vite.config.ts +++ b/packages/registry/php/test/browser/vite.config.ts @@ -2,10 +2,14 @@ import { fileURLToPath } from "url"; import path from "path"; import fs from "fs"; import type { Plugin } from "vite"; +import { resolvePackageRuntimeFile } from "../../../../../scripts/package-runtime-file"; +import { tryResolveBinary } from "../../../../../host/src/binary-resolver"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(__dirname, "../../../../.."); +const phpIcuRuntime = resolvePackageRuntimeFile(repoRoot, "php", "icu.dat"); + function resolveKernelArtifactsAlias(): Plugin { const KERNEL = "@kernel-wasm"; const ROOTFS = "@rootfs-vfs"; @@ -18,24 +22,21 @@ function resolveKernelArtifactsAlias(): Plugin { const query = queryIdx === -1 ? "" : source.slice(queryIdx); if (pathPart === KERNEL) { - const candidates = [ - path.resolve(repoRoot, "local-binaries/kernel.wasm"), - path.resolve(repoRoot, "binaries/kernel.wasm"), - ]; - const file = candidates.find((candidate) => fs.existsSync(candidate)); + const file = tryResolveBinary("kernel.wasm"); if (file) return file + query; this.error( - "kernel.wasm not found. Run `bash build.sh` from the repo root.\n" + - ` Looked at: ${candidates.join("\n Looked at: ")}`, + "kernel.wasm was not accepted from the standard local, fetched, or packaged locations. " + + "Run `bash build.sh` from the repo root or fetch package binaries.", ); } if (pathPart === ROOTFS) { - const file = path.resolve(repoRoot, "host/wasm/rootfs.vfs"); - if (fs.existsSync(file)) return file + query; + const file = tryResolveBinary("rootfs.vfs") + ?? tryResolveBinary("programs/rootfs.vfs"); + if (file) return file + query; this.error( - "rootfs.vfs not found. Run `bash build.sh` from the repo root.\n" + - ` Looked at: ${file}`, + "rootfs.vfs was not accepted from the standard local, fetched, or packaged locations. " + + "Run `bash build.sh` from the repo root or fetch the rootfs package.", ); } @@ -45,15 +46,26 @@ function resolveKernelArtifactsAlias(): Plugin { } function findPhpArtifact(name: string): string | null { + if (phpIcuRuntime) { + // All PHP artifacts served by this harness come from the single complete + // package tier selected with icu.dat. Never let per-file resolver priority + // compose local and fetched builds. + return phpIcuRuntime.closureHostPaths.get(`php/${name}`) ?? null; + } + if (name === "icu.dat") return null; const candidates = [ - path.resolve(repoRoot, "local-binaries/programs/wasm32/php", name), - path.resolve(repoRoot, "binaries/programs/wasm32/php", name), path.resolve(__dirname, "../../bin", name), + ...(name === "php.wasm" + ? [path.resolve(__dirname, "../../php-src/sapi/cli/php")] + : []), ]; - if (name === "php.wasm") { - candidates.push(path.resolve(__dirname, "../../php-src/sapi/cli/php")); - } - return candidates.find((candidate) => fs.existsSync(candidate)) ?? null; + return candidates.find((candidate) => { + try { + return fs.statSync(candidate).isFile(); + } catch { + return false; + } + }) ?? null; } function servePhpArtifacts(): Plugin { @@ -61,13 +73,38 @@ function servePhpArtifacts(): Plugin { name: "serve-php-artifacts", configureServer(server) { server.middlewares.use((req, res, next) => { + if (req.url === "/php-runtime-files/icu.dat") { + if (!phpIcuRuntime) { + res.statusCode = 404; + res.end("the declared PHP icu.dat runtime file is not materialized"); + return; + } + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ + artifact: phpIcuRuntime.artifact, + guestPath: phpIcuRuntime.guestPath, + mode: phpIcuRuntime.mode, + })); + return; + } const prefix = "/php-artifacts/"; if (!req.url?.startsWith(prefix)) { next(); return; } - const name = decodeURIComponent(req.url.slice(prefix.length)); - if (!/^[A-Za-z0-9._-]+$/.test(name)) { + let name: string; + try { + name = decodeURIComponent(req.url.slice(prefix.length)); + } catch { + res.statusCode = 400; + res.end("invalid percent-encoding in PHP artifact name"); + return; + } + if ( + name === "." + || name === ".." + || !/^[A-Za-z0-9._-]+$/.test(name) + ) { res.statusCode = 400; res.end("invalid PHP artifact name"); return; @@ -75,10 +112,10 @@ function servePhpArtifacts(): Plugin { const artifact = findPhpArtifact(name); if (!artifact) { res.statusCode = 404; - res.end( - `${name} not found. Run \`bash packages/registry/php/build-php.sh\` ` + - "or fetch package binaries.", - ); + res.end(name === "icu.dat" + ? "the declared PHP icu.dat runtime file is not materialized" + : `${name} not found. Run \`bash packages/registry/php/build-php.sh\` ` + + "or fetch package binaries."); return; } const data = fs.readFileSync(artifact); diff --git a/packages/registry/php/test/php-intl.test.ts b/packages/registry/php/test/php-intl.test.ts new file mode 100644 index 000000000..9999a4348 --- /dev/null +++ b/packages/registry/php/test/php-intl.test.ts @@ -0,0 +1,191 @@ +import { beforeAll, describe, it, expect } from "vitest"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCentralizedProgram } from "../../../../host/test/centralized-test-helper"; +import { tryResolveBinary } from "../../../../host/src/binary-resolver"; +import { MemoryFileSystem } from "../../../../host/src/vfs/memory-fs"; +import { + ensureDirRecursive, + writeVfsBinary, +} from "../../../../host/src/vfs/image-helpers"; +import { resolvePackageRuntimeFile } from "../../../../scripts/package-runtime-file"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const icuRuntime = resolvePackageRuntimeFile( + join(__dirname, "../../../.."), + "php", + "icu.dat", +); + +// intl is a RUNTIME-OPTIONAL side module: base php.wasm is built with +// --enable-intl=shared, so intl is NOT compiled in. intl.so is loaded on +// demand via `extension=intl.so`, and pulls its ICU common data from the +// separate icu.dat at runtime (udata_setCommonData in intl-icu-data-loader.c). +const phpBinaryPath = + icuRuntime?.closureHostPaths.get("php/php.wasm") ?? + join(__dirname, "../php-src/sapi/cli/php"); +const intlSoPath = icuRuntime?.closureHostPaths.get("php/intl.so"); +const rootfsPath = + tryResolveBinary("rootfs.vfs") ?? + tryResolveBinary("programs/rootfs.vfs") ?? + join(__dirname, "../../../../host/wasm/rootfs.vfs"); +const INTL_GUEST_PATH = "/usr/lib/php/extensions/intl.so"; +const PHP_INTL_VFS_MAX_BYTES = 256 * 1024 * 1024; +const O_RDONLY = 0; + +if (intlSoPath && !icuRuntime) { + throw new Error( + "PHP intl.so is present but the declared php:icu.dat runtime file is not materialized", + ); +} + +const READY = existsSync(phpBinaryPath) + && intlSoPath != null + && existsSync(rootfsPath); +let intlRootfsImage: Uint8Array; + +function readVfsBinary(fs: MemoryFileSystem, path: string): Uint8Array { + const size = fs.stat(path).size; + const bytes = new Uint8Array(size); + const fd = fs.open(path, O_RDONLY, 0); + let offset = 0; + try { + while (offset < bytes.length) { + const read = fs.read( + fd, + bytes.subarray(offset), + null, + bytes.length - offset, + ); + if (read <= 0) { + throw new Error( + `short VFS read for ${path}: ${offset} of ${bytes.length}`, + ); + } + offset += read; + } + } finally { + fs.close(fd); + } + return bytes; +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { + beforeAll(async () => { + const fs = MemoryFileSystem + .fromImage(new Uint8Array(readFileSync(rootfsPath))) + .rebaseToNewFileSystem(PHP_INTL_VFS_MAX_BYTES); + ensureDirRecursive(fs, dirname(INTL_GUEST_PATH)); + ensureDirRecursive(fs, dirname(icuRuntime!.guestPath)); + writeVfsBinary( + fs, + INTL_GUEST_PATH, + new Uint8Array(readFileSync(intlSoPath!)), + 0o755, + ); + const icuBytes = new Uint8Array(readFileSync(icuRuntime!.hostPath)); + writeVfsBinary( + fs, + icuRuntime!.guestPath, + icuBytes, + icuRuntime!.mode, + ); + const stagedIcuBytes = readVfsBinary(fs, icuRuntime!.guestPath); + expect(stagedIcuBytes.byteLength).toBe(icuBytes.byteLength); + expect(sha256(stagedIcuBytes)).toBe(sha256(icuBytes)); + intlRootfsImage = await fs.saveImage(); + }); + + // Proves the base binary is genuinely ICU-free / intl-free: intl only + // appears when explicitly loaded. This is the whole point of the design. + it("base php.wasm does NOT include intl", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-m"], + rootfsImage: intlRootfsImage, + }); + expect(exitCode).toBe(0); + expect(stdout.toLowerCase()).not.toContain("intl"); + }, 60_000); + + it("loads intl.so at runtime via extension=", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", + 'echo extension_loaded("intl") ? "intl-loaded" : "intl-missing";'], + rootfsImage: intlRootfsImage, + }); + expect(stdout).toContain("intl-loaded"); + expect(exitCode).toBe(0); + }, 60_000); + + // Exercises real ICU data (locale display names) to prove icu.dat is + // actually loaded and usable, not just that the module registered. + it("intl uses ICU data (Locale::getDisplayLanguage)", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", + 'echo Locale::getDisplayLanguage("fr", "en");'], + rootfsImage: intlRootfsImage, + }); + expect(stdout).toContain("French"); + expect(exitCode).toBe(0); + }, 60_000); + + // Collator sorting is a core ICU service that requires collation data. + it("intl Collator sorts with locale rules", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", ` + $c = new Collator("en_US"); + $a = ["banana", "apple", "cherry"]; + $c->sort($a); + echo implode(",", $a); + `], + rootfsImage: intlRootfsImage, + }); + expect(stdout).toContain("apple,banana,cherry"); + expect(exitCode).toBe(0); + }, 60_000); + + // dlopen occurs before fork. The child therefore replays intl.so without + // rerunning constructors and must retain ICU's common-data pointer from the + // copied process image; exercising ICU in both branches verifies that real + // side-module replay contract rather than only the synthetic linker shape. + it("intl and icu.dat survive pcntl_fork replay", async () => { + const { stdout, stderr, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", ` + $before = Locale::getDisplayLanguage("fr", "en"); + $pid = pcntl_fork(); + if ($pid < 0) { fwrite(STDERR, "fork-failed"); exit(20); } + if ($pid === 0) { + $child = Locale::getDisplayLanguage("fr", "en"); + echo "child=" . $child . "\\n"; + exit($child === "French" ? 0 : 21); + } + $status = 0; + $waited = pcntl_waitpid($pid, $status); + $c = new Collator("en_US"); + $a = ["banana", "apple", "cherry"]; + $c->sort($a); + echo "parent=" . $before . ":" . implode(",", $a) . "\\n"; + if ($waited !== $pid || !pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { + fwrite(STDERR, "child-status=" . $status); + exit(22); + } + `], + rootfsImage: intlRootfsImage, + }); + expect(stderr).toBe(""); + expect(stdout).toContain("child=French"); + expect(stdout).toContain("parent=French:apple,banana,cherry"); + expect(exitCode).toBe(0); + }, 60_000); +}); diff --git a/scripts/install-local-binary.sh b/scripts/install-local-binary.sh index dfd7a9776..daac62aa0 100755 --- a/scripts/install-local-binary.sh +++ b/scripts/install-local-binary.sh @@ -58,6 +58,14 @@ install_local_binary() { echo "install_local_binary: source file not found: $src" >&2 return 1 fi + local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" + case "$arch" in + wasm32|wasm64) ;; + *) + echo "install_local_binary: unsupported target arch '$arch' (expected wasm32 or wasm64)" >&2 + return 2 + ;; + esac # Repo root must be derived from this helper, not from the caller's # current directory: package builds often `cd` into an upstream git @@ -111,8 +119,6 @@ install_local_binary() { ;; esac - local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" - # Take everything from the FIRST dot in the source basename onward # so compound extensions like `.vfs.zst` round-trip intact (matches # the resolver's `place_binaries_symlinks` extension handling). @@ -177,3 +183,56 @@ install_local_binary() { echo " installed $resolver_dest (resolver scratch)" fi } + +# Install a declared non-Wasm `[[runtime_files]]` artifact into the same local +# resolver mirror used by published archives. This is intentionally separate +# from install_local_binary: data files must not pass Wasm/fork guards or be +# described as executable outputs. +install_local_runtime_file() { + local program="$1" + local src="$2" + local artifact="${3:-}" + + if [ -z "$program" ] || [ -z "$src" ]; then + echo "install_local_runtime_file: usage: install_local_runtime_file [artifact]" >&2 + return 2 + fi + if [ ! -f "$src" ] || [ -L "$src" ]; then + echo "install_local_runtime_file: source must be a regular non-symlink file: $src" >&2 + return 1 + fi + local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" + case "$arch" in + wasm32|wasm64) ;; + *) + echo "install_local_runtime_file: unsupported target arch '$arch' (expected wasm32 or wasm64)" >&2 + return 2 + ;; + esac + + local repo_root + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + local host_target + host_target="$(rustc -vV 2>/dev/null | awk '/^host/ {print $2}')" + if [ -z "$host_target" ]; then + echo "install_local_runtime_file: rustc did not report a host target" >&2 + return 1 + fi + local src_basename + src_basename="$(basename "$src")" + artifact="${artifact:-$src_basename}" + local rel + rel="$(cd "$repo_root" && \ + env -u CC -u CXX -u AR -u RANLIB -u CFLAGS -u CXXFLAGS -u CPPFLAGS -u LDFLAGS \ + cargo run -p xtask --target "$host_target" --quiet -- \ + build-deps runtime-file-path "$program" "$artifact")" || return 1 + if [ -z "$rel" ]; then + echo "install_local_runtime_file: manifest lookup returned an empty path" >&2 + return 1 + fi + + local dest="$repo_root/local-binaries/programs/$arch/$rel" + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" + echo " installed $dest" +} diff --git a/scripts/package-runtime-file.ts b/scripts/package-runtime-file.ts new file mode 100644 index 000000000..2045a36b4 --- /dev/null +++ b/scripts/package-runtime-file.ts @@ -0,0 +1,147 @@ +/** + * Repo-side bridge from package.toml `[[runtime_files]]` to VFS/test builders. + * + * Runtime-file metadata is a build/materialization contract, not a host-runtime + * API: published browser/rootfs images contain the installed bytes already. + * Repo tools query xtask so guest paths and modes are never duplicated in + * TypeScript fixtures. + */ +import { execFileSync } from "node:child_process"; +import { isAbsolute } from "node:path"; +import { tryResolveBinarySet } from "../host/src/binary-resolver"; + +export interface PackageRuntimeFileContract { + artifact: string; + guestPath: string; + mode: number; + mirrorPath: string; + /** Every program output + runtime file declared by this package. */ + closureMirrorPaths: string[]; +} + +export interface ResolvedPackageRuntimeFile extends PackageRuntimeFileContract { + hostPath: string; + /** Host paths keyed by resolver mirror path, all from one provenance root. */ + closureHostPaths: ReadonlyMap; +} + +let cachedHostTarget: string | undefined; + +function hostTarget(): string { + if (cachedHostTarget) return cachedHostTarget; + const output = execFileSync("rustc", ["-vV"], { encoding: "utf8" }); + const target = output.match(/^host:\s*(\S+)$/m)?.[1]; + if (!target) throw new Error("rustc -vV did not report a host target"); + cachedHostTarget = target; + return target; +} + +function hostCargoEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const name of [ + "CC", + "CXX", + "AR", + "RANLIB", + "CFLAGS", + "CXXFLAGS", + "CPPFLAGS", + "LDFLAGS", + ]) { + delete env[name]; + } + return env; +} + +export function readPackageRuntimeFileContract( + repoRoot: string, + packageName: string, + artifact: string, +): PackageRuntimeFileContract { + const raw = execFileSync( + "cargo", + [ + "run", + "-p", + "xtask", + "--target", + hostTarget(), + "--quiet", + "--", + "build-deps", + "runtime-file-metadata", + packageName, + artifact, + ], + { cwd: repoRoot, encoding: "utf8", env: hostCargoEnv() }, + ).trim(); + return parsePackageRuntimeFileContract(raw, packageName, artifact); +} + +/** Parse and validate xtask's structured runtime-file metadata. */ +export function parsePackageRuntimeFileContract( + raw: string, + packageName: string, + artifact: string, +): PackageRuntimeFileContract { + const parsed = JSON.parse(raw) as Record; + const contract: PackageRuntimeFileContract = { + artifact: parsed.artifact as string, + guestPath: parsed.guest_path as string, + mode: parsed.mode as number, + mirrorPath: parsed.mirror_path as string, + closureMirrorPaths: parsed.closure_mirror_paths as string[], + }; + const validMirrorPath = (value: unknown): value is string => + typeof value === "string" + && !isAbsolute(value) + && !value.includes("\\") + && !value.includes("\0") + && value + .split("/") + .every((part) => Boolean(part) && part !== "." && part !== ".."); + if ( + contract.artifact !== artifact + || typeof contract.guestPath !== "string" + || !contract.guestPath.startsWith("/") + || !Number.isInteger(contract.mode) + || contract.mode < 0 + || contract.mode > 0o777 + || !validMirrorPath(contract.mirrorPath) + || !Array.isArray(contract.closureMirrorPaths) + || contract.closureMirrorPaths.length === 0 + || !contract.closureMirrorPaths.every(validMirrorPath) + || new Set(contract.closureMirrorPaths).size !== contract.closureMirrorPaths.length + || !contract.closureMirrorPaths.includes(contract.mirrorPath) + ) { + throw new Error( + `invalid runtime-file metadata for ${packageName}:${artifact}: ${raw}`, + ); + } + return contract; +} + +export function resolvePackageRuntimeFile( + repoRoot: string, + packageName: string, + artifact: string, +): ResolvedPackageRuntimeFile | undefined { + const contract = readPackageRuntimeFileContract(repoRoot, packageName, artifact); + const hostPaths = tryResolveBinarySet( + contract.closureMirrorPaths.map((mirrorPath) => `programs/${mirrorPath}`), + ); + if (!hostPaths) return undefined; + const closureHostPaths = new Map( + contract.closureMirrorPaths.map((mirrorPath, index) => [ + mirrorPath, + hostPaths[index], + ]), + ); + const hostPath = closureHostPaths.get(contract.mirrorPath); + if (!hostPath) { + throw new Error( + `resolved package closure omitted ${packageName}:${contract.mirrorPath}`, + ); + } + return { ...contract, hostPath, closureHostPaths }; +} diff --git a/scripts/run-php-upstream-tests.ts b/scripts/run-php-upstream-tests.ts index d83bf9b62..3e39a135d 100644 --- a/scripts/run-php-upstream-tests.ts +++ b/scripts/run-php-upstream-tests.ts @@ -39,10 +39,10 @@ import { sep, } from "node:path"; import { NodeKernelHost } from "../host/src/node-kernel-host"; -import { tryResolveBinary } from "../host/src/binary-resolver"; import { ABI_SYSCALL_NAMES } from "../host/src/generated/abi"; import { ensureSourceExtract } from "../images/vfs/scripts/source-extract-helper"; import { preparePhpTestFixtures } from "../images/vfs/scripts/php-test-fixtures"; +import { resolvePackageRuntimeFile } from "./package-runtime-file"; const REPO_ROOT = resolve(new URL(".", import.meta.url).pathname, ".."); const LOCAL_PHP_SRC = join(REPO_ROOT, "packages/registry/php/php-src"); @@ -55,6 +55,7 @@ const BROWSER_DIR = join(REPO_ROOT, "apps/browser-demos"); const VITE_HOST = "127.0.0.1"; const VITE_PORT = Number(process.env.PHP_TEST_VITE_PORT ?? 5201); const BROWSER_EXTENSION_DIR = "/usr/lib/php/extensions"; +const PHP_ICU_RUNTIME = resolvePackageRuntimeFile(REPO_ROOT, "php", "icu.dat"); const RUN_TESTS_BASE_INI = [ "output_handler=", "open_basedir=", @@ -217,7 +218,7 @@ async function withTimeout( function resolvePhpBinary(): string { const candidate = process.env.PHP_WASM ?? - tryResolveBinary("programs/php/php.wasm") ?? + PHP_ICU_RUNTIME?.closureHostPaths.get("php/php.wasm") ?? join(LOCAL_PHP_SRC, "sapi/cli/php"); if (!candidate || !existsSync(candidate)) { throw new Error( @@ -230,7 +231,7 @@ function resolvePhpBinary(): string { function resolvePhpFpmBinary(phpPath: string): string | null { const explicit = process.env.PHP_FPM_WASM; if (explicit) return resolve(explicit); - const resolved = tryResolveBinary("programs/php/php-fpm.wasm"); + const resolved = PHP_ICU_RUNTIME?.closureHostPaths.get("php/php-fpm.wasm"); if (resolved) return resolved; const sibling = join(dirname(phpPath), "php-fpm.wasm"); return existsSync(sibling) ? sibling : null; @@ -547,10 +548,24 @@ function sharedExtensionPathsForPhp(phpPath: string): Map { } } } + // When the declared package closure is materialized, use its selected + // side-module paths directly. They were resolved together with php.wasm and + // icu.dat from one complete provenance tier. + for (const name of [ + "opcache", + "curl", + "phar", + "zend_test", + "zip", + "intl", + ]) { + const resolved = PHP_ICU_RUNTIME?.closureHostPaths.get(`php/${name}.so`); + if (resolved) out.set(name, resolved); + } const phpDir = dirname(phpPath); const opcachePath = process.env.PHP_OPCACHE_SO ?? - tryResolveBinary("programs/php/opcache.so") ?? + PHP_ICU_RUNTIME?.closureHostPaths.get("php/opcache.so") ?? join(phpDir, "opcache.so"); if (opcachePath && existsSync(opcachePath)) out.set("opcache", opcachePath); return out; @@ -989,6 +1004,26 @@ class NodePhpRunner implements PhpRunner { copyFileSync(srcPath, destPath); chmodSync(destPath, 0o755); } + if (this.sharedExtensionPaths.has("intl")) { + if (!PHP_ICU_RUNTIME) { + throw new Error( + "intl.so is available but the declared php:icu.dat runtime file is not materialized", + ); + } + const mountPoint = "/usr/lib"; + if (!PHP_ICU_RUNTIME.guestPath.startsWith(`${mountPoint}/`)) { + throw new Error( + `php:icu.dat guest path ${PHP_ICU_RUNTIME.guestPath} is outside ${mountPoint}`, + ); + } + const runtimeDest = join( + root, + ...PHP_ICU_RUNTIME.guestPath.slice(mountPoint.length + 1).split("/"), + ); + mkdirSync(dirname(runtimeDest), { recursive: true }); + copyFileSync(PHP_ICU_RUNTIME.hostPath, runtimeDest); + chmodSync(runtimeDest, PHP_ICU_RUNTIME.mode); + } this.extensionMountRoot = root; return root; } diff --git a/tools/xtask/src/archive_stage.rs b/tools/xtask/src/archive_stage.rs index b68c19a59..f0af1c8b5 100644 --- a/tools/xtask/src/archive_stage.rs +++ b/tools/xtask/src/archive_stage.rs @@ -16,6 +16,7 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; +use crate::build_deps::validate_cache_artifacts; use crate::pkg_manifest::{DepsManifest, ManifestKind, TargetArch}; /// Caller-supplied build provenance + the locally-computed cache-key @@ -63,6 +64,13 @@ pub fn stage_archive_with_options( )); } + // A direct archive-stage invocation must enforce the same declared + // artifact closure as a resolver build/fetch. In particular, do not ship + // an archive whose manifest promises a runtime file that the payload + // omits; the consumer should never be the first place that notices. + validate_cache_artifacts(target, cache_dir) + .map_err(|e| format!("archive_stage: invalid cache entry: {e}"))?; + let manifest_text = build_archive_manifest_text(target, arch, abi_version, opts)?; // Pre-flight: enumerate cache_dir BEFORE touching any tmp file so @@ -140,19 +148,74 @@ pub fn stage_archive_with_options( Ok(()) } -/// Recursively collect every regular file under `dir`. Symlinks and -/// other special files are not packed — the wasm cache layout is all -/// regular files (`lib/*.a`, `include/*.h`, `lib/pkgconfig/*.pc`). +/// Recursively collect every archive leaf under `dir`. Contained symlinks are +/// deliberately dereferenced into regular archive entries so compatibility +/// aliases survive the existing tar format. External/cyclic symlinks and +/// special files fail producer preflight instead of smuggling bytes from +/// outside the cache root or disappearing from the fetched artifact. fn collect_files(dir: &Path, out: &mut Vec) -> Result<(), String> { - for entry in fs::read_dir(dir).map_err(|e| format!("read_dir {}: {e}", dir.display()))? { - let entry = entry.map_err(|e| format!("read_dir entry: {e}"))?; + let canonical_root = fs::canonicalize(dir) + .map_err(|e| format!("resolve archive cache root {}: {e}", dir.display()))?; + let mut active_dirs = std::collections::BTreeSet::new(); + collect_files_inner(dir, &canonical_root, &mut active_dirs, out) +} + +fn collect_files_inner( + dir: &Path, + canonical_root: &Path, + active_dirs: &mut std::collections::BTreeSet, + out: &mut Vec, +) -> Result<(), String> { + let resolved_dir = fs::canonicalize(dir) + .map_err(|e| format!("resolve archive directory {}: {e}", dir.display()))?; + if !resolved_dir.starts_with(canonical_root) { + return Err(format!( + "archive directory {} resolves outside cache root {}", + dir.display(), + canonical_root.display() + )); + } + if !active_dirs.insert(resolved_dir.clone()) { + return Err(format!( + "archive directory symlink cycle reaches {}", + dir.display() + )); + } + let mut entries = fs::read_dir(dir) + .map_err(|e| format!("read_dir {}: {e}", dir.display()))? + .collect::, _>>() + .map_err(|e| format!("read_dir {}: {e}", dir.display()))?; + entries.sort_by_key(|entry| entry.path()); + for entry in entries { let p = entry.path(); - if p.is_dir() { - collect_files(&p, out)?; - } else if p.is_file() { + let link_metadata = fs::symlink_metadata(&p) + .map_err(|e| format!("stat archive entry {}: {e}", p.display()))?; + let resolved = fs::canonicalize(&p) + .map_err(|e| format!("resolve archive entry {}: {e}", p.display()))?; + if !resolved.starts_with(canonical_root) { + return Err(format!( + "archive entry {} resolves outside cache root {}", + p.display(), + canonical_root.display() + )); + } + let metadata = if link_metadata.file_type().is_symlink() { + fs::metadata(&p).map_err(|e| format!("follow archive symlink {}: {e}", p.display()))? + } else { + link_metadata + }; + if metadata.is_dir() { + collect_files_inner(&p, canonical_root, active_dirs, out)?; + } else if metadata.is_file() { out.push(p); + } else { + return Err(format!( + "archive entry {} is not a regular file, directory, or contained symlink", + p.display() + )); } } + active_dirs.remove(&resolved_dir); Ok(()) } @@ -367,6 +430,181 @@ headers = ["include/zlib.h"] assert!(!install_root.join("artifacts").exists()); } + #[test] + fn program_runtime_file_round_trips_through_archive_and_fetch() { + use crate::pkg_manifest::Binary; + use crate::remote_fetch::fetch_and_install; + use sha2::{Digest, Sha256}; + use std::io::Read; + + let dir = tempdir("program-runtime-round-trip"); + let registry = dir.join("registry/php"); + fs::create_dir_all(®istry).unwrap(); + let manifest_text = r#" +kind = "program" +name = "php" +version = "8.3.15" +depends_on = [] + +[source] +url = "https://example.test/php.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" + +[license] +spdx = "PHP-3.01" + +[[outputs]] +name = "php" +wasm = "php.wasm" + +[[runtime_files]] +artifact = "icu.dat" +guest_path = "/usr/lib/php/icu.dat" +mode = 420 +"#; + let manifest_path = registry.join("package.toml"); + fs::write(&manifest_path, manifest_text).unwrap(); + let manifest = DepsManifest::load(&manifest_path).unwrap(); + + let cache_dir = dir.join("cache_entry"); + fs::create_dir_all(&cache_dir).unwrap(); + fs::write( + cache_dir.join("php.wasm"), + b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b", + ) + .unwrap(); + let runtime_bytes = b"runtime-icu-data\0with-binary-bytes\xff"; + fs::write(cache_dir.join("icu.dat"), runtime_bytes).unwrap(); + + let archive_path = dir.join("php-out.tar.zst"); + let opts = StageOptions { + cache_key_sha: "c".repeat(64), + build_timestamp: "2026-07-12T00:00:00Z".to_string(), + build_host: "test-host".to_string(), + }; + stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 19, + &cache_dir, + &archive_path, + &opts, + ) + .unwrap(); + + let archive_bytes = fs::read(&archive_path).unwrap(); + let mut archive_hash = Sha256::new(); + archive_hash.update(&archive_bytes); + let binary = Binary { + archive_url: format!("file://{}", archive_path.display()), + archive_sha256: crate::util::hex(&Into::<[u8; 32]>::into(archive_hash.finalize())), + }; + let install_root = dir.join("install/canonical"); + fs::create_dir_all(install_root.parent().unwrap()).unwrap(); + fetch_and_install( + &binary, + &install_root, + &manifest, + TargetArch::Wasm32, + 19, + &opts.cache_key_sha, + ) + .unwrap(); + assert_eq!( + fs::read(install_root.join("icu.dat")).unwrap(), + runtime_bytes + ); + + // Pin the authored installation contract in the embedded manifest, + // not just the payload byte round-trip. + let decoder = zstd::stream::read::Decoder::new(&archive_bytes[..]).unwrap(); + let mut tar = tar::Archive::new(decoder); + let mut archived_manifest = None; + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + if entry.path().unwrap().as_ref() == Path::new("manifest.toml") { + let mut text = String::new(); + entry.read_to_string(&mut text).unwrap(); + archived_manifest = Some(text); + break; + } + } + let parsed = DepsManifest::parse_archived( + &archived_manifest.expect("archive must contain manifest.toml"), + registry, + ) + .unwrap(); + assert_eq!(parsed.runtime_files, manifest.runtime_files); + + fs::remove_file(cache_dir.join("icu.dat")).unwrap(); + let incomplete_archive = dir.join("php-incomplete.tar.zst"); + let err = stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 19, + &cache_dir, + &incomplete_archive, + &opts, + ) + .unwrap_err(); + assert!( + err.contains("runtime file") && err.contains("icu.dat"), + "got: {err}" + ); + assert!(!incomplete_archive.exists()); + } + + #[cfg(unix)] + #[test] + fn archive_dereferences_contained_symlinks_and_rejects_external_ones() { + use std::io::Read; + use std::os::unix::fs::symlink; + + let (cache_dir, archive_path, manifest, opts) = fixture_for_round_trip("symlink-safety"); + symlink("zlib.h", cache_dir.join("include/zconf.h")).unwrap(); + stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 4, + &cache_dir, + &archive_path, + &opts, + ) + .unwrap(); + + let bytes = fs::read(&archive_path).unwrap(); + let decoder = zstd::stream::read::Decoder::new(&bytes[..]).unwrap(); + let mut tar = tar::Archive::new(decoder); + let mut alias_bytes = None; + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + if entry.path().unwrap().as_ref() == Path::new("artifacts/include/zconf.h") { + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes).unwrap(); + alias_bytes = Some(bytes); + break; + } + } + assert_eq!(alias_bytes.as_deref(), Some(b"#ifndef ZLIB_H\n".as_slice())); + + fs::remove_file(cache_dir.join("include/zconf.h")).unwrap(); + let outside = archive_path.parent().unwrap().join("outside.dat"); + fs::write(&outside, b"outside").unwrap(); + symlink(&outside, cache_dir.join("include/zconf.h")).unwrap(); + let rejected = archive_path.parent().unwrap().join("external-link.tar.zst"); + let err = stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 4, + &cache_dir, + &rejected, + &opts, + ) + .unwrap_err(); + assert!(err.contains("outside cache root"), "got: {err}"); + assert!(!rejected.exists()); + } + #[test] fn embedded_manifest_round_trips_through_parse_archived() { use std::io::Read; @@ -483,7 +721,9 @@ headers = ["include/zlib.h"] stage_archive_with_options(&m, TargetArch::Wasm32, 4, &empty_cache, &archive, &opts) .unwrap_err(); assert!( - err.contains("contains no files") || err.contains("[outputs]"), + err.contains("missing from cache entry") + || err.contains("contains no files") + || err.contains("[outputs]"), "got: {err}" ); assert!( diff --git a/tools/xtask/src/build_deps.rs b/tools/xtask/src/build_deps.rs index 3053138ed..cd14e4657 100644 --- a/tools/xtask/src/build_deps.rs +++ b/tools/xtask/src/build_deps.rs @@ -309,7 +309,7 @@ pub fn compute_sha( // Fold in declared outputs so changing what a build is // expected to produce invalidates the cache. Without this, // renaming a program's `wasm = "..."` (or any library - // libs/headers/pkgconfig path) leaves cache_key_sha + // libs/headers/pkgconfig/files path) leaves cache_key_sha // unchanged — the resolver then serves a canonical // directory that doesn't match the new declaration and // archive-stage packs broken archives. Bug discovered in @@ -340,6 +340,17 @@ pub fn compute_sha( h.update(b"|"); } h.update(b"\n"); + // Preserve every existing package's cache key: the additive files + // field participates only when authored. A universally empty + // section would invalidate the entire package registry merely for + // learning a new output kind. + if !target.outputs.files.is_empty() { + h.update(b"outputs.files:v1\n"); + for s in &target.outputs.files { + h.update((s.len() as u64).to_le_bytes()); + h.update(s.as_bytes()); + } + } h.update(b"program_outputs:\n"); for out in &target.program_outputs { h.update(out.name.as_bytes()); @@ -351,6 +362,22 @@ pub fn compute_sha( } h.update(b"\n"); } + // Additive program runtime closure. Keep the section absent for + // existing manifests so learning this schema does not invalidate + // every historical package cache key. + if !target.runtime_files.is_empty() { + h.update(b"runtime_files:v1\n"); + for runtime_file in &target.runtime_files { + for field in [ + runtime_file.artifact.as_bytes(), + runtime_file.guest_path.as_bytes(), + ] { + h.update((field.len() as u64).to_le_bytes()); + h.update(field); + } + h.update(runtime_file.mode.to_le_bytes()); + } + } } } if !build_inputs.is_empty() { @@ -2059,7 +2086,7 @@ fn build_into_cache( } // Kind-aware validation. Library and program manifests carry a - // declared outputs list (libs/headers/pkgconfig or program wasms) + // declared outputs list (libs/headers/pkgconfig/files or program wasms) // that `validate_outputs` checks one-by-one. Source manifests have // no declared outputs — design 11 calls for emptiness as the only // signal — so we just verify the script populated OUT_DIR with at @@ -2350,24 +2377,209 @@ fn required_exports_for_program_output( } } -fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), String> { - if !matches!(target.kind, ManifestKind::Program) { - return Ok(()); +fn validate_declared_artifact( + target: &DepsManifest, + root: &Path, + rel: &str, + label: &str, + missing_suffix: &str, + require_regular_file: bool, +) -> Result { + let path = root.join(rel); + let metadata = std::fs::symlink_metadata(&path).map_err(|_| { + format!( + "{}: declared {} output {:?} {}", + target.spec(), + label, + rel, + missing_suffix + ) + })?; + if metadata.file_type().is_symlink() { + return Err(format!( + "{}: declared {} output {:?} must not be a symlink", + target.spec(), + label, + rel + )); } - for out in &target.program_outputs { - let path = dir.join(&out.wasm); - if !path.exists() { + let canonical_root = std::fs::canonicalize(root) + .map_err(|e| format!("{}: resolve package artifact root: {e}", target.spec()))?; + let resolved = std::fs::canonicalize(&path).map_err(|e| { + format!( + "{}: resolve declared {} output {:?}: {e}", + target.spec(), + label, + rel + ) + })?; + if !resolved.starts_with(&canonical_root) { + return Err(format!( + "{}: declared {} output {:?} resolves outside the package artifact root", + target.spec(), + label, + rel + )); + } + if require_regular_file && !metadata.is_file() { + return Err(format!( + "{}: declared {} output {:?} must be a regular file", + target.spec(), + label, + rel + )); + } + if !require_regular_file { + if metadata.is_file() { + return Ok(path); + } + if !metadata.is_dir() { return Err(format!( - "{}: declared wasm output {:?} missing from cache entry", + "{}: declared {} output {:?} must be a regular file or directory", target.spec(), - out.wasm + label, + rel )); } - validate_wasm_artifact_policy( - &path, - out.fork_instrumentation, - required_exports_for_program_output(target, out), - )?; + let mut active_dirs = BTreeSet::new(); + let leaf_count = validate_artifact_tree(&canonical_root, &path, &mut active_dirs)?; + if leaf_count == 0 { + return Err(format!( + "{}: declared {} output {:?} is an empty directory and cannot round-trip through an artifact archive", + target.spec(), + label, + rel + )); + } + } + Ok(path) +} + +/// Validate every reachable leaf below a declared artifact directory. +/// Internal symlinks are allowed because several library packages publish +/// compatibility aliases; external/cyclic links and special files are not. +fn validate_artifact_tree( + canonical_root: &Path, + path: &Path, + active_dirs: &mut BTreeSet, +) -> Result { + let link_metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("stat package artifact {}: {e}", path.display()))?; + let resolved = std::fs::canonicalize(path) + .map_err(|e| format!("resolve package artifact {}: {e}", path.display()))?; + if !resolved.starts_with(canonical_root) { + return Err(format!( + "package artifact {} resolves outside {}", + path.display(), + canonical_root.display() + )); + } + let metadata = if link_metadata.file_type().is_symlink() { + std::fs::metadata(path) + .map_err(|e| format!("follow package artifact symlink {}: {e}", path.display()))? + } else { + link_metadata + }; + if metadata.is_file() { + return Ok(1); + } + if !metadata.is_dir() { + return Err(format!( + "package artifact {} is not a regular file, directory, or contained symlink", + path.display() + )); + } + if !active_dirs.insert(resolved.clone()) { + return Err(format!( + "package artifact directory symlink cycle reaches {}", + path.display() + )); + } + let mut entries = std::fs::read_dir(path) + .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))? + .collect::, _>>() + .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))?; + entries.sort_by_key(|entry| entry.path()); + let mut leaves = 0usize; + for entry in entries { + leaves += validate_artifact_tree(canonical_root, &entry.path(), active_dirs)?; + } + active_dirs.remove(&resolved); + Ok(leaves) +} + +pub(crate) fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), String> { + match target.kind { + ManifestKind::Library => { + for rel in &target.outputs.libs { + validate_declared_artifact( + target, + dir, + rel, + "libs", + "missing from cache entry", + true, + )?; + } + for rel in &target.outputs.headers { + validate_declared_artifact( + target, + dir, + rel, + "headers", + "missing from cache entry", + false, + )?; + } + for rel in &target.outputs.pkgconfig { + validate_declared_artifact( + target, + dir, + rel, + "pkgconfig", + "missing from cache entry", + true, + )?; + } + for rel in &target.outputs.files { + validate_declared_artifact( + target, + dir, + rel, + "files", + "missing from cache entry", + true, + )?; + } + } + ManifestKind::Program => { + for out in &target.program_outputs { + let path = validate_declared_artifact( + target, + dir, + &out.wasm, + "wasm", + "missing from cache entry", + true, + )?; + validate_wasm_artifact_policy( + &path, + out.fork_instrumentation, + required_exports_for_program_output(target, out), + )?; + } + for runtime_file in &target.runtime_files { + validate_declared_artifact( + target, + dir, + &runtime_file.artifact, + "runtime file", + "missing from cache entry", + true, + )?; + } + } + ManifestKind::Source => {} } Ok(()) } @@ -2375,44 +2587,73 @@ fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), Str fn validate_outputs(target: &DepsManifest, out_dir: &Path) -> Result<(), String> { match target.kind { ManifestKind::Library => { - let check = |rel: &str, label: &str| -> Result<(), String> { - let p = out_dir.join(rel); - if !p.exists() { - return Err(format!( - "{}: declared {} output {:?} not produced by build script", - target.spec(), - label, - rel - )); - } - Ok(()) - }; for rel in &target.outputs.libs { - check(rel, "libs")?; + validate_declared_artifact( + target, + out_dir, + rel, + "libs", + "not produced by build script", + true, + )?; } for rel in &target.outputs.headers { - check(rel, "headers")?; + validate_declared_artifact( + target, + out_dir, + rel, + "headers", + "not produced by build script", + false, + )?; } for rel in &target.outputs.pkgconfig { - check(rel, "pkgconfig")?; + validate_declared_artifact( + target, + out_dir, + rel, + "pkgconfig", + "not produced by build script", + true, + )?; + } + for rel in &target.outputs.files { + validate_declared_artifact( + target, + out_dir, + rel, + "files", + "not produced by build script", + true, + )?; } } ManifestKind::Program => { for out in &target.program_outputs { - let p = out_dir.join(&out.wasm); - if !p.exists() { - return Err(format!( - "{}: declared wasm output {:?} not produced by build script", - target.spec(), - out.wasm - )); - } + let p = validate_declared_artifact( + target, + out_dir, + &out.wasm, + "wasm", + "not produced by build script", + true, + )?; validate_wasm_artifact_policy( &p, out.fork_instrumentation, required_exports_for_program_output(target, out), )?; } + for runtime_file in &target.runtime_files { + validate_declared_artifact( + target, + out_dir, + &runtime_file.artifact, + "runtime file", + "not produced by build script", + true, + )?; + } } // No outputs to validate for source-kind (Chunk C). ManifestKind::Source => return Ok(()), @@ -2599,7 +2840,7 @@ pub fn run(args: Vec) -> Result<(), String> { let mut it = rest.into_iter(); let sub = it.next().ok_or( "usage: xtask build-deps [--arch=wasm32|wasm64] [--binaries-dir ] [--fetch-only] \ - \ + \ [ []]", )?; let target = it.next(); @@ -2694,6 +2935,22 @@ pub fn run(args: Vec) -> Result<(), String> { })?; cmd_output_path(&manifest, &basename) } + "runtime-file-path" => { + let artifact = extra.ok_or_else(|| { + "build-deps runtime-file-path: missing \ + (usage: build-deps runtime-file-path )" + .to_string() + })?; + cmd_runtime_file_path(&manifest, &artifact) + } + "runtime-file-metadata" => { + let artifact = extra.ok_or_else(|| { + "build-deps runtime-file-metadata: missing \ + (usage: build-deps runtime-file-metadata )" + .to_string() + })?; + cmd_runtime_file_metadata(&manifest, &artifact) + } "output-fork-instrumentation" => { let basename = extra.ok_or_else(|| { "build-deps output-fork-instrumentation: missing \ @@ -2759,6 +3016,12 @@ fn cmd_parse(m: &DepsManifest) -> Result<(), String> { if !m.outputs.pkgconfig.is_empty() { println!("outputs.pkgconfig= {:?}", m.outputs.pkgconfig); } + if !m.outputs.files.is_empty() { + println!("outputs.files = {:?}", m.outputs.files); + } + if !m.runtime_files.is_empty() { + println!("runtime_files = {:?}", m.runtime_files); + } Ok(()) } @@ -2810,6 +3073,63 @@ fn cmd_output_path(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> Ok(()) } +/// `runtime-file-path `: print the mirror path +/// below `programs//` used by local and resolver materialization. +fn cmd_runtime_file_path(m: &DepsManifest, artifact: &str) -> Result<(), String> { + let rel = m.runtime_file_dest_rel(artifact)?; + println!("{}", rel.display()); + Ok(()) +} + +/// Structured installation contract for VFS/image builders. JSON avoids +/// consumers scraping Debug output and keeps guest path/mode authoritative. +fn cmd_runtime_file_metadata(m: &DepsManifest, artifact: &str) -> Result<(), String> { + let value = runtime_file_metadata_value(m, artifact)?; + println!( + "{}", + serde_json::to_string(&value).map_err(|e| format!("serialize runtime metadata: {e}"))? + ); + Ok(()) +} + +fn runtime_file_metadata_value( + m: &DepsManifest, + artifact: &str, +) -> Result { + let runtime_file = m + .runtime_files + .iter() + .find(|runtime_file| runtime_file.artifact == artifact) + .ok_or_else(|| { + format!( + "program {:?} has no [[runtime_files]] artifact {:?}", + m.name, artifact + ) + })?; + // A runtime file is meaningful only alongside the exact executable and + // side-module outputs produced by the same program package archive. Give + // repo-side consumers the complete resolver mirror closure so they can + // select one materialization tier atomically instead of resolving each + // member independently and accidentally mixing builds. + let closure_mirror_paths: Vec = m + .program_outputs + .iter() + .map(|output| m.output_dest_rel_for(output)) + .chain( + m.runtime_files + .iter() + .map(|runtime_file| m.runtime_file_dest_rel_for(runtime_file)), + ) + .collect(); + Ok(serde_json::json!({ + "artifact": runtime_file.artifact, + "guest_path": runtime_file.guest_path, + "mode": runtime_file.mode, + "mirror_path": m.runtime_file_dest_rel_for(runtime_file), + "closure_mirror_paths": closure_mirror_paths, + })) +} + fn cmd_output_fork_instrumentation(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> { let policy = m.output_fork_instrumentation(wasm_basename)?; println!("{}", policy.as_str()); @@ -2886,7 +3206,8 @@ fn cmd_resolve( } /// Place symlinks under `binaries_dir/programs//` pointing at -/// each declared `[[outputs]]` wasm in the cache canonical directory. +/// each declared `[[outputs]]` artifact and `[[runtime_files]]` file in the +/// cache canonical directory. /// /// Layout (per arch — wasm32 and wasm64 mirror in parallel): /// * 1 output: `/programs//.wasm`. @@ -2942,6 +3263,34 @@ fn place_binaries_symlinks( std::os::unix::fs::symlink(&src, &dest) .map_err(|e| format!("symlink {} -> {}: {e}", dest.display(), src.display()))?; } + for runtime_file in &m.runtime_files { + let src = canonical.join(&runtime_file.artifact); + let metadata = std::fs::symlink_metadata(&src).map_err(|e| { + format!( + "declared runtime file {} not found in cache at {}: {e}", + runtime_file.artifact, + src.display() + ) + })?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "declared runtime file {} is not a regular non-symlink file at {}", + runtime_file.artifact, + src.display() + )); + } + let dest = arch_root.join(m.runtime_file_dest_rel_for(runtime_file)); + let dest_dir = dest + .parent() + .ok_or_else(|| format!("dest path {} has no parent", dest.display()))?; + std::fs::create_dir_all(dest_dir) + .map_err(|e| format!("mkdir {}: {e}", dest_dir.display()))?; + if dest.exists() || dest.symlink_metadata().is_ok() { + let _ = std::fs::remove_file(&dest); + } + std::os::unix::fs::symlink(&src, &dest) + .map_err(|e| format!("symlink {} -> {}: {e}", dest.display(), src.display()))?; + } Ok(()) } @@ -4134,6 +4483,70 @@ fork_instrumentation = "disabled" ); } + #[test] + fn cache_key_sha_tracks_program_runtime_file_contract() { + let root = tempdir("sha-prog-runtime-file"); + write_program_manifest( + &root, + "php", + "1.0.0", + "[[outputs]]\nname = \"php\"\nwasm = \"php.wasm\"\n", + ); + let reg = Registry { + roots: vec![root.clone()], + }; + let baseline = sha_of(®, "php"); + let toml_path = root.join("php/package.toml"); + let original = std::fs::read_to_string(&toml_path).unwrap(); + + let with_runtime = format!( + "{original}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n" + ); + std::fs::write(&toml_path, &with_runtime).unwrap(); + let added = sha_of(®, "php"); + assert_ne!( + baseline, added, + "adding runtime closure must invalidate the key" + ); + + std::fs::write( + &toml_path, + with_runtime.replace("/usr/lib/php/icu.dat", "/opt/php/icu.dat"), + ) + .unwrap(); + let moved = sha_of(®, "php"); + assert_ne!( + added, moved, + "changing the guest path must invalidate the key" + ); + + std::fs::write(&toml_path, format!("{with_runtime}mode = 384\n")).unwrap(); + let remoded = sha_of(®, "php"); + assert_ne!( + added, remoded, + "changing runtime mode must invalidate the key" + ); + + // Length prefixes keep delimiter-bearing fields unambiguous. These + // two records collide under naive `artifact|guest` concatenation. + std::fs::write( + &toml_path, + format!("{original}\n[[runtime_files]]\nartifact = \"a|/b\"\nguest_path = \"/c\"\n"), + ) + .unwrap(); + let delimiter_a = sha_of(®, "php"); + std::fs::write( + &toml_path, + format!("{original}\n[[runtime_files]]\nartifact = \"a\"\nguest_path = \"/b|/c\"\n"), + ) + .unwrap(); + let delimiter_b = sha_of(®, "php"); + assert_ne!( + delimiter_a, delimiter_b, + "runtime hash fields must be framed" + ); + } + /// Pins behavior: program outputs are hashed in declaration order. /// Re-ordering DOES change cache_key_sha. We deliberately don't /// normalize because (a) the manifest preserves authored order @@ -4254,6 +4667,69 @@ fork_instrumentation = "disabled" ); } + #[test] + fn cache_key_sha_changes_when_library_runtime_file_added() { + let root = tempdir("sha-lib-runtime-file-added"); + write(&root, "libZ", "1.0.0", &[]); + let reg = Registry { + roots: vec![root.clone()], + }; + let sha_before = sha_of(®, "libZ"); + + let toml_path = root.join("libZ/package.toml"); + let text = std::fs::read_to_string(&toml_path).unwrap(); + std::fs::write( + &toml_path, + text.replace( + "libs = [\"lib/liblibZ.a\"]", + "libs = [\"lib/liblibZ.a\"]\nfiles = [\"share/libZ.dat\"]", + ), + ) + .unwrap(); + let sha_after = sha_of(®, "libZ"); + + assert_ne!( + sha_before, sha_after, + "adding a library runtime file output must invalidate the cache key" + ); + } + + #[test] + fn library_runtime_file_cache_keys_frame_delimiter_bearing_paths() { + let root = tempdir("sha-lib-runtime-file-framing"); + write(&root, "libZ", "1.0.0", &[]); + let reg = Registry { + roots: vec![root.clone()], + }; + let toml_path = root.join("libZ/package.toml"); + let original = std::fs::read_to_string(&toml_path).unwrap(); + + std::fs::write( + &toml_path, + original.replace( + "libs = [\"lib/liblibZ.a\"]", + "libs = [\"lib/liblibZ.a\"]\nfiles = [\"a|b\", \"c\"]", + ), + ) + .unwrap(); + let delimiter_a = sha_of(®, "libZ"); + + std::fs::write( + &toml_path, + original.replace( + "libs = [\"lib/liblibZ.a\"]", + "libs = [\"lib/liblibZ.a\"]\nfiles = [\"a\", \"b|c\"]", + ), + ) + .unwrap(); + let delimiter_b = sha_of(®, "libZ"); + + assert_ne!( + delimiter_a, delimiter_b, + "library runtime-file cache-key fields must be length framed" + ); + } + // --- ensure_built / build_into_cache tests --- /// Create a package.toml + build-.sh pair. The build script uses @@ -4486,41 +4962,149 @@ libs = ["lib/libC.a"] } #[test] - fn ensure_built_fails_when_script_exits_nonzero() { - let root = tempdir("built-badexit"); - let cache = tempdir("built-badexit-cache"); + fn ensure_built_fails_when_declared_runtime_file_missing() { + let root = tempdir("built-missing-runtime-file"); + let cache = tempdir("built-missing-runtime-file-cache"); write_lib( &root, - "libD", + "libRuntimeMissing", "1.0.0", &[], - "echo boom >&2\nexit 37", + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeMissing.a""#, r#"[outputs] -libs = ["lib/libD.a"] +libs = ["lib/libRuntimeMissing.a"] +files = ["share/runtime.dat"] "#, ); let reg = Registry { roots: vec![root] }; - let m = reg.load("libD").unwrap(); + let m = reg.load("libRuntimeMissing").unwrap(); let err = ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); - assert!(err.contains("exited"), "got: {err}"); - let sha = compute_sha( - &m, - ®, - TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), - ) - .unwrap(); - assert!(!canonical_path(&cache, &m, TEST_ARCH, &sha).exists()); + assert!(err.contains("declared files output"), "got: {err}"); + assert!(err.contains("share/runtime.dat"), "got: {err}"); } - /// Regression: build-script stdout must NOT leak to xtask's stdout. - /// - /// `cmd_resolve` consumers shell-capture xtask's stdout to read the - /// canonical cache path: + #[test] + fn ensure_built_accepts_declared_runtime_file() { + let root = tempdir("built-runtime-file"); + let cache = tempdir("built-runtime-file-cache"); + write_lib( + &root, + "libRuntime", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntime.a" +printf runtime > "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, + r#"[outputs] +libs = ["lib/libRuntime.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntime").unwrap(); + + let path = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + assert_eq!( + std::fs::read_to_string(path.join("share/runtime.dat")).unwrap(), + "runtime" + ); + } + + #[test] + fn ensure_built_rejects_declared_runtime_file_directory() { + let root = tempdir("built-runtime-file-directory"); + let cache = tempdir("built-runtime-file-directory-cache"); + write_lib( + &root, + "libRuntimeDirectory", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeDirectory.a""#, + r#"[outputs] +libs = ["lib/libRuntimeDirectory.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeDirectory").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must be a regular file"), "got: {err}"); + } + + #[cfg(unix)] + #[test] + fn ensure_built_rejects_declared_runtime_file_symlink_escape() { + let root = tempdir("built-runtime-file-symlink-escape"); + let cache = tempdir("built-runtime-file-symlink-escape-cache"); + let outside = root.join("outside.dat"); + std::fs::write(&outside, b"outside").unwrap(); + write_lib( + &root, + "libRuntimeSymlinkEscape", + "1.0.0", + &[], + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeSymlinkEscape.a" +ln -s {:?} "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, + outside + ), + r#"[outputs] +libs = ["lib/libRuntimeSymlinkEscape.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeSymlinkEscape").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must not be a symlink"), "got: {err}"); + } + + #[test] + fn ensure_built_fails_when_script_exits_nonzero() { + let root = tempdir("built-badexit"); + let cache = tempdir("built-badexit-cache"); + write_lib( + &root, + "libD", + "1.0.0", + &[], + "echo boom >&2\nexit 37", + r#"[outputs] +libs = ["lib/libD.a"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libD").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("exited"), "got: {err}"); + let sha = compute_sha( + &m, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap(); + assert!(!canonical_path(&cache, &m, TEST_ARCH, &sha).exists()); + } + + /// Regression: build-script stdout must NOT leak to xtask's stdout. + /// + /// `cmd_resolve` consumers shell-capture xtask's stdout to read the + /// canonical cache path: /// `PREFIX="$(cargo run -- build-deps resolve )"`. If the bash /// subprocess's stdout were inherited (the default), every chatty /// `echo` in the build script would land on xtask's stdout ahead of @@ -5205,6 +5789,49 @@ spdx = "TestLicense" name = "{output_name}" wasm = "{output_wasm}" +[compatibility] +target_arch = "{arch}" +abi_versions = [{abi_csv}] +cache_key_sha = "{cache_key_sha}" +"#, + "" + ) + } + + fn archived_program_runtime_manifest_text( + name: &str, + arch: &str, + abi_versions: &[u32], + cache_key_sha: &str, + ) -> String { + let abi_csv = abi_versions + .iter() + .map(|value| value.to_string()) + .collect::>() + .join(", "); + format!( + r#" +kind = "program" +name = "{name}" +version = "1.0.0" +revision = 1 +depends_on = [] + +[source] +url = "https://example.test/{name}.tar.gz" +sha256 = "{:0>64}" + +[license] +spdx = "MIT" + +[[outputs]] +name = "{name}" +wasm = "{name}.wasm" + +[[runtime_files]] +artifact = "icu.dat" +guest_path = "/usr/lib/php/icu.dat" + [compatibility] target_arch = "{arch}" abi_versions = [{abi_csv}] @@ -5287,6 +5914,33 @@ index_url = "{index_url}" std::fs::write(dir.join("build.toml"), build_toml).unwrap(); } + fn write_runtime_program_with_index(root: &Path, name: &str, index_url: &str) { + write_program( + root, + name, + "1.0.0", + &[], + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/{name}.wasm" +printf RUNTIME-BYTES > "$WASM_POSIX_DEP_OUT_DIR/icu.dat" +touch "$WASM_POSIX_DEP_OUT_DIR/via-build""#, + ), + &[(name, &format!("{name}.wasm"))], + ); + append_program_runtime_file(root, name, "icu.dat", "/usr/lib/php/icu.dat"); + let build_toml = format!( + r#"script_path = "{name}/build-{name}.sh" +repo_url = "https://example.test/repo.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + +[binary] +index_url = "{index_url}" +"#, + ); + fs::write(root.join(name).join("build.toml"), build_toml).unwrap(); + } + /// Stage an `index.toml` at `path` declaring `name@1.0.0` with a /// single Success entry for `arch` pointing at `archive_url` with /// the given `archive_sha256` and `cache_key_sha`. Mirrors what @@ -5655,6 +6309,185 @@ cache_key_sha = "{cache_key_hex}" ); } + #[test] + fn fetched_program_runtime_file_matches_source_mirror_layout() { + let root = tempdir("runtime-fetch-parity-reg"); + let remote_cache = tempdir("runtime-fetch-parity-remote-cache"); + let source_cache = tempdir("runtime-fetch-parity-source-cache"); + let remote_bin = tempdir("runtime-fetch-parity-remote-bin"); + let source_bin = tempdir("runtime-fetch-parity-source-bin"); + let archive_dir = tempdir("runtime-fetch-parity-archive"); + let index_dir = tempdir("runtime-fetch-parity-index"); + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + let name = "runtimeFetched"; + write_runtime_program_with_index(&root, name, &index_url); + + let reg = Registry { + roots: vec![root.clone()], + }; + let manifest = reg.load(name).unwrap(); + let cache_key_hex = hex(&compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap()); + let archived_manifest = archived_program_runtime_manifest_text( + name, + TEST_ARCH.as_str(), + &[TEST_ABI], + &cache_key_hex, + ); + let wasm_name = format!("{name}.wasm"); + let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; + let archive_bytes = crate::remote_fetch::build_test_archive( + &archived_manifest, + &[ + (wasm_name.as_str(), wasm_bytes.as_slice()), + ("icu.dat", b"RUNTIME-BYTES"), + ], + ); + let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); + fs::write(&archive_path, &archive_bytes).unwrap(); + stage_index_toml( + &index_path, + name, + TEST_ARCH, + &format!("file://{}", archive_path.display()), + &sha256_hex(&archive_bytes), + &cache_key_hex, + ); + + let remote_opts = ResolveOpts { + cache_root: &remote_cache, + local_libs: None, + force_source_build: None, + fetch_only: false, + repo_root: Some(&root), + binaries_dir: Some(&remote_bin), + }; + let remote_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &remote_opts).unwrap(); + assert!(!remote_path.join("via-build").exists()); + place_binaries_symlinks(&manifest, &remote_path, &remote_bin, TEST_ARCH).unwrap(); + + let force = BTreeSet::from([name.to_string()]); + let source_opts = ResolveOpts { + cache_root: &source_cache, + local_libs: None, + force_source_build: Some(&force), + fetch_only: false, + repo_root: Some(&root), + binaries_dir: Some(&source_bin), + }; + let source_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &source_opts).unwrap(); + assert!(source_path.join("via-build").exists()); + place_binaries_symlinks(&manifest, &source_path, &source_bin, TEST_ARCH).unwrap(); + + let mirror_rel = Path::new("programs/wasm32").join(name).join("icu.dat"); + assert_eq!( + fs::read(remote_bin.join(&mirror_rel)).unwrap(), + b"RUNTIME-BYTES" + ); + assert_eq!( + fs::read(source_bin.join(&mirror_rel)).unwrap(), + b"RUNTIME-BYTES" + ); + } + + #[test] + fn incomplete_fetched_runtime_file_falls_back_or_fails_fetch_only() { + let root = tempdir("runtime-fetch-incomplete-reg"); + let fallback_cache = tempdir("runtime-fetch-incomplete-fallback-cache"); + let fetch_only_cache = tempdir("runtime-fetch-incomplete-only-cache"); + let archive_dir = tempdir("runtime-fetch-incomplete-archive"); + let index_dir = tempdir("runtime-fetch-incomplete-index"); + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + let name = "runtimeIncomplete"; + write_runtime_program_with_index(&root, name, &index_url); + + let reg = Registry { + roots: vec![root.clone()], + }; + let manifest = reg.load(name).unwrap(); + let cache_key_hex = hex(&compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap()); + let archived_manifest = archived_program_runtime_manifest_text( + name, + TEST_ARCH.as_str(), + &[TEST_ABI], + &cache_key_hex, + ); + let wasm_name = format!("{name}.wasm"); + let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; + let archive_bytes = crate::remote_fetch::build_test_archive( + &archived_manifest, + &[(wasm_name.as_str(), wasm_bytes.as_slice())], + ); + let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); + fs::write(&archive_path, &archive_bytes).unwrap(); + stage_index_toml( + &index_path, + name, + TEST_ARCH, + &format!("file://{}", archive_path.display()), + &sha256_hex(&archive_bytes), + &cache_key_hex, + ); + + let fallback = ensure_built( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &resolve_opts(&fallback_cache, None), + ) + .unwrap(); + assert!(fallback.join("via-build").exists()); + assert_eq!( + fs::read(fallback.join("icu.dat")).unwrap(), + b"RUNTIME-BYTES" + ); + + let fetch_only_opts = ResolveOpts { + cache_root: &fetch_only_cache, + local_libs: None, + force_source_build: None, + fetch_only: true, + repo_root: Some(&root), + binaries_dir: None, + }; + let err = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &fetch_only_opts).unwrap_err(); + assert!(err.contains("fetch-only"), "got: {err}"); + assert!(!canonical_path( + &fetch_only_cache, + &manifest, + TEST_ARCH, + &compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap(), + ) + .join("via-build") + .exists()); + } + #[test] fn index_fetch_falls_through_on_archive_sha_mismatch() { let root = tempdir("idx-shafail-reg"); @@ -5947,6 +6780,15 @@ spdx = "MIT" } } + fn append_program_runtime_file(root: &Path, name: &str, artifact: &str, guest_path: &str) { + let manifest_path = root.join(name).join("package.toml"); + let mut text = fs::read_to_string(&manifest_path).unwrap(); + text.push_str(&format!( + "\n[[runtime_files]]\nartifact = {artifact:?}\nguest_path = {guest_path:?}\n" + )); + fs::write(manifest_path, text).unwrap(); + } + #[test] fn canonical_path_uses_programs_subdir_for_program_kind() { let m = DepsManifest::parse( @@ -6010,6 +6852,141 @@ wasm = "vim.wasm" assert!(err.contains("miss.wasm"), "got: {err}"); } + #[test] + fn program_runtime_file_is_required_and_cached_as_a_regular_file() { + let root = tempdir("prog-runtime-file"); + let cache = tempdir("prog-runtime-file-cache"); + write_program( + &root, + "runtimeprog", + "0.1.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimeprog.wasm" +printf runtime-data > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, + &[("runtimeprog", "runtimeprog.wasm")], + ); + append_program_runtime_file(&root, "runtimeprog", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("runtimeprog").unwrap(); + assert_eq!( + runtime_file_metadata_value(&m, "icu.dat").unwrap(), + serde_json::json!({ + "artifact": "icu.dat", + "guest_path": "/usr/lib/php/icu.dat", + "mode": 420, + "mirror_path": "runtimeprog/icu.dat", + "closure_mirror_paths": [ + "runtimeprog.wasm", + "runtimeprog/icu.dat", + ], + }) + ); + let path = + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap(); + assert_eq!(fs::read(path.join("icu.dat")).unwrap(), b"runtime-data"); + + fs::remove_file(path.join("icu.dat")).unwrap(); + let err = validate_cache_artifacts(&m, &path).unwrap_err(); + assert!( + err.contains("runtime file") && err.contains("missing"), + "got: {err}" + ); + } + + #[test] + fn runtime_file_metadata_lists_the_complete_multi_output_closure() { + let manifest = DepsManifest::parse( + r#"kind = "program" +name = "runtimeprog" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/runtimeprog.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "runtimeprog" +wasm = "bin/runtimeprog.wasm" +[[outputs]] +name = "module" +wasm = "extensions/module.so" +[[runtime_files]] +artifact = "share/icu.dat" +guest_path = "/usr/lib/runtimeprog/icu.dat" +[[runtime_files]] +artifact = "share/timezone.dat" +guest_path = "/usr/lib/runtimeprog/timezone.dat" +"#, + PathBuf::from("/x"), + ) + .unwrap(); + + let metadata = runtime_file_metadata_value(&manifest, "share/icu.dat").unwrap(); + assert_eq!( + metadata["closure_mirror_paths"], + serde_json::json!([ + "runtimeprog/runtimeprog.wasm", + "runtimeprog/module.so", + "runtimeprog/share/icu.dat", + "runtimeprog/share/timezone.dat", + ]) + ); + } + + #[test] + fn build_fails_when_program_runtime_file_is_missing() { + let root = tempdir("prog-runtime-file-missing"); + let cache = tempdir("prog-runtime-file-missing-cache"); + write_program( + &root, + "runtimemissing", + "0.1.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimemissing.wasm""#, + &[("runtimemissing", "runtimemissing.wasm")], + ); + append_program_runtime_file(&root, "runtimemissing", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("runtimemissing").unwrap(); + let err = + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap_err(); + assert!( + err.contains("runtime file") && err.contains("icu.dat"), + "got: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn build_rejects_program_runtime_file_symlink() { + let root = tempdir("prog-runtime-file-symlink"); + let cache = tempdir("prog-runtime-file-symlink-cache"); + let outside = root.join("outside.dat"); + fs::write(&outside, b"outside").unwrap(); + write_program( + &root, + "runtimesymlink", + "0.1.0", + &[], + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimesymlink.wasm" +ln -s {:?} "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, + outside + ), + &[("runtimesymlink", "runtimesymlink.wasm")], + ); + append_program_runtime_file(&root, "runtimesymlink", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("runtimesymlink").unwrap(); + let err = + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must not be a symlink"), "got: {err}"); + } + #[test] fn program_output_validation_rejects_legacy_asyncify_wasm() { let out = tempdir("prog-out-asyncify"); @@ -7287,6 +8264,36 @@ libs = ["lib/libF3b.a"] ); } + #[test] + fn cmd_resolve_materializes_program_runtime_file_under_package_directory() { + let root = tempdir("resolve-bdir-runtime-reg"); + let cache = tempdir("resolve-bdir-runtime-cache"); + let bin_dir = tempdir("resolve-bdir-runtime-bin"); + write_program( + &root, + "runtimebin", + "0.1.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimebin.wasm" +printf canonical-runtime > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, + &[("runtimebin", "runtimebin.wasm")], + ); + append_program_runtime_file(&root, "runtimebin", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { + roots: vec![root.clone()], + }; + let m = reg.load("runtimebin").unwrap(); + + cmd_resolve_with_test_cache(&m, ®, &root, TargetArch::Wasm32, &cache, Some(&bin_dir)) + .unwrap(); + + let runtime = bin_dir.join("programs/wasm32/runtimebin/icu.dat"); + assert!(runtime.symlink_metadata().unwrap().file_type().is_symlink()); + assert_eq!(fs::read(runtime).unwrap(), b"canonical-runtime"); + assert!(bin_dir.join("programs/wasm32/runtimebin.wasm").exists()); + } + #[test] fn cmd_resolve_with_binaries_dir_places_kernel_at_root() { // First-party kernel/userspace artifacts are consumed as diff --git a/tools/xtask/src/pkg_manifest.rs b/tools/xtask/src/pkg_manifest.rs index af12640d8..fa0e1bc80 100644 --- a/tools/xtask/src/pkg_manifest.rs +++ b/tools/xtask/src/pkg_manifest.rs @@ -180,6 +180,25 @@ pub struct ProgramOutput { pub fork_instrumentation: ForkInstrumentationPolicy, } +/// A non-Wasm file that a program package needs at runtime. +/// +/// `artifact` is produced in the package cache/archive; `guest_path` is where +/// VFS consumers install those bytes. Runtime files are mirrored separately +/// from `[[outputs]]` so the latter remains truthful about executable/archive +/// program artifacts and its legacy output-count layout stays unchanged. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeFile { + pub artifact: String, + pub guest_path: String, + #[serde(default = "default_runtime_file_mode")] + pub mode: u32, +} + +fn default_runtime_file_mode() -> u32 { + 0o644 +} + /// One entry in a manifest's `[[host_tools]]` array. Inline /// declaration on the consumer site — no separate registry entry, /// per design 10. @@ -414,6 +433,10 @@ pub struct DepsManifest { /// `validate_outputs` in the resolver (wired in Chunk B Task B.2). pub program_outputs: Vec, + /// Non-Wasm runtime closure declared by `kind = "program"` manifests. + /// Empty for library/source manifests. + pub runtime_files: Vec, + /// Build-time provenance + ABI compatibility. Always `None` for /// manifests parsed via [`DepsManifest::parse`] (source `package.toml`) /// and always `Some` for those parsed via @@ -699,6 +722,98 @@ fn validate_build_input_path(path: &str) -> Result<(), String> { Ok(()) } +/// Validate an artifact path before it can be joined to a build/cache root. +/// +/// This is shared by library and program outputs so neither manifest shape can +/// escape its package directory through an absolute path or `..` component. +fn validate_output_path(path: &str, context: &str) -> Result<(), String> { + if path.is_empty() { + return Err(format!("{context} must not be empty")); + } + if path.contains('\0') || path.contains('\\') { + return Err(format!( + "{context} must be a portable '/'-separated path, got {path:?}" + )); + } + if path.starts_with('/') || Path::new(path).is_absolute() { + return Err(format!( + "{context} must be relative to the package output directory, got {path:?}" + )); + } + for component in path.split('/') { + if component.is_empty() { + return Err(format!( + "{context} must not contain empty path components, got {path:?}" + )); + } + if component == "." { + return Err(format!("{context} must not contain '.', got {path:?}")); + } + if component == ".." { + return Err(format!("{context} must not contain '..', got {path:?}")); + } + } + Ok(()) +} + +fn validate_single_path_component(value: &str, context: &str) -> Result<(), String> { + if value.is_empty() { + return Err(format!("{context} must not be empty")); + } + if value == "." + || value == ".." + || value.contains('/') + || value.contains('\\') + || value.contains('\0') + { + return Err(format!( + "{context} must be a safe single path component, got {value:?}" + )); + } + Ok(()) +} + +/// Both paths describe files. Equal paths and ancestor/descendant pairs are +/// therefore unsatisfiable in one artifact or VFS closure. +fn file_paths_conflict(a: &str, b: &str) -> bool { + a == b + || a.strip_prefix(b) + .is_some_and(|suffix| suffix.starts_with('/')) + || b.strip_prefix(a) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +fn validate_runtime_guest_path(path: &str, context: &str) -> Result<(), String> { + if path.contains('\0') || path.contains('\\') { + return Err(format!( + "{context} must be a normalized absolute POSIX path, got {path:?}" + )); + } + if !path.starts_with('/') || path == "/" { + return Err(format!( + "{context} must be a non-root absolute POSIX path, got {path:?}" + )); + } + if path.ends_with('/') { + return Err(format!( + "{context} must not have a trailing '/', got {path:?}" + )); + } + for component in path[1..].split('/') { + if component.is_empty() { + return Err(format!( + "{context} must not contain empty path components, got {path:?}" + )); + } + if component == "." || component == ".." { + return Err(format!( + "{context} must not contain {component:?}, got {path:?}" + )); + } + } + Ok(()) +} + #[derive(Debug, Clone, Deserialize, Default)] pub struct Outputs { #[serde(default)] @@ -707,6 +822,10 @@ pub struct Outputs { pub headers: Vec, #[serde(default)] pub pkgconfig: Vec, + /// Runtime data or other package files that are neither link libraries, + /// headers, nor pkg-config metadata. + #[serde(default)] + pub files: Vec, } /// `name@version` reference, parsed from `depends_on` strings. @@ -735,6 +854,14 @@ impl DepRef { if name.contains('@') { return Err(format!("dep name {:?} must not contain '@'", name)); } + // Registry lookup joins the name directly below each registry root; + // cache/spec logic also treats the exact version as an identity token. + // Reject traversal and nested-path spellings before either value can + // reach filesystem interpolation. + validate_single_path_component(name, "dep name") + .map_err(|e| format!("dep reference {s:?}: {e}"))?; + validate_single_path_component(version, "dep version") + .map_err(|e| format!("dep reference {s:?}: {e}"))?; Ok(Self { name: name.into(), version: version.into(), @@ -752,6 +879,7 @@ impl std::fmt::Display for DepRef { /// validated [`DepsManifest`] so normalization (default build script, /// parsed DepRefs, etc.) lives in one place. #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] struct Raw { kind: ManifestKind, name: String, @@ -781,6 +909,8 @@ struct Raw { #[serde(default = "default_outputs_value")] outputs: toml::Value, #[serde(default)] + runtime_files: Vec, + #[serde(default)] compatibility: Option, // `binary` accepts two shapes: // * bare `[binary]` (archive_url + archive_sha256 directly), @@ -812,7 +942,63 @@ fn default_outputs_value() -> toml::Value { toml::Value::Table(toml::value::Table::new()) } +fn program_output_dest_rel( + package_name: &str, + output_count: usize, + out: &ProgramOutput, +) -> PathBuf { + let basename = Path::new(&out.wasm) + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or(&out.wasm); + let ext = match basename.find('.') { + Some(i) => &basename[i..], + None => "", + }; + let dest_name = format!("{}{}", out.name, ext); + if output_count > 1 { + Path::new(package_name).join(dest_name) + } else { + PathBuf::from(dest_name) + } +} + impl DepsManifest { + /// Resolver mirror path under `programs//` for a runtime file. + /// Runtime files always live below the package name, independently of the + /// number of executable `[[outputs]]` entries. + pub fn runtime_file_dest_rel_for(&self, runtime_file: &RuntimeFile) -> PathBuf { + Path::new(&self.name).join(&runtime_file.artifact) + } + + /// Runtime-file mirror path keyed by the exact declared artifact path. + /// Exact lookup keeps nested runtime closures representable and avoids + /// basename ambiguity (`a/data` versus `b/data`). + pub fn runtime_file_dest_rel(&self, artifact: &str) -> Result { + if self.kind != ManifestKind::Program { + return Err(format!( + "manifest {:?} is kind={:?}; runtime-file lookup is program-only", + self.name, self.kind + )); + } + let matches: Vec<&RuntimeFile> = self + .runtime_files + .iter() + .filter(|runtime_file| runtime_file.artifact == artifact) + .collect(); + match matches.as_slice() { + [runtime_file] => Ok(self.runtime_file_dest_rel_for(runtime_file)), + [] => Err(format!( + "program {:?} has no [[runtime_files]] artifact {:?}", + self.name, artifact + )), + _ => Err(format!( + "program {:?} has multiple [[runtime_files]] artifacts {:?}", + self.name, artifact + )), + } + } + /// Relative path under `programs//` where `out` should land /// once produced. Single source of truth for the placement /// convention; both `place_binaries_symlinks` (resolver-driven @@ -828,20 +1014,7 @@ impl DepsManifest { /// `` is everything from the first `.` onward in `out.wasm`'s /// basename, so `.vfs.zst`, `.tar.gz`, etc. round-trip intact. pub fn output_dest_rel_for(&self, out: &ProgramOutput) -> PathBuf { - let basename = Path::new(&out.wasm) - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or(&out.wasm); - let ext = match basename.find('.') { - Some(i) => &basename[i..], - None => "", - }; - let dest_name = format!("{}{}", out.name, ext); - if self.program_outputs.len() > 1 { - Path::new(&self.name).join(dest_name) - } else { - PathBuf::from(dest_name) - } + program_output_dest_rel(&self.name, self.program_outputs.len(), out) } /// Same as [`output_dest_rel_for`] but keyed by the `wasm` @@ -1187,12 +1360,18 @@ impl DepsManifest { if raw.name.is_empty() { return Err("name must not be empty".into()); } + validate_single_path_component(&raw.name, "name")?; if raw.name.contains('@') { return Err(format!("name {:?} must not contain '@'", raw.name)); } if raw.version.is_empty() { return Err("version must not be empty".into()); } + // `version` is interpolated into canonical cache directories and + // archive filenames. Keep it to one portable filesystem component so + // a manifest cannot escape those roots or create an ambiguous nested + // path before any resolver/archive code sees it. + validate_single_path_component(&raw.version, "version")?; // revision: archived → declared (validate_archived rejects None // up-front); source → defaults to 1 since source manifests no // longer carry it. Downstream code reads m.revision; the @@ -1330,8 +1509,59 @@ impl DepsManifest { }); } + let runtime_files = raw.runtime_files; + if !matches!(raw.kind, ManifestKind::Program) && !runtime_files.is_empty() { + return Err("[[runtime_files]] is allowed only for kind = \"program\"".into()); + } + let mut runtime_artifacts = BTreeSet::new(); + let mut runtime_guest_paths = BTreeSet::new(); + for (idx, runtime_file) in runtime_files.iter().enumerate() { + validate_output_path( + &runtime_file.artifact, + &format!("[[runtime_files]][{idx}].artifact"), + )?; + validate_runtime_guest_path( + &runtime_file.guest_path, + &format!("[[runtime_files]][{idx}].guest_path"), + )?; + if runtime_file.mode > 0o777 { + return Err(format!( + "[[runtime_files]][{idx}].mode must be between 0 and 0777, got {}", + runtime_file.mode + )); + } + if !runtime_artifacts.insert(runtime_file.artifact.clone()) { + return Err(format!( + "[[runtime_files]] declares artifact {:?} more than once", + runtime_file.artifact + )); + } + if !runtime_guest_paths.insert(runtime_file.guest_path.clone()) { + return Err(format!( + "[[runtime_files]] declares guest_path {:?} more than once", + runtime_file.guest_path + )); + } + } + for (index, left) in runtime_files.iter().enumerate() { + for right in runtime_files.iter().skip(index + 1) { + if file_paths_conflict(&left.artifact, &right.artifact) { + return Err(format!( + "[[runtime_files]] artifact paths {:?} and {:?} conflict as file/ancestor paths", + left.artifact, right.artifact + )); + } + if file_paths_conflict(&left.guest_path, &right.guest_path) { + return Err(format!( + "[[runtime_files]] guest paths {:?} and {:?} conflict as file/ancestor paths", + left.guest_path, right.guest_path + )); + } + } + } + // Dispatch on `kind` to decide whether `outputs` is the - // library shape (`[outputs]` table with libs/headers/pkgconfig) + // library shape (`[outputs]` table with libs/headers/pkgconfig/files) // or the program shape (`[[outputs]]` array-of-tables with // name/wasm). A mismatch between the two is rejected at parse // time: each kind enforces its own grammar. @@ -1346,6 +1576,16 @@ impl DepsManifest { .outputs .try_into() .map_err(|e| format!("parse [outputs] table: {e}"))?; + for (field, paths) in [ + ("libs", &outputs.libs), + ("headers", &outputs.headers), + ("pkgconfig", &outputs.pkgconfig), + ("files", &outputs.files), + ] { + for (idx, path) in paths.iter().enumerate() { + validate_output_path(path, &format!("[outputs].{field}[{idx}]"))?; + } + } (outputs, Vec::new()) } ManifestKind::Program => { @@ -1375,12 +1615,11 @@ impl DepsManifest { ); } for (idx, out) in program_outputs.iter().enumerate() { - if out.name.is_empty() { - return Err(format!("[[outputs]][{idx}].name must not be empty")); - } + validate_single_path_component(&out.name, &format!("[[outputs]][{idx}].name"))?; if out.wasm.is_empty() { return Err(format!("[[outputs]][{idx}].wasm must not be empty")); } + validate_output_path(&out.wasm, &format!("[[outputs]][{idx}].wasm"))?; } (Outputs::default(), program_outputs) } @@ -1407,6 +1646,56 @@ impl DepsManifest { } }; + if matches!(raw.kind, ManifestKind::Program) { + let mut output_artifacts: Vec<&str> = Vec::new(); + let mut output_mirrors: Vec = Vec::new(); + for (idx, out) in program_outputs.iter().enumerate() { + for (prior_idx, prior_artifact) in output_artifacts.iter().enumerate() { + if file_paths_conflict(prior_artifact, &out.wasm) { + return Err(format!( + "[[outputs]][{prior_idx}].wasm {:?} conflicts with [[outputs]][{idx}].wasm {:?}", + prior_artifact, out.wasm + )); + } + } + let mirror = program_output_dest_rel(&raw.name, program_outputs.len(), out); + for prior in &output_mirrors { + if file_paths_conflict(&prior.to_string_lossy(), &mirror.to_string_lossy()) { + return Err(format!( + "program outputs collide in the resolver mirror at {} and {}", + prior.display(), + mirror.display() + )); + } + } + output_artifacts.push(&out.wasm); + output_mirrors.push(mirror); + } + for (idx, runtime_file) in runtime_files.iter().enumerate() { + for out in &program_outputs { + if file_paths_conflict(&out.wasm, &runtime_file.artifact) { + return Err(format!( + "[[runtime_files]][{idx}].artifact {:?} conflicts with [[outputs]].wasm {:?}", + runtime_file.artifact, out.wasm + )); + } + } + let mirror = Path::new(&raw.name).join(&runtime_file.artifact); + for output_mirror in &output_mirrors { + if file_paths_conflict( + &output_mirror.to_string_lossy(), + &mirror.to_string_lossy(), + ) { + return Err(format!( + "[[runtime_files]][{idx}].artifact {:?} collides with a program output mirror at {}", + runtime_file.artifact, + output_mirror.display() + )); + } + } + } + } + // Default `arches` to `["wasm32"]` when omitted. Reject // duplicates so a manifest can't say `["wasm32", "wasm32"]`. let target_arches = if raw.arches.is_empty() { @@ -1432,6 +1721,7 @@ impl DepsManifest { build: raw.build, outputs, program_outputs, + runtime_files, compatibility: raw.compatibility, binary, host_tools, @@ -1587,6 +1877,7 @@ cache_key_sha = "111111111111111111111111111111111111111111111111111111111111111 assert_eq!(m.revision, 1); assert!(m.depends_on.is_empty()); assert_eq!(m.outputs.libs, vec!["lib/libz.a"]); + assert!(m.outputs.files.is_empty()); assert_eq!(m.spec(), "zlib@1.3.1"); // Fallback (no [build].script_path) resolves against self.dir, // so the repo_root argument is irrelevant. @@ -1596,6 +1887,105 @@ cache_key_sha = "111111111111111111111111111111111111111111111111111111111111111 ); } + #[test] + fn source_and_archived_manifests_reject_unsafe_version_path_components() { + for version in [".", "..", "../escape", "nested/version", r"nested\version"] { + for (label, fixture, archived) in [ + ("source", EXAMPLE, false), + ("archived", EXAMPLE_ARCHIVED, true), + ] { + let text = fixture.replace( + "version = \"1.3.1\"", + &format!("version = {version:?}"), + ); + let result = if archived { + DepsManifest::parse_archived(&text, PathBuf::from("/x")) + } else { + DepsManifest::parse(&text, PathBuf::from("/x")) + }; + let err = result.expect_err("unsafe version must fail before path interpolation"); + assert!( + err.contains("version") && err.contains("safe single path component"), + "{label} version {version:?}: got {err}" + ); + } + } + } + + #[test] + fn source_and_archived_manifests_reject_unknown_top_level_fields() { + for (label, fixture, archived) in [ + ("source", EXAMPLE, false), + ("archived", EXAMPLE_ARCHIVED, true), + ] { + let text = fixture.replace( + "depends_on = []", + "depends_on = []\nfuture_recipe_field = true", + ); + let result = if archived { + DepsManifest::parse_archived(&text, PathBuf::from("/x")) + } else { + DepsManifest::parse(&text, PathBuf::from("/x")) + }; + let err = result.expect_err("unknown top-level field must fail loudly"); + assert!( + err.contains("future_recipe_field") && err.contains("unknown field"), + "{label}: got {err}" + ); + } + } + + #[test] + fn parses_library_runtime_file_outputs() { + let text = EXAMPLE.replace( + "headers = [\"include/zlib.h\"]", + "headers = [\"include/zlib.h\"]\nfiles = [\"share/zlib/runtime.dat\"]", + ); + let m = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap(); + assert_eq!(m.outputs.files, vec!["share/zlib/runtime.dat"]); + } + + #[test] + fn rejects_library_output_paths_outside_the_package_root() { + for (path, expected) in [ + ("/tmp/runtime.dat", "must be relative"), + ("share/../../runtime.dat", "must not contain '..'"), + ] { + let text = EXAMPLE.replace( + "headers = [\"include/zlib.h\"]", + &format!("headers = [\"include/zlib.h\"]\nfiles = [{path:?}]"), + ); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[outputs].files[0]"), "got: {err}"); + assert!(err.contains(expected), "got: {err}"); + } + } + + #[test] + fn rejects_program_output_paths_outside_the_package_root() { + let text = r#" +kind = "program" +name = "escape" +version = "1.0.0" +kernel_abi = 8 +depends_on = [] + +[source] +url = "https://example.test/escape.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" + +[license] +spdx = "MIT" + +[[outputs]] +name = "escape" +wasm = "../escape.wasm" +"#; + let err = DepsManifest::parse(text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[[outputs]][0].wasm"), "got: {err}"); + assert!(err.contains("must not contain '..'"), "got: {err}"); + } + #[test] fn build_script_override_is_repo_root_relative() { // Phase A-bis Task 2: an explicit `[build].script_path` is @@ -1864,6 +2254,36 @@ spdx = "TestLicense" assert!(DepRef::parse("zlib@").is_err()); } + #[test] + fn depref_rejects_registry_and_version_path_traversal() { + for reference in [ + "../escape@1.0", + "nested/package@1.0", + r"nested\package@1.0", + "zlib@../escape", + "zlib@nested/version", + r"zlib@nested\version", + ".@1.0", + "zlib@..", + ] { + let err = DepRef::parse(reference).unwrap_err(); + assert!( + err.contains("safe single path component"), + "{reference:?}: got {err}" + ); + } + + let text = EXAMPLE.replace( + "depends_on = []", + r#"depends_on = ["../outside@1.0"]"#, + ); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("dep name") && err.contains("safe single path component"), + "got: {err}" + ); + } + #[test] fn depends_on_parsed_into_deprefs() { let text = EXAMPLE.replace( @@ -2113,6 +2533,194 @@ wasm = "vim.wasm" assert!(m.outputs.libs.is_empty()); assert!(m.outputs.headers.is_empty()); assert!(m.outputs.pkgconfig.is_empty()); + assert!(m.outputs.files.is_empty()); + assert!(m.runtime_files.is_empty()); + } + + #[test] + fn rejects_singular_runtime_file_table_typo() { + let text = format!( + "{PROGRAM_EXAMPLE}\n[[runtime_file]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n" + ); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("runtime_file") && err.contains("unknown field"), + "got: {err}" + ); + } + + fn program_with_runtime_file(artifact: &str, guest_path: &str, mode: Option) -> String { + format!( + "{PROGRAM_EXAMPLE}\n[[runtime_files]]\nartifact = {artifact:?}\nguest_path = {guest_path:?}\n{}", + mode.map(|value| format!("mode = {value}\n")).unwrap_or_default(), + ) + } + + #[test] + fn parses_program_runtime_file_and_stable_mirror_path() { + let text = program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", None); + let m = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap(); + assert_eq!( + m.runtime_files, + vec![RuntimeFile { + artifact: "icu.dat".into(), + guest_path: "/usr/lib/php/icu.dat".into(), + mode: 0o644, + }] + ); + assert_eq!( + m.runtime_file_dest_rel("icu.dat").unwrap(), + PathBuf::from("vim/icu.dat") + ); + // Runtime-file placement must not perturb the legacy single-output + // destination convention. + assert_eq!( + m.output_dest_rel("vim.wasm").unwrap(), + PathBuf::from("vim.wasm") + ); + + let nested = program_with_runtime_file("share/icu/icu.dat", "/usr/lib/php/icu.dat", None); + let nested = DepsManifest::parse(&nested, PathBuf::from("/x")).unwrap(); + assert_eq!( + nested.runtime_file_dest_rel("share/icu/icu.dat").unwrap(), + PathBuf::from("vim/share/icu/icu.dat") + ); + assert!(nested.runtime_file_dest_rel("icu.dat").is_err()); + } + + #[test] + fn rejects_unsafe_runtime_artifact_paths() { + for (artifact, expected) in [ + ("", "must not be empty"), + ("/icu.dat", "must be relative"), + ("../icu.dat", "must not contain '..'"), + ("./icu.dat", "must not contain '.'"), + ("share//icu.dat", "empty path components"), + ("share/icu.dat/", "empty path components"), + (r"share\icu.dat", "portable"), + ] { + let text = program_with_runtime_file(artifact, "/usr/lib/php/icu.dat", None); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[[runtime_files]][0].artifact"), "got: {err}"); + assert!(err.contains(expected), "got: {err}"); + } + } + + #[test] + fn rejects_unsafe_runtime_guest_paths() { + for (guest_path, expected) in [ + ("icu.dat", "non-root absolute"), + ("/", "non-root absolute"), + ("/usr/lib/php/", "trailing"), + ("/usr//php/icu.dat", "empty path components"), + ("/usr/./icu.dat", "must not contain"), + ("/usr/../icu.dat", "must not contain"), + (r"/usr/lib\icu.dat", "normalized absolute POSIX"), + ] { + let text = program_with_runtime_file("icu.dat", guest_path, None); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("[[runtime_files]][0].guest_path"), + "got: {err}" + ); + assert!(err.contains(expected), "got: {err}"); + } + } + + #[test] + fn rejects_runtime_file_on_non_program_and_invalid_mode() { + let library = format!( + "{EXAMPLE}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n" + ); + let err = DepsManifest::parse(&library, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("program"), "got: {err}"); + + let text = program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", Some(0o1000)); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("0777"), "got: {err}"); + + let unknown = format!( + "{PROGRAM_EXAMPLE}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\nmod = 384\n" + ); + let err = DepsManifest::parse(&unknown, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("unknown field") && err.contains("mod"), + "got: {err}" + ); + } + + #[test] + fn rejects_duplicate_and_colliding_runtime_files() { + let duplicate = format!( + "{}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/other.dat\"\n", + program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", None), + ); + let err = DepsManifest::parse(&duplicate, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("artifact") && err.contains("more than once"), + "got: {err}" + ); + + let duplicate_guest = format!( + "{}\n[[runtime_files]]\nartifact = \"other.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n", + program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", None), + ); + let err = DepsManifest::parse(&duplicate_guest, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("guest_path") && err.contains("more than once"), + "got: {err}" + ); + + let artifact_collision = PROGRAM_EXAMPLE.replace( + "wasm = \"vim.wasm\"", + "wasm = \"vim.wasm\"\n\n[[runtime_files]]\nartifact = \"vim.wasm\"\nguest_path = \"/usr/lib/vim.wasm\"", + ); + let err = DepsManifest::parse(&artifact_collision, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("conflict"), "got: {err}"); + + let prefix = format!( + "{}\n[[runtime_files]]\nartifact = \"share/icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n", + program_with_runtime_file("share", "/usr/lib/php", None), + ); + let err = DepsManifest::parse(&prefix, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("ancestor"), "got: {err}"); + + let output_prefix = PROGRAM_EXAMPLE.replace( + "wasm = \"vim.wasm\"", + "wasm = \"share\"\n\n[[runtime_files]]\nartifact = \"share/icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"", + ); + let err = DepsManifest::parse(&output_prefix, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("conflicts with [[outputs]].wasm"), + "got: {err}" + ); + } + + #[test] + fn rejects_unsafe_names_and_duplicate_program_mirrors() { + for package_name in ["../escape", "/absolute", "a/b", r"a\b"] { + let text = + PROGRAM_EXAMPLE.replacen("name = \"vim\"", &format!("name = {package_name:?}"), 1); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("safe single path component"), "got: {err}"); + } + + let unsafe_output = PROGRAM_EXAMPLE.replace( + "[[outputs]]\nname = \"vim\"", + "[[outputs]]\nname = \"../vim\"", + ); + let err = DepsManifest::parse(&unsafe_output, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[[outputs]][0].name"), "got: {err}"); + + let duplicate_mirror = PROGRAM_EXAMPLE.replace( + "[[outputs]]\nname = \"vim\"\nwasm = \"vim.wasm\"", + "[[outputs]]\nname = \"same\"\nwasm = \"a.wasm\"\n\n[[outputs]]\nname = \"same\"\nwasm = \"b.wasm\"", + ); + let err = DepsManifest::parse(&duplicate_mirror, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("collide") && err.contains("same.wasm"), + "got: {err}" + ); } #[test]