From 7a39fb94f35f8065f0d84b5ec6f3031b24340017 Mon Sep 17 00:00:00 2001 From: mho22 Date: Sun, 12 Jul 2026 05:05:20 -0400 Subject: [PATCH 01/12] php: add the initial runtime-optional intl side-module path Introduce the initial ICU and PIC libc++ package outputs, C++ weak-self-import coverage, and the PHP intl side-module link path while keeping ICU data outside the base PHP executable. Follow-up commits in this serial reconcile the ABI-19 build, fail-closed linker behavior, reproducibility, runtime closure, fork replay, and host/browser validation. This commit intentionally makes no standalone completion or validation claim. Co-authored-by: Claude Opus 4.8 (1M context) --- host/test/dylink.test.ts | 88 ++++++++++ packages/registry/icu/build-icu.sh | 176 +++++++++++++++++++ packages/registry/icu/build.toml | 10 ++ packages/registry/icu/package.toml | 45 +++++ packages/registry/libcxx/.gitignore | 2 + packages/registry/libcxx/build-libcxx.sh | 167 +++++++++++------- packages/registry/libcxx/build.toml | 5 +- packages/registry/libcxx/package.toml | 8 +- packages/registry/php/build-php.sh | 46 +++++ packages/registry/php/intl-icu-data-loader.c | 109 ++++++++++++ packages/registry/php/test/php-intl.test.ts | 95 ++++++++++ 11 files changed, 686 insertions(+), 65 deletions(-) create mode 100755 packages/registry/icu/build-icu.sh create mode 100644 packages/registry/icu/build.toml create mode 100644 packages/registry/icu/package.toml create mode 100644 packages/registry/php/intl-icu-data-loader.c create mode 100644 packages/registry/php/test/php-intl.test.ts diff --git a/host/test/dylink.test.ts b/host/test/dylink.test.ts index b493423e3..f07ba5820 100644 --- a/host/test/dylink.test.ts +++ b/host/test/dylink.test.ts @@ -912,3 +912,91 @@ describe.skipIf(!hasCompiler())("DynamicLinker", () => { expect(h1).toBe(h2); }); }); + +function hasWat2Wasm(): boolean { + try { + execFileSync("wat2wasm", ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +// A minimal dylink.0 section (MEM_INFO: mem/table size + align all 0), which +// marks a module as a side module. wat2wasm emits custom sections last, but the +// loader requires dylink.0 first, so it is injected as raw bytes below rather +// than declared in the WAT. +const DYLINK_SECTION = new Uint8Array([ + 0x00, 0x0f, // custom section, size 15 + 0x08, 0x64, 0x79, 0x6c, 0x69, 0x6e, 0x6b, 0x2e, 0x30, // name "dylink.0" + 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, // MEM_INFO subsection +]); + +/** + * Assemble a hand-written side module. Using WAT (rather than a compiled C/C++ + * fixture) lets these tests reproduce the exact import shapes a real C++ side + * module produces — a self-defined symbol that is *also* an env import, and an + * env.__cpp_exception tag — which no self-contained C fixture can emit. + */ +function assembleSideModule(wat: string, name: string): Uint8Array { + const dir = join(tmpdir(), "wasm-dylink-test"); + mkdirSync(dir, { recursive: true }); + const watPath = join(dir, `${name}.wat`); + const wasmPath = join(dir, `${name}.wasm`); + writeFileSync(watPath, wat); + execFileSync("wat2wasm", ["--enable-exceptions", watPath, "-o", wasmPath], + { stdio: "pipe" }); + const raw = new Uint8Array(readFileSync(wasmPath)); + const out = new Uint8Array(8 + DYLINK_SECTION.length + (raw.length - 8)); + out.set(raw.subarray(0, 8), 0); // magic + version + out.set(DYLINK_SECTION, 8); // dylink.0 first + out.set(raw.subarray(8), 8 + DYLINK_SECTION.length); + return out; +} + +describe.skipIf(!hasWat2Wasm())("weak self-import handling", () => { + function createLoadOptions(): LoadSharedLibraryOptions { + return { + memory: new WebAssembly.Memory({ initial: 1, maximum: 100, shared: true }), + table: new WebAssembly.Table({ initial: 1, element: "anyfunc" }), + stackPointer: new WebAssembly.Global({ value: "i32", mutable: true }, 65536), + heapPointer: { value: 1024 }, + globalSymbols: new Map(), + got: new Map(), + loadedLibraries: new Map(), + }; + } + + it("routes an unresolved env import to the module's own export", () => { + // `self_fn` is both imported from env and exported: the shape wasm-ld emits + // for an interposable weak C++ symbol the module also defines. + const lib = loadSharedLibrarySync("self-import.so", assembleSideModule(` + (module + (import "env" "self_fn" (func $self_fn (result i32))) + (func (export "self_fn") (result i32) (i32.const 42)) + (func (export "call_self") (result i32) (call $self_fn))) + `, "self-import"), createLoadOptions()); + expect((lib.exports.call_self as Function)()).toBe(42); + }); + + it("throws loudly when a self-import has no defining export", () => { + // A genuinely absent symbol must trap at call time, not silently return 0. + const lib = loadSharedLibrarySync("missing-import.so", assembleSideModule(` + (module + (import "env" "missing_fn" (func $missing (result i32))) + (func (export "call_missing") (result i32) (call $missing))) + `, "missing-import"), createLoadOptions()); + expect(() => (lib.exports.call_missing as Function)()).toThrow(/not provided/); + }); + + it("provides the __cpp_exception tag to -fwasm-exceptions modules", () => { + // C++ side modules import this tag; without the host providing it, the + // module fails to instantiate with "tag import requires a WebAssembly.Tag". + const lib = loadSharedLibrarySync("tag.so", assembleSideModule(` + (module + (import "env" "__cpp_exception" (tag $exc (param i32))) + (func (export "noop"))) + `, "tag"), createLoadOptions()); + expect(lib.exports.noop).toBeTypeOf("function"); + }); +}); diff --git a/packages/registry/icu/build-icu.sh b/packages/registry/icu/build-icu.sh new file mode 100755 index 000000000..0ec6757c4 --- /dev/null +++ b/packages/registry/icu/build-icu.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# +# Build ICU4C (libicuuc.a, libicui18n.a, libicudata.a stub + icu.dat) for +# wasm32-posix-kernel. +# +# ICU requires a TWO-STAGE build: +# +# Stage 1 (HOST): build ICU natively to produce the data-generation tools +# (genrb, pkgdata, icupkg, genccode, …) and the ICU common +# data. These run on the build machine. +# Stage 2 (CROSS): configure ICU for wasm32 with --with-cross-build pointing +# at the stage-1 build dir. The cross build reuses the host +# tools and host-generated data; it only compiles the C++ +# sources into wasm32 static libraries. +# +# Data is built in `archive` packaging mode, which emits the ICU common data as +# a standalone `icudtl.dat` file (NOT linked into libicudata.a — that +# becomes a stub). We install that file as `share/icu.dat`; PHP's intl side +# module loads it at runtime via udata_setCommonData() (the name `icu.dat` is +# deliberate and is NOT ICU's default-searched name). See +# packages/registry/php/build-php.sh for the intl side. +# +# Honors the dep-resolver build-script contract (see docs/package-management.md). +# When invoked via `cargo xtask build-deps resolve icu`, the resolver sets: +# WASM_POSIX_DEP_OUT_DIR # where to install +# WASM_POSIX_DEP_VERSION # upstream version (e.g. "74.2") +# WASM_POSIX_DEP_SOURCE_URL # tarball URL +# WASM_POSIX_DEP_SOURCE_SHA256 # expected sha256 of the tarball +# WASM_POSIX_DEP_LIBCXX_DIR # resolved libcxx prefix (direct dep) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +# shellcheck source=/dev/null +source "$REPO_ROOT/sdk/activate.sh" + +if ! command -v wasm32posix-cc &>/dev/null; then + echo "ERROR: wasm32posix-cc not found after sourcing sdk/activate.sh." >&2 + exit 1 +fi + +# --- Inputs from resolver, with ad-hoc fallbacks --- +ICU_VERSION="${WASM_POSIX_DEP_VERSION:-${ICU_VERSION:-74.2}}" +ICU_VER_UNDERSCORE="${ICU_VERSION//./_}" # 74.2 -> 74_2 +ICU_MAJOR="${ICU_VERSION%%.*}" # 74.2 -> 74 +INSTALL_DIR="${WASM_POSIX_DEP_OUT_DIR:-$SCRIPT_DIR/icu-install}" +SOURCE_URL="${WASM_POSIX_DEP_SOURCE_URL:-https://github.com/unicode-org/icu/releases/download/release-${ICU_MAJOR}-${ICU_VERSION#*.}/icu4c-${ICU_VER_UNDERSCORE}-src.tgz}" +SOURCE_SHA256="${WASM_POSIX_DEP_SOURCE_SHA256:-}" + +SYSROOT="${WASM_POSIX_SYSROOT:-$REPO_ROOT/sysroot}" +export WASM_POSIX_SYSROOT="$SYSROOT" + +SRC_ROOT="$SCRIPT_DIR/icu-src" # contains icu/ (with source/) +ICU_SRC="$SRC_ROOT/icu/source" +HOST_BUILD="$SCRIPT_DIR/host-build" # stage-1 native build (out-of-tree) + +# --- Resolve libcxx (ICU is C++), symlink into sysroot (mariadb pattern) --- +HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" +resolve_dep() { + (cd "$REPO_ROOT" && cargo run -p xtask --target "$HOST_TARGET" --quiet -- build-deps resolve "$1") +} +LIBCXX_PREFIX="${WASM_POSIX_DEP_LIBCXX_DIR:-}" +if [ -z "$LIBCXX_PREFIX" ]; then + echo "==> Resolving libcxx via cargo xtask build-deps..." + LIBCXX_PREFIX="$(resolve_dep libcxx)" +fi +[ -f "$LIBCXX_PREFIX/lib/libc++.a" ] || { echo "ERROR: libcxx resolve missing libc++.a at $LIBCXX_PREFIX" >&2; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++abi.a" ] || { echo "ERROR: libcxx resolve missing libc++abi.a at $LIBCXX_PREFIX" >&2; exit 1; } +[ -d "$LIBCXX_PREFIX/include/c++/v1" ] || { echo "ERROR: libcxx resolve missing include/c++/v1 at $LIBCXX_PREFIX" >&2; exit 1; } + +echo "==> Linking libcxx into sysroot ($LIBCXX_PREFIX)..." +mkdir -p "$SYSROOT/lib" "$SYSROOT/include/c++" +ln -sf "$LIBCXX_PREFIX/lib/libc++.a" "$SYSROOT/lib/libc++.a" +ln -sf "$LIBCXX_PREFIX/lib/libc++abi.a" "$SYSROOT/lib/libc++abi.a" +rm -rf "$SYSROOT/include/c++/v1" +ln -sfn "$LIBCXX_PREFIX/include/c++/v1" "$SYSROOT/include/c++/v1" + +# --- Fetch + verify source --- +if [ ! -d "$ICU_SRC" ]; then + echo "==> Downloading ICU $ICU_VERSION..." + TARBALL="/tmp/icu4c-${ICU_VER_UNDERSCORE}-src.tgz" + curl --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors -fsSL "$SOURCE_URL" -o "$TARBALL" + if [ -n "$SOURCE_SHA256" ]; then + echo "==> Verifying source sha256..." + echo "$SOURCE_SHA256 $TARBALL" | shasum -a 256 -c - + fi + mkdir -p "$SRC_ROOT" + tar xzf "$TARBALL" -C "$SRC_ROOT" # extracts icu/ + rm "$TARBALL" +fi + +NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" + +# ============================================================ +# Stage 1 — HOST build (native tools + data) +# ============================================================ +# Uses the host compiler (clang/clang++ from the dev shell), NOT the wasm +# wrappers. sdk/activate.sh only prepends SDK bin to PATH; it does not export +# CC/CXX, so an explicit host CC/CXX keeps this stage native. +if [ ! -x "$HOST_BUILD/bin/icupkg" ] && [ ! -x "$HOST_BUILD/bin/genccode" ]; then + echo "==> Stage 1: building ICU natively for host tools + data..." + rm -rf "$HOST_BUILD" + mkdir -p "$HOST_BUILD" + ( cd "$HOST_BUILD" + CC="${HOST_CC:-clang}" CXX="${HOST_CXX:-clang++}" \ + "$ICU_SRC/runConfigureICU" MacOSX \ + --enable-static --disable-shared \ + --disable-samples --disable-tests --disable-extras + make -j"$NPROC" + ) +else + echo "==> Stage 1: reusing existing host build at $HOST_BUILD" +fi + +# ============================================================ +# Stage 2 — CROSS build (wasm32 static libs) +# ============================================================ +echo "==> Stage 2: cross-configuring ICU for wasm32..." +# In-tree cross build (wasm32posix-configure runs ./configure in CWD). +# Scrub any prior cross-build state in the source tree. +cd "$ICU_SRC" +make distclean 2>/dev/null || true + +# ICU maps the configure host triple to a config/mh- makefile +# fragment. Our SDK forces --host=wasm32-unknown-none, whose OS component +# ("none") ICU does not recognize, so it selects the stock config/mh-unknown — +# a stub that hard-errors "configure could not detect your platform" and aborts +# `make`. ICU's own remedy (printed in that error) is to supply mh-unknown from +# a known platform. We use mh-linux: this is a --disable-shared --enable-static +# build, so mh-linux's Linux shared-library rules are never exercised; only its +# generic compile rules apply, driven by our wasm CC/CXX. Idempotent overwrite, +# re-applied every run because a fresh source extraction resets it. +cp "$ICU_SRC/config/mh-linux" "$ICU_SRC/config/mh-unknown" + +# C++ flags: ICU 74 needs C++17. libc++ headers come from the sysroot symlink. +# LDFLAGS carries -lc++ -lc++abi so configure's C++ link probes resolve. +# -fPIC: ICU's static libs are absorbed into intl.so, a wasm SIDE MODULE linked +# with `-shared --experimental-pic`. wasm-ld requires EVERY input object to be +# position-independent; a non-PIC ICU object triggers "R_WASM_MEMORY_ADDR_SLEB +# cannot be used against symbol ...; recompile with -fPIC" at the intl.so link. +CXXFLAGS="-O2 -std=c++17 -fPIC" \ +CFLAGS="-O2 -fPIC" \ +LDFLAGS="-lc++ -lc++abi" \ +wasm32posix-configure \ + --with-cross-build="$HOST_BUILD" \ + --enable-static --disable-shared \ + --disable-tools --disable-tests --disable-samples --disable-extras \ + --disable-layoutex \ + --with-data-packaging=archive \ + --prefix="$INSTALL_DIR" + +echo "==> Stage 2: building wasm32 libraries..." +make -j"$NPROC" + +echo "==> Installing to $INSTALL_DIR..." +rm -rf "$INSTALL_DIR" +make install + +# --- Stage the common data as icu.dat (see header) --- +DAT_SRC="$(find "$ICU_SRC/data" "$HOST_BUILD/data" -name "icudt${ICU_MAJOR}l.dat" 2>/dev/null | head -1 || true)" +if [ -z "$DAT_SRC" ]; then + echo "ERROR: could not locate icudt${ICU_MAJOR}l.dat after build" >&2 + exit 1 +fi +mkdir -p "$INSTALL_DIR/share" +cp "$DAT_SRC" "$INSTALL_DIR/share/icu.dat" +echo "==> staged $(basename "$DAT_SRC") -> $INSTALL_DIR/share/icu.dat ($(wc -c < "$INSTALL_DIR/share/icu.dat") bytes)" + +# --- Sanity: the static libs we promise (icuio included: PHP's PHP_SETUP_ICU +# requires the icu-io pkg-config module, so intl won't configure without it) --- +for lib in libicuuc.a libicui18n.a libicuio.a libicudata.a; do + [ -f "$INSTALL_DIR/lib/$lib" ] || { echo "ERROR: missing $INSTALL_DIR/lib/$lib" >&2; exit 1; } +done +echo "==> ICU build complete." +ls -lh "$INSTALL_DIR/lib/"*.a "$INSTALL_DIR/share/icu.dat" diff --git a/packages/registry/icu/build.toml b/packages/registry/icu/build.toml new file mode 100644 index 000000000..a7637ccfb --- /dev/null +++ b/packages/registry/icu/build.toml @@ -0,0 +1,10 @@ +script_path = "packages/registry/icu/build-icu.sh" +repo_url = "https://github.com/Automattic/kandelo.git" +commit = "e96fbe3964d5ec6784f00f4d49bfcf70a2030e22" +# Revision 3: ICU static libs rebuilt with -fPIC so they can be absorbed into +# intl.so (a wasm side module built with -shared --experimental-pic, which +# requires all inputs to be position-independent). +revision = 3 + +[binary] +index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/icu/package.toml b/packages/registry/icu/package.toml new file mode 100644 index 000000000..355f39786 --- /dev/null +++ b/packages/registry/icu/package.toml @@ -0,0 +1,45 @@ +kind = "library" +name = "icu" +version = "74.2" +kernel_abi = 7 +# ICU4C is C++; it links libc++/libc++abi from the libcxx package. +depends_on = ["libcxx@21.1.7"] +# wasm32 only for now — the sole consumer is PHP's intl side module, +# which is a wasm32 build. Add wasm64 only if a wasm64 consumer appears. +arches = ["wasm32"] + +# ICU4C for wasm32, built as static libraries: +# lib/libicuuc.a (common) +# lib/libicui18n.a (internationalization) +# lib/libicudata.a (stubdata — real data lives in the .dat, see below) +# include/unicode/*.h +# share/icu.dat (common data archive; renamed from icudt74l.dat per +# the intl side-module design — loaded at runtime via +# udata_setCommonData, NOT ICU's default name search) +# +# ICU requires a two-stage build: a HOST build (to generate the data and the +# genrb/pkgdata/icupkg tools) followed by a wasm32 cross build pointed at the +# host build via --with-cross-build. Data is built in `archive` packaging mode +# so the common data is emitted as a standalone icudt74l.dat instead of being +# linked into libicudata.a; we stage that file as `icu.dat`. + +[source] +url = "https://github.com/unicode-org/icu/releases/download/release-74-2/icu4c-74_2-src.tgz" +sha256 = "68db082212a96d6f53e35d60f47d38b962e9f9d207a74cfac78029ae8ff5e08c" + +[license] +spdx = "ICU" +url = "https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE" + +[build] +script_path = "packages/registry/icu/build-icu.sh" + +[outputs] +libs = ["lib/libicuuc.a", "lib/libicui18n.a", "lib/libicuio.a", "lib/libicudata.a"] +headers = ["include/unicode"] +# NOTE: the build also installs share/icu.dat (the ICU common data archive, +# staged from icudtl.dat). It is not declarable as a library output here +# (Outputs only takes libs/headers/pkgconfig); consumers such as PHP's intl +# side module read it directly from the resolved dep dir +# ($WASM_POSIX_DEP_ICU_DIR/share/icu.dat) and load it at runtime via +# udata_setCommonData(). See build-icu.sh. diff --git a/packages/registry/libcxx/.gitignore b/packages/registry/libcxx/.gitignore index 201dd024e..d13a937fe 100644 --- a/packages/registry/libcxx/.gitignore +++ b/packages/registry/libcxx/.gitignore @@ -1,5 +1,7 @@ node_modules/ # LLVM source clone (per major version) — build script populates on demand. llvm-project-*/ +# Per-arch LLVM source symlink tree the build script recreates each run. +llvm-source-*/ # Per-arch build trees produced by the build script. build-*/ diff --git a/packages/registry/libcxx/build-libcxx.sh b/packages/registry/libcxx/build-libcxx.sh index 96396ed8f..a89b35af2 100755 --- a/packages/registry/libcxx/build-libcxx.sh +++ b/packages/registry/libcxx/build-libcxx.sh @@ -103,7 +103,10 @@ if [ ! -d "$NIX_LIBUNWIND_SOURCE/libunwind" ]; then fi BUILD_DIR="$SCRIPT_DIR/build-${ARCH}" -LLVM_SRC_DIR="$BUILD_DIR/llvm-source" +# Assembled source tree lives OUTSIDE the build dirs so both the default +# (static, non-PIC) build and the position-independent build below can share it +# without one build's `rm -rf` deleting the other's source. +LLVM_SRC_DIR="$SCRIPT_DIR/llvm-source-${ARCH}" # --- Verify prerequisites --- if [ ! -f "$SYSROOT/lib/libc.a" ]; then @@ -136,10 +139,9 @@ echo "==> Building libc++ and libc++abi for ${ARCH}..." # SDK's `compileFlags` was updated in lock-step). WASM_C_FLAGS="--target=${WASM_TARGET} -matomics -mbulk-memory -mexception-handling -mllvm -wasm-enable-sjlj -mllvm -wasm-use-legacy-eh=false -fexceptions -fno-trapping-math --sysroot=${SYSROOT} -O2 -DNDEBUG" -# Always start with a fresh build tree so a cache-miss rebuild does -# not mix old + new cmake artifacts. -rm -rf "$BUILD_DIR" -mkdir -p "$BUILD_DIR" +# Start with a fresh source tree so a cache-miss rebuild does not mix old + new +# artifacts. (Each build dir is cleaned by build_libcxx_variant below.) +rm -rf "$LLVM_SRC_DIR" # Assemble the monorepo-shaped source tree expected by runtimes/CMakeLists.txt # from exact Nix source derivations. Nix's libcxx source carries runtimes/, @@ -160,63 +162,77 @@ for entry in "$NIX_LIBCXX_SOURCE/runtimes"/*; do done ln -s "$NIX_LIBUNWIND_SOURCE/libunwind" "$LLVM_SRC_DIR/libunwind" -cd "$BUILD_DIR" +NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" -cmake -G "Unix Makefiles" -S "$LLVM_SRC_DIR/runtimes" \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DCMAKE_SYSTEM_NAME=Generic \ - -DCMAKE_SYSTEM_PROCESSOR="${ARCH}" \ - -DCMAKE_C_COMPILER="$LLVM_CLANG" \ - -DCMAKE_CXX_COMPILER="$LLVM_CLANG" \ - -DCMAKE_AR="$LLVM_AR" \ - -DCMAKE_RANLIB="$LLVM_RANLIB" \ - -DCMAKE_NM="$LLVM_NM" \ - -DCMAKE_C_COMPILER_TARGET="${WASM_TARGET}" \ - -DCMAKE_CXX_COMPILER_TARGET="${WASM_TARGET}" \ - -DCMAKE_C_FLAGS="${WASM_C_FLAGS}" \ - -DCMAKE_CXX_FLAGS="${WASM_C_FLAGS}" \ - -DCMAKE_SYSROOT="${SYSROOT}" \ - -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ - \ - -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_ENABLE_STATIC=ON \ - -DLIBCXX_ENABLE_EXCEPTIONS=ON \ - -DLIBCXX_ENABLE_RTTI=ON \ - -DLIBCXX_HAS_MUSL_LIBC=ON \ - -DLIBCXX_HAS_PTHREAD_API=ON \ - -DLIBCXX_CXX_ABI=libcxxabi \ - -DLIBCXX_INCLUDE_BENCHMARKS=OFF \ - -DLIBCXX_INCLUDE_TESTS=OFF \ - -DLIBCXX_ENABLE_FILESYSTEM=ON \ - -DLIBCXX_ENABLE_MONOTONIC_CLOCK=ON \ - -DLIBCXX_ENABLE_RANDOM_DEVICE=OFF \ - -DLIBCXX_ENABLE_LOCALIZATION=ON \ - -DLIBCXX_ENABLE_WIDE_CHARACTERS=ON \ - -DLIBCXX_ENABLE_NEW_DELETE_DEFINITIONS=ON \ - \ - -DLIBCXXABI_ENABLE_SHARED=OFF \ - -DLIBCXXABI_ENABLE_STATIC=ON \ - -DLIBCXXABI_ENABLE_EXCEPTIONS=ON \ - -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ - -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ - -DLIBCXXABI_STATICALLY_LINK_UNWINDER_IN_STATIC_LIBRARY=ON \ - -DLIBCXXABI_ENABLE_THREADS=ON \ - -DLIBCXXABI_HAS_PTHREAD_API=ON \ - -DLIBCXXABI_INCLUDE_TESTS=OFF \ - \ - -DLIBUNWIND_ENABLE_SHARED=OFF \ - -DLIBUNWIND_ENABLE_STATIC=ON \ - -DLIBUNWIND_ENABLE_THREADS=ON \ - -DLIBUNWIND_USE_COMPILER_RT=OFF \ - -DLIBUNWIND_INCLUDE_TESTS=OFF \ - -DLIBUNWIND_HIDE_SYMBOLS=ON \ - \ - -DCMAKE_SIZEOF_VOID_P="${SIZEOF_VOID_P}" \ - 2>&1 | tail -20 +# Configure + build libc++/libc++abi/libunwind into with the given +# compile-flags string (plus any extra cmake args). Factored so the default +# static archives and the position-independent variant (below) share ONE cmake +# recipe and cannot drift apart. +build_libcxx_variant() { + local variant_build_dir="$1"; shift + local variant_c_flags="$1"; shift + rm -rf "$variant_build_dir" + mkdir -p "$variant_build_dir" + ( cd "$variant_build_dir" + cmake -G "Unix Makefiles" -S "$LLVM_SRC_DIR/runtimes" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DCMAKE_SYSTEM_NAME=Generic \ + -DCMAKE_SYSTEM_PROCESSOR="${ARCH}" \ + -DCMAKE_C_COMPILER="$LLVM_CLANG" \ + -DCMAKE_CXX_COMPILER="$LLVM_CLANG" \ + -DCMAKE_AR="$LLVM_AR" \ + -DCMAKE_RANLIB="$LLVM_RANLIB" \ + -DCMAKE_NM="$LLVM_NM" \ + -DCMAKE_C_COMPILER_TARGET="${WASM_TARGET}" \ + -DCMAKE_CXX_COMPILER_TARGET="${WASM_TARGET}" \ + -DCMAKE_C_FLAGS="${variant_c_flags}" \ + -DCMAKE_CXX_FLAGS="${variant_c_flags}" \ + -DCMAKE_SYSROOT="${SYSROOT}" \ + -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ + \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC=ON \ + -DLIBCXX_ENABLE_EXCEPTIONS=ON \ + -DLIBCXX_ENABLE_RTTI=ON \ + -DLIBCXX_HAS_MUSL_LIBC=ON \ + -DLIBCXX_HAS_PTHREAD_API=ON \ + -DLIBCXX_CXX_ABI=libcxxabi \ + -DLIBCXX_INCLUDE_BENCHMARKS=OFF \ + -DLIBCXX_INCLUDE_TESTS=OFF \ + -DLIBCXX_ENABLE_FILESYSTEM=ON \ + -DLIBCXX_ENABLE_MONOTONIC_CLOCK=ON \ + -DLIBCXX_ENABLE_RANDOM_DEVICE=OFF \ + -DLIBCXX_ENABLE_LOCALIZATION=ON \ + -DLIBCXX_ENABLE_WIDE_CHARACTERS=ON \ + -DLIBCXX_ENABLE_NEW_DELETE_DEFINITIONS=ON \ + \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DLIBCXXABI_ENABLE_STATIC=ON \ + -DLIBCXXABI_ENABLE_EXCEPTIONS=ON \ + -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ + -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ + -DLIBCXXABI_STATICALLY_LINK_UNWINDER_IN_STATIC_LIBRARY=ON \ + -DLIBCXXABI_ENABLE_THREADS=ON \ + -DLIBCXXABI_HAS_PTHREAD_API=ON \ + -DLIBCXXABI_INCLUDE_TESTS=OFF \ + \ + -DLIBUNWIND_ENABLE_SHARED=OFF \ + -DLIBUNWIND_ENABLE_STATIC=ON \ + -DLIBUNWIND_ENABLE_THREADS=ON \ + -DLIBUNWIND_USE_COMPILER_RT=OFF \ + -DLIBUNWIND_INCLUDE_TESTS=OFF \ + -DLIBUNWIND_HIDE_SYMBOLS=ON \ + \ + -DCMAKE_SIZEOF_VOID_P="${SIZEOF_VOID_P}" \ + "$@" \ + 2>&1 | tail -20 -echo "==> Compiling (this may take a few minutes)..." -NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" -make -j"$NPROC" cxx cxxabi unwind 2>&1 | tail -10 + echo "==> Compiling (this may take a few minutes)..." + make -j"$NPROC" cxx cxxabi unwind 2>&1 | tail -10 ) +} + +echo "==> Building default (static, non-PIC) libc++/libc++abi for ${ARCH}..." +build_libcxx_variant "$BUILD_DIR" "${WASM_C_FLAGS}" # --- Install into the resolver's OUT_DIR --- echo "==> Installing to $INSTALL_DIR..." @@ -298,7 +314,32 @@ if ! "$LLVM_CLANG" ${WASM_C_FLAGS} \ fi echo "==> Header smoke compile passed." +# --- Position-independent variant for wasm side modules --- +# The default archives above are non-PIC, which is correct for the common case: +# static linking into a main wasm module (php.wasm, mariadb, ruby). But a wasm +# SIDE MODULE (built with `-shared --experimental-pic`, e.g. PHP's intl.so, which +# statically absorbs libc++/libc++abi) requires EVERY input object to be +# position-independent, or wasm-ld fails with "relocation R_WASM_MEMORY_ADDR_SLEB +# cannot be used against symbol ...; recompile with -fPIC". Emit a parallel PIC +# pair alongside the defaults. This is purely additive: libc++.a / libc++abi.a +# and the header set above are untouched, so existing static consumers are +# unaffected; only side-module consumers reach for the -pic archives. +echo "==> Building position-independent libc++/libc++abi (for wasm side modules)..." +PIC_BUILD_DIR="$SCRIPT_DIR/build-${ARCH}-pic" +build_libcxx_variant "$PIC_BUILD_DIR" "${WASM_C_FLAGS} -fPIC" -DCMAKE_POSITION_INDEPENDENT_CODE=ON + +LIBCXX_PIC_A=$(find "$PIC_BUILD_DIR" -name "libc++.a" -not -path "*/CMakeFiles/*" | head -1) +LIBCXXABI_PIC_A=$(find "$PIC_BUILD_DIR" -name "libc++abi.a" -not -path "*/CMakeFiles/*" | head -1) +if [ -z "$LIBCXX_PIC_A" ] || [ -z "$LIBCXXABI_PIC_A" ]; then + echo "ERROR: PIC libraries not found under $PIC_BUILD_DIR" >&2 + exit 1 +fi +cp "$LIBCXX_PIC_A" "$INSTALL_DIR/lib/libc++-pic.a" +cp "$LIBCXXABI_PIC_A" "$INSTALL_DIR/lib/libc++abi-pic.a" + echo "==> Done!" -echo " libc++.a: $(wc -c < "$INSTALL_DIR/lib/libc++.a" | tr -d ' ') bytes" -echo " libc++abi.a: $(wc -c < "$INSTALL_DIR/lib/libc++abi.a" | tr -d ' ') bytes" -echo " headers: $INSTALL_DIR/include/c++/v1/" +echo " libc++.a: $(wc -c < "$INSTALL_DIR/lib/libc++.a" | tr -d ' ') bytes" +echo " libc++abi.a: $(wc -c < "$INSTALL_DIR/lib/libc++abi.a" | tr -d ' ') bytes" +echo " libc++-pic.a: $(wc -c < "$INSTALL_DIR/lib/libc++-pic.a" | tr -d ' ') bytes" +echo " libc++abi-pic.a: $(wc -c < "$INSTALL_DIR/lib/libc++abi-pic.a" | tr -d ' ') bytes" +echo " headers: $INSTALL_DIR/include/c++/v1/" diff --git a/packages/registry/libcxx/build.toml b/packages/registry/libcxx/build.toml index 875768a9e..9e316ba77 100644 --- a/packages/registry/libcxx/build.toml +++ b/packages/registry/libcxx/build.toml @@ -7,7 +7,10 @@ commit = "8c53383229fab78f97b098c3207a655159c03041" # source derivations, hard-fails on compiler/source version drift, and # installs headers from the build tree so the header set cannot drift # from the built library. -revision = 5 +# Revision 6: additionally emits position-independent libc++-pic.a / +# libc++abi-pic.a (a second -fPIC build) so wasm side modules like PHP's +# intl.so can statically absorb libc++. The non-PIC pair is unchanged. +revision = 6 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/libcxx/package.toml b/packages/registry/libcxx/package.toml index 49d406570..9fcc01e8c 100644 --- a/packages/registry/libcxx/package.toml +++ b/packages/registry/libcxx/package.toml @@ -42,5 +42,11 @@ url = "https://github.com/llvm/llvm-project/blob/main/LICENSE.TXT" script_path = "packages/registry/libcxx/build-libcxx.sh" [outputs] -libs = ["lib/libc++.a", "lib/libc++abi.a"] +# libc++.a / libc++abi.a are the default non-PIC archives for static linking +# into a main wasm module (php.wasm, mariadb, ruby). libc++-pic.a / +# libc++abi-pic.a are the position-independent variants required by wasm SIDE +# MODULES (`-shared --experimental-pic`), e.g. PHP's intl.so, which statically +# absorbs libc++ and would otherwise hit a wasm-ld "recompile with -fPIC" error. +# Additive: existing consumers keep using the non-PIC pair unchanged. +libs = ["lib/libc++.a", "lib/libc++abi.a", "lib/libc++-pic.a", "lib/libc++abi-pic.a"] headers = ["include/c++/v1"] diff --git a/packages/registry/php/build-php.sh b/packages/registry/php/build-php.sh index 976a00724..3f55b38bd 100755 --- a/packages/registry/php/build-php.sh +++ b/packages/registry/php/build-php.sh @@ -813,6 +813,11 @@ if [ ! -f Makefile ]; then # state. The second -u group is the measured subset that base PHP does not # otherwise pull into php.wasm; --export-all then exposes those symbols to # the side module. The post-build import-closure test guards this list. + # The following group forces libc symbols intl.so imports but base PHP + # never references (allocator, wide-char, math, and the pthread mutex/ + # cond/TLS that ICU's UMutex uses). They must resolve to php.wasm's own + # musl so intl.so shares one libc state — one allocator, one pthread key + # table; without -u they never enter php.wasm and intl.so fails to load. # # -Wl,-z,stack-size=4194304: 4 MB wasm stack. The default wasm-ld # stack is 64 KB, which sits ~100 KB above PHP's `alloc_globals` @@ -851,6 +856,13 @@ if [ ! -f Makefile ]; then -u inet_pton -u inet_ntop -u sched_yield -u alarm -u basename \ -u OCSP_basic_verify -u OCSP_cert_status_str -u OCSP_crl_reason_str \ -u OCSP_response_status_str -u SSL_alert_desc_string_long \ +-u aligned_alloc -u div -u modf -u round -u tanhf \ +-u swprintf -u wcstod -u wcstof -u wcstol -u wcstold \ +-u wcstoll -u wcstoul -u wcstoull -u wmemchr -u wmemcmp \ +-u pthread_cond_broadcast -u pthread_cond_destroy -u pthread_cond_signal \ +-u pthread_cond_timedwait -u pthread_cond_wait -u pthread_detach \ +-u pthread_getspecific -u pthread_key_create -u pthread_self \ +-u pthread_setspecific \ -Wl,-z,stack-size=4194304" \ ZLIB_CFLAGS="$ZLIB_CFLAGS_VALUE" \ ZLIB_LIBS="$ZLIB_LIBS_VALUE" \ @@ -883,6 +895,7 @@ if [ ! -f Makefile ]; then --enable-cli \ --enable-fpm \ --enable-opcache \ + --enable-intl=shared \ --enable-mbstring \ --disable-mbregex \ --enable-ctype \ @@ -1177,6 +1190,39 @@ wasm32posix-cc -shared -fPIC -o "$BIN_DIR/zip.so" \ "$LIBZIP_PREFIX/lib/libzip.a" echo "==> zip.so: $(wc -c < "$BIN_DIR/zip.so") bytes" +# Build intl as a shared .so, same libtool workaround as opcache: make compiles +# the PIC objects under ext/intl/**/.libs/ but the bundled libtool can't emit the +# final .so on this target, so we link it with `wasm32posix-cc -shared`. intl +# statically absorbs ICU and libc++/libc++abi so neither enters php.wasm; the ICU +# common data stays out of the .so as icu.dat (loaded by intl-icu-data-loader.c). +echo "==> Building intl.so (PHP extension)..." +make -j"$(sysctl -n hw.ncpu 2>/dev/null || nproc)" EXTRA_CFLAGS="$EXTRA_INC_LIBXML" ext/intl/intl.la || true + +# Compile the icu.dat loader (PIC) that feeds ICU its common data at dlopen. +wasm32posix-cc -fPIC -O2 -c "$SCRIPT_DIR/intl-icu-data-loader.c" \ + -I"$ICU_PREFIX/include" -o ext/intl/kandelo_icu_data_loader.o + +# Collect every PIC object libtool produced for ext/intl (top dir + the +# collator/, dateformat/, formatter/, … subdirs each have their own .libs/). +mapfile -t INTL_OBJS < <(find ext/intl -path '*/.libs/*.o' | sort) +[ "${#INTL_OBJS[@]}" -gt 0 ] || { echo "ERROR: no ext/intl PIC objects found — did 'make ext/intl/intl.la' compile?" >&2; exit 1; } +echo "==> linking intl.so from ${#INTL_OBJS[@]} objects + ICU static libs + libc++" + +# wasm-ld resolves archive back-references without --start-group, so the ICU +# archives are listed in dependency order (i18n -> io -> uc -> data), then +# libc++/libc++abi. A -shared PIC module requires every input to be PIC, so the +# libc++ PIC variants are named explicitly to win over the non-PIC sysroot ones. +wasm32posix-cc -shared -fPIC -o "$SCRIPT_DIR/bin/intl.so" \ + "${INTL_OBJS[@]}" \ + ext/intl/kandelo_icu_data_loader.o \ + "$ICU_PREFIX/lib/libicui18n.a" \ + "$ICU_PREFIX/lib/libicuio.a" \ + "$ICU_PREFIX/lib/libicuuc.a" \ + "$ICU_PREFIX/lib/libicudata.a" \ + "$LIBCXX_PREFIX/lib/libc++-pic.a" \ + "$LIBCXX_PREFIX/lib/libc++abi-pic.a" +echo "==> intl.so: $(wc -c < "$SCRIPT_DIR/bin/intl.so") bytes" + # Copy to bin/ with .wasm extension (needed for Vite browser demos) cp sapi/cli/php "$BIN_DIR/php.wasm" cp sapi/fpm/php-fpm "$BIN_DIR/php-fpm.wasm" diff --git a/packages/registry/php/intl-icu-data-loader.c b/packages/registry/php/intl-icu-data-loader.c new file mode 100644 index 000000000..77f5e60a5 --- /dev/null +++ b/packages/registry/php/intl-icu-data-loader.c @@ -0,0 +1,109 @@ +/* + * Feeds ICU its common data at intl.so load time. + * + * ICU ships as the standalone file icu.dat (see packages/registry/icu), but + * ICU's automatic loader only looks for the conventional icudt.dat + * name and would never find icu.dat on its own. So instead of embedding the + * ~30 MB blob in the .so, we hand it to ICU via udata_setCommonData() from a + * constructor: the side-module loader runs __wasm_call_ctors before PHP calls + * intl's MINIT, so the data is in place before any ICU service touches it. + * + * A missing/unreadable icu.dat is non-fatal at load (intl.so may be present + * without any code using intl) but stays loud: we warn to stderr and let ICU + * fail with U_MISSING_RESOURCE_ERROR when a service actually needs data, rather + * than silently succeeding. Path defaults to /usr/lib/php/icu.dat, overridable + * via KANDELO_ICU_DAT_PATH for VFS images that stage it elsewhere. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define KANDELO_ICU_DAT_DEFAULT "/usr/lib/php/icu.dat" + +static void kandelo_intl_load_icu_data(void) __attribute__((constructor)); + +static void kandelo_intl_load_icu_data(void) { + const char *path = getenv("KANDELO_ICU_DAT_PATH"); + if (path == NULL || path[0] == '\0') { + path = KANDELO_ICU_DAT_DEFAULT; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) { + fprintf(stderr, + "[intl] ICU data not loaded: cannot open %s. " + "intl functions will fail with U_MISSING_RESOURCE_ERROR. " + "Set KANDELO_ICU_DAT_PATH or stage icu.dat there.\n", + path); + return; + } + + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size <= 0) { + fprintf(stderr, "[intl] ICU data not loaded: cannot stat %s.\n", path); + close(fd); + return; + } + + /* + * ICU keeps this pointer for the life of the process, so the buffer must + * outlive this function and is deliberately never freed. A plain read (not + * mmap) sidesteps the VFS's emulated mmap and runs once per process. + */ + size_t size = (size_t) st.st_size; + void *buf = malloc(size); + if (buf == NULL) { + fprintf(stderr, "[intl] ICU data not loaded: OOM reading %s (%zu bytes).\n", + path, size); + close(fd); + return; + } + + size_t off = 0; + while (off < size) { + ssize_t n = read(fd, (char *) buf + off, size - off); + if (n < 0) { + fprintf(stderr, "[intl] ICU data not loaded: read error on %s.\n", path); + free(buf); + close(fd); + return; + } + if (n == 0) break; + off += (size_t) n; + } + close(fd); + + if (off != size) { + fprintf(stderr, "[intl] ICU data not loaded: short read on %s (%zu/%zu).\n", + path, off, size); + free(buf); + return; + } + + UErrorCode status = U_ZERO_ERROR; + udata_setCommonData(buf, &status); + if (U_FAILURE(status)) { + fprintf(stderr, + "[intl] udata_setCommonData(%s) failed: %s. " + "(Likely an ICU library/data version mismatch.)\n", + path, u_errorName(status)); + free(buf); + return; + } + + /* Force ICU to validate/initialize now so version skew surfaces at load. */ + status = U_ZERO_ERROR; + u_init(&status); + if (U_FAILURE(status)) { + fprintf(stderr, "[intl] u_init after loading %s failed: %s.\n", + path, u_errorName(status)); + } +} diff --git a/packages/registry/php/test/php-intl.test.ts b/packages/registry/php/test/php-intl.test.ts new file mode 100644 index 000000000..7fbb65598 --- /dev/null +++ b/packages/registry/php/test/php-intl.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect } from "vitest"; +import { existsSync, readdirSync, statSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { homedir } from "node:os"; +import { runCentralizedProgram } from "../../../../host/test/centralized-test-helper"; +import { tryResolveBinary } from "../../../../host/src/binary-resolver"; +import { NodePlatformIO } from "../../../../host/src/platform/node"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// intl is a RUNTIME-OPTIONAL side module: base php.wasm is built with +// --enable-intl=shared, so intl is NOT compiled in. intl.so is loaded on +// demand via `extension=intl.so`, and pulls its ICU common data from the +// separate icu.dat at runtime (udata_setCommonData in intl-icu-data-loader.c). +const phpBinaryPath = + tryResolveBinary("programs/php/php.wasm") ?? + join(__dirname, "../php-src/sapi/cli/php"); +const intlSoPath = tryResolveBinary("programs/php/intl.so"); + +// icu.dat lives in the icu package's resolver cache dir. Pick the newest +// non-temp build for the current arch. +function findIcuDat(): string | undefined { + const libsDir = join(homedir(), ".cache/kandelo/libs"); + if (!existsSync(libsDir)) return undefined; + const candidates = readdirSync(libsDir) + .filter((n) => n.startsWith("icu-") && n.includes("-wasm32-") && !n.includes(".tmp-")) + .map((n) => join(libsDir, n, "share", "icu.dat")) + .filter((p) => existsSync(p)); + if (candidates.length === 0) return undefined; + return candidates.sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0]; +} +const icuDatPath = findIcuDat(); + +const READY = existsSync(phpBinaryPath) && intlSoPath != null && icuDatPath != null; + +describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { + // Proves the base binary is genuinely ICU-free / intl-free: intl only + // appears when explicitly loaded. This is the whole point of the design. + it("base php.wasm does NOT include intl", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-m"], + // Same host-I/O adapter as the other cases: `php -m` needs no files, + // but it keeps the harness off the "default" rootfs.vfs image (not a + // fixture this package ships) so the run stays self-contained. + io: new NodePlatformIO(), + }); + expect(exitCode).toBe(0); + expect(stdout.toLowerCase()).not.toContain("intl"); + }, 60_000); + + it("loads intl.so at runtime via extension=", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${intlSoPath}`, "-r", + 'echo extension_loaded("intl") ? "intl-loaded" : "intl-missing";'], + env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], + io: new NodePlatformIO(), + }); + expect(stdout).toContain("intl-loaded"); + expect(exitCode).toBe(0); + }, 60_000); + + // Exercises real ICU data (locale display names) to prove icu.dat is + // actually loaded and usable, not just that the module registered. + it("intl uses ICU data (Locale::getDisplayLanguage)", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${intlSoPath}`, "-r", + 'echo Locale::getDisplayLanguage("fr", "en");'], + env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], + io: new NodePlatformIO(), + }); + expect(stdout).toContain("French"); + expect(exitCode).toBe(0); + }, 60_000); + + // Collator sorting is a core ICU service that requires collation data. + it("intl Collator sorts with locale rules", async () => { + const { stdout, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${intlSoPath}`, "-r", ` + $c = new Collator("en_US"); + $a = ["banana", "apple", "cherry"]; + $c->sort($a); + echo implode(",", $a); + `], + env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], + io: new NodePlatformIO(), + }); + expect(stdout).toContain("apple,banana,cherry"); + expect(exitCode).toBe(0); + }, 60_000); +}); From 1e66a26ad7da2360a19f6196fc598b204a561b1f Mon Sep 17 00:00:00 2001 From: mho22 Date: Sun, 12 Jul 2026 05:05:31 -0400 Subject: [PATCH 02/12] icu: statically link Linux host-tool runtimes ICU's native Stage-1 data tools must execute during the cross build. On the Nix Linux runner, link their libstdc++ and libgcc runtimes statically so Stage 2 does not depend on loader paths for host runtime shared libraries; keep the flags Linux-only because the macOS toolchain rejects them. Linux execution evidence remains the responsibility of the later package rebuild/CI validation in this serial. Co-authored-by: Claude Opus 4.8 (1M context) --- packages/registry/icu/build-icu.sh | 11 +++++++++++ packages/registry/icu/build.toml | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/registry/icu/build-icu.sh b/packages/registry/icu/build-icu.sh index 0ec6757c4..ebd224126 100755 --- a/packages/registry/icu/build-icu.sh +++ b/packages/registry/icu/build-icu.sh @@ -98,12 +98,23 @@ NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" # Uses the host compiler (clang/clang++ from the dev shell), NOT the wasm # wrappers. sdk/activate.sh only prepends SDK bin to PATH; it does not export # CC/CXX, so an explicit host CC/CXX keeps this stage native. +# +# On Linux, statically fold the GNU C++/GCC runtime into the data tools: the Nix +# CI runner has no libstdc++.so.6 on its loader path, so a dynamically linked +# icupkg/pkgdata (run here by Stage 2's make) aborts at exec with "cannot open +# shared object file". macOS clang links a self-contained libc++ and rejects the +# flags. LDFLAGS set here is honored: runConfigureICU re-exports it to configure. +case "$(uname -s)" in + Linux) HOST_LDFLAGS="-static-libstdc++ -static-libgcc" ;; + *) HOST_LDFLAGS="" ;; +esac if [ ! -x "$HOST_BUILD/bin/icupkg" ] && [ ! -x "$HOST_BUILD/bin/genccode" ]; then echo "==> Stage 1: building ICU natively for host tools + data..." rm -rf "$HOST_BUILD" mkdir -p "$HOST_BUILD" ( cd "$HOST_BUILD" CC="${HOST_CC:-clang}" CXX="${HOST_CXX:-clang++}" \ + LDFLAGS="$HOST_LDFLAGS" \ "$ICU_SRC/runConfigureICU" MacOSX \ --enable-static --disable-shared \ --disable-samples --disable-tests --disable-extras diff --git a/packages/registry/icu/build.toml b/packages/registry/icu/build.toml index a7637ccfb..731645e12 100644 --- a/packages/registry/icu/build.toml +++ b/packages/registry/icu/build.toml @@ -4,7 +4,9 @@ commit = "e96fbe3964d5ec6784f00f4d49bfcf70a2030e22" # Revision 3: ICU static libs rebuilt with -fPIC so they can be absorbed into # intl.so (a wasm side module built with -shared --experimental-pic, which # requires all inputs to be position-independent). -revision = 3 +# Revision 4: Stage-1 host tools statically link the GNU C++/GCC runtime on +# Linux so icupkg/pkgdata do not need libstdc++.so.6 on the Nix CI runner. +revision = 4 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" From d65da0d6b6e6f7e04a1f9008fc1ddd1daa1662c4 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 03:34:19 -0400 Subject: [PATCH 03/12] packages: make PHP intl dependencies reproducible ICU previously left its common data and pkg-config files outside the declared library output contract, embedded the resolver temporary prefix in libicuuc.a, and relied on undeclared native build tools. The libcxx revision provenance also pointed at a fork commit that did not contain the published PIC recipe. Add a backward-compatible outputs.files contract with parse, cache-key, missing-output, and accepted-output coverage. Declare ICU data/pkg-config outputs and host probes, build from a clean verified source tree, and compile the fallback data path as /usr/lib/php so clean archives are stable. Keep existing package keys unchanged when files is absent, and mark the ICU/libcxx producer revisions as unpublished Automattic work. Validation: the four focused runtime-file xtask tests passed; build-deps check reported 10 consistent tools across four consumers; the six ICU tools matched the dev-shell versions. Two clean ICU rev5 builds produced byte-identical four archives, 197 headers, and icu.dat; resolver validation accepted all declared outputs and rewrote the three .pc prefixes to the canonical cache path. Two clean libcxx rev6 builds in distinct cache locations produced byte-identical four archives and 1,680 headers. Scope limits: the clean producer comparisons were run on macOS/aarch64 in the repository dev shell. The repaired Linux host-tool build is not re-claimed here beyond the original PR CI evidence, and performance was not measured. --- docs/package-management.md | 5 +- packages/registry/icu/build-icu.sh | 59 ++++++++++++---- packages/registry/icu/build.toml | 6 +- packages/registry/icu/package.toml | 53 +++++++++++++-- packages/registry/libcxx/build.toml | 4 +- tools/xtask/src/build_deps.rs | 102 +++++++++++++++++++++++++++- tools/xtask/src/pkg_manifest.rs | 18 ++++- 7 files changed, 220 insertions(+), 27 deletions(-) diff --git a/docs/package-management.md b/docs/package-management.md index 58eecc441..1a54ca137 100644 --- a/docs/package-management.md +++ b/docs/package-management.md @@ -176,6 +176,7 @@ script_path = "packages/registry/zlib/build-zlib.sh" libs = ["lib/libz.a"] # must exist post-build headers = ["include/zlib.h", "include/zconf.h"] pkgconfig = ["lib/pkgconfig/zlib.pc"] +files = ["share/runtime-data.bin"] # other runtime data ``` `package.toml` **must NOT** carry `revision`, `[binary]`, @@ -353,7 +354,7 @@ that doesn't respect them cannot be cached safely. | Variable | Meaning | |---|---| -| `WASM_POSIX_DEP_OUT_DIR` | Temp dir the script must install into. Layout matches `outputs.libs` / `outputs.headers` / `outputs.pkgconfig` relative paths. | +| `WASM_POSIX_DEP_OUT_DIR` | Temp dir the script must install into. Layout matches `outputs.libs` / `outputs.headers` / `outputs.pkgconfig` / `outputs.files` relative paths. | | `WASM_POSIX_DEP_NAME` | `name` from package.toml. | | `WASM_POSIX_DEP_VERSION` | `version` from package.toml. | | `WASM_POSIX_DEP_REVISION` | Effective package revision after `build.toml` is overlaid. | @@ -371,7 +372,7 @@ from source; it does not restrict normal SDK users compiling against a published sysroot/libc++ artifact. After the script exits 0, the resolver verifies every path in -`outputs.{libs,headers,pkgconfig}` exists under `$WASM_POSIX_DEP_OUT_DIR`. +`outputs.{libs,headers,pkgconfig,files}` exists under `$WASM_POSIX_DEP_OUT_DIR`. A missing output fails the build (and the temp dir is cleaned up, so a retry starts clean). diff --git a/packages/registry/icu/build-icu.sh b/packages/registry/icu/build-icu.sh index ebd224126..2e9dd33fd 100755 --- a/packages/registry/icu/build-icu.sh +++ b/packages/registry/icu/build-icu.sh @@ -44,21 +44,40 @@ fi ICU_VERSION="${WASM_POSIX_DEP_VERSION:-${ICU_VERSION:-74.2}}" ICU_VER_UNDERSCORE="${ICU_VERSION//./_}" # 74.2 -> 74_2 ICU_MAJOR="${ICU_VERSION%%.*}" # 74.2 -> 74 +TARGET_ARCH="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" INSTALL_DIR="${WASM_POSIX_DEP_OUT_DIR:-$SCRIPT_DIR/icu-install}" SOURCE_URL="${WASM_POSIX_DEP_SOURCE_URL:-https://github.com/unicode-org/icu/releases/download/release-${ICU_MAJOR}-${ICU_VERSION#*.}/icu4c-${ICU_VER_UNDERSCORE}-src.tgz}" -SOURCE_SHA256="${WASM_POSIX_DEP_SOURCE_SHA256:-}" +SOURCE_SHA256="${WASM_POSIX_DEP_SOURCE_SHA256:-68db082212a96d6f53e35d60f47d38b962e9f9d207a74cfac78029ae8ff5e08c}" SYSROOT="${WASM_POSIX_SYSROOT:-$REPO_ROOT/sysroot}" export WASM_POSIX_SYSROOT="$SYSROOT" -SRC_ROOT="$SCRIPT_DIR/icu-src" # contains icu/ (with source/) +if [ "$TARGET_ARCH" != "wasm32" ]; then + echo "ERROR: ICU currently supports only wasm32, got $TARGET_ARCH" >&2 + exit 1 +fi + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/kandelo-icu.XXXXXX")" +cleanup() { + status=$? + trap - EXIT + if [ "${WASM_POSIX_KEEP_BUILD_DIR:-0}" = "1" ]; then + echo "==> Preserving ICU build directory: $WORK_DIR" >&2 + else + rm -rf "$WORK_DIR" + fi + exit "$status" +} +trap cleanup EXIT + +SRC_ROOT="$WORK_DIR/source" # contains icu/ (with source/) ICU_SRC="$SRC_ROOT/icu/source" -HOST_BUILD="$SCRIPT_DIR/host-build" # stage-1 native build (out-of-tree) +HOST_BUILD="$WORK_DIR/host-build" # stage-1 native build (out-of-tree) # --- Resolve libcxx (ICU is C++), symlink into sysroot (mariadb pattern) --- HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" resolve_dep() { - (cd "$REPO_ROOT" && cargo run -p xtask --target "$HOST_TARGET" --quiet -- build-deps resolve "$1") + (cd "$REPO_ROOT" && cargo run -p xtask --target "$HOST_TARGET" --quiet -- build-deps --arch "$TARGET_ARCH" resolve "$1") } LIBCXX_PREFIX="${WASM_POSIX_DEP_LIBCXX_DIR:-}" if [ -z "$LIBCXX_PREFIX" ]; then @@ -79,12 +98,10 @@ ln -sfn "$LIBCXX_PREFIX/include/c++/v1" "$SYSROOT/include/c++/v1" # --- Fetch + verify source --- if [ ! -d "$ICU_SRC" ]; then echo "==> Downloading ICU $ICU_VERSION..." - TARBALL="/tmp/icu4c-${ICU_VER_UNDERSCORE}-src.tgz" + TARBALL="$WORK_DIR/icu4c-${ICU_VER_UNDERSCORE}-src.tgz" curl --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors -fsSL "$SOURCE_URL" -o "$TARBALL" - if [ -n "$SOURCE_SHA256" ]; then - echo "==> Verifying source sha256..." - echo "$SOURCE_SHA256 $TARBALL" | shasum -a 256 -c - - fi + echo "==> Verifying source sha256..." + echo "$SOURCE_SHA256 $TARBALL" | shasum -a 256 -c - mkdir -p "$SRC_ROOT" tar xzf "$TARBALL" -C "$SRC_ROOT" # extracts icu/ rm "$TARBALL" @@ -92,6 +109,19 @@ fi NPROC="$(sysctl -n hw.ncpu 2>/dev/null || nproc)" +run_logged() { + local label="$1" + shift + local log="$WORK_DIR/$label.log" + if "$@" >"$log" 2>&1; then + tail -20 "$log" + return 0 + fi + echo "ERROR: $label failed; final log follows:" >&2 + tail -200 "$log" >&2 + return 1 +} + # ============================================================ # Stage 1 — HOST build (native tools + data) # ============================================================ @@ -118,7 +148,7 @@ if [ ! -x "$HOST_BUILD/bin/icupkg" ] && [ ! -x "$HOST_BUILD/bin/genccode" ]; the "$ICU_SRC/runConfigureICU" MacOSX \ --enable-static --disable-shared \ --disable-samples --disable-tests --disable-extras - make -j"$NPROC" + run_logged host-make make -j"$NPROC" ) else echo "==> Stage 1: reusing existing host build at $HOST_BUILD" @@ -162,11 +192,16 @@ wasm32posix-configure \ --prefix="$INSTALL_DIR" echo "==> Stage 2: building wasm32 libraries..." -make -j"$NPROC" +# ICU bakes ICUDATA_DIR into common/putil.ao when data packaging is `common`. +# The resolver install prefix is a random `.tmp-` directory, so leaving +# the generated default in place makes libicuuc.a differ on every clean build. +# The PHP extension stages icu.dat at this stable guest directory and calls +# udata_setCommonData() explicitly; use the same path for the fallback string. +run_logged wasm-make make -j"$NPROC" ICUDATA_DIR=/usr/lib/php echo "==> Installing to $INSTALL_DIR..." rm -rf "$INSTALL_DIR" -make install +run_logged wasm-install make install # --- Stage the common data as icu.dat (see header) --- DAT_SRC="$(find "$ICU_SRC/data" "$HOST_BUILD/data" -name "icudt${ICU_MAJOR}l.dat" 2>/dev/null | head -1 || true)" diff --git a/packages/registry/icu/build.toml b/packages/registry/icu/build.toml index 731645e12..7919c9137 100644 --- a/packages/registry/icu/build.toml +++ b/packages/registry/icu/build.toml @@ -1,12 +1,14 @@ script_path = "packages/registry/icu/build-icu.sh" repo_url = "https://github.com/Automattic/kandelo.git" -commit = "e96fbe3964d5ec6784f00f4d49bfcf70a2030e22" +commit = "UNPUBLISHED" # Revision 3: ICU static libs rebuilt with -fPIC so they can be absorbed into # intl.so (a wasm side module built with -shared --experimental-pic, which # requires all inputs to be position-independent). # Revision 4: Stage-1 host tools statically link the GNU C++/GCC runtime on # Linux so icupkg/pkgdata do not need libstdc++.so.6 on the Nix CI runner. -revision = 4 +# Revision 5: compile ICU's default data directory as the stable guest path +# /usr/lib/php instead of the resolver's random temporary install prefix. +revision = 5 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/icu/package.toml b/packages/registry/icu/package.toml index 355f39786..f49bdd7e7 100644 --- a/packages/registry/icu/package.toml +++ b/packages/registry/icu/package.toml @@ -11,8 +11,10 @@ arches = ["wasm32"] # ICU4C for wasm32, built as static libraries: # lib/libicuuc.a (common) # lib/libicui18n.a (internationalization) +# lib/libicuio.a (ICU stream I/O used by PHP's configure contract) # lib/libicudata.a (stubdata — real data lives in the .dat, see below) # include/unicode/*.h +# lib/pkgconfig/icu-{uc,i18n,io}.pc # share/icu.dat (common data archive; renamed from icudt74l.dat per # the intl side-module design — loaded at runtime via # udata_setCommonData, NOT ICU's default name search) @@ -34,12 +36,51 @@ url = "https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE" [build] script_path = "packages/registry/icu/build-icu.sh" +# Stage 1 compiles native ICU data tools; Stage 2 invokes GNU make around the +# SDK's wasm wrappers. Source fetching and direct (non-resolver) dependency +# fallback also need the declared network/Rust tools. Keep these probes aligned +# with the repository dev shell instead of accepting ambient host state. +[[host_tools]] +name = "clang" +version_constraint = ">=21.0" +probe = { args = ["--version"], version_regex = "clang version (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "clang++" +version_constraint = ">=21.0" +probe = { args = ["--version"], version_regex = "clang version (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "make" +version_constraint = ">=3.80" +probe = { args = ["--version"], version_regex = "GNU Make (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "curl" +version_constraint = ">=7.0" +probe = { args = ["--version"], version_regex = "curl (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "rustc" +version_constraint = ">=1.85" +probe = { args = ["--version"], version_regex = "rustc (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + +[[host_tools]] +name = "cargo" +version_constraint = ">=1.85" +probe = { args = ["--version"], version_regex = "cargo (\\d+\\.\\d+(?:\\.\\d+)?)" } +install_hints = { darwin = "run through scripts/dev-shell.sh", linux = "run through scripts/dev-shell.sh" } + [outputs] libs = ["lib/libicuuc.a", "lib/libicui18n.a", "lib/libicuio.a", "lib/libicudata.a"] headers = ["include/unicode"] -# NOTE: the build also installs share/icu.dat (the ICU common data archive, -# staged from icudtl.dat). It is not declarable as a library output here -# (Outputs only takes libs/headers/pkgconfig); consumers such as PHP's intl -# side module read it directly from the resolved dep dir -# ($WASM_POSIX_DEP_ICU_DIR/share/icu.dat) and load it at runtime via -# udata_setCommonData(). See build-icu.sh. +pkgconfig = ["lib/pkgconfig/icu-uc.pc", "lib/pkgconfig/icu-i18n.pc", "lib/pkgconfig/icu-io.pc"] +# ICU common data is a first-class declared runtime file, staged from +# icudtl.dat. Consumers read it from the resolved dependency directory at +# $WASM_POSIX_DEP_ICU_DIR/share/icu.dat and load it with udata_setCommonData(). +files = ["share/icu.dat"] diff --git a/packages/registry/libcxx/build.toml b/packages/registry/libcxx/build.toml index 9e316ba77..d6566a375 100644 --- a/packages/registry/libcxx/build.toml +++ b/packages/registry/libcxx/build.toml @@ -1,6 +1,6 @@ script_path = "packages/registry/libcxx/build-libcxx.sh" -repo_url = "https://github.com/brandonpayton/kandelo.git" -commit = "8c53383229fab78f97b098c3207a655159c03041" +repo_url = "https://github.com/Automattic/kandelo.git" +commit = "UNPUBLISHED" # Revision 4: libcxx + libcxxabi + libunwind rebuilt with # `-mllvm -wasm-use-legacy-eh=false` explicitly set. # Revision 5: libcxx builds only from flake.nix's exact Nix LLVM diff --git a/tools/xtask/src/build_deps.rs b/tools/xtask/src/build_deps.rs index 3053138ed..302aa49b7 100644 --- a/tools/xtask/src/build_deps.rs +++ b/tools/xtask/src/build_deps.rs @@ -309,7 +309,7 @@ pub fn compute_sha( // Fold in declared outputs so changing what a build is // expected to produce invalidates the cache. Without this, // renaming a program's `wasm = "..."` (or any library - // libs/headers/pkgconfig path) leaves cache_key_sha + // libs/headers/pkgconfig/files path) leaves cache_key_sha // unchanged — the resolver then serves a canonical // directory that doesn't match the new declaration and // archive-stage packs broken archives. Bug discovered in @@ -340,6 +340,18 @@ pub fn compute_sha( h.update(b"|"); } h.update(b"\n"); + // Preserve every existing package's cache key: the additive files + // field participates only when authored. A universally empty + // section would invalidate the entire package registry merely for + // learning a new output kind. + if !target.outputs.files.is_empty() { + h.update(b"outputs.files:\n"); + for s in &target.outputs.files { + h.update(s.as_bytes()); + h.update(b"|"); + } + h.update(b"\n"); + } h.update(b"program_outputs:\n"); for out in &target.program_outputs { h.update(out.name.as_bytes()); @@ -2059,7 +2071,7 @@ fn build_into_cache( } // Kind-aware validation. Library and program manifests carry a - // declared outputs list (libs/headers/pkgconfig or program wasms) + // declared outputs list (libs/headers/pkgconfig/files or program wasms) // that `validate_outputs` checks one-by-one. Source manifests have // no declared outputs — design 11 calls for emptiness as the only // signal — so we just verify the script populated OUT_DIR with at @@ -2396,6 +2408,9 @@ fn validate_outputs(target: &DepsManifest, out_dir: &Path) -> Result<(), String> for rel in &target.outputs.pkgconfig { check(rel, "pkgconfig")?; } + for rel in &target.outputs.files { + check(rel, "files")?; + } } ManifestKind::Program => { for out in &target.program_outputs { @@ -2759,6 +2774,9 @@ fn cmd_parse(m: &DepsManifest) -> Result<(), String> { if !m.outputs.pkgconfig.is_empty() { println!("outputs.pkgconfig= {:?}", m.outputs.pkgconfig); } + if !m.outputs.files.is_empty() { + println!("outputs.files = {:?}", m.outputs.files); + } Ok(()) } @@ -4254,6 +4272,33 @@ fork_instrumentation = "disabled" ); } + #[test] + fn cache_key_sha_changes_when_library_runtime_file_added() { + let root = tempdir("sha-lib-runtime-file-added"); + write(&root, "libZ", "1.0.0", &[]); + let reg = Registry { + roots: vec![root.clone()], + }; + let sha_before = sha_of(®, "libZ"); + + let toml_path = root.join("libZ/package.toml"); + let text = std::fs::read_to_string(&toml_path).unwrap(); + std::fs::write( + &toml_path, + text.replace( + "libs = [\"lib/liblibZ.a\"]", + "libs = [\"lib/liblibZ.a\"]\nfiles = [\"share/libZ.dat\"]", + ), + ) + .unwrap(); + let sha_after = sha_of(®, "libZ"); + + assert_ne!( + sha_before, sha_after, + "adding a library runtime file output must invalidate the cache key" + ); + } + // --- ensure_built / build_into_cache tests --- /// Create a package.toml + build-.sh pair. The build script uses @@ -4485,6 +4530,59 @@ libs = ["lib/libC.a"] } } + #[test] + fn ensure_built_fails_when_declared_runtime_file_missing() { + let root = tempdir("built-missing-runtime-file"); + let cache = tempdir("built-missing-runtime-file-cache"); + write_lib( + &root, + "libRuntimeMissing", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeMissing.a""#, + r#"[outputs] +libs = ["lib/libRuntimeMissing.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeMissing").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("declared files output"), "got: {err}"); + assert!(err.contains("share/runtime.dat"), "got: {err}"); + } + + #[test] + fn ensure_built_accepts_declared_runtime_file() { + let root = tempdir("built-runtime-file"); + let cache = tempdir("built-runtime-file-cache"); + write_lib( + &root, + "libRuntime", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntime.a" +printf runtime > "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, + r#"[outputs] +libs = ["lib/libRuntime.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntime").unwrap(); + + let path = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + assert_eq!( + std::fs::read_to_string(path.join("share/runtime.dat")).unwrap(), + "runtime" + ); + } + #[test] fn ensure_built_fails_when_script_exits_nonzero() { let root = tempdir("built-badexit"); diff --git a/tools/xtask/src/pkg_manifest.rs b/tools/xtask/src/pkg_manifest.rs index af12640d8..7defae9ac 100644 --- a/tools/xtask/src/pkg_manifest.rs +++ b/tools/xtask/src/pkg_manifest.rs @@ -707,6 +707,10 @@ pub struct Outputs { pub headers: Vec, #[serde(default)] pub pkgconfig: Vec, + /// Runtime data or other package files that are neither link libraries, + /// headers, nor pkg-config metadata. + #[serde(default)] + pub files: Vec, } /// `name@version` reference, parsed from `depends_on` strings. @@ -1331,7 +1335,7 @@ impl DepsManifest { } // Dispatch on `kind` to decide whether `outputs` is the - // library shape (`[outputs]` table with libs/headers/pkgconfig) + // library shape (`[outputs]` table with libs/headers/pkgconfig/files) // or the program shape (`[[outputs]]` array-of-tables with // name/wasm). A mismatch between the two is rejected at parse // time: each kind enforces its own grammar. @@ -1587,6 +1591,7 @@ cache_key_sha = "111111111111111111111111111111111111111111111111111111111111111 assert_eq!(m.revision, 1); assert!(m.depends_on.is_empty()); assert_eq!(m.outputs.libs, vec!["lib/libz.a"]); + assert!(m.outputs.files.is_empty()); assert_eq!(m.spec(), "zlib@1.3.1"); // Fallback (no [build].script_path) resolves against self.dir, // so the repo_root argument is irrelevant. @@ -1596,6 +1601,16 @@ cache_key_sha = "111111111111111111111111111111111111111111111111111111111111111 ); } + #[test] + fn parses_library_runtime_file_outputs() { + let text = EXAMPLE.replace( + "headers = [\"include/zlib.h\"]", + "headers = [\"include/zlib.h\"]\nfiles = [\"share/zlib/runtime.dat\"]", + ); + let m = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap(); + assert_eq!(m.outputs.files, vec!["share/zlib/runtime.dat"]); + } + #[test] fn build_script_override_is_repo_root_relative() { // Phase A-bis Task 2: an explicit `[build].script_path` is @@ -2113,6 +2128,7 @@ wasm = "vim.wasm" assert!(m.outputs.libs.is_empty()); assert!(m.outputs.headers.is_empty()); assert!(m.outputs.pkgconfig.is_empty()); + assert!(m.outputs.files.is_empty()); } #[test] From ebee6b7d61ea7e2ae6ccebef753859d3eed0e4e0 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 03:34:29 -0400 Subject: [PATCH 04/12] host: resolve C++ side-module imports fail-closed The original intl linker change synthesized delayed stubs for every missing env function. That can hide real ABI gaps on unexecuted paths, while name-only self-import detection can also mistake an imported-function re-export for a definition and recurse through its own trampoline. Preserve Batch 2 transactional and fork-replay behavior. Supply a trampoline only when the export section proves the same-name function index is module-defined, keep main-module interposition first, and leave every other unresolved import as an eager instantiation failure. At this point in the serial, provide the wasm-exceptions C++ tag required by the intl side module. Validation: host type generation passed; host/test/dylink.test.ts passed 37/37, including genuine self-definition, missing import, imported-function re-export, C++ tag, rollback, and fork-replay cases. The built ABI-19 intl.so had zero unresolved env/GOT symbols under this rule, and all four real PHP intl Node/kernel cases passed after the stricter index check. Scope limits: this intermediate commit creates the C++ tag at the side-module linker boundary. The later process-state replay commit in this same landing serial deliberately supersedes that ownership model with one process-owned tag and adds cross-side identity coverage. Browser behavior is validated later; performance was not measured. --- host/src/dylink.ts | 85 +++++++++++++++++++++++++++++++++++++++- host/test/dylink.test.ts | 37 +++++++++++++---- 2 files changed, 112 insertions(+), 10 deletions(-) diff --git a/host/src/dylink.ts b/host/src/dylink.ts index 5904ed89e..65de254bc 100644 --- a/host/src/dylink.ts +++ b/host/src/dylink.ts @@ -223,6 +223,44 @@ export function parseDylinkSection(wasmBytes: Uint8Array): DylinkMetadata | null return metadata; } +/** + * Return function exports whose indices refer to module-defined functions. + * + * WebAssembly.Module.exports() exposes only names and kinds. Dynamic-linker + * self-import handling also needs the function index: an imported function can + * be re-exported under the same name, but that is not a local definition and + * must not receive a trampoline back to itself. + * + * `module` has already validated the binary before this helper is called, so + * section bounds and LEB encodings are known to be structurally valid. + */ +function readDefinedFunctionExports( + wasmBytes: Uint8Array, + importedFunctionCount: number, +): Set { + const result = new Set(); + const offset = { value: 8 }; + while (offset.value < wasmBytes.length) { + const sectionId = wasmBytes[offset.value++]; + const sectionSize = readVarUint(wasmBytes, offset); + const sectionEnd = offset.value + sectionSize; + if (sectionId !== 7) { + offset.value = sectionEnd; + continue; + } + + const exportCount = readVarUint(wasmBytes, offset); + for (let i = 0; i < exportCount; i++) { + const name = readString(wasmBytes, offset); + const kind = wasmBytes[offset.value++]; + const index = readVarUint(wasmBytes, offset); + if (kind === 0 && index >= importedFunctionCount) result.add(name); + } + break; + } + return result; +} + /** Align a value up to the given alignment (must be power of 2). */ function alignUp(value: number, align: number): number { return (value + align - 1) & ~(align - 1); @@ -500,6 +538,26 @@ function instantiateSharedLibrary( const functionExports = new Set( moduleExports.filter((exp) => exp.kind === "function").map((exp) => exp.name), ); + const importedFunctionCount = moduleImports.filter((imp) => imp.kind === "function").length; + const definedFunctionExports = readDefinedFunctionExports( + wasmBytes, + importedFunctionCount, + ); + // wasm-ld can make an interposable C++ definition both an env import and a + // module export. The main process still wins when it supplies the symbol; + // otherwise route only this genuine self-definition back to the module. + // Do not manufacture trampolines for arbitrary unresolved imports: those + // remain instantiation errors instead of turning an ABI gap into a delayed + // failure on a possibly-unexecuted path. + const selfFunctionImports = new Set( + moduleImports + .filter((imp) => + imp.module === "env" + && imp.kind === "function" + && definedFunctionExports.has(imp.name) + ) + .map((imp) => imp.name), + ); const importsDynamicLookup = moduleImports.some((imp) => imp.module === "env" && imp.kind === "function" @@ -511,6 +569,15 @@ function instantiateSharedLibrary( && (imp.kind as string) === "tag" ); const longjmpTag = importsLongjmpTag ? resolveLongjmpTag(options) : undefined; + const importsCppExceptionTag = moduleImports.some((imp) => + imp.module === "env" + && imp.name === "__cpp_exception" + && (imp.kind as string) === "tag" + ); + const Tag = tagConstructor(); + const cppExceptionTag = importsCppExceptionTag && Tag + ? new Tag({ parameters: ["i32"] }) + : undefined; if (presentForkExports.length > 0 && !hasCompleteForkInstrumentation) { const missing = SIDE_MODULE_FORK_EXPORTS.filter((exportName) => @@ -763,19 +830,33 @@ function instantiateSharedLibrary( case "__table_base": return tableBaseGlobal; case "__stack_pointer": return options.stackPointer; case "__c_longjmp": return longjmpTag; + // C++ exceptions are currently confined to one side module. A + // cross-module exception contract would instead need one + // process-owned tag shared with every participating module. + case "__cpp_exception": return cppExceptionTag; case "fork": if (importsFork) return sideModuleForkImport; break; } const sym = options.globalSymbols.get(prop); if (sym !== undefined) return sym; + if (selfFunctionImports.has(prop)) { + return (...args: unknown[]) => { + const fn = instance?.exports[prop]; + if (typeof fn !== "function") { + throw new Error(`${name}: self import env.${prop} is unavailable`); + } + return (fn as Function)(...args); + }; + } return undefined; }, has(_target, prop: string) { if (["memory", "__indirect_function_table", "__memory_base", - "__table_base", "__stack_pointer", "__c_longjmp"].includes(prop)) return true; + "__table_base", "__stack_pointer", "__c_longjmp", + "__cpp_exception"].includes(prop)) return true; if (prop === "fork" && importsFork) return true; - return options.globalSymbols.has(prop); + return options.globalSymbols.has(prop) || selfFunctionImports.has(prop); }, }), "GOT.mem": new Proxy({} as Record, { diff --git a/host/test/dylink.test.ts b/host/test/dylink.test.ts index f07ba5820..2120cdb88 100644 --- a/host/test/dylink.test.ts +++ b/host/test/dylink.test.ts @@ -979,14 +979,35 @@ describe.skipIf(!hasWat2Wasm())("weak self-import handling", () => { expect((lib.exports.call_self as Function)()).toBe(42); }); - it("throws loudly when a self-import has no defining export", () => { - // A genuinely absent symbol must trap at call time, not silently return 0. - const lib = loadSharedLibrarySync("missing-import.so", assembleSideModule(` - (module - (import "env" "missing_fn" (func $missing (result i32))) - (func (export "call_missing") (result i32) (call $missing))) - `, "missing-import"), createLoadOptions()); - expect(() => (lib.exports.call_missing as Function)()).toThrow(/not provided/); + it("rejects a genuinely absent env import during instantiation", () => { + // Only an import that the side module itself exports gets a trampoline. + // A real ABI gap must remain an eager load failure rather than hiding on + // an unexecuted path behind a delayed stub. + expect(() => loadSharedLibrarySync( + "missing-import.so", + assembleSideModule(` + (module + (import "env" "missing_fn" (func $missing (result i32))) + (func (export "call_missing") (result i32) (call $missing))) + `, "missing-import"), + createLoadOptions(), + )).toThrow(); + }); + + it("does not mistake an imported-function re-export for a definition", () => { + // A same-name export can point straight back at the imported function. + // Treating that shape as a definition would recurse forever through the + // host trampoline instead of rejecting the unresolved import at dlopen. + expect(() => loadSharedLibrarySync( + "reexported-import.so", + assembleSideModule(` + (module + (import "env" "reexported_fn" (func $reexported (result i32))) + (export "reexported_fn" (func $reexported)) + (func (export "call_reexported") (result i32) (call $reexported))) + `, "reexported-import"), + createLoadOptions(), + )).toThrow(); }); it("provides the __cpp_exception tag to -fwasm-exceptions modules", () => { From 80ad441b64f5b6f2c9ec0792e214c68eac98abbc Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 03:34:41 -0400 Subject: [PATCH 05/12] php: build the complete intl side module at ABI 19 The original recipe was based on an older PHP tree, linked into the package source directory, and masked make failures with `|| true`. On the reconciled ABI-19 recipe that could publish a partial module: the first strict attempt exposed that only one newline-delimited object had been consumed. Preserve PHP 8.3.15, GNU libiconv, the Batch 2 extensions, forced shared-libc symbols, and fork instrumentation. Add ICU/libcxx as direct resolver dependencies, build every generated shared_objects_intl target without invoking the unsupported libtool link, then link all PIC objects and declared ICU/libcxx archives into OUT_DIR/intl.so. Reserve PHP revision 13 for the serial zip(11), curl(12), intl(13) Batch 3 order. Make the Node data fixture prefer an explicit resolved ICU file and restrict its cache fallback to the current ICU version/revision. Validation: a clean ABI-19 source resolve built 70 intl PIC objects and a 5,860,812-byte side module; php, php-fpm, opcache, phar, and zend_test were byte-identical across two clean temporary builds. The link consumed both libcxx PIC archives, and static inspection found all 3,040 intl imports resolved by PHP, the side module, or linker-owned objects. The focused Node/kernel suite passed 4/4: base PHP excludes intl, explicit loading succeeds, Locale reads icu.dat, and Collator applies locale rules. Scope limits: this evidence is for #839 on the ABI-19 Batch 2 base. The final #647 -> #648 -> #839 serial tree and real browser path are validated separately before recommendation. This adds no kernel/process ABI shape, so no ABI bump or snapshot change is included; performance was not measured. --- packages/registry/php/build-php.sh | 59 +++++++++++++++++++-- packages/registry/php/build.toml | 4 +- packages/registry/php/package.toml | 9 +++- packages/registry/php/test/php-intl.test.ts | 26 +++++++-- 4 files changed, 88 insertions(+), 10 deletions(-) diff --git a/packages/registry/php/build-php.sh b/packages/registry/php/build-php.sh index 3f55b38bd..f4a81f3f2 100755 --- a/packages/registry/php/build-php.sh +++ b/packages/registry/php/build-php.sh @@ -75,6 +75,13 @@ LIBZIP_PREFIX="${WASM_POSIX_DEP_LIBZIP_DIR:-}" [ -z "$LIBZIP_PREFIX" ] && { echo "==> Resolving libzip..."; LIBZIP_PREFIX="$(resolve_dep libzip)"; } LIBCURL_PREFIX="${WASM_POSIX_DEP_LIBCURL_DIR:-}" [ -z "$LIBCURL_PREFIX" ] && { echo "==> Resolving libcurl..."; LIBCURL_PREFIX="$(resolve_dep libcurl)"; } +# ICU and libcxx back only the runtime-loadable intl side module. Keep them as +# explicit direct dependencies so the recipe never succeeds because a prior +# local build happened to leave C++ headers or ICU archives in the sysroot. +ICU_PREFIX="${WASM_POSIX_DEP_ICU_DIR:-}" +[ -z "$ICU_PREFIX" ] && { echo "==> Resolving icu..."; ICU_PREFIX="$(resolve_dep icu)"; } +LIBCXX_PREFIX="${WASM_POSIX_DEP_LIBCXX_DIR:-}" +[ -z "$LIBCXX_PREFIX" ] && { echo "==> Resolving libcxx..."; LIBCXX_PREFIX="$(resolve_dep libcxx)"; } [ -f "$ZLIB_PREFIX/lib/libz.a" ] || { echo "ERROR: zlib resolve missing libz.a"; exit 1; } [ -f "$SQLITE_PREFIX/lib/libsqlite3.a" ] || { echo "ERROR: sqlite resolve missing libsqlite3.a"; exit 1; } [ -f "$OPENSSL_PREFIX/lib/libssl.a" ] || { echo "ERROR: openssl resolve missing libssl.a"; exit 1; } @@ -82,6 +89,16 @@ LIBCURL_PREFIX="${WASM_POSIX_DEP_LIBCURL_DIR:-}" [ -f "$LIBICONV_PREFIX/lib/libiconv.a" ] || { echo "ERROR: GNU libiconv resolve missing libiconv.a"; exit 1; } [ -f "$LIBZIP_PREFIX/lib/libzip.a" ] || { echo "ERROR: libzip resolve missing libzip.a"; exit 1; } [ -f "$LIBCURL_PREFIX/lib/libcurl.a" ] || { echo "ERROR: libcurl resolve missing libcurl.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicuuc.a" ] || { echo "ERROR: icu resolve missing libicuuc.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicui18n.a" ] || { echo "ERROR: icu resolve missing libicui18n.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicuio.a" ] || { echo "ERROR: icu resolve missing libicuio.a"; exit 1; } +[ -f "$ICU_PREFIX/lib/libicudata.a" ] || { echo "ERROR: icu resolve missing libicudata.a"; exit 1; } +[ -f "$ICU_PREFIX/share/icu.dat" ] || { echo "ERROR: icu resolve missing share/icu.dat"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++.a" ] || { echo "ERROR: libcxx resolve missing libc++.a"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++abi.a" ] || { echo "ERROR: libcxx resolve missing libc++abi.a"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++-pic.a" ] || { echo "ERROR: libcxx resolve missing libc++-pic.a"; exit 1; } +[ -f "$LIBCXX_PREFIX/lib/libc++abi-pic.a" ] || { echo "ERROR: libcxx resolve missing libc++abi-pic.a"; exit 1; } +[ -d "$LIBCXX_PREFIX/include/c++/v1" ] || { echo "ERROR: libcxx resolve missing include/c++/v1"; exit 1; } echo "==> zlib at $ZLIB_PREFIX" echo "==> sqlite at $SQLITE_PREFIX" echo "==> openssl at $OPENSSL_PREFIX" @@ -89,10 +106,24 @@ echo "==> libxml2 at $LIBXML2_PREFIX" echo "==> GNU libiconv at $LIBICONV_PREFIX" echo "==> libzip at $LIBZIP_PREFIX" echo "==> libcurl at $LIBCURL_PREFIX" +echo "==> icu at $ICU_PREFIX" +echo "==> libcxx at $LIBCXX_PREFIX" + +# The SDK's C++ driver reads libc++ from the worktree-local sysroot. Point that +# declared toolchain location at the resolver-owned dependency, matching the +# existing MariaDB/SpiderMonkey package contract. The intl side-module link +# below names the PIC archives explicitly; these non-PIC names are only for +# configure probes and any main-module C++ checks. +mkdir -p "$SYSROOT/lib" "$SYSROOT/include/c++" +ln -sf "$LIBCXX_PREFIX/lib/libc++.a" "$SYSROOT/lib/libc++.a" +ln -sf "$LIBCXX_PREFIX/lib/libc++abi.a" "$SYSROOT/lib/libc++abi.a" +ln -sf "$LIBCXX_PREFIX/lib/libc++.a" "$SYSROOT/lib/libstdc++.a" +rm -rf "$SYSROOT/include/c++/v1" +ln -sfn "$LIBCXX_PREFIX/include/c++/v1" "$SYSROOT/include/c++/v1" # Compose PKG_CONFIG_PATH for all deps so wasm32posix-configure's # pkg-config probes can find them in the cache instead of the sysroot. -DEP_PKG_CONFIG_PATH="$ZLIB_PREFIX/lib/pkgconfig:$SQLITE_PREFIX/lib/pkgconfig:$OPENSSL_PREFIX/lib/pkgconfig:$LIBXML2_PREFIX/lib/pkgconfig:$LIBICONV_PREFIX/lib/pkgconfig:$LIBZIP_PREFIX/lib/pkgconfig:$LIBCURL_PREFIX/lib/pkgconfig" +DEP_PKG_CONFIG_PATH="$ZLIB_PREFIX/lib/pkgconfig:$SQLITE_PREFIX/lib/pkgconfig:$OPENSSL_PREFIX/lib/pkgconfig:$LIBXML2_PREFIX/lib/pkgconfig:$LIBICONV_PREFIX/lib/pkgconfig:$LIBZIP_PREFIX/lib/pkgconfig:$LIBCURL_PREFIX/lib/pkgconfig:$ICU_PREFIX/lib/pkgconfig" # Compose -I and -L flags for defense-in-depth (autoconf raw probes). DEP_CPPFLAGS="-I$ZLIB_PREFIX/include -I$SQLITE_PREFIX/include -I$OPENSSL_PREFIX/include -I$LIBXML2_PREFIX/include -I$LIBICONV_PREFIX/include -I$LIBZIP_PREFIX/include -I$LIBCURL_PREFIX/include" @@ -137,6 +168,8 @@ REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBXML2_PREFIX" /usr/src/kandel REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBICONV_PREFIX" /usr/src/kandelo-deps/libiconv)" REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBZIP_PREFIX" /usr/src/kandelo-deps/libzip)" REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBCURL_PREFIX" /usr/src/kandelo-deps/libcurl)" +REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$ICU_PREFIX" /usr/src/kandelo-deps/icu)" +REPRODUCIBLE_PREFIX_MAPS+=" $(prefix_map_flags "$LIBCXX_PREFIX" /usr/src/kandelo-deps/libcxx)" echo "==> Downloading PHP $PHP_VERSION..." TARBALL="$WORK_DIR/php.tar.gz" @@ -1196,7 +1229,24 @@ echo "==> zip.so: $(wc -c < "$BIN_DIR/zip.so") bytes" # statically absorbs ICU and libc++/libc++abi so neither enters php.wasm; the ICU # common data stays out of the .so as icu.dat (loaded by intl-icu-data-loader.c). echo "==> Building intl.so (PHP extension)..." -make -j"$(sysctl -n hw.ncpu 2>/dev/null || nproc)" EXTRA_CFLAGS="$EXTRA_INC_LIBXML" ext/intl/intl.la || true +# Ask PHP's generated Makefile for the shared extension's complete object list +# and build those targets directly. Invoking intl.la would deliberately reach +# the unsupported libtool side-module link, while `|| true` would also hide a +# genuine compile failure and could link a partial extension. +INTL_LO_TARGETS_RAW="$(make -s --no-print-directory -f Makefile -f - print-intl-objects <<'MAKE' +.PHONY: print-intl-objects +print-intl-objects: + @printf '%s\n' $(shared_objects_intl) +MAKE +)" +mapfile -t INTL_LO_TARGETS <<< "$INTL_LO_TARGETS_RAW" +[ "${#INTL_LO_TARGETS[@]}" -gt 0 ] || { + echo "ERROR: PHP Makefile did not declare shared_objects_intl" >&2 + exit 1 +} +make -j"$(sysctl -n hw.ncpu 2>/dev/null || nproc)" \ + EXTRA_CFLAGS="$EXTRA_INC_LIBXML" \ + "${INTL_LO_TARGETS[@]}" # Compile the icu.dat loader (PIC) that feeds ICU its common data at dlopen. wasm32posix-cc -fPIC -O2 -c "$SCRIPT_DIR/intl-icu-data-loader.c" \ @@ -1212,7 +1262,7 @@ echo "==> linking intl.so from ${#INTL_OBJS[@]} objects + ICU static libs + libc # archives are listed in dependency order (i18n -> io -> uc -> data), then # libc++/libc++abi. A -shared PIC module requires every input to be PIC, so the # libc++ PIC variants are named explicitly to win over the non-PIC sysroot ones. -wasm32posix-cc -shared -fPIC -o "$SCRIPT_DIR/bin/intl.so" \ +wasm32posix-cc -shared -fPIC -o "$BIN_DIR/intl.so" \ "${INTL_OBJS[@]}" \ ext/intl/kandelo_icu_data_loader.o \ "$ICU_PREFIX/lib/libicui18n.a" \ @@ -1221,7 +1271,7 @@ wasm32posix-cc -shared -fPIC -o "$SCRIPT_DIR/bin/intl.so" \ "$ICU_PREFIX/lib/libicudata.a" \ "$LIBCXX_PREFIX/lib/libc++-pic.a" \ "$LIBCXX_PREFIX/lib/libc++abi-pic.a" -echo "==> intl.so: $(wc -c < "$SCRIPT_DIR/bin/intl.so") bytes" +echo "==> intl.so: $(wc -c < "$BIN_DIR/intl.so") bytes" # Copy to bin/ with .wasm extension (needed for Vite browser demos) cp sapi/cli/php "$BIN_DIR/php.wasm" @@ -1268,4 +1318,5 @@ if [ -z "${WASM_POSIX_DEP_OUT_DIR:-}" ]; then install_local_binary php "$BIN_DIR/phar.so" install_local_binary php "$BIN_DIR/zend_test.so" install_local_binary php "$BIN_DIR/zip.so" + install_local_binary php "$BIN_DIR/intl.so" fi diff --git a/packages/registry/php/build.toml b/packages/registry/php/build.toml index 699816789..554db487c 100644 --- a/packages/registry/php/build.toml +++ b/packages/registry/php/build.toml @@ -4,7 +4,9 @@ inputs = [ ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 12 +# Batch 3 extension landings advance this recipe once per byte-changing layer: +# zip (11), curl (12), then intl/ICU (13). +revision = 13 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/php/package.toml b/packages/registry/php/package.toml index 12da91d21..26d15124c 100644 --- a/packages/registry/php/package.toml +++ b/packages/registry/php/package.toml @@ -5,7 +5,7 @@ kernel_abi = 19 # The rebuilt executables import `__wasm_posix_vm_interrupt_after`, which is # part of the ABI-19 host contract. Older package archives must not satisfy this # recipe; rebuild the CLI, FPM, and side modules for this exact ABI epoch. -depends_on = ["zlib@1.3.1", "openssl@3.3.2", "sqlite@3.49.1", "libxml2@2.13.8", "libiconv@1.17", "libzip@1.11.4", "libcurl@8.11.1"] +depends_on = ["zlib@1.3.1", "openssl@3.3.2", "sqlite@3.49.1", "libxml2@2.13.8", "libiconv@1.17", "libzip@1.11.4", "libcurl@8.11.1", "icu@74.2", "libcxx@21.1.7"] # wasm32 only: build-php.sh invokes the wasm32 SDK explicitly. If a wasm64 PHP # build is introduced later, make the recipe architecture-aware and validate # every CLI, FPM, and extension output before advertising that architecture. @@ -76,3 +76,10 @@ wasm = "zend_test.so" [[outputs]] name = "zip" wasm = "zip.so" + +# intl is a normal, opt-in PHP extension. It statically absorbs the ICU and +# PIC libc++ archives, while ICU's common data remains a separately declared +# icu package file that VFS consumers stage at /usr/lib/php/icu.dat. +[[outputs]] +name = "intl" +wasm = "intl.so" diff --git a/packages/registry/php/test/php-intl.test.ts b/packages/registry/php/test/php-intl.test.ts index 7fbb65598..4d80fa12b 100644 --- a/packages/registry/php/test/php-intl.test.ts +++ b/packages/registry/php/test/php-intl.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { existsSync, readdirSync, statSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { homedir } from "node:os"; @@ -18,13 +18,31 @@ const phpBinaryPath = join(__dirname, "../php-src/sapi/cli/php"); const intlSoPath = tryResolveBinary("programs/php/intl.so"); -// icu.dat lives in the icu package's resolver cache dir. Pick the newest -// non-temp build for the current arch. +// icu.dat remains owned by the ICU library package, not copied into PHP's +// program outputs. CI/manual callers can name the exact resolved file; the +// cache fallback is restricted to this checkout's ICU version and revision so +// an unrelated newer ICU build cannot silently satisfy the test. function findIcuDat(): string | undefined { + const explicit = process.env.PHP_INTL_ICU_DATA; + if (explicit !== undefined) { + if (!existsSync(explicit) || !statSync(explicit).isFile()) { + throw new Error(`PHP_INTL_ICU_DATA is not a regular file: ${explicit}`); + } + return explicit; + } + + const icuDir = join(__dirname, "../../icu"); + const version = readFileSync(join(icuDir, "package.toml"), "utf8") + .match(/^version\s*=\s*"([^"]+)"/m)?.[1]; + const revision = readFileSync(join(icuDir, "build.toml"), "utf8") + .match(/^revision\s*=\s*(\d+)/m)?.[1]; + if (!version || !revision) return undefined; + const libsDir = join(homedir(), ".cache/kandelo/libs"); if (!existsSync(libsDir)) return undefined; + const expectedPrefix = `icu-${version}-rev${revision}-wasm32-`; const candidates = readdirSync(libsDir) - .filter((n) => n.startsWith("icu-") && n.includes("-wasm32-") && !n.includes(".tmp-")) + .filter((n) => n.startsWith(expectedPrefix) && !n.includes(".tmp-")) .map((n) => join(libsDir, n, "share", "icu.dat")) .filter((p) => existsSync(p)); if (candidates.length === 0) return undefined; From d940770611adfa3ff8436cb9aea40f35fbde6120 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 03:39:49 -0400 Subject: [PATCH 06/12] test: exercise intl exceptions and fork replay Instantiation alone did not prove that the host supplied the C++ exception tag with the expected payload signature, and synthetic fork-state tests did not prove that ICU common-data state survives replay of a real PHP side module. Make the WAT fixture throw and catch an i32 payload through env.__cpp_exception. Add a PHP case that loads intl and icu.dat before pcntl_fork, calls an ICU-backed Locale API in the replayed child, calls Locale and Collator again in the parent, and verifies wait/exit status. Validation: with the ABI-19 sysroot present, host/test/dylink.test.ts passed 37/37. packages/registry/php/test/php-intl.test.ts passed 5/5 against the resolver-built PHP/intl/ICU artifacts; the fork-replay case completed in both branches and reaped a zero-status child. Scope limits: this commit records Node/kernel evidence. The equivalent Chromium path is added and run only on the exact landed #647 -> #648 -> #839 serial tree; it is not claimed here. --- host/test/dylink.test.ts | 13 +++++--- packages/registry/php/test/php-intl.test.ts | 36 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/host/test/dylink.test.ts b/host/test/dylink.test.ts index 2120cdb88..aed7da704 100644 --- a/host/test/dylink.test.ts +++ b/host/test/dylink.test.ts @@ -1011,13 +1011,18 @@ describe.skipIf(!hasWat2Wasm())("weak self-import handling", () => { }); it("provides the __cpp_exception tag to -fwasm-exceptions modules", () => { - // C++ side modules import this tag; without the host providing it, the - // module fails to instantiate with "tag import requires a WebAssembly.Tag". + // C++ side modules import this i32-payload tag. Exercise an actual + // throw/catch so both the supplied signature and engine behavior are + // covered, not merely WebAssembly.Instance's value-type check. const lib = loadSharedLibrarySync("tag.so", assembleSideModule(` (module (import "env" "__cpp_exception" (tag $exc (param i32))) - (func (export "noop"))) + (func (export "throw_and_catch") (result i32) + (try (result i32) + (do + (throw $exc (i32.const 42))) + (catch $exc)))) `, "tag"), createLoadOptions()); - expect(lib.exports.noop).toBeTypeOf("function"); + expect((lib.exports.throw_and_catch as Function)()).toBe(42); }); }); diff --git a/packages/registry/php/test/php-intl.test.ts b/packages/registry/php/test/php-intl.test.ts index 4d80fa12b..8920f04d2 100644 --- a/packages/registry/php/test/php-intl.test.ts +++ b/packages/registry/php/test/php-intl.test.ts @@ -110,4 +110,40 @@ describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { expect(stdout).toContain("apple,banana,cherry"); expect(exitCode).toBe(0); }, 60_000); + + // dlopen occurs before fork. The child therefore replays intl.so without + // rerunning constructors and must retain ICU's common-data pointer from the + // copied process image; exercising ICU in both branches verifies that real + // side-module replay contract rather than only the synthetic linker shape. + it("intl and icu.dat survive pcntl_fork replay", async () => { + const { stdout, stderr, exitCode } = await runCentralizedProgram({ + programPath: phpBinaryPath, + argv: ["php", "-d", `extension=${intlSoPath}`, "-r", ` + $before = Locale::getDisplayLanguage("fr", "en"); + $pid = pcntl_fork(); + if ($pid < 0) { fwrite(STDERR, "fork-failed"); exit(20); } + if ($pid === 0) { + $child = Locale::getDisplayLanguage("fr", "en"); + echo "child=" . $child . "\\n"; + exit($child === "French" ? 0 : 21); + } + $status = 0; + $waited = pcntl_waitpid($pid, $status); + $c = new Collator("en_US"); + $a = ["banana", "apple", "cherry"]; + $c->sort($a); + echo "parent=" . $before . ":" . implode(",", $a) . "\\n"; + if ($waited !== $pid || !pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { + fwrite(STDERR, "child-status=" . $status); + exit(22); + } + `], + env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], + io: new NodePlatformIO(), + }); + expect(stderr).toBe(""); + expect(stdout).toContain("child=French"); + expect(stdout).toContain("parent=French:apple,banana,cherry"); + expect(exitCode).toBe(0); + }, 60_000); }); From 0f0de3016a24a5b3993e9aa884755a2c165a55e2 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 03:59:59 -0400 Subject: [PATCH 07/12] packages: remove producer paths from intl dependencies libc++abi archive members retained the absolute checkout path because both PIC and non-PIC CMake builds lacked compiler prefix maps. ICU metadata was likewise rewritten from the resolver temporary prefix to one producer cache path, so fetched or moved archives still pointed back to the build machine. Map the entire worktree, including assembled LLVM sources, both build variants, the smoke source, and sysroots, to /usr/src/kandelo in libcxx compiler paths and advance it to revision 7. Rewrite ICU pkg-config prefix fields to ${pcfiledir}/../.. and advance it to revision 6, keeping headers and libraries relative to each extracted package tree. Validation: two libcxx revision-7 builds from distinct checkout/output roots produced byte-identical PIC and non-PIC archives, and producer-path scans were clean. Two ICU revision-6 builds from distinct roots produced byte-identical declared archives and icu.dat. After moving an ICU prefix, pkg-config resolved the relocated include/library paths and a consumer compile/link succeeded. Scope limits: the reproducibility and relocation checks ran on macOS/aarch64 in the repository dev shell. Linux host-tool execution, the final PHP package, Node/browser runtime behavior, and aggregate gates are validated by later commits; performance was not measured. --- packages/registry/icu/build-icu.sh | 13 +++++++++++++ packages/registry/icu/build.toml | 4 +++- packages/registry/libcxx/build-libcxx.sh | 10 +++++++++- packages/registry/libcxx/build.toml | 5 ++++- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/registry/icu/build-icu.sh b/packages/registry/icu/build-icu.sh index 2e9dd33fd..79ea14e6b 100755 --- a/packages/registry/icu/build-icu.sh +++ b/packages/registry/icu/build-icu.sh @@ -203,6 +203,19 @@ echo "==> Installing to $INSTALL_DIR..." rm -rf "$INSTALL_DIR" run_logged wasm-install make install +# ICU's generated pkg-config metadata records --prefix verbatim. Resolver +# builds install into a temporary directory and later rewrite that to the +# producer's canonical cache path, which still breaks after an archive is +# fetched or moved elsewhere. Keep the metadata package-relative instead: each +# file lives in lib/pkgconfig, so pcfiledir/../.. is the current ICU prefix in +# every cache, extracted archive, or local overlay. +for pc in icu-uc.pc icu-i18n.pc icu-io.pc; do + pc_path="$INSTALL_DIR/lib/pkgconfig/$pc" + [ -f "$pc_path" ] || { echo "ERROR: missing ICU pkg-config output $pc_path" >&2; exit 1; } + sed 's|^prefix = .*|prefix = ${pcfiledir}/../..|' "$pc_path" > "$pc_path.tmp" + mv "$pc_path.tmp" "$pc_path" +done + # --- Stage the common data as icu.dat (see header) --- DAT_SRC="$(find "$ICU_SRC/data" "$HOST_BUILD/data" -name "icudt${ICU_MAJOR}l.dat" 2>/dev/null | head -1 || true)" if [ -z "$DAT_SRC" ]; then diff --git a/packages/registry/icu/build.toml b/packages/registry/icu/build.toml index 7919c9137..c2d176224 100644 --- a/packages/registry/icu/build.toml +++ b/packages/registry/icu/build.toml @@ -8,7 +8,9 @@ commit = "UNPUBLISHED" # Linux so icupkg/pkgdata do not need libstdc++.so.6 on the Nix CI runner. # Revision 5: compile ICU's default data directory as the stable guest path # /usr/lib/php instead of the resolver's random temporary install prefix. -revision = 5 +# Revision 6: make the three pkg-config prefixes relative to pcfiledir so a +# fetched or moved archive resolves its own headers and libraries. +revision = 6 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/libcxx/build-libcxx.sh b/packages/registry/libcxx/build-libcxx.sh index a89b35af2..22dd7420e 100755 --- a/packages/registry/libcxx/build-libcxx.sh +++ b/packages/registry/libcxx/build-libcxx.sh @@ -137,7 +137,15 @@ echo "==> Building libc++ and libc++abi for ${ARCH}..." # against the modern ABI; consumers linking against this libcxx # archive must also compile with `-wasm-use-legacy-eh=false` (the # SDK's `compileFlags` was updated in lock-step). -WASM_C_FLAGS="--target=${WASM_TARGET} -matomics -mbulk-memory -mexception-handling -mllvm -wasm-enable-sjlj -mllvm -wasm-use-legacy-eh=false -fexceptions -fno-trapping-math --sysroot=${SYSROOT} -O2 -DNDEBUG" +# +# LLVM records source/compilation paths in archive members. All assembled +# sources and both variant build directories live under REPO_ROOT, so one +# stable worktree mapping covers LLVM_SRC_DIR, BUILD_DIR, PIC_BUILD_DIR, the +# generated smoke source, and sysroot paths for both PIC and non-PIC builds. +# Without it, libc++abi.a and every side module absorbing it differ solely by +# the caller's checkout path. +REPRODUCIBLE_PREFIX_MAPS="-ffile-prefix-map=${REPO_ROOT}=/usr/src/kandelo -fdebug-prefix-map=${REPO_ROOT}=/usr/src/kandelo -fmacro-prefix-map=${REPO_ROOT}=/usr/src/kandelo" +WASM_C_FLAGS="--target=${WASM_TARGET} -matomics -mbulk-memory -mexception-handling -mllvm -wasm-enable-sjlj -mllvm -wasm-use-legacy-eh=false -fexceptions -fno-trapping-math --sysroot=${SYSROOT} -O2 -DNDEBUG ${REPRODUCIBLE_PREFIX_MAPS}" # Start with a fresh source tree so a cache-miss rebuild does not mix old + new # artifacts. (Each build dir is cleaned by build_libcxx_variant below.) diff --git a/packages/registry/libcxx/build.toml b/packages/registry/libcxx/build.toml index d6566a375..40b16e4ef 100644 --- a/packages/registry/libcxx/build.toml +++ b/packages/registry/libcxx/build.toml @@ -10,7 +10,10 @@ commit = "UNPUBLISHED" # Revision 6: additionally emits position-independent libc++-pic.a / # libc++abi-pic.a (a second -fPIC build) so wasm side modules like PHP's # intl.so can statically absorb libc++. The non-PIC pair is unchanged. -revision = 6 +# Revision 7: map the worktree (including the assembled LLVM source and both +# build variants) to a stable producer path so archives do not retain the +# caller's absolute checkout directory. +revision = 7 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" From 3d6d14db6883a14470cf4668e734c566ba70fa51 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 04:55:02 -0400 Subject: [PATCH 08/12] packages: declare non-Wasm program runtime files Problem: program manifests could describe only executable outputs, leaving required data blobs outside cache keys, archives, local mirrors, fetched-artifact validation, and VFS installation metadata. Existing archive collection could also follow external/cyclic symlinks, and unvalidated package/output paths could escape or collide in the mirror. Design: add a program-only runtime_files contract with normalized artifact and absolute guest paths, mode metadata, length-framed cache-key fields, exact nested-artifact lookup, collision checks, and regular-file validation on build/cache/fetch paths. Archive staging dereferences only contained symlinks and rejects escapes, cycles, specials, empty declared trees, and incomplete closures. Local and fetched materialization use one package-scoped mirror layout and a structured repository-build query. Compatibility: manifests with no runtime_files retain their historical cache keys. The previously additive library outputs.files section is versioned and length-framed now, before its first published consumer, while older output sections retain their encoding for cache compatibility. Focused evidence: pkg_manifest tests passed 112/112; archive_stage tests passed 8/8; runtime-filtered tests passed 19/19; delimiter framing, fetched/source mirror parity, incomplete-archive fallback/fetch-only failure, nested paths, collisions, and symlink safety are covered. The full xtask run reached 336 passed and the same 7 pre-existing malformed-Wasm/filename fixture failures observed on the base. Limits: shell helper syntax and host resolver precedence passed, but the full aggregate package build, published-archive round trip, full host/libc gate, and browser gate have not run. The metadata helper is a repository build/materialization API, not a guest or runtime host API. --- docs/binary-releases.md | 8 +- docs/package-management.md | 30 + host/src/binary-resolver.ts | 6 +- host/test/binary-resolver.test.ts | 18 +- scripts/install-local-binary.sh | 63 +- scripts/package-runtime-file.ts | 109 ++++ tools/xtask/src/archive_stage.rs | 258 ++++++++- tools/xtask/src/build_deps.rs | 932 ++++++++++++++++++++++++++++-- tools/xtask/src/pkg_manifest.rs | 522 ++++++++++++++++- 9 files changed, 1870 insertions(+), 76 deletions(-) create mode 100644 scripts/package-runtime-file.ts diff --git a/docs/binary-releases.md b/docs/binary-releases.md index aa6f26c13..e5b47b010 100644 --- a/docs/binary-releases.md +++ b/docs/binary-releases.md @@ -337,9 +337,11 @@ For each declared arch in the package's `arches = [...]` (default - `abi_versions` must contain the in-tree `ABI_VERSION`. - `cache_key_sha` must match the resolver's locally-computed cache-key sha (catches recipe drift). -6. Places `binaries/programs//.wasm` symlinks pointing - into the cache, so browser/Node demos can load by relative path - without re-fetching. +6. Places each program output under `binaries/programs//` using the + manifest's output layout, and places declared non-Wasm runtime files under + `binaries/programs///`. Both are symlinks into the + validated cache, so browser/Node image builders load the same bytes without + re-fetching. Local builds use the identical layout under `local-binaries/`. On any verification failure, the resolver logs a warning and falls through to a source build (the package's build script). This diff --git a/docs/package-management.md b/docs/package-management.md index 1a54ca137..cfb685af8 100644 --- a/docs/package-management.md +++ b/docs/package-management.md @@ -179,6 +179,36 @@ pkgconfig = ["lib/pkgconfig/zlib.pc"] files = ["share/runtime-data.bin"] # other runtime data ``` +Program packages use `[[outputs]]` for executable/side-module artifacts. A +non-Wasm file required at runtime is declared separately so it remains part of +the same reproducible archive and cache key: + +```toml +[[outputs]] +name = "php" +wasm = "php.wasm" + +[[runtime_files]] +artifact = "icu.dat" # relative to the package cache/archive +guest_path = "/usr/lib/php/icu.dat" # installation path in a VFS image +mode = 420 # optional decimal TOML; default 0644 +``` + +`[[runtime_files]]` is program-only. Artifact paths are normalized portable +relative paths; guest paths are normalized absolute POSIX paths; files, +ancestor paths, and resolver-mirror destinations may not collide. The resolver +requires regular non-symlink runtime files after fresh builds, cache hits, and +remote fetches. It mirrors them at +`{local-,}binaries/programs///` independently of the +number of `[[outputs]]` entries. Missing fetched files make the archive stale +and trigger source fallback (or a hard failure in fetch-only mode). + +Repo-side VFS/test builders query the authoritative path and mode with +`xtask build-deps runtime-file-metadata `; they must not +scan library caches or invent environment-only guest paths. Published VFS +images contain the installed bytes already, so this query is a build-tool +contract rather than a runtime host API. + `package.toml` **must NOT** carry `revision`, `[binary]`, `[build].repo_url`, or `[build].commit`. Those moved to `build.toml` during the binary-resolution-via-index-ledger migration; diff --git a/host/src/binary-resolver.ts b/host/src/binary-resolver.ts index 8eebfc5e0..1a7cbf07f 100644 --- a/host/src/binary-resolver.ts +++ b/host/src/binary-resolver.ts @@ -1,5 +1,6 @@ /** - * Resolve a binary (wasm, zip bundle, vfs image) from the repo's + * Resolve a packaged artifact (Wasm executable/side module, VFS image, + * archive, or declared runtime data file) from the repo's * `local-binaries/` or `binaries/` tree. * * Priority: @@ -81,13 +82,14 @@ function packageRoot(): string { } /** - * Resolve a binary relative to the binaries tree. + * Resolve an artifact relative to the binaries tree. * * Example paths: * `kernel.wasm` * `userspace.wasm` * `programs/vim.zip` (implicit wasm32 — see below) * `programs/git/git.wasm` (implicit wasm32) + * `programs/php/icu.dat` (implicit wasm32 runtime file) * `programs/wasm64/mariadb-vfs.vfs.zst` (explicit arch) * * Per-arch layout: `binaries/programs/` and `local-binaries/programs/` diff --git a/host/test/binary-resolver.test.ts b/host/test/binary-resolver.test.ts index c0444c53b..cc8ab229b 100644 --- a/host/test/binary-resolver.test.ts +++ b/host/test/binary-resolver.test.ts @@ -104,7 +104,7 @@ async function vfsImage( return compressed ? new Uint8Array(zstdCompressSync(image)) : image; } -function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst"): string { +function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst" | ".dat"): string { const testRoot = "programs/wasm32/__binary_resolver_test__"; const dir = `${testRoot}/${randomUUID()}`; cleanupDirs.add(join(localBinariesDir(), dir)); @@ -190,4 +190,20 @@ describe("binary resolver artifact policy", () => { expect(resolveBinary(relPath)).toBe(fetchedPath); }); + + it("prefers a local declared runtime data file over the fetched candidate", () => { + const relPath = fixtureRelPath(".dat"); + const localPath = writeCandidate( + localBinariesDir(), + relPath, + new TextEncoder().encode("local-runtime"), + ); + writeCandidate( + binariesDir(), + relPath, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(resolveBinary(relPath)).toBe(localPath); + }); }); diff --git a/scripts/install-local-binary.sh b/scripts/install-local-binary.sh index dfd7a9776..daac62aa0 100755 --- a/scripts/install-local-binary.sh +++ b/scripts/install-local-binary.sh @@ -58,6 +58,14 @@ install_local_binary() { echo "install_local_binary: source file not found: $src" >&2 return 1 fi + local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" + case "$arch" in + wasm32|wasm64) ;; + *) + echo "install_local_binary: unsupported target arch '$arch' (expected wasm32 or wasm64)" >&2 + return 2 + ;; + esac # Repo root must be derived from this helper, not from the caller's # current directory: package builds often `cd` into an upstream git @@ -111,8 +119,6 @@ install_local_binary() { ;; esac - local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" - # Take everything from the FIRST dot in the source basename onward # so compound extensions like `.vfs.zst` round-trip intact (matches # the resolver's `place_binaries_symlinks` extension handling). @@ -177,3 +183,56 @@ install_local_binary() { echo " installed $resolver_dest (resolver scratch)" fi } + +# Install a declared non-Wasm `[[runtime_files]]` artifact into the same local +# resolver mirror used by published archives. This is intentionally separate +# from install_local_binary: data files must not pass Wasm/fork guards or be +# described as executable outputs. +install_local_runtime_file() { + local program="$1" + local src="$2" + local artifact="${3:-}" + + if [ -z "$program" ] || [ -z "$src" ]; then + echo "install_local_runtime_file: usage: install_local_runtime_file [artifact]" >&2 + return 2 + fi + if [ ! -f "$src" ] || [ -L "$src" ]; then + echo "install_local_runtime_file: source must be a regular non-symlink file: $src" >&2 + return 1 + fi + local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" + case "$arch" in + wasm32|wasm64) ;; + *) + echo "install_local_runtime_file: unsupported target arch '$arch' (expected wasm32 or wasm64)" >&2 + return 2 + ;; + esac + + local repo_root + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + local host_target + host_target="$(rustc -vV 2>/dev/null | awk '/^host/ {print $2}')" + if [ -z "$host_target" ]; then + echo "install_local_runtime_file: rustc did not report a host target" >&2 + return 1 + fi + local src_basename + src_basename="$(basename "$src")" + artifact="${artifact:-$src_basename}" + local rel + rel="$(cd "$repo_root" && \ + env -u CC -u CXX -u AR -u RANLIB -u CFLAGS -u CXXFLAGS -u CPPFLAGS -u LDFLAGS \ + cargo run -p xtask --target "$host_target" --quiet -- \ + build-deps runtime-file-path "$program" "$artifact")" || return 1 + if [ -z "$rel" ]; then + echo "install_local_runtime_file: manifest lookup returned an empty path" >&2 + return 1 + fi + + local dest="$repo_root/local-binaries/programs/$arch/$rel" + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" + echo " installed $dest" +} diff --git a/scripts/package-runtime-file.ts b/scripts/package-runtime-file.ts new file mode 100644 index 000000000..b6637f836 --- /dev/null +++ b/scripts/package-runtime-file.ts @@ -0,0 +1,109 @@ +/** + * Repo-side bridge from package.toml `[[runtime_files]]` to VFS/test builders. + * + * Runtime-file metadata is a build/materialization contract, not a host-runtime + * API: published browser/rootfs images contain the installed bytes already. + * Repo tools query xtask so guest paths and modes are never duplicated in + * TypeScript fixtures. + */ +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { isAbsolute } from "node:path"; +import { tryResolveBinary } from "../host/src/binary-resolver"; + +export interface PackageRuntimeFileContract { + artifact: string; + guestPath: string; + mode: number; + mirrorPath: string; +} + +export interface ResolvedPackageRuntimeFile extends PackageRuntimeFileContract { + hostPath: string; +} + +let cachedHostTarget: string | undefined; + +function hostTarget(): string { + if (cachedHostTarget) return cachedHostTarget; + const output = execFileSync("rustc", ["-vV"], { encoding: "utf8" }); + const target = output.match(/^host:\s*(\S+)$/m)?.[1]; + if (!target) throw new Error("rustc -vV did not report a host target"); + cachedHostTarget = target; + return target; +} + +function hostCargoEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const name of [ + "CC", + "CXX", + "AR", + "RANLIB", + "CFLAGS", + "CXXFLAGS", + "CPPFLAGS", + "LDFLAGS", + ]) { + delete env[name]; + } + return env; +} + +export function readPackageRuntimeFileContract( + repoRoot: string, + packageName: string, + artifact: string, +): PackageRuntimeFileContract { + const raw = execFileSync( + "cargo", + [ + "run", + "-p", + "xtask", + "--target", + hostTarget(), + "--quiet", + "--", + "build-deps", + "runtime-file-metadata", + packageName, + artifact, + ], + { cwd: repoRoot, encoding: "utf8", env: hostCargoEnv() }, + ).trim(); + const parsed = JSON.parse(raw) as Record; + const contract: PackageRuntimeFileContract = { + artifact: parsed.artifact as string, + guestPath: parsed.guest_path as string, + mode: parsed.mode as number, + mirrorPath: parsed.mirror_path as string, + }; + if ( + contract.artifact !== artifact + || typeof contract.guestPath !== "string" + || !contract.guestPath.startsWith("/") + || !Number.isInteger(contract.mode) + || contract.mode < 0 + || contract.mode > 0o777 + || typeof contract.mirrorPath !== "string" + || isAbsolute(contract.mirrorPath) + || contract.mirrorPath.split("/").some((part) => !part || part === "." || part === "..") + ) { + throw new Error( + `invalid runtime-file metadata for ${packageName}:${artifact}: ${raw}`, + ); + } + return contract; +} + +export function resolvePackageRuntimeFile( + repoRoot: string, + packageName: string, + artifact: string, +): ResolvedPackageRuntimeFile | undefined { + const contract = readPackageRuntimeFileContract(repoRoot, packageName, artifact); + const hostPath = tryResolveBinary(`programs/${contract.mirrorPath}`); + if (!hostPath || !existsSync(hostPath)) return undefined; + return { ...contract, hostPath }; +} diff --git a/tools/xtask/src/archive_stage.rs b/tools/xtask/src/archive_stage.rs index b68c19a59..f0af1c8b5 100644 --- a/tools/xtask/src/archive_stage.rs +++ b/tools/xtask/src/archive_stage.rs @@ -16,6 +16,7 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; +use crate::build_deps::validate_cache_artifacts; use crate::pkg_manifest::{DepsManifest, ManifestKind, TargetArch}; /// Caller-supplied build provenance + the locally-computed cache-key @@ -63,6 +64,13 @@ pub fn stage_archive_with_options( )); } + // A direct archive-stage invocation must enforce the same declared + // artifact closure as a resolver build/fetch. In particular, do not ship + // an archive whose manifest promises a runtime file that the payload + // omits; the consumer should never be the first place that notices. + validate_cache_artifacts(target, cache_dir) + .map_err(|e| format!("archive_stage: invalid cache entry: {e}"))?; + let manifest_text = build_archive_manifest_text(target, arch, abi_version, opts)?; // Pre-flight: enumerate cache_dir BEFORE touching any tmp file so @@ -140,19 +148,74 @@ pub fn stage_archive_with_options( Ok(()) } -/// Recursively collect every regular file under `dir`. Symlinks and -/// other special files are not packed — the wasm cache layout is all -/// regular files (`lib/*.a`, `include/*.h`, `lib/pkgconfig/*.pc`). +/// Recursively collect every archive leaf under `dir`. Contained symlinks are +/// deliberately dereferenced into regular archive entries so compatibility +/// aliases survive the existing tar format. External/cyclic symlinks and +/// special files fail producer preflight instead of smuggling bytes from +/// outside the cache root or disappearing from the fetched artifact. fn collect_files(dir: &Path, out: &mut Vec) -> Result<(), String> { - for entry in fs::read_dir(dir).map_err(|e| format!("read_dir {}: {e}", dir.display()))? { - let entry = entry.map_err(|e| format!("read_dir entry: {e}"))?; + let canonical_root = fs::canonicalize(dir) + .map_err(|e| format!("resolve archive cache root {}: {e}", dir.display()))?; + let mut active_dirs = std::collections::BTreeSet::new(); + collect_files_inner(dir, &canonical_root, &mut active_dirs, out) +} + +fn collect_files_inner( + dir: &Path, + canonical_root: &Path, + active_dirs: &mut std::collections::BTreeSet, + out: &mut Vec, +) -> Result<(), String> { + let resolved_dir = fs::canonicalize(dir) + .map_err(|e| format!("resolve archive directory {}: {e}", dir.display()))?; + if !resolved_dir.starts_with(canonical_root) { + return Err(format!( + "archive directory {} resolves outside cache root {}", + dir.display(), + canonical_root.display() + )); + } + if !active_dirs.insert(resolved_dir.clone()) { + return Err(format!( + "archive directory symlink cycle reaches {}", + dir.display() + )); + } + let mut entries = fs::read_dir(dir) + .map_err(|e| format!("read_dir {}: {e}", dir.display()))? + .collect::, _>>() + .map_err(|e| format!("read_dir {}: {e}", dir.display()))?; + entries.sort_by_key(|entry| entry.path()); + for entry in entries { let p = entry.path(); - if p.is_dir() { - collect_files(&p, out)?; - } else if p.is_file() { + let link_metadata = fs::symlink_metadata(&p) + .map_err(|e| format!("stat archive entry {}: {e}", p.display()))?; + let resolved = fs::canonicalize(&p) + .map_err(|e| format!("resolve archive entry {}: {e}", p.display()))?; + if !resolved.starts_with(canonical_root) { + return Err(format!( + "archive entry {} resolves outside cache root {}", + p.display(), + canonical_root.display() + )); + } + let metadata = if link_metadata.file_type().is_symlink() { + fs::metadata(&p).map_err(|e| format!("follow archive symlink {}: {e}", p.display()))? + } else { + link_metadata + }; + if metadata.is_dir() { + collect_files_inner(&p, canonical_root, active_dirs, out)?; + } else if metadata.is_file() { out.push(p); + } else { + return Err(format!( + "archive entry {} is not a regular file, directory, or contained symlink", + p.display() + )); } } + active_dirs.remove(&resolved_dir); Ok(()) } @@ -367,6 +430,181 @@ headers = ["include/zlib.h"] assert!(!install_root.join("artifacts").exists()); } + #[test] + fn program_runtime_file_round_trips_through_archive_and_fetch() { + use crate::pkg_manifest::Binary; + use crate::remote_fetch::fetch_and_install; + use sha2::{Digest, Sha256}; + use std::io::Read; + + let dir = tempdir("program-runtime-round-trip"); + let registry = dir.join("registry/php"); + fs::create_dir_all(®istry).unwrap(); + let manifest_text = r#" +kind = "program" +name = "php" +version = "8.3.15" +depends_on = [] + +[source] +url = "https://example.test/php.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" + +[license] +spdx = "PHP-3.01" + +[[outputs]] +name = "php" +wasm = "php.wasm" + +[[runtime_files]] +artifact = "icu.dat" +guest_path = "/usr/lib/php/icu.dat" +mode = 420 +"#; + let manifest_path = registry.join("package.toml"); + fs::write(&manifest_path, manifest_text).unwrap(); + let manifest = DepsManifest::load(&manifest_path).unwrap(); + + let cache_dir = dir.join("cache_entry"); + fs::create_dir_all(&cache_dir).unwrap(); + fs::write( + cache_dir.join("php.wasm"), + b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b", + ) + .unwrap(); + let runtime_bytes = b"runtime-icu-data\0with-binary-bytes\xff"; + fs::write(cache_dir.join("icu.dat"), runtime_bytes).unwrap(); + + let archive_path = dir.join("php-out.tar.zst"); + let opts = StageOptions { + cache_key_sha: "c".repeat(64), + build_timestamp: "2026-07-12T00:00:00Z".to_string(), + build_host: "test-host".to_string(), + }; + stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 19, + &cache_dir, + &archive_path, + &opts, + ) + .unwrap(); + + let archive_bytes = fs::read(&archive_path).unwrap(); + let mut archive_hash = Sha256::new(); + archive_hash.update(&archive_bytes); + let binary = Binary { + archive_url: format!("file://{}", archive_path.display()), + archive_sha256: crate::util::hex(&Into::<[u8; 32]>::into(archive_hash.finalize())), + }; + let install_root = dir.join("install/canonical"); + fs::create_dir_all(install_root.parent().unwrap()).unwrap(); + fetch_and_install( + &binary, + &install_root, + &manifest, + TargetArch::Wasm32, + 19, + &opts.cache_key_sha, + ) + .unwrap(); + assert_eq!( + fs::read(install_root.join("icu.dat")).unwrap(), + runtime_bytes + ); + + // Pin the authored installation contract in the embedded manifest, + // not just the payload byte round-trip. + let decoder = zstd::stream::read::Decoder::new(&archive_bytes[..]).unwrap(); + let mut tar = tar::Archive::new(decoder); + let mut archived_manifest = None; + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + if entry.path().unwrap().as_ref() == Path::new("manifest.toml") { + let mut text = String::new(); + entry.read_to_string(&mut text).unwrap(); + archived_manifest = Some(text); + break; + } + } + let parsed = DepsManifest::parse_archived( + &archived_manifest.expect("archive must contain manifest.toml"), + registry, + ) + .unwrap(); + assert_eq!(parsed.runtime_files, manifest.runtime_files); + + fs::remove_file(cache_dir.join("icu.dat")).unwrap(); + let incomplete_archive = dir.join("php-incomplete.tar.zst"); + let err = stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 19, + &cache_dir, + &incomplete_archive, + &opts, + ) + .unwrap_err(); + assert!( + err.contains("runtime file") && err.contains("icu.dat"), + "got: {err}" + ); + assert!(!incomplete_archive.exists()); + } + + #[cfg(unix)] + #[test] + fn archive_dereferences_contained_symlinks_and_rejects_external_ones() { + use std::io::Read; + use std::os::unix::fs::symlink; + + let (cache_dir, archive_path, manifest, opts) = fixture_for_round_trip("symlink-safety"); + symlink("zlib.h", cache_dir.join("include/zconf.h")).unwrap(); + stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 4, + &cache_dir, + &archive_path, + &opts, + ) + .unwrap(); + + let bytes = fs::read(&archive_path).unwrap(); + let decoder = zstd::stream::read::Decoder::new(&bytes[..]).unwrap(); + let mut tar = tar::Archive::new(decoder); + let mut alias_bytes = None; + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + if entry.path().unwrap().as_ref() == Path::new("artifacts/include/zconf.h") { + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes).unwrap(); + alias_bytes = Some(bytes); + break; + } + } + assert_eq!(alias_bytes.as_deref(), Some(b"#ifndef ZLIB_H\n".as_slice())); + + fs::remove_file(cache_dir.join("include/zconf.h")).unwrap(); + let outside = archive_path.parent().unwrap().join("outside.dat"); + fs::write(&outside, b"outside").unwrap(); + symlink(&outside, cache_dir.join("include/zconf.h")).unwrap(); + let rejected = archive_path.parent().unwrap().join("external-link.tar.zst"); + let err = stage_archive_with_options( + &manifest, + TargetArch::Wasm32, + 4, + &cache_dir, + &rejected, + &opts, + ) + .unwrap_err(); + assert!(err.contains("outside cache root"), "got: {err}"); + assert!(!rejected.exists()); + } + #[test] fn embedded_manifest_round_trips_through_parse_archived() { use std::io::Read; @@ -483,7 +721,9 @@ headers = ["include/zlib.h"] stage_archive_with_options(&m, TargetArch::Wasm32, 4, &empty_cache, &archive, &opts) .unwrap_err(); assert!( - err.contains("contains no files") || err.contains("[outputs]"), + err.contains("missing from cache entry") + || err.contains("contains no files") + || err.contains("[outputs]"), "got: {err}" ); assert!( diff --git a/tools/xtask/src/build_deps.rs b/tools/xtask/src/build_deps.rs index 302aa49b7..719da88f7 100644 --- a/tools/xtask/src/build_deps.rs +++ b/tools/xtask/src/build_deps.rs @@ -345,12 +345,11 @@ pub fn compute_sha( // section would invalidate the entire package registry merely for // learning a new output kind. if !target.outputs.files.is_empty() { - h.update(b"outputs.files:\n"); + h.update(b"outputs.files:v1\n"); for s in &target.outputs.files { + h.update((s.len() as u64).to_le_bytes()); h.update(s.as_bytes()); - h.update(b"|"); } - h.update(b"\n"); } h.update(b"program_outputs:\n"); for out in &target.program_outputs { @@ -363,6 +362,22 @@ pub fn compute_sha( } h.update(b"\n"); } + // Additive program runtime closure. Keep the section absent for + // existing manifests so learning this schema does not invalidate + // every historical package cache key. + if !target.runtime_files.is_empty() { + h.update(b"runtime_files:v1\n"); + for runtime_file in &target.runtime_files { + for field in [ + runtime_file.artifact.as_bytes(), + runtime_file.guest_path.as_bytes(), + ] { + h.update((field.len() as u64).to_le_bytes()); + h.update(field); + } + h.update(runtime_file.mode.to_le_bytes()); + } + } } } if !build_inputs.is_empty() { @@ -2362,24 +2377,209 @@ fn required_exports_for_program_output( } } -fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), String> { - if !matches!(target.kind, ManifestKind::Program) { - return Ok(()); +fn validate_declared_artifact( + target: &DepsManifest, + root: &Path, + rel: &str, + label: &str, + missing_suffix: &str, + require_regular_file: bool, +) -> Result { + let path = root.join(rel); + let metadata = std::fs::symlink_metadata(&path).map_err(|_| { + format!( + "{}: declared {} output {:?} {}", + target.spec(), + label, + rel, + missing_suffix + ) + })?; + if metadata.file_type().is_symlink() { + return Err(format!( + "{}: declared {} output {:?} must not be a symlink", + target.spec(), + label, + rel + )); } - for out in &target.program_outputs { - let path = dir.join(&out.wasm); - if !path.exists() { + let canonical_root = std::fs::canonicalize(root) + .map_err(|e| format!("{}: resolve package artifact root: {e}", target.spec()))?; + let resolved = std::fs::canonicalize(&path).map_err(|e| { + format!( + "{}: resolve declared {} output {:?}: {e}", + target.spec(), + label, + rel + ) + })?; + if !resolved.starts_with(&canonical_root) { + return Err(format!( + "{}: declared {} output {:?} resolves outside the package artifact root", + target.spec(), + label, + rel + )); + } + if require_regular_file && !metadata.is_file() { + return Err(format!( + "{}: declared {} output {:?} must be a regular file", + target.spec(), + label, + rel + )); + } + if !require_regular_file { + if metadata.is_file() { + return Ok(path); + } + if !metadata.is_dir() { return Err(format!( - "{}: declared wasm output {:?} missing from cache entry", + "{}: declared {} output {:?} must be a regular file or directory", target.spec(), - out.wasm + label, + rel )); } - validate_wasm_artifact_policy( - &path, - out.fork_instrumentation, - required_exports_for_program_output(target, out), - )?; + let mut active_dirs = BTreeSet::new(); + let leaf_count = validate_artifact_tree(&canonical_root, &path, &mut active_dirs)?; + if leaf_count == 0 { + return Err(format!( + "{}: declared {} output {:?} is an empty directory and cannot round-trip through an artifact archive", + target.spec(), + label, + rel + )); + } + } + Ok(path) +} + +/// Validate every reachable leaf below a declared artifact directory. +/// Internal symlinks are allowed because several library packages publish +/// compatibility aliases; external/cyclic links and special files are not. +fn validate_artifact_tree( + canonical_root: &Path, + path: &Path, + active_dirs: &mut BTreeSet, +) -> Result { + let link_metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("stat package artifact {}: {e}", path.display()))?; + let resolved = std::fs::canonicalize(path) + .map_err(|e| format!("resolve package artifact {}: {e}", path.display()))?; + if !resolved.starts_with(canonical_root) { + return Err(format!( + "package artifact {} resolves outside {}", + path.display(), + canonical_root.display() + )); + } + let metadata = if link_metadata.file_type().is_symlink() { + std::fs::metadata(path) + .map_err(|e| format!("follow package artifact symlink {}: {e}", path.display()))? + } else { + link_metadata + }; + if metadata.is_file() { + return Ok(1); + } + if !metadata.is_dir() { + return Err(format!( + "package artifact {} is not a regular file, directory, or contained symlink", + path.display() + )); + } + if !active_dirs.insert(resolved.clone()) { + return Err(format!( + "package artifact directory symlink cycle reaches {}", + path.display() + )); + } + let mut entries = std::fs::read_dir(path) + .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))? + .collect::, _>>() + .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))?; + entries.sort_by_key(|entry| entry.path()); + let mut leaves = 0usize; + for entry in entries { + leaves += validate_artifact_tree(canonical_root, &entry.path(), active_dirs)?; + } + active_dirs.remove(&resolved); + Ok(leaves) +} + +pub(crate) fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), String> { + match target.kind { + ManifestKind::Library => { + for rel in &target.outputs.libs { + validate_declared_artifact( + target, + dir, + rel, + "libs", + "missing from cache entry", + true, + )?; + } + for rel in &target.outputs.headers { + validate_declared_artifact( + target, + dir, + rel, + "headers", + "missing from cache entry", + false, + )?; + } + for rel in &target.outputs.pkgconfig { + validate_declared_artifact( + target, + dir, + rel, + "pkgconfig", + "missing from cache entry", + true, + )?; + } + for rel in &target.outputs.files { + validate_declared_artifact( + target, + dir, + rel, + "files", + "missing from cache entry", + true, + )?; + } + } + ManifestKind::Program => { + for out in &target.program_outputs { + let path = validate_declared_artifact( + target, + dir, + &out.wasm, + "wasm", + "missing from cache entry", + true, + )?; + validate_wasm_artifact_policy( + &path, + out.fork_instrumentation, + required_exports_for_program_output(target, out), + )?; + } + for runtime_file in &target.runtime_files { + validate_declared_artifact( + target, + dir, + &runtime_file.artifact, + "runtime file", + "missing from cache entry", + true, + )?; + } + } + ManifestKind::Source => {} } Ok(()) } @@ -2387,47 +2587,73 @@ fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), Str fn validate_outputs(target: &DepsManifest, out_dir: &Path) -> Result<(), String> { match target.kind { ManifestKind::Library => { - let check = |rel: &str, label: &str| -> Result<(), String> { - let p = out_dir.join(rel); - if !p.exists() { - return Err(format!( - "{}: declared {} output {:?} not produced by build script", - target.spec(), - label, - rel - )); - } - Ok(()) - }; for rel in &target.outputs.libs { - check(rel, "libs")?; + validate_declared_artifact( + target, + out_dir, + rel, + "libs", + "not produced by build script", + true, + )?; } for rel in &target.outputs.headers { - check(rel, "headers")?; + validate_declared_artifact( + target, + out_dir, + rel, + "headers", + "not produced by build script", + false, + )?; } for rel in &target.outputs.pkgconfig { - check(rel, "pkgconfig")?; + validate_declared_artifact( + target, + out_dir, + rel, + "pkgconfig", + "not produced by build script", + true, + )?; } for rel in &target.outputs.files { - check(rel, "files")?; + validate_declared_artifact( + target, + out_dir, + rel, + "files", + "not produced by build script", + true, + )?; } } ManifestKind::Program => { for out in &target.program_outputs { - let p = out_dir.join(&out.wasm); - if !p.exists() { - return Err(format!( - "{}: declared wasm output {:?} not produced by build script", - target.spec(), - out.wasm - )); - } + let p = validate_declared_artifact( + target, + out_dir, + &out.wasm, + "wasm", + "not produced by build script", + true, + )?; validate_wasm_artifact_policy( &p, out.fork_instrumentation, required_exports_for_program_output(target, out), )?; } + for runtime_file in &target.runtime_files { + validate_declared_artifact( + target, + out_dir, + &runtime_file.artifact, + "runtime file", + "not produced by build script", + true, + )?; + } } // No outputs to validate for source-kind (Chunk C). ManifestKind::Source => return Ok(()), @@ -2614,7 +2840,7 @@ pub fn run(args: Vec) -> Result<(), String> { let mut it = rest.into_iter(); let sub = it.next().ok_or( "usage: xtask build-deps [--arch=wasm32|wasm64] [--binaries-dir ] [--fetch-only] \ - \ + \ [ []]", )?; let target = it.next(); @@ -2709,6 +2935,22 @@ pub fn run(args: Vec) -> Result<(), String> { })?; cmd_output_path(&manifest, &basename) } + "runtime-file-path" => { + let artifact = extra.ok_or_else(|| { + "build-deps runtime-file-path: missing \ + (usage: build-deps runtime-file-path )" + .to_string() + })?; + cmd_runtime_file_path(&manifest, &artifact) + } + "runtime-file-metadata" => { + let artifact = extra.ok_or_else(|| { + "build-deps runtime-file-metadata: missing \ + (usage: build-deps runtime-file-metadata )" + .to_string() + })?; + cmd_runtime_file_metadata(&manifest, &artifact) + } "output-fork-instrumentation" => { let basename = extra.ok_or_else(|| { "build-deps output-fork-instrumentation: missing \ @@ -2777,6 +3019,9 @@ fn cmd_parse(m: &DepsManifest) -> Result<(), String> { if !m.outputs.files.is_empty() { println!("outputs.files = {:?}", m.outputs.files); } + if !m.runtime_files.is_empty() { + println!("runtime_files = {:?}", m.runtime_files); + } Ok(()) } @@ -2828,6 +3073,47 @@ fn cmd_output_path(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> Ok(()) } +/// `runtime-file-path `: print the mirror path +/// below `programs//` used by local and resolver materialization. +fn cmd_runtime_file_path(m: &DepsManifest, artifact: &str) -> Result<(), String> { + let rel = m.runtime_file_dest_rel(artifact)?; + println!("{}", rel.display()); + Ok(()) +} + +/// Structured installation contract for VFS/image builders. JSON avoids +/// consumers scraping Debug output and keeps guest path/mode authoritative. +fn cmd_runtime_file_metadata(m: &DepsManifest, artifact: &str) -> Result<(), String> { + let value = runtime_file_metadata_value(m, artifact)?; + println!( + "{}", + serde_json::to_string(&value).map_err(|e| format!("serialize runtime metadata: {e}"))? + ); + Ok(()) +} + +fn runtime_file_metadata_value( + m: &DepsManifest, + artifact: &str, +) -> Result { + let runtime_file = m + .runtime_files + .iter() + .find(|runtime_file| runtime_file.artifact == artifact) + .ok_or_else(|| { + format!( + "program {:?} has no [[runtime_files]] artifact {:?}", + m.name, artifact + ) + })?; + Ok(serde_json::json!({ + "artifact": runtime_file.artifact, + "guest_path": runtime_file.guest_path, + "mode": runtime_file.mode, + "mirror_path": m.runtime_file_dest_rel_for(runtime_file), + })) +} + fn cmd_output_fork_instrumentation(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> { let policy = m.output_fork_instrumentation(wasm_basename)?; println!("{}", policy.as_str()); @@ -2904,7 +3190,8 @@ fn cmd_resolve( } /// Place symlinks under `binaries_dir/programs//` pointing at -/// each declared `[[outputs]]` wasm in the cache canonical directory. +/// each declared `[[outputs]]` artifact and `[[runtime_files]]` file in the +/// cache canonical directory. /// /// Layout (per arch — wasm32 and wasm64 mirror in parallel): /// * 1 output: `/programs//.wasm`. @@ -2960,6 +3247,34 @@ fn place_binaries_symlinks( std::os::unix::fs::symlink(&src, &dest) .map_err(|e| format!("symlink {} -> {}: {e}", dest.display(), src.display()))?; } + for runtime_file in &m.runtime_files { + let src = canonical.join(&runtime_file.artifact); + let metadata = std::fs::symlink_metadata(&src).map_err(|e| { + format!( + "declared runtime file {} not found in cache at {}: {e}", + runtime_file.artifact, + src.display() + ) + })?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "declared runtime file {} is not a regular non-symlink file at {}", + runtime_file.artifact, + src.display() + )); + } + let dest = arch_root.join(m.runtime_file_dest_rel_for(runtime_file)); + let dest_dir = dest + .parent() + .ok_or_else(|| format!("dest path {} has no parent", dest.display()))?; + std::fs::create_dir_all(dest_dir) + .map_err(|e| format!("mkdir {}: {e}", dest_dir.display()))?; + if dest.exists() || dest.symlink_metadata().is_ok() { + let _ = std::fs::remove_file(&dest); + } + std::os::unix::fs::symlink(&src, &dest) + .map_err(|e| format!("symlink {} -> {}: {e}", dest.display(), src.display()))?; + } Ok(()) } @@ -4152,6 +4467,70 @@ fork_instrumentation = "disabled" ); } + #[test] + fn cache_key_sha_tracks_program_runtime_file_contract() { + let root = tempdir("sha-prog-runtime-file"); + write_program_manifest( + &root, + "php", + "1.0.0", + "[[outputs]]\nname = \"php\"\nwasm = \"php.wasm\"\n", + ); + let reg = Registry { + roots: vec![root.clone()], + }; + let baseline = sha_of(®, "php"); + let toml_path = root.join("php/package.toml"); + let original = std::fs::read_to_string(&toml_path).unwrap(); + + let with_runtime = format!( + "{original}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n" + ); + std::fs::write(&toml_path, &with_runtime).unwrap(); + let added = sha_of(®, "php"); + assert_ne!( + baseline, added, + "adding runtime closure must invalidate the key" + ); + + std::fs::write( + &toml_path, + with_runtime.replace("/usr/lib/php/icu.dat", "/opt/php/icu.dat"), + ) + .unwrap(); + let moved = sha_of(®, "php"); + assert_ne!( + added, moved, + "changing the guest path must invalidate the key" + ); + + std::fs::write(&toml_path, format!("{with_runtime}mode = 384\n")).unwrap(); + let remoded = sha_of(®, "php"); + assert_ne!( + added, remoded, + "changing runtime mode must invalidate the key" + ); + + // Length prefixes keep delimiter-bearing fields unambiguous. These + // two records collide under naive `artifact|guest` concatenation. + std::fs::write( + &toml_path, + format!("{original}\n[[runtime_files]]\nartifact = \"a|/b\"\nguest_path = \"/c\"\n"), + ) + .unwrap(); + let delimiter_a = sha_of(®, "php"); + std::fs::write( + &toml_path, + format!("{original}\n[[runtime_files]]\nartifact = \"a\"\nguest_path = \"/b|/c\"\n"), + ) + .unwrap(); + let delimiter_b = sha_of(®, "php"); + assert_ne!( + delimiter_a, delimiter_b, + "runtime hash fields must be framed" + ); + } + /// Pins behavior: program outputs are hashed in declaration order. /// Re-ordering DOES change cache_key_sha. We deliberately don't /// normalize because (a) the manifest preserves authored order @@ -4299,6 +4678,42 @@ fork_instrumentation = "disabled" ); } + #[test] + fn library_runtime_file_cache_keys_frame_delimiter_bearing_paths() { + let root = tempdir("sha-lib-runtime-file-framing"); + write(&root, "libZ", "1.0.0", &[]); + let reg = Registry { + roots: vec![root.clone()], + }; + let toml_path = root.join("libZ/package.toml"); + let original = std::fs::read_to_string(&toml_path).unwrap(); + + std::fs::write( + &toml_path, + original.replace( + "libs = [\"lib/liblibZ.a\"]", + "libs = [\"lib/liblibZ.a\"]\nfiles = [\"a|b\", \"c\"]", + ), + ) + .unwrap(); + let delimiter_a = sha_of(®, "libZ"); + + std::fs::write( + &toml_path, + original.replace( + "libs = [\"lib/liblibZ.a\"]", + "libs = [\"lib/liblibZ.a\"]\nfiles = [\"a\", \"b|c\"]", + ), + ) + .unwrap(); + let delimiter_b = sha_of(®, "libZ"); + + assert_ne!( + delimiter_a, delimiter_b, + "library runtime-file cache-key fields must be length framed" + ); + } + // --- ensure_built / build_into_cache tests --- /// Create a package.toml + build-.sh pair. The build script uses @@ -4583,6 +4998,61 @@ files = ["share/runtime.dat"] ); } + #[test] + fn ensure_built_rejects_declared_runtime_file_directory() { + let root = tempdir("built-runtime-file-directory"); + let cache = tempdir("built-runtime-file-directory-cache"); + write_lib( + &root, + "libRuntimeDirectory", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeDirectory.a""#, + r#"[outputs] +libs = ["lib/libRuntimeDirectory.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeDirectory").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must be a regular file"), "got: {err}"); + } + + #[cfg(unix)] + #[test] + fn ensure_built_rejects_declared_runtime_file_symlink_escape() { + let root = tempdir("built-runtime-file-symlink-escape"); + let cache = tempdir("built-runtime-file-symlink-escape-cache"); + let outside = root.join("outside.dat"); + std::fs::write(&outside, b"outside").unwrap(); + write_lib( + &root, + "libRuntimeSymlinkEscape", + "1.0.0", + &[], + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeSymlinkEscape.a" +ln -s {:?} "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, + outside + ), + r#"[outputs] +libs = ["lib/libRuntimeSymlinkEscape.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeSymlinkEscape").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must not be a symlink"), "got: {err}"); + } + #[test] fn ensure_built_fails_when_script_exits_nonzero() { let root = tempdir("built-badexit"); @@ -5303,6 +5773,49 @@ spdx = "TestLicense" name = "{output_name}" wasm = "{output_wasm}" +[compatibility] +target_arch = "{arch}" +abi_versions = [{abi_csv}] +cache_key_sha = "{cache_key_sha}" +"#, + "" + ) + } + + fn archived_program_runtime_manifest_text( + name: &str, + arch: &str, + abi_versions: &[u32], + cache_key_sha: &str, + ) -> String { + let abi_csv = abi_versions + .iter() + .map(|value| value.to_string()) + .collect::>() + .join(", "); + format!( + r#" +kind = "program" +name = "{name}" +version = "1.0.0" +revision = 1 +depends_on = [] + +[source] +url = "https://example.test/{name}.tar.gz" +sha256 = "{:0>64}" + +[license] +spdx = "MIT" + +[[outputs]] +name = "{name}" +wasm = "{name}.wasm" + +[[runtime_files]] +artifact = "icu.dat" +guest_path = "/usr/lib/php/icu.dat" + [compatibility] target_arch = "{arch}" abi_versions = [{abi_csv}] @@ -5385,6 +5898,33 @@ index_url = "{index_url}" std::fs::write(dir.join("build.toml"), build_toml).unwrap(); } + fn write_runtime_program_with_index(root: &Path, name: &str, index_url: &str) { + write_program( + root, + name, + "1.0.0", + &[], + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/{name}.wasm" +printf RUNTIME-BYTES > "$WASM_POSIX_DEP_OUT_DIR/icu.dat" +touch "$WASM_POSIX_DEP_OUT_DIR/via-build""#, + ), + &[(name, &format!("{name}.wasm"))], + ); + append_program_runtime_file(root, name, "icu.dat", "/usr/lib/php/icu.dat"); + let build_toml = format!( + r#"script_path = "{name}/build-{name}.sh" +repo_url = "https://example.test/repo.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + +[binary] +index_url = "{index_url}" +"#, + ); + fs::write(root.join(name).join("build.toml"), build_toml).unwrap(); + } + /// Stage an `index.toml` at `path` declaring `name@1.0.0` with a /// single Success entry for `arch` pointing at `archive_url` with /// the given `archive_sha256` and `cache_key_sha`. Mirrors what @@ -5753,6 +6293,185 @@ cache_key_sha = "{cache_key_hex}" ); } + #[test] + fn fetched_program_runtime_file_matches_source_mirror_layout() { + let root = tempdir("runtime-fetch-parity-reg"); + let remote_cache = tempdir("runtime-fetch-parity-remote-cache"); + let source_cache = tempdir("runtime-fetch-parity-source-cache"); + let remote_bin = tempdir("runtime-fetch-parity-remote-bin"); + let source_bin = tempdir("runtime-fetch-parity-source-bin"); + let archive_dir = tempdir("runtime-fetch-parity-archive"); + let index_dir = tempdir("runtime-fetch-parity-index"); + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + let name = "runtimeFetched"; + write_runtime_program_with_index(&root, name, &index_url); + + let reg = Registry { + roots: vec![root.clone()], + }; + let manifest = reg.load(name).unwrap(); + let cache_key_hex = hex(&compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap()); + let archived_manifest = archived_program_runtime_manifest_text( + name, + TEST_ARCH.as_str(), + &[TEST_ABI], + &cache_key_hex, + ); + let wasm_name = format!("{name}.wasm"); + let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; + let archive_bytes = crate::remote_fetch::build_test_archive( + &archived_manifest, + &[ + (wasm_name.as_str(), wasm_bytes.as_slice()), + ("icu.dat", b"RUNTIME-BYTES"), + ], + ); + let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); + fs::write(&archive_path, &archive_bytes).unwrap(); + stage_index_toml( + &index_path, + name, + TEST_ARCH, + &format!("file://{}", archive_path.display()), + &sha256_hex(&archive_bytes), + &cache_key_hex, + ); + + let remote_opts = ResolveOpts { + cache_root: &remote_cache, + local_libs: None, + force_source_build: None, + fetch_only: false, + repo_root: Some(&root), + binaries_dir: Some(&remote_bin), + }; + let remote_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &remote_opts).unwrap(); + assert!(!remote_path.join("via-build").exists()); + place_binaries_symlinks(&manifest, &remote_path, &remote_bin, TEST_ARCH).unwrap(); + + let force = BTreeSet::from([name.to_string()]); + let source_opts = ResolveOpts { + cache_root: &source_cache, + local_libs: None, + force_source_build: Some(&force), + fetch_only: false, + repo_root: Some(&root), + binaries_dir: Some(&source_bin), + }; + let source_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &source_opts).unwrap(); + assert!(source_path.join("via-build").exists()); + place_binaries_symlinks(&manifest, &source_path, &source_bin, TEST_ARCH).unwrap(); + + let mirror_rel = Path::new("programs/wasm32").join(name).join("icu.dat"); + assert_eq!( + fs::read(remote_bin.join(&mirror_rel)).unwrap(), + b"RUNTIME-BYTES" + ); + assert_eq!( + fs::read(source_bin.join(&mirror_rel)).unwrap(), + b"RUNTIME-BYTES" + ); + } + + #[test] + fn incomplete_fetched_runtime_file_falls_back_or_fails_fetch_only() { + let root = tempdir("runtime-fetch-incomplete-reg"); + let fallback_cache = tempdir("runtime-fetch-incomplete-fallback-cache"); + let fetch_only_cache = tempdir("runtime-fetch-incomplete-only-cache"); + let archive_dir = tempdir("runtime-fetch-incomplete-archive"); + let index_dir = tempdir("runtime-fetch-incomplete-index"); + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + let name = "runtimeIncomplete"; + write_runtime_program_with_index(&root, name, &index_url); + + let reg = Registry { + roots: vec![root.clone()], + }; + let manifest = reg.load(name).unwrap(); + let cache_key_hex = hex(&compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap()); + let archived_manifest = archived_program_runtime_manifest_text( + name, + TEST_ARCH.as_str(), + &[TEST_ABI], + &cache_key_hex, + ); + let wasm_name = format!("{name}.wasm"); + let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; + let archive_bytes = crate::remote_fetch::build_test_archive( + &archived_manifest, + &[(wasm_name.as_str(), wasm_bytes.as_slice())], + ); + let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); + fs::write(&archive_path, &archive_bytes).unwrap(); + stage_index_toml( + &index_path, + name, + TEST_ARCH, + &format!("file://{}", archive_path.display()), + &sha256_hex(&archive_bytes), + &cache_key_hex, + ); + + let fallback = ensure_built( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &resolve_opts(&fallback_cache, None), + ) + .unwrap(); + assert!(fallback.join("via-build").exists()); + assert_eq!( + fs::read(fallback.join("icu.dat")).unwrap(), + b"RUNTIME-BYTES" + ); + + let fetch_only_opts = ResolveOpts { + cache_root: &fetch_only_cache, + local_libs: None, + force_source_build: None, + fetch_only: true, + repo_root: Some(&root), + binaries_dir: None, + }; + let err = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &fetch_only_opts).unwrap_err(); + assert!(err.contains("fetch-only"), "got: {err}"); + assert!(!canonical_path( + &fetch_only_cache, + &manifest, + TEST_ARCH, + &compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap(), + ) + .join("via-build") + .exists()); + } + #[test] fn index_fetch_falls_through_on_archive_sha_mismatch() { let root = tempdir("idx-shafail-reg"); @@ -6045,6 +6764,15 @@ spdx = "MIT" } } + fn append_program_runtime_file(root: &Path, name: &str, artifact: &str, guest_path: &str) { + let manifest_path = root.join(name).join("package.toml"); + let mut text = fs::read_to_string(&manifest_path).unwrap(); + text.push_str(&format!( + "\n[[runtime_files]]\nartifact = {artifact:?}\nguest_path = {guest_path:?}\n" + )); + fs::write(manifest_path, text).unwrap(); + } + #[test] fn canonical_path_uses_programs_subdir_for_program_kind() { let m = DepsManifest::parse( @@ -6108,6 +6836,96 @@ wasm = "vim.wasm" assert!(err.contains("miss.wasm"), "got: {err}"); } + #[test] + fn program_runtime_file_is_required_and_cached_as_a_regular_file() { + let root = tempdir("prog-runtime-file"); + let cache = tempdir("prog-runtime-file-cache"); + write_program( + &root, + "runtimeprog", + "0.1.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimeprog.wasm" +printf runtime-data > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, + &[("runtimeprog", "runtimeprog.wasm")], + ); + append_program_runtime_file(&root, "runtimeprog", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("runtimeprog").unwrap(); + assert_eq!( + runtime_file_metadata_value(&m, "icu.dat").unwrap(), + serde_json::json!({ + "artifact": "icu.dat", + "guest_path": "/usr/lib/php/icu.dat", + "mode": 420, + "mirror_path": "runtimeprog/icu.dat", + }) + ); + let path = + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap(); + assert_eq!(fs::read(path.join("icu.dat")).unwrap(), b"runtime-data"); + + fs::remove_file(path.join("icu.dat")).unwrap(); + let err = validate_cache_artifacts(&m, &path).unwrap_err(); + assert!( + err.contains("runtime file") && err.contains("missing"), + "got: {err}" + ); + } + + #[test] + fn build_fails_when_program_runtime_file_is_missing() { + let root = tempdir("prog-runtime-file-missing"); + let cache = tempdir("prog-runtime-file-missing-cache"); + write_program( + &root, + "runtimemissing", + "0.1.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimemissing.wasm""#, + &[("runtimemissing", "runtimemissing.wasm")], + ); + append_program_runtime_file(&root, "runtimemissing", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("runtimemissing").unwrap(); + let err = + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap_err(); + assert!( + err.contains("runtime file") && err.contains("icu.dat"), + "got: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn build_rejects_program_runtime_file_symlink() { + let root = tempdir("prog-runtime-file-symlink"); + let cache = tempdir("prog-runtime-file-symlink-cache"); + let outside = root.join("outside.dat"); + fs::write(&outside, b"outside").unwrap(); + write_program( + &root, + "runtimesymlink", + "0.1.0", + &[], + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimesymlink.wasm" +ln -s {:?} "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, + outside + ), + &[("runtimesymlink", "runtimesymlink.wasm")], + ); + append_program_runtime_file(&root, "runtimesymlink", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("runtimesymlink").unwrap(); + let err = + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must not be a symlink"), "got: {err}"); + } + #[test] fn program_output_validation_rejects_legacy_asyncify_wasm() { let out = tempdir("prog-out-asyncify"); @@ -7385,6 +8203,36 @@ libs = ["lib/libF3b.a"] ); } + #[test] + fn cmd_resolve_materializes_program_runtime_file_under_package_directory() { + let root = tempdir("resolve-bdir-runtime-reg"); + let cache = tempdir("resolve-bdir-runtime-cache"); + let bin_dir = tempdir("resolve-bdir-runtime-bin"); + write_program( + &root, + "runtimebin", + "0.1.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimebin.wasm" +printf canonical-runtime > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, + &[("runtimebin", "runtimebin.wasm")], + ); + append_program_runtime_file(&root, "runtimebin", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { + roots: vec![root.clone()], + }; + let m = reg.load("runtimebin").unwrap(); + + cmd_resolve_with_test_cache(&m, ®, &root, TargetArch::Wasm32, &cache, Some(&bin_dir)) + .unwrap(); + + let runtime = bin_dir.join("programs/wasm32/runtimebin/icu.dat"); + assert!(runtime.symlink_metadata().unwrap().file_type().is_symlink()); + assert_eq!(fs::read(runtime).unwrap(), b"canonical-runtime"); + assert!(bin_dir.join("programs/wasm32/runtimebin.wasm").exists()); + } + #[test] fn cmd_resolve_with_binaries_dir_places_kernel_at_root() { // First-party kernel/userspace artifacts are consumed as diff --git a/tools/xtask/src/pkg_manifest.rs b/tools/xtask/src/pkg_manifest.rs index 7defae9ac..6d3cd0788 100644 --- a/tools/xtask/src/pkg_manifest.rs +++ b/tools/xtask/src/pkg_manifest.rs @@ -180,6 +180,25 @@ pub struct ProgramOutput { pub fork_instrumentation: ForkInstrumentationPolicy, } +/// A non-Wasm file that a program package needs at runtime. +/// +/// `artifact` is produced in the package cache/archive; `guest_path` is where +/// VFS consumers install those bytes. Runtime files are mirrored separately +/// from `[[outputs]]` so the latter remains truthful about executable/archive +/// program artifacts and its legacy output-count layout stays unchanged. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeFile { + pub artifact: String, + pub guest_path: String, + #[serde(default = "default_runtime_file_mode")] + pub mode: u32, +} + +fn default_runtime_file_mode() -> u32 { + 0o644 +} + /// One entry in a manifest's `[[host_tools]]` array. Inline /// declaration on the consumer site — no separate registry entry, /// per design 10. @@ -414,6 +433,10 @@ pub struct DepsManifest { /// `validate_outputs` in the resolver (wired in Chunk B Task B.2). pub program_outputs: Vec, + /// Non-Wasm runtime closure declared by `kind = "program"` manifests. + /// Empty for library/source manifests. + pub runtime_files: Vec, + /// Build-time provenance + ABI compatibility. Always `None` for /// manifests parsed via [`DepsManifest::parse`] (source `package.toml`) /// and always `Some` for those parsed via @@ -699,6 +722,98 @@ fn validate_build_input_path(path: &str) -> Result<(), String> { Ok(()) } +/// Validate an artifact path before it can be joined to a build/cache root. +/// +/// This is shared by library and program outputs so neither manifest shape can +/// escape its package directory through an absolute path or `..` component. +fn validate_output_path(path: &str, context: &str) -> Result<(), String> { + if path.is_empty() { + return Err(format!("{context} must not be empty")); + } + if path.contains('\0') || path.contains('\\') { + return Err(format!( + "{context} must be a portable '/'-separated path, got {path:?}" + )); + } + if path.starts_with('/') || Path::new(path).is_absolute() { + return Err(format!( + "{context} must be relative to the package output directory, got {path:?}" + )); + } + for component in path.split('/') { + if component.is_empty() { + return Err(format!( + "{context} must not contain empty path components, got {path:?}" + )); + } + if component == "." { + return Err(format!("{context} must not contain '.', got {path:?}")); + } + if component == ".." { + return Err(format!("{context} must not contain '..', got {path:?}")); + } + } + Ok(()) +} + +fn validate_single_path_component(value: &str, context: &str) -> Result<(), String> { + if value.is_empty() { + return Err(format!("{context} must not be empty")); + } + if value == "." + || value == ".." + || value.contains('/') + || value.contains('\\') + || value.contains('\0') + { + return Err(format!( + "{context} must be a safe single path component, got {value:?}" + )); + } + Ok(()) +} + +/// Both paths describe files. Equal paths and ancestor/descendant pairs are +/// therefore unsatisfiable in one artifact or VFS closure. +fn file_paths_conflict(a: &str, b: &str) -> bool { + a == b + || a.strip_prefix(b) + .is_some_and(|suffix| suffix.starts_with('/')) + || b.strip_prefix(a) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +fn validate_runtime_guest_path(path: &str, context: &str) -> Result<(), String> { + if path.contains('\0') || path.contains('\\') { + return Err(format!( + "{context} must be a normalized absolute POSIX path, got {path:?}" + )); + } + if !path.starts_with('/') || path == "/" { + return Err(format!( + "{context} must be a non-root absolute POSIX path, got {path:?}" + )); + } + if path.ends_with('/') { + return Err(format!( + "{context} must not have a trailing '/', got {path:?}" + )); + } + for component in path[1..].split('/') { + if component.is_empty() { + return Err(format!( + "{context} must not contain empty path components, got {path:?}" + )); + } + if component == "." || component == ".." { + return Err(format!( + "{context} must not contain {component:?}, got {path:?}" + )); + } + } + Ok(()) +} + #[derive(Debug, Clone, Deserialize, Default)] pub struct Outputs { #[serde(default)] @@ -785,6 +900,8 @@ struct Raw { #[serde(default = "default_outputs_value")] outputs: toml::Value, #[serde(default)] + runtime_files: Vec, + #[serde(default)] compatibility: Option, // `binary` accepts two shapes: // * bare `[binary]` (archive_url + archive_sha256 directly), @@ -816,7 +933,63 @@ fn default_outputs_value() -> toml::Value { toml::Value::Table(toml::value::Table::new()) } +fn program_output_dest_rel( + package_name: &str, + output_count: usize, + out: &ProgramOutput, +) -> PathBuf { + let basename = Path::new(&out.wasm) + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or(&out.wasm); + let ext = match basename.find('.') { + Some(i) => &basename[i..], + None => "", + }; + let dest_name = format!("{}{}", out.name, ext); + if output_count > 1 { + Path::new(package_name).join(dest_name) + } else { + PathBuf::from(dest_name) + } +} + impl DepsManifest { + /// Resolver mirror path under `programs//` for a runtime file. + /// Runtime files always live below the package name, independently of the + /// number of executable `[[outputs]]` entries. + pub fn runtime_file_dest_rel_for(&self, runtime_file: &RuntimeFile) -> PathBuf { + Path::new(&self.name).join(&runtime_file.artifact) + } + + /// Runtime-file mirror path keyed by the exact declared artifact path. + /// Exact lookup keeps nested runtime closures representable and avoids + /// basename ambiguity (`a/data` versus `b/data`). + pub fn runtime_file_dest_rel(&self, artifact: &str) -> Result { + if self.kind != ManifestKind::Program { + return Err(format!( + "manifest {:?} is kind={:?}; runtime-file lookup is program-only", + self.name, self.kind + )); + } + let matches: Vec<&RuntimeFile> = self + .runtime_files + .iter() + .filter(|runtime_file| runtime_file.artifact == artifact) + .collect(); + match matches.as_slice() { + [runtime_file] => Ok(self.runtime_file_dest_rel_for(runtime_file)), + [] => Err(format!( + "program {:?} has no [[runtime_files]] artifact {:?}", + self.name, artifact + )), + _ => Err(format!( + "program {:?} has multiple [[runtime_files]] artifacts {:?}", + self.name, artifact + )), + } + } + /// Relative path under `programs//` where `out` should land /// once produced. Single source of truth for the placement /// convention; both `place_binaries_symlinks` (resolver-driven @@ -832,20 +1005,7 @@ impl DepsManifest { /// `` is everything from the first `.` onward in `out.wasm`'s /// basename, so `.vfs.zst`, `.tar.gz`, etc. round-trip intact. pub fn output_dest_rel_for(&self, out: &ProgramOutput) -> PathBuf { - let basename = Path::new(&out.wasm) - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or(&out.wasm); - let ext = match basename.find('.') { - Some(i) => &basename[i..], - None => "", - }; - let dest_name = format!("{}{}", out.name, ext); - if self.program_outputs.len() > 1 { - Path::new(&self.name).join(dest_name) - } else { - PathBuf::from(dest_name) - } + program_output_dest_rel(&self.name, self.program_outputs.len(), out) } /// Same as [`output_dest_rel_for`] but keyed by the `wasm` @@ -1191,6 +1351,7 @@ impl DepsManifest { if raw.name.is_empty() { return Err("name must not be empty".into()); } + validate_single_path_component(&raw.name, "name")?; if raw.name.contains('@') { return Err(format!("name {:?} must not contain '@'", raw.name)); } @@ -1334,6 +1495,57 @@ impl DepsManifest { }); } + let runtime_files = raw.runtime_files; + if !matches!(raw.kind, ManifestKind::Program) && !runtime_files.is_empty() { + return Err("[[runtime_files]] is allowed only for kind = \"program\"".into()); + } + let mut runtime_artifacts = BTreeSet::new(); + let mut runtime_guest_paths = BTreeSet::new(); + for (idx, runtime_file) in runtime_files.iter().enumerate() { + validate_output_path( + &runtime_file.artifact, + &format!("[[runtime_files]][{idx}].artifact"), + )?; + validate_runtime_guest_path( + &runtime_file.guest_path, + &format!("[[runtime_files]][{idx}].guest_path"), + )?; + if runtime_file.mode > 0o777 { + return Err(format!( + "[[runtime_files]][{idx}].mode must be between 0 and 0777, got {}", + runtime_file.mode + )); + } + if !runtime_artifacts.insert(runtime_file.artifact.clone()) { + return Err(format!( + "[[runtime_files]] declares artifact {:?} more than once", + runtime_file.artifact + )); + } + if !runtime_guest_paths.insert(runtime_file.guest_path.clone()) { + return Err(format!( + "[[runtime_files]] declares guest_path {:?} more than once", + runtime_file.guest_path + )); + } + } + for (index, left) in runtime_files.iter().enumerate() { + for right in runtime_files.iter().skip(index + 1) { + if file_paths_conflict(&left.artifact, &right.artifact) { + return Err(format!( + "[[runtime_files]] artifact paths {:?} and {:?} conflict as file/ancestor paths", + left.artifact, right.artifact + )); + } + if file_paths_conflict(&left.guest_path, &right.guest_path) { + return Err(format!( + "[[runtime_files]] guest paths {:?} and {:?} conflict as file/ancestor paths", + left.guest_path, right.guest_path + )); + } + } + } + // Dispatch on `kind` to decide whether `outputs` is the // library shape (`[outputs]` table with libs/headers/pkgconfig/files) // or the program shape (`[[outputs]]` array-of-tables with @@ -1350,6 +1562,16 @@ impl DepsManifest { .outputs .try_into() .map_err(|e| format!("parse [outputs] table: {e}"))?; + for (field, paths) in [ + ("libs", &outputs.libs), + ("headers", &outputs.headers), + ("pkgconfig", &outputs.pkgconfig), + ("files", &outputs.files), + ] { + for (idx, path) in paths.iter().enumerate() { + validate_output_path(path, &format!("[outputs].{field}[{idx}]"))?; + } + } (outputs, Vec::new()) } ManifestKind::Program => { @@ -1379,12 +1601,11 @@ impl DepsManifest { ); } for (idx, out) in program_outputs.iter().enumerate() { - if out.name.is_empty() { - return Err(format!("[[outputs]][{idx}].name must not be empty")); - } + validate_single_path_component(&out.name, &format!("[[outputs]][{idx}].name"))?; if out.wasm.is_empty() { return Err(format!("[[outputs]][{idx}].wasm must not be empty")); } + validate_output_path(&out.wasm, &format!("[[outputs]][{idx}].wasm"))?; } (Outputs::default(), program_outputs) } @@ -1411,6 +1632,56 @@ impl DepsManifest { } }; + if matches!(raw.kind, ManifestKind::Program) { + let mut output_artifacts: Vec<&str> = Vec::new(); + let mut output_mirrors: Vec = Vec::new(); + for (idx, out) in program_outputs.iter().enumerate() { + for (prior_idx, prior_artifact) in output_artifacts.iter().enumerate() { + if file_paths_conflict(prior_artifact, &out.wasm) { + return Err(format!( + "[[outputs]][{prior_idx}].wasm {:?} conflicts with [[outputs]][{idx}].wasm {:?}", + prior_artifact, out.wasm + )); + } + } + let mirror = program_output_dest_rel(&raw.name, program_outputs.len(), out); + for prior in &output_mirrors { + if file_paths_conflict(&prior.to_string_lossy(), &mirror.to_string_lossy()) { + return Err(format!( + "program outputs collide in the resolver mirror at {} and {}", + prior.display(), + mirror.display() + )); + } + } + output_artifacts.push(&out.wasm); + output_mirrors.push(mirror); + } + for (idx, runtime_file) in runtime_files.iter().enumerate() { + for out in &program_outputs { + if file_paths_conflict(&out.wasm, &runtime_file.artifact) { + return Err(format!( + "[[runtime_files]][{idx}].artifact {:?} conflicts with [[outputs]].wasm {:?}", + runtime_file.artifact, out.wasm + )); + } + } + let mirror = Path::new(&raw.name).join(&runtime_file.artifact); + for output_mirror in &output_mirrors { + if file_paths_conflict( + &output_mirror.to_string_lossy(), + &mirror.to_string_lossy(), + ) { + return Err(format!( + "[[runtime_files]][{idx}].artifact {:?} collides with a program output mirror at {}", + runtime_file.artifact, + output_mirror.display() + )); + } + } + } + } + // Default `arches` to `["wasm32"]` when omitted. Reject // duplicates so a manifest can't say `["wasm32", "wasm32"]`. let target_arches = if raw.arches.is_empty() { @@ -1436,6 +1707,7 @@ impl DepsManifest { build: raw.build, outputs, program_outputs, + runtime_files, compatibility: raw.compatibility, binary, host_tools, @@ -1611,6 +1883,47 @@ cache_key_sha = "111111111111111111111111111111111111111111111111111111111111111 assert_eq!(m.outputs.files, vec!["share/zlib/runtime.dat"]); } + #[test] + fn rejects_library_output_paths_outside_the_package_root() { + for (path, expected) in [ + ("/tmp/runtime.dat", "must be relative"), + ("share/../../runtime.dat", "must not contain '..'"), + ] { + let text = EXAMPLE.replace( + "headers = [\"include/zlib.h\"]", + &format!("headers = [\"include/zlib.h\"]\nfiles = [{path:?}]"), + ); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[outputs].files[0]"), "got: {err}"); + assert!(err.contains(expected), "got: {err}"); + } + } + + #[test] + fn rejects_program_output_paths_outside_the_package_root() { + let text = r#" +kind = "program" +name = "escape" +version = "1.0.0" +kernel_abi = 8 +depends_on = [] + +[source] +url = "https://example.test/escape.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" + +[license] +spdx = "MIT" + +[[outputs]] +name = "escape" +wasm = "../escape.wasm" +"#; + let err = DepsManifest::parse(text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[[outputs]][0].wasm"), "got: {err}"); + assert!(err.contains("must not contain '..'"), "got: {err}"); + } + #[test] fn build_script_override_is_repo_root_relative() { // Phase A-bis Task 2: an explicit `[build].script_path` is @@ -2129,6 +2442,181 @@ wasm = "vim.wasm" assert!(m.outputs.headers.is_empty()); assert!(m.outputs.pkgconfig.is_empty()); assert!(m.outputs.files.is_empty()); + assert!(m.runtime_files.is_empty()); + } + + fn program_with_runtime_file(artifact: &str, guest_path: &str, mode: Option) -> String { + format!( + "{PROGRAM_EXAMPLE}\n[[runtime_files]]\nartifact = {artifact:?}\nguest_path = {guest_path:?}\n{}", + mode.map(|value| format!("mode = {value}\n")).unwrap_or_default(), + ) + } + + #[test] + fn parses_program_runtime_file_and_stable_mirror_path() { + let text = program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", None); + let m = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap(); + assert_eq!( + m.runtime_files, + vec![RuntimeFile { + artifact: "icu.dat".into(), + guest_path: "/usr/lib/php/icu.dat".into(), + mode: 0o644, + }] + ); + assert_eq!( + m.runtime_file_dest_rel("icu.dat").unwrap(), + PathBuf::from("vim/icu.dat") + ); + // Runtime-file placement must not perturb the legacy single-output + // destination convention. + assert_eq!( + m.output_dest_rel("vim.wasm").unwrap(), + PathBuf::from("vim.wasm") + ); + + let nested = program_with_runtime_file("share/icu/icu.dat", "/usr/lib/php/icu.dat", None); + let nested = DepsManifest::parse(&nested, PathBuf::from("/x")).unwrap(); + assert_eq!( + nested.runtime_file_dest_rel("share/icu/icu.dat").unwrap(), + PathBuf::from("vim/share/icu/icu.dat") + ); + assert!(nested.runtime_file_dest_rel("icu.dat").is_err()); + } + + #[test] + fn rejects_unsafe_runtime_artifact_paths() { + for (artifact, expected) in [ + ("", "must not be empty"), + ("/icu.dat", "must be relative"), + ("../icu.dat", "must not contain '..'"), + ("./icu.dat", "must not contain '.'"), + ("share//icu.dat", "empty path components"), + ("share/icu.dat/", "empty path components"), + (r"share\icu.dat", "portable"), + ] { + let text = program_with_runtime_file(artifact, "/usr/lib/php/icu.dat", None); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[[runtime_files]][0].artifact"), "got: {err}"); + assert!(err.contains(expected), "got: {err}"); + } + } + + #[test] + fn rejects_unsafe_runtime_guest_paths() { + for (guest_path, expected) in [ + ("icu.dat", "non-root absolute"), + ("/", "non-root absolute"), + ("/usr/lib/php/", "trailing"), + ("/usr//php/icu.dat", "empty path components"), + ("/usr/./icu.dat", "must not contain"), + ("/usr/../icu.dat", "must not contain"), + (r"/usr/lib\icu.dat", "normalized absolute POSIX"), + ] { + let text = program_with_runtime_file("icu.dat", guest_path, None); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("[[runtime_files]][0].guest_path"), + "got: {err}" + ); + assert!(err.contains(expected), "got: {err}"); + } + } + + #[test] + fn rejects_runtime_file_on_non_program_and_invalid_mode() { + let library = format!( + "{EXAMPLE}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n" + ); + let err = DepsManifest::parse(&library, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("program"), "got: {err}"); + + let text = program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", Some(0o1000)); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("0777"), "got: {err}"); + + let unknown = format!( + "{PROGRAM_EXAMPLE}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\nmod = 384\n" + ); + let err = DepsManifest::parse(&unknown, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("unknown field") && err.contains("mod"), + "got: {err}" + ); + } + + #[test] + fn rejects_duplicate_and_colliding_runtime_files() { + let duplicate = format!( + "{}\n[[runtime_files]]\nartifact = \"icu.dat\"\nguest_path = \"/other.dat\"\n", + program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", None), + ); + let err = DepsManifest::parse(&duplicate, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("artifact") && err.contains("more than once"), + "got: {err}" + ); + + let duplicate_guest = format!( + "{}\n[[runtime_files]]\nartifact = \"other.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n", + program_with_runtime_file("icu.dat", "/usr/lib/php/icu.dat", None), + ); + let err = DepsManifest::parse(&duplicate_guest, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("guest_path") && err.contains("more than once"), + "got: {err}" + ); + + let artifact_collision = PROGRAM_EXAMPLE.replace( + "wasm = \"vim.wasm\"", + "wasm = \"vim.wasm\"\n\n[[runtime_files]]\nartifact = \"vim.wasm\"\nguest_path = \"/usr/lib/vim.wasm\"", + ); + let err = DepsManifest::parse(&artifact_collision, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("conflict"), "got: {err}"); + + let prefix = format!( + "{}\n[[runtime_files]]\nartifact = \"share/icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n", + program_with_runtime_file("share", "/usr/lib/php", None), + ); + let err = DepsManifest::parse(&prefix, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("ancestor"), "got: {err}"); + + let output_prefix = PROGRAM_EXAMPLE.replace( + "wasm = \"vim.wasm\"", + "wasm = \"share\"\n\n[[runtime_files]]\nartifact = \"share/icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"", + ); + let err = DepsManifest::parse(&output_prefix, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("conflicts with [[outputs]].wasm"), + "got: {err}" + ); + } + + #[test] + fn rejects_unsafe_names_and_duplicate_program_mirrors() { + for package_name in ["../escape", "/absolute", "a/b", r"a\b"] { + let text = + PROGRAM_EXAMPLE.replacen("name = \"vim\"", &format!("name = {package_name:?}"), 1); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("safe single path component"), "got: {err}"); + } + + let unsafe_output = PROGRAM_EXAMPLE.replace( + "[[outputs]]\nname = \"vim\"", + "[[outputs]]\nname = \"../vim\"", + ); + let err = DepsManifest::parse(&unsafe_output, PathBuf::from("/x")).unwrap_err(); + assert!(err.contains("[[outputs]][0].name"), "got: {err}"); + + let duplicate_mirror = PROGRAM_EXAMPLE.replace( + "[[outputs]]\nname = \"vim\"\nwasm = \"vim.wasm\"", + "[[outputs]]\nname = \"same\"\nwasm = \"a.wasm\"\n\n[[outputs]]\nname = \"same\"\nwasm = \"b.wasm\"", + ); + let err = DepsManifest::parse(&duplicate_mirror, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("collide") && err.contains("same.wasm"), + "got: {err}" + ); } #[test] From 96bd76fe96e3fa8ef8846fb6fff2dd11c9d69eba Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 04:55:20 -0400 Subject: [PATCH 09/12] host: preserve side-module process state across fork replay Problem: replay instantiated a fresh side module without restoring its live TLS-base global, and each module could receive a different C++ exception-tag identity. Pthread workers also have separate instances, tables, and tags, so allowing them to race process dlopen or inherit the process archive could corrupt replay. Design: keep process-owned pointer-width-correct exception/longjmp tags, archive and validate each module's exact positive TLS base, restore only that global over the fork-copied live TLS bytes, and arbitrate process dlopen against pthread fork with a host-private atomic record. Pthread dlopen now fails through dlerror; pthread fork returns ENOTSUP once the process archive is nonempty. Focused evidence: host declaration generation passed; dylink plus binary-resolver Vitest ran 33 tests passed with 16 compiler-dependent skips; the compiled fork/dlopen replay file ran 3/3, including mutated C++ thread_local state plus throw/catch and the real pthread boundary. Limits: the i64 cases exercise 64-bit pointer encoding only, not a memory64 end-to-end runtime. The full host, libc, ABI-classifier, and real browser aggregate gates have not yet run. The archive-size change is intended to remain host-private and transient; the ABI check is still required before landing. --- docs/architecture.md | 21 +- docs/fork-instrumentation.md | 20 +- docs/posix-status.md | 10 +- host/src/browser-kernel-worker-entry.ts | 1 + host/src/dylink.ts | 171 +++++++++++++- host/src/node-kernel-worker-entry.ts | 1 + host/src/worker-main.ts | 288 +++++++++++++++++++++-- host/src/worker-protocol.ts | 3 + host/test/centralized-test-helper.ts | 25 +- host/test/dylink.test.ts | 256 +++++++++++++++++++- host/test/fork-dlopen-replay-e2e.test.ts | 233 +++++++++++++++++- 11 files changed, 978 insertions(+), 51 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 67e369071..328dd391f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -263,7 +263,26 @@ five-export fallback, while ABI 18 and later require role claims and reject stale call-graph artifacts. ABI 19 combines that contract with live main's concurrent/nested pthread continuation-buffer repair. Dlopen replay records both the parent's memory base and exact table base, including null gaps -left by failed loads. The supported +left by failed loads. TLS-bearing side modules additionally record their live, +positive `__tls_base`. A child restores the pointer-width-correct mutable +global without calling `__wasm_init_tls`, because copied memory already holds +the parent's live TLS bytes and reinitialization would reset C++ unwinder state +and application `thread_local` values. C++ exceptions and longjmp use one +canonical pointer-width tag identity across the main image and all side +modules; a main-exported tag wins over the host-created fallback. + +The dlopen replay list and its atomic pthread-fork lock live in a transient, +host-private control record. The same host build writes and reads that record +during one process lifetime; guest code and persisted artifacts never +interpret it. Changing that record's size is therefore not a guest ABI change, +while the public ABI snapshot/classifier remains authoritative. + +Pthread workers have separate Wasm instances, tables, and exception tags, none +of which can be structured-cloned from the process worker. `dlopen()` from a +pthread therefore fails normally with `dlerror()`. If the process has loaded a +side module, `fork()` from a pthread returns `ENOTSUP`; an atomic process lock +excludes a racing main-worker dlopen across the pthread's archive check, +unwind, memory copy, and parent rewind. The supported direct-main-to-side boundary and the remaining opaque cross-side callback limitation are specified in [fork-instrumentation.md](fork-instrumentation.md#fork-from-a-dlopened-side-module). diff --git a/docs/fork-instrumentation.md b/docs/fork-instrumentation.md index cfb1137bb..674f0adfc 100644 --- a/docs/fork-instrumentation.md +++ b/docs/fork-instrumentation.md @@ -229,9 +229,23 @@ function pointers passed through main-module memory or the shared table cannot currently be attributed to their originating module; using such a pointer to create a side A -> side B -> fork path is unsupported and is not yet guaranteed to fail before control-flow corruption. A future module-activation protocol is -required to close that residual. Fork from a pthread into a dlopened side -module is also unsupported; pthread workers do not install the side-module -coordinator. +required to close that residual. + +Pthread workers do not own the process worker's side-module instances, table, +or exception-tag identities. `dlopen()` from a pthread consequently returns +NULL with a precise `dlerror()`. Once the process main worker has published a +dlopen archive entry, `fork()` from a pthread returns `ENOTSUP` without +creating a child. A host-private atomic lock prevents main-worker dlopen from +racing the pthread's archive check and is held through unwind, SYS_FORK/memory +copy, and parent rewind; the child clears its copied lock before replay. Fork +from a pthread remains supported while that process-wide archive is empty. + +For TLS-bearing side modules, each archive entry also preserves the live +positive `__tls_base`. Replay restores only that mutable global using the +process pointer type. It does not call `__wasm_init_tls`: the child memory copy +already contains live TLS, and reinitialization would overwrite C++ landing-pad +and application `thread_local` state. TLS-relative exports relocate from that +base, while `__tls_size` and `__tls_align` remain scalar constants. Every participating module still uses the fixed 16 KiB save-buffer limit described below. A dynamically allocated side buffer avoids overlap with the diff --git a/docs/posix-status.md b/docs/posix-status.md index 720684f3b..0c67d9d83 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -615,7 +615,10 @@ These PHP needs are well-handled by the current kernel: - Memory: anonymous mmap, munmap, brk - Multi-process: fork (kernel syscall), exec (host-initiated), waitpid (kernel syscall) - Networking: AF_INET TCP (connect, bind, listen, accept, send, recv), getaddrinfo -- Dynamic linking: dlopen, dlsym, dlclose, dlerror (Wasm dylink) +- Dynamic linking: dlopen, dlsym, dlclose, dlerror (Wasm dylink on the process + worker). Pthread workers cannot share the process's Wasm table/tag graph, so + pthread `dlopen` fails and pthread `fork` after a process dlopen returns + `ENOTSUP`. - POSIX timers: partial `SIGEV_SIGNAL`/`SIGEV_NONE` timer_create, timer_settime, timer_gettime, timer_delete. The PHP package imports a cooperative Wasm host hook because native `SIGEV_THREAD_ID` delivery is unavailable; that package @@ -639,7 +642,10 @@ These require features fundamentally unavailable in the Wasm architecture: - **Wasm FP exceptions (110 math tests):** WebAssembly has no floating-point exception flags (`fenv.h`). All `fe*` math tests fail. `long double` variants pass because they use software fp128. - **No pthread_cancel:** Wasm has no async cancellation mechanism or cancel-point assembly. `pthread_create` works; `pthread_cancel` does not. -- **No dlopen/TLS:** `tls_get_new-dtv_dso` requires loading a shared library with TLS at runtime. +- **No musl DTV expansion for arbitrary DSOs:** `tls_get_new-dtv_dso` requires + native-style per-thread dynamic TLS-vector growth. Kandelo can replay the + fixed TLS reservation of a Wasm side module across process `fork`, but that + does not implement musl's general pthread DTV contract. ### Linker Requirements for Signal Handlers diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index cea749224..0f4fba8b3 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -1530,6 +1530,7 @@ async function handleClone( programBytes: processInfo.programBytes, programModule: threadModule, memory, + processChannelOffset: processInfo.channelOffset, channelOffset: alloc.channelOffset, fnPtr, argPtr, diff --git a/host/src/dylink.ts b/host/src/dylink.ts index 65de254bc..26a99a4b0 100644 --- a/host/src/dylink.ts +++ b/host/src/dylink.ts @@ -284,6 +284,8 @@ export interface LoadedSharedLibrary { name: string; /** Fork save buffer for an instrumented side module importing env.fork. */ forkBufAddr?: number; + /** Thread-local-storage base captured from the parent instance. */ + tlsBase?: number; /** Whether this module can originate a coordinated env.fork unwind. */ forkCapable: boolean; /** Function/GOT.func imports used for conservative cross-side isolation. */ @@ -338,6 +340,11 @@ export interface DylinkReplayOptions { tableBase: number; /** Exact side-module save buffer copied from the fork parent. */ forkBufAddr?: number; + /** Exact mutable `__tls_base` value from the fork parent. The child memory + * already contains the parent's live TLS bytes, so replay restores only + * this instance-local global and deliberately does not call + * `__wasm_init_tls`, which would reset those bytes to the initial image. */ + tlsBase?: number; } /** @@ -368,6 +375,12 @@ export interface LoadSharedLibraryOptions { * is retained on this options object for subsequent loads. */ longjmpTag?: WebAssembly.Tag; + /** + * Process-owned C++ exception tag shared by every C++ side module. C++ + * exceptions crossing side-module calls require tag identity as well as a + * matching payload type, so this must not be allocated per dlopen. + */ + cppExceptionTag?: WebAssembly.Tag; /** Process pointer width, which also determines the __c_longjmp payload. */ ptrWidth?: 4 | 8; /** Immutable symbol names exported by the main module. */ @@ -401,6 +414,17 @@ export function createLongjmpTag(ptrWidth: 4 | 8): WebAssembly.Tag | undefined { : undefined; } +/** Create the process-owned C++ exception tag for the target pointer width. */ +export function createCppExceptionTag(ptrWidth: 4 | 8): WebAssembly.Tag | undefined { + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new TypeError(`invalid process pointer width ${String(ptrWidth)}`); + } + const Tag = tagConstructor(); + return Tag + ? new Tag({ parameters: [ptrWidth === 8 ? "i64" : "i32"] }) + : undefined; +} + /** Reject lookalike values before handing an exception-tag import to Wasm. */ export function requireLongjmpTag(value: unknown, context: string): WebAssembly.Tag { const Tag = tagConstructor(); @@ -413,6 +437,18 @@ export function requireLongjmpTag(value: unknown, context: string): WebAssembly. return value; } +/** Reject lookalike values before handing the C++ tag import to Wasm. */ +export function requireCppExceptionTag(value: unknown, context: string): WebAssembly.Tag { + const Tag = tagConstructor(); + if (!Tag) { + throw new Error(`${context}: this WebAssembly runtime does not support exception tags`); + } + if (!(value instanceof Tag)) { + throw new TypeError(`${context}: __cpp_exception must be an actual WebAssembly.Tag`); + } + return value; +} + function validateLongjmpConfiguration(options: LoadSharedLibraryOptions): void { const ptrWidth = options.ptrWidth ?? 4; if (ptrWidth !== 4 && ptrWidth !== 8) { @@ -432,6 +468,17 @@ function resolveLongjmpTag(options: LoadSharedLibraryOptions): WebAssembly.Tag { return options.longjmpTag; } +function resolveCppExceptionTag(options: LoadSharedLibraryOptions): WebAssembly.Tag { + validateLongjmpConfiguration(options); + if (options.cppExceptionTag !== undefined) { + return requireCppExceptionTag(options.cppExceptionTag, "dynamic linker"); + } + const ptrWidth = options.ptrWidth ?? 4; + const tag = createCppExceptionTag(ptrWidth); + options.cppExceptionTag = requireCppExceptionTag(tag, "dynamic linker"); + return options.cppExceptionTag; +} + const SIDE_DYNAMIC_LOOKUP_IMPORTS = new Set([ "__wasm_dlopen", "__wasm_dlsym", @@ -574,9 +621,8 @@ function instantiateSharedLibrary( && imp.name === "__cpp_exception" && (imp.kind as string) === "tag" ); - const Tag = tagConstructor(); - const cppExceptionTag = importsCppExceptionTag && Tag - ? new Tag({ parameters: ["i32"] }) + const cppExceptionTag = importsCppExceptionTag + ? resolveCppExceptionTag(options) : undefined; if (presentForkExports.length > 0 && !hasCompleteForkInstrumentation) { @@ -830,9 +876,6 @@ function instantiateSharedLibrary( case "__table_base": return tableBaseGlobal; case "__stack_pointer": return options.stackPointer; case "__c_longjmp": return longjmpTag; - // C++ exceptions are currently confined to one side module. A - // cross-module exception contract would instead need one - // process-owned tag shared with every participating module. case "__cpp_exception": return cppExceptionTag; case "fork": if (importsFork) return sideModuleForkImport; @@ -874,6 +917,103 @@ function instantiateSharedLibrary( // Instantiate synchronously after validating the side-module fork contract. instance = new WebAssembly.Instance(module, imports); + // A threaded wasm-ld side module initializes its mutable __tls_base from + // __memory_base in the start function. Fork-child memory already carries + // the parent's `__wasm_init_memory_flag == 2`, so the fresh child instance + // skips that initialization and otherwise leaves __tls_base at zero. + // Capture the live parent value and restore it during replay without + // calling __wasm_init_tls: the latter would overwrite copied, live TLS + // state (including the C++ unwinder's landing-pad context) with .tdata. + const tlsSizeExport = instance.exports.__tls_size; + const tlsSize = tlsSizeExport instanceof WebAssembly.Global + ? Number(tlsSizeExport.value) + : 0; + let tlsBase: number | undefined; + if (metadata.tlsExports.size > 0 && !(tlsSizeExport instanceof WebAssembly.Global)) { + throw new Error(`${name}: TLS exports require an exported __tls_size global`); + } + if (!Number.isSafeInteger(tlsSize) || tlsSize < 0) { + throw new Error(`${name}: invalid side-module TLS size ${String(tlsSize)}`); + } + if (tlsSize > 0) { + const tlsBaseExport = instance.exports.__tls_base; + const tlsAlignExport = instance.exports.__tls_align; + if (!(tlsBaseExport instanceof WebAssembly.Global)) { + throw new Error( + `${name}: TLS-bearing side modules must export mutable __tls_base for fork replay`, + ); + } + if (!(tlsAlignExport instanceof WebAssembly.Global)) { + throw new Error(`${name}: TLS-bearing side modules must export __tls_align`); + } + const tlsAlign = Number(tlsAlignExport.value); + if ( + !Number.isSafeInteger(tlsAlign) + || tlsAlign <= 0 + || (tlsAlign & (tlsAlign - 1)) !== 0 + ) { + throw new Error(`${name}: invalid side-module TLS alignment ${String(tlsAlign)}`); + } + + const initialRawTlsBase = tlsBaseExport.value; + const expectedTlsBaseType = (options.ptrWidth ?? 4) === 8 ? "bigint" : "number"; + if (typeof initialRawTlsBase !== expectedTlsBaseType) { + throw new Error( + `${name}: __tls_base type does not match the ${(options.ptrWidth ?? 4) * 8}-bit process pointer width`, + ); + } + try { + // A self-assignment is the only portable reflection available for + // distinguishing a mutable WebAssembly.Global from an immutable one. + tlsBaseExport.value = initialRawTlsBase; + } catch { + throw new Error(`${name}: exported __tls_base must be mutable for fork replay`); + } + if (replay) { + if (!Number.isSafeInteger(replay.tlsBase) || replay.tlsBase! <= 0) { + throw new Error(`${name}: fork replay is missing a valid side-module TLS base`); + } + try { + tlsBaseExport.value = typeof initialRawTlsBase === "bigint" + ? BigInt(replay.tlsBase!) + : replay.tlsBase!; + } catch { + throw new Error(`${name}: exported __tls_base must be mutable for fork replay`); + } + } + tlsBase = Number(tlsBaseExport.value); + // Address zero is reserved as the archive's explicit "no TLS" sentinel. + // A real TLS allocation cannot live there: the process memory allocator + // always returns a positive address and the null page must stay invalid. + if (!Number.isSafeInteger(tlsBase) || tlsBase <= 0) { + throw new Error(`${name}: invalid side-module TLS base ${String(tlsBase)}`); + } + if (tlsBase % tlsAlign !== 0) { + throw new Error( + `${name}: side-module TLS base 0x${tlsBase.toString(16)} is not aligned to ${tlsAlign}`, + ); + } + const tlsEnd = tlsBase + tlsSize; + const moduleMemoryEnd = memoryBase + metadata.memorySize; + if ( + !Number.isSafeInteger(tlsEnd) + || tlsBase < memoryBase + || tlsEnd > moduleMemoryEnd + ) { + throw new Error( + `${name}: TLS range 0x${tlsBase.toString(16)}..0x${tlsEnd.toString(16)} ` + + `escapes module reservation 0x${memoryBase.toString(16)}..0x${moduleMemoryEnd.toString(16)}`, + ); + } + if (tlsEnd > options.memory.buffer.byteLength) { + throw new Error( + `${name}: TLS range 0x${tlsBase.toString(16)}..0x${tlsEnd.toString(16)} exceeds memory`, + ); + } + } else if (replay?.tlsBase !== undefined) { + throw new Error(`${name}: fork replay supplied TLS state for a module without TLS`); + } + // Relocate exports: data address globals need memoryBase added const relocatedExports: Record = {}; for (const [exportName, exportValue] of Object.entries(instance.exports)) { @@ -882,9 +1022,23 @@ function instantiateSharedLibrary( (exportValue as any).value = (exportValue as any).value; relocatedExports[exportName] = exportValue; } catch { + // These are scalar ABI facts, not data addresses. + if (exportName === "__tls_size" || exportName === "__tls_align") { + relocatedExports[exportName] = exportValue; + continue; + } + const rawValue = exportValue.value; + const relocationBase = metadata.tlsExports.has(exportName) + ? tlsBase + : memoryBase; + if (relocationBase === undefined) { + throw new Error(`${name}: TLS export ${exportName} has no live TLS base`); + } relocatedExports[exportName] = new WebAssembly.Global( - { value: "i32", mutable: false }, - (exportValue as WebAssembly.Global).value + memoryBase, + { value: typeof rawValue === "bigint" ? "i64" : "i32", mutable: false }, + typeof rawValue === "bigint" + ? rawValue + BigInt(relocationBase) + : rawValue + relocationBase, ); } } else { @@ -945,6 +1099,7 @@ function instantiateSharedLibrary( metadata, name, forkBufAddr: sideForkBufAddr || undefined, + tlsBase, forkCapable: importsFork, functionImports, functionExports, diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index f91aa5881..c5515c60a 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -1370,6 +1370,7 @@ async function handleClone( programBytes: processInfo.programBytes, programModule: threadModule, memory, + processChannelOffset: processInfo.channelOffset, channelOffset: alloc.channelOffset, fnPtr, argPtr, diff --git a/host/src/worker-main.ts b/host/src/worker-main.ts index c38e94b49..4d63ce182 100644 --- a/host/src/worker-main.ts +++ b/host/src/worker-main.ts @@ -11,11 +11,13 @@ import type { WorkerToHostMessage, } from "./worker-protocol"; import { + createCppExceptionTag, createLongjmpTag, DynamicLinker, FORK_CAP_DYLINK_MAIN, forkInstrumentRoleAvailable, readForkInstrumentCapabilityClaim, + requireCppExceptionTag, requireLongjmpTag, type LoadedSharedLibrary, type SideModuleForkState, @@ -198,6 +200,40 @@ export interface DlopenSupport { beginSideModuleForkRewind: () => void; /** Reject a leaked active side-module identity on a normal main return. */ assertNoActiveSideModuleFork: () => void; + /** Clear a fork parent's copied archive lock in the child's private memory. */ + resetForkChildLock: () => void; +} + +/** + * Thread workers instantiate a separate Wasm module/table/tag graph, so they + * cannot safely load or invoke process side modules. Keep dlopen's ordinary C + * failure contract (NULL plus dlerror text) instead of letting a generic + * unresolved-import stub trap the pthread. + */ +function buildUnsupportedThreadDlopenImports( + memory: WebAssembly.Memory, +): Record { + const message = new TextEncoder().encode( + "dlopen is unsupported from pthread workers; load side modules on the process main worker", + ); + const n = (value: number | bigint): number => + typeof value === "bigint" ? Number(value) : value; + return { + __wasm_dlopen: (): number => 0, + __wasm_dlsym: (): number => 0, + __wasm_dlclose: (): number => -1, + __wasm_dlerror: (bufPtr: number | bigint, bufMax: number | bigint): number => { + const ptr = n(bufPtr); + const max = n(bufMax); + if (!Number.isSafeInteger(ptr) || !Number.isSafeInteger(max) || ptr < 0 || max <= 0) { + return 0; + } + const len = Math.min(message.length, max, memory.buffer.byteLength - ptr); + if (len <= 0) return 0; + new Uint8Array(memory.buffer, ptr, len).set(message.subarray(0, len)); + return len; + }, + }; } /** @@ -208,18 +244,22 @@ export interface DlopenSupport { * don't use dlopen. * * Each successful dlopen is also persisted into a per-process archive - * (linked list in linear memory, head pointer at a fixed slot below - * forkBufAddr) so the fork child can replay them via `replayDlopens`. + * (linked list in linear memory, with control slots below the main process + * channel's fork buffer) so the fork child can replay them via + * `replayDlopens`. The archive anchor is deliberately independent of the + * call-site rewind buffer: a fork issued by a pthread rewinds from that + * thread's buffer but still inherits the one process-wide dlopen archive. */ function buildDlopenImports( memory: WebAssembly.Memory, channelOffset: number, - forkBufAddr: number, + archiveControlAddr: number, getTable: () => WebAssembly.Table | undefined, getStackPointer: () => WebAssembly.Global | undefined, getInstance: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8, longjmpTag: WebAssembly.Tag | undefined, + cppExceptionTag: WebAssembly.Tag | undefined, mainHasDylinkForkRole: boolean, ): DlopenSupport { let linker: DynamicLinker | null = null; @@ -233,8 +273,12 @@ function buildDlopenImports( const sideForkOffset = ptrWidth === 8 ? DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64 : DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM32; - const headSlot = forkBufAddr - headOffset; - const activeSideForkSlot = forkBufAddr - sideForkOffset; + const lockOffset = ptrWidth === 8 + ? DLOPEN_LOCK_OFFSET_WASM64 + : DLOPEN_LOCK_OFFSET_WASM32; + const headSlot = archiveControlAddr - headOffset; + const activeSideForkSlot = archiveControlAddr - sideForkOffset; + const archiveLock = new Int32Array(memory.buffer, archiveControlAddr - lockOffset, 1); const entrySize = ptrWidth === 8 ? DLOPEN_ENTRY_SIZE_WASM64 : DLOPEN_ENTRY_SIZE_WASM32; const readPtr = (view: DataView, addr: number): number => @@ -243,7 +287,44 @@ function buildDlopenImports( if (ptrWidth === 8) view.setBigUint64(addr, BigInt(value), true); else view.setUint32(addr, value, true); }; + const readArchiveHead = (): number => ptrWidth === 8 + ? Number(Atomics.load(new BigUint64Array(memory.buffer, headSlot, 1), 0)) + : Atomics.load(new Uint32Array(memory.buffer, headSlot, 1), 0); + const writeArchiveHead = (value: number): void => { + if (ptrWidth === 8) { + Atomics.store(new BigUint64Array(memory.buffer, headSlot, 1), 0, BigInt(value)); + } else { + Atomics.store(new Uint32Array(memory.buffer, headSlot, 1), 0, value); + } + }; const linkerAllocations = new Map(); + let hostDlopenError: string | null = null; + let mainDlopenDepth = 0; + const acquireMainDlopenLock = (): boolean => { + if (mainDlopenDepth > 0) { + mainDlopenDepth++; + return true; + } + const owner = Atomics.compareExchange(archiveLock, 0, 0, 1); + if (owner !== 0) { + hostDlopenError = owner === 2 + ? "dlopen is temporarily unavailable while a pthread is forking" + : "dlopen is temporarily unavailable while another dlopen operation owns the process lock"; + return false; + } + mainDlopenDepth = 1; + return true; + }; + const releaseMainDlopenLock = (): void => { + if (mainDlopenDepth <= 0) { + throw new Error("dlopen process lock released without ownership"); + } + mainDlopenDepth--; + if (mainDlopenDepth === 0) { + Atomics.store(archiveLock, 0, 0); + Atomics.notify(archiveLock, 0); + } + }; // The kernel mmap allocator. Shared with the linker, but also used // directly by persistArchiveEntry to obtain blocks for the archive. @@ -319,6 +400,7 @@ function buildDlopenImports( const RESERVED = new Set([ "memory", "__indirect_function_table", "__memory_base", "__table_base", "__stack_pointer", "__c_longjmp", + "__cpp_exception", ]); const globalSymbols = new Map(); const inst = getInstance(); @@ -332,6 +414,19 @@ function buildDlopenImports( } const mainModuleSymbols = new Set(globalSymbols.keys()); + // A main-defined/exported tag is the process ABI authority. If the main + // image instead imports and re-exports the host tag, the identity is the + // same; if it has no export, retain the process-owned fallback created + // before main instantiation. Every side module must receive this one + // canonical identity for cross-module exception propagation. + const exportedLongjmpTag = inst?.exports.__c_longjmp; + const canonicalLongjmpTag = exportedLongjmpTag === undefined + ? longjmpTag + : requireLongjmpTag(exportedLongjmpTag, "main module export"); + const exportedCppExceptionTag = inst?.exports.__cpp_exception; + const canonicalCppExceptionTag = exportedCppExceptionTag === undefined + ? cppExceptionTag + : requireCppExceptionTag(exportedCppExceptionTag, "main module export"); const mainFork = inst?.exports.fork; const mainForkState = inst?.exports.wpk_fork_state; const sideModuleFork = mainHasDylinkForkRole @@ -386,7 +481,8 @@ function buildDlopenImports( globalSymbols, got: new Map(), loadedLibraries, - longjmpTag, + longjmpTag: canonicalLongjmpTag, + cppExceptionTag: canonicalCppExceptionTag, ptrWidth, mainModuleSymbols, sideModuleFork, @@ -409,6 +505,7 @@ function buildDlopenImports( memoryBase: number, tableBase: number, sideForkBufAddr: number, + tlsBase: number, ): void => { const nameBytes = encoder.encode(name); const nameLen = nameBytes.length; @@ -429,6 +526,7 @@ function buildDlopenImports( view.setBigUint64(entry + 40, BigInt(memoryBase), true); view.setBigUint64(entry + 48, BigInt(tableBase), true); view.setBigUint64(entry + 56, BigInt(sideForkBufAddr), true); + view.setBigUint64(entry + 64, BigInt(tlsBase), true); } else { view.setUint32(entry + 0, 0, true); view.setUint32(entry + 4, namePtr, true); @@ -438,15 +536,19 @@ function buildDlopenImports( view.setUint32(entry + 20, memoryBase, true); view.setUint32(entry + 24, tableBase, true); view.setUint32(entry + 28, sideForkBufAddr, true); + view.setUint32(entry + 32, tlsBase, true); } new Uint8Array(memory.buffer, namePtr, nameLen).set(nameBytes); new Uint8Array(memory.buffer, bytesPtr, bytes.length).set(bytes); // Append to tail (preserves insertion order). - const head = readPtr(view, headSlot); + const head = readArchiveHead(); if (head === 0) { - writePtr(view, headSlot, entry); + // Publish only after the complete entry and payload are visible. A + // pthread fork acquire-loads this word before deciding whether it can + // safely fork without access to the process side-module graph. + writeArchiveHead(entry); return; } let cursor = head; @@ -462,7 +564,7 @@ function buildDlopenImports( const replayDlopens = (): void => { const view = new DataView(memory.buffer); - let cursor = readPtr(view, headSlot); + let cursor = readArchiveHead(); if (cursor === 0) return; // Force linker creation: it's lazily built on the first C-side @@ -479,6 +581,7 @@ function buildDlopenImports( let memoryBase: number; let tableBase: number; let sideForkBufAddr: number; + let tlsBase: number; if (ptrWidth === 8) { next = Number(view.getBigUint64(cursor + 0, true)); namePtr = Number(view.getBigUint64(cursor + 8, true)); @@ -488,6 +591,7 @@ function buildDlopenImports( memoryBase = Number(view.getBigUint64(cursor + 40, true)); tableBase = Number(view.getBigUint64(cursor + 48, true)); sideForkBufAddr = Number(view.getBigUint64(cursor + 56, true)); + tlsBase = Number(view.getBigUint64(cursor + 64, true)); } else { next = view.getUint32(cursor + 0, true); namePtr = view.getUint32(cursor + 4, true); @@ -497,6 +601,7 @@ function buildDlopenImports( memoryBase = view.getUint32(cursor + 20, true); tableBase = view.getUint32(cursor + 24, true); sideForkBufAddr = view.getUint32(cursor + 28, true); + tlsBase = view.getUint32(cursor + 32, true); } // Copy name + bytes out of shared memory before passing to @@ -513,6 +618,7 @@ function buildDlopenImports( memoryBase, tableBase, forkBufAddr: sideForkBufAddr || undefined, + tlsBase: tlsBase === 0 ? undefined : tlsBase, }); if (handle === 0) { throw new Error(`dlopen(${name}): ${lk.dlerror() || "unknown"}`); @@ -523,6 +629,12 @@ function buildDlopenImports( throw new Error(`${name}: fork replay restored a mismatched save buffer`); } } + if (tlsBase !== 0) { + const loaded = loadedLibraries.get(name); + if (!loaded || loaded.tlsBase !== tlsBase) { + throw new Error(`${name}: fork replay restored a mismatched TLS base`); + } + } cursor = next; } @@ -598,9 +710,17 @@ function buildDlopenImports( } }; + const resetForkChildLock = (): void => { + Atomics.store(archiveLock, 0, 0); + Atomics.notify(archiveLock, 0); + }; + const imports: Record = { __wasm_dlopen: (bytesPtr: number, bytesLen: number, namePtr: number, nameLen: number): number => { + if (!acquireMainDlopenLock()) return 0; + hostDlopenError = null; + try { const bytes = new Uint8Array(memory.buffer, bytesPtr, bytesLen); // Copy bytes since memory.buffer may detach during Wasm instantiation const bytesCopy = new Uint8Array(bytes); @@ -627,9 +747,13 @@ function buildDlopenImports( loaded.memoryBase, loaded.tableBase, loaded.forkBufAddr ?? 0, + loaded.tlsBase ?? 0, ); } return handle; + } finally { + releaseMainDlopenLock(); + } }, __wasm_dlsym: (handle: number, namePtr: number, nameLen: number): number => { @@ -647,7 +771,8 @@ function buildDlopenImports( }, __wasm_dlerror: (bufPtr: number, bufMax: number): number => { - const err = getLinker().dlerror(); + const err = hostDlopenError ?? getLinker().dlerror(); + hostDlopenError = null; if (!err) return 0; const encoded = encoder.encode(err); const len = Math.min(encoded.length, bufMax); @@ -662,6 +787,7 @@ function buildDlopenImports( completeSideModuleForkUnwind, beginSideModuleForkRewind, assertNoActiveSideModuleFork, + resetForkChildLock, }; } @@ -677,6 +803,7 @@ function buildImportObject( getInstance?: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8 = 4, longjmpTag?: WebAssembly.Tag, + cppExceptionTag?: WebAssembly.Tag, postVmInterruptTimer?: ( timedOutPtr: number, vmInterruptPtr: number, @@ -711,6 +838,13 @@ function buildImportObject( ) as unknown as WebAssembly.ExportValue; } + if (moduleImports.some(i => i.module === "env" && i.name === "__cpp_exception" && (i.kind as string) === "tag")) { + envImports.__cpp_exception = requireCppExceptionTag( + cppExceptionTag, + "process module", + ) as unknown as WebAssembly.ExportValue; + } + // Add dlopen imports if provided if (dlopenImports) { Object.assign(envImports, dlopenImports); @@ -939,16 +1073,32 @@ function buildImportObject( /** Size of the fork save buffer used by wpk_fork_* instrumentation */ const FORK_BUF_SIZE = FORK_SAVE_BUFFER_SIZE; -// Slot below forkBufAddr that stores the head pointer of the dlopen -// archive linked list. Fork's memcpy carries the parent's archive into -// the child intact; the child walks it to replay each dlopen before -// wpk_fork rewind. +// Host-private control slots below the process main channel's fork buffer. +// Fork's memcpy carries the parent's dlopen archive into the child intact; +// the child walks it to replay each module before wpk_fork rewind. These are +// intentionally not relative to a pthread's rewind buffer. const DLOPEN_HEAD_OFFSET_WASM32 = 12; const DLOPEN_HEAD_OFFSET_WASM64 = 24; const DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM32 = 16; const DLOPEN_ACTIVE_SIDE_FORK_OFFSET_WASM64 = 32; -const DLOPEN_ENTRY_SIZE_WASM32 = 32; -const DLOPEN_ENTRY_SIZE_WASM64 = 64; +// Atomic host-private arbitration between process-main dlopen and pthread +// fork. The pthread holds value 2 from its pre-unwind archive check through +// memory-copy/SYS_FORK and parent rewind; process dlopen holds value 1 until +// the complete archive entry is published. A fork child clears its copied +// value before replay because its memory is already independent. +const DLOPEN_LOCK_OFFSET_WASM32 = 20; +const DLOPEN_LOCK_OFFSET_WASM64 = 40; +// Each entry also carries the side module's instance-local TLS base. Fork +// copies the TLS bytes in memory, but a new replay instance's mutable global +// must be restored explicitly. Zero is the explicit no-TLS sentinel; TLS +// allocations are required to have a positive base. +// +// This is a host-private, transient replay record: the same host build writes +// and reads it around one fork, and neither guest code nor persisted package +// artifacts interpret the layout. Enlarging it therefore does not alter the +// guest/kernel ABI. The ABI classifier/check still guards the public contract. +const DLOPEN_ENTRY_SIZE_WASM32 = 40; +const DLOPEN_ENTRY_SIZE_WASM64 = 72; const WPK_FORK_EXPORTS = [ "wpk_fork_unwind_begin", @@ -1119,6 +1269,7 @@ export async function centralizedWorkerMain( // --- SDK module path (existing) --- const processLongjmpTag = createLongjmpTag(ptrWidth); + const processCppExceptionTag = createCppExceptionTag(ptrWidth); let kernelExitStatus: number | null = null; const kernelImports = buildKernelImports( memory, @@ -1140,6 +1291,7 @@ export async function centralizedWorkerMain( // Fork state — captured by kernel_fork closure let forkResult = 0; const forkBufAddr = initData.forkBufAddr ?? channelOffset - FORK_BUF_SIZE; + const dlopenArchiveControlAddr = channelOffset - FORK_BUF_SIZE; if (hasForkInstrumentation) { // Override kernel_fork with fork-instrumentation-aware version. @@ -1170,16 +1322,17 @@ export async function centralizedWorkerMain( const dlopenSupport = buildDlopenImports( memory, channelOffset, - forkBufAddr, + dlopenArchiveControlAddr, () => processInstance?.exports.__indirect_function_table as WebAssembly.Table | undefined, () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + processCppExceptionTag, hasDylinkForkRole, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -1191,6 +1344,9 @@ export async function centralizedWorkerMain( }); const instance = await WebAssembly.instantiate(module, importObject); processInstance = instance; + if (initData.isForkChild) { + dlopenSupport.resetForkChildLock(); + } verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); // For the fork-parent case (initial launch, not a fork child), install @@ -1336,16 +1492,17 @@ export async function centralizedWorkerMain( const dlopenSupport = buildDlopenImports( memory, channelOffset, - forkBufAddr, + dlopenArchiveControlAddr, () => processInstance?.exports.__indirect_function_table as WebAssembly.Table | undefined, () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + processCppExceptionTag, false, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, processCppExceptionTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -1993,11 +2150,30 @@ export async function centralizedThreadWorkerMain( port: MessagePort, initData: CentralizedThreadInitMessage, ): Promise { - const { memory, channelOffset, pid, tid, fnPtr, argPtr, stackPtr, tlsPtr, ctidPtr } = initData; + const { + memory, + processChannelOffset, + channelOffset, + pid, + tid, + fnPtr, + argPtr, + stackPtr, + tlsPtr, + ctidPtr, + } = initData; const tlsOffset = initData.tlsOffset ?? initData.tlsAllocAddr; const ptrWidth = initData.ptrWidth ?? 4; let threadInstance: WebAssembly.Instance | undefined; + let processDlopenLock: Int32Array | undefined; + let pthreadForkLockHeld = false; + const releasePthreadForkLock = (): void => { + if (!pthreadForkLockHeld || !processDlopenLock) return; + Atomics.store(processDlopenLock, 0, 0); + Atomics.notify(processDlopenLock, 0); + pthreadForkLockHeld = false; + }; try { // Strip the start section AND neuter the constructor function body to prevent @@ -2014,6 +2190,37 @@ export async function centralizedThreadWorkerMain( const moduleExports = WebAssembly.Module.exports(module); const hasForkInstrumentation = hasCompleteForkInstrumentation(moduleExports, pid); const forkBufAddr = channelOffset - FORK_BUF_SIZE; + const processArchiveHeadOffset = ptrWidth === 8 + ? DLOPEN_HEAD_OFFSET_WASM64 + : DLOPEN_HEAD_OFFSET_WASM32; + const processArchiveHeadAddr = processChannelOffset + - FORK_BUF_SIZE + - processArchiveHeadOffset; + const processArchiveLockOffset = ptrWidth === 8 + ? DLOPEN_LOCK_OFFSET_WASM64 + : DLOPEN_LOCK_OFFSET_WASM32; + const processArchiveLockAddr = processChannelOffset + - FORK_BUF_SIZE + - processArchiveLockOffset; + if ( + !Number.isSafeInteger(processArchiveHeadAddr) + || processArchiveHeadAddr <= 0 + || processArchiveHeadAddr + ptrWidth > memory.buffer.byteLength + || !Number.isSafeInteger(processArchiveLockAddr) + || processArchiveLockAddr <= 0 + || processArchiveLockAddr + 4 > memory.buffer.byteLength + ) { + throw new Error( + `pid=${pid} tid=${tid}: invalid process dlopen archive anchor ` + + `${String(processArchiveHeadAddr)}`, + ); + } + processDlopenLock = new Int32Array(memory.buffer, processArchiveLockAddr, 1); + const processHasDlopenArchive = (): boolean => { + return ptrWidth === 8 + ? Atomics.load(new BigUint64Array(memory.buffer, processArchiveHeadAddr, 1), 0) !== 0n + : Atomics.load(new Uint32Array(memory.buffer, processArchiveHeadAddr, 1), 0) !== 0; + }; let forkResult = 0; let kernelThreadExitStatus: number | null = null; @@ -2034,14 +2241,35 @@ export async function centralizedThreadWorkerMain( const state = getState(); if (state === 2) { (threadInstance.exports.wpk_fork_rewind_end as () => void)(); + releasePthreadForkLock(); return forkResult; } - (threadInstance.exports.wpk_fork_unwind_begin as (addr: number) => void)(forkBufAddr); + // Side modules live in the process main worker's module/table/tag + // graph. A pthread worker cannot replay that graph into its own + // instance, so fork must fail before unwind once the process has ever + // loaded a side module. The head is read live from shared memory so a + // dlopen after pthread creation is still observed. + if (Atomics.compareExchange(processDlopenLock!, 0, 0, 2) !== 0) { + return -95; // ENOTSUP: process-main dlopen is active + } + pthreadForkLockHeld = true; + if (processHasDlopenArchive()) { + releasePthreadForkLock(); + return -95; // ENOTSUP: pthreads cannot replay process side modules + } + + try { + (threadInstance.exports.wpk_fork_unwind_begin as (addr: number) => void)(forkBufAddr); + } catch (error) { + releasePthreadForkLock(); + throw error; + } return 0; }; } else { kernelImports.kernel_fork = (): number => { + if (processHasDlopenArchive()) return -95; // ENOTSUP throw new Error( `pid=${pid} tid=${tid}: kernel_fork reached without complete ` + "wasm-fork-instrument exports. Rebuild the program with " + @@ -2050,8 +2278,10 @@ export async function centralizedThreadWorkerMain( }; } const threadLongjmpTag = createLongjmpTag(ptrWidth); - const importObject = buildImportObject(module, memory, kernelImports, channelOffset, undefined, - () => threadInstance, ptrWidth, threadLongjmpTag, + const threadCppExceptionTag = createCppExceptionTag(ptrWidth); + const threadDlopenImports = buildUnsupportedThreadDlopenImports(memory); + const importObject = buildImportObject(module, memory, kernelImports, channelOffset, threadDlopenImports, + () => threadInstance, ptrWidth, threadLongjmpTag, threadCppExceptionTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -2134,8 +2364,17 @@ export async function centralizedThreadWorkerMain( const forkState = getState(); if (forkState === 1) { unwindEnd(); + // Close the race where the process main worker dlopens after this + // pthread began unwinding but before it completed. Rewind locally + // with ENOTSUP and do not create a child. + if (processHasDlopenArchive()) { + forkResult = -95; + needsRewind = true; + continue; + } const childPid = sendForkSyscall(memory, channelOffset); if (childPid < 0) { + releasePthreadForkLock(); throw new Error(`Fork failed: errno=${-childPid}`); } forkResult = childPid; @@ -2192,6 +2431,7 @@ export async function centralizedThreadWorkerMain( tid, } satisfies WorkerToHostMessage); } catch (err) { + releasePthreadForkLock(); const message = err instanceof Error ? `${err.message}\n${err.stack ?? ""}` : String(err); diff --git a/host/src/worker-protocol.ts b/host/src/worker-protocol.ts index c837a91ab..2b80c6359 100644 --- a/host/src/worker-protocol.ts +++ b/host/src/worker-protocol.ts @@ -70,6 +70,9 @@ export interface CentralizedThreadInitMessage { programBytes: ArrayBuffer; programModule?: WebAssembly.Module; memory: WebAssembly.Memory; + /** Main process channel offset. The thread reads the process-wide dlopen + * archive head relative to this live shared-memory anchor before fork. */ + processChannelOffset: number; channelOffset: number; fnPtr: number; argPtr: number; diff --git a/host/test/centralized-test-helper.ts b/host/test/centralized-test-helper.ts index c0c808675..6fe9a32d3 100644 --- a/host/test/centralized-test-helper.ts +++ b/host/test/centralized-test-helper.ts @@ -140,6 +140,9 @@ export interface RunProgramOptions { captureForkCount?: boolean; /** Use the canonical rootfs image in worker-thread mode. Defaults to true. */ useDefaultRootfs?: boolean; + /** Exact VFS image for tests that stage package runtime files. Overrides + * `useDefaultRootfs`; omitted means the canonical image. */ + rootfsImage?: "default" | ArrayBuffer | Uint8Array; } export interface RunProgramResult { @@ -211,7 +214,8 @@ async function runInWorkerThread(options: RunProgramOptions): Promise { stdout += new TextDecoder().decode(data); @@ -316,6 +320,7 @@ async function runOnMainThread(options: RunProgramOptions): Promise(); const processPtrWidths = new Map(); const forkReplayContexts = new Map(); + let mainThreadForkCount: bigint | undefined; const pid = 100; @@ -525,6 +530,10 @@ async function runOnMainThread(options: RunProgramOptions): Promise { if (exitPid === pid) { + if (options.captureForkCount) { + mainThreadForkCount = kernelWorker.getForkCount(exitPid); + } kernelWorker.unregisterProcess(exitPid); processProgramBytes.delete(exitPid); processLayouts.delete(exitPid); @@ -694,5 +708,12 @@ async function runOnMainThread(options: RunProgramOptions): Promise { + const bytes = [...new TextEncoder().encode(exportName)]; + if (bytes.length >= 128) throw new Error("test TLS export name is too long"); + return [bytes.length, ...bytes, 1]; // WASM_DYLINK_FLAG_TLS + }); + const exportInfo = tlsExports.length > 0 + ? [3, 1 + exportInfoBody.length, tlsExports.length, ...exportInfoBody] + : []; + const payload = new Uint8Array(1 + dylinkName.length + 6 + exportInfo.length); payload[0] = dylinkName.length; payload.set(dylinkName, 1); payload.set([1, 4, memorySize, 0, tableSize, 0], 1 + dylinkName.length); + payload.set(exportInfo, 1 + dylinkName.length + 6); const section = new Uint8Array(2 + payload.length); section[0] = 0; section[1] = payload.length; @@ -202,6 +214,67 @@ describe.skipIf(typeof WebAssembly.Tag !== "function")("longjmp tag identity", ( }); }); +describe.skipIf(typeof WebAssembly.Tag !== "function")("C++ exception tag identity", () => { + const cases = [ + { ptrWidth: 4 as const, wasmType: "i32", value: 42 }, + { ptrWidth: 8 as const, wasmType: "i64", value: 42n }, + ]; + + it.each(cases)( + "shares one process-owned $wasmType tag across throwing and catching side modules", + ({ ptrWidth, wasmType, value }) => { + const throwerBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $cpp (import "env" "__cpp_exception") (param ${wasmType})) + (func (export "throw_cpp") (param $value ${wasmType}) + local.get $value + throw $cpp)) + `, `cpp-thrower-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const catcherBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "throw_cpp" (func $throw_cpp (param ${wasmType}))) + (tag $cpp (import "env" "__cpp_exception") (param ${wasmType})) + (func (export "catch_cpp") (param $value ${wasmType}) (result ${wasmType}) + (try (result ${wasmType}) + (do + local.get $value + call $throw_cpp + unreachable) + (catch $cpp)))) + `, `cpp-catcher-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.ptrWidth = ptrWidth; + options.cppExceptionTag = createCppExceptionTag(ptrWidth)!; + + loadSharedLibrarySync(`libcpp-thrower-${wasmType}.so`, throwerBytes, options); + const processTag = options.cppExceptionTag; + const catcher = loadSharedLibrarySync( + `libcpp-catcher-${wasmType}.so`, + catcherBytes, + options, + ); + + expect(options.cppExceptionTag).toBe(processTag); + expect((catcher.exports.catch_cpp as Function)(value)).toBe(value); + }, + ); + + it("rejects a lookalike process C++ tag before instantiation", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $cpp (import "env" "__cpp_exception") (param i32))) + `, "invalid-cpp-tag", undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.cppExceptionTag = {} as WebAssembly.Tag; + + expect(() => loadSharedLibrarySync("libinvalid-cpp-tag.so", wasmBytes, options)) + .toThrow(/__cpp_exception must be an actual WebAssembly\.Tag/); + }); +}); + describe.skipIf(!hasCompiler())("dylink.0 parser", () => { it("parses a simple shared library", () => { const wasmBytes = buildSharedLib( @@ -735,6 +808,185 @@ describe("dylink symbol interposition", () => { }); describe("dylink replay layout and rollback", () => { + it("restores copied live TLS without re-running side-module TLS initialization", () => { + const tlsSide = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__memory_base" (global $memory_base i32)) + (global $tls_base (export "__tls_base") (mut i32) (i32.const 0)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4)) + (global (export "__wasm_lpad_context") i32 (i32.const 8)) + (func $init_tls (export "__wasm_init_tls") (param $base i32) + local.get $base + global.set $tls_base + local.get $base + i32.const 42 + i32.store) + (func $start + global.get $memory_base + i32.load + i32.eqz + if + global.get $memory_base + i32.const 2 + i32.store + global.get $memory_base + i32.const 8 + i32.add + call $init_tls + end) + (start $start) + (func (export "get_tls") (result i32) + global.get $tls_base + i32.load) + (func (export "set_tls") (param $value i32) + global.get $tls_base + local.get $value + i32.store)) + `, "replay-live-tls", undefined, 0, 16, [], ["__wasm_lpad_context"]); + const parent = createSideForkLoadOptions(); + const loaded = loadSharedLibrarySync("libtls.so", tlsSide, parent); + expect(loaded.tlsBase).toBe(1032); + expect((loaded.exports.__wasm_lpad_context as WebAssembly.Global).value) + .toBe(loaded.tlsBase! + 8); + expect((loaded.exports.__tls_size as WebAssembly.Global).value).toBe(4); + expect((loaded.exports.__tls_align as WebAssembly.Global).value).toBe(4); + expect((loaded.exports.get_tls as Function)()).toBe(42); + (loaded.exports.set_tls as Function)(99); + + const child = createSideForkLoadOptions(); + new Uint8Array(child.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + const replayed = loadSharedLibrarySync("libtls.so", tlsSide, child, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: loaded.tlsBase, + }); + + expect(replayed.tlsBase).toBe(loaded.tlsBase); + expect((replayed.exports.__wasm_lpad_context as WebAssembly.Global).value) + .toBe(replayed.tlsBase! + 8); + expect((replayed.exports.get_tls as Function)()).toBe(99); + + const ambiguous = createSideForkLoadOptions(); + new Uint8Array(ambiguous.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + expect(() => loadSharedLibrarySync("libtls.so", tlsSide, ambiguous, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: 0, + })).toThrow(/missing a valid side-module TLS base/); + }); + + it("restores an i64 TLS-base global for the 64-bit pointer contract", () => { + const tlsSide = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__memory_base" (global $memory_base i32)) + (global $tls_base (export "__tls_base") (mut i64) (i64.const 0)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4)) + (func $init_tls (param $base i32) + local.get $base + i64.extend_i32_u + global.set $tls_base + local.get $base + i32.const 17 + i32.store) + (func $start + global.get $memory_base + i32.load + i32.eqz + if + global.get $memory_base + i32.const 2 + i32.store + global.get $memory_base + i32.const 8 + i32.add + call $init_tls + end) + (start $start) + (func (export "get_tls") (result i32) + global.get $tls_base + i32.wrap_i64 + i32.load) + (func (export "set_tls") (param $value i32) + global.get $tls_base + i32.wrap_i64 + local.get $value + i32.store)) + `, "replay-live-tls-i64", undefined, 0, 16); + const parent = createSideForkLoadOptions(); + parent.ptrWidth = 8; + const loaded = loadSharedLibrarySync("libtls64.so", tlsSide, parent); + (loaded.exports.set_tls as Function)(71); + + const child = createSideForkLoadOptions(); + child.ptrWidth = 8; + new Uint8Array(child.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + const replayed = loadSharedLibrarySync("libtls64.so", tlsSide, child, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: loaded.tlsBase, + }); + + expect(replayed.tlsBase).toBe(loaded.tlsBase); + expect((replayed.exports.__tls_base as WebAssembly.Global).value) + .toBe(BigInt(loaded.tlsBase!)); + expect((replayed.exports.get_tls as Function)()).toBe(71); + }); + + it("rejects incomplete, immutable, and out-of-reservation TLS state", () => { + const missingBase = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4))) + `, "tls-missing-base", undefined, 0, 16); + expect(() => loadSharedLibrarySync( + "libtls-missing-base.so", + missingBase, + createSideForkLoadOptions(), + )).toThrow(/must export mutable __tls_base/); + + const immutableBase = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (global (export "__tls_base") i32 (i32.const 1032)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4))) + `, "tls-immutable-base", undefined, 0, 16); + expect(() => loadSharedLibrarySync( + "libtls-immutable-base.so", + immutableBase, + createSideForkLoadOptions(), + )).toThrow(/must be mutable/); + + const live = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (import "env" "__memory_base" (global $memory_base i32)) + (global $tls_base (export "__tls_base") (mut i32) (i32.const 0)) + (global (export "__tls_size") i32 (i32.const 4)) + (global (export "__tls_align") i32 (i32.const 4)) + (func $start + global.get $memory_base + i32.const 8 + i32.add + global.set $tls_base) + (start $start)) + `, "tls-outside-reservation", undefined, 0, 16); + const parent = createSideForkLoadOptions(); + const loaded = loadSharedLibrarySync("libtls-outside.so", live, parent); + const child = createSideForkLoadOptions(); + new Uint8Array(child.memory.buffer).set(new Uint8Array(parent.memory.buffer)); + expect(() => loadSharedLibrarySync("libtls-outside.so", live, child, { + memoryBase: loaded.memoryBase, + tableBase: loaded.tableBase, + tlsBase: loaded.memoryBase + 16, + })).toThrow(/escapes module reservation/); + }); + it("pads replay to the exact parent table base and rejects overshoot", () => { const wasmBytes = buildDylinkWat(` (module diff --git a/host/test/fork-dlopen-replay-e2e.test.ts b/host/test/fork-dlopen-replay-e2e.test.ts index 57f4cf17a..7a5b0375d 100644 --- a/host/test/fork-dlopen-replay-e2e.test.ts +++ b/host/test/fork-dlopen-replay-e2e.test.ts @@ -13,8 +13,8 @@ * This fixture is expected to FAIL until that fix lands. */ import { describe, it, expect, beforeAll } from "vitest"; -import { execSync } from "node:child_process"; -import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { execFileSync, execSync } from "node:child_process"; +import { readFileSync, writeFileSync, mkdirSync, existsSync, realpathSync } from "node:fs"; import { join, dirname } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; @@ -23,13 +23,25 @@ import { NodePlatformIO } from "../src/platform/node"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = join(__dirname, "../.."); -const SYSROOT = join(REPO_ROOT, "sysroot"); +const SYSROOT = process.env.KANDELO_TEST_SYSROOT ?? join(REPO_ROOT, "sysroot"); const GLUE_DIR = join(REPO_ROOT, "libc", "glue"); -const LLVM_BIN = process.env.LLVM_BIN || "/opt/homebrew/opt/llvm@21/bin"; -const CLANG = `${LLVM_BIN}/clang`; -const WASM_LD = process.env.LLVM_BIN - ? `${LLVM_BIN}/wasm-ld` - : "/opt/homebrew/bin/wasm-ld"; +const clangDriver = process.env.CLANG ?? "clang"; + +function llvmTool(name: "clang" | "clang++" | "wasm-ld"): string { + const override = name === "wasm-ld" ? process.env.WASM_LD : undefined; + if (override) return override; + try { + return execFileSync(clangDriver, [`-print-prog-name=${name}`], { + encoding: "utf8", + }).trim() || name; + } catch { + return name; + } +} + +const CLANG = llvmTool("clang"); +const CLANGXX = llvmTool("clang++"); +const WASM_LD = llvmTool("wasm-ld"); const FORK_INSTRUMENT = join(REPO_ROOT, "scripts", "run-wasm-fork-instrument.sh"); const hasSysroot = existsSync(join(SYSROOT, "lib", "libc.a")); @@ -38,6 +50,39 @@ const hasKernel = existsSync(join(REPO_ROOT, "binaries", "kernel.wasm")) || const BUILD_DIR = join(tmpdir(), "wasm-fork-dlopen-replay-e2e"); +function findLibcxxPrefix(): string | undefined { + const explicit = process.env.KANDELO_LIBCXX_PREFIX; + if ( + explicit + && existsSync(join(explicit, "lib", "libc++-pic.a")) + && existsSync(join(explicit, "lib", "libc++abi-pic.a")) + ) { + return explicit; + } + const sysrootArchive = join(SYSROOT, "lib", "libc++.a"); + if (!existsSync(sysrootArchive)) return undefined; + const prefix = dirname(dirname(realpathSync(sysrootArchive))); + return existsSync(join(prefix, "lib", "libc++-pic.a")) + && existsSync(join(prefix, "lib", "libc++abi-pic.a")) + ? prefix + : undefined; +} + +const libcxxPrefix = findLibcxxPrefix(); +const hasCppPrerequisites = hasSysroot && hasKernel && libcxxPrefix !== undefined; + +if (process.env.KANDELO_REQUIRE_CPP_DYLINK_FORK_E2E === "1" && !hasCppPrerequisites) { + throw new Error( + "C++ dlopen/fork e2e was required but kernel.wasm, sysroot/libc.a, or libcxx PIC archives are missing", + ); +} + +const CPP_RUNTIME_MAIN_EXPORTS = [ + "getenv", "fprintf", "fflush", "malloc", "strlen", "memcmp", "realloc", + "free", "fwrite", "vfprintf", "fputc", "abort", "memchr", "snprintf", + "aligned_alloc", "strcmp", "pthread_mutex_lock", "pthread_mutex_unlock", "calloc", +]; + /** Build a shared Wasm library (.so side module) from C source. */ function buildSharedLib(source: string, name: string): string { const srcPath = join(BUILD_DIR, `${name}.c`); @@ -58,8 +103,46 @@ function buildSharedLib(source: string, name: string): string { return soPath; } +/** Build a real C++ EH side module, including its TLS-bearing unwinder. */ +function buildCppSharedLib(source: string, name: string): string { + if (!libcxxPrefix) throw new Error("libcxx PIC prefix unavailable"); + const srcPath = join(BUILD_DIR, `${name}.cpp`); + const objPath = join(BUILD_DIR, `${name}.o`); + const soPath = join(BUILD_DIR, `${name}.so`); + writeFileSync(srcPath, source); + execFileSync(CLANGXX, [ + "--target=wasm32-unknown-unknown", + `--sysroot=${SYSROOT}`, + "-nostdlib", + "-fPIC", + "-O2", + "-fwasm-exceptions", + "-matomics", + "-mbulk-memory", + `-I${join(libcxxPrefix, "include", "c++", "v1")}`, + "-c", + srcPath, + "-o", + objPath, + ], { stdio: "pipe" }); + execFileSync(WASM_LD, [ + "--experimental-pic", + "--shared", + "--shared-memory", + "--export-all", + "--allow-undefined", + "--export=__tls_base", + "-o", + soPath, + objPath, + join(libcxxPrefix, "lib", "libc++-pic.a"), + join(libcxxPrefix, "lib", "libc++abi-pic.a"), + ], { stdio: "pipe" }); + return soPath; +} + /** Build a main program with dlopen + fork support. */ -function buildMainProgram(source: string, name: string): string { +function buildMainProgram(source: string, name: string, forceExports: string[] = []): string { const srcPath = join(BUILD_DIR, `${name}.c`); const wasmPath = join(BUILD_DIR, `${name}.wasm`); @@ -97,6 +180,8 @@ function buildMainProgram(source: string, name: string): string { "-Wl,--export=__tls_align", "-Wl,--export=__stack_pointer", "-Wl,--export=__wasm_thread_init", + ...(forceExports.length > 0 ? ["-Wl,--export-all"] : []), + ...forceExports.map((symbol) => `-Wl,-u,${symbol}`), ]; const allArgs = [...cflags, srcPath, ...linkFlags, "-o", wasmPath]; @@ -188,4 +273,134 @@ describe.skipIf(!hasSysroot || !hasKernel)("fork after dlopen end-to-end", () => expect(result.exitCode).toBe(0); expect(result.stdout).toContain("ok"); }); + + it("fails pthread dlopen and fork after process dlopen without creating a child", { timeout: 30_000 }, async () => { + const soPath = buildSharedLib( + `int pthread_boundary_fixture(void) { return 1; }`, + "libpthreadboundary", + ); + const wasmPath = buildMainProgram(` + #include + #include + #include + #include + #include + #include + + static const char *side_path; + static int thread_result; + + static void *run_thread(void *unused) { + (void)unused; + void *nested = dlopen(side_path, RTLD_NOW); + const char *error = dlerror(); + if (nested != NULL || error == NULL || strstr(error, "pthread workers") == NULL) { + thread_result = 1; + return NULL; + } + errno = 0; + pid_t child = fork(); + if (child != -1 || errno != ENOTSUP) { + thread_result = 2; + return NULL; + } + thread_result = 0; + return NULL; + } + + int main(int argc, char **argv) { + side_path = argv[1]; + void *side = dlopen(side_path, RTLD_NOW); + if (!side) { fprintf(stderr, "main dlopen: %s\\n", dlerror()); return 2; } + pthread_t thread; + if (pthread_create(&thread, NULL, run_thread, NULL) != 0) return 3; + if (pthread_join(thread, NULL) != 0) return 4; + if (thread_result != 0) return 10 + thread_result; + puts("pthread dylink boundary ok"); + return 0; + } + `, "test-pthread-dylink-boundary"); + + const result = await runCentralizedProgram({ + programPath: wasmPath, + argv: ["pthread-dylink-boundary", soPath], + timeout: 30_000, + io: io(), + captureForkCount: true, + }); + + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("pthread dylink boundary ok"); + expect(result.forkCount).toBe(0n); + }); + + it.skipIf(!hasCppPrerequisites)( + "child preserves side-module TLS for a real compiled C++ throw/catch", + { timeout: 30_000 }, + async () => { + const soPath = buildCppSharedLib(` + thread_local int cpp_tls_marker = 7; + extern "C" void cpp_set_tls_marker(int value) { cpp_tls_marker = value; } + extern "C" int cpp_get_tls_marker(void) { return cpp_tls_marker; } + extern "C" int cpp_throw_and_catch(int value) { + try { throw value; } + catch (int caught) { return caught; } + } + `, "libcppthrow"); + const sideModule = new WebAssembly.Module( + new Uint8Array(readFileSync(soPath)) as unknown as BufferSource, + ); + const sideImports = WebAssembly.Module.imports(sideModule); + const sideExports = WebAssembly.Module.exports(sideModule); + expect(sideImports.some((entry) => + entry.module === "env" + && entry.name === "__cpp_exception" + && (entry.kind as string) === "tag" + )).toBe(true); + expect(sideExports.map((entry) => entry.name)).toEqual( + expect.arrayContaining(["__tls_base", "__tls_size", "__wasm_init_tls"]), + ); + + const wasmPath = buildMainProgram(` + #include + #include + #include + #include + #include + typedef int (*cpp_throw_fn)(int); + typedef void (*cpp_set_marker_fn)(int); + typedef int (*cpp_get_marker_fn)(void); + int main(int argc, char **argv) { + void *lib = dlopen(argv[1], RTLD_NOW); + if (!lib) { fprintf(stderr, "dlopen: %s\\n", dlerror()); return 2; } + cpp_throw_fn run = (cpp_throw_fn)dlsym(lib, "cpp_throw_and_catch"); + cpp_set_marker_fn set_marker = (cpp_set_marker_fn)dlsym(lib, "cpp_set_tls_marker"); + cpp_get_marker_fn get_marker = (cpp_get_marker_fn)dlsym(lib, "cpp_get_tls_marker"); + if (!run || !set_marker || !get_marker || run(41) != 41) return 3; + set_marker(99); + if (get_marker() != 99) return 4; + pid_t pid = fork(); + if (pid == 0) _exit(get_marker() == 99 && run(42) == 42 ? 0 : 5); + if (pid < 0) return 6; + int status = 0; + if (waitpid(pid, &status, 0) != pid) return 7; + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) return 8; + puts("cpp throw after fork ok"); + return 0; + } + `, "test-cpp-throw-after-dlopen-fork", CPP_RUNTIME_MAIN_EXPORTS); + + const result = await runCentralizedProgram({ + programPath: wasmPath, + argv: ["cpp-throw-main", soPath], + timeout: 30_000, + io: io(), + }); + + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("cpp throw after fork ok"); + }, + ); }); From 60a091b2a84173fdce5a90e75720fca60e532be4 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 05:00:42 -0400 Subject: [PATCH 10/12] php: materialize intl ICU data through the package closure Problem: intl.so needs an ICU data blob at runtime, but the original branch relied on cache scanning and host-only path overrides. That made local, fetched, Node, VFS, and browser executions observe different closures and allowed a present extension to run without its matching data. Design: PHP copies the exact resolver-selected ICU data into its program archive as a declared runtime file at /usr/lib/php/icu.dat. Repository image/test builders query structured package metadata for the guest path and mode, fail when intl exists without its closure, and exercise the same resolver-selected bytes on Node and browser. The PHP build exports __tls_base for fork-safe side-module replay and tracks the loader source as a cache-key input. Focused evidence: runtime metadata resolved to php/icu.dat, /usr/lib/php/icu.dat, mode 0644; the local mirror is byte-identical to both reproducible ICU rev6 outputs; shell syntax, host declarations, and the PHP browser Vite bundle passed. The dev middleware served all 30,782,896 ICU bytes and returned 400 for dot, encoded-dot, and malformed-percent artifact paths. The base-PHP focused case passed against an ABI-19 base-manifest image. Limits: the four intl runtime cases intentionally remain unvalidated on the stale cached intl.so, which the repaired host correctly rejects because it does not export mutable __tls_base. A fresh PHP rebuild, artifact inspection, Node intl/fork tests, real Chromium run, and full aggregate gates are still required. The focused base-manifest rootfs omitted unrelated lazy package entries and is not canonical browser evidence. --- .../vfs/scripts/build-php-test-vfs-image.ts | 37 ++++- packages/registry/php/build-php.sh | 10 +- packages/registry/php/build.toml | 1 + packages/registry/php/intl-icu-data-loader.c | 13 +- packages/registry/php/package.toml | 11 +- .../php/test/browser/php-browser.spec.ts | 96 ++++++++++--- packages/registry/php/test/browser/run-php.ts | 134 +++++++++++++----- .../registry/php/test/browser/vite.config.ts | 79 ++++++++--- packages/registry/php/test/php-intl.test.ts | 105 +++++++------- scripts/run-php-upstream-tests.ts | 22 +++ 10 files changed, 361 insertions(+), 147 deletions(-) diff --git a/images/vfs/scripts/build-php-test-vfs-image.ts b/images/vfs/scripts/build-php-test-vfs-image.ts index fa527c5ac..a9a23552f 100644 --- a/images/vfs/scripts/build-php-test-vfs-image.ts +++ b/images/vfs/scripts/build-php-test-vfs-image.ts @@ -34,6 +34,7 @@ import { findRepoRoot, tryResolveBinary } from "../../../host/src/binary-resolve import { preparePhpTestFixtures } from "./php-test-fixtures"; import { ensureSourceExtract } from "./source-extract-helper"; import { saveImage, walkAndWrite } from "./vfs-image-helpers"; +import { resolvePackageRuntimeFile } from "../../../scripts/package-runtime-file"; const REPO_ROOT = findRepoRoot(); const PHP_FIXTURE_ROOT = join(REPO_ROOT, "tests/php-fixtures"); @@ -43,6 +44,7 @@ const PHP_WASM = process.env.PHP_WASM ?? join(LOCAL_PHP_SRC, "sapi/cli/php"); const OPCACHE_SO = process.env.PHP_OPCACHE_SO ?? tryResolveBinary("programs/php/opcache.so"); +const ICU_RUNTIME = resolvePackageRuntimeFile(REPO_ROOT, "php", "icu.dat"); const PHP_EXTENSION_DIRS = [ dirname(PHP_WASM), ...((process.env.PHP_EXTENSION_DIR ?? "") @@ -50,10 +52,16 @@ const PHP_EXTENSION_DIRS = [ .map((path) => path.trim()) .filter(Boolean)), ]; +const INTL_SO = tryResolveBinary("programs/php/intl.so") + ?? [...PHP_EXTENSION_DIRS] + .reverse() + .map((dir) => join(dir, "intl.so")) + .find((path) => existsSync(path)); const PHP_FPM_WASM = process.env.PHP_FPM_WASM ?? tryResolveBinary("programs/php/php-fpm.wasm"); const ROOTFS_VFS = process.env.ROOTFS_VFS - ?? tryResolveBinary("programs/rootfs/rootfs.vfs") + ?? tryResolveBinary("rootfs.vfs") + ?? tryResolveBinary("programs/rootfs.vfs") ?? join(REPO_ROOT, "host/wasm/rootfs.vfs"); const OUT_FILE = process.env.PHP_TEST_VFS_OUT ?? join(REPO_ROOT, "apps/browser-demos/public/php-test.vfs.zst"); @@ -116,6 +124,13 @@ function phpTestVfsFingerprint(sourceRoot: string): string { hashInputPath(hash, `extensions-${index}`, extensionDir); } hashInputPath(hash, "opcache", OPCACHE_SO); + hashInputPath(hash, "intl", INTL_SO); + hashInputPath(hash, "intl-icu-data", ICU_RUNTIME?.hostPath); + if (ICU_RUNTIME) { + hash.update( + `runtime-contract\0${ICU_RUNTIME.artifact}\0${ICU_RUNTIME.guestPath}\0${ICU_RUNTIME.mode}\0`, + ); + } return hash.digest("hex"); } @@ -283,6 +298,11 @@ async function main() { `rootfs.vfs not found at ${ROOTFS_VFS}. Build the rootfs package or set ROOTFS_VFS`, ); } + if (INTL_SO && !ICU_RUNTIME) { + throw new Error( + "PHP intl.so is present but the declared php:icu.dat runtime file is not materialized", + ); + } const phpSourceInput = resolvePhpSource(); if (!existsSync(phpSourceInput)) { throw new Error(`php-src not found at ${phpSourceInput}`); @@ -358,6 +378,21 @@ async function main() { new Uint8Array(readFileSync(OPCACHE_SO)), ); } + if (INTL_SO && ICU_RUNTIME) { + writeVfsBinary( + fs, + "/usr/lib/php/extensions/intl.so", + new Uint8Array(readFileSync(INTL_SO)), + 0o755, + ); + ensureDirRecursive(fs, dirname(ICU_RUNTIME.guestPath)); + writeVfsBinary( + fs, + ICU_RUNTIME.guestPath, + new Uint8Array(readFileSync(ICU_RUNTIME.hostPath)), + ICU_RUNTIME.mode, + ); + } const phptDirs = collectPhptDirs(phpSrc); const supportDirs = collectPhptSupportDirs(phpSrc, phptDirs); diff --git a/packages/registry/php/build-php.sh b/packages/registry/php/build-php.sh index f4a81f3f2..9349334c3 100755 --- a/packages/registry/php/build-php.sh +++ b/packages/registry/php/build-php.sh @@ -1262,7 +1262,7 @@ echo "==> linking intl.so from ${#INTL_OBJS[@]} objects + ICU static libs + libc # archives are listed in dependency order (i18n -> io -> uc -> data), then # libc++/libc++abi. A -shared PIC module requires every input to be PIC, so the # libc++ PIC variants are named explicitly to win over the non-PIC sysroot ones. -wasm32posix-cc -shared -fPIC -o "$BIN_DIR/intl.so" \ +wasm32posix-cc -shared -fPIC -Wl,--export=__tls_base -o "$BIN_DIR/intl.so" \ "${INTL_OBJS[@]}" \ ext/intl/kandelo_icu_data_loader.o \ "$ICU_PREFIX/lib/libicui18n.a" \ @@ -1273,6 +1273,13 @@ wasm32posix-cc -shared -fPIC -o "$BIN_DIR/intl.so" \ "$LIBCXX_PREFIX/lib/libc++abi-pic.a" echo "==> intl.so: $(wc -c < "$BIN_DIR/intl.so") bytes" +# Make the program package's runtime closure self-contained. ICU remains the +# build dependency/source of truth; PHP's archive carries these exact resolved +# bytes so normal local/fetched materialization can stage the declared guest +# file without inspecting the library cache. +cp "$ICU_PREFIX/share/icu.dat" "$BIN_DIR/icu.dat" +chmod 0644 "$BIN_DIR/icu.dat" + # Copy to bin/ with .wasm extension (needed for Vite browser demos) cp sapi/cli/php "$BIN_DIR/php.wasm" cp sapi/fpm/php-fpm "$BIN_DIR/php-fpm.wasm" @@ -1319,4 +1326,5 @@ if [ -z "${WASM_POSIX_DEP_OUT_DIR:-}" ]; then install_local_binary php "$BIN_DIR/zend_test.so" install_local_binary php "$BIN_DIR/zip.so" install_local_binary php "$BIN_DIR/intl.so" + install_local_runtime_file php "$BIN_DIR/icu.dat" fi diff --git a/packages/registry/php/build.toml b/packages/registry/php/build.toml index 554db487c..a733f0df5 100644 --- a/packages/registry/php/build.toml +++ b/packages/registry/php/build.toml @@ -1,6 +1,7 @@ script_path = "packages/registry/php/build-php.sh" inputs = [ "packages/registry/php/build-php.sh", + "packages/registry/php/intl-icu-data-loader.c", ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" diff --git a/packages/registry/php/intl-icu-data-loader.c b/packages/registry/php/intl-icu-data-loader.c index 77f5e60a5..1f88fb168 100644 --- a/packages/registry/php/intl-icu-data-loader.c +++ b/packages/registry/php/intl-icu-data-loader.c @@ -11,8 +11,8 @@ * A missing/unreadable icu.dat is non-fatal at load (intl.so may be present * without any code using intl) but stays loud: we warn to stderr and let ICU * fail with U_MISSING_RESOURCE_ERROR when a service actually needs data, rather - * than silently succeeding. Path defaults to /usr/lib/php/icu.dat, overridable - * via KANDELO_ICU_DAT_PATH for VFS images that stage it elsewhere. + * than silently succeeding. The PHP package runtime-file contract installs the + * exact matching bytes at /usr/lib/php/icu.dat on every host. */ #include @@ -26,22 +26,19 @@ #include #include -#define KANDELO_ICU_DAT_DEFAULT "/usr/lib/php/icu.dat" +#define KANDELO_ICU_DAT_PATH "/usr/lib/php/icu.dat" static void kandelo_intl_load_icu_data(void) __attribute__((constructor)); static void kandelo_intl_load_icu_data(void) { - const char *path = getenv("KANDELO_ICU_DAT_PATH"); - if (path == NULL || path[0] == '\0') { - path = KANDELO_ICU_DAT_DEFAULT; - } + const char *path = KANDELO_ICU_DAT_PATH; int fd = open(path, O_RDONLY); if (fd < 0) { fprintf(stderr, "[intl] ICU data not loaded: cannot open %s. " "intl functions will fail with U_MISSING_RESOURCE_ERROR. " - "Set KANDELO_ICU_DAT_PATH or stage icu.dat there.\n", + "Rebuild/materialize the complete PHP package runtime closure.\n", path); return; } diff --git a/packages/registry/php/package.toml b/packages/registry/php/package.toml index 26d15124c..1a34dd52e 100644 --- a/packages/registry/php/package.toml +++ b/packages/registry/php/package.toml @@ -78,8 +78,15 @@ name = "zip" wasm = "zip.so" # intl is a normal, opt-in PHP extension. It statically absorbs the ICU and -# PIC libc++ archives, while ICU's common data remains a separately declared -# icu package file that VFS consumers stage at /usr/lib/php/icu.dat. +# PIC libc++ archives. PHP copies the exact resolved ICU common-data bytes into +# its own declared runtime closure for VFS consumers. [[outputs]] name = "intl" wasm = "intl.so" + +# Non-Wasm runtime closure for intl. The resolver/archive mirrors this at +# programs//php/icu.dat; VFS consumers install it at the declared guest +# path instead of scanning a library cache or injecting a host-only override. +[[runtime_files]] +artifact = "icu.dat" +guest_path = "/usr/lib/php/icu.dat" diff --git a/packages/registry/php/test/browser/php-browser.spec.ts b/packages/registry/php/test/browser/php-browser.spec.ts index 9aef3208f..ec59da5d0 100644 --- a/packages/registry/php/test/browser/php-browser.spec.ts +++ b/packages/registry/php/test/browser/php-browser.spec.ts @@ -8,46 +8,48 @@ * The browser harness runs multiple PHP tests (inline, file-based, extensions) * and reports all results as JSON in the #results element. * - * Run: npx playwright test --config packages/registry/php/test/browser/playwright.config.ts + * Run the default suite: + * npx playwright test --config packages/registry/php/test/browser/playwright.config.ts + * Both cases resolve their complete PHP package closure through the normal + * local/published binary mirrors. */ import { test, expect } from "@playwright/test"; import { existsSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { resolvePackageRuntimeFile } from "../../../../../scripts/package-runtime-file"; +import { tryResolveBinary } from "../../../../../host/src/binary-resolver"; const __dirname = dirname(fileURLToPath(import.meta.url)); const repoRoot = join(__dirname, "../../../../.."); -const hasKernelWasm = [ - join(repoRoot, "local-binaries/kernel.wasm"), - join(repoRoot, "binaries/kernel.wasm"), -].some((candidate) => existsSync(candidate)); -const hasPhpWasm = [ - join(repoRoot, "local-binaries/programs/wasm32/php/php.wasm"), - join(repoRoot, "binaries/programs/wasm32/php/php.wasm"), - join(repoRoot, "packages/registry/php/php-src/sapi/cli/php"), -].some((candidate) => existsSync(candidate)); -const hasZipSo = [ - join(repoRoot, "local-binaries/programs/wasm32/php/zip.so"), - join(repoRoot, "binaries/programs/wasm32/php/zip.so"), - join(repoRoot, "packages/registry/php/bin/zip.so"), -].some((candidate) => existsSync(candidate)); -const hasCurlSo = [ - join(repoRoot, "local-binaries/programs/wasm32/php/curl.so"), - join(repoRoot, "binaries/programs/wasm32/php/curl.so"), - join(repoRoot, "packages/registry/php/bin/curl.so"), -].some((candidate) => existsSync(candidate)); -const hasRootfsVfs = existsSync(join(repoRoot, "host/wasm/rootfs.vfs")); +const hasKernelWasm = tryResolveBinary("kernel.wasm") != null; +const hasPhpWasm = tryResolveBinary("programs/php/php.wasm") != null + || existsSync(join(repoRoot, "packages/registry/php/php-src/sapi/cli/php")); +const hasZipSo = tryResolveBinary("programs/php/zip.so") != null + || existsSync(join(repoRoot, "packages/registry/php/bin/zip.so")); +const hasCurlSo = tryResolveBinary("programs/php/curl.so") != null + || existsSync(join(repoRoot, "packages/registry/php/bin/curl.so")); +const hasIntlSo = tryResolveBinary("programs/php/intl.so") != null + || existsSync(join(repoRoot, "packages/registry/php/bin/intl.so")); +const phpIcuRuntime = resolvePackageRuntimeFile(repoRoot, "php", "icu.dat"); +if (hasIntlSo && !phpIcuRuntime) { + throw new Error( + "PHP intl.so is present but the declared php:icu.dat runtime file is not materialized", + ); +} +const hasRootfsVfs = tryResolveBinary("rootfs.vfs") != null + || tryResolveBinary("programs/rootfs.vfs") != null; test.skip(!hasKernelWasm, "kernel.wasm is not built or fetched"); test.skip(!hasPhpWasm, "php.wasm is not built or fetched"); -test.skip(!hasZipSo, "zip.so is not built or fetched"); -test.skip(!hasCurlSo, "curl.so is not built or fetched"); test.skip(!hasRootfsVfs, "rootfs.vfs is not built"); test("PHP CLI runs in the browser (inline, file, session, SQLite, fileinfo, XML, OpenSSL, extensions)", async ({ page, }) => { + test.skip(!hasZipSo, "zip.so is not built or fetched"); + test.skip(!hasCurlSo, "curl.so is not built or fetched"); const runtimeErrors: string[] = []; page.on("console", (msg) => { if (msg.type() === "error") runtimeErrors.push(`console: ${msg.text()}`); @@ -138,3 +140,51 @@ test("PHP CLI runs in the browser (inline, file, session, SQLite, fileinfo, XML, }); expect(runtimeErrors).toEqual([]); }); + +test("PHP intl and ICU data survive browser side-module fork replay", async ({ page }) => { + test.skip(!hasIntlSo, "intl.so is not built or fetched"); + + const runtimeErrors: string[] = []; + page.on("console", (msg) => { + if (msg.type() === "error") runtimeErrors.push(`console: ${msg.text()}`); + }); + page.on("pageerror", (error) => { + runtimeErrors.push(`pageerror: ${error.message}`); + }); + page.on("requestfailed", (request) => { + runtimeErrors.push( + `requestfailed: ${request.url()} ${request.failure()?.errorText ?? "failed"}`, + ); + }); + page.on("response", (response) => { + if (response.status() >= 400) { + runtimeErrors.push(`response: ${response.status()} ${response.url()}`); + } + }); + + await page.goto("/?intl=1"); + await page.waitForFunction( + () => { + const status = document.getElementById("status"); + return status?.textContent === "done" || status?.textContent === "error"; + }, + { timeout: 120_000 }, + ); + + const status = await page.locator("#status").textContent(); + const stderr = await page.locator("#stderr").textContent(); + const resultsText = await page.locator("#results").textContent(); + const exitCode = await page.locator("#exit-code").textContent(); + if (status === "error" || exitCode !== "0") { + console.log("INTL STDERR:", stderr); + console.log("INTL RESULTS:", resultsText); + } + + expect(status).toBe("done"); + expect(exitCode).toBe("0"); + expect(stderr).toBe(""); + const results = JSON.parse(resultsText!); + expect(results.intlFork).toContain("child=French"); + expect(results.intlFork).toContain("parent=French:apple,banana,cherry"); + expect(runtimeErrors).toEqual([]); +}); diff --git a/packages/registry/php/test/browser/run-php.ts b/packages/registry/php/test/browser/run-php.ts index 0c69ab3e7..107cd1dce 100644 --- a/packages/registry/php/test/browser/run-php.ts +++ b/packages/registry/php/test/browser/run-php.ts @@ -30,13 +30,18 @@ interface TestResult { exitCode: number; } +interface BinaryFixture { + bytes: ArrayBuffer; + mode: number; +} + async function runPhp( phpBytes: ArrayBuffer, kernelBytes: ArrayBuffer, rootfsBytes: ArrayBuffer, files: Record, - binaryFiles: Record, argv: string[], + binaryFiles: Record = {}, ): Promise { let stdout = ""; let stderr = ""; @@ -53,10 +58,9 @@ async function runPhp( for (const [path, content] of Object.entries(files)) { writeVfsFile(memfs, path, content); } - for (const [path, content] of Object.entries(binaryFiles)) { - const separator = path.lastIndexOf("/"); - if (separator > 0) ensureDirRecursive(memfs, path.slice(0, separator)); - writeVfsBinary(memfs, path, new Uint8Array(content)); + for (const [path, fixture] of Object.entries(binaryFiles)) { + ensureDirRecursive(memfs, path.slice(0, path.lastIndexOf("/")) || "/"); + writeVfsBinary(memfs, path, new Uint8Array(fixture.bytes), fixture.mode); } const vfsImage = await memfs.saveImage(); @@ -99,76 +103,130 @@ async function runPhp( async function main() { try { - const [kernelBytes, rootfsBytes, phpBytes, zipBytes, curlBytes] = await Promise.all([ + const [kernelBytes, rootfsBytes, phpBytes] = await Promise.all([ fetch(kernelWasmUrl).then((r) => r.arrayBuffer()), fetch(rootfsVfsUrl).then((r) => r.arrayBuffer()), fetch("/php-artifacts/php.wasm").then((r) => r.arrayBuffer()), + ]); + + // Run the dedicated intl/fork contract only for its Playwright case. The + // server exposes bytes and installation metadata from PHP's declared + // runtime closure; this browser path does not duplicate the ICU guest path. + if (new URL(window.location.href).searchParams.has("intl")) { + const [intlBytes, icuDataBytes, icuContract] = await Promise.all([ + fetch("/php-artifacts/intl.so").then((r) => r.arrayBuffer()), + fetch("/php-artifacts/icu.dat").then((r) => r.arrayBuffer()), + fetch("/php-runtime-files/icu.dat").then(async (response) => { + if (!response.ok) throw new Error(await response.text()); + return await response.json() as { + artifact: string; + guestPath: string; + mode: number; + }; + }), + ]); + const intlResult = await runPhp( + phpBytes, + kernelBytes, + rootfsBytes, + {}, + ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=intl.so", "-r", ` + $before = Locale::getDisplayLanguage("fr", "en"); + $pid = pcntl_fork(); + if ($pid < 0) { fwrite(STDERR, "fork-failed"); exit(20); } + if ($pid === 0) { + $child = Locale::getDisplayLanguage("fr", "en"); + echo "child=" . $child . "\\n"; + exit($child === "French" ? 0 : 21); + } + $status = 0; + $waited = pcntl_waitpid($pid, $status); + $c = new Collator("en_US"); + $a = ["banana", "apple", "cherry"]; + $c->sort($a); + echo "parent=" . $before . ":" . implode(",", $a) . "\\n"; + if ($waited !== $pid || !pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { + fwrite(STDERR, "child-status=" . $status); + exit(22); + } + `], + { + "/usr/lib/php/extensions/intl.so": { bytes: intlBytes, mode: 0o755 }, + [icuContract.guestPath]: { bytes: icuDataBytes, mode: icuContract.mode }, + }, + ); + stdoutEl.textContent = intlResult.stdout; + stderrEl.textContent = intlResult.stderr; + exitCodeEl.textContent = String(intlResult.exitCode); + resultsEl.textContent = JSON.stringify({ intlFork: intlResult.stdout.trim() }); + statusEl.textContent = "done"; + return; + } + + const [zipBytes, curlBytes] = await Promise.all([ fetch("/php-artifacts/zip.so").then((r) => r.arrayBuffer()), fetch("/php-artifacts/curl.so").then((r) => r.arrayBuffer()), ]); + const binaryFiles: Record = { + "/usr/lib/php/extensions/zip.so": { bytes: zipBytes, mode: 0o755 }, + "/usr/lib/php/extensions/curl.so": { bytes: curlBytes, mode: 0o755 }, + }; const files = { "/home/script.php": '', "/home/ext_test.php": ' mb_strlen("hello"), "ctype" => ctype_alpha("hello") ? "yes" : "no"]); ?>', }; - const binaryFiles = { - "/usr/lib/php/extensions/zip.so": zipBytes, - "/usr/lib/php/extensions/curl.so": curlBytes, - }; // Test 1: Hello World (inline) - const r1 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", 'echo "Hello World\n";']); + const r1 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", 'echo "Hello World\n";'], binaryFiles); // Test 2: File-based execution - const r2 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "/home/script.php"]); + const r2 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "/home/script.php"], binaryFiles); // Test 3: Extensions (mbstring + ctype) - const r3 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "/home/ext_test.php"]); + const r3 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "/home/ext_test.php"], binaryFiles); // Test 4: Session - const r4 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", 'session_start(); echo strlen(session_id()) > 0 ? "session-ok" : "fail";']); + const r4 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", 'session_start(); echo strlen(session_id()) > 0 ? "session-ok" : "fail";'], binaryFiles); // Test 5: SQLite3 in-memory - const r5 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$db=new SQLite3(":memory:");$db->exec("CREATE TABLE t(v TEXT)");$db->exec("INSERT INTO t VALUES(\'sqlite-ok\')");echo $db->querySingle("SELECT v FROM t");']); + const r5 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$db=new SQLite3(":memory:");$db->exec("CREATE TABLE t(v TEXT)");$db->exec("INSERT INTO t VALUES(\'sqlite-ok\')");echo $db->querySingle("SELECT v FROM t");'], binaryFiles); // Test 6: fileinfo - const r6 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$f=new finfo(FILEINFO_MIME_TYPE);echo $f->buffer("GIF89a");']); + const r6 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$f=new finfo(FILEINFO_MIME_TYPE);echo $f->buffer("GIF89a");'], binaryFiles); // Test 7: SimpleXML - const r7 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$x=new SimpleXMLElement("xml-ok");echo $x->i;']); + const r7 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$x=new SimpleXMLElement("xml-ok");echo $x->i;'], binaryFiles); // Test 8: rootfs OpenSSL defaults are present and key + CSR generation succeeds. - const r8 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, - ["php", "-r", '$k=openssl_pkey_new();$c=$k?openssl_csr_new(["commonName"=>"kandelo.test"],$k):false;if(!$k||!$c){while($e=openssl_error_string()){fwrite(STDERR,$e."\\n");}exit(1);}echo "openssl-defaults-ok";']); + const r8 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, + ["php", "-r", '$k=openssl_pkey_new();$c=$k?openssl_csr_new(["commonName"=>"kandelo.test"],$k):false;if(!$k||!$c){while($e=openssl_error_string()){fwrite(STDERR,$e."\\n");}exit(1);}echo "openssl-defaults-ok";'], binaryFiles); // Test 9: load the packaged zip side module from the kernel-owned VFS and // prove that a DEFLATE archive survives close/reopen in the browser host. - const r9 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, + const r9 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=zip.so", "-r", - '$p="/tmp/browser-zip-smoke.zip";$z=new ZipArchive;if($z->open($p,ZipArchive::CREATE|ZipArchive::OVERWRITE)!==true)exit(10);if(!$z->addFromString("hello.txt","browser-zip-ok"))exit(11);if(!$z->setCompressionName("hello.txt",ZipArchive::CM_DEFLATE))exit(12);if(!$z->close())exit(13);$r=new ZipArchive;if($r->open($p)!==true)exit(14);$s=$r->statName("hello.txt");if($s===false||$s["comp_method"]!==ZipArchive::CM_DEFLATE)exit(15);echo $r->getFromName("hello.txt");$r->close();']); + '$p="/tmp/browser-zip-smoke.zip";$z=new ZipArchive;if($z->open($p,ZipArchive::CREATE|ZipArchive::OVERWRITE)!==true)exit(10);if(!$z->addFromString("hello.txt","browser-zip-ok"))exit(11);if(!$z->setCompressionName("hello.txt",ZipArchive::CM_DEFLATE))exit(12);if(!$z->close())exit(13);$r=new ZipArchive;if($r->open($p)!==true)exit(14);$s=$r->statName("hello.txt");if($s===false||$s["comp_method"]!==ZipArchive::CM_DEFLATE)exit(15);echo $r->getFromName("hello.txt");$r->close();'], binaryFiles); // Test 10: load the packaged curl side module from the same browser VFS - // path and call into the linked libcurl implementation. The next test - // exercises browser TCP; this one isolates the PHP/dlopen side-module - // contract on the real browser host. - const r10 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, + // path and call into the linked libcurl implementation. + const r10 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=curl.so", "-r", - 'echo json_encode(["loaded"=>extension_loaded("curl"),"version"=>curl_version()["version"],"constant"=>defined("CURLOPT_URL"),"handle"=>is_object(curl_init())]);']); + 'echo json_encode(["loaded"=>extension_loaded("curl"),"version"=>curl_version()["version"],"constant"=>defined("CURLOPT_URL"),"handle"=>is_object(curl_init())]);'], binaryFiles); - // Test 11: serve one HTTP response on the kernel's loopback network and - // fetch it with libcurl from a fork child. Loading curl.so before fork - // also exercises browser-side dlopen replay in the child process. - const r11 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, binaryFiles, + // Test 11: exercise browser TCP with libcurl from a fork child. Loading + // curl.so before fork also verifies browser-side dlopen replay. + const r11 = await runPhp(phpBytes, kernelBytes, rootfsBytes, files, ["php", "-n", "-d", "extension_dir=/usr/lib/php/extensions", "-d", "extension=curl.so", "-r", - '$server=stream_socket_server("tcp://127.0.0.1:0",$errno,$error);if($server===false){fwrite(STDERR,"$errno:$error");exit(10);}$address=stream_socket_get_name($server,false);$pid=pcntl_fork();if($pid<0){fwrite(STDERR,"fork failed");exit(11);}if($pid===0){fclose($server);$ch=curl_init("http://$address/probe");curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);curl_setopt($ch,CURLOPT_TIMEOUT,10);$body=curl_exec($ch);if($body===false){fwrite(STDERR,curl_error($ch));exit(12);}echo json_encode(["body"=>$body,"status"=>curl_getinfo($ch,CURLINFO_RESPONSE_CODE)]);exit(0);}$client=stream_socket_accept($server,10);if($client===false){fwrite(STDERR,"accept failed");exit(13);}$request="";while(!str_contains($request,"\\r\\n\\r\\n")){$chunk=fread($client,4096);if($chunk===false||$chunk===""){fwrite(STDERR,"request read failed");exit(14);}$request.=$chunk;}fwrite($client,"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 16\\r\\nConnection: close\\r\\n\\r\\nkandelo-curl-ok\\n");fclose($client);fclose($server);pcntl_waitpid($pid,$status);if(!pcntl_wifexited($status)||pcntl_wexitstatus($status)!==0)exit(15);']); + '$server=stream_socket_server("tcp://127.0.0.1:0",$errno,$error);if($server===false){fwrite(STDERR,"$errno:$error");exit(10);}$address=stream_socket_get_name($server,false);$pid=pcntl_fork();if($pid<0){fwrite(STDERR,"fork failed");exit(11);}if($pid===0){fclose($server);$ch=curl_init("http://$address/probe");curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);curl_setopt($ch,CURLOPT_TIMEOUT,10);$body=curl_exec($ch);if($body===false){fwrite(STDERR,curl_error($ch));exit(12);}echo json_encode(["body"=>$body,"status"=>curl_getinfo($ch,CURLINFO_RESPONSE_CODE)]);exit(0);}$client=stream_socket_accept($server,10);if($client===false){fwrite(STDERR,"accept failed");exit(13);}$request="";while(!str_contains($request,"\\r\\n\\r\\n")){$chunk=fread($client,4096);if($chunk===false||$chunk===""){fwrite(STDERR,"request read failed");exit(14);}$request.=$chunk;}fwrite($client,"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 16\\r\\nConnection: close\\r\\n\\r\\nkandelo-curl-ok\\n");fclose($client);fclose($server);pcntl_waitpid($pid,$status);if(!pcntl_wifexited($status)||pcntl_wexitstatus($status)!==0)exit(15);'], binaryFiles); const results = { hello: r1.stdout.trim(), diff --git a/packages/registry/php/test/browser/vite.config.ts b/packages/registry/php/test/browser/vite.config.ts index b7a649210..42d8d3db8 100644 --- a/packages/registry/php/test/browser/vite.config.ts +++ b/packages/registry/php/test/browser/vite.config.ts @@ -2,10 +2,14 @@ import { fileURLToPath } from "url"; import path from "path"; import fs from "fs"; import type { Plugin } from "vite"; +import { resolvePackageRuntimeFile } from "../../../../../scripts/package-runtime-file"; +import { tryResolveBinary } from "../../../../../host/src/binary-resolver"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(__dirname, "../../../../.."); +const phpIcuRuntime = resolvePackageRuntimeFile(repoRoot, "php", "icu.dat"); + function resolveKernelArtifactsAlias(): Plugin { const KERNEL = "@kernel-wasm"; const ROOTFS = "@rootfs-vfs"; @@ -18,24 +22,21 @@ function resolveKernelArtifactsAlias(): Plugin { const query = queryIdx === -1 ? "" : source.slice(queryIdx); if (pathPart === KERNEL) { - const candidates = [ - path.resolve(repoRoot, "local-binaries/kernel.wasm"), - path.resolve(repoRoot, "binaries/kernel.wasm"), - ]; - const file = candidates.find((candidate) => fs.existsSync(candidate)); + const file = tryResolveBinary("kernel.wasm"); if (file) return file + query; this.error( - "kernel.wasm not found. Run `bash build.sh` from the repo root.\n" + - ` Looked at: ${candidates.join("\n Looked at: ")}`, + "kernel.wasm was not accepted from the standard local, fetched, or packaged locations. " + + "Run `bash build.sh` from the repo root or fetch package binaries.", ); } if (pathPart === ROOTFS) { - const file = path.resolve(repoRoot, "host/wasm/rootfs.vfs"); - if (fs.existsSync(file)) return file + query; + const file = tryResolveBinary("rootfs.vfs") + ?? tryResolveBinary("programs/rootfs.vfs"); + if (file) return file + query; this.error( - "rootfs.vfs not found. Run `bash build.sh` from the repo root.\n" + - ` Looked at: ${file}`, + "rootfs.vfs was not accepted from the standard local, fetched, or packaged locations. " + + "Run `bash build.sh` from the repo root or fetch the rootfs package.", ); } @@ -45,15 +46,22 @@ function resolveKernelArtifactsAlias(): Plugin { } function findPhpArtifact(name: string): string | null { + if (name === "icu.dat") return phpIcuRuntime?.hostPath ?? null; + const resolved = tryResolveBinary(`programs/php/${name}`); + if (resolved) return resolved; const candidates = [ - path.resolve(repoRoot, "local-binaries/programs/wasm32/php", name), - path.resolve(repoRoot, "binaries/programs/wasm32/php", name), path.resolve(__dirname, "../../bin", name), + ...(name === "php.wasm" + ? [path.resolve(__dirname, "../../php-src/sapi/cli/php")] + : []), ]; - if (name === "php.wasm") { - candidates.push(path.resolve(__dirname, "../../php-src/sapi/cli/php")); - } - return candidates.find((candidate) => fs.existsSync(candidate)) ?? null; + return candidates.find((candidate) => { + try { + return fs.statSync(candidate).isFile(); + } catch { + return false; + } + }) ?? null; } function servePhpArtifacts(): Plugin { @@ -61,13 +69,38 @@ function servePhpArtifacts(): Plugin { name: "serve-php-artifacts", configureServer(server) { server.middlewares.use((req, res, next) => { + if (req.url === "/php-runtime-files/icu.dat") { + if (!phpIcuRuntime) { + res.statusCode = 404; + res.end("the declared PHP icu.dat runtime file is not materialized"); + return; + } + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ + artifact: phpIcuRuntime.artifact, + guestPath: phpIcuRuntime.guestPath, + mode: phpIcuRuntime.mode, + })); + return; + } const prefix = "/php-artifacts/"; if (!req.url?.startsWith(prefix)) { next(); return; } - const name = decodeURIComponent(req.url.slice(prefix.length)); - if (!/^[A-Za-z0-9._-]+$/.test(name)) { + let name: string; + try { + name = decodeURIComponent(req.url.slice(prefix.length)); + } catch { + res.statusCode = 400; + res.end("invalid percent-encoding in PHP artifact name"); + return; + } + if ( + name === "." + || name === ".." + || !/^[A-Za-z0-9._-]+$/.test(name) + ) { res.statusCode = 400; res.end("invalid PHP artifact name"); return; @@ -75,10 +108,10 @@ function servePhpArtifacts(): Plugin { const artifact = findPhpArtifact(name); if (!artifact) { res.statusCode = 404; - res.end( - `${name} not found. Run \`bash packages/registry/php/build-php.sh\` ` + - "or fetch package binaries.", - ); + res.end(name === "icu.dat" + ? "the declared PHP icu.dat runtime file is not materialized" + : `${name} not found. Run \`bash packages/registry/php/build-php.sh\` ` + + "or fetch package binaries."); return; } const data = fs.readFileSync(artifact); diff --git a/packages/registry/php/test/php-intl.test.ts b/packages/registry/php/test/php-intl.test.ts index 8920f04d2..4d5172eaf 100644 --- a/packages/registry/php/test/php-intl.test.ts +++ b/packages/registry/php/test/php-intl.test.ts @@ -1,11 +1,15 @@ -import { describe, it, expect } from "vitest"; -import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { beforeAll, describe, it, expect } from "vitest"; +import { existsSync, readFileSync } from "node:fs"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; -import { homedir } from "node:os"; import { runCentralizedProgram } from "../../../../host/test/centralized-test-helper"; import { tryResolveBinary } from "../../../../host/src/binary-resolver"; -import { NodePlatformIO } from "../../../../host/src/platform/node"; +import { MemoryFileSystem } from "../../../../host/src/vfs/memory-fs"; +import { + ensureDirRecursive, + writeVfsBinary, +} from "../../../../host/src/vfs/image-helpers"; +import { resolvePackageRuntimeFile } from "../../../../scripts/package-runtime-file"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -17,52 +21,55 @@ const phpBinaryPath = tryResolveBinary("programs/php/php.wasm") ?? join(__dirname, "../php-src/sapi/cli/php"); const intlSoPath = tryResolveBinary("programs/php/intl.so"); +const rootfsPath = + tryResolveBinary("rootfs.vfs") ?? + tryResolveBinary("programs/rootfs.vfs") ?? + join(__dirname, "../../../../host/wasm/rootfs.vfs"); +const icuRuntime = resolvePackageRuntimeFile( + join(__dirname, "../../../.."), + "php", + "icu.dat", +); +const INTL_GUEST_PATH = "/usr/lib/php/extensions/intl.so"; -// icu.dat remains owned by the ICU library package, not copied into PHP's -// program outputs. CI/manual callers can name the exact resolved file; the -// cache fallback is restricted to this checkout's ICU version and revision so -// an unrelated newer ICU build cannot silently satisfy the test. -function findIcuDat(): string | undefined { - const explicit = process.env.PHP_INTL_ICU_DATA; - if (explicit !== undefined) { - if (!existsSync(explicit) || !statSync(explicit).isFile()) { - throw new Error(`PHP_INTL_ICU_DATA is not a regular file: ${explicit}`); - } - return explicit; - } - - const icuDir = join(__dirname, "../../icu"); - const version = readFileSync(join(icuDir, "package.toml"), "utf8") - .match(/^version\s*=\s*"([^"]+)"/m)?.[1]; - const revision = readFileSync(join(icuDir, "build.toml"), "utf8") - .match(/^revision\s*=\s*(\d+)/m)?.[1]; - if (!version || !revision) return undefined; - - const libsDir = join(homedir(), ".cache/kandelo/libs"); - if (!existsSync(libsDir)) return undefined; - const expectedPrefix = `icu-${version}-rev${revision}-wasm32-`; - const candidates = readdirSync(libsDir) - .filter((n) => n.startsWith(expectedPrefix) && !n.includes(".tmp-")) - .map((n) => join(libsDir, n, "share", "icu.dat")) - .filter((p) => existsSync(p)); - if (candidates.length === 0) return undefined; - return candidates.sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0]; +if (intlSoPath && !icuRuntime) { + throw new Error( + "PHP intl.so is present but the declared php:icu.dat runtime file is not materialized", + ); } -const icuDatPath = findIcuDat(); -const READY = existsSync(phpBinaryPath) && intlSoPath != null && icuDatPath != null; +const READY = existsSync(phpBinaryPath) + && intlSoPath != null + && existsSync(rootfsPath); +let intlRootfsImage: Uint8Array; describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { + beforeAll(async () => { + const fs = MemoryFileSystem.fromImage(new Uint8Array(readFileSync(rootfsPath))); + ensureDirRecursive(fs, dirname(INTL_GUEST_PATH)); + ensureDirRecursive(fs, dirname(icuRuntime!.guestPath)); + writeVfsBinary( + fs, + INTL_GUEST_PATH, + new Uint8Array(readFileSync(intlSoPath!)), + 0o755, + ); + writeVfsBinary( + fs, + icuRuntime!.guestPath, + new Uint8Array(readFileSync(icuRuntime!.hostPath)), + icuRuntime!.mode, + ); + intlRootfsImage = await fs.saveImage(); + }); + // Proves the base binary is genuinely ICU-free / intl-free: intl only // appears when explicitly loaded. This is the whole point of the design. it("base php.wasm does NOT include intl", async () => { const { stdout, exitCode } = await runCentralizedProgram({ programPath: phpBinaryPath, argv: ["php", "-m"], - // Same host-I/O adapter as the other cases: `php -m` needs no files, - // but it keeps the harness off the "default" rootfs.vfs image (not a - // fixture this package ships) so the run stays self-contained. - io: new NodePlatformIO(), + rootfsImage: intlRootfsImage, }); expect(exitCode).toBe(0); expect(stdout.toLowerCase()).not.toContain("intl"); @@ -71,10 +78,9 @@ describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { it("loads intl.so at runtime via extension=", async () => { const { stdout, exitCode } = await runCentralizedProgram({ programPath: phpBinaryPath, - argv: ["php", "-d", `extension=${intlSoPath}`, "-r", + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", 'echo extension_loaded("intl") ? "intl-loaded" : "intl-missing";'], - env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], - io: new NodePlatformIO(), + rootfsImage: intlRootfsImage, }); expect(stdout).toContain("intl-loaded"); expect(exitCode).toBe(0); @@ -85,10 +91,9 @@ describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { it("intl uses ICU data (Locale::getDisplayLanguage)", async () => { const { stdout, exitCode } = await runCentralizedProgram({ programPath: phpBinaryPath, - argv: ["php", "-d", `extension=${intlSoPath}`, "-r", + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", 'echo Locale::getDisplayLanguage("fr", "en");'], - env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], - io: new NodePlatformIO(), + rootfsImage: intlRootfsImage, }); expect(stdout).toContain("French"); expect(exitCode).toBe(0); @@ -98,14 +103,13 @@ describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { it("intl Collator sorts with locale rules", async () => { const { stdout, exitCode } = await runCentralizedProgram({ programPath: phpBinaryPath, - argv: ["php", "-d", `extension=${intlSoPath}`, "-r", ` + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", ` $c = new Collator("en_US"); $a = ["banana", "apple", "cherry"]; $c->sort($a); echo implode(",", $a); `], - env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], - io: new NodePlatformIO(), + rootfsImage: intlRootfsImage, }); expect(stdout).toContain("apple,banana,cherry"); expect(exitCode).toBe(0); @@ -118,7 +122,7 @@ describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { it("intl and icu.dat survive pcntl_fork replay", async () => { const { stdout, stderr, exitCode } = await runCentralizedProgram({ programPath: phpBinaryPath, - argv: ["php", "-d", `extension=${intlSoPath}`, "-r", ` + argv: ["php", "-d", `extension=${INTL_GUEST_PATH}`, "-r", ` $before = Locale::getDisplayLanguage("fr", "en"); $pid = pcntl_fork(); if ($pid < 0) { fwrite(STDERR, "fork-failed"); exit(20); } @@ -138,8 +142,7 @@ describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { exit(22); } `], - env: [`KANDELO_ICU_DAT_PATH=${icuDatPath}`], - io: new NodePlatformIO(), + rootfsImage: intlRootfsImage, }); expect(stderr).toBe(""); expect(stdout).toContain("child=French"); diff --git a/scripts/run-php-upstream-tests.ts b/scripts/run-php-upstream-tests.ts index d83bf9b62..b2afb9d2d 100644 --- a/scripts/run-php-upstream-tests.ts +++ b/scripts/run-php-upstream-tests.ts @@ -43,6 +43,7 @@ import { tryResolveBinary } from "../host/src/binary-resolver"; import { ABI_SYSCALL_NAMES } from "../host/src/generated/abi"; import { ensureSourceExtract } from "../images/vfs/scripts/source-extract-helper"; import { preparePhpTestFixtures } from "../images/vfs/scripts/php-test-fixtures"; +import { resolvePackageRuntimeFile } from "./package-runtime-file"; const REPO_ROOT = resolve(new URL(".", import.meta.url).pathname, ".."); const LOCAL_PHP_SRC = join(REPO_ROOT, "packages/registry/php/php-src"); @@ -55,6 +56,7 @@ const BROWSER_DIR = join(REPO_ROOT, "apps/browser-demos"); const VITE_HOST = "127.0.0.1"; const VITE_PORT = Number(process.env.PHP_TEST_VITE_PORT ?? 5201); const BROWSER_EXTENSION_DIR = "/usr/lib/php/extensions"; +const PHP_ICU_RUNTIME = resolvePackageRuntimeFile(REPO_ROOT, "php", "icu.dat"); const RUN_TESTS_BASE_INI = [ "output_handler=", "open_basedir=", @@ -989,6 +991,26 @@ class NodePhpRunner implements PhpRunner { copyFileSync(srcPath, destPath); chmodSync(destPath, 0o755); } + if (this.sharedExtensionPaths.has("intl")) { + if (!PHP_ICU_RUNTIME) { + throw new Error( + "intl.so is available but the declared php:icu.dat runtime file is not materialized", + ); + } + const mountPoint = "/usr/lib"; + if (!PHP_ICU_RUNTIME.guestPath.startsWith(`${mountPoint}/`)) { + throw new Error( + `php:icu.dat guest path ${PHP_ICU_RUNTIME.guestPath} is outside ${mountPoint}`, + ); + } + const runtimeDest = join( + root, + ...PHP_ICU_RUNTIME.guestPath.slice(mountPoint.length + 1).split("/"), + ); + mkdirSync(dirname(runtimeDest), { recursive: true }); + copyFileSync(PHP_ICU_RUNTIME.hostPath, runtimeDest); + chmodSync(runtimeDest, PHP_ICU_RUNTIME.mode); + } this.extensionMountRoot = root; return root; } From 1006904374e0d0688cf87eb9f3f1c829483d7b41 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 05:30:48 -0400 Subject: [PATCH 11/12] packages: keep runtime closures in one artifact tier Reject unsafe manifest and dependency path components before registry, cache, and archive interpolation, and make package.toml's top-level schema fail closed so a misspelled runtime_files declaration cannot disappear silently. Expose every program output and runtime file in structured runtime metadata. Resolve the complete set from one local, fetched, or installed-package root, allowing whole-tier fallback while refusing mixed partial closures. Route the PHP Node, VFS, and browser consumers through that selected closure. Validation: - pkg_manifest focused suite: 116 passed - runtime metadata focused test: 1 passed - binary resolver/runtime metadata Vitest: 14 passed - live build-deps manifest check: passed - host declaration build: passed - PHP browser Vite production build: passed with ABI-19 kernel/rootfs fixtures - ABI snapshot/version check: passed - full xtask suite: 342 passed; the same 7 pre-existing archive/program fixture failures remain The full host, libc, PHPT, and manual browser suites were not run for this focused repair. Performance was not measured. --- docs/package-management.md | 18 ++- host/src/binary-resolver.ts | 124 ++++++++++++++--- host/test/binary-resolver.test.ts | 125 +++++++++++++++++- host/test/package-runtime-file.test.ts | 64 +++++++++ .../vfs/scripts/build-php-test-vfs-image.ts | 10 +- .../php/test/browser/php-browser.spec.ts | 10 +- .../registry/php/test/browser/vite.config.ts | 10 +- packages/registry/php/test/php-intl.test.ts | 14 +- scripts/package-runtime-file.ts | 54 ++++++-- scripts/run-php-upstream-tests.ts | 21 ++- tools/xtask/src/build_deps.rs | 61 +++++++++ tools/xtask/src/pkg_manifest.rs | 104 +++++++++++++++ 12 files changed, 556 insertions(+), 59 deletions(-) create mode 100644 host/test/package-runtime-file.test.ts diff --git a/docs/package-management.md b/docs/package-management.md index cfb685af8..6a6e8a145 100644 --- a/docs/package-management.md +++ b/docs/package-management.md @@ -150,8 +150,8 @@ The split is load-bearing post the Required fields: ```toml -name = "zlib" # logical library name -version = "1.3.1" # upstream version +name = "zlib" # logical library name; one safe path component +version = "1.3.1" # upstream version; one safe path component depends_on = [] # ["zlib@1.3.1", ...] — exact versions, no ranges [source] @@ -207,7 +207,19 @@ Repo-side VFS/test builders query the authoritative path and mode with `xtask build-deps runtime-file-metadata `; they must not scan library caches or invent environment-only guest paths. Published VFS images contain the installed bytes already, so this query is a build-tool -contract rather than a runtime host API. +contract rather than a runtime host API. The structured metadata also lists +the package's complete mirror closure (every `[[outputs]]` artifact plus every +`[[runtime_files]]` file). Repo-side consumers resolve that set from one +complete provenance root: a partial local override may fall back wholesale to +a complete fetched package, but local, fetched, and installed-package tiers +are never combined. If artifacts exist but no tier has the complete accepted +closure, resolution fails loudly. + +Top-level keys are closed-schema: misspellings such as `[[runtime_file]]` +(singular) are rejected instead of silently dropping a runtime dependency. +Package names, versions, dependency names, and exact dependency-version tokens +must each be safe single filesystem components; `/`, `\`, NUL, `.` and `..` +spellings are rejected before cache, archive, or registry path construction. `package.toml` **must NOT** carry `revision`, `[binary]`, `[build].repo_url`, or `[build].commit`. Those moved to `build.toml` diff --git a/host/src/binary-resolver.ts b/host/src/binary-resolver.ts index 1a7cbf07f..f309f24f3 100644 --- a/host/src/binary-resolver.ts +++ b/host/src/binary-resolver.ts @@ -111,20 +111,64 @@ function applyDefaultArch(relPath: string): string { return `programs/wasm32/${tail}`; } -function packagedBinaryCandidates(relPath: string): string[] { - const root = packageRoot(); +function packagedBinaryCandidates( + relPath: string, + root = join(packageRoot(), "wasm"), +): string[] { const adjusted = applyDefaultArch(relPath); - const candidates = [join(root, "wasm", adjusted)]; + const candidates = [join(root, adjusted)]; if (relPath === "kernel.wasm") { - candidates.push(join(root, "wasm", "kandelo-kernel.wasm")); + candidates.push(join(root, "kandelo-kernel.wasm")); } else if (relPath === "userspace.wasm") { - candidates.push(join(root, "wasm", "wasm_posix_userspace.wasm")); + candidates.push(join(root, "wasm_posix_userspace.wasm")); } else if (relPath === "rootfs.vfs") { - candidates.push(join(root, "wasm", "rootfs.vfs")); + candidates.push(join(root, "rootfs.vfs")); } return candidates; } +interface BinaryCandidateTier { + label: string; + root: string; + candidatesFor(relPath: string): string[]; +} + +/** + * Ordered provenance roots used by both single-artifact and package-closure + * resolution. Keeping the grouping explicit lets a closure fall back as a + * unit without ever combining local, fetched, and installed-package bytes. + */ +function binaryCandidateTiers(): BinaryCandidateTier[] { + const tiers: BinaryCandidateTier[] = []; + try { + const repo = findRepoRoot(); + for (const [label, root] of [ + ["local-binaries", join(repo, "local-binaries")], + ["binaries", join(repo, "binaries")], + ] as const) { + tiers.push({ + label, + root, + candidatesFor(relPath: string): string[] { + return [join(root, applyDefaultArch(relPath))]; + }, + }); + } + } catch { + // Installed npm consumers do not carry a source repo root. + } + + const root = join(packageRoot(), "wasm"); + tiers.push({ + label: "installed package", + root, + candidatesFor(relPath: string): string[] { + return packagedBinaryCandidates(relPath, root); + }, + }); + return tiers; +} + let cachedForkInstrumentationDisabledOutputs: Set | null = null; interface ProgramOutputPolicy { @@ -284,22 +328,11 @@ export function resolveBinary(relPath: string): string { const adjusted = applyDefaultArch(relPath); const checked: string[] = []; const candidates: string[] = []; - try { - const repo = findRepoRoot(); - const local = join(repo, "local-binaries", adjusted); - checked.push(local); - candidates.push(local); - const fetched = join(repo, "binaries", adjusted); - checked.push(fetched); - candidates.push(fetched); - } catch { - // Installed npm consumers do not have a source repo root. Fall - // through to packaged assets below. - } - const packaged = packagedBinaryCandidates(relPath); - for (const candidate of packaged) { - checked.push(candidate); - candidates.push(candidate); + for (const tier of binaryCandidateTiers()) { + for (const candidate of tier.candidatesFor(relPath)) { + checked.push(candidate); + candidates.push(candidate); + } } const candidate = chooseBinaryCandidate(candidates, relPath); if (candidate) return candidate; @@ -322,6 +355,53 @@ export function tryResolveBinary(relPath: string): string | null { } } +/** + * Resolve a related artifact set from one complete provenance tier. + * + * A partial or policy-invalid local tier is skipped as a whole when a later + * tier is complete. If artifacts exist across the candidate roots but no + * single root contains an accepted complete set, this throws instead of + * silently composing a package from unrelated builds. It returns `null` only + * when none of the requested artifacts exists in any tier. + * + * Returned paths preserve `relPaths` order and are guaranteed to share the + * same local, fetched, or installed-package root. + */ +export function tryResolveBinarySet(relPaths: readonly string[]): string[] | null { + if (relPaths.length === 0) return []; + + let anyExisting = false; + const incomplete: string[] = []; + for (const tier of binaryCandidateTiers()) { + const selected: string[] = []; + const unavailable: string[] = []; + for (const relPath of relPaths) { + const candidates = tier.candidatesFor(relPath); + const existing = candidates.filter((candidate) => existsSync(candidate)); + anyExisting ||= existing.length > 0; + const candidate = chooseBinaryCandidate(candidates, relPath); + if (candidate) { + selected.push(candidate); + } else if (existing.length > 0) { + unavailable.push(`${relPath} (rejected by artifact policy)`); + } else { + unavailable.push(`${relPath} (missing)`); + } + } + if (unavailable.length === 0) return selected; + incomplete.push( + ` ${tier.label} (${tier.root}): ${unavailable.join(", ")}`, + ); + } + + if (!anyExisting) return null; + throw new Error( + "Package artifact closure is incomplete: no single provenance tier " + + "contains every accepted artifact, and tiers will not be mixed.\n" + + incomplete.join("\n"), + ); +} + /** Returns the absolute path of binaries/ whether or not it exists. */ export function binariesDir(): string { return join(findRepoRoot(), "binaries"); diff --git a/host/test/binary-resolver.test.ts b/host/test/binary-resolver.test.ts index cc8ab229b..ce77e9de3 100644 --- a/host/test/binary-resolver.test.ts +++ b/host/test/binary-resolver.test.ts @@ -7,6 +7,7 @@ import { binariesDir, localBinariesDir, resolveBinary, + tryResolveBinarySet, } from "../src/binary-resolver"; import { ABI_VERSION } from "../src/generated/abi"; import { @@ -104,7 +105,7 @@ async function vfsImage( return compressed ? new Uint8Array(zstdCompressSync(image)) : image; } -function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst" | ".dat"): string { +function fixtureClosureRelPaths(names: readonly string[]): string[] { const testRoot = "programs/wasm32/__binary_resolver_test__"; const dir = `${testRoot}/${randomUUID()}`; cleanupDirs.add(join(localBinariesDir(), dir)); @@ -115,7 +116,11 @@ function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst" | ".dat"): stri cleanupEmptyDirs.add(join(root, "programs")); cleanupEmptyDirs.add(root); } - return `${dir}/artifact${extension}`; + return names.map((name) => `${dir}/${name}`); +} + +function fixtureRelPath(extension: ".wasm" | ".vfs" | ".vfs.zst" | ".dat"): string { + return fixtureClosureRelPaths([`artifact${extension}`])[0]; } function candidatePath(root: string, relPath: string): string { @@ -207,3 +212,119 @@ describe("binary resolver artifact policy", () => { expect(resolveBinary(relPath)).toBe(localPath); }); }); + +describe("binary resolver package closures", () => { + it("returns a complete local closure from one provenance root", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + const wasmPath = writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const dataPath = writeCandidate( + localBinariesDir(), + dataRel, + new TextEncoder().encode("local-runtime"), + ); + writeCandidate( + binariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(tryResolveBinarySet([wasmRel, dataRel])).toEqual([wasmPath, dataPath]); + }); + + it("falls back wholesale from a partial local closure to complete fetched bytes", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const fetchedWasm = writeCandidate( + binariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const fetchedData = writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(tryResolveBinarySet([wasmRel, dataRel])).toEqual([ + fetchedWasm, + fetchedData, + ]); + }); + + it("rejects complementary partial tiers instead of mixing a closure", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(() => tryResolveBinarySet([wasmRel, dataRel])).toThrow( + /no single provenance tier.*tiers will not be mixed/s, + ); + }); + + it("falls back wholesale when a local closure member fails artifact policy", () => { + const [wasmRel, dataRel] = fixtureClosureRelPaths([ + "program.wasm", + "runtime.dat", + ]); + writeCandidate( + localBinariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION - 1), + ); + writeCandidate( + localBinariesDir(), + dataRel, + new TextEncoder().encode("local-runtime"), + ); + const fetchedWasm = writeCandidate( + binariesDir(), + wasmRel, + executableWasmWithAbi(ABI_VERSION), + ); + const fetchedData = writeCandidate( + binariesDir(), + dataRel, + new TextEncoder().encode("fetched-runtime"), + ); + + expect(tryResolveBinarySet([wasmRel, dataRel])).toEqual([ + fetchedWasm, + fetchedData, + ]); + }); + + it("returns null only when no closure member exists in any tier", () => { + const relPaths = fixtureClosureRelPaths(["program.wasm", "runtime.dat"]); + expect(tryResolveBinarySet(relPaths)).toBeNull(); + }); +}); diff --git a/host/test/package-runtime-file.test.ts b/host/test/package-runtime-file.test.ts new file mode 100644 index 000000000..47dc49007 --- /dev/null +++ b/host/test/package-runtime-file.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { + parsePackageRuntimeFileContract, + readPackageRuntimeFileContract, +} from "../../scripts/package-runtime-file"; +import { findRepoRoot } from "../src/binary-resolver"; + +function metadata(overrides: Record = {}): string { + return JSON.stringify({ + artifact: "icu.dat", + guest_path: "/usr/lib/php/icu.dat", + mode: 0o644, + mirror_path: "php/icu.dat", + closure_mirror_paths: ["php/php.wasm", "php/intl.so", "php/icu.dat"], + ...overrides, + }); +} + +describe("package runtime-file closure metadata", () => { + it("reports every declared PHP output and runtime file", () => { + const contract = readPackageRuntimeFileContract( + findRepoRoot(), + "php", + "icu.dat", + ); + expect(contract.closureMirrorPaths).toEqual([ + "php/php.wasm", + "php/php-fpm.wasm", + "php/opcache.so", + "php/curl.so", + "php/phar.so", + "php/zend_test.so", + "php/zip.so", + "php/intl.so", + "php/icu.dat", + ]); + }, 120_000); + + it("rejects duplicate or incomplete closure path metadata", () => { + expect(() => parsePackageRuntimeFileContract( + metadata({ + closure_mirror_paths: ["php/php.wasm", "php/icu.dat", "php/icu.dat"], + }), + "php", + "icu.dat", + )).toThrow(/invalid runtime-file metadata/); + + expect(() => parsePackageRuntimeFileContract( + metadata({ closure_mirror_paths: ["php/php.wasm", "php/intl.so"] }), + "php", + "icu.dat", + )).toThrow(/invalid runtime-file metadata/); + }); + + it("rejects closure mirror traversal before binary resolution", () => { + expect(() => parsePackageRuntimeFileContract( + metadata({ + closure_mirror_paths: ["php/php.wasm", "../outside", "php/icu.dat"], + }), + "php", + "icu.dat", + )).toThrow(/invalid runtime-file metadata/); + }); +}); diff --git a/images/vfs/scripts/build-php-test-vfs-image.ts b/images/vfs/scripts/build-php-test-vfs-image.ts index a9a23552f..68f66178d 100644 --- a/images/vfs/scripts/build-php-test-vfs-image.ts +++ b/images/vfs/scripts/build-php-test-vfs-image.ts @@ -39,12 +39,12 @@ import { resolvePackageRuntimeFile } from "../../../scripts/package-runtime-file const REPO_ROOT = findRepoRoot(); const PHP_FIXTURE_ROOT = join(REPO_ROOT, "tests/php-fixtures"); const LOCAL_PHP_SRC = join(REPO_ROOT, "packages/registry/php/php-src"); +const ICU_RUNTIME = resolvePackageRuntimeFile(REPO_ROOT, "php", "icu.dat"); const PHP_WASM = process.env.PHP_WASM - ?? tryResolveBinary("programs/php/php.wasm") + ?? ICU_RUNTIME?.closureHostPaths.get("php/php.wasm") ?? join(LOCAL_PHP_SRC, "sapi/cli/php"); const OPCACHE_SO = process.env.PHP_OPCACHE_SO - ?? tryResolveBinary("programs/php/opcache.so"); -const ICU_RUNTIME = resolvePackageRuntimeFile(REPO_ROOT, "php", "icu.dat"); + ?? ICU_RUNTIME?.closureHostPaths.get("php/opcache.so"); const PHP_EXTENSION_DIRS = [ dirname(PHP_WASM), ...((process.env.PHP_EXTENSION_DIR ?? "") @@ -52,13 +52,13 @@ const PHP_EXTENSION_DIRS = [ .map((path) => path.trim()) .filter(Boolean)), ]; -const INTL_SO = tryResolveBinary("programs/php/intl.so") +const INTL_SO = ICU_RUNTIME?.closureHostPaths.get("php/intl.so") ?? [...PHP_EXTENSION_DIRS] .reverse() .map((dir) => join(dir, "intl.so")) .find((path) => existsSync(path)); const PHP_FPM_WASM = process.env.PHP_FPM_WASM - ?? tryResolveBinary("programs/php/php-fpm.wasm"); + ?? ICU_RUNTIME?.closureHostPaths.get("php/php-fpm.wasm"); const ROOTFS_VFS = process.env.ROOTFS_VFS ?? tryResolveBinary("rootfs.vfs") ?? tryResolveBinary("programs/rootfs.vfs") diff --git a/packages/registry/php/test/browser/php-browser.spec.ts b/packages/registry/php/test/browser/php-browser.spec.ts index ec59da5d0..f0e81012e 100644 --- a/packages/registry/php/test/browser/php-browser.spec.ts +++ b/packages/registry/php/test/browser/php-browser.spec.ts @@ -23,16 +23,16 @@ import { tryResolveBinary } from "../../../../../host/src/binary-resolver"; const __dirname = dirname(fileURLToPath(import.meta.url)); const repoRoot = join(__dirname, "../../../../.."); +const phpIcuRuntime = resolvePackageRuntimeFile(repoRoot, "php", "icu.dat"); const hasKernelWasm = tryResolveBinary("kernel.wasm") != null; -const hasPhpWasm = tryResolveBinary("programs/php/php.wasm") != null +const hasPhpWasm = phpIcuRuntime?.closureHostPaths.has("php/php.wasm") === true || existsSync(join(repoRoot, "packages/registry/php/php-src/sapi/cli/php")); -const hasZipSo = tryResolveBinary("programs/php/zip.so") != null +const hasZipSo = phpIcuRuntime?.closureHostPaths.has("php/zip.so") === true || existsSync(join(repoRoot, "packages/registry/php/bin/zip.so")); -const hasCurlSo = tryResolveBinary("programs/php/curl.so") != null +const hasCurlSo = phpIcuRuntime?.closureHostPaths.has("php/curl.so") === true || existsSync(join(repoRoot, "packages/registry/php/bin/curl.so")); -const hasIntlSo = tryResolveBinary("programs/php/intl.so") != null +const hasIntlSo = phpIcuRuntime?.closureHostPaths.has("php/intl.so") === true || existsSync(join(repoRoot, "packages/registry/php/bin/intl.so")); -const phpIcuRuntime = resolvePackageRuntimeFile(repoRoot, "php", "icu.dat"); if (hasIntlSo && !phpIcuRuntime) { throw new Error( "PHP intl.so is present but the declared php:icu.dat runtime file is not materialized", diff --git a/packages/registry/php/test/browser/vite.config.ts b/packages/registry/php/test/browser/vite.config.ts index 42d8d3db8..87f4aa688 100644 --- a/packages/registry/php/test/browser/vite.config.ts +++ b/packages/registry/php/test/browser/vite.config.ts @@ -46,9 +46,13 @@ function resolveKernelArtifactsAlias(): Plugin { } function findPhpArtifact(name: string): string | null { - if (name === "icu.dat") return phpIcuRuntime?.hostPath ?? null; - const resolved = tryResolveBinary(`programs/php/${name}`); - if (resolved) return resolved; + if (phpIcuRuntime) { + // All PHP artifacts served by this harness come from the single complete + // package tier selected with icu.dat. Never let per-file resolver priority + // compose local and fetched builds. + return phpIcuRuntime.closureHostPaths.get(`php/${name}`) ?? null; + } + if (name === "icu.dat") return null; const candidates = [ path.resolve(__dirname, "../../bin", name), ...(name === "php.wasm" diff --git a/packages/registry/php/test/php-intl.test.ts b/packages/registry/php/test/php-intl.test.ts index 4d5172eaf..c5897563a 100644 --- a/packages/registry/php/test/php-intl.test.ts +++ b/packages/registry/php/test/php-intl.test.ts @@ -12,24 +12,24 @@ import { import { resolvePackageRuntimeFile } from "../../../../scripts/package-runtime-file"; const __dirname = dirname(fileURLToPath(import.meta.url)); +const icuRuntime = resolvePackageRuntimeFile( + join(__dirname, "../../../.."), + "php", + "icu.dat", +); // intl is a RUNTIME-OPTIONAL side module: base php.wasm is built with // --enable-intl=shared, so intl is NOT compiled in. intl.so is loaded on // demand via `extension=intl.so`, and pulls its ICU common data from the // separate icu.dat at runtime (udata_setCommonData in intl-icu-data-loader.c). const phpBinaryPath = - tryResolveBinary("programs/php/php.wasm") ?? + icuRuntime?.closureHostPaths.get("php/php.wasm") ?? join(__dirname, "../php-src/sapi/cli/php"); -const intlSoPath = tryResolveBinary("programs/php/intl.so"); +const intlSoPath = icuRuntime?.closureHostPaths.get("php/intl.so"); const rootfsPath = tryResolveBinary("rootfs.vfs") ?? tryResolveBinary("programs/rootfs.vfs") ?? join(__dirname, "../../../../host/wasm/rootfs.vfs"); -const icuRuntime = resolvePackageRuntimeFile( - join(__dirname, "../../../.."), - "php", - "icu.dat", -); const INTL_GUEST_PATH = "/usr/lib/php/extensions/intl.so"; if (intlSoPath && !icuRuntime) { diff --git a/scripts/package-runtime-file.ts b/scripts/package-runtime-file.ts index b6637f836..2045a36b4 100644 --- a/scripts/package-runtime-file.ts +++ b/scripts/package-runtime-file.ts @@ -7,19 +7,22 @@ * TypeScript fixtures. */ import { execFileSync } from "node:child_process"; -import { existsSync } from "node:fs"; import { isAbsolute } from "node:path"; -import { tryResolveBinary } from "../host/src/binary-resolver"; +import { tryResolveBinarySet } from "../host/src/binary-resolver"; export interface PackageRuntimeFileContract { artifact: string; guestPath: string; mode: number; mirrorPath: string; + /** Every program output + runtime file declared by this package. */ + closureMirrorPaths: string[]; } export interface ResolvedPackageRuntimeFile extends PackageRuntimeFileContract { hostPath: string; + /** Host paths keyed by resolver mirror path, all from one provenance root. */ + closureHostPaths: ReadonlyMap; } let cachedHostTarget: string | undefined; @@ -72,13 +75,31 @@ export function readPackageRuntimeFileContract( ], { cwd: repoRoot, encoding: "utf8", env: hostCargoEnv() }, ).trim(); + return parsePackageRuntimeFileContract(raw, packageName, artifact); +} + +/** Parse and validate xtask's structured runtime-file metadata. */ +export function parsePackageRuntimeFileContract( + raw: string, + packageName: string, + artifact: string, +): PackageRuntimeFileContract { const parsed = JSON.parse(raw) as Record; const contract: PackageRuntimeFileContract = { artifact: parsed.artifact as string, guestPath: parsed.guest_path as string, mode: parsed.mode as number, mirrorPath: parsed.mirror_path as string, + closureMirrorPaths: parsed.closure_mirror_paths as string[], }; + const validMirrorPath = (value: unknown): value is string => + typeof value === "string" + && !isAbsolute(value) + && !value.includes("\\") + && !value.includes("\0") + && value + .split("/") + .every((part) => Boolean(part) && part !== "." && part !== ".."); if ( contract.artifact !== artifact || typeof contract.guestPath !== "string" @@ -86,9 +107,12 @@ export function readPackageRuntimeFileContract( || !Number.isInteger(contract.mode) || contract.mode < 0 || contract.mode > 0o777 - || typeof contract.mirrorPath !== "string" - || isAbsolute(contract.mirrorPath) - || contract.mirrorPath.split("/").some((part) => !part || part === "." || part === "..") + || !validMirrorPath(contract.mirrorPath) + || !Array.isArray(contract.closureMirrorPaths) + || contract.closureMirrorPaths.length === 0 + || !contract.closureMirrorPaths.every(validMirrorPath) + || new Set(contract.closureMirrorPaths).size !== contract.closureMirrorPaths.length + || !contract.closureMirrorPaths.includes(contract.mirrorPath) ) { throw new Error( `invalid runtime-file metadata for ${packageName}:${artifact}: ${raw}`, @@ -103,7 +127,21 @@ export function resolvePackageRuntimeFile( artifact: string, ): ResolvedPackageRuntimeFile | undefined { const contract = readPackageRuntimeFileContract(repoRoot, packageName, artifact); - const hostPath = tryResolveBinary(`programs/${contract.mirrorPath}`); - if (!hostPath || !existsSync(hostPath)) return undefined; - return { ...contract, hostPath }; + const hostPaths = tryResolveBinarySet( + contract.closureMirrorPaths.map((mirrorPath) => `programs/${mirrorPath}`), + ); + if (!hostPaths) return undefined; + const closureHostPaths = new Map( + contract.closureMirrorPaths.map((mirrorPath, index) => [ + mirrorPath, + hostPaths[index], + ]), + ); + const hostPath = closureHostPaths.get(contract.mirrorPath); + if (!hostPath) { + throw new Error( + `resolved package closure omitted ${packageName}:${contract.mirrorPath}`, + ); + } + return { ...contract, hostPath, closureHostPaths }; } diff --git a/scripts/run-php-upstream-tests.ts b/scripts/run-php-upstream-tests.ts index b2afb9d2d..3e39a135d 100644 --- a/scripts/run-php-upstream-tests.ts +++ b/scripts/run-php-upstream-tests.ts @@ -39,7 +39,6 @@ import { sep, } from "node:path"; import { NodeKernelHost } from "../host/src/node-kernel-host"; -import { tryResolveBinary } from "../host/src/binary-resolver"; import { ABI_SYSCALL_NAMES } from "../host/src/generated/abi"; import { ensureSourceExtract } from "../images/vfs/scripts/source-extract-helper"; import { preparePhpTestFixtures } from "../images/vfs/scripts/php-test-fixtures"; @@ -219,7 +218,7 @@ async function withTimeout( function resolvePhpBinary(): string { const candidate = process.env.PHP_WASM ?? - tryResolveBinary("programs/php/php.wasm") ?? + PHP_ICU_RUNTIME?.closureHostPaths.get("php/php.wasm") ?? join(LOCAL_PHP_SRC, "sapi/cli/php"); if (!candidate || !existsSync(candidate)) { throw new Error( @@ -232,7 +231,7 @@ function resolvePhpBinary(): string { function resolvePhpFpmBinary(phpPath: string): string | null { const explicit = process.env.PHP_FPM_WASM; if (explicit) return resolve(explicit); - const resolved = tryResolveBinary("programs/php/php-fpm.wasm"); + const resolved = PHP_ICU_RUNTIME?.closureHostPaths.get("php/php-fpm.wasm"); if (resolved) return resolved; const sibling = join(dirname(phpPath), "php-fpm.wasm"); return existsSync(sibling) ? sibling : null; @@ -549,10 +548,24 @@ function sharedExtensionPathsForPhp(phpPath: string): Map { } } } + // When the declared package closure is materialized, use its selected + // side-module paths directly. They were resolved together with php.wasm and + // icu.dat from one complete provenance tier. + for (const name of [ + "opcache", + "curl", + "phar", + "zend_test", + "zip", + "intl", + ]) { + const resolved = PHP_ICU_RUNTIME?.closureHostPaths.get(`php/${name}.so`); + if (resolved) out.set(name, resolved); + } const phpDir = dirname(phpPath); const opcachePath = process.env.PHP_OPCACHE_SO ?? - tryResolveBinary("programs/php/opcache.so") ?? + PHP_ICU_RUNTIME?.closureHostPaths.get("php/opcache.so") ?? join(phpDir, "opcache.so"); if (opcachePath && existsSync(opcachePath)) out.set("opcache", opcachePath); return out; diff --git a/tools/xtask/src/build_deps.rs b/tools/xtask/src/build_deps.rs index 719da88f7..cd14e4657 100644 --- a/tools/xtask/src/build_deps.rs +++ b/tools/xtask/src/build_deps.rs @@ -3106,11 +3106,27 @@ fn runtime_file_metadata_value( m.name, artifact ) })?; + // A runtime file is meaningful only alongside the exact executable and + // side-module outputs produced by the same program package archive. Give + // repo-side consumers the complete resolver mirror closure so they can + // select one materialization tier atomically instead of resolving each + // member independently and accidentally mixing builds. + let closure_mirror_paths: Vec = m + .program_outputs + .iter() + .map(|output| m.output_dest_rel_for(output)) + .chain( + m.runtime_files + .iter() + .map(|runtime_file| m.runtime_file_dest_rel_for(runtime_file)), + ) + .collect(); Ok(serde_json::json!({ "artifact": runtime_file.artifact, "guest_path": runtime_file.guest_path, "mode": runtime_file.mode, "mirror_path": m.runtime_file_dest_rel_for(runtime_file), + "closure_mirror_paths": closure_mirror_paths, })) } @@ -6860,6 +6876,10 @@ printf runtime-data > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, "guest_path": "/usr/lib/php/icu.dat", "mode": 420, "mirror_path": "runtimeprog/icu.dat", + "closure_mirror_paths": [ + "runtimeprog.wasm", + "runtimeprog/icu.dat", + ], }) ); let path = @@ -6874,6 +6894,47 @@ printf runtime-data > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, ); } + #[test] + fn runtime_file_metadata_lists_the_complete_multi_output_closure() { + let manifest = DepsManifest::parse( + r#"kind = "program" +name = "runtimeprog" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/runtimeprog.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "runtimeprog" +wasm = "bin/runtimeprog.wasm" +[[outputs]] +name = "module" +wasm = "extensions/module.so" +[[runtime_files]] +artifact = "share/icu.dat" +guest_path = "/usr/lib/runtimeprog/icu.dat" +[[runtime_files]] +artifact = "share/timezone.dat" +guest_path = "/usr/lib/runtimeprog/timezone.dat" +"#, + PathBuf::from("/x"), + ) + .unwrap(); + + let metadata = runtime_file_metadata_value(&manifest, "share/icu.dat").unwrap(); + assert_eq!( + metadata["closure_mirror_paths"], + serde_json::json!([ + "runtimeprog/runtimeprog.wasm", + "runtimeprog/module.so", + "runtimeprog/share/icu.dat", + "runtimeprog/share/timezone.dat", + ]) + ); + } + #[test] fn build_fails_when_program_runtime_file_is_missing() { let root = tempdir("prog-runtime-file-missing"); diff --git a/tools/xtask/src/pkg_manifest.rs b/tools/xtask/src/pkg_manifest.rs index 6d3cd0788..fa0e1bc80 100644 --- a/tools/xtask/src/pkg_manifest.rs +++ b/tools/xtask/src/pkg_manifest.rs @@ -854,6 +854,14 @@ impl DepRef { if name.contains('@') { return Err(format!("dep name {:?} must not contain '@'", name)); } + // Registry lookup joins the name directly below each registry root; + // cache/spec logic also treats the exact version as an identity token. + // Reject traversal and nested-path spellings before either value can + // reach filesystem interpolation. + validate_single_path_component(name, "dep name") + .map_err(|e| format!("dep reference {s:?}: {e}"))?; + validate_single_path_component(version, "dep version") + .map_err(|e| format!("dep reference {s:?}: {e}"))?; Ok(Self { name: name.into(), version: version.into(), @@ -871,6 +879,7 @@ impl std::fmt::Display for DepRef { /// validated [`DepsManifest`] so normalization (default build script, /// parsed DepRefs, etc.) lives in one place. #[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] struct Raw { kind: ManifestKind, name: String, @@ -1358,6 +1367,11 @@ impl DepsManifest { if raw.version.is_empty() { return Err("version must not be empty".into()); } + // `version` is interpolated into canonical cache directories and + // archive filenames. Keep it to one portable filesystem component so + // a manifest cannot escape those roots or create an ambiguous nested + // path before any resolver/archive code sees it. + validate_single_path_component(&raw.version, "version")?; // revision: archived → declared (validate_archived rejects None // up-front); source → defaults to 1 since source manifests no // longer carry it. Downstream code reads m.revision; the @@ -1873,6 +1887,54 @@ cache_key_sha = "111111111111111111111111111111111111111111111111111111111111111 ); } + #[test] + fn source_and_archived_manifests_reject_unsafe_version_path_components() { + for version in [".", "..", "../escape", "nested/version", r"nested\version"] { + for (label, fixture, archived) in [ + ("source", EXAMPLE, false), + ("archived", EXAMPLE_ARCHIVED, true), + ] { + let text = fixture.replace( + "version = \"1.3.1\"", + &format!("version = {version:?}"), + ); + let result = if archived { + DepsManifest::parse_archived(&text, PathBuf::from("/x")) + } else { + DepsManifest::parse(&text, PathBuf::from("/x")) + }; + let err = result.expect_err("unsafe version must fail before path interpolation"); + assert!( + err.contains("version") && err.contains("safe single path component"), + "{label} version {version:?}: got {err}" + ); + } + } + } + + #[test] + fn source_and_archived_manifests_reject_unknown_top_level_fields() { + for (label, fixture, archived) in [ + ("source", EXAMPLE, false), + ("archived", EXAMPLE_ARCHIVED, true), + ] { + let text = fixture.replace( + "depends_on = []", + "depends_on = []\nfuture_recipe_field = true", + ); + let result = if archived { + DepsManifest::parse_archived(&text, PathBuf::from("/x")) + } else { + DepsManifest::parse(&text, PathBuf::from("/x")) + }; + let err = result.expect_err("unknown top-level field must fail loudly"); + assert!( + err.contains("future_recipe_field") && err.contains("unknown field"), + "{label}: got {err}" + ); + } + } + #[test] fn parses_library_runtime_file_outputs() { let text = EXAMPLE.replace( @@ -2192,6 +2254,36 @@ spdx = "TestLicense" assert!(DepRef::parse("zlib@").is_err()); } + #[test] + fn depref_rejects_registry_and_version_path_traversal() { + for reference in [ + "../escape@1.0", + "nested/package@1.0", + r"nested\package@1.0", + "zlib@../escape", + "zlib@nested/version", + r"zlib@nested\version", + ".@1.0", + "zlib@..", + ] { + let err = DepRef::parse(reference).unwrap_err(); + assert!( + err.contains("safe single path component"), + "{reference:?}: got {err}" + ); + } + + let text = EXAMPLE.replace( + "depends_on = []", + r#"depends_on = ["../outside@1.0"]"#, + ); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("dep name") && err.contains("safe single path component"), + "got: {err}" + ); + } + #[test] fn depends_on_parsed_into_deprefs() { let text = EXAMPLE.replace( @@ -2445,6 +2537,18 @@ wasm = "vim.wasm" assert!(m.runtime_files.is_empty()); } + #[test] + fn rejects_singular_runtime_file_table_typo() { + let text = format!( + "{PROGRAM_EXAMPLE}\n[[runtime_file]]\nartifact = \"icu.dat\"\nguest_path = \"/usr/lib/php/icu.dat\"\n" + ); + let err = DepsManifest::parse(&text, PathBuf::from("/x")).unwrap_err(); + assert!( + err.contains("runtime_file") && err.contains("unknown field"), + "got: {err}" + ); + } + fn program_with_runtime_file(artifact: &str, guest_path: &str, mode: Option) -> String { format!( "{PROGRAM_EXAMPLE}\n[[runtime_files]]\nartifact = {artifact:?}\nguest_path = {guest_path:?}\n{}", From 5ec4d899c19995837a26044520491b4433fd0ba3 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sun, 12 Jul 2026 05:38:55 -0400 Subject: [PATCH 12/12] vfs: reject incomplete image file staging Make the shared VFS image writer honor positive partial writes until all bytes are staged, reject zero/negative/invalid progress, and close its descriptor on every exit path. Route the browser-demo wrappers through that shared implementation. Give the Node PHP intl fixture a 256 MiB rebased filesystem before adding the 30.8 MB ICU data file and verify the staged bytes by size and SHA-256. Preserve the browser fixture's proven 256 MiB growable restore and byte-check its injected side-module/runtime fixtures without re-reading the large PHP executable on every invocation. Validation: - focused VFS image-helper tests: 5 passed - exact ABI-19 PHP intl Node suite: 5 passed, including ICU data and fork replay - exact ABI-19 Chromium intl/fork case: 1 passed - host declaration build: passed - PHP browser Vite production build: passed The full browser app typecheck was attempted and remains blocked by pre-existing unrelated type errors. Its production build was attempted and remains blocked by absent coreutils/node/nc binary assets in this isolated worktree. The full host, libc, and manual browser suites were not run. Performance was not measured. --- apps/browser-demos/lib/init/vfs-utils.ts | 38 +------ docs/architecture.md | 6 ++ host/src/vfs/image-helpers.ts | 23 +++- host/test/vfs/image-helpers.test.ts | 100 ++++++++++++++++++ packages/registry/php/test/browser/run-php.ts | 50 ++++++++- packages/registry/php/test/php-intl.test.ts | 43 +++++++- 6 files changed, 219 insertions(+), 41 deletions(-) create mode 100644 host/test/vfs/image-helpers.test.ts diff --git a/apps/browser-demos/lib/init/vfs-utils.ts b/apps/browser-demos/lib/init/vfs-utils.ts index b7f78e0b9..8adb13779 100644 --- a/apps/browser-demos/lib/init/vfs-utils.ts +++ b/apps/browser-demos/lib/init/vfs-utils.ts @@ -5,39 +5,10 @@ * and are used by demo build scripts that construct VFS images. */ import type { MemoryFileSystem } from "../../../../host/src/vfs/memory-fs"; - -const encoder = new TextEncoder(); - -/** - * Write a text file to the VFS. Opens with O_WRONLY|O_CREAT|O_TRUNC, - * writes the encoded content, and closes the fd. - */ -export function writeVfsFile( - fs: MemoryFileSystem, - path: string, - content: string, - mode = 0o644, -): void { - const data = encoder.encode(content); - const fd = fs.open(path, 0o1101, mode); // O_WRONLY | O_CREAT | O_TRUNC - fs.write(fd, data, 0, data.length); - fs.close(fd); -} - -/** - * Write a binary file to the VFS. Opens with O_WRONLY|O_CREAT|O_TRUNC, - * writes the raw bytes, and closes the fd. - */ -export function writeVfsBinary( - fs: MemoryFileSystem, - path: string, - data: Uint8Array, - mode = 0o755, -): void { - const fd = fs.open(path, 0o1101, mode); // O_WRONLY | O_CREAT | O_TRUNC - fs.write(fd, data, 0, data.length); - fs.close(fd); -} +export { + writeVfsBinary, + writeVfsFile, +} from "../../../../host/src/vfs/image-helpers"; /** * Create a directory, ignoring EEXIST errors. @@ -79,4 +50,3 @@ export function ensureDirRecursive( ensureDir(fs, current, mode); } } - diff --git a/docs/architecture.md b/docs/architecture.md index 328dd391f..949923027 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -615,6 +615,12 @@ const restored = MemoryFileSystem.fromImage(image, { maxByteLength: 1024 * 1024 The image must also have been built with a large enough filesystem maximum, for example `MemoryFileSystem.create(sab, 1024 * 1024 * 1024)`. `fromImage(..., { maxByteLength })` only controls the restored buffer's runtime growth ceiling; `statfs`/`df` and allocation remain capped by the image superblock maximum. +A consumer that must stage files larger than the image's recorded allocation +ceiling first calls `rebaseToNewFileSystem(requiredMaxBytes)`. Shared image +helpers never treat a partial file as complete: `writeVfsBinary` advances over +positive short writes and throws on zero/negative progress or an underlying +filesystem error, while still closing the descriptor. + Kandelo browser UI presets use this approach. Each image builder pre-populates a VFS with runtime files, directory structure, configs, and symlinks, then saves it as a `.vfs.zst` file (zstd-compressed; `saveImage()` compresses on write). At runtime, the UI fetches the file and `MemoryFileSystem.fromImage` decompresses transparently - restoring the image replaces thousands of individual file writes with a single buffer copy. The empty regions of the SharedFS allocator compress to almost nothing, so a 32 MB filesystem with a few MB of real content typically ships as a 1-3 MB download. There are two consumption patterns for VFS images, depending on whether the demo wants the kernel worker to fully own the filesystem: diff --git a/host/src/vfs/image-helpers.ts b/host/src/vfs/image-helpers.ts index aa1ba8230..606c12798 100644 --- a/host/src/vfs/image-helpers.ts +++ b/host/src/vfs/image-helpers.ts @@ -28,8 +28,27 @@ export function writeVfsBinary( mode = 0o755, ): void { const fd = fs.open(path, O_WRONLY_CREAT_TRUNC, mode); - fs.write(fd, data, 0, data.length); - fs.close(fd); + try { + let offset = 0; + while (offset < data.length) { + const remaining = data.subarray(offset); + const written = fs.write(fd, remaining, offset, remaining.length); + if ( + !Number.isInteger(written) + || written <= 0 + || written > remaining.length + ) { + const detail = written < 0 + ? `write failed with error code ${written}` + : `write made invalid progress ` + + `(${written} of ${remaining.length} remaining bytes)`; + throw new Error(`Failed to stage complete VFS file ${path}: ${detail}`); + } + offset += written; + } + } finally { + fs.close(fd); + } } /** mkdir, swallowing EEXIST. */ diff --git a/host/test/vfs/image-helpers.test.ts b/host/test/vfs/image-helpers.test.ts new file mode 100644 index 000000000..547595e24 --- /dev/null +++ b/host/test/vfs/image-helpers.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFileSystem } from "../../src/vfs/memory-fs"; +import { writeVfsBinary, writeVfsFile } from "../../src/vfs/image-helpers"; +import { + writeVfsBinary as writeBrowserVfsBinary, + writeVfsFile as writeBrowserVfsFile, +} from "../../../apps/browser-demos/lib/init/vfs-utils"; + +const O_RDONLY = 0; + +function readFile(fs: MemoryFileSystem, path: string): Uint8Array { + const size = fs.stat(path).size; + const bytes = new Uint8Array(size); + const fd = fs.open(path, O_RDONLY, 0); + try { + const read = fs.read(fd, bytes, null, bytes.length); + if (read !== bytes.length) { + throw new Error(`short test read: ${read} of ${bytes.length}`); + } + } finally { + fs.close(fd); + } + return bytes; +} + +describe("VFS image write helpers", () => { + it("backs browser-demo writers with the shared strict helpers", () => { + expect(writeBrowserVfsBinary).toBe(writeVfsBinary); + expect(writeBrowserVfsFile).toBe(writeVfsFile); + }); + + it("stages every byte of a binary file", () => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + const data = new Uint8Array(256 * 1024); + for (let i = 0; i < data.length; i++) data[i] = i & 0xff; + + writeVfsBinary(fs, "/payload.bin", data, 0o640); + + expect(fs.stat("/payload.bin").size).toBe(data.length); + expect(readFile(fs, "/payload.bin")).toEqual(data); + }); + + it("reports terminal ENOSPC after preserving a positive partial write", () => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(128 * 1024)); + const data = new Uint8Array(1024 * 1024).fill(0xa5); + + expect(() => writeVfsBinary(fs, "/partial.bin", data)).toThrow(); + expect(fs.stat("/partial.bin").size).toBeGreaterThan(0); + expect(fs.stat("/partial.bin").size).toBeLessThan(data.length); + }); + + it("continues from the correct offset after a positive short write", () => { + const data = new Uint8Array([1, 2, 3, 4]); + const close = vi.fn(); + const write = vi.fn() + .mockReturnValueOnce(2) + .mockReturnValueOnce(2); + const fs = { + open: vi.fn(() => 7), + write, + close, + } as unknown as MemoryFileSystem; + + writeVfsBinary(fs, "/fixture.bin", data); + + expect(write).toHaveBeenCalledTimes(2); + expect(write.mock.calls[0][1]).toEqual(new Uint8Array([1, 2, 3, 4])); + expect(write.mock.calls[0].slice(2)).toEqual([0, 4]); + expect(write.mock.calls[1][1]).toEqual(new Uint8Array([3, 4])); + expect(write.mock.calls[1].slice(2)).toEqual([2, 2]); + expect(close).toHaveBeenCalledTimes(1); + expect(close).toHaveBeenCalledWith(7); + }); + + it("closes the descriptor for zero, negative, invalid, and thrown writes", () => { + const data = new Uint8Array([1, 2, 3, 4]); + const outcomes: Array = [ + 0, + -28, + 5, + new Error("ENOSPC"), + ]; + + for (const outcome of outcomes) { + const close = vi.fn(); + const fs = { + open: vi.fn(() => 7), + write: vi.fn(() => { + if (outcome instanceof Error) throw outcome; + return outcome; + }), + close, + } as unknown as MemoryFileSystem; + + expect(() => writeVfsBinary(fs, "/fixture.bin", data)).toThrow(); + expect(close).toHaveBeenCalledTimes(1); + expect(close).toHaveBeenCalledWith(7); + } + }); +}); diff --git a/packages/registry/php/test/browser/run-php.ts b/packages/registry/php/test/browser/run-php.ts index 107cd1dce..4e026ecc2 100644 --- a/packages/registry/php/test/browser/run-php.ts +++ b/packages/registry/php/test/browser/run-php.ts @@ -23,6 +23,9 @@ const statusEl = document.getElementById("status")!; const resultsEl = document.getElementById("results")!; const PHP_PATH = "/usr/local/bin/php"; +const PHP_BROWSER_VFS_MAX_BYTES = 256 * 1024 * 1024; +const O_RDONLY = 0; +const VERIFY_CHUNK_BYTES = 64 * 1024; interface TestResult { stdout: string; @@ -35,6 +38,44 @@ interface BinaryFixture { mode: number; } +function assertVfsBinaryRoundTrip( + fs: MemoryFileSystem, + path: string, + expected: Uint8Array, +): void { + const size = fs.stat(path).size; + if (size !== expected.byteLength) { + throw new Error( + `staged VFS file ${path} has ${size} bytes; expected ${expected.byteLength}`, + ); + } + + const fd = fs.open(path, O_RDONLY, 0); + const chunk = new Uint8Array(Math.min(VERIFY_CHUNK_BYTES, expected.byteLength)); + let offset = 0; + try { + while (offset < expected.byteLength) { + const wanted = Math.min(chunk.byteLength, expected.byteLength - offset); + const read = fs.read(fd, chunk, null, wanted); + if (read <= 0) { + throw new Error( + `short VFS verification read for ${path}: ${offset} of ${expected.byteLength}`, + ); + } + for (let i = 0; i < read; i++) { + if (chunk[i] !== expected[offset + i]) { + throw new Error( + `staged VFS file ${path} differs at byte ${offset + i}`, + ); + } + } + offset += read; + } + } finally { + fs.close(fd); + } +} + async function runPhp( phpBytes: ArrayBuffer, kernelBytes: ArrayBuffer, @@ -48,19 +89,22 @@ async function runPhp( const decoder = new TextDecoder(); const memfs = MemoryFileSystem.fromImage(new Uint8Array(rootfsBytes), { - maxByteLength: 256 * 1024 * 1024, + maxByteLength: PHP_BROWSER_VFS_MAX_BYTES, }); for (const dir of ["/tmp", "/root", "/home", "/dev"]) ensureDir(memfs, dir); memfs.chmod("/tmp", 0o777); memfs.chmod("/root", 0o700); ensureDirRecursive(memfs, "/usr/local/bin"); - writeVfsBinary(memfs, PHP_PATH, new Uint8Array(phpBytes)); + const phpData = new Uint8Array(phpBytes); + writeVfsBinary(memfs, PHP_PATH, phpData); for (const [path, content] of Object.entries(files)) { writeVfsFile(memfs, path, content); } for (const [path, fixture] of Object.entries(binaryFiles)) { ensureDirRecursive(memfs, path.slice(0, path.lastIndexOf("/")) || "/"); - writeVfsBinary(memfs, path, new Uint8Array(fixture.bytes), fixture.mode); + const data = new Uint8Array(fixture.bytes); + writeVfsBinary(memfs, path, data, fixture.mode); + assertVfsBinaryRoundTrip(memfs, path, data); } const vfsImage = await memfs.saveImage(); diff --git a/packages/registry/php/test/php-intl.test.ts b/packages/registry/php/test/php-intl.test.ts index c5897563a..9999a4348 100644 --- a/packages/registry/php/test/php-intl.test.ts +++ b/packages/registry/php/test/php-intl.test.ts @@ -1,4 +1,5 @@ import { beforeAll, describe, it, expect } from "vitest"; +import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; @@ -31,6 +32,8 @@ const rootfsPath = tryResolveBinary("programs/rootfs.vfs") ?? join(__dirname, "../../../../host/wasm/rootfs.vfs"); const INTL_GUEST_PATH = "/usr/lib/php/extensions/intl.so"; +const PHP_INTL_VFS_MAX_BYTES = 256 * 1024 * 1024; +const O_RDONLY = 0; if (intlSoPath && !icuRuntime) { throw new Error( @@ -43,9 +46,41 @@ const READY = existsSync(phpBinaryPath) && existsSync(rootfsPath); let intlRootfsImage: Uint8Array; +function readVfsBinary(fs: MemoryFileSystem, path: string): Uint8Array { + const size = fs.stat(path).size; + const bytes = new Uint8Array(size); + const fd = fs.open(path, O_RDONLY, 0); + let offset = 0; + try { + while (offset < bytes.length) { + const read = fs.read( + fd, + bytes.subarray(offset), + null, + bytes.length - offset, + ); + if (read <= 0) { + throw new Error( + `short VFS read for ${path}: ${offset} of ${bytes.length}`, + ); + } + offset += read; + } + } finally { + fs.close(fd); + } + return bytes; +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { beforeAll(async () => { - const fs = MemoryFileSystem.fromImage(new Uint8Array(readFileSync(rootfsPath))); + const fs = MemoryFileSystem + .fromImage(new Uint8Array(readFileSync(rootfsPath))) + .rebaseToNewFileSystem(PHP_INTL_VFS_MAX_BYTES); ensureDirRecursive(fs, dirname(INTL_GUEST_PATH)); ensureDirRecursive(fs, dirname(icuRuntime!.guestPath)); writeVfsBinary( @@ -54,12 +89,16 @@ describe.skipIf(!READY)("PHP intl as a runtime-loadable side module", () => { new Uint8Array(readFileSync(intlSoPath!)), 0o755, ); + const icuBytes = new Uint8Array(readFileSync(icuRuntime!.hostPath)); writeVfsBinary( fs, icuRuntime!.guestPath, - new Uint8Array(readFileSync(icuRuntime!.hostPath)), + icuBytes, icuRuntime!.mode, ); + const stagedIcuBytes = readVfsBinary(fs, icuRuntime!.guestPath); + expect(stagedIcuBytes.byteLength).toBe(icuBytes.byteLength); + expect(sha256(stagedIcuBytes)).toBe(sha256(icuBytes)); intlRootfsImage = await fs.saveImage(); });