From e694277a2cdfe0840a64396be5d62bc96e9baf84 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Jul 2026 16:54:35 -0700 Subject: [PATCH 1/3] feat: bundle native binaries in root package --- .github/workflows/ci.yml | 25 +-- .github/workflows/release.yml | 127 +++---------- .gitignore | 4 + CHANGELOG.md | 4 +- README.md | 20 +- RELEASE-PREREQS.md | 24 +-- docs/archive.md | 2 +- docs/errors.md | 2 +- docs/index.md | 2 +- docs/install.md | 8 +- docs/migrating-to-0.5.md | 15 +- docs/native-helper.md | 12 +- docs/native.md | 26 +-- docs/permissions.md | 4 +- docs/root.md | 2 +- docs/testing.md | 2 +- native/index.d.ts | 105 ----------- native/index.js | 179 ------------------ native/package.json | 35 +--- npm/darwin-arm64/package.json | 14 -- npm/darwin-x64/package.json | 14 -- npm/linux-arm64-gnu/package.json | 15 -- npm/linux-arm64-musl/package.json | 15 -- npm/linux-x64-gnu/package.json | 15 -- npm/linux-x64-musl/package.json | 15 -- npm/win32-x64-msvc/package.json | 14 -- package.json | 6 +- pnpm-lock.yaml | 41 +--- pnpm-workspace.yaml | 1 - scripts/assemble-native-binaries.mjs | 41 ++++ scripts/bench-publish.mjs | 5 +- scripts/check-release-packages.mjs | 209 +++++++++++--------- scripts/harden-native-loader.mjs | 45 ----- scripts/native-loader.cjs | 179 ------------------ scripts/native-mode-smoke.mjs | 14 +- scripts/native-smoke.mjs | 5 +- scripts/native-targets.mjs | 52 +++++ scripts/prepack-build.mjs | 35 ++-- scripts/release-graph-smoke.mjs | 241 ------------------------ scripts/stage-host-native.mjs | 20 ++ src/archive-kind.ts | 4 +- src/archive-read.ts | 2 +- src/archive.ts | 2 +- src/native-binding.ts | 121 ++++++++++++ src/native.ts | 174 ++++++++++++++++- src/owner-dacl.ts | 2 +- src/private-directory.ts | 4 +- test/native-archive-equivalence.test.ts | 5 +- test/native-integration.test.ts | 13 +- test/native-loader.test.ts | 21 ++- test/native-publish-equivalence.test.ts | 5 +- test/private-directory.test.ts | 5 +- test/symlink-dotdot-escape.test.ts | 8 +- 53 files changed, 708 insertions(+), 1252 deletions(-) delete mode 100644 native/index.d.ts delete mode 100644 native/index.js delete mode 100644 npm/darwin-arm64/package.json delete mode 100644 npm/darwin-x64/package.json delete mode 100644 npm/linux-arm64-gnu/package.json delete mode 100644 npm/linux-arm64-musl/package.json delete mode 100644 npm/linux-x64-gnu/package.json delete mode 100644 npm/linux-x64-musl/package.json delete mode 100644 npm/win32-x64-msvc/package.json create mode 100644 scripts/assemble-native-binaries.mjs delete mode 100644 scripts/harden-native-loader.mjs delete mode 100644 scripts/native-loader.cjs create mode 100644 scripts/native-targets.mjs delete mode 100644 scripts/release-graph-smoke.mjs create mode 100644 scripts/stage-host-native.mjs create mode 100644 src/native-binding.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef1dea5..6f746c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,12 +93,12 @@ jobs: - name: Test native crate run: cargo test --workspace --locked - - name: Build host binding - run: pnpm native:build - - name: Build package run: pnpm build + - name: Build and stage host binding + run: pnpm native:build + - name: Test native mode auto run: node scripts/native-mode-smoke.mjs auto @@ -126,8 +126,8 @@ jobs: npm install --global pnpm@10.34.5 pnpm install --frozen-lockfile cargo test --workspace --locked - pnpm native:build pnpm build + pnpm native:build node scripts/native-mode-smoke.mjs auto node scripts/native-mode-smoke.mjs require node scripts/native-mode-smoke.mjs off @@ -160,8 +160,8 @@ jobs: - name: Audit run: cargo audit --deny warnings - release-graph-smoke: - name: Release graph smoke (${{ matrix.os }}) + package-smoke: + name: Bundled package smoke (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: @@ -191,15 +191,18 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Build host binding + - name: Build package + run: pnpm build + + - name: Build and stage host binding run: pnpm native:build - - name: Pack and smoke test release graph - run: pnpm release-graph:smoke + - name: Pack and smoke test bundled package + run: pnpm package:smoke - name: Upload behavior proof uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: release-graph-proof-${{ runner.os }}-${{ runner.arch }} - path: release-graph-proof.json + name: bundled-package-proof-${{ runner.os }}-${{ runner.arch }} + path: release-artifacts/manifest.json if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bdddb28..7897179 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Validate tag, changelog, and package versions + - name: Validate tag, changelog, and versions env: RELEASE_TAG: ${{ github.ref_name }} run: | @@ -63,25 +63,19 @@ jobs: git merge-base --is-ancestor "${release_commit}" origin/main node <<'NODE' const { readFileSync } = require("node:fs"); - const { join } = require("node:path"); const version = process.env.RELEASE_TAG.slice(1); - const dirs = [ - ".", "native", "npm/linux-x64-gnu", "npm/linux-x64-musl", - "npm/linux-arm64-gnu", "npm/linux-arm64-musl", "npm/darwin-x64", - "npm/darwin-arm64", "npm/win32-x64-msvc", - ]; - for (const dir of dirs) { - const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); - if (pkg.version !== version) throw new Error(`${pkg.name} is ${pkg.version}, expected ${version}`); + const pkg = JSON.parse(readFileSync("package.json", "utf8")); + if (pkg.name !== "@openclaw/fs-safe" || pkg.version !== version) { + throw new Error(`root package is ${pkg.name}@${pkg.version}, expected @openclaw/fs-safe@${version}`); + } + const nativePkg = JSON.parse(readFileSync("native/package.json", "utf8")); + if (!nativePkg.private || nativePkg.version !== version) { + throw new Error(`private native build workspace must declare ${version}`); } const cargo = readFileSync("native/Cargo.toml", "utf8"); if (!cargo.match(new RegExp(`^version = "${version.replaceAll(".", "\\.")}"$`, "m"))) { throw new Error(`native/Cargo.toml does not declare ${version}`); } - const loader = readFileSync("native/index.js", "utf8"); - if (!loader.includes("require('./package.json').version")) { - throw new Error("native loader must derive its version check from package.json"); - } NODE package_version="$(node -p "require('./package.json').version")" [ "${RELEASE_TAG}" = "v${package_version}" ] @@ -166,15 +160,15 @@ jobs: - name: Build native binding if: matrix.cross == 'none' - run: pnpm --filter @openclaw/fs-safe-native exec napi build --release --platform --target ${{ matrix.target }} --output-dir ../artifacts + run: pnpm --filter @openclaw/fs-safe-native-build exec napi build --release --platform --target ${{ matrix.target }} --output-dir ../artifacts - name: Build native binding with napi cross toolchain if: matrix.cross == 'napi' - run: pnpm --filter @openclaw/fs-safe-native exec napi build --release --platform --target ${{ matrix.target }} --use-napi-cross --output-dir ../artifacts + run: pnpm --filter @openclaw/fs-safe-native-build exec napi build --release --platform --target ${{ matrix.target }} --use-napi-cross --output-dir ../artifacts - name: Build native binding with Zig if: matrix.cross == 'zig' - run: pnpm --filter @openclaw/fs-safe-native exec napi build --release --platform --target ${{ matrix.target }} --cross-compile --output-dir ../artifacts + run: pnpm --filter @openclaw/fs-safe-native-build exec napi build --release --platform --target ${{ matrix.target }} --cross-compile --output-dir ../artifacts - name: Upload binding uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -185,7 +179,7 @@ jobs: retention-days: 1 collect: - name: Validate release packages + name: Assemble and validate @openclaw/fs-safe if: github.ref_protected needs: prebuild runs-on: ubuntu-latest @@ -210,32 +204,32 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Download bindings + - name: Build package + run: pnpm build + + - name: Download all seven bindings uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: pattern: binding-* path: artifacts merge-multiple: true - - name: Assemble platform packages - run: pnpm --filter @openclaw/fs-safe-native exec napi artifacts --output-dir ../artifacts --npm-dir ../npm + - name: Assemble bundled bindings + run: node scripts/assemble-native-binaries.mjs --source artifacts --destination dist/native - - name: Build root package - run: pnpm build - - - name: Pack and smoke test every package + - name: Pack and smoke test the single package run: node scripts/check-release-packages.mjs --output release-artifacts - - name: Upload release packages + - name: Upload release package uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: release-packages + name: release-package path: release-artifacts/* if-no-files-found: error retention-days: 7 - publish-platforms: - name: Publish ${{ matrix.package }} + publish: + name: Publish @openclaw/fs-safe if: github.ref_protected needs: collect runs-on: ubuntu-latest @@ -243,17 +237,6 @@ jobs: permissions: contents: read id-token: write - strategy: - fail-fast: false - matrix: - package: - - "@openclaw/fs-safe-native-linux-x64-gnu" - - "@openclaw/fs-safe-native-linux-x64-musl" - - "@openclaw/fs-safe-native-linux-arm64-gnu" - - "@openclaw/fs-safe-native-linux-arm64-musl" - - "@openclaw/fs-safe-native-darwin-x64" - - "@openclaw/fs-safe-native-darwin-arm64" - - "@openclaw/fs-safe-native-win32-x64-msvc" steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -267,73 +250,19 @@ jobs: - name: Update npm run: npm install --global npm@${{ env.NPM_VERSION }} - - name: Download release packages + - name: Download release package uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - name: release-packages + name: release-package path: release-artifacts - - name: Publish or verify - run: node scripts/publish-or-verify.mjs --package "${{ matrix.package }}" --artifacts release-artifacts - - publish-native: - name: Publish @openclaw/fs-safe-native - if: github.ref_protected - needs: publish-platforms - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - id-token: write - steps: - - name: Check out - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: ${{ env.NODE_VERSION }} - registry-url: https://registry.npmjs.org - - name: Update npm - run: npm install --global npm@${{ env.NPM_VERSION }} - - name: Download release packages - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: release-packages - path: release-artifacts - - name: Publish or verify - run: node scripts/publish-or-verify.mjs --package "@openclaw/fs-safe-native" --artifacts release-artifacts - - publish-root: - name: Publish @openclaw/fs-safe - if: github.ref_protected - needs: publish-native - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - id-token: write - steps: - - name: Check out - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: ${{ env.NODE_VERSION }} - registry-url: https://registry.npmjs.org - - name: Update npm - run: npm install --global npm@${{ env.NPM_VERSION }} - - name: Download release packages - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: release-packages - path: release-artifacts - name: Publish or verify run: node scripts/publish-or-verify.mjs --package "@openclaw/fs-safe" --artifacts release-artifacts release: name: Publish GitHub Release if: github.ref_protected - needs: publish-root + needs: publish runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -349,10 +278,10 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} - - name: Download release packages + - name: Download release package uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - name: release-packages + name: release-package path: release-artifacts - name: Generate release notes and proof diff --git a/.gitignore b/.gitignore index c8078c0..f6d4d8b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,12 @@ dist/ node_modules target/ native/*.node +native/index.js +native/index.d.ts coverage/ release-graph-proof*.json +artifacts*/ +release-artifacts*/ *.tsbuildinfo .clawpatch/ .deepsec/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d981aa9..7780898 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.5.0 - 2026-07-26 +## 0.5.0 - Unreleased ### Highlights @@ -9,7 +9,7 @@ - Add native fd-relative ZIP and TAR extraction/read support with gzip, zstd, and bzip2 streaming; TypeScript evaluates the shared entry policy before Rust creates any output, and zstd/bzip2 report a typed native-required error when no binding is available. - Bound PAX, GNU long-name/link, and sparse metadata with one `maxMetaEntryBytes` policy shared by node-tar and the native fixed-header metering reader, including typed failures for oversized or malformed metadata. - Add `Root.walk()` subtree pruning and partial directory-error reporting for bounded best-effort consumers, plus a shared `maxEntryPathComponents` archive limit that rejects implicit-directory depth attacks before either extraction path creates output. -- Ship the optional `@openclaw/fs-safe-native` helper and seven platform packages from this repository for fd-relative opens, guarded directory creation and hardlinks, atomic no-replace rename, and file identity checks. +- Bundle all seven prebuilt native binaries inside the single `@openclaw/fs-safe` package for fd-relative opens, guarded directory creation and hardlinks, atomic no-replace rename, and file identity checks. This deliberately increases the package size in exchange for deterministic installs with no optional platform packages, downloads, postinstall step, or consumer Rust build; unsupported platforms silently use the guarded JavaScript fallback in `auto` mode. ### Security and Correctness diff --git a/README.md b/README.md index 51ae6b1..7cd1e51 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,10 @@ pnpm add @openclaw/fs-safe Node 22 or newer. Core root/path/json/temp helpers avoid framework dependencies. Archive helpers use optional `jszip` and `tar` dependencies for ZIP/TAR support; installs that omit optional dependencies can still use every non-archive subpath. -The optional native package supplies fd-relative and atomic no-replace -primitives that Node does not expose directly. Configure its lazy loader before -first use when you need a strict environment policy: +The package bundles prebuilt native bindings for seven supported targets. They +supply fd-relative and atomic no-replace primitives that Node does not expose +directly. Configure the lazy loader before first use when you need a strict +environment policy: ```ts import { configureFsSafeNative } from "@openclaw/fs-safe"; @@ -71,10 +72,13 @@ configureFsSafeNative({ mode: "off" }); // guarded JavaScript only configureFsSafeNative({ mode: "require" }); // fail closed if the binding is unavailable ``` -Equivalent env var: `FS_SAFE_NATIVE_MODE=auto|off|require`. The seven platform -packages are prebuilt; consumers do not need Rust. Without a matching package, -`auto` retains lexical and canonical root checks, no-follow opens, guarded -temp+rename writes, and post-write identity verification. See the [native +Equivalent env var: `FS_SAFE_NATIVE_MODE=auto|off|require`. All seven binaries +ship inside `@openclaw/fs-safe`; there are no platform packages, postinstall +steps, downloads, or consumer Rust builds. This makes the tarball larger than +a per-platform package, but makes installation deterministic. On a platform +without a bundled binary, `auto` silently retains lexical and canonical root +checks, no-follow opens, guarded temp+rename writes, and post-write identity +verification. See the [native helper policy](docs/native-helper.md) for the exact boundary and deployment tradeoff, and [native architecture](docs/native.md) for the platform mechanisms and policy ownership model. @@ -85,7 +89,7 @@ and guarded JavaScript results are best-effort. See the [security model](docs/se ## Migrating from the Python helper -Version 0.5 replaces the persistent Python worker with optional prebuilt native +Version 0.5 replaces the persistent Python worker with bundled prebuilt native bindings. The modes map directly: `configureFsSafePython({ mode: "auto" })` becomes `configureFsSafeNative({ mode: "auto" })`, and likewise for `off` and `require`. Replace `FS_SAFE_PYTHON_MODE` with `FS_SAFE_NATIVE_MODE`; remove diff --git a/RELEASE-PREREQS.md b/RELEASE-PREREQS.md index d280e27..3fa475d 100644 --- a/RELEASE-PREREQS.md +++ b/RELEASE-PREREQS.md @@ -4,19 +4,9 @@ Complete this checklist in npm before pushing the first `v0.5.0` tag. The releas ## Package setup -Create or confirm these public packages under the `@openclaw` scope: +Exactly one package is published: `@openclaw/fs-safe`. It already exists, is public, and is owned by the OpenClaw npm organization. -- `@openclaw/fs-safe-native-linux-x64-gnu` -- `@openclaw/fs-safe-native-linux-x64-musl` -- `@openclaw/fs-safe-native-linux-arm64-gnu` -- `@openclaw/fs-safe-native-linux-arm64-musl` -- `@openclaw/fs-safe-native-darwin-x64` -- `@openclaw/fs-safe-native-darwin-arm64` -- `@openclaw/fs-safe-native-win32-x64-msvc` -- `@openclaw/fs-safe-native` -- `@openclaw/fs-safe` - -For every package above, open npm package settings and configure the same trusted publisher: +Open its npm package settings and configure this trusted publisher: | npm trusted-publisher field | Required value | |---|---| @@ -26,14 +16,14 @@ For every package above, open npm package settings and configure the same truste | Workflow filename | `release.yml` | | Environment | Leave empty; this workflow does not use a GitHub environment. | -Confirm each package is public, the OpenClaw npm organization owns it, and the publishing maintainers have 2FA enabled. Do not add `NPM_TOKEN` or an automation token to GitHub. +Confirm the publishing maintainers have 2FA enabled. Do not add `NPM_TOKEN` or an automation token to GitHub. The former native loader and platform package names were never published, so there is nothing to deprecate or unpublish. ## Release commit and tag -- Replace `0.5.0-dev.0` with `0.5.0` in the root, native loader, Rust crate, all seven platform package manifests, and generated native loader version checks. -- Run `pnpm install` so `pnpm-lock.yaml` matches the stable package versions. +- Confirm `0.5.0` in the root package, private native build workspace, and Rust crate. +- Run `pnpm install` so `pnpm-lock.yaml` matches the stable package version. - Change `## 0.5.0 - Unreleased` in `CHANGELOG.md` to the release date. -- Run `pnpm check`, `pnpm test:security`, `cargo test --workspace --locked`, `pnpm docs:site`, and `git diff --check` on the release commit. +- Run `pnpm check`, `pnpm test:security`, `cargo test --workspace --locked`, `cargo clippy --workspace --locked -- -D warnings`, `pnpm docs:site`, and `git diff --check` on the release commit. - Merge the release commit to `main`, then create an annotated, protected `v0.5.0` tag on that exact commit. -The workflow validates the protected annotated tag, `main` ancestry, all nine package versions, package bytes, install/import behavior, and changelog-derived release notes before it publishes anything. +The workflow validates the protected annotated tag, `main` ancestry, package and crate versions, all seven non-empty native binaries, the single tarball's bytes, install/import behavior, native loading and fallback behavior, and changelog-derived release notes before it publishes anything. diff --git a/docs/archive.md b/docs/archive.md index 20cf8af..be8d766 100644 --- a/docs/archive.md +++ b/docs/archive.md @@ -1,6 +1,6 @@ # Archive extraction -`@openclaw/fs-safe/archive` extracts ZIP and TAR archives behind one API, with traversal checks, blocked-link-type rejection, and entry-count and byte budgets. When the optional native binding is available, Rust streams ZIP, TAR, gzip, zstd, and bzip2 while TypeScript remains the sole policy owner; every accepted output is created fd-relative in a private staging root. Extraction then merges through the same safe-open boundary used by direct writes — a symlinked entry can't trick the merge into following an out-of-tree path. +`@openclaw/fs-safe/archive` extracts ZIP and TAR archives behind one API, with traversal checks, blocked-link-type rejection, and entry-count and byte budgets. When the bundled native binding is available for the current platform, Rust streams ZIP, TAR, gzip, zstd, and bzip2 while TypeScript remains the sole policy owner; every accepted output is created fd-relative in a private staging root. Extraction then merges through the same safe-open boundary used by direct writes — a symlinked entry can't trick the merge into following an out-of-tree path. The guarded JavaScript fallback uses optional runtime dependencies: `jszip` for ZIP and `tar` for TAR/gzip. The native path does not use those packages. Installs diff --git a/docs/errors.md b/docs/errors.md index 2ba83ec..3235ede 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -90,7 +90,7 @@ type FsSafeErrorCode = | `device-path` | A read/open target is a known unsafe device or process-fd path. | `/dev/zero`, `/dev/random`, `/dev/stdin`, `/dev/fd/*`, `/proc/*/fd/*`, or a Windows reserved device name. | | `hardlink` | Read or copy with `hardlinks: "reject"` saw `nlink > 1`. | File is hardlinked — possibly an alias of an out-of-tree inode. | | `helper-failed` | A native mechanism or multi-step operational helper failed. | Inspect `cause` and any operation-specific `details`; retrying may be unsafe if the operation partially completed. | -| `helper-unavailable` | Required native binding could not be loaded. | Missing/incompatible optional platform package or `FS_SAFE_NATIVE_MODE=require`. `auto` falls back where possible; `require` fails closed. | +| `helper-unavailable` | Required native binding could not be loaded. | Unsupported platform, missing/incompatible bundled binary, or `FS_SAFE_NATIVE_MODE=require`. `auto` falls back where possible; `require` fails closed. | | `insecure-permissions` | A secure file or path permission check found a mode/ACL that allows broader access than requested. | File or directory is group/world writable/readable; Windows ACL grants broad read. | | `invalid-path` | Input was empty, contained NUL, was an unparseable URL, or otherwise unusable. | Caller didn't validate input; input was a network path on Windows. | | `not-empty` | `remove()` on a non-empty directory. | Use `replaceDirectoryAtomic` or remove children first. | diff --git a/docs/index.md b/docs/index.md index bf2aab9..47936ef 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,7 +51,7 @@ await fs.remove("notes/archive/today.txt"); |---|---| | [`root()`](root.md) | One boundary for read/write/move/remove and bounded recursive walking inside a trusted directory. | | [`@openclaw/fs-safe/config`](config.md) | Process-global native helper and lock-option defaults. | -| [Native helper policy](native-helper.md) | Choose `auto`, `off`, or `require` for optional native primitives. | +| [Native helper policy](native-helper.md) | Choose `auto`, `off`, or `require` for bundled native primitives. | | [Native architecture](native.md) | Understand the thin syscall layer, beneath model, platform mechanisms, and fallback boundary. | | [`replaceFileAtomic`](atomic.md) | Sibling-temp + rename, fsync hooks, mode preservation, copy fallback. | | [`@openclaw/fs-safe/durability`](durability.md) | Pinned directory identities, durable creation, exclusive publication, streaming SHA-256, provenance receipts, and sync-failure policy. | diff --git a/docs/install.md b/docs/install.md index 809b730..432912e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -87,16 +87,18 @@ Use the main entry for the common surface, or the focused subpaths when you want `@openclaw/fs-safe` lists `jszip` and `tar` as optional dependencies for [archive extraction](archive.md). They are loaded lazily and only required when ZIP/TAR helpers run. Installs that omit optional dependencies can still import and use every non-archive subpath; archive calls fail with a clear missing-optional-dependency message. -There are no peer dependencies. Native helpers are optional prebuilt packages, so consumers do not run a native build during installation. +There are no peer dependencies. The single npm package bundles all seven native binaries, so consumers do not run a native build, download platform code, or execute a postinstall step. Shipping every target increases the tarball size compared with per-platform packages, intentionally trading bandwidth for deterministic installation. Upgrading an existing consumer? Follow [Migrating to 0.5](migrating-to-0.5.md) before choosing a native mode or accepting the new archive clamp default. ## Native helper policy -The optional native package provides fd-relative open/link/mkdir primitives, +The bundled native binaries provide fd-relative open/link/mkdir primitives, atomic no-replace rename, and file identity checks. The default is `auto`: use -the platform binding when it loads, otherwise keep the guarded JavaScript path. +the matching binary when it loads, otherwise silently keep the guarded +JavaScript path. Platforms without one of the seven bundled targets therefore +continue through the documented fallback in `auto` mode. ```ts import { configureFsSafeNative } from "@openclaw/fs-safe/config"; diff --git a/docs/migrating-to-0.5.md b/docs/migrating-to-0.5.md index c3025a4..10f1ddb 100644 --- a/docs/migrating-to-0.5.md +++ b/docs/migrating-to-0.5.md @@ -8,18 +8,19 @@ description: "Ordered checklist for moving a 0.4 consumer from the Python helper Use this checklist from top to bottom. Version 0.5 replaces the Python worker, changes the default archive mode policy, and adds explicit contracts for publication, walking, locks, secrets, and native-only features. Nothing in this -guide requires a Rust toolchain: supported native packages are prebuilt and -optional. +guide requires a Rust toolchain: all supported native binaries are prebuilt and +bundled in `@openclaw/fs-safe`. ## 1. Update the package and runtime - Run on Node.js 22 or newer. - Update `@openclaw/fs-safe` and regenerate every lock or shrinkwrap file your deployment consumes. -- Keep optional dependencies enabled if you want automatic native loading and - JavaScript ZIP/TAR support. An install that omits optional packages can still - import fs-safe, but native-only features and missing JS archive decoders fail - with actionable errors. +- Keep optional dependencies enabled if you want JavaScript ZIP/TAR support. + Native loading no longer depends on optional packages because all seven + binaries ship in `@openclaw/fs-safe`. An install that omits optional packages + can still import fs-safe, but missing JS archive decoders fail with actionable + errors. If you call `resolveRootPath()` or `resolveRootPathSync()` directly, upgrade to 0.5: versions through 0.4.7 could approve an in-root symlink traversal that @@ -41,7 +42,7 @@ configureFsSafeNative({ mode: "auto" }); | `configureFsSafePython({ mode })` | `configureFsSafeNative({ mode })` | | `FS_SAFE_PYTHON_MODE` | `FS_SAFE_NATIVE_MODE` | | `OPENCLAW_FS_SAFE_PYTHON_MODE` | `OPENCLAW_FS_SAFE_NATIVE_MODE` | -| `pythonPath`, `FS_SAFE_PYTHON`, pinned-Python aliases | Nothing; native packages do not use an interpreter | +| `pythonPath`, `FS_SAFE_PYTHON`, pinned-Python aliases | Nothing; bundled native binaries do not use an interpreter | The old names warn once and map `auto`, `off`, or `require` so a shipped 0.4 deployment does not silently change policy. Interpreter paths are ignored and diff --git a/docs/native-helper.md b/docs/native-helper.md index a25646c..31f849b 100644 --- a/docs/native-helper.md +++ b/docs/native-helper.md @@ -1,11 +1,11 @@ --- title: Native helper policy -description: "How fs-safe loads its optional native filesystem primitives and how auto, require, and off affect guarded fallbacks." +description: "How fs-safe loads its bundled native filesystem primitives and how auto, require, and off affect guarded fallbacks." --- # Native helper policy -`@openclaw/fs-safe` installs `@openclaw/fs-safe-native` as an optional dependency. Its loader selects one of seven prebuilt packages for Linux x64/arm64 (glibc or musl), macOS x64/arm64, or Windows x64. Consumers do not compile Rust during installation. +`@openclaw/fs-safe` itself contains seven prebuilt binaries for Linux x64/arm64 (glibc or musl), macOS x64/arm64, and Windows x64. The loader selects `dist/native//fs-safe-native.node` without optional platform packages, downloads, postinstall scripts, or a consumer Rust build. Carrying every target makes the npm tarball larger, but every installation receives the same complete artifact. ```ts import { configureFsSafeNative } from "@openclaw/fs-safe/config"; @@ -21,15 +21,15 @@ The equivalent environment variables are `FS_SAFE_NATIVE_MODE` and `OPENCLAW_FS_ | Mode | Behavior | |---|---| -| `auto` | Prefer native primitives when the current platform package loads; otherwise use the guarded JavaScript path. | -| `off` | Do not load a platform package. Use the guarded JavaScript path deterministically. | +| `auto` | Prefer native primitives when the current bundled binary loads; otherwise silently use the guarded JavaScript path. | +| `off` | Do not load a bundled binary. Use the guarded JavaScript path deterministically. | | `require` | Throw `FsSafeError("helper-unavailable")` instead of falling back when an operation needs the native binding and it cannot load. | -Configure the mode once during startup. Loading is lazy and cached; changing from `auto` to `require` after a failed load changes failure policy but does not repeatedly probe the package. +Configure the mode once during startup. Loading is lazy and cached; changing from `auto` to `require` after a failed load changes failure policy but does not repeatedly probe the binary. ## Native boundary -The native package exposes policy-free filesystem mechanisms: beneath-root +The bundled native layer exposes policy-free filesystem mechanisms: beneath-root open/mkdir/link, no-replace rename, identity reads, archive decode/execution, clone/copy/hash workers, and Windows security descriptor calls. The TypeScript layer owns policy, retries, filters, budgets, modes, cleanup, error diff --git a/docs/native.md b/docs/native.md index 90e3aa2..38b60db 100644 --- a/docs/native.md +++ b/docs/native.md @@ -1,21 +1,23 @@ --- title: Native architecture -description: "The optional native binding, fd-relative beneath model, platform mechanisms, loader security, and JavaScript fallback contract." +description: "The bundled native bindings, fd-relative beneath model, platform mechanisms, loader security, and JavaScript fallback contract." --- # Native architecture -The optional `@openclaw/fs-safe-native` package supplies mechanisms that Node -does not expose directly. It is deliberately not a second policy engine. +`@openclaw/fs-safe` bundles native bindings that supply mechanisms Node does +not expose directly. The Rust layer is deliberately not a second policy engine. TypeScript owns trusted-root selection, path validation, archive filtering, budgets, modes, identity fencing, cleanup decisions, and error normalization. Rust receives already-decided relative operations and performs the smallest platform syscall sequence that can preserve the boundary. Every operation that has an equivalent safe Node implementation keeps that -guarded JavaScript path. Native loading is lazy and optional; installs do not -compile Rust. Native-only formats and creation-time Windows DACL guarantees -fail explicitly instead of substituting a weaker implementation. +guarded JavaScript path. Native loading is lazy; installs do not compile Rust, +run postinstall code, or fetch binaries. The npm tarball carries all seven +supported targets, so it is larger than a per-platform package by design. +Native-only formats and creation-time Windows DACL guarantees fail explicitly +instead of substituting a weaker implementation. ## The beneath model @@ -111,13 +113,15 @@ infer native loading from timing. ## Loader security -Importing fs-safe never executes a platform detector. Linux libc selection uses +Importing fs-safe never executes a child process. Linux libc selection uses the Node process report, conventional musl library filenames, and the ELF `PT_INTERP` field of `process.execPath`. If all probes are inconclusive, the -loader conservatively attempts the glibc package and lets normal module loading -fail into `auto` fallback. After napi-rs generates bindings and declarations, -the hardening script replaces its broad platform/WASI template with the -checked-in seven-target loader this repository actually ships. Tests reject +loader conservatively attempts the bundled glibc binary and lets normal module +loading fail into `auto` fallback. The loader requires only +`dist/native//fs-safe-native.node`; it never probes optional packages, +downloads code, or runs a postinstall step. A missing or incompatible binary +silently selects the JavaScript fallback in `auto`, throws typed +`helper-unavailable` in `require`, and is never inspected in `off`. Tests reject `child_process`, `exec`, or `spawn` usage in the loader. ## Related pages diff --git a/docs/permissions.md b/docs/permissions.md index 11434d2..b14d600 100644 --- a/docs/permissions.md +++ b/docs/permissions.md @@ -70,7 +70,7 @@ group. Trusted defaults include the current user, SYSTEM, and Administrators. The parser is on the advanced surface so tests and CLIs can process captured `icacls` output without spawning a process. -When the optional native binding is available, `inspectPathPermissions()` +When the bundled native binding is available, `inspectPathPermissions()` reads the owner and DACL directly with Windows security APIs. It classifies the current user, LocalSystem, and built-in Administrators as trusted and reports the world/group read/write facts consumed by secure reads. Descriptor forms it @@ -120,7 +120,7 @@ Object-specific and other ACE layouts are not guessed: they are omitted, `complete` becomes false, and their numeric types appear in `unsupportedAceTypes`, allowing a security-sensitive caller to fail closed. Non-Windows systems return `{ status: "unsupported-platform", platform }`. -Windows requires the optional native binding; if it is unavailable or forced +Windows requires the bundled native binding; if it is unavailable or forced off, the call throws `FsSafeError("helper-unavailable")`. The existing coarse `inspectPathPermissions()` API still owns its compatibility fallback and trust classification. diff --git a/docs/root.md b/docs/root.md index d38d1f0..b985736 100644 --- a/docs/root.md +++ b/docs/root.md @@ -121,7 +121,7 @@ These do not pin a later operation. They are safe to expose to UIs and decision ## Native helper mode -Create-only writes prefer the optional native helper for fd-relative opens and +Create-only writes prefer the bundled native helper for fd-relative opens and atomic no-replace rename. Operations without native wiring retain their guarded JavaScript implementations. diff --git a/docs/testing.md b/docs/testing.md index 96c6419..cc5cc13 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -17,7 +17,7 @@ The double-underscore prefix is a deliberate "hands off" signal: production code ## When to reach for hooks - Reproduce a TOCTOU race deterministically: simulate a symlink swap between resolve and open, or between write and rename. -- Force guarded JavaScript behavior without removing native packages from your runners. +- Force guarded JavaScript behavior without removing bundled binaries from your runners. - Inject latency to test cancellation/timeout paths. If you don't need to inject a race, you don't need hooks — most tests should drive the library through normal calls and assert on observable behavior. diff --git a/native/index.d.ts b/native/index.d.ts deleted file mode 100644 index 62d6dd8..0000000 --- a/native/index.d.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* auto-generated by NAPI-RS */ -/* eslint-disable */ -export declare function cloneFileExclusive(sourceFd: number, targetRootFd: number, targetRelPath: string): number - -export declare function copyFileRangeExclusive(sourceFd: number, targetRootFd: number, targetRelPath: string): Promise - -export declare function createPrivateDirectory(path: string): void - -export declare function extractArchiveNative(path: string, kind: string, rootFd: number, plan: Array, maxMetaEntryBytes: number, signal: AbortSignal): Promise - -export interface FileHash { - bytes: number - digest: string -} - -export interface FileIdentity { - dev: number - ino: number - mode: number - nlink: number - size: number - isFile: boolean - isDirectory: boolean - isSymbolicLink: boolean -} - -export declare function fstatIdentity(fd: number): FileIdentity - -export declare function inspectArchiveNative(path: string, kind: string, maxEntries: number, maxMetaEntryBytes: number, maxManifestBytes: number, signal: AbortSignal): Promise> - -export declare function linkBeneath(sourceRootFd: number, sourceRelPath: string, targetRootFd: number, targetRelPath: string): void - -export declare function mkdirBeneath(rootFd: number, relPath: string, mode: number): void - -export interface NativeArchiveEntry { - index: number - path: string - kind: string - size: number - mode: number -} - -export interface NativeArchivePlanEntry { - index: number - path: string - kind: string - size: number - mode: number -} - -export interface NativeCopyResult { - fd: number - bytes: number - errorCode?: string - errorMessage?: string -} - -export declare function openBeneath(rootFd: number, relPath: string, flags: number): OpenBeneathResult - -export interface OpenBeneathResult { - fd: number - containment: 'kernel-atomic' | 'best-effort' -} - -export declare function readArchiveEntryNative(path: string, kind: string, requested: string, maxBytes: number, maxEntries: number, maxMetaEntryBytes: number, signal: AbortSignal): Promise - -export declare function readOwnerAndDacl(path: string): WindowsSecurityFacts - -export declare function renameNoReplace(sourceRootFd: number, sourceRelPath: string, targetRootFd: number, targetRelPath: string): void - -export declare function sha256File(fd: number): Promise - -export interface WindowsAccessControlEntry { - sid: string - mask: number - aceType: string - flags: WindowsAceFlags -} - -export interface WindowsAceFlags { - raw: number - objectInherit: boolean - containerInherit: boolean - noPropagateInherit: boolean - inheritOnly: boolean - inherited: boolean - successfulAccess: boolean - failedAccess: boolean -} - -export interface WindowsSecurityFacts { - ownerSid: string - currentUserSid: string - ownerClass: string - worldWritable: boolean - groupWritable: boolean - worldReadable: boolean - groupReadable: boolean - fallbackRequired: boolean - daclPresent: boolean - isLocal: boolean - aceListComplete: boolean - unsupportedAceTypes: Array - aces: Array -} diff --git a/native/index.js b/native/index.js deleted file mode 100644 index 687acf4..0000000 --- a/native/index.js +++ /dev/null @@ -1,179 +0,0 @@ -// prettier-ignore -/* eslint-disable */ -// @ts-nocheck -/* Checked-in N-API loader for the seven platform packages this project ships. */ - -const { closeSync, openSync, readSync, readdirSync } = require('fs') -const loadErrors = [] - -const isFileMusl = (file) => file.includes('libc.musl-') || file.includes('ld-musl-') - -const isMuslFromReport = () => { - try { - if (!process.report || typeof process.report.getReport !== 'function') return null - const report = process.report.getReport() - if (report?.header?.glibcVersionRuntime) return false - if (report?.sharedObjects?.some(isFileMusl)) return true - } catch { - // Continue with filesystem and ELF inspection. - } - return null -} - -const isMuslFromFilesystem = () => { - for (const directory of ['/lib', '/usr/lib']) { - try { - if (readdirSync(directory).some(isFileMusl)) return true - } catch { - // A missing or unreadable conventional library directory is inconclusive. - } - } - return null -} - -const readUInt = (buffer, offset, bytes, littleEndian) => { - if (offset < 0 || offset + bytes > buffer.length) return null - if (bytes === 2) return littleEndian ? buffer.readUInt16LE(offset) : buffer.readUInt16BE(offset) - if (bytes === 4) return littleEndian ? buffer.readUInt32LE(offset) : buffer.readUInt32BE(offset) - if (bytes === 8) { - const value = littleEndian ? buffer.readBigUInt64LE(offset) : buffer.readBigUInt64BE(offset) - return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null - } - return null -} - -const isMuslFromElfInterpreter = () => { - let fd - try { - fd = openSync(process.execPath, 'r') - const header = Buffer.alloc(64) - if (readSync(fd, header, 0, header.length, 0) < 52) return null - if (!header.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return null - - const elfClass = header[4] - const dataEncoding = header[5] - if ((elfClass !== 1 && elfClass !== 2) || (dataEncoding !== 1 && dataEncoding !== 2)) { - return null - } - const littleEndian = dataEncoding === 1 - const is64Bit = elfClass === 2 - const tableOffset = readUInt(header, is64Bit ? 32 : 28, is64Bit ? 8 : 4, littleEndian) - const entrySize = readUInt(header, is64Bit ? 54 : 42, 2, littleEndian) - const entryCount = readUInt(header, is64Bit ? 56 : 44, 2, littleEndian) - if (tableOffset === null || entrySize === null || entryCount === null || - entrySize < (is64Bit ? 56 : 32) || entryCount > 1024) return null - - const programHeader = Buffer.alloc(entrySize) - for (let index = 0; index < entryCount; index += 1) { - const offset = tableOffset + index * entrySize - if (readSync(fd, programHeader, 0, entrySize, offset) !== entrySize) return null - if (readUInt(programHeader, 0, 4, littleEndian) !== 3) continue // PT_INTERP - const interpreterOffset = readUInt(programHeader, is64Bit ? 8 : 4, is64Bit ? 8 : 4, littleEndian) - const interpreterSize = readUInt(programHeader, is64Bit ? 32 : 16, is64Bit ? 8 : 4, littleEndian) - if (interpreterOffset === null || interpreterSize === null || - interpreterSize < 1 || interpreterSize > 4096) return null - const interpreter = Buffer.alloc(interpreterSize) - if (readSync(fd, interpreter, 0, interpreterSize, interpreterOffset) !== interpreterSize) { - return null - } - return isFileMusl(interpreter.toString('utf8')) - } - } catch { - return null - } finally { - if (fd !== undefined) { - try { closeSync(fd) } catch { - // Best-effort detection must never turn close failure into import failure. - } - } - } - return null -} - -const isMusl = () => { - if (process.platform !== 'linux') return false - for (const detector of [isMuslFromReport, isMuslFromFilesystem, isMuslFromElfInterpreter]) { - const result = detector() - if (result !== null) return result - } - // Unknown Linux libc: try the glibc package and let its require fail normally. - return false -} - -const targetSuffix = () => { - if (process.platform === 'win32' && process.arch === 'x64') return 'win32-x64-msvc' - if (process.platform === 'darwin' && process.arch === 'x64') return 'darwin-x64' - if (process.platform === 'darwin' && process.arch === 'arm64') return 'darwin-arm64' - if (process.platform === 'linux' && (process.arch === 'x64' || process.arch === 'arm64')) { - return `linux-${process.arch}-${isMusl() ? 'musl' : 'gnu'}` - } - return null -} - -const tryRequire = (specifier) => { - try { - return require(specifier) - } catch (error) { - loadErrors.push(error) - return null - } -} - -const requireNative = () => { - if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { - const explicit = tryRequire(process.env.NAPI_RS_NATIVE_LIBRARY_PATH) - if (explicit) return explicit - } - - const suffix = targetSuffix() - if (!suffix) { - loadErrors.push(new Error(`Unsupported OS or architecture: ${process.platform}-${process.arch}`)) - return null - } - const local = tryRequire(`./fs-safe-native.${suffix}.node`) - if (local) return local - - const packageName = `@openclaw/fs-safe-native-${suffix}` - const binding = tryRequire(packageName) - if (!binding) return null - if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - const expectedVersion = require('./package.json').version - const bindingVersion = require(`${packageName}/package.json`).version - if (bindingVersion !== expectedVersion) { - loadErrors.push(new Error( - `Native binding package version mismatch, expected ${expectedVersion} but got ${bindingVersion}. ` + - 'Reinstall dependencies to fix this issue.', - )) - return null - } - } - return binding -} - -const nativeBinding = requireNative() -if (!nativeBinding) { - const error = new Error( - `Cannot find native binding for ${process.platform}-${process.arch}; ` + - 'install the matching optional @openclaw/fs-safe-native platform package.', - ) - error.cause = loadErrors.reduceRight((cause, current) => { - current.cause = cause - return current - }, undefined) - throw error -} - -module.exports = nativeBinding -module.exports.cloneFileExclusive = nativeBinding.cloneFileExclusive -module.exports.copyFileRangeExclusive = nativeBinding.copyFileRangeExclusive -module.exports.createPrivateDirectory = nativeBinding.createPrivateDirectory -module.exports.extractArchiveNative = nativeBinding.extractArchiveNative -module.exports.fstatIdentity = nativeBinding.fstatIdentity -module.exports.inspectArchiveNative = nativeBinding.inspectArchiveNative -module.exports.linkBeneath = nativeBinding.linkBeneath -module.exports.mkdirBeneath = nativeBinding.mkdirBeneath -module.exports.openBeneath = nativeBinding.openBeneath -module.exports.readArchiveEntryNative = nativeBinding.readArchiveEntryNative -module.exports.readOwnerAndDacl = nativeBinding.readOwnerAndDacl -module.exports.renameNoReplace = nativeBinding.renameNoReplace -module.exports.sha256File = nativeBinding.sha256File diff --git a/native/package.json b/native/package.json index 46b978e..5f7aecd 100644 --- a/native/package.json +++ b/native/package.json @@ -1,7 +1,8 @@ { - "name": "@openclaw/fs-safe-native", + "name": "@openclaw/fs-safe-native-build", "version": "0.5.0", - "description": "Native fd-relative filesystem primitives for @openclaw/fs-safe.", + "description": "Private build workspace for the native bindings bundled by @openclaw/fs-safe.", + "private": true, "license": "MIT", "author": "OpenClaw Team ", "repository": { @@ -9,22 +10,9 @@ "url": "git+https://github.com/openclaw/fs-safe.git", "directory": "native" }, - "main": "index.js", - "types": "index.d.ts", - "files": [ - "index.js", - "index.d.ts" - ], - "publishConfig": { - "access": "public", - "provenance": true - }, - "engines": { - "node": ">=22" - }, "napi": { "binaryName": "fs-safe-native", - "packageName": "@openclaw/fs-safe-native", + "packageName": "@openclaw/fs-safe", "targets": [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", @@ -36,18 +24,7 @@ ] }, "scripts": { - "artifacts": "napi artifacts --npm-dir ../npm", - "build": "napi build --platform --release && node ../scripts/harden-native-loader.mjs", - "build:debug": "napi build --platform && node ../scripts/harden-native-loader.mjs", - "prepublishOnly": "napi prepublish -t npm" - }, - "optionalDependencies": { - "@openclaw/fs-safe-native-darwin-arm64": "0.5.0", - "@openclaw/fs-safe-native-darwin-x64": "0.5.0", - "@openclaw/fs-safe-native-linux-arm64-gnu": "0.5.0", - "@openclaw/fs-safe-native-linux-arm64-musl": "0.5.0", - "@openclaw/fs-safe-native-linux-x64-gnu": "0.5.0", - "@openclaw/fs-safe-native-linux-x64-musl": "0.5.0", - "@openclaw/fs-safe-native-win32-x64-msvc": "0.5.0" + "build": "napi build --platform --release && node ../scripts/stage-host-native.mjs", + "build:debug": "napi build --platform && node ../scripts/stage-host-native.mjs" } } diff --git a/npm/darwin-arm64/package.json b/npm/darwin-arm64/package.json deleted file mode 100644 index 9b0339a..0000000 --- a/npm/darwin-arm64/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@openclaw/fs-safe-native-darwin-arm64", - "version": "0.5.0", - "description": "Native binding for @openclaw/fs-safe on macOS arm64.", - "license": "MIT", - "author": "OpenClaw Team ", - "repository": "github:openclaw/fs-safe", - "main": "fs-safe-native.darwin-arm64.node", - "files": ["fs-safe-native.darwin-arm64.node"], - "os": ["darwin"], - "cpu": ["arm64"], - "engines": { "node": ">=22" }, - "publishConfig": { "access": "public", "provenance": true } -} diff --git a/npm/darwin-x64/package.json b/npm/darwin-x64/package.json deleted file mode 100644 index 0a9d36b..0000000 --- a/npm/darwin-x64/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@openclaw/fs-safe-native-darwin-x64", - "version": "0.5.0", - "description": "Native binding for @openclaw/fs-safe on macOS x64.", - "license": "MIT", - "author": "OpenClaw Team ", - "repository": "github:openclaw/fs-safe", - "main": "fs-safe-native.darwin-x64.node", - "files": ["fs-safe-native.darwin-x64.node"], - "os": ["darwin"], - "cpu": ["x64"], - "engines": { "node": ">=22" }, - "publishConfig": { "access": "public", "provenance": true } -} diff --git a/npm/linux-arm64-gnu/package.json b/npm/linux-arm64-gnu/package.json deleted file mode 100644 index 55297b0..0000000 --- a/npm/linux-arm64-gnu/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "@openclaw/fs-safe-native-linux-arm64-gnu", - "version": "0.5.0", - "description": "Native binding for @openclaw/fs-safe on Linux arm64 glibc.", - "license": "MIT", - "author": "OpenClaw Team ", - "repository": "github:openclaw/fs-safe", - "main": "fs-safe-native.linux-arm64-gnu.node", - "files": ["fs-safe-native.linux-arm64-gnu.node"], - "os": ["linux"], - "cpu": ["arm64"], - "libc": ["glibc"], - "engines": { "node": ">=22" }, - "publishConfig": { "access": "public", "provenance": true } -} diff --git a/npm/linux-arm64-musl/package.json b/npm/linux-arm64-musl/package.json deleted file mode 100644 index f36d29f..0000000 --- a/npm/linux-arm64-musl/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "@openclaw/fs-safe-native-linux-arm64-musl", - "version": "0.5.0", - "description": "Native binding for @openclaw/fs-safe on Linux arm64 musl.", - "license": "MIT", - "author": "OpenClaw Team ", - "repository": "github:openclaw/fs-safe", - "main": "fs-safe-native.linux-arm64-musl.node", - "files": ["fs-safe-native.linux-arm64-musl.node"], - "os": ["linux"], - "cpu": ["arm64"], - "libc": ["musl"], - "engines": { "node": ">=22" }, - "publishConfig": { "access": "public", "provenance": true } -} diff --git a/npm/linux-x64-gnu/package.json b/npm/linux-x64-gnu/package.json deleted file mode 100644 index a46a1db..0000000 --- a/npm/linux-x64-gnu/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "@openclaw/fs-safe-native-linux-x64-gnu", - "version": "0.5.0", - "description": "Native binding for @openclaw/fs-safe on Linux x64 glibc.", - "license": "MIT", - "author": "OpenClaw Team ", - "repository": "github:openclaw/fs-safe", - "main": "fs-safe-native.linux-x64-gnu.node", - "files": ["fs-safe-native.linux-x64-gnu.node"], - "os": ["linux"], - "cpu": ["x64"], - "libc": ["glibc"], - "engines": { "node": ">=22" }, - "publishConfig": { "access": "public", "provenance": true } -} diff --git a/npm/linux-x64-musl/package.json b/npm/linux-x64-musl/package.json deleted file mode 100644 index be0fa72..0000000 --- a/npm/linux-x64-musl/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "@openclaw/fs-safe-native-linux-x64-musl", - "version": "0.5.0", - "description": "Native binding for @openclaw/fs-safe on Linux x64 musl.", - "license": "MIT", - "author": "OpenClaw Team ", - "repository": "github:openclaw/fs-safe", - "main": "fs-safe-native.linux-x64-musl.node", - "files": ["fs-safe-native.linux-x64-musl.node"], - "os": ["linux"], - "cpu": ["x64"], - "libc": ["musl"], - "engines": { "node": ">=22" }, - "publishConfig": { "access": "public", "provenance": true } -} diff --git a/npm/win32-x64-msvc/package.json b/npm/win32-x64-msvc/package.json deleted file mode 100644 index 4a03b59..0000000 --- a/npm/win32-x64-msvc/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@openclaw/fs-safe-native-win32-x64-msvc", - "version": "0.5.0", - "description": "Native binding for @openclaw/fs-safe on Windows x64.", - "license": "MIT", - "author": "OpenClaw Team ", - "repository": "github:openclaw/fs-safe", - "main": "fs-safe-native.win32-x64-msvc.node", - "files": ["fs-safe-native.win32-x64-msvc.node"], - "os": ["win32"], - "cpu": ["x64"], - "engines": { "node": ">=22" }, - "publishConfig": { "access": "public", "provenance": true } -} diff --git a/package.json b/package.json index f26b897..0900d63 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "dist/**/*.js", "dist/**/*.d.ts", "dist/**/*.d.ts.map", + "dist/native/**/*.node", "docs/**/*.md", "README.md", "CHANGELOG.md", @@ -130,10 +131,10 @@ "test:security": "vitest run test/fs-safe.test.ts test/read-boundary-bypass.test.ts test/write-boundary-bypass.test.ts test/additional-boundary-bypass.test.ts test/adversarial-boundary-payloads.test.ts", "check": "pnpm lint:file-size && pnpm lint:fs-boundary && pnpm build && pnpm test && node scripts/check-pack.mjs", "docs:site": "node scripts/build-docs-site.mjs", - "native:build": "pnpm --filter @openclaw/fs-safe-native build", + "native:build": "pnpm --filter @openclaw/fs-safe-native-build build", "native:test": "cargo test --manifest-path native/Cargo.toml", "pack:check": "pnpm build && node scripts/check-pack.mjs", - "release-graph:smoke": "node scripts/release-graph-smoke.mjs", + "package:smoke": "node scripts/check-release-packages.mjs --allow-host-only --output release-artifacts", "check:changed": "pnpm run check", "release:notes": "node scripts/release-notes.mjs", "test:changed": "pnpm run test", @@ -143,7 +144,6 @@ "crabbox:warmup": "crabbox warmup" }, "optionalDependencies": { - "@openclaw/fs-safe-native": "0.5.0", "jszip": "^3.10.1", "tar": "7.5.21" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a6ebf48..bfaf78b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,9 +27,6 @@ importers: specifier: ^4.1.10 version: 4.1.10(@types/node@26.1.1)(@vitest/coverage-v8@4.1.10)(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)) optionalDependencies: - '@openclaw/fs-safe-native': - specifier: 0.5.0 - version: link:native jszip: specifier: ^3.10.1 version: 3.10.1 @@ -37,43 +34,7 @@ importers: specifier: 7.5.21 version: 7.5.21 - native: - optionalDependencies: - '@openclaw/fs-safe-native-darwin-arm64': - specifier: 0.5.0 - version: link:../npm/darwin-arm64 - '@openclaw/fs-safe-native-darwin-x64': - specifier: 0.5.0 - version: link:../npm/darwin-x64 - '@openclaw/fs-safe-native-linux-arm64-gnu': - specifier: 0.5.0 - version: link:../npm/linux-arm64-gnu - '@openclaw/fs-safe-native-linux-arm64-musl': - specifier: 0.5.0 - version: link:../npm/linux-arm64-musl - '@openclaw/fs-safe-native-linux-x64-gnu': - specifier: 0.5.0 - version: link:../npm/linux-x64-gnu - '@openclaw/fs-safe-native-linux-x64-musl': - specifier: 0.5.0 - version: link:../npm/linux-x64-musl - '@openclaw/fs-safe-native-win32-x64-msvc': - specifier: 0.5.0 - version: link:../npm/win32-x64-msvc - - npm/darwin-arm64: {} - - npm/darwin-x64: {} - - npm/linux-arm64-gnu: {} - - npm/linux-arm64-musl: {} - - npm/linux-x64-gnu: {} - - npm/linux-x64-musl: {} - - npm/win32-x64-msvc: {} + native: {} packages: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1de7f9c..735f30d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,5 @@ packages: - native - - npm/* linkWorkspacePackages: true diff --git a/scripts/assemble-native-binaries.mjs b/scripts/assemble-native-binaries.mjs new file mode 100644 index 0000000..4abccd5 --- /dev/null +++ b/scripts/assemble-native-binaries.mjs @@ -0,0 +1,41 @@ +import { + copyFileSync, + mkdirSync, + readdirSync, + rmSync, + statSync, +} from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; +import { nativeTargets } from "./native-targets.mjs"; + +const sourceIndex = process.argv.indexOf("--source"); +const destinationIndex = process.argv.indexOf("--destination"); +const sourceRoot = resolve(sourceIndex >= 0 ? process.argv[sourceIndex + 1] : "artifacts"); +const destinationRoot = resolve( + destinationIndex >= 0 ? process.argv[destinationIndex + 1] : "dist/native", +); + +function filesBelow(directory) { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = join(directory, entry.name); + return entry.isDirectory() ? filesBelow(path) : [path]; + }); +} + +const artifacts = filesBelow(sourceRoot); +rmSync(destinationRoot, { recursive: true, force: true }); +for (const target of nativeTargets) { + const matches = artifacts.filter((path) => basename(path) === target.artifact); + if (matches.length !== 1) { + throw new Error( + `${target.label} requires exactly one ${target.artifact}; found ${matches.length}`, + ); + } + const [source] = matches; + if (statSync(source).size === 0) throw new Error(`${target.artifact} is empty`); + const destination = join(destinationRoot, target.label, "fs-safe-native.node"); + mkdirSync(dirname(destination), { recursive: true }); + copyFileSync(source, destination); + if (statSync(destination).size === 0) throw new Error(`${destination} is empty after copy`); + console.log(`${target.label}: ${statSync(destination).size} bytes`); +} diff --git a/scripts/bench-publish.mjs b/scripts/bench-publish.mjs index 7a34150..d8e4a95 100644 --- a/scripts/bench-publish.mjs +++ b/scripts/bench-publish.mjs @@ -1,4 +1,3 @@ -import { createRequire } from "node:module"; import fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; @@ -10,12 +9,12 @@ import { configureFsSafeNative, } from "../dist/native-config.js"; import { + __loadBundledNativeForTest, __resetNativeLoaderForTest, __setNativeLoaderForTest, } from "../dist/native.js"; -const require = createRequire(import.meta.url); -const native = require("../native"); +const native = __loadBundledNativeForTest(); const sizes = [ { label: "4 KiB", bytes: 4 * 1024, iterations: 7 }, { label: "1 MiB", bytes: 1024 * 1024, iterations: 7 }, diff --git a/scripts/check-release-packages.mjs b/scripts/check-release-packages.mjs index 36ca127..22343a8 100644 --- a/scripts/check-release-packages.mjs +++ b/scripts/check-release-packages.mjs @@ -4,133 +4,160 @@ import { mkdtempSync, mkdirSync, readFileSync, + renameSync, rmSync, statSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { hostNativeTarget, nativeTargets } from "./native-targets.mjs"; const outputIndex = process.argv.indexOf("--output"); const outputDir = resolve(outputIndex >= 0 ? process.argv[outputIndex + 1] : "release-artifacts"); +const allowHostOnly = process.argv.includes("--allow-host-only"); mkdirSync(outputDir, { recursive: true }); -const packageDirs = [ - "npm/linux-x64-gnu", - "npm/linux-x64-musl", - "npm/linux-arm64-gnu", - "npm/linux-arm64-musl", - "npm/darwin-x64", - "npm/darwin-arm64", - "npm/win32-x64-msvc", - "native", - ".", -]; - -const packages = packageDirs.map((directory) => ({ - directory, - packageJson: JSON.parse(readFileSync(join(directory, "package.json"), "utf8")), -})); -const rootVersion = packages.at(-1).packageJson.version; - -function normalizeRepository(repository) { - const value = typeof repository === "string" ? repository : repository?.url; - return String(value ?? "") - .replace(/^github:/u, "https://github.com/") - .replace(/^git\+/u, "") - .replace(/\.git$/iu, "") - .replace(/\/+$/u, ""); +const pkg = JSON.parse(readFileSync("package.json", "utf8")); +if (pkg.name !== "@openclaw/fs-safe") throw new Error(`unexpected package name ${pkg.name}`); +if (pkg.author !== "OpenClaw Team ") { + throw new Error("root package has unexpected author metadata"); } - -for (const entry of packages) { - const pkg = entry.packageJson; - if (pkg.version !== rootVersion) { - throw new Error(`${pkg.name} version ${pkg.version} does not match ${rootVersion}`); - } - if (pkg.author !== "OpenClaw Team ") { - throw new Error(`${pkg.name} has unexpected author metadata`); - } - if (normalizeRepository(pkg.repository) !== "https://github.com/openclaw/fs-safe") { - throw new Error(`${pkg.name} has unexpected repository metadata`); - } - if (pkg.publishConfig?.access !== "public" || pkg.publishConfig?.provenance !== true) { - throw new Error(`${pkg.name} must publish publicly with provenance`); - } +if (pkg.publishConfig?.access !== "public" || pkg.publishConfig?.provenance !== true) { + throw new Error("root package must publish publicly with provenance"); +} +if (pkg.optionalDependencies?.["@openclaw/fs-safe-native"]) { + throw new Error("root package must not depend on the retired native loader package"); } -const nativePackage = packages.find((entry) => entry.directory === "native").packageJson; -for (const [name, version] of Object.entries(nativePackage.optionalDependencies ?? {})) { - if (version !== rootVersion || !packages.some((entry) => entry.packageJson.name === name)) { - throw new Error(`native optional dependency ${name}@${version} is not a release package`); - } +const expectedTargets = allowHostOnly ? [hostNativeTarget()].filter(Boolean) : nativeTargets; +if (expectedTargets.length === 0) { + throw new Error(`no bundled native target for ${process.platform}-${process.arch}`); } -const rootPackage = packages.at(-1).packageJson; -if (rootPackage.optionalDependencies?.["@openclaw/fs-safe-native"] !== rootVersion) { - throw new Error("root package must depend on the matching native package version"); +for (const target of expectedTargets) { + const binary = join("dist", "native", target.label, "fs-safe-native.node"); + if (!existsSync(binary) || statSync(binary).size === 0) { + throw new Error(`missing or empty bundled binary ${binary}`); + } } -const manifest = []; -for (const entry of packages) { - const packed = JSON.parse(execFileSync( +const packed = JSON.parse( + execFileSync( "npm", ["pack", "--json", "--ignore-scripts", "--pack-destination", outputDir], - { cwd: resolve(entry.directory), encoding: "utf8" }, - )); - const [artifact] = packed; - if (!artifact?.filename || !artifact?.integrity || artifact.version !== rootVersion) { - throw new Error(`npm pack returned incomplete metadata for ${entry.packageJson.name}`); - } - const paths = new Set(artifact.files.map((file) => file.path)); - if (entry.directory.startsWith("npm/")) { - const binary = entry.packageJson.main; - if (!paths.has(binary) || !existsSync(join(entry.directory, binary))) { - throw new Error(`${entry.packageJson.name} is missing ${binary}`); - } - if (statSync(join(entry.directory, binary)).size === 0) { - throw new Error(`${entry.packageJson.name} contains an empty native binary`); - } - } else if (entry.directory === "native") { - for (const expected of ["index.js", "index.d.ts", "package.json"]) { - if (!paths.has(expected)) throw new Error(`${entry.packageJson.name} is missing ${expected}`); - } - } else { - for (const expected of ["dist/index.js", "dist/index.d.ts", "package.json"]) { - if (!paths.has(expected)) throw new Error(`${entry.packageJson.name} is missing ${expected}`); - } + { encoding: "utf8" }, + ), +); +const [artifact] = packed; +if (!artifact?.filename || !artifact?.integrity || artifact.version !== pkg.version) { + throw new Error("npm pack returned incomplete root-package metadata"); +} +const paths = new Set(artifact.files.map((file) => file.path)); +for (const expected of ["dist/index.js", "dist/index.d.ts", "package.json"]) { + if (!paths.has(expected)) throw new Error(`packed root package is missing ${expected}`); +} +for (const target of expectedTargets) { + const binary = `dist/native/${target.label}/fs-safe-native.node`; + if (!paths.has(binary)) throw new Error(`packed root package is missing ${binary}`); +} +if (!allowHostOnly) { + const packedNativePaths = [...paths].filter((path) => path.endsWith("/fs-safe-native.node")); + if (packedNativePaths.length !== nativeTargets.length) { + throw new Error(`packed root package has ${packedNativePaths.length} native binaries, expected 7`); } - manifest.push({ - name: entry.packageJson.name, - version: rootVersion, - filename: artifact.filename, - integrity: artifact.integrity, - }); } +const manifest = [ + { + name: pkg.name, + version: pkg.version, + filename: artifact.filename, + integrity: artifact.integrity, + size: artifact.size, + unpackedSize: artifact.unpackedSize, + }, +]; writeFileSync(join(outputDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); const smoke = mkdtempSync(join(tmpdir(), "fs-safe-release-smoke-")); try { writeFileSync(join(smoke, "package.json"), '{"private":true,"type":"module"}\n'); - const wanted = [ - "@openclaw/fs-safe", - "@openclaw/fs-safe-native", - "@openclaw/fs-safe-native-linux-x64-gnu", - ].map((name) => join(outputDir, manifest.find((entry) => entry.name === name).filename)); execFileSync( "npm", - ["install", "--force", "--ignore-scripts", "--no-audit", "--no-fund", "--no-package-lock", ...wanted], + [ + "install", + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--no-package-lock", + join(outputDir, artifact.filename), + ], { cwd: smoke, stdio: "pipe" }, ); execFileSync( process.execPath, - ["--input-type=module", "--eval", "await import('@openclaw/fs-safe'); await import('@openclaw/fs-safe/config');"], + [ + "--input-type=module", + "--eval", + "await import('@openclaw/fs-safe'); await import('@openclaw/fs-safe/config');", + ], { cwd: smoke, stdio: "pipe" }, ); - execFileSync( + + const fixture = join(smoke, "fixture.txt"); + writeFileSync(fixture, "abc"); + const nativeProof = execFileSync( process.execPath, - ["--eval", "const n=require('@openclaw/fs-safe-native'); if(typeof n.openBeneath!=='function') process.exit(1)"], - { cwd: smoke, stdio: "pipe" }, - ); + [ + "--input-type=module", + "--eval", + "import {configureFsSafeNative} from '@openclaw/fs-safe';" + + "import {sha256File} from '@openclaw/fs-safe/durability';" + + "configureFsSafeNative({mode:'require'});" + + `const result=await sha256File(${JSON.stringify(fixture)});` + + "if(result.digest!=='ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad')throw new Error('native hash mismatch');" + + "console.log(JSON.stringify(result));", + ], + { cwd: smoke, encoding: "utf8" }, + ).trim(); + + const installedNative = join(smoke, "node_modules", "@openclaw", "fs-safe", "dist", "native"); + const hiddenNative = `${installedNative}.removed`; + renameSync(installedNative, hiddenNative); + const fallbackProof = execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + "import {configureFsSafeNative} from '@openclaw/fs-safe';" + + "import {sha256File} from '@openclaw/fs-safe/durability';" + + "configureFsSafeNative({mode:'auto'});" + + `const result=await sha256File(${JSON.stringify(fixture)});` + + "if(result.digest!=='ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad')throw new Error('fallback hash mismatch');" + + "console.log(JSON.stringify(result));", + ], + { cwd: smoke, encoding: "utf8" }, + ).trim(); + const requiredProof = execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + "import {configureFsSafeNative} from '@openclaw/fs-safe';" + + "import {sha256File} from '@openclaw/fs-safe/durability';" + + "configureFsSafeNative({mode:'require'});" + + `try{await sha256File(${JSON.stringify(fixture)});throw new Error('native binding unexpectedly loaded')}` + + "catch(error){if(error.code!=='helper-unavailable')throw error;console.log(error.code)}", + ], + { cwd: smoke, encoding: "utf8" }, + ).trim(); + console.log(`native binding: ${nativeProof}`); + console.log(`auto fallback without bundled directory: ${fallbackProof}`); + console.log(`required mode without bundled directory: ${requiredProof}`); } finally { rmSync(smoke, { recursive: true, force: true }); } + +console.log(`packed tarball: ${artifact.filename}`); +console.log(`gzipped size: ${artifact.size} bytes`); +console.log(`unpacked size: ${artifact.unpackedSize} bytes`); diff --git a/scripts/harden-native-loader.mjs b/scripts/harden-native-loader.mjs deleted file mode 100644 index 4a9f983..0000000 --- a/scripts/harden-native-loader.mjs +++ /dev/null @@ -1,45 +0,0 @@ -import { readFileSync, writeFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; - -const root = fileURLToPath(new URL("../", import.meta.url)); -const loaderPath = fileURLToPath(new URL("../native/index.js", import.meta.url)); -const typesPath = fileURLToPath(new URL("../native/index.d.ts", import.meta.url)); -const templatePath = fileURLToPath(new URL("./native-loader.cjs", import.meta.url)); -const loader = readFileSync(loaderPath, "utf8"); -const hardened = readFileSync(templatePath, "utf8"); - -if (hardened !== loader) { - writeFileSync(loaderPath, hardened); - console.log(`hardened ${loaderPath.slice(root.length)}`); -} - -const generatedTypes = readFileSync(typesPath, "utf8"); -const hardenedTypes = generatedTypes - .replace( - /inspectArchiveNative\(([^)]*)\): Promise/, - "inspectArchiveNative($1): Promise>", - ) - .replace( - /extractArchiveNative\(([^)]*)\): Promise/, - "extractArchiveNative($1): Promise", - ) - .replace( - /readArchiveEntryNative\(([^)]*)\): Promise/, - "readArchiveEntryNative($1): Promise", - ) - .replace( - /sha256File\(([^)]*)\): Promise/, - "sha256File($1): Promise", - ) - .replace( - /copyFileRangeExclusive\(([^)]*)\): Promise/, - "copyFileRangeExclusive($1): Promise", - ) - .replace( - /containment: string/, - "containment: 'kernel-atomic' | 'best-effort'", - ); -if (hardenedTypes !== generatedTypes) { - writeFileSync(typesPath, hardenedTypes); - console.log(`hardened ${typesPath.slice(root.length)}`); -} diff --git a/scripts/native-loader.cjs b/scripts/native-loader.cjs deleted file mode 100644 index 687acf4..0000000 --- a/scripts/native-loader.cjs +++ /dev/null @@ -1,179 +0,0 @@ -// prettier-ignore -/* eslint-disable */ -// @ts-nocheck -/* Checked-in N-API loader for the seven platform packages this project ships. */ - -const { closeSync, openSync, readSync, readdirSync } = require('fs') -const loadErrors = [] - -const isFileMusl = (file) => file.includes('libc.musl-') || file.includes('ld-musl-') - -const isMuslFromReport = () => { - try { - if (!process.report || typeof process.report.getReport !== 'function') return null - const report = process.report.getReport() - if (report?.header?.glibcVersionRuntime) return false - if (report?.sharedObjects?.some(isFileMusl)) return true - } catch { - // Continue with filesystem and ELF inspection. - } - return null -} - -const isMuslFromFilesystem = () => { - for (const directory of ['/lib', '/usr/lib']) { - try { - if (readdirSync(directory).some(isFileMusl)) return true - } catch { - // A missing or unreadable conventional library directory is inconclusive. - } - } - return null -} - -const readUInt = (buffer, offset, bytes, littleEndian) => { - if (offset < 0 || offset + bytes > buffer.length) return null - if (bytes === 2) return littleEndian ? buffer.readUInt16LE(offset) : buffer.readUInt16BE(offset) - if (bytes === 4) return littleEndian ? buffer.readUInt32LE(offset) : buffer.readUInt32BE(offset) - if (bytes === 8) { - const value = littleEndian ? buffer.readBigUInt64LE(offset) : buffer.readBigUInt64BE(offset) - return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null - } - return null -} - -const isMuslFromElfInterpreter = () => { - let fd - try { - fd = openSync(process.execPath, 'r') - const header = Buffer.alloc(64) - if (readSync(fd, header, 0, header.length, 0) < 52) return null - if (!header.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return null - - const elfClass = header[4] - const dataEncoding = header[5] - if ((elfClass !== 1 && elfClass !== 2) || (dataEncoding !== 1 && dataEncoding !== 2)) { - return null - } - const littleEndian = dataEncoding === 1 - const is64Bit = elfClass === 2 - const tableOffset = readUInt(header, is64Bit ? 32 : 28, is64Bit ? 8 : 4, littleEndian) - const entrySize = readUInt(header, is64Bit ? 54 : 42, 2, littleEndian) - const entryCount = readUInt(header, is64Bit ? 56 : 44, 2, littleEndian) - if (tableOffset === null || entrySize === null || entryCount === null || - entrySize < (is64Bit ? 56 : 32) || entryCount > 1024) return null - - const programHeader = Buffer.alloc(entrySize) - for (let index = 0; index < entryCount; index += 1) { - const offset = tableOffset + index * entrySize - if (readSync(fd, programHeader, 0, entrySize, offset) !== entrySize) return null - if (readUInt(programHeader, 0, 4, littleEndian) !== 3) continue // PT_INTERP - const interpreterOffset = readUInt(programHeader, is64Bit ? 8 : 4, is64Bit ? 8 : 4, littleEndian) - const interpreterSize = readUInt(programHeader, is64Bit ? 32 : 16, is64Bit ? 8 : 4, littleEndian) - if (interpreterOffset === null || interpreterSize === null || - interpreterSize < 1 || interpreterSize > 4096) return null - const interpreter = Buffer.alloc(interpreterSize) - if (readSync(fd, interpreter, 0, interpreterSize, interpreterOffset) !== interpreterSize) { - return null - } - return isFileMusl(interpreter.toString('utf8')) - } - } catch { - return null - } finally { - if (fd !== undefined) { - try { closeSync(fd) } catch { - // Best-effort detection must never turn close failure into import failure. - } - } - } - return null -} - -const isMusl = () => { - if (process.platform !== 'linux') return false - for (const detector of [isMuslFromReport, isMuslFromFilesystem, isMuslFromElfInterpreter]) { - const result = detector() - if (result !== null) return result - } - // Unknown Linux libc: try the glibc package and let its require fail normally. - return false -} - -const targetSuffix = () => { - if (process.platform === 'win32' && process.arch === 'x64') return 'win32-x64-msvc' - if (process.platform === 'darwin' && process.arch === 'x64') return 'darwin-x64' - if (process.platform === 'darwin' && process.arch === 'arm64') return 'darwin-arm64' - if (process.platform === 'linux' && (process.arch === 'x64' || process.arch === 'arm64')) { - return `linux-${process.arch}-${isMusl() ? 'musl' : 'gnu'}` - } - return null -} - -const tryRequire = (specifier) => { - try { - return require(specifier) - } catch (error) { - loadErrors.push(error) - return null - } -} - -const requireNative = () => { - if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { - const explicit = tryRequire(process.env.NAPI_RS_NATIVE_LIBRARY_PATH) - if (explicit) return explicit - } - - const suffix = targetSuffix() - if (!suffix) { - loadErrors.push(new Error(`Unsupported OS or architecture: ${process.platform}-${process.arch}`)) - return null - } - const local = tryRequire(`./fs-safe-native.${suffix}.node`) - if (local) return local - - const packageName = `@openclaw/fs-safe-native-${suffix}` - const binding = tryRequire(packageName) - if (!binding) return null - if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - const expectedVersion = require('./package.json').version - const bindingVersion = require(`${packageName}/package.json`).version - if (bindingVersion !== expectedVersion) { - loadErrors.push(new Error( - `Native binding package version mismatch, expected ${expectedVersion} but got ${bindingVersion}. ` + - 'Reinstall dependencies to fix this issue.', - )) - return null - } - } - return binding -} - -const nativeBinding = requireNative() -if (!nativeBinding) { - const error = new Error( - `Cannot find native binding for ${process.platform}-${process.arch}; ` + - 'install the matching optional @openclaw/fs-safe-native platform package.', - ) - error.cause = loadErrors.reduceRight((cause, current) => { - current.cause = cause - return current - }, undefined) - throw error -} - -module.exports = nativeBinding -module.exports.cloneFileExclusive = nativeBinding.cloneFileExclusive -module.exports.copyFileRangeExclusive = nativeBinding.copyFileRangeExclusive -module.exports.createPrivateDirectory = nativeBinding.createPrivateDirectory -module.exports.extractArchiveNative = nativeBinding.extractArchiveNative -module.exports.fstatIdentity = nativeBinding.fstatIdentity -module.exports.inspectArchiveNative = nativeBinding.inspectArchiveNative -module.exports.linkBeneath = nativeBinding.linkBeneath -module.exports.mkdirBeneath = nativeBinding.mkdirBeneath -module.exports.openBeneath = nativeBinding.openBeneath -module.exports.readArchiveEntryNative = nativeBinding.readArchiveEntryNative -module.exports.readOwnerAndDacl = nativeBinding.readOwnerAndDacl -module.exports.renameNoReplace = nativeBinding.renameNoReplace -module.exports.sha256File = nativeBinding.sha256File diff --git a/scripts/native-mode-smoke.mjs b/scripts/native-mode-smoke.mjs index 9ebcb88..52c586d 100644 --- a/scripts/native-mode-smoke.mjs +++ b/scripts/native-mode-smoke.mjs @@ -1,8 +1,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { createRequire } from "node:module"; import { configureFsSafeNative, root } from "../dist/index.js"; +import { sha256File } from "../dist/file-hash.js"; const mode = process.argv[2]; if (!new Set(["auto", "require", "off"]).has(mode)) { @@ -10,11 +10,13 @@ if (!new Set(["auto", "require", "off"]).has(mode)) { } configureFsSafeNative({ mode }); if (mode !== "off") { - const require = createRequire(import.meta.url); - const binding = require("../native"); - if (typeof binding.openBeneath !== "function") throw new Error("native binding did not load"); -} else { - process.env.NAPI_RS_NATIVE_LIBRARY_PATH = path.join(os.tmpdir(), "must-not-load.node"); + const fixture = path.join(os.tmpdir(), `fs-safe-native-mode-${process.pid}`); + await fs.writeFile(fixture, "native"); + try { + await sha256File(fixture); + } finally { + await fs.rm(fixture, { force: true }); + } } const directory = await fs.mkdtemp(path.join(os.tmpdir(), `fs-safe-mode-${mode}-`)); diff --git a/scripts/native-smoke.mjs b/scripts/native-smoke.mjs index 15ed0a2..8afb4e8 100644 --- a/scripts/native-smoke.mjs +++ b/scripts/native-smoke.mjs @@ -1,11 +1,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { createRequire } from "node:module"; +import { __loadBundledNativeForTest } from "../dist/native.js"; -const require = createRequire(import.meta.url); console.log("native smoke: load binding"); -const native = require("../native"); +const native = __loadBundledNativeForTest(); const root = fs.mkdtempSync(path.join(os.tmpdir(), "fs-safe-native-smoke-")); const rootFd = fs.openSync(root, fs.constants.O_RDONLY); try { diff --git a/scripts/native-targets.mjs b/scripts/native-targets.mjs new file mode 100644 index 0000000..3f5c3d2 --- /dev/null +++ b/scripts/native-targets.mjs @@ -0,0 +1,52 @@ +export const nativeTargets = [ + { + label: "linux-x64-gnu", + rust: "x86_64-unknown-linux-gnu", + artifact: "fs-safe-native.linux-x64-gnu.node", + }, + { + label: "linux-x64-musl", + rust: "x86_64-unknown-linux-musl", + artifact: "fs-safe-native.linux-x64-musl.node", + }, + { + label: "linux-arm64-gnu", + rust: "aarch64-unknown-linux-gnu", + artifact: "fs-safe-native.linux-arm64-gnu.node", + }, + { + label: "linux-arm64-musl", + rust: "aarch64-unknown-linux-musl", + artifact: "fs-safe-native.linux-arm64-musl.node", + }, + { + label: "darwin-x64", + rust: "x86_64-apple-darwin", + artifact: "fs-safe-native.darwin-x64.node", + }, + { + label: "darwin-arm64", + rust: "aarch64-apple-darwin", + artifact: "fs-safe-native.darwin-arm64.node", + }, + { + label: "win32-x64-msvc", + rust: "x86_64-pc-windows-msvc", + artifact: "fs-safe-native.win32-x64-msvc.node", + }, +]; + +export function hostNativeTarget() { + const platformArch = `${process.platform}-${process.arch}`; + if (platformArch === "darwin-x64" || platformArch === "darwin-arm64") { + return nativeTargets.find((target) => target.label === platformArch); + } + if (platformArch === "win32-x64") { + return nativeTargets.find((target) => target.label === "win32-x64-msvc"); + } + if (platformArch === "linux-x64" || platformArch === "linux-arm64") { + const libc = process.report?.getReport().header?.glibcVersionRuntime ? "gnu" : "musl"; + return nativeTargets.find((target) => target.label === `${platformArch}-${libc}`); + } + return undefined; +} diff --git a/scripts/prepack-build.mjs b/scripts/prepack-build.mjs index 6e224b8..32390fd 100644 --- a/scripts/prepack-build.mjs +++ b/scripts/prepack-build.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { existsSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; import { dirname, resolve } from "node:path"; @@ -19,21 +19,26 @@ function resolveTypeScriptCompiler() { } } -function run(command, args, env = {}) { - const result = spawnSync(command, args, { - stdio: "inherit", - env: { ...process.env, ...env }, - }); - if (result.status !== 0) { - process.exit(result.status ?? 1); - } -} - -let tscBin = resolveTypeScriptCompiler(); - +const tscBin = resolveTypeScriptCompiler(); if (!tscBin || !existsSync(tscBin)) { throw new Error("TypeScript compiler is unavailable; run pnpm install before packing"); } -rmSync("dist", { recursive: true, force: true }); -run(process.execPath, [tscBin, "-p", "tsconfig.json"]); +const nativeDirectory = resolve("dist/native"); +const heldNativeDirectory = resolve(".fs-safe-prepack-native"); +rmSync(heldNativeDirectory, { recursive: true, force: true }); +if (existsSync(nativeDirectory)) renameSync(nativeDirectory, heldNativeDirectory); + +try { + rmSync("dist", { recursive: true, force: true }); + const result = spawnSync(process.execPath, [tscBin, "-p", "tsconfig.json"], { + stdio: "inherit", + env: process.env, + }); + if (result.status !== 0) process.exitCode = result.status ?? 1; +} finally { + if (existsSync(heldNativeDirectory)) { + mkdirSync("dist", { recursive: true }); + renameSync(heldNativeDirectory, nativeDirectory); + } +} diff --git a/scripts/release-graph-smoke.mjs b/scripts/release-graph-smoke.mjs deleted file mode 100644 index 59f32cd..0000000 --- a/scripts/release-graph-smoke.mjs +++ /dev/null @@ -1,241 +0,0 @@ -import { execFileSync } from "node:child_process"; -import { - copyFileSync, - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - readdirSync, - rmSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { basename, dirname, join, resolve } from "node:path"; - -const repositoryRoot = resolve(import.meta.dirname, ".."); -const proofPath = resolve(repositoryRoot, process.env.FS_SAFE_RELEASE_GRAPH_PROOF ?? "release-graph-proof.json"); -const workspace = mkdtempSync(join(tmpdir(), "fs-safe-release-graph-")); -const packsDirectory = join(workspace, "packs"); -const scratchDirectory = join(workspace, "consumer"); -const pnpmCli = process.env.npm_execpath; - -if (!pnpmCli || !existsSync(pnpmCli) || !basename(pnpmCli).startsWith("pnpm")) { - throw new Error("release graph smoke must run through pnpm"); -} - -const npmCli = resolveNpmCli(); -const cleanEnvironment = Object.fromEntries( - Object.entries(process.env).filter(([key]) => !key.toLowerCase().startsWith("npm_config_")), -); - -const host = resolveHostPackage(); -const nativeBinary = join(repositoryRoot, "native", host.binary); -const packageBinary = join(repositoryRoot, host.directory, host.binary); -const previousPackageBinary = existsSync(packageBinary) ? readFileSync(packageBinary) : undefined; - -try { - mkdirSync(packsDirectory); - mkdirSync(scratchDirectory); - if (!existsSync(nativeBinary)) { - throw new Error(`host native binary was not built: native/${host.binary}`); - } - copyFileSync(nativeBinary, packageBinary); - - const packageDirectories = [ - ...readdirSync(join(repositoryRoot, "npm"), { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .map((entry) => `npm/${entry.name}`) - .toSorted(), - "native", - ".", - ]; - const tarballs = new Map(); - for (const directory of packageDirectories) { - const packageRoot = resolve(repositoryRoot, directory); - const packageJson = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")); - const before = new Set(existsSync(packsDirectory) ? readdirSync(packsDirectory) : []); - runPnpm(["--dir", packageRoot, "pack", "--pack-destination", packsDirectory]); - const filename = readdirSync(packsDirectory).find((candidate) => !before.has(candidate)); - if (!filename) { - throw new Error(`pnpm pack did not produce an artifact for ${packageJson.name}`); - } - tarballs.set(packageJson.name, join(packsDirectory, filename)); - } - - const dependencies = Object.fromEntries( - ["@openclaw/fs-safe", "@openclaw/fs-safe-native", host.packageName].map((name) => [ - name, - `file:${tarballs.get(name)}`, - ]), - ); - writeFileSync( - join(scratchDirectory, "package.json"), - `${JSON.stringify({ private: true, type: "module", dependencies }, null, 2)}\n`, - ); - runNpm(["install", "--force", "--ignore-scripts", "--no-audit", "--no-fund", "--no-package-lock"], { - cwd: scratchDirectory, - }); - copyFileSync( - join(repositoryRoot, "scripts", "packed-final-smoke.mjs"), - join(scratchDirectory, "packed-final-smoke.mjs"), - ); - - const nativeOutput = runNode( - `import fs from 'node:fs';import os from 'node:os';import path from 'node:path';` + - `import {configureFsSafeNative} from '@openclaw/fs-safe';` + - `import {sha256File} from '@openclaw/fs-safe/durability';` + - `import {readOwnerAndDacl} from '@openclaw/fs-safe/permissions';` + - `const imported=await import('@openclaw/fs-safe-native');const n=imported.default??imported;` + - `if(typeof n.openBeneath!=='function')throw new Error('openBeneath missing');` + - `configureFsSafeNative({mode:'require'});const d=fs.mkdtempSync(path.join(os.tmpdir(),'fs-safe-hash-'));` + - `try{const p=path.join(d,'input');fs.writeFileSync(p,'abc');const h=await sha256File(p);` + - `if(h.digest!=='ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad')` + - `throw new Error('native hash mismatch');` + - `const a=readOwnerAndDacl(d);` + - `if(process.platform==='win32'&&(a.status!=='supported'||!/^s-/i.test(a.currentUserSid)))` + - `throw new Error('current process user SID missing');` + - `console.log(JSON.stringify({nativeBinding:'loaded',openBeneath:typeof n.openBeneath,publicHash:h,publicDacl:a}));}` + - `finally{fs.rmSync(d,{recursive:true,force:true});}`, - ); - const offOutput = runNode( - `import fs from 'node:fs';import os from 'node:os';import path from 'node:path';` + - `import {configureFsSafeNative} from '@openclaw/fs-safe';` + - `import {publishFileExclusive} from '@openclaw/fs-safe/durability';` + - `configureFsSafeNative({mode:'off'});` + - `const d=fs.mkdtempSync(path.join(os.tmpdir(),'fs-safe-off-'));` + - `try{const s=path.join(d,'source');const t=path.join(d,'target');fs.writeFileSync(s,'fallback');` + - `const r=await publishFileExclusive({sourcePath:s,targetPath:t,strategy:'link-or-copy'});` + - `if(fs.readFileSync(t,'utf8')!=='fallback')throw new Error('fallback content mismatch');` + - `console.log(JSON.stringify({nativeMode:'off',fallback:r.method}));}` + - `finally{fs.rmSync(d,{recursive:true,force:true});}`, - { NAPI_RS_NATIVE_LIBRARY_PATH: join(workspace, "must-not-load.node") }, - ); - const finalRoundOutput = runNodeFile("packed-final-smoke.mjs"); - - rmSync(join(scratchDirectory, "node_modules", "@openclaw", "fs-safe-native"), { - force: true, - recursive: true, - }); - rmSync(join(scratchDirectory, "node_modules", "@openclaw", host.packageName.slice("@openclaw/".length)), { - force: true, - recursive: true, - }); - const requireOutput = runNode( - `import fs from 'node:fs';import os from 'node:os';import path from 'node:path';` + - `import {configureFsSafeNative} from '@openclaw/fs-safe';` + - `import {publishFileExclusive} from '@openclaw/fs-safe/durability';` + - `configureFsSafeNative({mode:'require'});` + - `const d=fs.mkdtempSync(path.join(os.tmpdir(),'fs-safe-require-'));` + - `try{const s=path.join(d,'source');fs.writeFileSync(s,'require');` + - `try{await publishFileExclusive({sourcePath:s,targetPath:path.join(d,'target'),strategy:'rename-noreplace'});` + - `throw new Error('required native binding unexpectedly available');}` + - `catch(e){if(e.code!=='helper-unavailable')throw e;` + - `console.log(JSON.stringify({nativeMode:'require',bindingRemoved:true,errorCode:e.code}));}}` + - `finally{fs.rmSync(d,{recursive:true,force:true});}`, - ); - - const proof = { - schemaVersion: 1, - platform: `${process.platform}-${process.arch}`, - packagesPacked: packageDirectories.length, - hostPackage: host.packageName, - native: JSON.parse(nativeOutput), - fallback: JSON.parse(offOutput), - finalRound: JSON.parse(finalRoundOutput), - missingRequiredBinding: JSON.parse(requireOutput), - }; - writeFileSync(proofPath, `${JSON.stringify(proof, null, 2)}\n`); - console.log("release graph smoke: PASS"); - console.log(JSON.stringify(proof, null, 2)); -} finally { - if (previousPackageBinary) { - writeFileSync(packageBinary, previousPackageBinary); - } else { - rmSync(packageBinary, { force: true }); - } - rmSync(workspace, { force: true, recursive: true }); -} - -function resolveHostPackage() { - const key = `${process.platform}-${process.arch}`; - const hosts = { - "darwin-arm64": { - directory: "npm/darwin-arm64", - packageName: "@openclaw/fs-safe-native-darwin-arm64", - binary: "fs-safe-native.darwin-arm64.node", - }, - "darwin-x64": { - directory: "npm/darwin-x64", - packageName: "@openclaw/fs-safe-native-darwin-x64", - binary: "fs-safe-native.darwin-x64.node", - }, - "linux-arm64": { - directory: "npm/linux-arm64-gnu", - packageName: "@openclaw/fs-safe-native-linux-arm64-gnu", - binary: "fs-safe-native.linux-arm64-gnu.node", - }, - "linux-x64": { - directory: "npm/linux-x64-gnu", - packageName: "@openclaw/fs-safe-native-linux-x64-gnu", - binary: "fs-safe-native.linux-x64-gnu.node", - }, - "win32-x64": { - directory: "npm/win32-x64-msvc", - packageName: "@openclaw/fs-safe-native-win32-x64-msvc", - binary: "fs-safe-native.win32-x64-msvc.node", - }, - }; - const hostPackage = hosts[key]; - if (!hostPackage) { - throw new Error(`release graph smoke does not support ${key}`); - } - return hostPackage; -} - -function runPnpm(args) { - return execFileSync(process.execPath, [pnpmCli, ...args], { - cwd: repositoryRoot, - encoding: "utf8", - env: cleanEnvironment, - stdio: "pipe", - }); -} - -function runNpm(args, options) { - return execFileSync(process.execPath, [npmCli, ...args], { - ...options, - encoding: "utf8", - env: cleanEnvironment, - stdio: "pipe", - }); -} - -function runNode(source, additionalEnvironment = {}) { - return execFileSync(process.execPath, ["--input-type=module", "--eval", source], { - cwd: scratchDirectory, - encoding: "utf8", - env: { ...cleanEnvironment, ...additionalEnvironment }, - stdio: "pipe", - }).trim(); -} - -function runNodeFile(filename) { - return execFileSync(process.execPath, [filename], { - cwd: scratchDirectory, - encoding: "utf8", - env: cleanEnvironment, - stdio: "pipe", - }).trim(); -} - -function resolveNpmCli() { - const candidates = [ - join(dirname(process.execPath), "..", "lib", "node_modules", "npm", "bin", "npm-cli.js"), - join(dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js"), - ]; - const npm = candidates.find((candidate) => existsSync(candidate)); - if (!npm) { - throw new Error("could not resolve npm-cli.js from the current Node installation"); - } - return npm; -} diff --git a/scripts/stage-host-native.mjs b/scripts/stage-host-native.mjs new file mode 100644 index 0000000..5430574 --- /dev/null +++ b/scripts/stage-host-native.mjs @@ -0,0 +1,20 @@ +import { copyFileSync, mkdirSync, statSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { hostNativeTarget } from "./native-targets.mjs"; + +const repositoryRoot = resolve(import.meta.dirname, ".."); +const target = hostNativeTarget(); +if (!target) throw new Error(`no bundled native target for ${process.platform}-${process.arch}`); + +const source = resolve(repositoryRoot, "native", target.artifact); +const destination = resolve( + repositoryRoot, + "dist", + "native", + target.label, + "fs-safe-native.node", +); +if (statSync(source).size === 0) throw new Error(`${target.artifact} is empty`); +mkdirSync(dirname(destination), { recursive: true }); +copyFileSync(source, destination); +console.log(`staged ${target.label} binding at dist/native/${target.label}/fs-safe-native.node`); diff --git a/src/archive-kind.ts b/src/archive-kind.ts index bf8342b..25946f1 100644 --- a/src/archive-kind.ts +++ b/src/archive-kind.ts @@ -16,8 +16,8 @@ function requireNativeArchiveKind(kind: "tar-bzip2" | "tar-zstd"): ArchiveKind { if (!getNativeBinding()) { throw new FsSafeError( "helper-unavailable", - `${kind} archives require the optional native binding; install the matching ` + - "@openclaw/fs-safe-native platform package and use FS_SAFE_NATIVE_MODE=auto or require", + `${kind} archives require a supported bundled native binding; ` + + "use FS_SAFE_NATIVE_MODE=auto or require on a supported platform", ); } return kind; diff --git a/src/archive-read.ts b/src/archive-read.ts index 0187b14..c7772a0 100644 --- a/src/archive-read.ts +++ b/src/archive-read.ts @@ -252,7 +252,7 @@ export async function readArchiveEntry( if (kind === "tar-zstd" || kind === "tar-bzip2") { throw new FsSafeError( "helper-unavailable", - `${kind} archives require the optional native binding; install the matching platform package`, + `${kind} archives require a supported bundled native binding`, ); } return kind === "zip" diff --git a/src/archive.ts b/src/archive.ts index ca54ed6..5cbf2d4 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -334,7 +334,7 @@ export async function extractArchive(params: ExtractArchiveOptions): Promise; + createPrivateDirectory(path: string): void; + extractArchiveNative( + path: string, + kind: string, + rootFd: number, + plan: NativeArchivePlanEntry[], + maxMetaEntryBytes: number, + signal: AbortSignal, + ): Promise; + fstatIdentity(fd: number): NativeFileIdentity; + inspectArchiveNative( + path: string, + kind: string, + maxEntries: number, + maxMetaEntryBytes: number, + maxManifestBytes: number, + signal: AbortSignal, + ): Promise; + linkBeneath( + sourceRootFd: number, + sourceRelPath: string, + targetRootFd: number, + targetRelPath: string, + ): void; + mkdirBeneath(rootFd: number, relPath: string, mode: number): void; + openBeneath(rootFd: number, relPath: string, flags: number): NativeOpenBeneathResult; + readArchiveEntryNative( + path: string, + kind: string, + requested: string, + maxBytes: number, + maxEntries: number, + maxMetaEntryBytes: number, + signal: AbortSignal, + ): Promise; + readOwnerAndDacl(path: string): NativeWindowsSecurityFacts; + renameNoReplace( + sourceRootFd: number, + sourceRelPath: string, + targetRootFd: number, + targetRelPath: string, + ): void; + sha256File(fd: number): Promise; +} diff --git a/src/native.ts b/src/native.ts index 690164b..dec935e 100644 --- a/src/native.ts +++ b/src/native.ts @@ -1,20 +1,178 @@ +import { + closeSync, + openSync, + readSync, + readdirSync, +} from "node:fs"; import { createRequire } from "node:module"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { FsSafeError } from "./errors.js"; +import type { NativeBinding } from "./native-binding.js"; import { getFsSafeNativeConfig } from "./native-config.js"; -export type NativeBinding = typeof import("@openclaw/fs-safe-native"); +export type { NativeBinding } from "./native-binding.js"; const require = createRequire(import.meta.url); +const bundledNativeDirectory = fileURLToPath(new URL("../dist/native/", import.meta.url)); let binding: NativeBinding | undefined; let loadError: unknown; let attempted = false; -let loadBinding = (): NativeBinding => require("@openclaw/fs-safe-native") as NativeBinding; +let loadBinding = loadBundledBinding; -export function getNativeBinding(): NativeBinding | undefined { - const { mode } = getFsSafeNativeConfig(); - if (mode === "off") { +function isMuslFilename(file: string): boolean { + return file.includes("libc.musl-") || file.includes("ld-musl-"); +} + +function isMuslFromReport(): boolean | undefined { + try { + if (!process.report || typeof process.report.getReport !== "function") return undefined; + const report = process.report.getReport() as { + header?: { glibcVersionRuntime?: string }; + sharedObjects?: string[]; + }; + if (report.header?.glibcVersionRuntime) return false; + if (report.sharedObjects?.some(isMuslFilename)) return true; + } catch { + // Continue with filesystem and ELF inspection. + } + return undefined; +} + +function isMuslFromFilesystem(): boolean | undefined { + for (const directory of ["/lib", "/usr/lib"]) { + try { + if (readdirSync(directory).some(isMuslFilename)) return true; + } catch { + // A missing or unreadable conventional library directory is inconclusive. + } + } + return undefined; +} + +function readUInt( + buffer: Buffer, + offset: number, + bytes: 2 | 4 | 8, + littleEndian: boolean, +): number | undefined { + if (offset < 0 || offset + bytes > buffer.length) return undefined; + if (bytes === 2) return littleEndian ? buffer.readUInt16LE(offset) : buffer.readUInt16BE(offset); + if (bytes === 4) return littleEndian ? buffer.readUInt32LE(offset) : buffer.readUInt32BE(offset); + const value = littleEndian ? buffer.readBigUInt64LE(offset) : buffer.readBigUInt64BE(offset); + return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : undefined; +} + +function isMuslFromElfInterpreter(): boolean | undefined { + let fd: number | undefined; + try { + fd = openSync(process.execPath, "r"); + const header = Buffer.alloc(64); + if (readSync(fd, header, 0, header.length, 0) < 52) return undefined; + if (!header.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return undefined; + + const elfClass = header[4]; + const dataEncoding = header[5]; + if ((elfClass !== 1 && elfClass !== 2) || (dataEncoding !== 1 && dataEncoding !== 2)) { + return undefined; + } + const littleEndian = dataEncoding === 1; + const is64Bit = elfClass === 2; + const tableOffset = readUInt(header, is64Bit ? 32 : 28, is64Bit ? 8 : 4, littleEndian); + const entrySize = readUInt(header, is64Bit ? 54 : 42, 2, littleEndian); + const entryCount = readUInt(header, is64Bit ? 56 : 44, 2, littleEndian); + if ( + tableOffset === undefined || + entrySize === undefined || + entryCount === undefined || + entrySize < (is64Bit ? 56 : 32) || + entryCount > 1024 + ) { + return undefined; + } + + const programHeader = Buffer.alloc(entrySize); + for (let index = 0; index < entryCount; index += 1) { + const offset: number = tableOffset + index * entrySize; + if (readSync(fd, programHeader, 0, entrySize, offset) !== entrySize) return undefined; + if (readUInt(programHeader, 0, 4, littleEndian) !== 3) continue; + const interpreterOffset = readUInt( + programHeader, + is64Bit ? 8 : 4, + is64Bit ? 8 : 4, + littleEndian, + ); + const interpreterSize = readUInt( + programHeader, + is64Bit ? 32 : 16, + is64Bit ? 8 : 4, + littleEndian, + ); + if ( + interpreterOffset === undefined || + interpreterSize === undefined || + interpreterSize < 1 || + interpreterSize > 4096 + ) { + return undefined; + } + const interpreter = Buffer.alloc(interpreterSize); + if ( + readSync(fd, interpreter, 0, interpreterSize, interpreterOffset) !== interpreterSize + ) { + return undefined; + } + return isMuslFilename(interpreter.toString("utf8")); + } + } catch { return undefined; + } finally { + if (fd !== undefined) { + try { + closeSync(fd); + } catch { + // Best-effort detection must never turn close failure into import failure. + } + } + } + return undefined; +} + +function isMusl(): boolean { + if (process.platform !== "linux") return false; + for (const detector of [isMuslFromReport, isMuslFromFilesystem, isMuslFromElfInterpreter]) { + const result = detector(); + if (result !== undefined) return result; + } + // Unknown Linux libc: try the glibc binary and let its require fail normally. + return false; +} + +function bundledTarget(): string | undefined { + if (process.platform === "win32" && process.arch === "x64") return "win32-x64-msvc"; + if (process.platform === "darwin" && process.arch === "x64") return "darwin-x64"; + if (process.platform === "darwin" && process.arch === "arm64") return "darwin-arm64"; + if (process.platform === "linux" && (process.arch === "x64" || process.arch === "arm64")) { + return `linux-${process.arch}-${isMusl() ? "musl" : "gnu"}`; + } + return undefined; +} + +function loadBundledBinding(): NativeBinding { + const target = bundledTarget(); + if (!target) { + throw new Error(`Unsupported OS or architecture: ${process.platform}-${process.arch}`); } + return require(join(bundledNativeDirectory, target, "fs-safe-native.node")) as NativeBinding; +} + +export function __loadBundledNativeForTest(): NativeBinding { + return loadBundledBinding(); +} + +export function getNativeBinding(): NativeBinding | undefined { + const { mode } = getFsSafeNativeConfig(); + if (mode === "off") return undefined; if (!attempted) { attempted = true; try { @@ -23,9 +181,7 @@ export function getNativeBinding(): NativeBinding | undefined { loadError = error; } } - if (binding) { - return binding; - } + if (binding) return binding; if (mode === "require") { throw new FsSafeError("helper-unavailable", "native fs-safe helper is unavailable", { cause: loadError, @@ -55,5 +211,5 @@ export function __resetNativeLoaderForTest(): void { binding = undefined; loadError = undefined; attempted = false; - loadBinding = () => require("@openclaw/fs-safe-native") as NativeBinding; + loadBinding = loadBundledBinding; } diff --git a/src/owner-dacl.ts b/src/owner-dacl.ts index 6880f18..68f4f48 100644 --- a/src/owner-dacl.ts +++ b/src/owner-dacl.ts @@ -44,7 +44,7 @@ export function readOwnerAndDacl(targetPath: string): OwnerAndDaclResult { if (!native) { throw new FsSafeError( "helper-unavailable", - "Windows owner and DACL facts require the optional native binding", + "Windows owner and DACL facts require the bundled native binding", ); } const facts = native.readOwnerAndDacl(targetPath); diff --git a/src/private-directory.ts b/src/private-directory.ts index 16d7cf9..14a142c 100644 --- a/src/private-directory.ts +++ b/src/private-directory.ts @@ -13,7 +13,7 @@ export async function createPrivateDirectory( if (platform !== "win32") { throw new FsSafeError( "helper-unavailable", - "private-directory creation is available only on Windows through the optional native binding", + "private-directory creation is available only on Windows through the bundled native binding", ); } @@ -21,7 +21,7 @@ export async function createPrivateDirectory( if (!native) { throw new FsSafeError( "helper-unavailable", - "private Windows directory creation requires the optional native binding", + "private Windows directory creation requires the bundled native binding", ); } native.createPrivateDirectory(targetPath); diff --git a/test/native-archive-equivalence.test.ts b/test/native-archive-equivalence.test.ts index a609c2b..b551d18 100644 --- a/test/native-archive-equivalence.test.ts +++ b/test/native-archive-equivalence.test.ts @@ -1,4 +1,3 @@ -import { createRequire } from "node:module"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -14,14 +13,14 @@ import { import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; import { __resetNativeLoaderForTest, + __loadBundledNativeForTest, __setNativeLoaderForTest, type NativeBinding, } from "../src/native.js"; -const require = createRequire(import.meta.url); let native: NativeBinding | undefined; try { - native = require("../native") as NativeBinding; + native = __loadBundledNativeForTest(); } catch { // JS-only jobs intentionally exercise the fallback without a built binding. } diff --git a/test/native-integration.test.ts b/test/native-integration.test.ts index 82496be..973b959 100644 --- a/test/native-integration.test.ts +++ b/test/native-integration.test.ts @@ -2,20 +2,21 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { createRequire } from "node:module"; import { afterEach, describe, expect, it } from "vitest"; import { configureFsSafeNative } from "../src/native-config.js"; import { acquireFileLock } from "../src/file-lock.js"; -import { __resetNativeLoaderForTest, __setNativeLoaderForTest } from "../src/native.js"; +import { + __loadBundledNativeForTest, + __resetNativeLoaderForTest, + __setNativeLoaderForTest, + type NativeBinding, +} from "../src/native.js"; import { publishFileExclusive } from "../src/publish-file.js"; import { runPinnedWriteHelper } from "../src/pinned-write.js"; -type NativeBinding = typeof import("../native/index.js"); - -const require = createRequire(import.meta.url); let native: NativeBinding | undefined; try { - native = require("../native") as NativeBinding; + native = __loadBundledNativeForTest(); } catch { // Native artifacts are built by dedicated platform jobs. The ordinary JS // matrix deliberately proves that installation without them still works. diff --git a/test/native-loader.test.ts b/test/native-loader.test.ts index 3fc404c..f06e026 100644 --- a/test/native-loader.test.ts +++ b/test/native-loader.test.ts @@ -8,6 +8,7 @@ import { getFsSafeNativeConfig, } from "../src/native-config.js"; import { + __loadBundledNativeForTest, __resetNativeLoaderForTest, __setNativeLoaderForTest, getNativeBinding, @@ -108,11 +109,25 @@ describe("native helper configuration", () => { }); }); -describe("native package loader", () => { +describe("bundled native loader", () => { + let hostBinding: NativeBinding | undefined; + try { + hostBinding = __loadBundledNativeForTest(); + } catch { + // Ordinary JavaScript-only jobs intentionally run without a built binding. + } + + it.runIf(Boolean(hostBinding))("loads the bundled binary for the host target", () => { + expect(hostBinding?.openBeneath).toBeTypeOf("function"); + expect(hostBinding?.sha256File).toBeTypeOf("function"); + }); + it("contains no import-time process execution path", () => { - const loader = readFileSync(fileURLToPath(new URL("../native/index.js", import.meta.url)), "utf8"); + const loader = readFileSync(fileURLToPath(new URL("../src/native.ts", import.meta.url)), "utf8"); expect(loader).not.toMatch(/(?:child_process|execSync|execFileSync|spawnSync|\bspawn\s*\()/); expect(loader).toContain("isMuslFromElfInterpreter"); - expect(loader).toContain("Unknown Linux libc: try the glibc package"); + expect(loader).toContain("Unknown Linux libc: try the glibc binary"); + expect(loader).toContain('"fs-safe-native.node"'); + expect(loader).not.toContain("@openclaw/fs-safe-native"); }); }); diff --git a/test/native-publish-equivalence.test.ts b/test/native-publish-equivalence.test.ts index b3e4eed..cad758a 100644 --- a/test/native-publish-equivalence.test.ts +++ b/test/native-publish-equivalence.test.ts @@ -1,7 +1,6 @@ import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; import fsSync from "node:fs"; -import { createRequire } from "node:module"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -9,15 +8,15 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; import { __resetNativeLoaderForTest, + __loadBundledNativeForTest, __setNativeLoaderForTest, type NativeBinding, } from "../src/native.js"; import { publishFileExclusive } from "../src/publish-file.js"; -const require = createRequire(import.meta.url); let native: NativeBinding | undefined; try { - native = require("../native") as NativeBinding; + native = __loadBundledNativeForTest(); } catch { // JS-only jobs intentionally exercise the fallback without a built binding. } diff --git a/test/private-directory.test.ts b/test/private-directory.test.ts index f3fa9bc..01712dc 100644 --- a/test/private-directory.test.ts +++ b/test/private-directory.test.ts @@ -1,4 +1,3 @@ -import { createRequire } from "node:module"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -6,16 +5,16 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; import { __resetNativeLoaderForTest, + __loadBundledNativeForTest, __setNativeLoaderForTest, type NativeBinding, } from "../src/native.js"; import { inspectPathPermissions } from "../src/permissions.js"; import { createPrivateDirectory } from "../src/private-directory.js"; -const require = createRequire(import.meta.url); let native: NativeBinding | undefined; try { - native = require("../native") as NativeBinding; + native = __loadBundledNativeForTest(); } catch { // Ordinary JS jobs do not build a host binding. } diff --git a/test/symlink-dotdot-escape.test.ts b/test/symlink-dotdot-escape.test.ts index 0becca0..dd7d8b6 100644 --- a/test/symlink-dotdot-escape.test.ts +++ b/test/symlink-dotdot-escape.test.ts @@ -1,21 +1,19 @@ import os from "node:os"; import path from "node:path"; -import { createRequire } from "node:module"; import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; import { afterEach, describe, expect, it } from "vitest"; import { configureFsSafeNative, FsSafeError, root, type Root } from "../src/index.js"; import { __resetNativeLoaderForTest, + __loadBundledNativeForTest, __setNativeLoaderForTest, + type NativeBinding, } from "../src/native.js"; import { resolveRootPath, resolveRootPathSync } from "../src/root-path.js"; -type NativeBinding = typeof import("../native/index.js"); - -const require = createRequire(import.meta.url); let native: NativeBinding | undefined; try { - native = require("../native") as NativeBinding; + native = __loadBundledNativeForTest(); } catch { // Ordinary JavaScript-only jobs intentionally run without a platform artifact. } From edc2705dc39d97ea10a8d07706f7fbddd95bef77 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Jul 2026 16:59:58 -0700 Subject: [PATCH 2/3] fix: resolve npm cli on Windows --- scripts/check-release-packages.mjs | 34 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/scripts/check-release-packages.mjs b/scripts/check-release-packages.mjs index 22343a8..9017a0a 100644 --- a/scripts/check-release-packages.mjs +++ b/scripts/check-release-packages.mjs @@ -4,19 +4,45 @@ import { mkdtempSync, mkdirSync, readFileSync, + realpathSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { hostNativeTarget, nativeTargets } from "./native-targets.mjs"; const outputIndex = process.argv.indexOf("--output"); const outputDir = resolve(outputIndex >= 0 ? process.argv[outputIndex + 1] : "release-artifacts"); const allowHostOnly = process.argv.includes("--allow-host-only"); mkdirSync(outputDir, { recursive: true }); +const npmCli = resolveNpmCli(); + +function resolveNpmCli() { + const candidates = [ + process.env.npm_execpath, + join(dirname(process.execPath), "..", "lib", "node_modules", "npm", "bin", "npm-cli.js"), + join(dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js"), + ]; + if (process.platform !== "win32") { + try { + candidates.push(realpathSync(execFileSync("which", ["npm"], { encoding: "utf8" }).trim())); + } catch { + // The standard bundled paths remain valid on supported non-Windows installations. + } + } + const resolved = candidates.find( + (candidate) => candidate && basename(candidate) === "npm-cli.js" && existsSync(candidate), + ); + if (!resolved) throw new Error("could not resolve npm-cli.js from the current Node installation"); + return resolved; +} + +function runNpm(args, options) { + return execFileSync(process.execPath, [npmCli, ...args], options); +} const pkg = JSON.parse(readFileSync("package.json", "utf8")); if (pkg.name !== "@openclaw/fs-safe") throw new Error(`unexpected package name ${pkg.name}`); @@ -42,8 +68,7 @@ for (const target of expectedTargets) { } const packed = JSON.parse( - execFileSync( - "npm", + runNpm( ["pack", "--json", "--ignore-scripts", "--pack-destination", outputDir], { encoding: "utf8" }, ), @@ -82,8 +107,7 @@ writeFileSync(join(outputDir, "manifest.json"), `${JSON.stringify(manifest, null const smoke = mkdtempSync(join(tmpdir(), "fs-safe-release-smoke-")); try { writeFileSync(join(smoke, "package.json"), '{"private":true,"type":"module"}\n'); - execFileSync( - "npm", + runNpm( [ "install", "--ignore-scripts", From 72a4f03e7c377ea686e5ce7d293adbeaae47014a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Jul 2026 17:05:32 -0700 Subject: [PATCH 3/3] test: cover bundled native target detection --- src/native.ts | 40 ++++++++++++++++++++++++++++++++------ test/native-loader.test.ts | 24 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/native.ts b/src/native.ts index dec935e..6df4688 100644 --- a/src/native.ts +++ b/src/native.ts @@ -148,16 +148,24 @@ function isMusl(): boolean { return false; } -function bundledTarget(): string | undefined { - if (process.platform === "win32" && process.arch === "x64") return "win32-x64-msvc"; - if (process.platform === "darwin" && process.arch === "x64") return "darwin-x64"; - if (process.platform === "darwin" && process.arch === "arm64") return "darwin-arm64"; - if (process.platform === "linux" && (process.arch === "x64" || process.arch === "arm64")) { - return `linux-${process.arch}-${isMusl() ? "musl" : "gnu"}`; +function targetFor( + platform: NodeJS.Platform, + arch: string, + musl: boolean, +): string | undefined { + if (platform === "win32" && arch === "x64") return "win32-x64-msvc"; + if (platform === "darwin" && arch === "x64") return "darwin-x64"; + if (platform === "darwin" && arch === "arm64") return "darwin-arm64"; + if (platform === "linux" && (arch === "x64" || arch === "arm64")) { + return `linux-${arch}-${musl ? "musl" : "gnu"}`; } return undefined; } +function bundledTarget(): string | undefined { + return targetFor(process.platform, process.arch, isMusl()); +} + function loadBundledBinding(): NativeBinding { const target = bundledTarget(); if (!target) { @@ -170,6 +178,26 @@ export function __loadBundledNativeForTest(): NativeBinding { return loadBundledBinding(); } +export function __nativeLoaderDetectorsForTest(): { + report: boolean | undefined; + filesystem: boolean | undefined; + elfInterpreter: boolean | undefined; +} { + return { + report: isMuslFromReport(), + filesystem: isMuslFromFilesystem(), + elfInterpreter: isMuslFromElfInterpreter(), + }; +} + +export function __nativeTargetForTest( + platform: NodeJS.Platform, + arch: string, + musl = false, +): string | undefined { + return targetFor(platform, arch, musl); +} + export function getNativeBinding(): NativeBinding | undefined { const { mode } = getFsSafeNativeConfig(); if (mode === "off") return undefined; diff --git a/test/native-loader.test.ts b/test/native-loader.test.ts index f06e026..a96c2b6 100644 --- a/test/native-loader.test.ts +++ b/test/native-loader.test.ts @@ -9,6 +9,8 @@ import { } from "../src/native-config.js"; import { __loadBundledNativeForTest, + __nativeLoaderDetectorsForTest, + __nativeTargetForTest, __resetNativeLoaderForTest, __setNativeLoaderForTest, getNativeBinding, @@ -122,6 +124,28 @@ describe("bundled native loader", () => { expect(hostBinding?.sha256File).toBeTypeOf("function"); }); + it("maps every bundled target without probing the host", () => { + expect(__nativeTargetForTest("linux", "x64")).toBe("linux-x64-gnu"); + expect(__nativeTargetForTest("linux", "x64", true)).toBe("linux-x64-musl"); + expect(__nativeTargetForTest("linux", "arm64")).toBe("linux-arm64-gnu"); + expect(__nativeTargetForTest("linux", "arm64", true)).toBe("linux-arm64-musl"); + expect(__nativeTargetForTest("darwin", "x64")).toBe("darwin-x64"); + expect(__nativeTargetForTest("darwin", "arm64")).toBe("darwin-arm64"); + expect(__nativeTargetForTest("win32", "x64")).toBe("win32-x64-msvc"); + expect(__nativeTargetForTest("freebsd", "x64")).toBeUndefined(); + expect(__nativeTargetForTest("linux", "ppc64")).toBeUndefined(); + }); + + it("runs only non-executing libc probes", () => { + const detected = __nativeLoaderDetectorsForTest(); + expect([true, false, undefined]).toContain(detected.report); + expect([true, undefined]).toContain(detected.filesystem); + expect([true, false, undefined]).toContain(detected.elfInterpreter); + if (process.platform === "linux") { + expect(detected.elfInterpreter).toBeTypeOf("boolean"); + } + }); + it("contains no import-time process execution path", () => { const loader = readFileSync(fileURLToPath(new URL("../src/native.ts", import.meta.url)), "utf8"); expect(loader).not.toMatch(/(?:child_process|execSync|execFileSync|spawnSync|\bspawn\s*\()/);