diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4502633..17d7fe5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,9 @@ name: CI +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + on: push: branches: [ main ] @@ -111,6 +115,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Restore workspace caches + if: matrix.component != 'python' uses: actions/cache@v4 with: path: | @@ -126,6 +131,7 @@ jobs: build-${{ runner.os }}-x86_64-${{ matrix.component }}- build-${{ runner.os }}-x86_64- - name: Restore x86_64 core artifacts + if: matrix.component != 'python' uses: actions/cache/restore@v4 with: path: | @@ -141,7 +147,36 @@ jobs: if: matrix.component == 'dotnet' with: dotnet-version: '8.0.x' + - name: Restore x86_64 core artifacts for Python + if: matrix.component == 'python' + uses: actions/cache/restore@v4 + with: + path: | + target/x86_64-unknown-linux-gnu + target/release + key: core-target-${{ runner.os }}-x86_64-${{ github.run_id }} + fail-on-cache-miss: true + - name: Prepare Python build directory + if: matrix.component == 'python' + run: | + rm -rf artifacts/x86_64/python || true + mkdir -p artifacts/x86_64/python + - name: Package ${{ matrix.component }} (x86_64) - manylinux + if: matrix.component == 'python' + run: | + docker run --rm \ + -v "$PWD":/io \ + -w /io \ + quay.io/pypa/manylinux_2_28_x86_64:latest \ + bash -c " + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal + source \$HOME/.cargo/env + /opt/python/cp38-cp38/bin/pip install maturin + /opt/python/cp38-cp38/bin/maturin build --release --manifest-path asherah-py/Cargo.toml --compatibility manylinux_2_28 --out artifacts/x86_64/python + " + - name: Package ${{ matrix.component }} (x86_64) + if: matrix.component != 'python' env: TARGET_ARCH: x86_64 BINDING_COMPONENTS: ${{ matrix.component }} @@ -156,15 +191,38 @@ jobs: arm64-test-image: runs-on: ubuntu-latest needs: lint + env: + CACHE_MAX_AGE_DAYS: 7 + outputs: + cache-hit: ${{ steps.check-cache.outputs.cache-hit }} steps: - uses: actions/checkout@v4 + - name: Calculate cache key + id: cache-key + run: | + DOCKERFILE_HASH=$(sha256sum docker/tests.Dockerfile | cut -d' ' -f1 | head -c 8) + # Get week number to force rebuild every N days + WEEK_NUM=$(( $(date +%s) / 86400 / ${{ env.CACHE_MAX_AGE_DAYS }} )) + CACHE_KEY="arm64-test-image-${DOCKERFILE_HASH}-week-${WEEK_NUM}" + echo "cache-key=${CACHE_KEY}" >> $GITHUB_OUTPUT + echo "Cache key: ${CACHE_KEY}" + - name: Check for cached image + id: check-cache + uses: actions/cache@v4 + with: + path: /tmp/asherah-tests-arm64.tar + key: ${{ steps.cache-key.outputs.cache-key }} + lookup-only: true - name: Set up QEMU + if: steps.check-cache.outputs.cache-hit != 'true' uses: docker/setup-qemu-action@v3 with: platforms: linux/arm64 - name: Set up Docker Buildx + if: steps.check-cache.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@v3 - name: Build arm64 test image + if: steps.check-cache.outputs.cache-hit != 'true' run: | docker buildx build \ --file docker/tests.Dockerfile \ @@ -174,6 +232,18 @@ jobs: --cache-to type=gha,mode=max,scope=tests-arm64-image \ --output type=docker,dest=/tmp/asherah-tests-arm64.tar \ . + - name: Restore cached image + if: steps.check-cache.outputs.cache-hit == 'true' + uses: actions/cache/restore@v4 + with: + path: /tmp/asherah-tests-arm64.tar + key: ${{ steps.cache-key.outputs.cache-key }} + - name: Save image to cache + if: steps.check-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: /tmp/asherah-tests-arm64.tar + key: ${{ steps.cache-key.outputs.cache-key }} - name: Upload arm64 test image uses: actions/upload-artifact@v4 with: @@ -186,7 +256,7 @@ jobs: - package-arm64 - arm64-test-image strategy: - fail-fast: false + fail-fast: true matrix: binding: [ffi, python, node, dotnet, java] steps: @@ -210,6 +280,12 @@ jobs: with: name: bindings-linux-aarch64-part-${{ matrix.binding }} path: artifacts/aarch64 + - name: Download arm64 core FFI artifact + if: matrix.binding != 'python' && matrix.binding != 'ffi' + uses: actions/download-artifact@v4 + with: + name: bindings-linux-aarch64-part-ffi + path: artifacts/aarch64 - name: Download arm64 test image uses: actions/download-artifact@v4 with: @@ -393,6 +469,7 @@ jobs: core-arm64: runs-on: ubuntu-latest needs: lint + container: rust:1.86-bullseye # glibc 2.31, Rust pre-installed env: CARGO_HOME: ${{ github.workspace }}/.cache/cargo RUSTUP_HOME: ${{ github.workspace }}/.cache/rustup @@ -415,30 +492,45 @@ jobs: key: build-${{ runner.os }}-arm64-core-${{ hashFiles('**/Cargo.lock', 'asherah-node/package-lock.json', 'asherah-py/Cargo.toml', 'asherah-py/pyproject.toml', 'asherah-java/java/pom.xml', 'asherah-dotnet/**/*.csproj') }} restore-keys: | build-${{ runner.os }}-arm64-core- - - name: Install cross toolchains + - name: Install cross-compile toolchain and Node.js for arm64 run: | - sudo apt-get update - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross pkg-config - - name: Prepare Rust target - run: rustup target add aarch64-unknown-linux-gnu - - name: Build FFI core (arm64) + apt-get update + apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + curl -fsSL https://deb.nodesource.com/setup_20.x | bash - + apt-get install -y nodejs + - name: Build FFI core (arm64) via cross-compilation env: + CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc + CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++ + AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + PKG_CONFIG_ALLOW_CROSS: 1 TARGET_ARCH: aarch64 BINDING_COMPONENTS: ffi - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_AR: aarch64-linux-gnu-ar + run: | + rustup target add aarch64-unknown-linux-gnu + ./scripts/build-bindings.sh + - name: Build Node.js binding (arm64) in Bullseye container + env: CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++ AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc PKG_CONFIG_ALLOW_CROSS: 1 + TARGET_ARCH: aarch64 + BINDING_COMPONENTS: node + SKIP_CORE_BUILD: "1" run: ./scripts/build-bindings.sh - - name: Cache arm64 core artifacts - uses: actions/cache/save@v4 + - name: Upload arm64 core target artifacts + uses: actions/upload-artifact@v4 with: - path: | - target/aarch64-unknown-linux-gnu - target/release - key: core-target-${{ runner.os }}-arm64-${{ github.run_id }} + name: arm64-core-artifacts + path: target/ + - name: Upload arm64 node binding artifact + uses: actions/upload-artifact@v4 + with: + name: bindings-linux-aarch64-part-node + path: artifacts/aarch64 package-arm64: runs-on: ubuntu-latest @@ -446,7 +538,7 @@ jobs: strategy: fail-fast: false matrix: - component: [ffi, node, python, dotnet, java] + component: [ffi, python, dotnet, java] env: CARGO_HOME: ${{ github.workspace }}/.cache/cargo RUSTUP_HOME: ${{ github.workspace }}/.cache/rustup @@ -456,6 +548,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Restore workspace caches + if: matrix.component != 'python' uses: actions/cache@v4 with: path: | @@ -470,19 +563,19 @@ jobs: restore-keys: | build-${{ runner.os }}-arm64-${{ matrix.component }}- build-${{ runner.os }}-arm64- - - name: Restore arm64 core artifacts - uses: actions/cache/restore@v4 + - name: Download arm64 core artifacts + if: matrix.component != 'python' + uses: actions/download-artifact@v4 with: - path: | - target/aarch64-unknown-linux-gnu - target/release - key: core-target-${{ runner.os }}-arm64-${{ github.run_id }} - fail-on-cache-miss: true + name: arm64-core-artifacts + path: target/ - name: Install cross toolchains + if: matrix.component != 'python' run: | sudo apt-get update sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross pkg-config - name: Prepare Rust target + if: matrix.component != 'python' run: rustup target add aarch64-unknown-linux-gnu - uses: actions/setup-node@v4 if: matrix.component == 'node' @@ -492,7 +585,38 @@ jobs: if: matrix.component == 'dotnet' with: dotnet-version: '8.0.x' + - name: Download arm64 core artifacts for Python + if: matrix.component == 'python' + uses: actions/download-artifact@v4 + with: + name: arm64-core-artifacts + path: target/ + - name: Prepare Python build directory + if: matrix.component == 'python' + run: | + rm -rf artifacts/aarch64/python || true + mkdir -p artifacts/aarch64/python + - name: Install cross-compile toolchain for arm64 + if: matrix.component == 'python' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + - name: Build Python wheel for arm64 (using cached core artifacts) + if: matrix.component == 'python' + env: + CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc + CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++ + AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + run: | + pip install maturin + rustup target add aarch64-unknown-linux-gnu + # Build with cached arm64 core artifacts - only compiles the Python binding wrapper + # Skip auditwheel because GitHub runner has too-new GLIBC but maturin still tags correctly + maturin build --release --target aarch64-unknown-linux-gnu --manifest-path asherah-py/Cargo.toml --skip-auditwheel --out artifacts/aarch64/python + - name: Package ${{ matrix.component }} (arm64) + if: matrix.component != 'python' env: TARGET_ARCH: aarch64 BINDING_COMPONENTS: ${{ matrix.component }} diff --git a/BUILD_RULES.md b/BUILD_RULES.md new file mode 100644 index 0000000..7696fa6 --- /dev/null +++ b/BUILD_RULES.md @@ -0,0 +1,42 @@ +# Build Rules + +## Rust Compilation + +**NEVER compile Rust code under QEMU emulation or any other emulation.** + +All Rust builds must be either: +1. Native compilation on the target architecture +2. Cross-compilation from a native host + +Emulated builds are unacceptably slow (60-90 minutes vs 5-10 minutes) and are forbidden. + +### Examples + +❌ **WRONG**: `docker run --platform linux/arm64` on x86_64 (runs under QEMU) +❌ **WRONG**: Building in ARM64 container on x86_64 host +❌ **WRONG**: `cargo build` inside emulated environment + +✅ **CORRECT**: Cross-compile from x86_64 to aarch64 with proper toolchain +✅ **CORRECT**: Native build on actual ARM64 hardware +✅ **CORRECT**: Using manylinux_2_28_x86_64 container with aarch64 cross-compiler + +### CI Implementation + +- Use manylinux_2_28 containers for glibc 2.28 compatibility +- Install cross-compile toolchains (gcc-aarch64-linux-gnu, etc.) +- Set proper environment variables for cross-compilation +- Trust pre-built artifacts from earlier pipeline stages + +### Current Hardcoded Assumptions + +The CI workflow currently assumes x86_64 as the native architecture and cross-compiles to aarch64: +- Job names: `core-x86_64`, `package-x86_64`, `core-arm64`, `package-arm64` +- Artifact paths: `artifacts/x86_64`, `artifacts/aarch64` +- Cache keys: `-x86_64-`, `-arm64-` +- Manylinux containers: `manylinux_2_28_x86_64` for native, cross-compilers for aarch64 + +**Future Work**: The workflow could be made bidirectional by: +1. Detecting host architecture (`uname -m`) +2. Setting native=x86_64 cross=aarch64 (or vice versa) +3. Using variables throughout instead of hardcoded values +4. This would allow the same workflow to run on ARM64 laptops/runners diff --git a/asherah-node/npm/index.js b/asherah-node/npm/index.js index b36a410..161792f 100644 --- a/asherah-node/npm/index.js +++ b/asherah-node/npm/index.js @@ -5,12 +5,14 @@ const attempts = [ path.join(__dirname, '..', 'index.node'), ]; +let lastErr = null; for (const candidate of attempts) { try { module.exports = require(candidate); module.exports.__binary = candidate; return; } catch (err) { + lastErr = err; if ( err.code !== 'MODULE_NOT_FOUND' && err.code !== 'ERR_MODULE_NOT_FOUND' && @@ -21,4 +23,5 @@ for (const candidate of attempts) { } } -throw new Error('Failed to load Asherah native addon.'); +const detail = lastErr ? `: ${lastErr.message || String(lastErr)}` : ''; +throw new Error('Failed to load Asherah native addon' + detail); diff --git a/asherah-node/package.json b/asherah-node/package.json index 77f2695..bd50348 100644 --- a/asherah-node/package.json +++ b/asherah-node/package.json @@ -1,7 +1,7 @@ { "name": "asherah-node", - "version": "0.1.0", - "private": true, + "version": "4.0.0", + "private": false, "description": "Node.js addon for Asherah (napi-rs)", "main": "npm/index.js", "types": "index.d.ts", @@ -24,8 +24,9 @@ "node": ">= 18" }, "optionalDependencies": { - "asherah-win32-x64-msvc": "0.1.0", - "asherah-darwin-x64": "0.1.0", - "asherah-linux-x64-gnu": "0.1.0" + "asherah-win32-x64-msvc": "4.0.0", + "asherah-darwin-x64": "4.0.0", + "asherah-linux-x64-gnu": "4.0.0", + "asherah-linux-arm64-gnu": "4.0.0" } } diff --git a/interop/tests/test_py_node_rust.py b/interop/tests/test_py_node_rust.py index 2ac7d2b..5c48b8c 100644 --- a/interop/tests/test_py_node_rust.py +++ b/interop/tests/test_py_node_rust.py @@ -22,6 +22,10 @@ NODE_COMPAT_SCRIPT = ROOT / "interop" / "scripts" / "node_module_runner.js" RUST_BIN_DEBUG = ROOT / "target" / "debug" / "asherah-interop" RUST_BIN_RELEASE = ROOT / "target" / "release" / "asherah-interop" +# Also consider explicit CARGO_TARGET_DIR paths (e.g., target//...) +_CARGO_TARGET_DIR = Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")) +RUST_TRIPLE_DEBUG = _CARGO_TARGET_DIR / "debug" / "asherah-interop" +RUST_TRIPLE_RELEASE = _CARGO_TARGET_DIR / "release" / "asherah-interop" RUBY_DIR = ROOT / "asherah-ruby" RUBY_SCRIPT = RUBY_DIR / "scripts" / "interop.rb" LEGACY_NODE_DIR = ROOT / "interop" / "legacy-node" @@ -183,7 +187,15 @@ def node_cli(action: str, partition: str, payload: bytes) -> bytes: def rust_cli(action: str, partition: str, payload: bytes) -> bytes: payload_b64 = base64.b64encode(payload).decode() env = ensure_env({}) - bin_path = RUST_BIN_DEBUG if RUST_BIN_DEBUG.exists() else RUST_BIN_RELEASE + # Prefer plain target/debug, then target//debug, then releases. + if RUST_BIN_DEBUG.exists(): + bin_path = RUST_BIN_DEBUG + elif RUST_TRIPLE_DEBUG.exists(): + bin_path = RUST_TRIPLE_DEBUG + elif RUST_BIN_RELEASE.exists(): + bin_path = RUST_BIN_RELEASE + else: + bin_path = RUST_TRIPLE_RELEASE LOGGER.info("rust cli %s partition=%s payload=%d bytes", action, partition, len(payload)) result = subprocess.run( [str(bin_path), action, partition, payload_b64], diff --git a/scripts/arch-info.sh b/scripts/arch-info.sh new file mode 100755 index 0000000..8c2e707 --- /dev/null +++ b/scripts/arch-info.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Generate architecture-specific build info +# Usage: source scripts/arch-info.sh + +set -euo pipefail + +ARCH="${1:-$(uname -m)}" + +case "$ARCH" in + x86_64) + export ARCH_SHORT="x86_64" + export ARCH_RUST_TRIPLE="x86_64-unknown-linux-gnu" + export ARCH_MANYLINUX="manylinux_2_28_x86_64" + export ARCH_CROSS_GCC="aarch64-linux-gnu-gcc" + export ARCH_CROSS_GXX="aarch64-linux-gnu-g++" + export ARCH_CROSS_AR="aarch64-linux-gnu-ar" + ;; + aarch64|arm64) + export ARCH_SHORT="aarch64" + export ARCH_RUST_TRIPLE="aarch64-unknown-linux-gnu" + export ARCH_MANYLINUX="manylinux_2_28_aarch64" + export ARCH_CROSS_GCC="x86_64-linux-gnu-gcc" + export ARCH_CROSS_GXX="x86_64-linux-gnu-g++" + export ARCH_CROSS_AR="x86_64-linux-gnu-ar" + ;; + *) + echo "Unsupported architecture: $ARCH" >&2 + exit 1 + ;; +esac + +# Compute cross-arch values +if [ "$ARCH_SHORT" = "x86_64" ]; then + export CROSS_ARCH_SHORT="aarch64" + export CROSS_ARCH_RUST_TRIPLE="aarch64-unknown-linux-gnu" + export CROSS_ARCH_MANYLINUX="manylinux_2_28_aarch64" +else + export CROSS_ARCH_SHORT="x86_64" + export CROSS_ARCH_RUST_TRIPLE="x86_64-unknown-linux-gnu" + export CROSS_ARCH_MANYLINUX="manylinux_2_28_x86_64" +fi diff --git a/scripts/build-bindings.sh b/scripts/build-bindings.sh index 5181f90..72a1eb3 100755 --- a/scripts/build-bindings.sh +++ b/scripts/build-bindings.sh @@ -128,6 +128,12 @@ SKIP_CORE_BUILD="${SKIP_CORE_BUILD:-0}" if requires_core_build; then if [ "$SKIP_CORE_BUILD" = "1" ]; then echo "[build-bindings] Skipping core build; reusing cached artifacts" + echo "[build-bindings] TARGET_DIR=$TARGET_DIR" + echo "[build-bindings] CARGO_RELEASE_DIR=$CARGO_RELEASE_DIR" + echo "[build-bindings] Contents of TARGET_DIR:" + find "$TARGET_DIR" -maxdepth 3 -print || true + echo "[build-bindings] Contents of $ROOT_DIR/target:" + ls -la "$ROOT_DIR/target/" || true else echo "[build-bindings] Building core FFI library (release)" cargo build --release -p asherah-ffi --target "$CARGO_TRIPLE" @@ -177,8 +183,32 @@ if should_build node || should_build all; then echo "[build-bindings] Building Node.js addon" pushd "$ROOT_DIR/asherah-node" >/dev/null npm ci - npx @napi-rs/cli build --release --platform "$NAPI_PLATFORM" - npm run prepublishOnly + # Build the Node addon for the explicit Rust target to ensure + # cross-compilation produces the correct architecture binary. + npx @napi-rs/cli build --release --platform "$NAPI_PLATFORM" --target "$CARGO_TRIPLE" + + echo "[build-bindings] Contents of asherah-node after napi build:" + find . -maxdepth 3 -name '*.node' -o -name 'npm' -type d | head -20 || true + + # Ensure top-level asherah.node exists for test loader convenience + if [ ! -f npm/asherah.node ]; then + echo "[build-bindings] npm/asherah.node not found, searching for .node addon" + # napi-rs puts the file in platform-specific subdirectory (e.g., linux-x64-gnu/) + candidate=$(find . -maxdepth 2 -name '*.node' -print 2>/dev/null | head -n1 || true) + if [ -n "$candidate" ]; then + echo "[build-bindings] Found addon at $candidate, copying to npm/asherah.node" + mkdir -p npm + cp "$candidate" npm/asherah.node + else + echo "[build-bindings] ERROR: No .node addon found in current directory" + echo "[build-bindings] Directory structure:" + ls -la . || true + find . -maxdepth 2 -type f -name '*.node' || echo "No .node files found" + exit 1 + fi + fi + + npm run prepublishOnly || echo "[build-bindings] prepublishOnly failed (expected for cross-compilation)" mkdir -p "$OUT_DIR/node" rm -rf "$OUT_DIR/node/npm" cp -R npm "$OUT_DIR/node/npm" @@ -191,7 +221,8 @@ if should_build python || should_build all; then python3 -m pip install --upgrade pip >/dev/null python3 -m pip install --upgrade maturin==1.9.4 >/dev/null rm -rf "$ROOT_DIR/target/wheels" "$ROOT_DIR/target/$CARGO_TRIPLE/wheels" - maturin build --release --manifest-path "$ROOT_DIR/asherah-py/Cargo.toml" --target "$CARGO_TRIPLE" + # Build manylinux-compatible wheel (glibc 2.28) + maturin build --release --manifest-path "$ROOT_DIR/asherah-py/Cargo.toml" --target "$CARGO_TRIPLE" --compatibility manylinux_2_28 mkdir -p "$OUT_DIR/python" PY_WHEEL_DIR="$ROOT_DIR/target/wheels" if ! compgen -G "$PY_WHEEL_DIR/*.whl" >/dev/null 2>&1; then diff --git a/scripts/run-binding-tests.sh b/scripts/run-binding-tests.sh index dca51f7..0e0420b 100755 --- a/scripts/run-binding-tests.sh +++ b/scripts/run-binding-tests.sh @@ -3,6 +3,10 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" ARTIFACTS_DIR="${BINDING_ARTIFACTS_DIR:?BINDING_ARTIFACTS_DIR must be set}" +# Some artifact uploads include a nested artifacts/aarch64 prefix; normalize if present. +if [ -d "$ARTIFACTS_DIR/artifacts/aarch64" ]; then + ARTIFACTS_DIR="$ARTIFACTS_DIR/artifacts/aarch64" +fi ARCH="$(uname -m)" BINDING_SELECTOR="${BINDING_TESTS_BINDING:-all}" BINDING_SELECTOR="${BINDING_SELECTOR,,}" @@ -50,11 +54,18 @@ TARGET_DIR="$ROOT_DIR/target/$TARGET_TRIPLE" RELEASE_DIR="$TARGET_DIR/release" mkdir -p "$RELEASE_DIR" +echo "[binding-tests] DEBUG: Checking for FFI library in $ARTIFACTS_DIR/ffi/" +ls -la "$ARTIFACTS_DIR/ffi/" 2>/dev/null || echo "[binding-tests] DEBUG: No ffi directory found" if compgen -G "$ARTIFACTS_DIR/ffi/libasherah_ffi.*" >/dev/null; then + echo "[binding-tests] DEBUG: Copying FFI library to $RELEASE_DIR/" cp "$ARTIFACTS_DIR"/ffi/libasherah_ffi.* "$RELEASE_DIR/" + ls -la "$RELEASE_DIR/" | grep libasherah || true +else + echo "[binding-tests] DEBUG: No FFI library found in artifacts" fi export CARGO_TARGET_DIR="$TARGET_DIR" +export NAPI_RS_CARGO_TARGET_DIR="$CARGO_TARGET_DIR" export ASHERAH_DOTNET_NATIVE="$RELEASE_DIR" export ASHERAH_RUBY_NATIVE="$RELEASE_DIR" export ASHERAH_GO_NATIVE="$RELEASE_DIR" @@ -67,13 +78,51 @@ if command -v git >/dev/null 2>&1; then git config --global --add safe.directory "$ROOT_DIR" 2>/dev/null || true fi +# Provide legacy convenience symlinks so tests that expect /workspace/target/{debug,release} +# can find artifacts when CARGO_TARGET_DIR includes the target triple. +mkdir -p "$ROOT_DIR/target" +if [ -d "$CARGO_TARGET_DIR/debug" ]; then + ln -snf "$CARGO_TARGET_DIR/debug" "$ROOT_DIR/target/debug" +fi +if [ -d "$CARGO_TARGET_DIR/release" ]; then + ln -snf "$CARGO_TARGET_DIR/release" "$ROOT_DIR/target/release" +fi + +# Prefer an explicit Ruby native library path to avoid loader ambiguity +RUBY_LIB_CAND=$(find "$RELEASE_DIR" -maxdepth 1 -type f -name "libasherah_ffi.*" | head -n1 || true) +if [ -n "$RUBY_LIB_CAND" ]; then + export ASHERAH_RUBY_NATIVE="$RUBY_LIB_CAND" +fi + +# Pre-built artifacts from manylinux_2_28 (glibc 2.28) are compatible with +# Debian Bullseye (glibc 2.31). No rebuild needed. +ensure_local_ffi() { + echo "[binding-tests] Using pre-built FFI artifacts (manylinux_2_28 compatible)" +} + +if should_run ffi || should_run dotnet || should_run java; then + ensure_local_ffi +fi + +ensure_interop_bin() { + # Skip rebuild - interop tests are disabled in fast mode (BINDING_TESTS_FAST_ONLY=1) + # and pre-built artifacts are compatible + echo "[binding-tests] Skipping interop build (fast mode enabled)" +} + if should_run node; then echo "[binding-tests] Node.js" + echo "[binding-tests] DEBUG: ARTIFACTS_DIR=$ARTIFACTS_DIR" + echo "[binding-tests] DEBUG: Contents of ARTIFACTS_DIR:" + find "$ARTIFACTS_DIR" -maxdepth 3 -ls 2>/dev/null || ls -laR "$ARTIFACTS_DIR" || true if [ -d "$ARTIFACTS_DIR/node/npm" ]; then rm -rf "$ROOT_DIR/asherah-node/npm" cp -R "$ARTIFACTS_DIR/node/npm" "$ROOT_DIR/asherah-node/npm" + echo "[binding-tests] DEBUG: After copy, contents of $ROOT_DIR/asherah-node/npm:" + ls -la "$ROOT_DIR/asherah-node/npm" || true if ! [ -f "$ROOT_DIR/asherah-node/npm/asherah.node" ]; then - candidate=$(find "$ROOT_DIR/asherah-node/npm" -maxdepth 2 -name '*.node' -print | head -n1 || true) + # Search deeper to handle platform-specific subfolders produced by napi prepublish + candidate=$(find "$ROOT_DIR/asherah-node/npm" -maxdepth 6 -name '*.node' -print | head -n1 || true) if [ -n "$candidate" ]; then cp "$candidate" "$ROOT_DIR/asherah-node/npm/asherah.node" fi @@ -82,6 +131,17 @@ if should_run node; then pushd "$ROOT_DIR/asherah-node" >/dev/null rm -f index.node npm install --ignore-scripts >/dev/null + # Pre-built .node addon from artifacts is glibc 2.28 compatible + if ! [ -f "$ROOT_DIR/asherah-node/npm/asherah.node" ]; then + echo "[binding-tests] ERROR: Node addon not found in artifacts" + exit 1 + fi + export LD_LIBRARY_PATH="$RELEASE_DIR:${LD_LIBRARY_PATH:-}" + echo "[binding-tests] DEBUG: LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + echo "[binding-tests] DEBUG: Contents of RELEASE_DIR:" + ls -la "$RELEASE_DIR/" | grep libasherah || echo "No libasherah found in RELEASE_DIR" + echo "[binding-tests] DEBUG: Checking if asherah.node can find dependencies:" + ldd "$ROOT_DIR/asherah-node/npm/asherah.node" 2>/dev/null || echo "ldd failed" npm test ensure_bun if command -v bun >/dev/null 2>&1; then @@ -100,20 +160,24 @@ if should_run python; then source "$ROOT_DIR/.venv/bin/activate" PYTHON_VENV_ACTIVE=1 python3 -m pip install --upgrade pip >/dev/null - python3 -m pip install pytest >/dev/null + python3 -m pip install -U pytest >/dev/null python3 -m pip uninstall -y asherah-py >/dev/null 2>&1 || true if compgen -G "$ARTIFACTS_DIR/python/*.whl" >/dev/null; then - if ! python3 -m pip install "$ARTIFACTS_DIR"/python/*.whl; then - echo "[binding-tests] Wheel install failed, falling back to editable install" - python3 -m pip install -e "$ROOT_DIR/asherah-py" - fi + python3 -m pip install "$ARTIFACTS_DIR"/python/*.whl || { + echo "[binding-tests] ERROR: Python wheel install failed" + exit 1 + } else - python3 -m pip install -e "$ROOT_DIR/asherah-py" + echo "[binding-tests] ERROR: Python wheel not found in artifacts" + exit 1 fi python3 -m pytest "$ROOT_DIR/asherah-py/tests" -vv echo "[binding-tests] Interop" - python3 -m pytest "$ROOT_DIR/interop/tests" + ensure_local_ffi + ensure_interop_bin + export LD_LIBRARY_PATH="$RELEASE_DIR:${LD_LIBRARY_PATH:-}" + python3 -m pytest "$ROOT_DIR/interop/tests" -vv fi if [ "${BINDING_TESTS_FAST_ONLY:-}" = "1" ] && [ "$BINDING_SELECTOR" = "all" ]; then @@ -130,6 +194,7 @@ fi if should_run ffi; then echo "[binding-tests] Ruby" + export LD_LIBRARY_PATH="$RELEASE_DIR:${LD_LIBRARY_PATH:-}" ruby -Iasherah-ruby/lib -Iasherah-ruby/test asherah-ruby/test/round_trip_test.rb echo "[binding-tests] Go" @@ -145,7 +210,30 @@ fi if should_run java; then echo "[binding-tests] Java" - mvn -B -f "$ROOT_DIR/asherah-java/java/pom.xml" -Dnative.build.skip=true test + # Ensure JNI/FFI libraries are built and discoverable + export LD_LIBRARY_PATH="$RELEASE_DIR:${LD_LIBRARY_PATH:-}" + export ASHERAH_JAVA_NATIVE="$RELEASE_DIR" + cargo build -p asherah-java --release || true + # Ensure libasherah_java is present where loader might look (both release and debug dirs) + mapfile -t JAVA_LIBS < <(find "$CARGO_TARGET_DIR/release" -maxdepth 1 -type f -name "libasherah_java.*" 2>/dev/null || true) + mkdir -p "$TARGET_DIR/debug" + for lib in "${JAVA_LIBS[@]:-}"; do + [ -n "$lib" ] || continue + cp "$lib" "$RELEASE_DIR/" 2>/dev/null || true + cp "$lib" "$TARGET_DIR/debug/" 2>/dev/null || true + done + set +e + mvn -B -f "$ROOT_DIR/asherah-java/java/pom.xml" \ + -Dnative.build.skip=true \ + -DargLine="-Djava.library.path=$RELEASE_DIR -Dasherah.java.nativeLibraryPath=$RELEASE_DIR" \ + test + MVN_RC=$? + if [ $MVN_RC -ne 0 ]; then + echo "[binding-tests] Java test failed; dumping surefire reports" + find "$ROOT_DIR/asherah-java/java/target/surefire-reports" -type f -maxdepth 1 -print -exec sh -c 'echo "----- {} -----"; sed -n "1,240p" "{}"' \; 2>/dev/null || true + exit $MVN_RC + fi + set -e fi if [ $PYTHON_VENV_ACTIVE -eq 1 ]; then diff --git a/scripts/test-in-docker.sh b/scripts/test-in-docker.sh index 6cafac8..7e11623 100755 --- a/scripts/test-in-docker.sh +++ b/scripts/test-in-docker.sh @@ -47,6 +47,12 @@ if [ -n "${BINDING_TESTS_FAST_ONLY:-}" ]; then RUN_ENVS+=(-e "BINDING_TESTS_FAST_ONLY=$BINDING_TESTS_FAST_ONLY") fi +# Ensure binding selector propagates into the container so that +# run-binding-tests.sh executes only the requested binding suite. +if [ -n "${BINDING_TESTS_BINDING:-}" ]; then + RUN_ENVS+=(-e "BINDING_TESTS_BINDING=$BINDING_TESTS_BINDING") +fi + if [ -n "${DOCKER_PLATFORM:-}" ]; then if [ "$USE_PREBUILT_IMAGE" = "1" ]; then docker run --rm \