diff --git a/.github/scripts/run-tests-with-miri.sh b/.github/scripts/run-tests-with-miri.sh new file mode 100755 index 0000000..4b8e7a2 --- /dev/null +++ b/.github/scripts/run-tests-with-miri.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Parallelizes the integration tests when running with Miri# +# +# Usage: .github/scripts/run-tests-with-miri.sh +# Env: MIRI_TEST_TIMEOUT_SECS (default 300) +# MIRI_TEST_JOBS (default: nproc) +# MIRI_TEST_TARGETS (default: all targets, space-separated) +# MIRIFLAGS (passed to `cargo miri test`) +set -uo pipefail + +manifest_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$manifest_dir" + +cargo miri setup + +timeout_secs="${MIRI_TEST_TIMEOUT_SECS:-300}" +jobs="${MIRI_TEST_JOBS:-$(nproc)}" + +target_flag() { + case "$1" in + lib) echo "--lib" ;; + *) echo "--test $1" ;; + esac +} +export -f target_flag + +# shellcheck disable=SC2206 +targets=(${MIRI_TEST_TARGETS:-lib core_integration regression_integration custom_page_sizes_integration statistics_integration}) + +pairs_file="$(mktemp)" +results_file="$(mktemp)" +trap 'rm -f "$pairs_file" "$results_file"' EXIT + +for label in "${targets[@]}"; do + target="$(target_flag "$label")" + RUSTFLAGS="--cfg miri" cargo test --quiet $target -- --list 2>/dev/null | + grep ': test$' | sed 's/: test$//' | + sed "s/^/${label}\t/" \ + >>"$pairs_file" +done + +total=$(wc -l <"$pairs_file") +echo "Running $total tests:" +echo "Executors: $jobs" +echo "Timeout: ${timeout_secs}s" + +export TIMEOUT_SECS="$timeout_secs" + +run_one() { + local label="$1" name="$2" + local target + target="$(target_flag "$label")" + + local out status + out="$(timeout "$TIMEOUT_SECS" cargo miri test --features miri-soft-floats $target -- --exact "$name" 2>&1)" + status=$? + + if [ "$status" -eq 0 ]; then + printf 'PASS\t%s\t%s\n' "$label" "$name" + elif [ "$status" -eq 124 ]; then + printf 'TIMEOUT\t%s\t%s\n' "$label" "$name" + echo "!!! TIMED OUT after ${TIMEOUT_SECS}s: $label :: $name" >&2 + else + printf 'FAIL\t%s\t%s\n' "$label" "$name" + echo "!!! FAILED: $label :: $name" >&2 + echo "$out" >&2 + fi +} +export -f run_one + +xargs -P "$jobs" -L1 bash -c 'run_one "$@"' _ <"$pairs_file" >>"$results_file" + +pass_count=$(grep -c '^PASS' "$results_file" || true) +timeout_lines=$(grep '^TIMEOUT' "$results_file" || true) +fail_lines=$(grep '^FAIL' "$results_file" || true) + +echo +echo "$total / $pass_count tests passed." + +if [ -n "$timeout_lines" ]; then + echo "Timed out:" + echo "$timeout_lines" | awk -F'\t' '{print " - " $2 " :: " $3}' +fi +if [ -n "$fail_lines" ]; then + echo "Failed:" + echo "$fail_lines" | awk -F'\t' '{print " - " $2 " :: " $3}' +fi + +if [ -n "$timeout_lines" ] || [ -n "$fail_lines" ]; then + exit 1 +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38d880d..230e623 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -320,7 +320,9 @@ jobs: components: miri - name: Run Miri - run: cargo miri test -j4 --no-fail-fast --verbose + env: + MIRIFLAGS: -Zmiri-disable-isolation + run: cargo miri test --lib --no-fail-fast --verbose --features miri-soft-floats coverage: name: Code Coverage diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml new file mode 100644 index 0000000..3cc436f --- /dev/null +++ b/.github/workflows/daily.yml @@ -0,0 +1,52 @@ +name: Daily CI Jobs + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +permissions: + contents: read + +env: + RUST_BACKTRACE: 1 + CARGO_TERM_COLOR: always + +jobs: + miri-integration: + name: Miri Integration Tests + runs-on: ubuntu-latest + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: miri + + - name: Install WABT (WebAssembly Binary Toolkit) + run: | + WABT_VERSION=1.0.41 + WABT_PLATFORM="linux-x64" + wget https://github.com/WebAssembly/wabt/releases/download/${WABT_VERSION}/wabt-${WABT_VERSION}-${WABT_PLATFORM}.tar.gz + tar -xzf wabt-${WABT_VERSION}-${WABT_PLATFORM}.tar.gz + sudo cp wabt-${WABT_VERSION}/bin/* /usr/local/bin/ + shell: bash + + - name: Convert wast files for Miri + run: cargo test --test miri_wast_convert -- --ignored + + - name: Run Miri + env: + MIRIFLAGS: -Zmiri-disable-isolation + MIRI_TEST_TIMEOUT_SECS: 300 + MIRI_TEST_JOBS: 4 + MIRI_TEST_TARGETS: core_integration regression_integration custom_page_sizes_integration statistics_integration + run: ./.github/scripts/run-tests-with-miri.sh diff --git a/Cargo.toml b/Cargo.toml index 7588bd7..91d2615 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,9 @@ default = [] # by the verifier. Useful for fuzzing/regression testing strict-assertions = [] +# Disables inline asm (e.g. x86 `sqrtss`) for Miri +miri-soft-floats = ["libm/force-soft-floats"] + [dependencies] libm = "0.2.16" @@ -65,3 +68,6 @@ panic = "abort" opt-level = 3 lto = "thin" codegen-units = 1 + +[target.'cfg(miri)'.dependencies] +libm = { version = "0.2.16", features = ["force-soft-floats"] } diff --git a/tests/core_integration.rs b/tests/core_integration.rs index 9d45d9c..16ac31b 100644 --- a/tests/core_integration.rs +++ b/tests/core_integration.rs @@ -1,5 +1,3 @@ -#![cfg(not(miri))] - mod util; use spacewasm::vec; use util::{run_wast_test_file, spectest_host_module}; @@ -14,6 +12,7 @@ fn address() { } #[test] +#[cfg_attr(miri, ignore = "stack recursion")] fn call() { run("core/call"); } @@ -209,6 +208,7 @@ fn memory() { } #[test] +#[cfg_attr(miri, ignore = "stack recursion")] fn skip_stack_guard_page() { run("core/skip-stack-guard-page"); } @@ -244,6 +244,7 @@ fn labels() { } #[test] +#[cfg_attr(miri, ignore = "malloc too slow")] fn memory_grow() { run("core/memory_grow"); } @@ -309,6 +310,7 @@ fn elem() { } #[test] +#[cfg_attr(miri, ignore = "stack recursion")] fn fac() { run("core/fac"); } diff --git a/tests/custom_page_sizes_integration.rs b/tests/custom_page_sizes_integration.rs index baa6de4..6f62b16 100644 --- a/tests/custom_page_sizes_integration.rs +++ b/tests/custom_page_sizes_integration.rs @@ -1,5 +1,3 @@ -#![cfg(not(miri))] - mod util; use spacewasm::vec; use util::{run_wast_test_file, spectest_host_module}; diff --git a/tests/miri_wast_convert.rs b/tests/miri_wast_convert.rs new file mode 100644 index 0000000..5ef8e1f --- /dev/null +++ b/tests/miri_wast_convert.rs @@ -0,0 +1,9 @@ +#![cfg(not(miri))] + +mod util; + +#[test] +#[ignore] +fn convert() { + util::convert_wast_for_miri(); +} diff --git a/tests/regression_integration.rs b/tests/regression_integration.rs index a9a4238..0abf273 100644 --- a/tests/regression_integration.rs +++ b/tests/regression_integration.rs @@ -1,5 +1,3 @@ -#![cfg(not(miri))] - mod util; use std::{ops::ControlFlow, sync::Mutex}; diff --git a/tests/util/spectest.rs b/tests/util/spectest.rs index 4f02ace..1df2188 100644 --- a/tests/util/spectest.rs +++ b/tests/util/spectest.rs @@ -28,13 +28,16 @@ use std::ops::ControlFlow; use std::panic::catch_unwind; use std::path::Path; use std::path::PathBuf; -use std::process::Command as ProcessCommand; use std::ptr::NonNull; use std::rc::Rc; +use std::sync::{Arc, Mutex}; -type SubtestLogType = Arc>>>>>; +#[cfg(not(miri))] +use std::process::Command as ProcessCommand; +#[cfg(not(miri))] use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; + +type SubtestLogType = Arc>>>>>; #[derive(Debug, Deserialize, Serialize)] struct TestFile { @@ -1035,11 +1038,67 @@ fn check_initialization_error(result: InterpreterResult, text: &str) { } } -// Simple temp directory that cleans up on drop +/// Wrapper for `wast2json` +#[cfg(not(miri))] +fn wast2json(source_wast_path: &Path, out_dir: &Path, test_filename: &str) { + let output = ProcessCommand::new("wast2json") + .arg(source_wast_path) + .arg("--enable-custom-page-sizes") + .arg("-o") + .arg(out_dir.join(format!("{}.json", test_filename))) + .current_dir(out_dir) + .output() + .unwrap_or_else(|e| panic!("Failed to run wast2json: {e}")); + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + panic!("wast2json failed: {}", stderr); + } +} + +#[cfg(not(miri))] +pub fn convert_wast_for_miri() { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let tests_dir = PathBuf::from(manifest_dir).join("tests"); + let converted_root = PathBuf::from(manifest_dir).join("target").join("miri-wast"); + + let _ = std::fs::remove_dir_all(&converted_root); + + // The `tests/` subdirectories containing `.wast` files. Hard-coded + // rather than walked, since it's a fixed, small set; add to this list + // if a new `.wast` suite subdirectory is introduced. + let wast_dirs: &[&str] = &["core", "regression", "custom-page-sizes"]; + + for subdir in wast_dirs { + let dir = tests_dir.join(subdir); + for entry in + std::fs::read_dir(&dir).unwrap_or_else(|e| panic!("Failed to read {dir:?}: {e}")) + { + let wast_path = entry.unwrap().path(); + if wast_path.extension().is_none_or(|ext| ext != "wast") { + continue; + } + + let rel = wast_path + .strip_prefix(&tests_dir) + .unwrap() + .with_extension(""); + let test_filename = rel.file_stem().unwrap().to_string_lossy().to_string(); + + let out_dir = converted_root.join(&rel); + std::fs::create_dir_all(&out_dir).unwrap(); + + wast2json(&wast_path, &out_dir, &test_filename); + } + } +} + +#[cfg(not(miri))] struct TempDir { path: PathBuf, } +#[cfg(not(miri))] impl TempDir { fn new() -> std::io::Result { static COUNTER: AtomicU64 = AtomicU64::new(0); @@ -1056,6 +1115,7 @@ impl TempDir { } } +#[cfg(not(miri))] impl Drop for TempDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.path); @@ -1416,9 +1476,11 @@ pub fn run_wast_test_file(test_name: &str, host_modules: HostModuleFactory) { .join(format!("{}.wast", test_name)); // Create a temporary directory for generated files + #[cfg(not(miri))] let temp_dir = TempDir::new().unwrap_or_else(|e| panic!("Failed to create temp directory: {e}")); - let temp_path = temp_dir.path(); + #[cfg(not(miri))] + let temp_path = temp_dir.path().to_path_buf(); // Extract just the filename (without directory path) for the JSON output let test_filename = PathBuf::from(test_name) @@ -1427,20 +1489,29 @@ pub fn run_wast_test_file(test_name: &str, host_modules: HostModuleFactory) { .to_string_lossy() .to_string(); - // Run wast2json to generate Wasm modules and JSON descriptor - let output = ProcessCommand::new("wast2json") - .arg(&source_wast_path) - .arg("--enable-custom-page-sizes") - .arg("-o") - .arg(temp_path.join(format!("{}.json", test_filename))) - .current_dir(temp_path) - .output() - .unwrap_or_else(|e| panic!("Failed to run wast2json: {e}")); + // Pre-convert tests for Miri to run + #[cfg(not(miri))] + wast2json(&source_wast_path, &temp_path, &test_filename); + #[cfg(not(miri))] + let test_dir = temp_path; + + #[cfg(miri)] + let test_dir = { + let dir = PathBuf::from(manifest_dir) + .join("target") + .join("miri-wast") + .join(test_name); + + if !dir.join(format!("{}.json", test_filename)).exists() { + panic!( + "Converted wast files missing at {}. Run `cargo test --test miri_wast_convert \ + -- --ignored` (with `wast2json` on PATH) before `cargo miri test`.", + dir.display() + ); + } - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - panic!("wast2json failed: {}", stderr); - } + dir + }; let wast_line = Arc::new(Mutex::new(None)); #[allow(clippy::arc_with_non_send_sync)] @@ -1448,7 +1519,7 @@ pub fn run_wast_test_file(test_name: &str, host_modules: HostModuleFactory) { match catch_unwind(|| { run_wast_test_file_inner( - temp_path.to_path_buf(), + test_dir, &test_filename, host_modules, wast_line.clone(),