diff --git a/python/unit-tests/test_ramulator2.py b/python/unit-tests/test_ramulator2.py deleted file mode 100644 index 9e098d1f1..000000000 --- a/python/unit-tests/test_ramulator2.py +++ /dev/null @@ -1,42 +0,0 @@ -import sys -import os -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) - -from assassyn.utils import repo_path - -from assassyn.ramulator2 import PyRamulator, Request - -home = repo_path() -sim = PyRamulator(f"{home}/tools/c-ramulator2-wrapper/configs/example_config.yaml") - -is_write = False -v = 0 # counter - -for i in range(200): - plused = v + 1 - we = v & 1 - re = not we - raddr = v & 0xFF - waddr = plused & 0xFF - addr = waddr if is_write else raddr - - def callback(req: Request, i=i): # capture i in closure - print( - f"Cycle {i + 3 + (req.depart - req.arrive)}: Request completed: {req.addr} the data is: {req.addr - 1}", - flush=True, - ) - - ok = sim.send_request(addr, is_write, callback, i) - write_success = "true" if ok else "false" - if is_write: - print( - f"Cycle {i + 2}: Write request sent for address {addr}, success or not (true or false){write_success}", - flush=True, - ) - - is_write = not is_write - sim.frontend_tick() - sim.memory_system_tick() - v = plused - -sim.finish() \ No newline at end of file diff --git a/python/unit-tests/test_ramulator2_combined.py b/python/unit-tests/test_ramulator2_combined.py deleted file mode 100644 index c7db520ea..000000000 --- a/python/unit-tests/test_ramulator2_combined.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python3 -"""Combined Ramulator2 test and cross-validation for pytest. - -This module combines the Python Ramulator2 test with cross-language validation -to ensure behavioral consistency across C++, Rust, and Python implementations. -""" -import os -import sys -import difflib -import subprocess -import shlex -from typing import Dict, Tuple -import pytest - -# Add the python directory to the path -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) - -from assassyn.utils import repo_path -from assassyn.ramulator2 import PyRamulator, Request - - -def run_command(command: str, workdir: str, env: Dict[str, str] | None = None) -> Tuple[int, str, str]: - """Run a command and return exit code, stdout, and stderr.""" - proc = subprocess.Popen( - command, - cwd=workdir, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - executable="/bin/bash", - env=env, - ) - out, err = proc.communicate() - return proc.returncode, out, err - - -def get_expected_targets(home: str) -> Dict[str, Tuple[str, str]]: - """Get the command and working directory for each language implementation.""" - return { - "cpp": ( - os.path.join(home, "tools/c-ramulator2-wrapper/build/bin/test"), - os.path.join(home, "tools/c-ramulator2-wrapper/build/bin"), - ), - "rust": ( - "cargo test --quiet --test test_ramulator2 -- --nocapture", - os.path.join(home, "tools/rust-sim-runtime"), - ), - "python": ( - f"python -u {shlex.quote(os.path.join(home, 'python/unit-tests/test_ramulator2.py'))}", - home, - ), - } - - -def build_cpp_if_needed(home: str) -> None: - """Build C++ executable if it doesn't exist.""" - cpp_exe = os.path.join(home, "tools/c-ramulator2-wrapper/build/bin/test") - if os.path.isfile(cpp_exe) and os.access(cpp_exe, os.X_OK): - return - build_dir = os.path.join(home, "tools/c-ramulator2-wrapper/build") - os.makedirs(build_dir, exist_ok=True) - cmake_cmd = "cmake .." - make_cmd = "make -j" - code, out, err = run_command(cmake_cmd, build_dir, env=os.environ.copy()) - if code != 0: - raise RuntimeError(f"CMake configuration failed in {build_dir}:\n{err}") - code, out, err = run_command(make_cmd, build_dir, env=os.environ.copy()) - if code != 0: - raise RuntimeError(f"Make build failed in {build_dir}:\n{err}") - - -def build_rust_if_needed(home: str) -> None: - """Build Rust test if needed.""" - workdir = os.path.join(home, "tools/rust-sim-runtime") - code, out, err = run_command("cargo test --quiet --test test_ramulator2", workdir, env=os.environ.copy()) - if code != 0: - raise RuntimeError(f"Cargo test build failed in {workdir}:\n{err}") - - -def normalize_output(text: str) -> str: - """Normalize output for comparison by removing blank lines and trailing whitespace.""" - lines = text.split('\n') - non_empty_lines = [line for line in lines if line.strip() != ''] - return '\n'.join(non_empty_lines).rstrip() - - -def filter_rust_output(text: str) -> str: - """Filter out Cargo test harness noise from Rust output.""" - lines = text.split('\n') - filtered_lines = [] - for line in lines: - # Skip Cargo test harness lines - if (line.startswith("running ") and " test" in line) or \ - line.startswith("test result:") or \ - line.strip() == "." or \ - line.strip() == "": - continue - filtered_lines.append(line) - return '\n'.join(filtered_lines).rstrip() - - -def run_python_test() -> str: - """Run the Python Ramulator2 test and return the output.""" - home = repo_path() - sim = PyRamulator(f"{home}/tools/c-ramulator2-wrapper/configs/example_config.yaml") - - is_write = False - v = 0 # counter - output_lines = [] - - for i in range(200): - plused = v + 1 - we = v & 1 - re = not we - raddr = v & 0xFF - waddr = plused & 0xFF - addr = waddr if is_write else raddr - - def callback(req: Request, i=i): # capture i in closure - output_lines.append( - f"Cycle {i + 3 + (req.depart - req.arrive)}: Request completed: {req.addr} the data is: {req.addr - 1}" - ) - - ok = sim.send_request(addr, is_write, callback, i) - write_success = "true" if ok else "false" - if is_write: - output_lines.append( - f"Cycle {i + 2}: Write request sent for address {addr}, success or not (true or false){write_success}" - ) - - is_write = not is_write - sim.frontend_tick() - sim.memory_system_tick() - v = plused - - sim.finish() - return '\n'.join(output_lines) - - -def test_python_ramulator2(): - """Test Python Ramulator2 wrapper functionality.""" - output = run_python_test() - - # Basic validation - check that we got some output - assert len(output) > 0, "Python test should produce output" - - # Check for expected patterns in the output - assert "Write request sent" in output, "Should contain write requests" - assert "Request completed" in output, "Should contain completed requests" - - # Check that we have reasonable number of operations - write_count = output.count("Write request sent") - completed_count = output.count("Request completed") - assert write_count > 0, "Should have some write requests" - assert completed_count > 0, "Should have some completed requests" - - -def test_cross_language_validation(): - """Test that all language implementations produce identical output.""" - home = repo_path() - targets = get_expected_targets(home) - - # Build artifacts if missing - try: - build_cpp_if_needed(home) - build_rust_if_needed(home) - except Exception as e: - pytest.skip(f"Failed to build required artifacts: {e}") - - # Base environment: ensure ASSASSYN_HOME is set for all children - base_env = os.environ.copy() - base_env["ASSASSYN_HOME"] = home - - results: Dict[str, Tuple[int, str, str]] = {} - - for lang, (cmd_or_path, workdir) in targets.items(): - # Per-language environment - env = base_env.copy() - if lang == "cpp": - command = shlex.quote(cmd_or_path) - # Help the C++ binary find shared libraries at runtime - wrapper_lib_dir = os.path.join(home, "tools/c-ramulator2-wrapper/build/lib") - ramulator_lib_dir = os.path.join(home, "3rd-party/ramulator2") - existing_ld = env.get("LD_LIBRARY_PATH", "") - ld_parts = [p for p in [wrapper_lib_dir, ramulator_lib_dir, existing_ld] if p] - env["LD_LIBRARY_PATH"] = ":".join(ld_parts) - else: - command = cmd_or_path - - code, out, err = run_command(command, workdir, env=env) - results[lang] = (code, out, err) - - # Check that command succeeded - assert code == 0, f"{lang} implementation failed with exit code {code}. Stderr: {err}" - - # Normalize outputs for comparison - norm = {} - for lang, (code, out, err) in results.items(): - if lang == "rust": - norm[lang] = normalize_output(filter_rust_output(out)) - else: - norm[lang] = normalize_output(out) - - # Compare all outputs - languages = list(norm.keys()) - base = languages[0] - for other in languages[1:]: - if norm[base] != norm[other]: - diff = difflib.unified_diff( - norm[base].splitlines(keepends=True), - norm[other].splitlines(keepends=True), - fromfile=base, - tofile=other, - n=3, - ) - diff_text = "".join(diff) - pytest.fail(f"Output differs between {base} and {other}:\n{diff_text}") - - -if __name__ == "__main__": - # Allow running as standalone script for debugging - test_python_ramulator2() - test_cross_language_validation() - print("All tests passed!") diff --git a/python/unit-tests/compare_ramulator2_outputs.py b/python/unit-tests/test_trilang_ramulator2_valid.py similarity index 73% rename from python/unit-tests/compare_ramulator2_outputs.py rename to python/unit-tests/test_trilang_ramulator2_valid.py index babc17509..39cdd0841 100644 --- a/python/unit-tests/compare_ramulator2_outputs.py +++ b/python/unit-tests/test_trilang_ramulator2_valid.py @@ -16,7 +16,9 @@ import subprocess import sys from typing import Dict, Tuple +from io import StringIO from assassyn.utils import repo_path +from assassyn.ramulator2 import PyRamulator, Request def run_command(command: str, workdir: str, env: Dict[str, str] | None = None) -> Tuple[int, str, str]: @@ -86,6 +88,68 @@ def build_rust_if_needed(home: str, debug: bool = False) -> None: raise RuntimeError(f"Cargo test build failed in {workdir}:\n{err}") +def run_python_test() -> Tuple[int, str, str]: + """Run the Python Ramulator2 test directly and return output.""" + try: + home = repo_path() + sim = PyRamulator(f"{home}/tools/c-ramulator2-wrapper/configs/example_config.yaml") + + is_write = False + v = 0 # counter + output_lines = [] + + for i in range(200): + plused = v + 1 + we = v & 1 + re = not we + raddr = v & 0xFF + waddr = plused & 0xFF + addr = waddr if is_write else raddr + + def callback(req: Request, i=i): # capture i in closure + output_lines.append( + f"Cycle {i + 3 + (req.depart - req.arrive)}: Request completed: {req.addr} the data is: {req.addr - 1}" + ) + + ok = sim.send_request(addr, is_write, callback, i) + write_success = "true" if ok else "false" + if is_write: + output_lines.append( + f"Cycle {i + 2}: Write request sent for address {addr}, success or not (true or false){write_success}" + ) + + is_write = not is_write + sim.frontend_tick() + sim.memory_system_tick() + v = plused + + # Suppress stdout during finish() to avoid statistics output + old_stdout = sys.stdout + sys.stdout = StringIO() + sim.finish() + sys.stdout = old_stdout + + output = '\n'.join(output_lines) + return 0, output, "" + except Exception as e: # noqa: BLE001 + return 1, "", str(e) + + +def filter_stats_output(text: str) -> str: + """Filter out Ramulator2 statistics output.""" + # The stats appear at the end of the output, starting with "Frontend:" or "MemorySystem:" + # Simply truncate everything from the first occurrence of these markers onwards + lines = text.split('\n') + filtered_lines = [] + + for line in lines: + if line.startswith('Frontend:') or line.startswith('MemorySystem:'): + break # Stop processing, discard the rest + filtered_lines.append(line) + + return '\n'.join(filtered_lines).rstrip() + + def compare_texts(name_a: str, text_a: str, name_b: str, text_b: str) -> str: if text_a == text_b: return "" @@ -147,6 +211,17 @@ def main() -> int: for lang, (cmd_or_path, workdir) in targets.items(): if args.skip and lang in args.skip: continue + + # Special handling for Python - run directly instead of subprocess + if lang == "python": + if args.debug: + sys.stderr.write(f"[DEBUG] {lang} running direct Python test\n") + code, out, err = run_python_test() + results[lang] = (code, out, err) + if code != 0: + failures[lang] = f"Non-zero exit ({code}). Stderr:\n{err}\nStdout:\n{out}" + continue + # Per-language environment env = base_env.copy() if lang == "cpp": @@ -184,21 +259,20 @@ def main() -> int: sys.stderr.write(f"[ERROR] {lang}: {msg}\n") return 2 - # Show raw outputs if requested - if args.show_outputs: - for lang, (code, out, err) in results.items(): - print(f"\n=== {lang.upper()} OUTPUT ===") - print("STDOUT:") - print(out) - if err: - print("STDERR:") - print(err) - print(f"Exit code: {code}") - print("=" * 50) - # Normalize outputs slightly (strip trailing whitespace) norm = {k: v[1].rstrip() for k, v in results.items()} + # Filter out stats from all languages + for lang in norm: + norm[lang] = filter_stats_output(norm[lang]) + + # Show filtered outputs if requested + if args.show_outputs: + for lang in norm: + print(f"\n=== {lang.upper()} OUTPUT (FILTERED) ===") + print(norm[lang]) + print("=" * 50) + # Filter out Cargo test harness noise from Rust output if "rust" in norm: lines = norm["rust"].split('\n') @@ -213,7 +287,7 @@ def main() -> int: filtered_lines.append(line) norm["rust"] = '\n'.join(filtered_lines).rstrip() - # Normalize whitespace differences in statistics sections for all languages + # Normalize whitespace differences for all languages import re for lang in norm: # Remove ALL blank lines to eliminate formatting differences diff --git a/python/unit-tests/trilang-ramulator2-valid.md b/python/unit-tests/trilang-ramulator2-valid.md new file mode 100644 index 000000000..1cb378e0c --- /dev/null +++ b/python/unit-tests/trilang-ramulator2-valid.md @@ -0,0 +1,105 @@ +# Triple Language Ramulator2 Cross Validation + +The `test_trilang_ramulator2_valid.py` script runs three different Ramulator2 wrapper implementations and compares their outputs to ensure behavioral consistency. + +## Core Functionality + +The script runs these three implementations with the same configuration and request sequence: + +- **C++ Implementation**: `tools/c-ramulator2-wrapper/test.cpp` (executable: `build/bin/test`) +- **Rust Implementation**: `tools/rust-sim-runtime/tests/test_ramulator2.rs` (via `cargo test`) +- **Python Implementation**: Runs directly via the Ramulator2 Python wrapper API + +## Implementation Details + +### Python Direct Execution + +Unlike C++ and Rust which are executed as subprocesses, the Python implementation is executed directly within the validation script using the `PyRamulator` class. This approach: + +- Avoids overhead from subprocess invocation +- Allows direct stdout suppression during statistics collection +- Captures output via Python list collection rather than stdout parsing + +### Statistics Output Handling + +Ramulator2's `finalize()` method prints detailed statistics in YAML format. To ensure fair comparison of only the simulation output (cycle messages), the script: + +1. **Python**: Suppresses stdout by redirecting to `StringIO` during `finish()` call +2. **C++/Rust**: Filters out statistics lines (starting with `Frontend:` or `MemorySystem:`) from subprocess output + +## Usage + +### Basic Usage +```bash +python python/unit-tests/test_trilang_ramulator2_valid.py +``` + +### Command Line Options + +- `--skip `: Skip running a specific language implementation (`cpp`, `rust`, or `python`) +- `--debug`: Enable verbose debugging output with command/env details +- `--show-outputs`: Display filtered outputs from all languages before comparison + +## Output Processing + +The script processes and normalizes outputs for fair comparison: + +1. **Statistics Filtering**: Removes Ramulator2 statistics output (lines starting with `Frontend:` or `MemorySystem:`) from all implementations +2. **Python Output Suppression**: Suppresses stdout from Python implementation's `finish()` call to avoid statistics output +3. **Rust Test Harness Filtering**: Removes Rust test harness noise (`running 1 test`, `test result: ok...`) +4. **Blank Line Removal**: Strips all blank lines to eliminate formatting differences +5. **Whitespace Normalization**: Strips trailing whitespace from all outputs + +## Return Codes + +- **0**: All outputs are identical +- **1**: Outputs differ (shows unified diff) +- **2**: Command execution failed + +## Example Output + +### Success +```bash +$ python python/unit-tests/test_trilang_ramulator2_valid.py +All outputs are identical across implementations. +``` + +### Failure with Differences +```bash +$ python python/unit-tests/test_trilang_ramulator2_valid.py +[DIFF] cpp vs rust: +--- cpp ++++ rust +@@ -1,3 +1,2 @@ + Cycle 3: Write request sent for address 2, success or not (true or false)true + Cycle 9: Request completed: 2 the data is: 1 +-Cycle 5: Write request sent for address 4, success or not (true or false)true ++Cycle 5: Write request sent for address 4, success or not (true or false)false +``` + +### Viewing Filtered Outputs +```bash +$ python python/unit-tests/test_trilang_ramulator2_valid.py --show-outputs + +=== CPP OUTPUT (FILTERED) === +Cycle 2: Write request sent for address 1, success or not (true or false)true +Cycle 3: Request completed: 0 the data is: -1 +... + +=== RUST OUTPUT (FILTERED) === +Cycle 2: Write request sent for address 1, success or not (true or false)true +Cycle 3: Request completed: 0 the data is: -1 +... + +=== PYTHON OUTPUT (FILTERED) === +Cycle 2: Write request sent for address 1, success or not (true or false)true +Cycle 3: Request completed: 0 the data is: -1 +... +``` + +## Related Files + +- `python/unit-tests/test_trilang_ramulator2_valid.py`: Main validation script +- `python/assassyn/ramulator2.py`: Python Ramulator2 wrapper implementation +- `tools/c-ramulator2-wrapper/test.cpp`: C++ implementation +- `tools/rust-sim-runtime/tests/test_ramulator2.rs`: Rust implementation diff --git a/python/unit-tests/trilang-x-valid.md b/python/unit-tests/trilang-x-valid.md deleted file mode 100644 index 096d9213f..000000000 --- a/python/unit-tests/trilang-x-valid.md +++ /dev/null @@ -1,64 +0,0 @@ -# Triple Language Ramulator2 Cross Validation - -The `compare_ramulator2_outputs.py` script runs three different Ramulator2 wrapper implementations and compares their outputs to ensure behavioral consistency. - -## Core Functionality - -The script runs these three implementations with the same configuration and request sequence: - -- **C++ Implementation**: `tools/c-ramulator2-wrapper/test.cpp` (executable: `build/bin/test`) -- **Rust Implementation**: `tools/rust-sim-runtime/tests/test_ramulator2.rs` (via `cargo test`) -- **Python Implementation**: `python/unit-tests/test_ramulator2.py` - -## Usage - -### Basic Usage -```bash -python python/unit-tests/compare_ramulator2_outputs.py -``` - -### Command Line Options - -- `--skip `: Skip running a specific language implementation -- `--debug`: Enable verbose debugging output -- `--show-outputs`: Display raw outputs before comparison - -## Output Processing - -The script normalizes outputs for fair comparison: -1. Removes Rust test harness noise (`running 1 test`, `test result: ok...`) -2. Strips blank lines and trailing whitespace -3. Compares normalized outputs - -## Return Codes - -- **0**: All outputs are identical -- **1**: Outputs differ (shows unified diff) -- **2**: Command execution failed - -## Example Output - -### Success -```bash -$ python python/unit-tests/compare_ramulator2_outputs.py -All outputs are identical across implementations. -``` - -### Failure with Differences -```bash -$ python python/unit-tests/compare_ramulator2_outputs.py -[DIFF] cpp vs rust: ---- cpp -+++ rust -@@ -1,3 +1,2 @@ - Cycle 3: Write request sent for address 2, success or not (true or false)true - Cycle 9: Request completed: 2 the data is: 1 --Cycle 5: Write request sent for address 4, success or not (true or false)true -+Cycle 5: Write request sent for address 4, success or not (true or false)false -``` - -## Related Files - -- `python/unit-tests/test_ramulator2.py`: Python implementation -- `tools/c-ramulator2-wrapper/test.cpp`: C++ implementation -- `tools/rust-sim-runtime/tests/test_ramulator2.rs`: Rust implementation