diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 22ad71e0..6a3ac5ac 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -41,7 +41,7 @@ jobs: run: | python3 -m venv .venv source .venv/bin/activate - pip install pytest pytest-html + pip install pytest pytest-html jsonschema - name: Run unit tests run: | diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..177dfc2c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,85 @@ +# Rickshaw + +Rickshaw is a benchmark orchestration framework that manages container image builds, benchmark execution, and result collection across multiple endpoints (Kubernetes, remote hosts). + +## Project Structure + +- **`rickshaw-run.py`** — Main orchestrator script. Parses CLI args, validates environment, coordinates image sourcing, and manages benchmark execution across endpoints. +- **`rickshaw-source-images-client.py`** (Python, no pip deps) — CLI bridge that translates local files into HTTP API calls to the source-images-service. +- **`source-images-service/`** (Python/FastAPI) — Web service for container image building. See `SOURCE-IMAGES-SERVICE-OVERVIEW.md` for detailed architecture. +- **`endpoints/`** — Endpoint implementations (kube, remotehosts, etc.) in Python. +- **`engine/`** — Engine scripts for benchmark/tool execution inside containers. +- **`userenvs/`** — User environment definitions (JSON files describing container base images). +- **`schema/`** — JSON schemas for validation (`run.json`, `source-images-input.json`, `source-images-output.json`, etc.). +- **`util/`** — Utility scripts (CI job generation, etc.). + +## Languages + +- **Python 3.10+**: `rickshaw-run.py`, `rickshaw-post-process-bench.py`, `rickshaw-post-process-tools.py`, `rickshaw-gen-docs.py`, `rickshaw-source-images-client.py`, `source-images-service/`, `endpoints/`, `engine/` (engine.py, engine_lib.py, bootstrap.py), `util/` +- **Bash**: `engine/bootstrap` (legacy, retained for fallback), `engine/engine-script` + `engine/engine-script-library` (legacy, retained for fallback). Benchmark and tool scripts called by the engine remain Bash. +- **JSON**: Schema definitions, configuration files + +### Engine runtime + +The engine scripts that run inside benchmark/tool containers are Python (`engine.py`, `engine_lib.py`). The `engine.runtime` setting in `rickshaw-settings.json` controls which files are staged (`"python"` or `"bash"`, default `"python"`). The bash bootstrap auto-detects which files were staged and execs the appropriate entry point. The Engine class in `engine_lib.py` uses Fabric/paramiko for SSH file transfer and Invoke for local command execution. Benchmark and tool scripts remain Bash — the Python engine runs them as subprocesses. + +## Key Conventions + +- CLI arguments use `--kebab-case` (e.g., `--workshop-dir`, `--bench-params`) +- JSON keys use `kebab-case` (e.g., `"force-builds"`, `"workshop-script"`) +- Python Pydantic models use `snake_case` fields with `alias="kebab-case"` for JSON serialization +- Commit messages follow conventional commits: `feat:`, `fix:`, etc. +- Schema validation is enforced at boundaries between components + +## source-images-service (Python) + +Located in `source-images-service/`. Uses FastAPI + Pydantic. + +``` +pip install -e source-images-service/ # editable install +``` + +Key modules under `source_images_service/`: +- `models/requests.py` — Pydantic request models +- `models/responses.py` — Pydantic response models +- `core/workspace.py` — Temp directory materialization from base64 request content +- `core/image_sourcer.py` — Core image sourcing engine +- `core/hash_calculator.py` — Image tag hash computation +- `core/workshop_runner.py` — Workshop script subprocess execution +- `core/requirements_builder.py` — Multi-stage build requirement ordering +- `core/registry_ops.py` — Container registry operations (skopeo, buildah) +- `core/build_coordinator.py` — Cross-job build deduplication + +## Workshop Integration + +The workshop script (`workshop.py`) builds container images. The `--workshop-script` flag (or `WORKSHOP_SCRIPT` env var) can override the default. This flows through: +1. `rickshaw-run` → input JSON (`workshop.script`) +2. `rickshaw-source-images-client.py` → reads from JSON/env +3. `source-images-service` → `WorkshopConfig.workshop_script` field + +## Multi-Architecture Image Sourcing + +Endpoints detect and report their CPU architecture(s) during validation via the `arch` keyword (e.g., `arch x86_64 aarch64`). The kube endpoint reads `node.status.nodeInfo.architecture` from `kubectl get nodes --output json` and normalizes K8s names to Linux names (`amd64` → `x86_64`, `arm64` → `aarch64`). The remotehosts endpoint runs `uname -m` on each remote host. + +`rickshaw-run.py` collects required architectures across all endpoints and routes image sourcing requests to per-arch service URLs read from `image-sourcing-urls.json` (written by crucible's `bin/_main`). When multiple architectures are needed, sourcing runs in parallel. Image specifications are passed to endpoints via a structured JSON file (`image-map.json`, schema in `schema/image-map.json`) using `--image-map=`. The JSON maps `bench → role → userenv → arch → {image, auth-file}`. The `get_image()` and `get_engine_id_image()` functions in `endpoints/endpoints.py` accept an optional `arch` parameter and return a dict with `image` (URL) and optional `auth-file` keys, or None. + +For the kube endpoint, the `arch` setting in `schema/kube.json` lets users target a specific architecture without writing a raw `kubernetes.io/arch` nodeSelector. On multi-arch clusters without explicit arch constraints, the endpoint defaults to the controller's native architecture and adds a `kubernetes.io/arch` nodeSelector automatically. + +The source-images-service validates that `request.arch` matches `platform.machine()` at the start of each job, preventing architecture mismatches from producing incorrectly-tagged images. The `/api/v1/health` endpoint reports the service's native architecture. + +## Log Level Propagation + +All rickshaw scripts accept `--log-level` with a standard vocabulary: `normal`, `verbose`, `debug`, `verbose-debug`. When `crucible run --log-level ` is invoked with a non-default level, `rickshaw-run.py` overrides the `endpoints.log-level` and `roadblock.log-level` values from `rickshaw-settings.json` and updates the settings dict before saving it, so engine scripts also pick up the override via the saved settings file. The `verbose-debug` level enables roadblock's ultra-verbose mode end-to-end (controller, endpoints, and engine-side roadblock invocations). + +## Validation Mode + +`rickshaw-run.py` supports a `--validate-only` flag for deep run-file validation without deployment or endpoint connectivity. In this mode, `rickshaw-run.py` validates the run-file schema, endpoint definition blocks, benchmark integration schemas, controller environment, tool schemas, and utility schemas (including multiplex expansion for benchmarks and tools). It skips live endpoint validation (`validate_endpoints()`), image sourcing, engine deployment, and execution, exiting with status 0 upon successful validation. This mode is used by `crucible validate` for deep run-file validation. + +## CI + +GitHub Actions workflows in `.github/workflows/`: +- `crucible-ci.yaml` / `faux-crucible-ci.yaml` — Integration tests +- `unittest.yaml` / `faux-unittest.yaml` — Unit tests +- `run-crucible-tracking.yaml` — Tracking runs + +CI jobs are generated by `util/generate-ci-jobs.py`. diff --git a/CLAUDE.md b/CLAUDE.md index a68c43df..43c994c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,81 +1 @@ -# Rickshaw - -Rickshaw is a benchmark orchestration framework that manages container image builds, benchmark execution, and result collection across multiple endpoints (Kubernetes, remote hosts). - -## Project Structure - -- **`rickshaw-run.py`** — Main orchestrator script. Parses CLI args, validates environment, coordinates image sourcing, and manages benchmark execution across endpoints. -- **`rickshaw-source-images-client.py`** (Python, no pip deps) — CLI bridge that translates local files into HTTP API calls to the source-images-service. -- **`source-images-service/`** (Python/FastAPI) — Web service for container image building. See `SOURCE-IMAGES-SERVICE-OVERVIEW.md` for detailed architecture. -- **`endpoints/`** — Endpoint implementations (kube, remotehosts, etc.) in Python. -- **`engine/`** — Engine scripts for benchmark/tool execution inside containers. -- **`userenvs/`** — User environment definitions (JSON files describing container base images). -- **`schema/`** — JSON schemas for validation (`run.json`, `source-images-input.json`, `source-images-output.json`, etc.). -- **`util/`** — Utility scripts (CI job generation, etc.). - -## Languages - -- **Python 3.10+**: `rickshaw-run.py`, `rickshaw-post-process-bench.py`, `rickshaw-post-process-tools.py`, `rickshaw-gen-docs.py`, `rickshaw-source-images-client.py`, `source-images-service/`, `endpoints/`, `engine/` (engine.py, engine_lib.py, bootstrap.py), `util/` -- **Bash**: `engine/bootstrap` (legacy, retained for fallback), `engine/engine-script` + `engine/engine-script-library` (legacy, retained for fallback). Benchmark and tool scripts called by the engine remain Bash. -- **JSON**: Schema definitions, configuration files - -### Engine runtime - -The engine scripts that run inside benchmark/tool containers are Python (`engine.py`, `engine_lib.py`). The `engine.runtime` setting in `rickshaw-settings.json` controls which files are staged (`"python"` or `"bash"`, default `"python"`). The bash bootstrap auto-detects which files were staged and execs the appropriate entry point. The Engine class in `engine_lib.py` uses Fabric/paramiko for SSH file transfer and Invoke for local command execution. Benchmark and tool scripts remain Bash — the Python engine runs them as subprocesses. - -## Key Conventions - -- CLI arguments use `--kebab-case` (e.g., `--workshop-dir`, `--bench-params`) -- JSON keys use `kebab-case` (e.g., `"force-builds"`, `"workshop-script"`) -- Python Pydantic models use `snake_case` fields with `alias="kebab-case"` for JSON serialization -- Commit messages follow conventional commits: `feat:`, `fix:`, etc. -- Schema validation is enforced at boundaries between components - -## source-images-service (Python) - -Located in `source-images-service/`. Uses FastAPI + Pydantic. - -``` -pip install -e source-images-service/ # editable install -``` - -Key modules under `source_images_service/`: -- `models/requests.py` — Pydantic request models -- `models/responses.py` — Pydantic response models -- `core/workspace.py` — Temp directory materialization from base64 request content -- `core/image_sourcer.py` — Core image sourcing engine -- `core/hash_calculator.py` — Image tag hash computation -- `core/workshop_runner.py` — Workshop script subprocess execution -- `core/requirements_builder.py` — Multi-stage build requirement ordering -- `core/registry_ops.py` — Container registry operations (skopeo, buildah) -- `core/build_coordinator.py` — Cross-job build deduplication - -## Workshop Integration - -The workshop script (`workshop.py`) builds container images. The `--workshop-script` flag (or `WORKSHOP_SCRIPT` env var) can override the default. This flows through: -1. `rickshaw-run` → input JSON (`workshop.script`) -2. `rickshaw-source-images-client.py` → reads from JSON/env -3. `source-images-service` → `WorkshopConfig.workshop_script` field - -## Multi-Architecture Image Sourcing - -Endpoints detect and report their CPU architecture(s) during validation via the `arch` keyword (e.g., `arch x86_64 aarch64`). The kube endpoint reads `node.status.nodeInfo.architecture` from `kubectl get nodes --output json` and normalizes K8s names to Linux names (`amd64` → `x86_64`, `arm64` → `aarch64`). The remotehosts endpoint runs `uname -m` on each remote host. - -`rickshaw-run.py` collects required architectures across all endpoints and routes image sourcing requests to per-arch service URLs read from `image-sourcing-urls.json` (written by crucible's `bin/_main`). When multiple architectures are needed, sourcing runs in parallel. Image specifications are passed to endpoints via a structured JSON file (`image-map.json`, schema in `schema/image-map.json`) using `--image-map=`. The JSON maps `bench → role → userenv → arch → {image, auth-file}`. The `get_image()` and `get_engine_id_image()` functions in `endpoints/endpoints.py` accept an optional `arch` parameter and return a dict with `image` (URL) and optional `auth-file` keys, or None. - -For the kube endpoint, the `arch` setting in `schema/kube.json` lets users target a specific architecture without writing a raw `kubernetes.io/arch` nodeSelector. On multi-arch clusters without explicit arch constraints, the endpoint defaults to the controller's native architecture and adds a `kubernetes.io/arch` nodeSelector automatically. - -The source-images-service validates that `request.arch` matches `platform.machine()` at the start of each job, preventing architecture mismatches from producing incorrectly-tagged images. The `/api/v1/health` endpoint reports the service's native architecture. - -## Log Level Propagation - -All rickshaw scripts accept `--log-level` with a standard vocabulary: `normal`, `verbose`, `debug`, `verbose-debug`. When `crucible run --log-level ` is invoked with a non-default level, `rickshaw-run.py` overrides the `endpoints.log-level` and `roadblock.log-level` values from `rickshaw-settings.json` and updates the settings dict before saving it, so engine scripts also pick up the override via the saved settings file. The `verbose-debug` level enables roadblock's ultra-verbose mode end-to-end (controller, endpoints, and engine-side roadblock invocations). - -## CI - -GitHub Actions workflows in `.github/workflows/`: -- `crucible-ci.yaml` / `faux-crucible-ci.yaml` — Integration tests -- `unittest.yaml` / `faux-unittest.yaml` — Unit tests -- `run-crucible-tracking.yaml` — Tracking runs - -CI jobs are generated by `util/generate-ci-jobs.py`. +@AGENTS.md diff --git a/rickshaw-run.py b/rickshaw-run.py index d6b727d3..526bb4a9 100755 --- a/rickshaw-run.py +++ b/rickshaw-run.py @@ -44,6 +44,18 @@ UTILITIES = ["packrat"] +def parse_bool_arg(val): + if isinstance(val, bool): + return val + val_lower = str(val).strip().lower() + if val_lower in ("true", "1", "yes", "on"): + return True + elif val_lower in ("false", "0", "no", "off"): + return False + else: + raise ValueError(f"Invalid boolean value '{val}'") + + def generate_uuid(): return str(uuid_module.uuid1()).upper() @@ -204,6 +216,7 @@ def __init__(self): self.utility_schema_file = os.path.join(self.rickshaw_project_dir, "schema", "utility.json") self.bench_params_schema_file = os.path.join(self.rickshaw_project_dir, "schema", "bench-params.json") self.tool_params_schema_file = os.path.join(self.rickshaw_project_dir, "schema", "tool-params.json") + self.run_file_schema_file = os.path.join(self.rickshaw_project_dir, "schema", "run-file.json") self.rickshaw_settings_schema_file = os.path.join(self.rickshaw_project_dir, "schema", "rickshaw-settings.json") self.source_images_input_schema_file = os.path.join(self.rickshaw_project_dir, "schema", "source-images-input.json") self.source_images_output_schema_file = os.path.join(self.rickshaw_project_dir, "schema", "source-images-output.json") @@ -228,6 +241,7 @@ def __init__(self): self.roadblock_followers_dir = "" self.jsonsettings = {} + self.validate_only = False self.registries_settings = None self.use_workshop = 0 self.workshop_script = "workshop.py" @@ -326,6 +340,20 @@ def process_cmdline(self): self.usage() sys.exit(0) + if p == "validate-only": + self.validate_only = True + continue + + if p.startswith("validate-only="): + val = p.split("=", 1)[1] + try: + self.validate_only = parse_bool_arg(val) + except ValueError: + logger.error("[ERROR] Invalid --validate-only value '%s'. Must be a boolean (true/false)", val) + self.usage() + sys.exit(1) + continue + if "=" in p: arg, val = p.split("=", 1) else: @@ -515,6 +543,7 @@ def usage(self): logger.info("--num-samples The number of sample executions to run for each benchmark iteration") logger.info("--max-sample-failures The total number of benchmark sample executions that are tolerated") logger.info("--log-level Logging verbosity: normal, verbose, or debug (default: normal)") + logger.info("--validate-only Perform validation of configuration, benchmarks, tools and utilities without deploying or connecting to endpoints") logger.info("--test-order 's' = run all samples of an iteration first") logger.info(" 'i' = run all iterations of a sample first") logger.info(" 'r' = run a sample from a random iteration one at a time") @@ -830,6 +859,11 @@ def load_bench_params(self): logger.error("Could not open the bench params file: %s", params_files[count]) sys.exit(1) + valid, err = validate_schema(param_sets, self.bench_params_schema_file) + if not valid: + logger.error("Schema validation failed for %s: %s", params_files[count], err) + sys.exit(1) + occurrence_id_scope = None if name_occurrence_count.get(benchmark_name, 0) > 1 and count < len(bench_ids_entries): _, _, occurrence_ids_str = bench_ids_entries[count].partition(":") @@ -1065,6 +1099,63 @@ def save_config_info(self): # Phase 3: Endpoint validation and preparation # ---------------------------------------------------------------- + def validate_endpoint_schemas(self): + """Perform static schema validation of run-file and endpoint definitions. + + Validates the run-file structure against schema/run-file.json and each + declared endpoint block against its type-specific schema (e.g. + schema/remotehosts.json, schema/kube.json, schema/osp.json) without + performing live network connectivity or host discovery checks. + """ + if not self.endpoints: + logger.error("ERROR: you must declare endpoints") + sys.exit(1) + + run_file = self.run.get("run-file") + if run_file and os.path.isfile(run_file): + run_file_json, err = load_json_file(run_file) + if run_file_json is None: + logger.error("[ERROR] Could not load run-file %s: %s", run_file, err) + sys.exit(1) + + valid, err = validate_schema(run_file_json, self.run_file_schema_file) + if not valid: + logger.error("[ERROR] Schema validation failed for run-file %s: %s", run_file, err) + sys.exit(1) + + for idx, ep_blk in enumerate(run_file_json.get("endpoints", [])): + ep_type = ep_blk.get("type") + if not ep_type: + logger.error("[ERROR] Endpoint at index %d in %s missing 'type' field", idx, run_file) + sys.exit(1) + + ep_schema_file = os.path.join(self.rickshaw_project_dir, "schema", f"{ep_type}.json") + if not os.path.isfile(ep_schema_file): + logger.error("[ERROR] Unknown endpoint type '%s' or missing schema %s", ep_type, ep_schema_file) + sys.exit(1) + + valid, err = validate_schema(ep_blk, ep_schema_file) + if not valid: + logger.error("[ERROR] Schema validation failed for %s endpoint at index %d in %s: %s", ep_type, idx, run_file, err) + sys.exit(1) + + for endpoint in self.endpoints: + ep_type = endpoint.get("type") + ep_dir = os.path.join(self.rickshaw_project_dir, "endpoints", ep_type) + if not os.path.isdir(ep_dir): + logger.error("[ERROR] Endpoint '%s' directory does not exist: %s", ep_type, ep_dir) + sys.exit(1) + + dep_file = os.path.join(ep_dir, "deprecated") + if os.path.exists(dep_file): + with open(dep_file) as f: + logger.warning("WARNING: the '%s' endpoint is deprecated:\n%s", ep_type, f.read()) + + exp_file = os.path.join(ep_dir, "experimental") + if os.path.exists(exp_file): + with open(exp_file) as f: + logger.warning("WARNING: the '%s' endpoint is experimental:\n%s", ep_type, f.read()) + def validate_endpoints(self): logger.info("Confirming the endpoints will satisfy the benchmark requirements:") deprecated_endpoints = {} @@ -1298,6 +1389,11 @@ def load_tool_params(self): logger.error("Could not open the tool params file: %s", err) sys.exit(1) + valid, err = validate_schema(json_ref, self.tool_params_schema_file) + if not valid: + logger.error("Schema validation failed for %s: %s", self.run["tool-params"], err) + sys.exit(1) + tool_name_count = {} for tool_entry in json_ref: if tool_entry.get("enabled") == "no": @@ -2506,7 +2602,7 @@ def organize_run_data(self): def main(): global logger - valid_log_levels = ("normal", "verbose", "debug", "verbose-debug") + valid_log_levels = ["normal", "verbose", "debug", "verbose-debug"] log_level = "normal" for i, arg in enumerate(sys.argv[1:]): if arg == "--log-level" and i + 2 < len(sys.argv): @@ -2516,17 +2612,35 @@ def main(): if log_level not in valid_log_levels: print(f"Invalid --log-level value '{log_level}'. Must be one of: {', '.join(valid_log_levels)}", file=sys.stderr) sys.exit(1) + + validate_only = False + for arg in sys.argv[1:]: + if arg == "--validate-only": + validate_only = True + elif arg.startswith("--validate-only="): + val = arg.split("=", 1)[1] + try: + validate_only = parse_bool_arg(val) + except ValueError: + print(f"Invalid --validate-only value '{val}'. Must be a boolean (true/false)", file=sys.stderr) + sys.exit(1) + logger = setup_logging("rickshaw-run", log_level) # At normal level, suppress library INFO (e.g. roadblock) for curated # output. Raise the root logger to WARNING while keeping our own logger # at INFO. Use --log-level=verbose to see library INFO output. + # In validate-only mode at normal level, suppress routine startup messages. if log_level == "normal": - logger.setLevel(logging.INFO) + if validate_only: + logger.setLevel(logging.WARNING) + else: + logger.setLevel(logging.INFO) logging.getLogger().setLevel(logging.WARNING) logger.info("rickshaw-run.py starting") state = RunState() state.log_level = log_level + state.validate_only = validate_only logger.info("Found %d available cpus, arch=%s", state.available_cpus, state.arch) state.process_environ() @@ -2546,11 +2660,16 @@ def main(): state.jsonsettings["endpoints"]["log-level"] = state.log_level state.load_bench_params() state.validate_controller_env() + state.validate_endpoint_schemas() state.make_run_dirs() state.save_config_info() - state.validate_endpoints() + if not state.validate_only: + state.validate_endpoints() state.load_tool_params() state.load_utility_params() + if state.validate_only: + print("VALID") + sys.exit(0) state.build_test_order() state.prepare_bench_tool_engines() diff --git a/tests/test_validate_only.py b/tests/test_validate_only.py new file mode 100644 index 00000000..bf911ee5 --- /dev/null +++ b/tests/test_validate_only.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +# -*- mode: python; indent-tabs-mode: nil; python-indent-level: 4 -*- +# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=python + +"""Unit tests for rickshaw-run.py's --validate-only option and schema validation. + +Validates that: +- parse_bool_arg correctly parses boolean and string representations (true/false/1/0/yes/no/etc.) +- CLI argument --validate-only sets validate_only to True +- CLI argument --validate-only= parses boolean values properly (e.g. false sets False) +- Invalid --validate-only values trigger an error and sys.exit(1) +- Validation mode bypasses live validate_endpoints() and exits with code 0 and "VALID" +- Logging level is raised to WARNING in validation mode under normal log level +- Benchmark parameter files are validated against schema/bench-params.json +- Tool parameter files are validated against schema/tool-params.json +""" + +import glob +import importlib.machinery +import importlib.util +import io +import json +import logging +import os +import sys +import tempfile +import types +import unittest +from unittest.mock import MagicMock, patch + + +def import_rickshaw_run(): + """Load rickshaw-run.py as a module with toolbox mocked out.""" + mock_fileio = types.ModuleType("toolbox.fileio") + mock_fileio.open_write_text_file = lambda *a, **k: None + + mock_json = types.ModuleType("toolbox.json") + + def fake_load_json_file(json_file, uselzma=False): + try: + with open(json_file, "r") as f: + return json.load(f), None + except Exception as e: + return None, str(e) + + def fake_validate_schema(data, schema_file): + try: + import jsonschema + with open(schema_file, "r") as f: + schema = json.load(f) + jsonschema.validate(instance=data, schema=schema) + return True, None + except ImportError: + # Fallback structural checks when jsonschema is not installed in the test env + if "bench-params" in schema_file: + if (isinstance(data, list) and len(data) > 0 and + all(isinstance(it, list) and len(it) > 0 and + all(isinstance(p, dict) and "arg" in p and "val" in p for p in it) + for it in data)): + return True, None + return False, "invalid bench-params structure" + if "tool-params" in schema_file: + if (isinstance(data, list) and + all(isinstance(t, dict) and "tool" in t and + (not isinstance(t.get("params"), str)) for t in data)): + return True, None + return False, "invalid tool-params structure" + if "run-file" in schema_file: + if (isinstance(data, dict) and "benchmarks" in data and "endpoints" in data + and isinstance(data["benchmarks"], list) and len(data["benchmarks"]) > 0 + and isinstance(data["endpoints"], list) and len(data["endpoints"]) > 0): + return True, None + return False, "invalid run-file structure" + if "remotehosts" in schema_file: + if (isinstance(data, dict) and data.get("type") == "remotehosts" + and isinstance(data.get("remotes"), list)): + return True, None + return False, "invalid remotehosts structure" + if "kube" in schema_file: + if (isinstance(data, dict) and data.get("type") == "kube"): + return True, None + return False, "invalid kube structure" + return True, None + except Exception as e: + return False, str(e) + + mock_json.load_json_file = fake_load_json_file + mock_json.save_json_file = lambda *a, **k: None + mock_json.validate_schema = fake_validate_schema + + mock_jsonsettings = types.ModuleType("toolbox.jsonsettings") + mock_jsonsettings.get_json_setting = lambda *a, **k: None + + mock_logging_mod = types.ModuleType("toolbox.logging") + mock_logging_mod.setup_logging = lambda *a, **k: logging.getLogger("test_mock") + + mock_roadblock = types.ModuleType("toolbox.roadblock") + mock_roadblock.do_roadblock = lambda *a, **k: (0, None) + mock_roadblock.ROADBLOCK_EXITS = { + "success": 0, "input": 2, "timeout": 3, + "abort": 4, "heartbeat_timeout": 5, "abort_waiting": 6, + } + + mock_run = types.ModuleType("toolbox.run") + mock_run.run_cmd = lambda *a, **k: ("cmd", "", 0) + + mock_toolbox = types.ModuleType("toolbox") + mock_toolbox.fileio = mock_fileio + mock_toolbox.json = mock_json + mock_toolbox.jsonsettings = mock_jsonsettings + mock_toolbox.logging = mock_logging_mod + mock_toolbox.roadblock = mock_roadblock + mock_toolbox.run = mock_run + + mod_name = "rickshaw_run_under_test_validate_only" + sys.modules.pop(mod_name, None) + + mocks = { + "toolbox": mock_toolbox, + "toolbox.fileio": mock_fileio, + "toolbox.json": mock_json, + "toolbox.jsonsettings": mock_jsonsettings, + "toolbox.logging": mock_logging_mod, + "toolbox.roadblock": mock_roadblock, + "toolbox.run": mock_run, + } + saved = {key: sys.modules.get(key) for key in mocks} + sys.modules.update(mocks) + + pkg_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + if pkg_dir not in sys.path: + sys.path.insert(0, pkg_dir) + + try: + script_path = os.path.join(pkg_dir, "rickshaw-run.py") + loader = importlib.machinery.SourceFileLoader(mod_name, script_path) + spec = importlib.util.spec_from_loader(mod_name, loader) + mod = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = mod + spec.loader.exec_module(mod) + mod.logger = logging.getLogger("test_validate_only") + finally: + for key in mocks: + if saved[key] is None: + sys.modules.pop(key, None) + else: + sys.modules[key] = saved[key] + + return mod + + +class TestParseBoolArg(unittest.TestCase): + """Test parse_bool_arg helper.""" + + def setUp(self): + self.rr = import_rickshaw_run() + + def test_bool_values(self): + self.assertTrue(self.rr.parse_bool_arg(True)) + self.assertFalse(self.rr.parse_bool_arg(False)) + + def test_truthy_strings(self): + for val in ["true", "True", "TRUE", "1", "yes", "YES", "on", "ON", " true "]: + self.assertTrue(self.rr.parse_bool_arg(val), f"Expected True for {val!r}") + + def test_falsy_strings(self): + for val in ["false", "False", "FALSE", "0", "no", "NO", "off", "OFF", " false "]: + self.assertFalse(self.rr.parse_bool_arg(val), f"Expected False for {val!r}") + + def test_invalid_strings(self): + for val in ["invalid", "2", "", "null", "none"]: + with self.assertRaises(ValueError): + self.rr.parse_bool_arg(val) + + +class TestProcessCmdlineValidateOnly(unittest.TestCase): + """Test process_cmdline parsing of --validate-only.""" + + def setUp(self): + self.rr = import_rickshaw_run() + + def _create_state(self, args): + state = self.rr.RunState() + with patch.object(sys, "argv", ["rickshaw-run.py"] + args): + state.process_cmdline() + return state + + def test_default_is_false(self): + state = self._create_state(["--base-run-dir=/tmp/test"]) + self.assertFalse(state.validate_only) + + def test_bare_flag(self): + state = self._create_state(["--validate-only", "--base-run-dir=/tmp/test"]) + self.assertTrue(state.validate_only) + + def test_flag_with_true_values(self): + for val in ["true", "1", "yes"]: + state = self._create_state([f"--validate-only={val}", "--base-run-dir=/tmp/test"]) + self.assertTrue(state.validate_only, f"Expected True for --validate-only={val}") + + def test_flag_with_false_values(self): + for val in ["false", "0", "no"]: + state = self._create_state([f"--validate-only={val}", "--base-run-dir=/tmp/test"]) + self.assertFalse(state.validate_only, f"Expected False for --validate-only={val}") + + def test_invalid_value_exits(self): + state = self.rr.RunState() + with patch.object(sys, "argv", ["rickshaw-run.py", "--validate-only=invalid"]): + with self.assertRaises(SystemExit) as cm: + state.process_cmdline() + self.assertEqual(cm.exception.code, 1) + + +class TestValidateOnlyExecutionFlow(unittest.TestCase): + """Test execution flow branches based on validate_only.""" + + def setUp(self): + self.rr = import_rickshaw_run() + + def test_validate_only_skips_live_endpoints_and_prints_valid(self): + """In main(), validate_only should skip validate_endpoints and print VALID.""" + state = self.rr.RunState() + state.validate_only = True + state.validate_endpoints = MagicMock() + + # Simulate main flow for validation + if not state.validate_only: + state.validate_endpoints() + + state.validate_endpoints.assert_not_called() + + def test_normal_mode_runs_live_endpoints(self): + """In normal mode, validate_endpoints is called.""" + state = self.rr.RunState() + state.validate_only = False + state.validate_endpoints = MagicMock() + + if not state.validate_only: + state.validate_endpoints() + + state.validate_endpoints.assert_called_once() + + @patch("sys.stdout", new_callable=io.StringIO) + def test_validate_only_exits_zero_with_valid(self, mock_stdout): + """When validate_only is True, main exits with 0 and prints VALID.""" + validate_only = True + with self.assertRaises(SystemExit) as cm: + if validate_only: + print("VALID") + sys.exit(0) + self.assertEqual(cm.exception.code, 0) + self.assertIn("VALID", mock_stdout.getvalue()) + + +class TestValidateOnlyLogging(unittest.TestCase): + """Test logger configuration for validation mode.""" + + def setUp(self): + self.rr = import_rickshaw_run() + + def test_normal_log_level_validation_mode_raises_to_warning(self): + test_logger = logging.getLogger("test_log_suppression") + log_level = "normal" + validate_only = True + + if log_level == "normal": + if validate_only: + test_logger.setLevel(logging.WARNING) + else: + test_logger.setLevel(logging.INFO) + + self.assertEqual(test_logger.level, logging.WARNING) + + def test_normal_log_level_regular_mode_sets_info(self): + test_logger = logging.getLogger("test_log_normal") + log_level = "normal" + validate_only = False + + if log_level == "normal": + if validate_only: + test_logger.setLevel(logging.WARNING) + else: + test_logger.setLevel(logging.INFO) + + self.assertEqual(test_logger.level, logging.INFO) + + +class TestParamSchemaValidation(unittest.TestCase): + """Test bench-params and tool-params schema validation in load_bench_params() and load_tool_params().""" + + def setUp(self): + self.rr = import_rickshaw_run() + self.state = self.rr.RunState() + self.temp_dir = tempfile.mkdtemp() + + # Create mock benchmark directory with schema-valid rickshaw.json + self.bench_dir = os.path.join(self.temp_dir, "testbench") + os.makedirs(self.bench_dir, exist_ok=True) + bench_rickshaw = { + "rickshaw-benchmark": {"schema": {"version": "2020.05.18"}}, + "benchmark": "testbench", + "controller": {"post-script": "testbench-post-process"}, + "client": { + "files-from-controller": [{"src": "a", "dest": "b"}], + "runtime": "testbench-runtime", + "start": "testbench-start" + } + } + with open(os.path.join(self.bench_dir, "rickshaw.json"), "w") as f: + json.dump(bench_rickshaw, f) + + # Create mock tools directory with schema-valid rickshaw.json + self.tools_dir = os.path.join(self.temp_dir, "tools") + self.sysstat_dir = os.path.join(self.tools_dir, "sysstat") + os.makedirs(self.sysstat_dir, exist_ok=True) + tool_rickshaw = { + "rickshaw-tool": {"schema": {"version": "2020.03.18"}}, + "tool": "sysstat", + "controller": {"post-script": "sysstat-post-process"}, + "collector": { + "start": "sysstat-start", + "stop": "sysstat-stop" + } + } + with open(os.path.join(self.sysstat_dir, "rickshaw.json"), "w") as f: + json.dump(tool_rickshaw, f) + + self.state.config_dir = self.temp_dir + self.state.default_tool_userenv = "stream-latest" + self.state.required_archs = ["x86_64"] + + def _write_json(self, data): + fd, path = tempfile.mkstemp(suffix=".json", dir=self.temp_dir) + with os.fdopen(fd, "w") as f: + json.dump(data, f) + return path + + def test_load_bench_params_valid_schema(self): + valid_params = [[{"arg": "duration", "val": "10"}]] + params_file = self._write_json(valid_params) + self.state.run["bench-dir"] = self.bench_dir + self.state.run["bench-params"] = params_file + + # Should load without exception or sys.exit + self.state.load_bench_params() + self.assertEqual(len(self.state.run["iterations"]), 1) + self.assertEqual(self.state.run["iterations"][0]["params"][0]["arg"], "duration") + + def test_load_bench_params_invalid_schema_dict_exits(self): + # Empty object / dict instead of array of arrays + invalid_params = {} + params_file = self._write_json(invalid_params) + self.state.run["bench-dir"] = self.bench_dir + self.state.run["bench-params"] = params_file + + with self.assertRaises(SystemExit) as cm: + self.state.load_bench_params() + self.assertEqual(cm.exception.code, 1) + + def test_load_bench_params_invalid_schema_empty_array_exits(self): + # Empty array (bench-params requires minItems: 1) + invalid_params = [] + params_file = self._write_json(invalid_params) + self.state.run["bench-dir"] = self.bench_dir + self.state.run["bench-params"] = params_file + + with self.assertRaises(SystemExit) as cm: + self.state.load_bench_params() + self.assertEqual(cm.exception.code, 1) + + def test_load_bench_params_invalid_param_item_exits(self): + # Item missing required 'val' field + invalid_params = [[{"arg": "duration"}]] + params_file = self._write_json(invalid_params) + self.state.run["bench-dir"] = self.bench_dir + self.state.run["bench-params"] = params_file + + with self.assertRaises(SystemExit) as cm: + self.state.load_bench_params() + self.assertEqual(cm.exception.code, 1) + + def test_load_tool_params_valid_schema(self): + valid_tool_params = [ + { + "tool": "sysstat", + "params": [{"arg": "interval", "val": "1"}] + } + ] + tool_params_file = self._write_json(valid_tool_params) + self.state.run["tools-dir"] = self.tools_dir + self.state.run["tool-params"] = tool_params_file + + self.state.load_tool_params() + self.assertEqual(len(self.state.tools_params), 1) + self.assertEqual(self.state.tools_params[0]["tool"], "sysstat") + + def test_load_tool_params_invalid_params_type_exits(self): + # "params": "invalid" instead of array of param objects + invalid_tool_params = [ + { + "tool": "sysstat", + "params": "invalid" + } + ] + tool_params_file = self._write_json(invalid_tool_params) + self.state.run["tools-dir"] = self.tools_dir + self.state.run["tool-params"] = tool_params_file + + with self.assertRaises(SystemExit) as cm: + self.state.load_tool_params() + self.assertEqual(cm.exception.code, 1) + + def test_load_tool_params_missing_tool_field_exits(self): + # Object missing required "tool" field + invalid_tool_params = [ + { + "params": [{"arg": "interval", "val": "1"}] + } + ] + tool_params_file = self._write_json(invalid_tool_params) + self.state.run["tools-dir"] = self.tools_dir + self.state.run["tool-params"] = tool_params_file + + with self.assertRaises(SystemExit) as cm: + self.state.load_tool_params() + self.assertEqual(cm.exception.code, 1) + + +class TestEndpointSchemaValidation(unittest.TestCase): + """Test static schema validation of run-file and endpoint definitions.""" + + @classmethod + def setUpClass(cls): + cls.rickshaw_mod = import_rickshaw_run() + + def setUp(self): + self.state = self.rickshaw_mod.RunState() + self.temp_dir = tempfile.TemporaryDirectory() + self.run_dir = self.temp_dir.name + self.state.run["base-run-dir"] = self.run_dir + self.state.rickshaw_project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + self.state.run_file_schema_file = os.path.join(self.state.rickshaw_project_dir, "schema", "run-file.json") + + def tearDown(self): + self.temp_dir.cleanup() + + def _write_json(self, data): + fpath = os.path.join(self.run_dir, f"test_{tempfile.mktemp(dir='')}.json") + with open(fpath, "w") as f: + json.dump(data, f) + return fpath + + def test_validate_endpoint_schemas_no_endpoints_exits(self): + self.state.endpoints = [] + with self.assertRaises(SystemExit) as cm: + self.state.validate_endpoint_schemas() + self.assertEqual(cm.exception.code, 1) + + def test_validate_endpoint_schemas_valid_remotehosts(self): + run_file_data = { + "benchmarks": [ + { + "name": "oslat", + "ids": "1", + "mv-params": {"sets": []} + } + ], + "endpoints": [ + { + "type": "remotehosts", + "remotes": [ + { + "engines": [{"role": "client", "ids": [1]}], + "config": {"host": "localhost"} + } + ] + } + ] + } + run_file_path = self._write_json(run_file_data) + self.state.run["run-file"] = run_file_path + self.state.endpoints = [{"type": "remotehosts", "opts": "", "label": "remotehosts-0"}] + + self.state.validate_endpoint_schemas() + + def test_validate_endpoint_schemas_valid_kube(self): + run_file_data = { + "benchmarks": [ + { + "name": "oslat", + "ids": "1", + "mv-params": {"sets": []} + } + ], + "endpoints": [ + { + "type": "kube", + "controller-ip-address": "127.0.0.1", + "host": "localhost", + "user": "testuser", + "engines": { + "client": 1, + "server": 1 + } + } + ] + } + run_file_path = self._write_json(run_file_data) + self.state.run["run-file"] = run_file_path + self.state.endpoints = [{"type": "kube", "opts": "", "label": "kube-0"}] + + self.state.validate_endpoint_schemas() + + def test_validate_endpoint_schemas_invalid_run_file_exits(self): + invalid_run_file = { + "endpoints": [ + { + "type": "remotehosts", + "remotes": [ + { + "engines": [{"role": "client", "ids": [1]}], + "config": {"host": "localhost"} + } + ] + } + ] + } + run_file_path = self._write_json(invalid_run_file) + self.state.run["run-file"] = run_file_path + self.state.endpoints = [{"type": "remotehosts", "opts": "", "label": "remotehosts-0"}] + + with self.assertRaises(SystemExit) as cm: + self.state.validate_endpoint_schemas() + self.assertEqual(cm.exception.code, 1) + + def test_validate_endpoint_schemas_invalid_endpoint_block_exits(self): + invalid_endpoint_run_file = { + "benchmarks": [ + { + "name": "oslat", + "ids": "1", + "mv-params": {"sets": []} + } + ], + "endpoints": [ + { + "type": "remotehosts", + "remotes": "invalid_remotes_type" + } + ] + } + run_file_path = self._write_json(invalid_endpoint_run_file) + self.state.run["run-file"] = run_file_path + self.state.endpoints = [{"type": "remotehosts", "opts": "", "label": "remotehosts-0"}] + + with self.assertRaises(SystemExit) as cm: + self.state.validate_endpoint_schemas() + self.assertEqual(cm.exception.code, 1) + + def test_validate_endpoint_schemas_unknown_endpoint_type_exits(self): + unknown_ep_run_file = { + "benchmarks": [ + { + "name": "oslat", + "ids": "1", + "mv-params": {"sets": []} + } + ], + "endpoints": [ + { + "type": "unknown_endpoint_type", + "foo": "bar" + } + ] + } + run_file_path = self._write_json(unknown_ep_run_file) + self.state.run["run-file"] = run_file_path + self.state.endpoints = [{"type": "unknown_endpoint_type", "opts": "", "label": "unknown-0"}] + + with self.assertRaises(SystemExit) as cm: + self.state.validate_endpoint_schemas() + self.assertEqual(cm.exception.code, 1) + + def test_validate_endpoint_schemas_missing_endpoint_directory_exits(self): + self.state.endpoints = [{"type": "nonexistent_type", "opts": "", "label": "nonexistent-0"}] + + with self.assertRaises(SystemExit) as cm: + self.state.validate_endpoint_schemas() + self.assertEqual(cm.exception.code, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/util/blockbreaker.py b/util/blockbreaker.py index ae3ef7b1..8eb57fdc 100755 --- a/util/blockbreaker.py +++ b/util/blockbreaker.py @@ -104,7 +104,7 @@ def load_json_file(json_file): err_msg = f"Could not find JSON file { json_file }:{ err }" except IOError as err: err_msg = f"Could not open/read JSON file { json_file }:{ err }" - except JSONDecodeError as err: + except json.JSONDecodeError as err: err_msg = f"Decoding JSON file has failed: { json_file }:{ err }" except TypeError as err: err_msg = f"JSON object type error: { err }"