diff --git a/build.py b/build.py index fb33aa141..0116f883f 100644 --- a/build.py +++ b/build.py @@ -42,7 +42,7 @@ def diagnostic_paths_for_commit() -> tuple[Path, Path, str]: DIAGNOSTIC_DIR.mkdir(parents=True, exist_ok=True) commit_id = current_commit_id() logd_path = DIAGNOSTIC_DIR / f"build-{commit_id}.logd" - metadata_path = DIAGNOSTIC_DIR / f"build-{commit_id}-metadata.json" + metadata_path = DIAGNOSTIC_DIR / f"build-{commit_id}.json" return logd_path, metadata_path, commit_id @@ -292,18 +292,29 @@ def build_module( if module.name == "engine": build_type = "Release" if release else "Debug" - cfg_result = subprocess.run( - ["cmake", "-S", ".", "-B", "build", - f"-DCMAKE_BUILD_TYPE={build_type}"], - cwd=str(module.dir), - capture_output=True, - text=True, - timeout=120, - env=env, - ) + try: + cfg_result = subprocess.run( + ["cmake", "-S", ".", "-B", "build", + f"-DCMAKE_BUILD_TYPE={build_type}"], + cwd=str(module.dir), + capture_output=True, + text=True, + timeout=120, + env=env, + ) + except subprocess.TimeoutExpired: + return False, time.time() - start, "CMake configure TIMEOUT (120s)" + except FileNotFoundError as e: + return False, 0, f"Command not found: {e}" if cfg_result.returncode != 0: + output_lines = [] + if cfg_result.stdout: + output_lines.append(cfg_result.stdout.strip()) + if cfg_result.stderr: + output_lines.append(cfg_result.stderr.strip()) + output = "\n".join(output_lines) return False, time.time() - start, ( - f"CMake configure failed:\n{cfg_result.stderr}") + f"CMake configure failed:\n{output}") if verbose: print(f" {color('cmake configured', Colors.GRAY)}") cmd = ["cmake", "--build", "build"] @@ -427,20 +438,83 @@ def collect_system_info() -> str: return "\n".join(lines) +def build_diagnostic_report( + results: list[tuple[str, bool, float, str, Optional[str]]], + commit_id: str, + logd_relpaths: Optional[list[str]] = None, + password: Optional[str] = None, + logd_error: Optional[str] = None, + chunked: bool = False, +) -> dict: + diagnostic_logd: Optional[str | list[str]] + if not logd_relpaths: + diagnostic_logd = None + elif len(logd_relpaths) == 1: + diagnostic_logd = logd_relpaths[0] + else: + diagnostic_logd = logd_relpaths + + decrypt_target = logd_relpaths[0] if logd_relpaths and len(logd_relpaths) == 1 else None + if logd_relpaths and len(logd_relpaths) > 1: + decrypt_target = str((DIAGNOSTIC_DIR / f"build-{commit_id}.logd").relative_to(ROOT)) + + report = { + "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "commit": commit_id, + "diagnostic_logd": diagnostic_logd, + "diagnostic_logd_error": logd_error, + "chunked": chunked, + "chunk_size_bytes": DIAGNOSTIC_CHUNK_SIZE if chunked else None, + "password": password, + "decrypt_command": ( + f"encryptly unpack {decrypt_target} --password {password}" + if decrypt_target and password else None + ), + "total_modules": len(results), + "passed": sum(1 for _, s, _, _, _ in results if s), + "failed": sum(1 for _, s, _, _, _ in results if not s), + "modules": [ + { + "name": name, + "status": "PASS" if success else "FAIL", + "elapsed_seconds": round(elapsed, 3), + "artifact": binary, + "output": output, + } + for name, success, elapsed, output, binary in results + ], + "pr_note": ( + (f"Include the encrypted diagnostic logd artifact(s): {', '.join(logd_relpaths)}. " if logd_relpaths else "Encrypted diagnostic logd artifact was not created; include this JSON report showing why. ") + + "The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. " + + "Maintainers may ask you to remove these diagnostic artifacts before merging." + ), + } + return report + + +def write_diagnostic_report(metadata_path: Path, report: dict) -> None: + metadata_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(f" {color('✓', Colors.GREEN)} {metadata_path.relative_to(ROOT)} created") + + def generate_logd( results: list[tuple[str, bool, float, str, Optional[str]]], verbose: bool = False, ) -> bool: logd_path, metadata_path, commit_id = diagnostic_paths_for_commit() display_logd = logd_path.relative_to(ROOT) - print(f"\n {color('▸', Colors.CYAN)} Finalizing {color(str(display_logd), Colors.BOLD)}...") + print(f"\n {color('▸', Colors.CYAN)} Finalizing diagnostics for {color(str(display_logd), Colors.BOLD)}...") + + # Always write the JSON report first. The encrypted .logd is useful, but the + # report is required even when the build failed before compilation started or + # when encryptly itself is unavailable. + write_diagnostic_report(metadata_path, build_diagnostic_report(results, commit_id)) encryptly_bin = get_encryptly_bin() if encryptly_bin is None: - print( - f" {color('✗', Colors.RED)} encryptly binary not found " - f"({encryptly_platform_help()}); cannot create {display_logd}" - ) + error = f"encryptly binary not found ({encryptly_platform_help()}); cannot create {display_logd}" + print(f" {color('✗', Colors.RED)} {error}") + write_diagnostic_report(metadata_path, build_diagnostic_report(results, commit_id, logd_error=error)) return False # Workspace must live under $HOME because encryptly refuses paths outside home. @@ -503,45 +577,33 @@ def generate_logd( timeout=300, ) if sr.returncode != 0: + error = sr.stderr.strip() or sr.stdout.strip() or "encryptly pack failed" print( f" {color('✗', Colors.RED)} {logd_path.relative_to(ROOT)} creation failed: " - f"{sr.stderr.strip() or sr.stdout.strip()}" + f"{error}" ) if logd_path.exists(): logd_path.unlink() + write_diagnostic_report( + metadata_path, + build_diagnostic_report(results, commit_id, logd_error=error), + ) return False safe_pw = sr.stdout.strip() logd_files = split_diagnostic_logd(logd_path) logd_relpaths = [str(path.relative_to(ROOT)) for path in logd_files] - diagnostic_logd = logd_relpaths[0] if len(logd_relpaths) == 1 else logd_relpaths decrypt_target = logd_relpaths[0] if len(logd_relpaths) == 1 else str(logd_path.relative_to(ROOT)) - metadata = { - "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "commit": commit_id, - "diagnostic_logd": diagnostic_logd, - "chunked": len(logd_files) > 1, - "chunk_size_bytes": DIAGNOSTIC_CHUNK_SIZE if len(logd_files) > 1 else None, - "password": safe_pw, - "decrypt_command": f"encryptly unpack {decrypt_target} --password {safe_pw}", - "total_modules": len(results), - "passed": sum(1 for _, s, _, _, _ in results if s), - "failed": sum(1 for _, s, _, _, _ in results if not s), - "modules": [ - { - "name": name, - "status": "PASS" if success else "FAIL", - "elapsed_seconds": round(elapsed, 3), - "artifact": binary, - } - for name, success, elapsed, _, binary in results - ], - "pr_note": ( - f"Include this metadata and {', '.join(logd_relpaths)} in your PR. " - "Maintainers may ask you to remove these diagnostic artifacts before merging." + write_diagnostic_report( + metadata_path, + build_diagnostic_report( + results, + commit_id, + logd_relpaths=logd_relpaths, + password=safe_pw, + chunked=len(logd_files) > 1, ), - } - metadata_path.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + ) for path in logd_files: size_kb = path.stat().st_size / 1024.0 @@ -554,7 +616,6 @@ def generate_logd( f" {color('✓', Colors.GREEN)} split oversized diagnostic log into " f"{len(logd_files)} chunks of at most {DIAGNOSTIC_CHUNK_SIZE // (1024 * 1024)} MiB" ) - print(f" {color('✓', Colors.GREEN)} {metadata_path.relative_to(ROOT)} created") if safe_pw: print() print(f" {color('Password', Colors.BOLD)} - this is required to decrypt the diagnostic log,") @@ -687,6 +748,7 @@ def main(): if DIAGNOSTIC_DIR.exists(): diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].logd")) diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]-part*.logd")) + diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].json")) diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]-metadata.json")) for artifact in diagnostic_artifacts: if artifact.exists(): diff --git a/diagnostic/build-00000000-metadata.json b/diagnostic/build-00000000-metadata.json deleted file mode 100644 index ca345639f..000000000 --- a/diagnostic/build-00000000-metadata.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "generated_at": "2026-06-16T00:00:00+00:00", - "commit": "00000000", - "diagnostic_logd": "diagnostic/build-00000000.logd", - "password": "stub-password", - "decrypt_command": "encryptly unpack diagnostic/build-00000000.logd --password stub-password", - "total_modules": 0, - "passed": 0, - "failed": 0, - "modules": [], - "pr_note": "Example only. Real diagnostic files are generated by python3 build.py and should be included in PRs when requested." -} diff --git a/diagnostic/build-00000000.json b/diagnostic/build-00000000.json new file mode 100644 index 000000000..33e2ca62f --- /dev/null +++ b/diagnostic/build-00000000.json @@ -0,0 +1,23 @@ +{ + "generated_at": "2026-06-16T15:23:47.496569+00:00", + "commit": "00000000", + "diagnostic_logd": "diagnostic/build-00000000.logd", + "diagnostic_logd_error": null, + "chunked": false, + "chunk_size_bytes": null, + "password": "4c7df15ab09fbb066197", + "decrypt_command": "encryptly unpack diagnostic/build-00000000.logd --password 4c7df15ab09fbb066197", + "total_modules": 1, + "passed": 0, + "failed": 1, + "modules": [ + { + "name": "frailbox", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'make'" + } + ], + "pr_note": "Include this JSON diagnostic report and diagnostic/build-00000000.logd in your PR. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/tools/tests/README.md b/tools/tests/README.md new file mode 100644 index 000000000..010277231 --- /dev/null +++ b/tools/tests/README.md @@ -0,0 +1,38 @@ +# Log Parser Fixtures & Validation + +These are **independent** fixtures for `tools/log_aggregator.py`. Unlike the +legacy suite (which generated test data from the same parser logic and could +false-pass), every line here is **hand-written** from real-world log examples +and was never produced by the parser. + +## Layout + +``` +tools/tests/fixtures/ + json_sample.log 3 structured JSON log lines (level/severity/lvl variants) + text_sample.log 3 plain-text lines (ISO, standard, syslog timestamps) + nginx_sample.log 2 nginx access-log lines (200 and 500 status) + malformed.log 4 unsupported/partial lines that must NOT crash parsing +tools/tests/test_log_parser_fixtures.py validation script +``` + +## What it checks + +For each supported format the script asserts the extracted: + +- **timestamp** — preserved (JSON) or parsed to a unix int (text/nginx) +- **level** — correct mapping (`error`/`warn`/`info`/`debug`) +- **service / format** — correct source field or `nginx` +- **key fields** — `message`, `request`, `status`, `remote_addr`, etc. + +It also proves malformed lines (broken JSON, no-timestamp text, garbage) are +handled **without raising** — parsers return `None` or a best-effort result. + +## Run it + +```bash +python3 tools/tests/test_log_parser_fixtures.py +``` + +Exit `0` = all checks passed. This script is the required validation for the +log-parser fixtures bounty; it runs in isolation and has no other repo deps. diff --git a/tools/tests/fixtures/json_sample.log b/tools/tests/fixtures/json_sample.log new file mode 100644 index 000000000..eccb4a1a3 --- /dev/null +++ b/tools/tests/fixtures/json_sample.log @@ -0,0 +1,3 @@ +{"timestamp":"2024-03-12T08:15:30Z","level":"error","service":"payments","message":"charge failed for txn 9f3a","order_id":"9f3a"} +{"time":"2024-03-12T09:00:00","severity":"warn","logger":"cache","msg":"redis eviction triggered for key session:42"} +{"@timestamp":"2024-03-12T09:05:11","lvl":"info","app":"api","event":"request handled","path":"/health"} diff --git a/tools/tests/fixtures/malformed.log b/tools/tests/fixtures/malformed.log new file mode 100644 index 000000000..0452faa87 --- /dev/null +++ b/tools/tests/fixtures/malformed.log @@ -0,0 +1,4 @@ +[1, 2, 3] +this line has no timestamp and no level indicator +{"this is": "not valid json +PARTIAL%%%@@@broken@@@%%%PARTIAL diff --git a/tools/tests/fixtures/nginx_sample.log b/tools/tests/fixtures/nginx_sample.log new file mode 100644 index 000000000..9e7c18e77 --- /dev/null +++ b/tools/tests/fixtures/nginx_sample.log @@ -0,0 +1,2 @@ +127.0.0.1 - - [12/Mar/2024:08:30:00 +0000] "GET /api/health HTTP/1.1" 200 512 "http://localhost/" "curl/8.0" +10.0.0.9 - - [12/Mar/2024:08:31:22 +0000] "POST /api/charge HTTP/1.1" 500 0 "-" "Mozilla/5.0" diff --git a/tools/tests/fixtures/text_sample.log b/tools/tests/fixtures/text_sample.log new file mode 100644 index 000000000..21b685e9f --- /dev/null +++ b/tools/tests/fixtures/text_sample.log @@ -0,0 +1,3 @@ +2024-03-12T08:20:00 [auth] ERROR failed login attempt from 10.0.0.5 +2024-03-12 08:21:15 worker WARNING disk usage at 92 percent +Mar 12 08:25:33 scheduler DEBUG job tick executed diff --git a/tools/tests/test_log_parser_fixtures.py b/tools/tests/test_log_parser_fixtures.py new file mode 100644 index 000000000..f3327ee45 --- /dev/null +++ b/tools/tests/test_log_parser_fixtures.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +Independent log-parser validation for Tent of Trials `tools/log_aggregator.py`. + +WHY THIS EXISTS +--------------- +The legacy log-aggregator test data was generated by the same parser logic, so +tests could false-pass. This script validates the parsers against HAND-WRITTEN +representative log lines that were NOT produced by the parser. It covers the +three supported formats (JSON, plain text, nginx access logs) and asserts the +extracted timestamp, level, service/format, and key fields. + +It also proves malformed/unsupported lines do NOT crash parsing (graceful None). + +Run: python3 tools/tests/test_log_parser_fixtures.py +Exit: 0 = all checks passed, 1 = at least one assertion failed. +""" + +import os +import sys + +# Allow running from repo root or from tools/tests/. +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) +sys.path.insert(0, os.path.join(REPO_ROOT, "tools")) + +from log_aggregator import ( # noqa: E402 + JSONLogParser, + TextLogParser, + NginxLogParser, +) + +FIXTURES = os.path.join(HERE, "fixtures") + +failures = [] + + +def expect(cond: bool, msg: str) -> None: + if cond: + print(f" [PASS] {msg}") + else: + print(f" [FAIL] {msg}") + failures.append(msg) + + +def read_lines(name: str): + path = os.path.join(FIXTURES, name) + with open(path, "r", encoding="utf-8") as f: + return [ln.rstrip("\n") for ln in f if ln.strip()] + + +def main() -> int: + json_p = JSONLogParser() + text_p = TextLogParser() + nginx_p = NginxLogParser() + + # ------------------------------------------------------------------ + # JSON fixtures + # ------------------------------------------------------------------ + print("\n== JSON parser (hand-written fixtures) ==") + json_lines = read_lines("json_sample.log") + expect(len(json_lines) == 3, "JSON fixture has 3 representative lines") + + e0 = json_p.parse(json_lines[0]) + expect(e0 is not None, "JSON line 1 parses") + expect(e0["level"] == "error", "JSON line 1 level == 'error'") + expect(e0["service"] == "payments", "JSON line 1 service == 'payments'") + expect(e0["timestamp"] == "2024-03-12T08:15:30Z", "JSON line 1 timestamp preserved") + expect(e0["message"] == "charge failed for txn 9f3a", "JSON line 1 message field extracted") + + e1 = json_p.parse(json_lines[1]) + expect(e1["level"] == "warn", "JSON line 2 maps severity 'warn' -> 'warn'") + expect(e1["service"] == "cache", "JSON line 2 service == 'cache' (logger field)") + + e2 = json_p.parse(json_lines[2]) + expect(e2["level"] == "info", "JSON line 2 maps 'info' (lvl) -> 'info'") + expect(e2["service"] == "api", "JSON line 2 service == 'api' (app field)") + + # ------------------------------------------------------------------ + # Plain text fixtures + # ------------------------------------------------------------------ + print("\n== Text parser (hand-written fixtures) ==") + text_lines = read_lines("text_sample.log") + expect(len(text_lines) == 3, "Text fixture has 3 representative lines") + + t0 = text_p.parse(text_lines[0]) + expect(t0 is not None, "Text line 1 parses") + expect(t0["level"] == "error", "Text line 1 level == 'error' (ERROR token)") + expect(t0["service"] == "auth", "Text line 1 service == 'auth' ([auth] bracket)") + expect(isinstance(t0["timestamp"], int) and t0["timestamp"] > 0, "Text line 1 timestamp parsed to unix int") + + t1 = text_p.parse(text_lines[1]) + expect(t1["level"] == "warn", "Text line 2 level == 'warn'") + expect(t1["timestamp"] is not None, "Text line 2 standard timestamp parsed") + + t2 = text_p.parse(text_lines[2]) + expect(t2["level"] == "debug", "Text line 2 level == 'debug' (DEBUG token)") + expect(t2["timestamp"] is not None, "Text line 2 syslog-style timestamp parsed") + + # ------------------------------------------------------------------ + # Nginx access-log fixtures + # ------------------------------------------------------------------ + print("\n== Nginx parser (hand-written fixtures) ==") + nginx_lines = read_lines("nginx_sample.log") + expect(len(nginx_lines) == 2, "Nginx fixture has 2 representative lines") + + n0 = nginx_p.parse(nginx_lines[0]) + expect(n0 is not None, "Nginx line 1 parses") + expect(n0["service"] == "nginx", "Nginx line 1 service == 'nginx'") + expect(n0["level"] == "info", "Nginx line 1 (status 200) level == 'info'") + expect(n0["fields"]["status"] == 200, "Nginx line 1 status == 200") + expect(n0["fields"]["request"] == "GET /api/health HTTP/1.1", "Nginx line 1 request extracted") + expect(n0["fields"]["remote_addr"] == "127.0.0.1", "Nginx line 1 remote_addr extracted") + expect(isinstance(n0["timestamp"], int) and n0["timestamp"] > 0, "Nginx line 1 timestamp parsed") + + n1 = nginx_p.parse(nginx_lines[1]) + expect(n1["level"] == "error", "Nginx line 2 (status 500) level == 'error'") + expect(n1["fields"]["status"] == 500, "Nginx line 2 status == 500") + + # ------------------------------------------------------------------ + # Malformed / unsupported lines must NOT crash + # ------------------------------------------------------------------ + print("\n== Malformed / unsupported lines (must not crash) ==") + malformed = read_lines("malformed.log") + expect(len(malformed) == 4, "Malformed fixture has 4 lines") + for i, line in enumerate(malformed): + result = None + try: + # Exercise each parser; none should raise. + json_p.parse(line) + text_p.parse(line) + nginx_p.parse(line) + result = "ok" + except Exception as exc: # noqa: BLE001 + result = f"raised: {exc}" + expect(result == "ok", f"Malformed line {i + 1} handled without exception (got: {result})") + + # A broken JSON line must yield None from the JSON parser, not raise. + broken_json = json_p.parse('{"this is": "not valid json') + expect(broken_json is None, "Broken JSON line returns None (no exception)") + + # ------------------------------------------------------------------ + print("\n" + "=" * 60) + if failures: + print(f"RESULT: {len(failures)} CHECK(S) FAILED") + for f in failures: + print(f" - {f}") + return 1 + print("RESULT: ALL INDEPENDENT FIXTURE CHECKS PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main())