From 7dee24d68d3301b3bd62468b102c70f7f322a2dc Mon Sep 17 00:00:00 2001 From: Gowtham Date: Sun, 31 May 2026 22:26:55 -0600 Subject: [PATCH 01/55] Add redaction-first log import --- README.md | 4 ++ docs/commands.md | 2 +- docs/real-log-dogfood.md | 112 ++++++++++++++++++++++++++++++++++++++ redline/cli.py | 14 +++++ redline/import_logs.py | 10 ++++ redline/mcp.py | 4 ++ tests/test_cli_config.py | 53 ++++++++++++++++++ tests/test_import_logs.py | 55 +++++++++++++++++++ tests/test_mcp.py | 1 + tests/test_packaging.py | 2 + 10 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 docs/real-log-dogfood.md diff --git a/README.md b/README.md index 220f766..7dba6bb 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,9 @@ redline is built around the full prompt-regression loop: - `redline watch`: collect prompt-response observations from logs, Python functions, OpenAI/Anthropic-compatible SDK calls, or ASGI apps, with best-effort common secrets and PII redacted before write by default. +- `redline import`: normalize exported team logs into redline JSONL, with the + same best-effort redaction enabled by default. Use `--no-redact` only for + reviewed local-only logs. - `RedlineMiddleware`: capture bounded JSON FastAPI or ASGI request/response pairs locally, with optional skip diagnostics. - `redline redact --check`: scan logs for common secrets and PII, then write a scrubbed copy when needed. Redaction is best-effort pattern matching, not a privacy boundary; review sensitive logs before sharing. @@ -477,6 +480,7 @@ bash scripts/release_check.sh - [docs/launch.md](docs/launch.md): public alpha launch plan - [docs/troubleshooting.md](docs/troubleshooting.md): first-run and CI failure recovery - [docs/commands.md](docs/commands.md): compact CLI command reference +- [docs/real-log-dogfood.md](docs/real-log-dogfood.md): redaction-first real-log test protocol - [docs/dogfood.md](docs/dogfood.md): first-user dogfood protocol - [docs/case-studies.md](docs/case-studies.md): reproducible dogfood case studies - [docs/internet-dogfood-sources.md](docs/internet-dogfood-sources.md): public prompt-response datasets for dogfood sourcing diff --git a/docs/commands.md b/docs/commands.md index 58a23ce..01c1c1d 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -10,7 +10,7 @@ output and examples after installing. | `redline init` | Write `redline.json`, runner files, and optional CI workflow. | `--runner`, `--copy-runner`, `--github-action`, `--force` | | `redline doctor` | Check config, suite, replay, reports, audit, and team workflow. | `--strict`, `--json` | | `redline watch` | Collect prompt-response observations or print snippets. | `--log`, `--stats`, `--snippet`, `--follow` | -| `redline import` | Normalize exported JSONL fields into redline `prompt`/`response` logs. | `--input-field`, `--output-field`, `--context-field`, `--metadata-field`, `--limit`, `--out` | +| `redline import` | Normalize exported JSONL fields into redline `prompt`/`response` logs, with best-effort redaction on by default. | `--input-field`, `--output-field`, `--context-field`, `--metadata-field`, `--limit`, `--out`, `--no-redact` | | `redline redact` | Check or write best-effort redacted logs. | `--check`, `--out`, `--json` | | `redline cluster` | Inspect behavior groups before suite generation. | `--max-cases`, `--json` | | `redline suite` | Generate a suite from baseline JSONL logs. | `--out`, `--max-cases`, `--all-cases`, `--owner` | diff --git a/docs/real-log-dogfood.md b/docs/real-log-dogfood.md new file mode 100644 index 0000000..5a4b385 --- /dev/null +++ b/docs/real-log-dogfood.md @@ -0,0 +1,112 @@ +# Real Log Dogfood + +Use this protocol when testing redline on real prompt/output logs from a team, +agent, support workflow, extraction job, or public dataset. The goal is to learn +whether redline catches useful regressions without leaking private data. + +## Rule Zero + +Do not send raw customer logs, secrets, API keys, or private prompts to another +person. Keep the raw export local. Share only sanitized reports or short +hand-redacted examples. + +## 1. Inspect the Export + +Open a few rows before importing: + +```bash +python -m json.tool < first-row.json +``` + +Find the field that contains the user prompt or instruction, and the field that +contains the model or agent response. Common examples: + +| Source shape | Input field | Output field | +| --- | --- | --- | +| redline JSONL | `prompt` | `response` | +| Dolly-style public data | `instruction` | `response` | +| OpenAI wrapper log | `request.messages` | `response.output_text` | +| Langfuse/Helicone-style export | `input` | `output` | +| Support traces | `ticket.text` | `assistant.reply` | + +## 2. Import With Redaction On + +`redline import` redacts common secrets and PII by default before writing the +normalized baseline file. + +```bash +redline import raw-export.jsonl \ + --input-field instruction \ + --output-field response \ + --metadata-field category \ + --out .redline/dogfood/baseline.jsonl +``` + +The command prints how many values were redacted. If the count is non-zero, +inspect the normalized output before generating a suite. + +Use `--no-redact` only for local-only test fixtures that have already been +reviewed. + +## 3. Generate a Suite + +```bash +redline suite .redline/dogfood/baseline.jsonl \ + --out .redline/dogfood/suite.json +``` + +Then inspect the selected cases: + +```bash +redline cases .redline/dogfood/suite.json +redline summary .redline/dogfood/suite.json +``` + +## 4. Compare a Candidate + +If you already have candidate outputs: + +```bash +redline diff .redline/dogfood/suite.json .redline/dogfood/candidate.jsonl \ + --compact \ + --out-json .redline/dogfood/diff.json \ + --out-html .redline/dogfood/diff.html \ + --fail-on none +``` + +If you have a runner configured: + +```bash +redline eval .redline/dogfood/suite.json \ + --prompt prompts/v2.txt \ + --compact \ + --fail-on none +``` + +## 5. Report Back + +For early dogfood, collect this information: + +- Source type: public dataset, app logs, trace export, SDK capture, or manual JSONL. +- Rows imported and redaction count. +- Suite cases generated and behavior groups covered. +- Blocking regressions found, if any. +- False positives that felt noisy. +- First command or error message that caused confusion. +- Whether the result would have changed a shipping decision. + +Do not share raw logs. Share the redline report only after reviewing it for +private content. + +## What Good Looks Like + +A useful dogfood run does not need to prove semantic correctness. It should show +whether redline caught concrete structural drift: + +- JSON keys disappeared. +- A table became prose. +- A safe task became a refusal. +- Required numbers, URLs, owners, or action items were dropped. +- A pinned requirement failed. + +Those are the regressions redline is designed to catch quickly and locally. diff --git a/redline/cli.py b/redline/cli.py index ce803f5..cbbb3e3 100644 --- a/redline/cli.py +++ b/redline/cli.py @@ -251,6 +251,8 @@ def build_parser() -> argparse.ArgumentParser: ) import_parser.add_argument("--limit", type=int, help="maximum records to import") import_parser.add_argument("--out", required=True, help="redline JSONL output path") + import_parser.add_argument("--no-redact", action="store_true", help="write raw values without import redaction") + import_parser.add_argument("--redaction-placeholder", default=DEFAULT_PLACEHOLDER, help="replacement text for import redaction") import_parser.add_argument("--json", action="store_true", help="print machine-readable JSON") import_parser.set_defaults(func=cmd_import) @@ -738,6 +740,8 @@ def cmd_import(args: argparse.Namespace) -> int: id_field=args.id_field, metadata_fields=args.metadata_field, limit=args.limit, + redact=not args.no_redact, + placeholder=args.redaction_placeholder, ) if args.json: print(json.dumps(report, indent=2, sort_keys=True)) @@ -749,6 +753,16 @@ def cmd_import(args: argparse.Namespace) -> int: print(f"Appended context: {report['context_field']}") if report["metadata_fields"]: print(f"Copied metadata: {', '.join(report['metadata_fields'])}") + if report["redacted"]: + print( + "Redaction: " + f"best-effort common secrets/PII scanned; {report['redactions']} value(s) redacted" + ) + patterns = report.get("redaction_patterns") + if isinstance(patterns, dict) and patterns: + print(f"Redaction hits: {', '.join(f'{key}={value}' for key, value in patterns.items())}") + else: + print("Redaction: disabled; review sensitive logs before sharing or committing") print(f"Wrote {Path(str(report['output']))}.") print() print("Next:") diff --git a/redline/import_logs.py b/redline/import_logs.py index 2baa500..bbaba31 100644 --- a/redline/import_logs.py +++ b/redline/import_logs.py @@ -5,6 +5,7 @@ from typing import Any from .io import iter_jsonl, write_jsonl +from .redact import DEFAULT_PLACEHOLDER, redact_object _MISSING = object() @@ -20,15 +21,20 @@ def import_jsonl_log( id_field: str | None = None, metadata_fields: list[str] | None = None, limit: int | None = None, + redact: bool = True, + placeholder: str = DEFAULT_PLACEHOLDER, ) -> dict[str, Any]: if limit is not None and limit < 1: raise ValueError("--limit must be 1 or greater") metadata_paths = metadata_fields or [] rows: list[dict[str, Any]] = [] + redaction_counts: dict[str, int] = {} source = Path(path) for line_number, row in iter_jsonl(source): if limit is not None and len(rows) >= limit: break + if redact: + row = redact_object(row, placeholder=placeholder, counts=redaction_counts) prompt = _required_field(source, line_number, row, input_field, "input") response = _required_field(source, line_number, row, output_field, "output") imported: dict[str, Any] = { @@ -50,6 +56,7 @@ def import_jsonl_log( if not rows: raise ValueError(f"{path} contains no JSONL records") write_jsonl(output, rows) + redactions = sum(redaction_counts.values()) return { "source": str(path), "output": str(output), @@ -59,6 +66,9 @@ def import_jsonl_log( "context_field": context_field or "", "id_field": id_field or "", "metadata_fields": metadata_paths, + "redacted": redact, + "redactions": redactions, + "redaction_patterns": dict(sorted(redaction_counts.items())), } diff --git a/redline/mcp.py b/redline/mcp.py index 93e89fb..ed70db8 100644 --- a/redline/mcp.py +++ b/redline/mcp.py @@ -541,6 +541,8 @@ def _tools() -> list[ToolSpec]: "description": "Source field paths copied into metadata.", }, "limit": _integer("Maximum records to import."), + "no_redact": _boolean("Write raw values without import redaction."), + "redaction_placeholder": _string("Replacement text for import redaction."), "json": _boolean("Print machine-readable JSON."), }, required=("path", "out"), @@ -890,6 +892,8 @@ def _build_import(arguments: dict[str, Any]) -> list[str]: _add_option(args, "--id-field", arguments.get("id_field")) _add_repeated_options(args, "--metadata-field", arguments.get("metadata_fields")) _add_option(args, "--limit", arguments.get("limit")) + _add_flag(args, "--no-redact", arguments.get("no_redact")) + _add_option(args, "--redaction-placeholder", arguments.get("redaction_placeholder")) _add_flag(args, "--json", arguments.get("json")) return args diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index 3d2097b..54763dd 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -2081,6 +2081,59 @@ def test_watch_reports_default_redaction_and_supports_raw_opt_out(self) -> None: finally: os.chdir(previous) + def test_import_reports_default_redaction_and_supports_raw_opt_out(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + previous = Path.cwd() + os.chdir(root) + try: + Path("downloaded.jsonl").write_text( + '{"instruction": "Email ada@example.com", "response": "ok"}\n', + encoding="utf-8", + ) + + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual( + main( + [ + "import", + "downloaded.jsonl", + "--input-field", + "instruction", + "--output-field", + "response", + "--out", + "baseline.jsonl", + ] + ), + 0, + ) + + self.assertIn("1 value(s) redacted", output.getvalue()) + self.assertNotIn("ada@example.com", Path("baseline.jsonl").read_text(encoding="utf-8")) + + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual( + main( + [ + "import", + "downloaded.jsonl", + "--input-field", + "instruction", + "--output-field", + "response", + "--out", + "raw.jsonl", + "--no-redact", + ] + ), + 0, + ) + self.assertIn("ada@example.com", Path("raw.jsonl").read_text(encoding="utf-8")) + finally: + os.chdir(previous) + def test_eval_uses_configured_replay_command(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_import_logs.py b/tests/test_import_logs.py index b9e08d2..9f6172e 100644 --- a/tests/test_import_logs.py +++ b/tests/test_import_logs.py @@ -68,6 +68,61 @@ def test_import_jsonl_log_limits_records(self) -> None: self.assertEqual(report["records"], 1) self.assertEqual(len(read_jsonl_records(output, "prompt", "response")), 1) + def test_import_jsonl_log_redacts_common_secrets_by_default(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "downloaded.jsonl" + output = root / "baseline.jsonl" + source.write_text( + json.dumps( + { + "instruction": "Email ada@example.com with the result.", + "response": "Used key sk-ant-" + ("a" * 24), + "metadata": {"api_key": "secret-value"}, + } + ) + + "\n", + encoding="utf-8", + ) + + report = import_jsonl_log( + source, + output=output, + input_field="instruction", + output_field="response", + metadata_fields=["metadata"], + ) + + self.assertTrue(report["redacted"]) + self.assertEqual(report["redactions"], 3) + records = read_jsonl_records(output, "prompt", "response") + self.assertNotIn("ada@example.com", records[0].prompt) + self.assertNotIn("sk-ant-", records[0].response) + self.assertEqual(records[0].raw["metadata"]["metadata"]["api_key"], "[REDACTED]") + + def test_import_jsonl_log_can_disable_redaction_for_local_only_logs(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "downloaded.jsonl" + output = root / "baseline.jsonl" + source.write_text( + '{"instruction": "Email ada@example.com", "response": "ok"}\n', + encoding="utf-8", + ) + + report = import_jsonl_log( + source, + output=output, + input_field="instruction", + output_field="response", + redact=False, + ) + + self.assertFalse(report["redacted"]) + self.assertEqual(report["redactions"], 0) + records = read_jsonl_records(output, "prompt", "response") + self.assertIn("ada@example.com", records[0].prompt) + def test_import_jsonl_log_reports_missing_fields(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 5e2c8af..385b2fa 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -77,6 +77,7 @@ def test_import_tool_normalizes_external_jsonl(self) -> None: self.assertFalse(result["isError"]) self.assertEqual(result["structuredContent"]["exit_code"], 0) self.assertEqual(result["structuredContent"]["json"]["records"], 1) + self.assertTrue(result["structuredContent"]["json"]["redacted"]) self.assertTrue(wrote_output) def test_eval_and_diff_tools_do_not_accept_dynamic_commands(self) -> None: diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 72c270c..eb1a454 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -238,6 +238,7 @@ def test_readme_marks_repo_only_script_commands(self) -> None: self.assertIn("zero runtime", readme) self.assertIn("docs/troubleshooting.md", readme) self.assertIn("docs/commands.md", readme) + self.assertIn("docs/real-log-dogfood.md", readme) self.assertIn("redline judges", readme) self.assertIn("feature-level", readme) self.assertIn("From a repo checkout, record the public demo", readme) @@ -284,6 +285,7 @@ def test_troubleshooting_and_command_reference_are_public_docs(self) -> None: self.assertIn("Neutral does not mean safe", troubleshooting) self.assertIn("| `redline eval` |", commands) self.assertIn("| `redline import` |", commands) + self.assertIn("--no-redact", commands) self.assertIn("| `redline dashboard` |", commands) self.assertIn("| `redline-mcp` |", commands) self.assertIn("Default fail-on statuses", commands) From 4383556d3bc44400b4c87a1c1dc2c4d16946e3e5 Mon Sep 17 00:00:00 2001 From: Gowtham Date: Sun, 31 May 2026 22:31:58 -0600 Subject: [PATCH 02/55] Clarify behavior grouping language --- README.md | 13 +++++++------ docs/ai-session-dogfood-prompts.jsonl | 2 +- docs/commands.md | 2 +- docs/launch.md | 4 ++-- docs/runners.md | 2 +- redline/benchmark.py | 4 ++-- redline/cli.py | 4 ++-- redline/clusters.py | 8 ++++---- redline/dashboard.py | 2 +- redline/doctor.py | 4 ++-- redline/history.py | 4 ++-- redline/mcp.py | 4 ++-- redline/summary.py | 22 +++++++++++----------- tests/test_clusters.py | 6 +++--- tests/test_dashboard.py | 2 +- tests/test_doctor.py | 2 +- tests/test_history.py | 2 +- tests/test_summary.py | 10 +++++----- 18 files changed, 49 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 7dba6bb..a031aa1 100644 --- a/README.md +++ b/README.md @@ -200,10 +200,11 @@ redline is deterministic and local-first by default. Optional judge commands are available for ambiguous `changed` cases, but redline does not call a cloud model unless you explicitly configure that command. -Suite generation groups logs by deterministic behavior signatures, not opaque -embedding clusters. It picks one representative per group first, then adds -high-variance edges and evenly spread prompt-diverse samples from large clusters -when the case budget allows. +Suite generation does not run statistical or embedding clustering by default. +It groups logs by deterministic behavior signatures, such as prompt intent, +response shape, length bucket, and JSON schema. It picks one representative per +group first, then adds high-variance edges and evenly spread prompt-diverse +samples from large groups when the case budget allows. ## Trust Boundary @@ -230,7 +231,7 @@ redline is built around the full prompt-regression loop: - `RedlineMiddleware`: capture bounded JSON FastAPI or ASGI request/response pairs locally, with optional skip diagnostics. - `redline redact --check`: scan logs for common secrets and PII, then write a scrubbed copy when needed. Redaction is best-effort pattern matching, not a privacy boundary; review sensitive logs before sharing. -- `redline cluster`: inspect behavior groups before suite generation. +- `redline cluster`: inspect deterministic behavior-signature groups before suite generation. - `redline suite`: generate a representative eval suite from baseline logs. - `redline prompts`: scan many prompt files and write or check a versionable prompt-to-suite manifest. Add `--check-suites` in CI when every prompt should already have a built and valid suite. @@ -427,7 +428,7 @@ redline summary redline-suite.json `doctor` shows whether the suite has explicit requirements or recorded judgments before you rely on structural checks in CI. -`summary` reports cluster/case coverage, owner coverage, accepted baseline +`summary` reports behavior-group/case coverage, owner coverage, accepted baseline history, approver coverage, and explicit guard coverage for cases with requirements or recorded judgments so teams can review suite readiness before CI. `dashboard` also shows audit checkpoint evidence when `.redline/audit-checkpoint.json` diff --git a/docs/ai-session-dogfood-prompts.jsonl b/docs/ai-session-dogfood-prompts.jsonl index e75ab61..ad0534b 100644 --- a/docs/ai-session-dogfood-prompts.jsonl +++ b/docs/ai-session-dogfood-prompts.jsonl @@ -2,7 +2,7 @@ {"case_id":"task_02","prompt":"Design a first-run experience for redline. A new user cloned the repo and has five minutes. What exact commands should they run, what should they see, and where are they most likely to get confused?"} {"case_id":"task_03","prompt":"Write a concise config reference for redline.json. Include fields for suite path, input/output fields, max cases, replay command, judge command, reports, logs, and fail-on behavior."} {"case_id":"task_04","prompt":"Create a runner adapter guide for a team using an HTTP API. Their endpoint accepts {\"prompt\":\"...\"} and returns {\"response\":\"...\"}. Show the script, environment variables, and redline command."} -{"case_id":"task_05","prompt":"Review this CLI output and tell me whether it is clear enough for a first-time user. Suggest exact wording improvements:\nGenerated 5 cases from 5 records.\nDetected 5 behavioral clusters.\nWrote redline-suite.json."} +{"case_id":"task_05","prompt":"Review this CLI output and tell me whether it is clear enough for a first-time user. Suggest exact wording improvements:\nGenerated 5 cases from 5 records.\nDetected 5 behavior-signature groups.\nWrote redline-suite.json."} {"case_id":"task_06","prompt":"I changed an LLM system prompt and now responses are shorter. Give me a realistic before/after prompt-output log with 5 JSONL rows where the shorter candidate silently loses important behavior."} {"case_id":"task_07","prompt":"Act as a code reviewer for a Python CLI eval tool. What are the top 10 bugs or product risks you would look for before calling it public alpha?"} {"case_id":"task_08","prompt":"Write a GitHub Actions workflow for redline. It should install the package, run doctor, validate the suite, run eval, upload reports, and fail on regressions."} diff --git a/docs/commands.md b/docs/commands.md index 01c1c1d..25d7e19 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -12,7 +12,7 @@ output and examples after installing. | `redline watch` | Collect prompt-response observations or print snippets. | `--log`, `--stats`, `--snippet`, `--follow` | | `redline import` | Normalize exported JSONL fields into redline `prompt`/`response` logs, with best-effort redaction on by default. | `--input-field`, `--output-field`, `--context-field`, `--metadata-field`, `--limit`, `--out`, `--no-redact` | | `redline redact` | Check or write best-effort redacted logs. | `--check`, `--out`, `--json` | -| `redline cluster` | Inspect behavior groups before suite generation. | `--max-cases`, `--json` | +| `redline cluster` | Inspect deterministic behavior-signature groups before suite generation. | `--max-cases`, `--json` | | `redline suite` | Generate a suite from baseline JSONL logs. | `--out`, `--max-cases`, `--all-cases`, `--owner` | | `redline suite add` | Pin a hand-picked edge case. | `--prompt`, `--response`, `--include`, `--exclude`, `--owner` | | `redline cases` | List generated suite case IDs and coverage. | `--json` | diff --git a/docs/launch.md b/docs/launch.md index b8da4dc..8b7bbd4 100644 --- a/docs/launch.md +++ b/docs/launch.md @@ -101,7 +101,7 @@ Short post: I built redline, a local-first tool that turns prompt-response logs into eval suites automatically. -You point it at existing JSONL logs, it clusters behavior, picks representative +You point it at existing JSONL logs, groups behavior by deterministic signatures, picks representative cases, and catches regressions when a prompt change drops structure, numbers, URLs, entities, refusals, tables, code blocks, or required fields. @@ -123,7 +123,7 @@ Longer post: Most eval tools start with "write test cases." redline starts from the logs you already have. -It watches or imports prompt-response JSONL, clusters observed behavior, builds +It watches or imports prompt-response JSONL, groups observed behavior by deterministic signatures, builds a representative suite, and compares new prompt runs against the accepted baseline. The default checks are deterministic and local: JSON validity, missing keys, empty outputs, refusals, URLs, numbers, entities, code blocks, tables, diff --git a/docs/runners.md b/docs/runners.md index cb30b24..a6187a2 100644 --- a/docs/runners.md +++ b/docs/runners.md @@ -247,7 +247,7 @@ python runners/braintrust_suite_export.py redline-suite.json \ ``` What it does: writes one JSONL row per redline case with `input`, `expected`, -and redline metadata such as case ID, cluster, owner, and source line. Import +and redline metadata such as case ID, behavior signature, owner, and source line. Import that JSONL into Braintrust when you want Braintrust to mirror the same dataset redline gates locally. diff --git a/redline/benchmark.py b/redline/benchmark.py index e479a93..1d7b9bd 100644 --- a/redline/benchmark.py +++ b/redline/benchmark.py @@ -186,7 +186,7 @@ def format_benchmark_report(report: dict[str, Any], *, command_name: str = "benc "Mode: static estimate; no replay commands are executed", f"{suite_label + ':':<22}{report['suite']}", f"Cases: {report['cases']}", - f"Behavioral clusters: {report['clusters']}", + f"Behavior groups: {report['clusters']}", f"Records seen: {report['records_seen']}", f"Workers: {report['workers']}", f"Timeout per case: {_seconds(report['timeout_seconds'])}", @@ -250,7 +250,7 @@ def format_benchmark_markdown(report: dict[str, Any], *, command_name: str = "be "| --- | --- |", f"| {'Prompt manifest' if report.get('is_prompt_manifest') else 'Suite'} | `{report['suite']}` |", f"| Cases | {report['cases']} |", - f"| Behavioral clusters | {report['clusters']} |", + f"| Behavior groups | {report['clusters']} |", f"| Workers | {report['workers']} |", f"| Timeout per case | {_seconds(report['timeout_seconds'])} |", f"| Worst-case eval budget | {_duration(report['worst_case_seconds'])} |", diff --git a/redline/cli.py b/redline/cli.py index cbbb3e3..26e7ff9 100644 --- a/redline/cli.py +++ b/redline/cli.py @@ -322,7 +322,7 @@ def build_parser() -> argparse.ArgumentParser: audit_parser.add_argument("--json", action="store_true", help="print machine-readable JSON") audit_parser.set_defaults(func=cmd_audit) - cluster_parser = subparsers.add_parser("cluster", help="analyze behavioral clusters in a log") + cluster_parser = subparsers.add_parser("cluster", help="inspect deterministic behavior-signature groups in a log") cluster_parser.add_argument("log", nargs="?", help="JSONL prompt-response log; defaults to watched log") cluster_parser.add_argument("--config", default=DEFAULT_CONFIG_PATH, help="config path to read") cluster_parser.add_argument("--input-field", help="JSONL input field") @@ -1026,7 +1026,7 @@ def cmd_suite(args: argparse.Namespace) -> int: print(f"Generated {summary['cases']} cases from {summary['records_seen']} records.") if summary.get("duplicate_prompt_response_pairs"): print(f"Skipped {summary['duplicate_prompt_response_pairs']} duplicate prompt-response pairs.") - print(f"Detected {summary['clusters']} behavioral clusters.") + print(f"Detected {summary['clusters']} behavior-signature groups.") if summary.get("selection") != "all": unique_pairs = int(summary.get("unique_prompt_response_pairs", summary["records_seen"])) cases = int(summary["cases"]) diff --git a/redline/clusters.py b/redline/clusters.py index e3caa31..8335481 100644 --- a/redline/clusters.py +++ b/redline/clusters.py @@ -46,11 +46,11 @@ def format_cluster_report(suite: dict[str, Any]) -> str: lines = [ "redline cluster", "", - f"Identified {report['clusters']} behavioral clusters from {report['unique_prompt_response_pairs']} unique pairs.", + f"Identified {report['clusters']} behavior-signature groups from {report['unique_prompt_response_pairs']} unique pairs.", f"Records seen: {report['records_seen']} Duplicate pairs: {report['duplicate_prompt_response_pairs']}", - f"High-risk clusters: {report['high_risk_clusters']}", - f"High-variance clusters: {report['high_variance_clusters']}", - f"Failure-pattern clusters: {report['failure_pattern_clusters']}", + f"High-risk groups: {report['high_risk_clusters']}", + f"High-variance groups: {report['high_variance_clusters']}", + f"Failure-pattern groups: {report['failure_pattern_clusters']}", f"Suggested eval suite: {report['suggested_cases']} representative cases.", f"Case coverage: {report['suggested_cases']}/{report['unique_prompt_response_pairs']} ({_percent(report['case_coverage'])})", "", diff --git a/redline/dashboard.py b/redline/dashboard.py index d9e673f..e5731fd 100644 --- a/redline/dashboard.py +++ b/redline/dashboard.py @@ -882,7 +882,7 @@ def _trend_cluster_diagnosis(value: Any) -> str: ) if not rows: return "" - return "

Cluster diagnosis

" + return "

Behavior group diagnosis

" def _trend_class(direction: str) -> str: diff --git a/redline/doctor.py b/redline/doctor.py index e9004a5..71fce81 100644 --- a/redline/doctor.py +++ b/redline/doctor.py @@ -235,7 +235,7 @@ def _coverage_check( non_ascii_records = _manifest_int(manifest_summary, "non_ascii_records") judge_configured = "judge" in config message += ( - f"; high-risk clusters={high_risk_clusters}; " + f"; high-risk groups={high_risk_clusters}; " f"requirements={requirements_count}; " f"judge={'yes' if judge_configured else 'no'}; " f"explicit guards={explicit_guard_cases}/{cases_count}" @@ -261,7 +261,7 @@ def _coverage_check( non_ascii_records = int(summary_report.get("non_ascii_records") or 0) judge_configured = "judge" in config message += ( - f"; high-risk clusters={high_risk_clusters}; " + f"; high-risk groups={high_risk_clusters}; " f"requirements={requirements_count}; " f"judge={'yes' if judge_configured else 'no'}; " f"explicit guards={explicit_guard_cases}/{cases_count}" diff --git a/redline/history.py b/redline/history.py index c3ff184..b2032dd 100644 --- a/redline/history.py +++ b/redline/history.py @@ -156,9 +156,9 @@ def format_markdown_history(entries: list[dict[str, Any]], *, limit: int | None if cluster_rows: lines.extend( [ - "## Cluster Diagnosis", + "## Behavior Group Diagnosis", "", - "| Cluster | Blocking Delta | Latest Blocking | Changed Delta |", + "| Behavior Group | Blocking Delta | Latest Blocking | Changed Delta |", "| --- | ---: | ---: | ---: |", ] ) diff --git a/redline/mcp.py b/redline/mcp.py index ed70db8..50d0190 100644 --- a/redline/mcp.py +++ b/redline/mcp.py @@ -418,7 +418,7 @@ def _build_suite_from_logs_prompt(arguments: dict[str, Any]) -> str: "Use this workflow:\n" "1. Call `redline_suite` for the baseline log and write the suite output.\n" "2. Call `redline_validate` on the generated suite.\n" - "3. Call `redline_summary` so I can see coverage, clusters, pinned cases, and next steps.\n" + "3. Call `redline_summary` so I can see coverage, behavior groups, pinned cases, and next steps.\n" "4. If coverage is low or cases look unclear, call `redline_cases` and `redline_case` to inspect representative cases before recommending pins.\n" "5. Call `redline_budget` so I can see expected CI runtime before enabling a gate.\n" "6. Explain what behavior redline can catch, what still needs human review, and the exact `redline suite add` command I can run for must-cover edge cases.\n" @@ -640,7 +640,7 @@ def _tools() -> list[ToolSpec]: ), ToolSpec( "redline_summary", - "Summarize suite or prompt-manifest provenance, coverage, clusters, owners, requirements, and next steps.", + "Summarize suite or prompt-manifest provenance, coverage, behavior groups, owners, requirements, and next steps.", _schema( { "suite_path": _string("Suite or prompt manifest JSON path. Defaults to config."), diff --git a/redline/summary.py b/redline/summary.py index e20774f..adbdfad 100644 --- a/redline/summary.py +++ b/redline/summary.py @@ -329,8 +329,8 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None f"Records seen: {summary['records_seen']}", f"Unique pairs: {summary['unique_prompt_response_pairs']}", f"Duplicate pairs: {summary['duplicate_prompt_response_pairs']}", - f"Behavioral clusters: {summary['clusters']}", - f"Cluster coverage: {summary['covered_clusters']}/{summary['clusters']} ({_percent(summary['cluster_coverage'])})", + f"Behavior groups: {summary['clusters']}", + f"Group coverage: {summary['covered_clusters']}/{summary['clusters']} ({_percent(summary['cluster_coverage'])})", f"Representative cases: {summary['cases']}", f"Case coverage: {summary['cases']}/{summary['unique_prompt_response_pairs']} ({_percent(summary['case_coverage'])})", f"Pinned cases: {summary['pinned_cases']}", @@ -340,10 +340,10 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None f"Approved baselines: {summary['approved_baselines']}/{summary['accepted_baselines']}", f"Explicit guard coverage: {summary['explicit_guard_cases']}/{summary['cases']} ({_percent(summary['explicit_guard_coverage'])})", f"Max cases: {summary['max_cases']}", - f"High-risk clusters: {summary['high_risk_clusters']}", - f"Medium-risk clusters: {summary['medium_risk_clusters']}", - f"High-variance clusters: {summary['high_variance_clusters']}", - f"Failure-pattern clusters: {summary['failure_pattern_clusters']:>2}", + f"High-risk groups: {summary['high_risk_clusters']}", + f"Medium-risk groups: {summary['medium_risk_clusters']}", + f"High-variance groups: {summary['high_variance_clusters']}", + f"Failure-pattern groups: {summary['failure_pattern_clusters']:>2}", f"Non-ASCII records: {summary['non_ascii_records']}", f"Cases with requirements: {summary['requirements']:>2}", "", @@ -365,7 +365,7 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None top_clusters = summary["top_clusters"] if top_clusters: - lines.append("Top clusters:") + lines.append("Top groups:") for cluster in top_clusters: marker = " high-variance" if cluster["high_variance"] else "" flags = cluster["failure_patterns"] @@ -398,8 +398,8 @@ def format_prompt_manifest_summary(report: dict[str, Any]) -> str: f"Invalid suites: {report['invalid_suite_count']}", f"Records seen: {report['records_seen']}", f"Unique pairs: {report['unique_prompt_response_pairs']}", - f"Behavioral clusters: {report['clusters']}", - f"Cluster coverage: {report['covered_clusters']}/{report['clusters']} ({_percent(report['cluster_coverage'])})", + f"Behavior groups: {report['clusters']}", + f"Group coverage: {report['covered_clusters']}/{report['clusters']} ({_percent(report['cluster_coverage'])})", f"Representative cases: {report['cases']}", f"Case coverage: {report['cases']}/{report['unique_prompt_response_pairs']} ({_percent(report['case_coverage'])})", f"Pinned cases: {report['pinned_cases']}", @@ -408,8 +408,8 @@ def format_prompt_manifest_summary(report: dict[str, Any]) -> str: f"Accepted baselines: {report['accepted_baselines']}", f"Approved baselines: {report['approved_baselines']}/{report['accepted_baselines']}", f"Explicit guard coverage: {report['explicit_guard_cases']}/{report['cases']} ({_percent(report['explicit_guard_coverage'])})", - f"High-risk clusters: {report['high_risk_clusters']}", - f"Medium-risk clusters: {report['medium_risk_clusters']}", + f"High-risk groups: {report['high_risk_clusters']}", + f"Medium-risk groups: {report['medium_risk_clusters']}", f"Non-ASCII records: {report['non_ascii_records']}", f"Cases with requirements: {report['requirements']:>2}", "", diff --git a/tests/test_clusters.py b/tests/test_clusters.py index b0e8ef7..5ae4f50 100644 --- a/tests/test_clusters.py +++ b/tests/test_clusters.py @@ -69,10 +69,10 @@ def test_format_cluster_report_is_readable(self) -> None: output = format_cluster_report(suite) self.assertIn("redline cluster", output) - self.assertIn("Identified 1 behavioral clusters from 1 unique pairs", output) + self.assertIn("Identified 1 behavior-signature groups from 1 unique pairs", output) self.assertIn("Records seen: 1 Duplicate pairs: 0", output) - self.assertIn("High-risk clusters: 0", output) - self.assertIn("Failure-pattern clusters: 0", output) + self.assertIn("High-risk groups: 0", output) + self.assertIn("Failure-pattern groups: 0", output) self.assertIn("Suggested eval suite: 1 representative cases.", output) self.assertIn("Case coverage: 1/1 (100.0%)", output) self.assertIn("RISK", output) diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index ad90aeb..5e7dfd5 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -326,7 +326,7 @@ def test_dashboard_trend_shows_cluster_diagnosis(self) -> None: html = format_dashboard_html(dashboard) - self.assertIn("

Cluster diagnosis

", html) + self.assertIn("

Behavior group diagnosis

", html) self.assertIn("structured JSON prompt -> JSON response (short)", html) self.assertIn("blocking +1", html) self.assertIn("changed -1", html) diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 8c2de55..237dac6 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -352,7 +352,7 @@ def test_doctor_calibrates_coverage_for_high_risk_suite_without_semantic_checks( coverage = next(check for check in report["checks"] if check["name"] == "coverage") self.assertEqual(coverage["status"], "ok") - self.assertIn("high-risk clusters=1", coverage["message"]) + self.assertIn("high-risk groups=1", coverage["message"]) self.assertIn("requirements=0; judge=no", coverage["message"]) self.assertIn("explicit guards=0/1", coverage["message"]) self.assertIn("add requirements or a judge", coverage["message"]) diff --git a/tests/test_history.py b/tests/test_history.py index fc364f4..7bc1755 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -211,7 +211,7 @@ def test_format_markdown_history_includes_cluster_diagnosis(self) -> None: ] ) - self.assertIn("## Cluster Diagnosis", text) + self.assertIn("## Behavior Group Diagnosis", text) self.assertIn("structured JSON prompt -> JSON response (short)", text) self.assertIn("| structured JSON prompt -> JSON response (short) | +1 | 1 | 0 |", text) diff --git a/tests/test_summary.py b/tests/test_summary.py index 530ea02..09c696e 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -92,7 +92,7 @@ def test_format_suite_summary_is_readable(self) -> None: self.assertIn("Records seen:", output) self.assertIn("Unique pairs:", output) self.assertIn("Duplicate pairs:", output) - self.assertIn("Cluster coverage:", output) + self.assertIn("Group coverage:", output) self.assertIn("Case coverage:", output) self.assertIn("Pinned cases:", output) self.assertIn("Owned cases:", output) @@ -100,9 +100,9 @@ def test_format_suite_summary_is_readable(self) -> None: self.assertIn("Accepted baselines:", output) self.assertIn("Approved baselines:", output) self.assertIn("Explicit guard coverage:", output) - self.assertIn("High-risk clusters:", output) - self.assertIn("Failure-pattern clusters:", output) - self.assertIn("Top clusters:", output) + self.assertIn("High-risk groups:", output) + self.assertIn("Failure-pattern groups:", output) + self.assertIn("Top groups:", output) self.assertIn("structured JSON prompt -> JSON response", output) def test_suite_summary_surfaces_non_ascii_calibration(self) -> None: @@ -273,7 +273,7 @@ def test_suite_summary_recommends_more_coverage_when_budget_is_tight(self) -> No self.assertEqual(summary["cluster_coverage"], 0.5) self.assertIn("Increase --max-cases", summary["next_steps"][0]) self.assertIn("redline suite add redline-suite.json --prompt-file", output) - self.assertIn("Cluster coverage: 1/2 (50.0%)", output) + self.assertIn("Group coverage: 1/2 (50.0%)", output) self.assertIn("Next:", output) From dda6a0b193f16eccd0e661a3b86b5773d1635b9d Mon Sep 17 00:00:00 2001 From: Gowtham Date: Sun, 31 May 2026 22:35:14 -0600 Subject: [PATCH 03/55] Add suite readiness scoring --- README.md | 7 ++-- docs/commands.md | 2 +- redline/summary.py | 92 +++++++++++++++++++++++++++++++++++++++++++ tests/test_summary.py | 9 +++++ 4 files changed, 106 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a031aa1..e9962e6 100644 --- a/README.md +++ b/README.md @@ -428,9 +428,10 @@ redline summary redline-suite.json `doctor` shows whether the suite has explicit requirements or recorded judgments before you rely on structural checks in CI. -`summary` reports behavior-group/case coverage, owner coverage, accepted baseline -history, approver coverage, and explicit guard coverage for cases with -requirements or recorded judgments so teams can review suite readiness before CI. +`summary` reports a suite readiness score, behavior-group/case coverage, owner +coverage, accepted baseline history, approver coverage, and explicit guard +coverage for cases with requirements or recorded judgments so teams can review +suite readiness before CI. `dashboard` also shows audit checkpoint evidence when `.redline/audit-checkpoint.json` is present. diff --git a/docs/commands.md b/docs/commands.md index 25d7e19..c4cf8eb 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -19,7 +19,7 @@ output and examples after installing. | `redline case` | Show one full suite case. | `--json` | | `redline require` | Add deterministic include/exclude requirements. | `--include`, `--exclude`, `--owner` | | `redline prompts` | Build or check a prompt-to-suite manifest. | `--suite-dir`, `--out`, `--check`, `--check-suites` | -| `redline summary` | Summarize suite or manifest readiness. | `--json` | +| `redline summary` | Summarize suite or manifest readiness, including suite score and coverage gaps. | `--json` | | `redline validate` | Validate suite or manifest structure and freshness. | `--strict`, `--json` | | `redline budget` | Estimate CI runtime without replaying prompts. | `--workers`, `--timeout`, `--max-seconds`, `--measure-local` | | `redline benchmark` | Compatibility alias for `redline budget`. | Same as `budget` | diff --git a/redline/summary.py b/redline/summary.py index adbdfad..bb00e92 100644 --- a/redline/summary.py +++ b/redline/summary.py @@ -154,6 +154,7 @@ def suite_summary(suite: dict[str, Any]) -> dict[str, Any]: "requirements": requirements_count, "top_clusters": top_clusters, } + result["suite_readiness"] = _suite_readiness(result) result["next_steps"] = _summary_next_steps(result) return result @@ -333,6 +334,7 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None f"Group coverage: {summary['covered_clusters']}/{summary['clusters']} ({_percent(summary['cluster_coverage'])})", f"Representative cases: {summary['cases']}", f"Case coverage: {summary['cases']}/{summary['unique_prompt_response_pairs']} ({_percent(summary['case_coverage'])})", + f"Suite readiness: {_format_readiness(summary['suite_readiness'])}", f"Pinned cases: {summary['pinned_cases']}", f"Owned cases: {summary['owned_cases']}/{summary['cases']}", f"Owner rule coverage: {summary['owner_rule_cases']}/{summary['owned_cases']} ({_percent(summary['owner_rule_coverage'])})", @@ -374,6 +376,15 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None lines.append(f" {cluster['size']:>4} {cluster['behavior']}{marker}") lines.append("") + readiness = summary.get("suite_readiness") + if isinstance(readiness, dict): + reasons = readiness.get("reasons") + if isinstance(reasons, list) and reasons: + lines.append("Readiness signals:") + for reason in reasons: + lines.append(f"- {reason}") + lines.append("") + next_steps = summary["next_steps"] if next_steps: lines.append("Next:") @@ -484,6 +495,87 @@ def _summary_next_steps(summary: dict[str, Any], *, suite_path: str | None = Non return steps +def _suite_readiness(summary: dict[str, Any]) -> dict[str, Any]: + cases = int(summary.get("cases") or 0) + if cases == 0: + return { + "score": 0, + "label": "empty", + "reasons": ["no suite cases are available yet"], + } + + score = 0.0 + score += 35.0 * float(summary.get("cluster_coverage") or 0.0) + score += 20.0 * float(summary.get("case_coverage") or 0.0) + score += 20.0 * float(summary.get("explicit_guard_coverage") or 0.0) + owner_coverage = _ratio(int(summary.get("owned_cases") or 0), cases) or 0.0 + score += 10.0 * owner_coverage + if int(summary.get("high_risk_clusters") or 0) == 0 or int(summary.get("requirements") or 0): + score += 10.0 + if int(summary.get("non_ascii_records") or 0) == 0: + score += 5.0 + + rounded = int(round(min(100.0, max(0.0, score)))) + if rounded >= 80: + label = "strong" + elif rounded >= 55: + label = "usable" + else: + label = "needs_work" + + return { + "score": rounded, + "label": label, + "reasons": _readiness_reasons(summary, owner_coverage=owner_coverage), + } + + +def _readiness_reasons(summary: dict[str, Any], *, owner_coverage: float) -> list[str]: + reasons = [] + cluster_coverage = float(summary.get("cluster_coverage") or 0.0) + case_coverage = float(summary.get("case_coverage") or 0.0) + explicit_guard_coverage = float(summary.get("explicit_guard_coverage") or 0.0) + + if cluster_coverage >= 1.0: + reasons.append("all detected behavior-signature groups have at least one selected case") + else: + reasons.append("some behavior-signature groups are not represented in the suite") + + if case_coverage >= 0.8: + reasons.append("case budget covers most unique prompt-response pairs") + else: + reasons.append("case budget is sampling a small share of unique prompt-response pairs") + + if explicit_guard_coverage >= 0.5: + reasons.append("many cases have requirements or recorded judgments") + elif explicit_guard_coverage > 0: + reasons.append("some cases have requirements or recorded judgments") + else: + reasons.append("no cases have explicit requirements or recorded judgments yet") + + if owner_coverage >= 1.0: + reasons.append("all cases have owners") + elif owner_coverage > 0: + reasons.append("some cases still need owners") + else: + reasons.append("no cases have owners yet") + + if int(summary.get("high_risk_clusters") or 0) and int(summary.get("requirements") or 0) == 0: + reasons.append("high-risk groups need explicit requirements or a judge before CI gating") + if int(summary.get("non_ascii_records") or 0): + reasons.append("non-ASCII records need extra review because entity/refusal heuristics are English-oriented") + + return reasons + + +def _format_readiness(value: object) -> str: + if not isinstance(value, dict): + return "" + score = int(value.get("score") or 0) + label = str(value.get("label") or "unknown").replace("_", " ") + return f"{score}/100 ({label})" + + def _manifest_summary_status(summary: dict[str, Any]) -> str: if int(summary["invalid_suite_count"]): return "invalid" diff --git a/tests/test_summary.py b/tests/test_summary.py index 09c696e..bc2536a 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -54,6 +54,12 @@ def test_suite_summary_counts_cases_clusters_and_judgments(self) -> None: self.assertEqual(summary["judgment_cases"], 1) self.assertEqual(summary["explicit_guard_cases"], 1) self.assertEqual(summary["explicit_guard_coverage"], 0.5) + self.assertEqual(summary["suite_readiness"]["score"], 80) + self.assertEqual(summary["suite_readiness"]["label"], "strong") + self.assertIn( + "many cases have requirements or recorded judgments", + summary["suite_readiness"]["reasons"], + ) self.assertEqual(summary["judgments"], {"expected": 1}) self.assertEqual(summary["requirements"], 1) self.assertEqual(summary["failure_pattern_clusters"], 0) @@ -94,6 +100,7 @@ def test_format_suite_summary_is_readable(self) -> None: self.assertIn("Duplicate pairs:", output) self.assertIn("Group coverage:", output) self.assertIn("Case coverage:", output) + self.assertIn("Suite readiness:", output) self.assertIn("Pinned cases:", output) self.assertIn("Owned cases:", output) self.assertIn("Owner rule coverage:", output) @@ -103,6 +110,7 @@ def test_format_suite_summary_is_readable(self) -> None: self.assertIn("High-risk groups:", output) self.assertIn("Failure-pattern groups:", output) self.assertIn("Top groups:", output) + self.assertIn("Readiness signals:", output) self.assertIn("structured JSON prompt -> JSON response", output) def test_suite_summary_surfaces_non_ascii_calibration(self) -> None: @@ -271,6 +279,7 @@ def test_suite_summary_recommends_more_coverage_when_budget_is_tight(self) -> No self.assertEqual(summary["clusters"], 2) self.assertEqual(summary["case_coverage"], 0.5) self.assertEqual(summary["cluster_coverage"], 0.5) + self.assertEqual(summary["suite_readiness"]["label"], "needs_work") self.assertIn("Increase --max-cases", summary["next_steps"][0]) self.assertIn("redline suite add redline-suite.json --prompt-file", output) self.assertIn("Group coverage: 1/2 (50.0%)", output) From 12062cb20cc32fe29c0b4fd5e3b761f145d351ab Mon Sep 17 00:00:00 2001 From: Gowtham Date: Sun, 31 May 2026 22:37:08 -0600 Subject: [PATCH 04/55] Document redline methodology --- README.md | 3 ++ docs/methodology.md | 104 ++++++++++++++++++++++++++++++++++++++++ tests/test_packaging.py | 10 ++++ 3 files changed, 117 insertions(+) create mode 100644 docs/methodology.md diff --git a/README.md b/README.md index e9962e6..e824620 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,8 @@ redline is deterministic and local-first by default. Optional judge commands are available for ambiguous `changed` cases, but redline does not call a cloud model unless you explicitly configure that command. +Methodology details live in [docs/methodology.md](docs/methodology.md). + Suite generation does not run statistical or embedding clustering by default. It groups logs by deterministic behavior signatures, such as prompt intent, response shape, length bucket, and JSON schema. It picks one representative per @@ -481,6 +483,7 @@ bash scripts/release_check.sh - [docs/release.md](docs/release.md): package, tag, PyPI, and MCP Registry release flow - [docs/launch.md](docs/launch.md): public alpha launch plan - [docs/troubleshooting.md](docs/troubleshooting.md): first-run and CI failure recovery +- [docs/methodology.md](docs/methodology.md): behavior grouping, case selection, scoring, and trust boundaries - [docs/commands.md](docs/commands.md): compact CLI command reference - [docs/real-log-dogfood.md](docs/real-log-dogfood.md): redaction-first real-log test protocol - [docs/dogfood.md](docs/dogfood.md): first-user dogfood protocol diff --git a/docs/methodology.md b/docs/methodology.md new file mode 100644 index 0000000..262b05f --- /dev/null +++ b/docs/methodology.md @@ -0,0 +1,104 @@ +# Methodology + +redline is a deterministic regression detector for prompt-response systems. It +does not claim that a green run proves semantic safety. It turns logs into a +reviewable suite, checks high-signal behavioral changes, and leaves factual, +tone, policy, hallucination, and subtle reasoning judgment to explicit +requirements, optional judges, or humans. + +## Input Model + +redline expects prompt-response observations as JSONL. The canonical fields are +`prompt` and `response`, but `redline import` can normalize nested exports from +application logs, observability tools, or model gateways. + +Import is redaction-first by default. The importer redacts common secrets and +PII before extracting fields so copied metadata is treated with the same local +privacy boundary as prompt and response text. Use `--no-redact` only for local +fixtures you have already inspected. + +## Behavior Grouping + +redline does not run statistical, embedding, or semantic clustering by default. +It groups observations by deterministic behavior signatures: + +- prompt intent, such as classify, summarize, transform, generate, or answer +- response shape, such as JSON, table, code block, bullet list, numbered list, + refusal, empty output, or prose +- response length bucket +- JSON schema shape when the response is valid JSON + +The CLI command is named `redline cluster` for compatibility, but the public +model is behavior-signature grouping. These groups are explainable and stable; +they are not a claim that two prompts are semantically equivalent. + +## Case Selection + +Suite generation chooses cases in this order: + +1. One representative from each behavior group. +2. High-risk groups with obvious failure patterns, such as invalid JSON, + malformed tables, empty outputs, or refusals. +3. High-variance edges where same-shape outputs vary substantially in length. +4. Prompt-diverse samples from large groups when the case budget allows. +5. User-pinned cases from `redline suite add`. + +This strategy optimizes for useful regression coverage from existing logs, not +perfect coverage of every product risk. If a scenario matters, pin it. + +## Regression Signals + +The deterministic diff engine checks for high-signal changes: + +- lost JSON validity or missing JSON keys +- lost table, list, or code-block structure +- missing numbers, URLs, ticket IDs, and likely entities +- empty outputs +- newly refused safe-looking tasks +- allow/deny polarity flips +- large same-shape content drift +- explicit include/exclude requirements + +These checks are intentionally conservative. They catch common prompt +regressions quickly in CI, but they do not replace domain review. + +## Suite Readiness Score + +`redline summary` reports a suite readiness score from 0 to 100. The score is a +suite-health signal, not a model-quality score. It combines: + +- behavior-group coverage +- unique prompt-response pair coverage +- explicit requirements or recorded judgments +- owner coverage +- high-risk group calibration +- non-English calibration warnings + +Use the score to decide what to improve before CI gating. A high score means the +suite is better prepared for review and automation. It does not mean the +candidate prompt is safe to ship. + +## Judges + +Optional judges are for ambiguous `changed` cases and semantic risks that +deterministic checks cannot prove. A judge can be a local script, a local model, +or a cloud LLM command you control. redline routes only the configured review +surface and records judge output as evidence. + +Default redline behavior is local and deterministic. No cloud judge is called +unless you configure one. + +## Calibration + +Use redline as a safety loop: + +1. Import or watch real logs. +2. Generate a suite. +3. Inspect `redline summary` and `redline cases`. +4. Pin must-cover edge cases. +5. Add requirements for details that must not disappear. +6. Run `redline eval` or `redline diff` before prompt changes ship. +7. Mark expected changes and accept new baselines after review. + +The strongest evidence for redline is not a score. It is a concrete report that +says which production behavior changed and why it matters. diff --git a/tests/test_packaging.py b/tests/test_packaging.py index eb1a454..6b303cf 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -103,6 +103,16 @@ def test_demo_gif_script_records_or_writes_transcript(self) -> None: self.assertIn("redline-demo-transcript.txt", text) self.assertIn("redline demo --public --compact", text) + def test_methodology_doc_calibrates_behavior_grouping_and_score(self) -> None: + text = Path("docs/methodology.md").read_text(encoding="utf-8") + readme = Path("README.md").read_text(encoding="utf-8") + + self.assertIn("does not run statistical, embedding, or semantic clustering", text) + self.assertIn("behavior-signature grouping", text) + self.assertIn("suite readiness score", text) + self.assertIn("not a model-quality score", text) + self.assertIn("docs/methodology.md", readme) + def test_release_check_builds_and_smokes_installed_wheel(self) -> None: script = Path("scripts/release_check.sh").read_text(encoding="utf-8") From bfd4c41620998b7b9581b234c35af198fa32e44d Mon Sep 17 00:00:00 2001 From: Gowtham Date: Sun, 31 May 2026 22:39:32 -0600 Subject: [PATCH 05/55] Explain why report findings matter --- redline/reports.py | 43 ++++++++++++++++++++++++++++++++++++++++++- tests/test_reports.py | 6 ++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/redline/reports.py b/redline/reports.py index 6cb1b48..27042b7 100644 --- a/redline/reports.py +++ b/redline/reports.py @@ -132,6 +132,10 @@ def format_markdown_report(result: dict[str, Any], *, title: str = "redline diff if metadata: lines.extend(metadata) lines.append("") + impact = _why_this_matters(item) + if impact: + lines.append(f"Why this matters: {impact}") + lines.append("") for reason in item["reasons"]: lines.append(f"- {reason}") lines.append("") @@ -482,6 +486,35 @@ def _metadata_lines(item: dict[str, Any]) -> list[str]: return lines +def _why_this_matters(item: dict[str, Any]) -> str: + status = str(item.get("status") or "").lower() + reasons = item.get("reasons") + text = " ".join(str(reason).lower() for reason in reasons) if isinstance(reasons, list) else "" + if status == "missing": + return "Replay did not produce a candidate output, so this baseline behavior is untested." + if "valid json" in text or "json keys" in text: + return "Downstream code may fail if consumers expect parseable JSON or required fields." + if "markdown table" in text or "table structure" in text: + return "A consumer expecting rows and columns may receive unstructured prose instead." + if "code block" in text: + return "A developer or tool expecting copyable code may lose the executable structure." + if "bullet list" in text or "numbered list" in text: + return "A workflow expecting ordered or scannable steps may become harder to review." + if "missing numbers" in text or "missing urls" in text or "missing entities" in text: + return "Concrete details used for decisions, routing, or compliance may have been dropped." + if "newly refuses" in text or "refusal" in text: + return "A previously supported safe workflow may now be blocked." + if "became empty" in text: + return "The user may receive no usable answer." + if "content changed substantially" in text or "much shorter" in text: + return "Meaning may have changed enough to require human review before acceptance." + if status == "regression": + return "This case changed in a way redline treats as blocking before shipping." + if status == "changed": + return "Review whether this behavioral change is intentional before accepting it." + return "" + + def _result_warnings(result: dict[str, Any]) -> list[str]: warnings = result.get("warnings") if not isinstance(warnings, list): @@ -1054,7 +1087,12 @@ def _annotation_message(item: dict[str, Any]) -> str: .changed { background: var(--changed); } .improved { background: var(--improved); } .accepted, .ignored, .neutral { background: var(--neutral); } -.meta, .prompt { color: var(--muted); font-size: 13px; } +.meta, .prompt, .impact { color: var(--muted); font-size: 13px; } +.impact { + border-left: 3px solid var(--changed); + margin: 10px 0; + padding-left: 10px; +} .reasons { margin: 10px 0 14px 20px; padding: 0; } .responses { display: grid; @@ -1344,6 +1382,9 @@ def _html_case(item: dict[str, Any]) -> str: lines.append(f'
{_h(metadata)}
') if prompt: lines.append(f'

Prompt: {_h(prompt)}

') + impact = _why_this_matters(item) + if impact: + lines.append(f'

Why this matters: {_h(impact)}

') if isinstance(reasons, list) and reasons: lines.append('