diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5d3b5c0..8bbbec6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -21,10 +21,10 @@ jobs:
python-version: ["3.10", "3.11", "3.12", "3.13"]
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v7
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml
index 53c7fa8..e4eff07 100644
--- a/.github/workflows/pages.yml
+++ b/.github/workflows/pages.yml
@@ -16,7 +16,7 @@ permissions:
concurrency:
group: pages
- cancel-in-progress: false
+ cancel-in-progress: true
jobs:
deploy:
@@ -26,13 +26,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v7
- name: Configure Pages
- uses: actions/configure-pages@v5
+ uses: actions/configure-pages@v6
- name: Upload static site
- uses: actions/upload-pages-artifact@v3
+ uses: actions/upload-pages-artifact@v5
with:
path: site
- name: Deploy to GitHub Pages
id: deployment
- uses: actions/deploy-pages@v4
+ uses: actions/deploy-pages@v5
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 3bf2b0f..fe72576 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -15,10 +15,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v4
+ uses: actions/checkout@v7
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v6
with:
python-version: "3.11"
cache: "pip"
@@ -30,14 +30,14 @@ jobs:
run: bash scripts/certify_release.sh "$RUNNER_TEMP/redline-certify"
- name: Upload release distributions
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
name: redline-release-distributions
path: ${{ runner.temp }}/redline-certify/dist/*
if-no-files-found: error
- name: Upload certification summary
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
name: redline-release-certification
path: ${{ runner.temp }}/redline-certify/certification.txt
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e47bec0..76c3724 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,75 @@
# Changelog
+## 0.3.0
+
+redline 0.3.0 turns the alpha from a CLI-first regression checker into a
+guided local product loop: import logs, generate a suite, run a quick check,
+review cases in the app, and carry the result into CI or an MCP client.
+
+Highlights:
+
+- Add a guided local app workflow for first-run demos, report review, suite
+ health, benchmark evidence, history, and next-step commands.
+- Add `redline quick-check` and `redline status` so new users can get from two
+ logs to a reviewed result without knowing the full command graph.
+- Expand log import with field detection, auto-mapping, previews, presets, and
+ redaction-first docs for real team exports.
+- Add suite readiness, methodology, coverage, calibration, and per-case impact
+ signals so users can understand what redline selected and why it matters.
+- Expand MCP coverage so AI coding assistants can run quick checks, imports,
+ status review, and dashboard rendering with the same trust boundary as the
+ CLI.
+- Refresh the README, GitHub Pages site, screenshots, and case studies around
+ reproducible product proof, including a 100-row Databricks Dolly dogfood run.
+
+### Added
+
+- Add `redline app --demo` for a one-command local product demo with generated
+ reports and review workflow.
+- Add `redline quick-check` for temporary suite generation, diffing, report
+ output, and optional app opening from two local logs.
+- Add `redline status` to summarize config, suites, reports, history, audit
+ evidence, and the next command a user should run.
+- Add import field detection, `--auto-map`, preview output, richer diagnostics,
+ and source guides for public datasets and team log exports.
+- Add suite readiness scoring and explicit messaging that readiness measures
+ suite health, not model quality or candidate safety.
+- Add report methodology and suite coverage metadata to JSON, Markdown, HTML,
+ dashboard, and summary surfaces.
+- Add calibration examples and profile documentation for strict versus review
+ behavior.
+- Add per-case impact guidance in reports and dashboard review queues.
+- Add MCP tools and prompts for quick-check, status, auto import, and safer
+ review workflows inside AI coding assistants.
+- Add a public internet dogfood case study using a local 100-row Databricks
+ Dolly sample, with raw third-party rows kept out of git.
+
+### Changed
+
+- Revamp the local dashboard into an app-style review surface with report
+ cards, ship readiness, case review, benchmark evidence, and history context.
+- Refresh GitHub Pages and README positioning around product proof instead of
+ only command reference.
+- Update proof screenshots from real local app and HTML report artifacts.
+- Position deterministic structural checks as a merge gate while keeping the
+ trust boundary visible in CLI, docs, and reports.
+- Improve CLI next-step output by quoting paths and commands that users can
+ copy safely.
+- Surface HTML reports and app-opening commands after first-run and eval flows.
+- Improve summary and dashboard wording for stochastic suites, English-centric
+ heuristics, and semantic review gaps.
+
+### Fixed
+
+- Ignore dashboard sidecar JSON artifacts such as Slack payloads and schemas so
+ dashboard/app report counts stay clean.
+- Validate suite schema compatibility more explicitly before users rely on a
+ generated suite.
+- Fix README and PyPI product-surface rendering by removing the fragile top SVG
+ README hero and using current raw asset URLs for proof images.
+- Quote printed app, demo, status, and CLI next-step commands to avoid broken
+ copy/paste paths.
+
## 0.2.1
redline 0.2.1 is a packaging polish patch for PyPI rendering.
diff --git a/README.md b/README.md
index 220f766..dae77d6 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,6 @@
-
-
-
-
-
-
-
+
redline
Catch prompt regressions before they ship.
@@ -44,26 +38,50 @@ Install from PyPI:
python -m pip install redline-ai
```
-Run the public proof:
+Run the guided local app with the public proof loaded:
+
+```bash
+redline app --demo
+```
+
+This generates the public demo reports, opens the local product app, and shows
+the full import -> suite -> eval -> review workflow. The demo catches ten
+synthetic regressions without API keys, private logs, a cloud account, or an LLM
+judge.
+
+Prefer terminal output first:
```bash
redline demo --public --compact
```
-The demo catches ten synthetic regressions without API keys, private logs, a
-cloud account, or an LLM judge. It writes JSON, Markdown, and self-contained
-HTML reports under `.redline/demo`.
+The demo writes JSON, Markdown, and self-contained HTML reports under
+`.redline/demo`.
+
+Ask redline what to do next:
+
+```bash
+redline status --reports-dir .redline/demo/reports
+```
+
+`status` reads local config, suites, reports, history, and audit evidence, then
+prints the next command instead of leaving you to infer the workflow.
-Open the local demo report index:
+Open the guided local product app on existing reports:
```bash
-redline dashboard --reports-dir .redline/demo/reports --open
+redline app --reports-dir .redline/demo/reports
```
-On headless CI or remote shells, skip `--open` and use the printed HTML path:
+The app is a local, copy-command workflow: import logs, generate suites, run
+evals, review regressions, record history, and export CI/MCP setup without the
+browser executing shell commands.
+
+On headless CI or remote shells, skip browser opening and use the printed HTML
+path:
```bash
-redline dashboard --reports-dir .redline/demo/reports --out .redline/dashboard.html
+redline app --reports-dir .redline/demo/reports --no-open --out .redline/app.html
```
@@ -71,7 +89,7 @@ redline dashboard --reports-dir .redline/demo/reports --out .redline/dashboard.h
- `redline: command not found`: run `python -m pip install redline-ai`, then
confirm `python -m pip show redline-ai`.
-- Dashboard did not open: use `--out .redline/dashboard.html` and open or
+- App did not open: use `--no-open --out .redline/app.html` and open or
upload that file from your environment.
- Suite not found: run `redline suite logs/baseline.jsonl --out redline-suite.json`.
- Validation failed: run `redline validate redline-suite.json --strict` and fix
@@ -82,15 +100,24 @@ redline dashboard --reports-dir .redline/demo/reports --out .redline/dashboard.h
Full guide: [docs/troubleshooting.md](docs/troubleshooting.md).
-
+
## Product Proof
-These are real local artifacts generated by `redline demo --public --compact`.
+redline has two proof paths: a fast first-run demo and a larger public-data
+dogfood run.
+
+| Proof | Command or data | Result |
+| --- | --- | --- |
+| First-run demo | `redline demo --public --compact` | 10 synthetic regressions caught locally with no API keys. |
+| Internet dogfood | 100 prompt-response rows sampled from [Databricks Dolly 15k](https://huggingface.co/datasets/databricks/databricks-dolly-15k) | 51 regressions, 27 changed cases, 22 neutral controls, and 0 dashboard warnings. |
+| Release gate | tests, lint, type check, action smoke, and release build | Package, CI, report, dashboard, and MCP paths are validated before publish. |
+
+These screenshots are local artifacts from the 100-row internet dogfood run.
| Dashboard | HTML report |
| --- | --- |
-|  |  |
+|  |  |
## What Is redline?
@@ -111,6 +138,17 @@ small.
redline gives you three primitives that cover the prompt-regression loop:
+For a first pass on two local logs, use one command:
+
+```bash
+redline quick-check logs/baseline.jsonl logs/candidate.jsonl --open
+```
+
+It generates a temporary suite, writes JSON/Markdown/HTML reports plus a guided
+local app under `.redline/quick-check`, opens the focused HTML report, and
+prints the concrete behavioral diff. Use `--open-app` when you want the guided
+review workflow to open instead of only the focused report.
+
### 1. Logs
Start with prompt-response data you already have. Import JSONL, convert exports
@@ -118,11 +156,23 @@ from tools like Langfuse or Helicone, capture OpenAI/Anthropic SDK calls, or add
bounded FastAPI/ASGI middleware.
```bash
+redline import downloaded.jsonl --detect
+redline import downloaded.jsonl --auto-map --preview 3
+redline import downloaded.jsonl --auto-map --out logs/baseline.jsonl
+redline import downloaded.jsonl --input-field instruction --output-field response --preview 3
redline import downloaded.jsonl --input-field instruction --output-field response --out logs/baseline.jsonl
+redline import langfuse-export.jsonl --preset langfuse --out logs/baseline.jsonl
redline suite logs/baseline.jsonl --out redline-suite.json
redline cases redline-suite.json
```
+Use `--detect` when you do not know the field names. Use `--preview` when the
+export is new to you; it shows mapped, redacted sample rows without writing a
+baseline file.
+
+Suite generation prints a readiness score and improvement suggestions. That
+score measures suite health, not model quality or candidate safety.
+
### 2. Suite
redline groups behavior into deterministic signatures and selects
@@ -200,10 +250,21 @@ redline is deterministic and local-first by default. Optional judge commands are
available for ambiguous `changed` cases, but redline does not call a cloud model
unless you explicitly configure that command.
-Suite generation groups logs by deterministic behavior signatures, not opaque
-embedding clusters. It picks one representative per group first, then adds
-high-variance edges and evenly spread prompt-diverse samples from large clusters
-when the case budget allows.
+That is the point. redline is designed to be the fast merge-blocking gate for
+regressions that break production systems: invalid JSON, missing required
+fields, lost tables, empty answers, dropped URLs, changed refusal behavior, and
+explicit requirement failures. LLM judges are useful for semantic review, but
+they are slower, cost money, and can be flaky in CI. redline keeps the default
+gate deterministic, reproducible, and cheap, then lets you add judges only where
+the structural signal is not enough.
+
+Methodology details live in [docs/methodology.md](docs/methodology.md).
+
+Suite generation does not run statistical or embedding clustering by default.
+It groups logs by deterministic behavior signatures, such as prompt intent,
+response shape, length bucket, and JSON schema. It picks one representative per
+group first, then adds high-variance edges and evenly spread prompt-diverse
+samples from large groups when the case budget allows.
## Trust Boundary
@@ -224,10 +285,13 @@ redline is built around the full prompt-regression loop:
- `redline watch`: collect prompt-response observations from logs, Python
functions, OpenAI/Anthropic-compatible SDK calls, or ASGI apps, with
best-effort common secrets and PII redacted before write by default.
+- `redline import`: normalize exported team logs into redline JSONL, with the
+ same best-effort redaction enabled by default. Use `--no-redact` only for
+ reviewed local-only logs.
- `RedlineMiddleware`: capture bounded JSON FastAPI or ASGI request/response pairs locally, with optional skip diagnostics.
- `redline redact --check`: scan logs for common secrets and PII, then write a scrubbed copy when needed.
Redaction is best-effort pattern matching, not a privacy boundary; review sensitive logs before sharing.
-- `redline cluster`: inspect behavior groups before suite generation.
+- `redline cluster`: inspect deterministic behavior-signature groups before suite generation.
- `redline suite`: generate a representative eval suite from baseline logs.
- `redline prompts`: scan many prompt files and write or check a versionable prompt-to-suite manifest.
Add `--check-suites` in CI when every prompt should already have a built and valid suite.
@@ -247,11 +311,16 @@ redline is built around the full prompt-regression loop:
`--out-checkpoint .redline/audit-checkpoint.json` to persist that evidence,
then `--checkpoint .redline/audit-checkpoint.json` to verify against it later.
- `redline sbom`: write CycloneDX SBOM release evidence for security review.
+- `redline app`: open the guided local product surface for importing logs,
+ generating suites, reviewing regressions, recording history, and wiring CI/MCP.
+- `redline status`: show project readiness and the next command from local
+ evidence, including the guided app command, first review case, its reason, and
+ why it matters.
- `redline history`, `redline compare`, and `redline dashboard`: track quality
- over time and inspect reports locally. The dashboard surfaces feature-level
- rollups, prompt-level eval rows, benchmark evidence, and a latest-report
- review queue when reports come from a prompt manifest. It also warns when
- reports exist without benchmark evidence from the same project.
+ over time and inspect report artifacts locally. The dashboard surfaces
+ feature-level rollups, prompt-level eval rows, benchmark evidence, and a
+ latest-report review queue when reports come from a prompt manifest. It also
+ warns when reports exist without benchmark evidence from the same project.
- `redline summary`: inspect suite readiness, or pass `redline-prompts.json` to
roll up multi-prompt suite coverage, owners, requirements, and missing suites.
- `redline-mcp`: let AI coding assistants run checks inside Claude, Codex,
@@ -311,7 +380,7 @@ redline-mcp
Use [docs/mcp.md](docs/mcp.md) to wire redline into an MCP client. The MCP
surface exposes safe capture-readiness, privacy, audit, scale, read,
-case-inspection, eval, and report tools plus workflow prompts like
+quick-check, case-inspection, eval, and report tools plus workflow prompts like
`setup_redline_project`, `check_prompt_change`, `build_suite_from_logs`, and
`review_candidate_outputs`.
It can also list or copy runner adapters and optional judge templates during setup.
@@ -330,7 +399,7 @@ redline init --runner stdio --copy-runner --github-action
Use redline as a composite GitHub Action from another repo:
```yaml
-- uses: gowtham0992/redline@v0.2.1
+- uses: gowtham0992/redline@v0.3.0
with:
prompt-path: prompts/v2.txt
benchmark-max-seconds: "300"
@@ -424,9 +493,10 @@ redline summary redline-suite.json
`doctor` shows whether the suite has explicit requirements or recorded
judgments before you rely on structural checks in CI.
-`summary` reports cluster/case coverage, owner coverage, accepted baseline
-history, approver coverage, and explicit guard coverage for cases with
-requirements or recorded judgments so teams can review suite readiness before CI.
+`summary` reports a suite readiness score, behavior-group/case coverage, owner
+coverage, accepted baseline history, approver coverage, and explicit guard
+coverage for cases with requirements or recorded judgments so teams can review
+suite readiness before CI.
`dashboard` also shows audit checkpoint evidence when `.redline/audit-checkpoint.json`
is present.
@@ -476,7 +546,11 @@ bash scripts/release_check.sh
- [docs/release.md](docs/release.md): package, tag, PyPI, and MCP Registry release flow
- [docs/launch.md](docs/launch.md): public alpha launch plan
- [docs/troubleshooting.md](docs/troubleshooting.md): first-run and CI failure recovery
+- [docs/import-guides.md](docs/import-guides.md): Langfuse, Helicone, OpenAI chat, Datadog, and custom log import recipes
+- [docs/methodology.md](docs/methodology.md): behavior grouping, case selection, scoring, and trust boundaries
+- [docs/calibration.md](docs/calibration.md): tiny fixture showing regressions, changed cases, and neutral cases
- [docs/commands.md](docs/commands.md): compact CLI command reference
+- [docs/real-log-dogfood.md](docs/real-log-dogfood.md): redaction-first real-log test protocol
- [docs/dogfood.md](docs/dogfood.md): first-user dogfood protocol
- [docs/case-studies.md](docs/case-studies.md): reproducible dogfood case studies
- [docs/internet-dogfood-sources.md](docs/internet-dogfood-sources.md): public prompt-response datasets for dogfood sourcing
diff --git a/docs/ai-session-dogfood-prompts.jsonl b/docs/ai-session-dogfood-prompts.jsonl
index e75ab61..ad0534b 100644
--- a/docs/ai-session-dogfood-prompts.jsonl
+++ b/docs/ai-session-dogfood-prompts.jsonl
@@ -2,7 +2,7 @@
{"case_id":"task_02","prompt":"Design a first-run experience for redline. A new user cloned the repo and has five minutes. What exact commands should they run, what should they see, and where are they most likely to get confused?"}
{"case_id":"task_03","prompt":"Write a concise config reference for redline.json. Include fields for suite path, input/output fields, max cases, replay command, judge command, reports, logs, and fail-on behavior."}
{"case_id":"task_04","prompt":"Create a runner adapter guide for a team using an HTTP API. Their endpoint accepts {\"prompt\":\"...\"} and returns {\"response\":\"...\"}. Show the script, environment variables, and redline command."}
-{"case_id":"task_05","prompt":"Review this CLI output and tell me whether it is clear enough for a first-time user. Suggest exact wording improvements:\nGenerated 5 cases from 5 records.\nDetected 5 behavioral clusters.\nWrote redline-suite.json."}
+{"case_id":"task_05","prompt":"Review this CLI output and tell me whether it is clear enough for a first-time user. Suggest exact wording improvements:\nGenerated 5 cases from 5 records.\nDetected 5 behavior-signature groups.\nWrote redline-suite.json."}
{"case_id":"task_06","prompt":"I changed an LLM system prompt and now responses are shorter. Give me a realistic before/after prompt-output log with 5 JSONL rows where the shorter candidate silently loses important behavior."}
{"case_id":"task_07","prompt":"Act as a code reviewer for a Python CLI eval tool. What are the top 10 bugs or product risks you would look for before calling it public alpha?"}
{"case_id":"task_08","prompt":"Write a GitHub Actions workflow for redline. It should install the package, run doctor, validate the suite, run eval, upload reports, and fail on regressions."}
diff --git a/docs/calibration.md b/docs/calibration.md
new file mode 100644
index 0000000..6b91980
--- /dev/null
+++ b/docs/calibration.md
@@ -0,0 +1,48 @@
+# Calibration Examples
+
+Use this fixture when you want to see redline's trust boundary in a tiny,
+inspectable dataset. It is not a benchmark. It is a calibration exercise that
+shows what deterministic checks catch, what they mark for review, and what a
+green case means.
+
+```bash
+redline suite examples/calibration_baseline.jsonl \
+ --out /tmp/redline-calibration-suite.json \
+ --all-cases
+
+redline diff /tmp/redline-calibration-suite.json \
+ examples/calibration_candidate.jsonl \
+ --fail-on none
+```
+
+Expected result:
+
+- Two regressions: lost JSON structure, and dropped refund details.
+- One changed case: the reply added apologetic tone.
+- One neutral case: the response is unchanged.
+
+The neutral case is deliberately boring. It means redline found no configured
+behavioral change in that case. It does not prove factual correctness,
+hallucination safety, policy compliance, or subtle reasoning quality.
+
+## Why This Matters
+
+The fixture demonstrates the intended operating model:
+
+- structural losses should block a prompt release;
+- tone or wording shifts should be reviewed before acceptance;
+- unchanged or neutral cases still inherit the global trust boundary;
+- must-cover product requirements should be pinned with `redline suite add` or
+ `redline require`.
+
+For semantic risks, add requirements or configure an optional judge. Keep the
+default deterministic checks as the fast local gate, and use judges where the
+product risk is not visible from structure or concrete detail loss.
+
+For exploratory data with noisy entity extraction, compare strict and review
+profiles:
+
+```bash
+redline diff /tmp/redline-calibration-suite.json examples/calibration_candidate.jsonl --profile strict --fail-on none
+redline diff /tmp/redline-calibration-suite.json examples/calibration_candidate.jsonl --profile review --fail-on none
+```
diff --git a/docs/case-studies.md b/docs/case-studies.md
index 7d8c678..d042412 100644
--- a/docs/case-studies.md
+++ b/docs/case-studies.md
@@ -1,8 +1,9 @@
# Case Studies
-These are repo-local dogfood runs that anyone can reproduce from a fresh
-checkout. They are not customer case studies yet. They are the proof fixtures we
-use until external users share anonymized logs.
+These are dogfood runs we use until external users share anonymized logs. The
+first two are repo-local fixtures that anyone can reproduce from a fresh
+checkout. The public internet dogfood run uses third-party rows that stay under
+`.redline/private/`, so only the protocol and aggregate result are committed.
The important property is that the data is checked in, deterministic, and
launch-safe. If a future redline change stops catching these regressions, the
@@ -131,6 +132,68 @@ Expected result: long-form assistant differences appear as `changed` review
items unless a candidate loses stronger deterministic signals such as required
structure, URLs, refusals, or empty output.
+## Case Study 4: Public Internet Dogfood
+
+Scenario: download a public instruction-response dataset, import a 100-row
+sample, create a deterministic local "candidate got shorter" variant, then
+verify that redline diagnoses the regression pattern across reports and
+dashboard surfaces.
+
+Source:
+
+- [Databricks Dolly 15k](https://huggingface.co/datasets/databricks/databricks-dolly-15k)
+- Local raw and normalized rows stay under `.redline/private/`.
+
+Protocol used:
+
+```bash
+curl -L \
+ https://huggingface.co/datasets/databricks/databricks-dolly-15k/resolve/main/databricks-dolly-15k.jsonl \
+ -o .redline/private/internet-dogfood-100/dolly-raw.jsonl
+
+redline import .redline/private/internet-dogfood-100/dolly-raw.jsonl \
+ --preset dolly \
+ --limit 100 \
+ --out .redline/private/internet-dogfood-100/dolly-baseline-100.jsonl
+
+redline suite .redline/private/internet-dogfood-100/dolly-baseline-100.jsonl \
+ --out .redline/private/internet-dogfood-100/dolly-suite-100.json \
+ --all-cases
+```
+
+The candidate file for this run was generated locally by applying deterministic
+shortening, refusal, empty-output, and generic-answer mutations to the imported
+baseline. It stays under `.redline/private/` with the downloaded source rows.
+For real dogfood, replace that mutation step with your actual candidate model or
+prompt runner.
+
+```bash
+redline diff .redline/private/internet-dogfood-100/dolly-suite-100.json \
+ .redline/private/internet-dogfood-100/dolly-candidate-100.jsonl \
+ --compact \
+ --fail-on none
+```
+
+Current result:
+
+```text
+redline diff: cases=100 regression=51 changed=27 improved=0 accepted=0 ignored=0 missing=0 neutral=22
+Diagnosis: Candidate got shorter, lost required structure, dropped concrete details, returned empty outputs, and changed content substantially; fix blocking cases before shipping.
+```
+
+What redline catches:
+
+- Shorter candidate outputs that drop concrete details.
+- Empty outputs and newly refused safe requests.
+- Structure loss in code, lists, and other format-sensitive answers.
+- Suite coverage across 100/100 cases and 20/20 behavior groups.
+- Dashboard sidecar filtering: 1 report, 1 benchmark, 1 history entry, and 0
+ warnings in the local product app.
+
+Why this matters: the demo proves the first-run promise; this run proves the
+same local loop can process a larger public prompt-response sample and produce
+dashboard/report evidence without cloud services or private data.
+
## External Case Studies Still Needed
The repo fixtures prove the loop and protect regressions in redline itself. The
diff --git a/docs/commands.md b/docs/commands.md
index 58a23ce..1308788 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -7,20 +7,23 @@ output and examples after installing.
| --- | --- | --- |
| `redline` | Show first-run help. | `--version` |
| `redline demo` | Generate and diff the bundled demo logs. | `--public`, `--compact` |
+| `redline status` | Show project readiness, latest local evidence, app command, first review case, and the next command to run. | `--reports-dir`, `--history`, `--checkpoint`, `--limit`, `--json` |
+| `redline app` | Open the guided local product app for import, suite, eval, review, history, and integration commands. | `--demo`, `--reports-dir`, `--history`, `--checkpoint`, `--out`, `--no-open`, `--json` |
| `redline init` | Write `redline.json`, runner files, and optional CI workflow. | `--runner`, `--copy-runner`, `--github-action`, `--force` |
| `redline doctor` | Check config, suite, replay, reports, audit, and team workflow. | `--strict`, `--json` |
| `redline watch` | Collect prompt-response observations or print snippets. | `--log`, `--stats`, `--snippet`, `--follow` |
-| `redline import` | Normalize exported JSONL fields into redline `prompt`/`response` logs. | `--input-field`, `--output-field`, `--context-field`, `--metadata-field`, `--limit`, `--out` |
+| `redline import` | Normalize exported JSONL fields into redline `prompt`/`response` logs, with best-effort redaction on by default. | `--list-presets`, `--detect`, `--auto-map`, `--preview`, `--preset`, `--input-field`, `--output-field`, `--context-field`, `--metadata-field`, `--limit`, `--out`, `--no-redact` |
| `redline redact` | Check or write best-effort redacted logs. | `--check`, `--out`, `--json` |
-| `redline cluster` | Inspect behavior groups before suite generation. | `--max-cases`, `--json` |
+| `redline cluster` | Inspect deterministic behavior-signature groups before suite generation. | `--max-cases`, `--json` |
| `redline suite` | Generate a suite from baseline JSONL logs. | `--out`, `--max-cases`, `--all-cases`, `--owner` |
| `redline suite add` | Pin a hand-picked edge case. | `--prompt`, `--response`, `--include`, `--exclude`, `--owner` |
| `redline cases` | List generated suite case IDs and coverage. | `--json` |
| `redline case` | Show one full suite case. | `--json` |
| `redline require` | Add deterministic include/exclude requirements. | `--include`, `--exclude`, `--owner` |
| `redline prompts` | Build or check a prompt-to-suite manifest. | `--suite-dir`, `--out`, `--check`, `--check-suites` |
-| `redline summary` | Summarize suite or manifest readiness. | `--json` |
+| `redline summary` | Summarize suite or manifest readiness, including suite score and coverage gaps. | `--json` |
| `redline validate` | Validate suite or manifest structure and freshness. | `--strict`, `--json` |
+| `redline quick-check` | Generate a temporary suite from baseline JSONL, diff candidate JSONL, and write reports plus a guided local app in one first-run command. | `--input-field`, `--output-field`, `--out-dir`, `--max-cases`, `--all-cases`, `--profile`, `--fail-on`, `--open`, `--open-app` |
| `redline budget` | Estimate CI runtime without replaying prompts. | `--workers`, `--timeout`, `--max-seconds`, `--measure-local` |
| `redline benchmark` | Compatibility alias for `redline budget`. | Same as `budget` |
| `redline eval` | Replay suite cases through a configured runner. | `--prompt`, `--replay`, `--workers`, `--judge`, `--fail-on`, `--compact` |
@@ -29,7 +32,7 @@ output and examples after installing.
| `redline accept` | Promote reviewed candidate outputs into the suite baseline. | `--candidate`, `--all-expected`, `--approver`, `--note` |
| `redline history` | Append and summarize trend history. | `--label`, `--out`, `--out-md`, `--fail-on` |
| `redline compare` | Compare two redline reports. | `--fail-on`, `--out-json`, `--out-html` |
-| `redline dashboard` | Render a self-contained local HTML dashboard. | `--reports-dir`, `--history`, `--checkpoint`, `--out`, `--open` |
+| `redline dashboard` | Render a self-contained local HTML dashboard artifact. | `--reports-dir`, `--history`, `--checkpoint`, `--out`, `--style classic\|app`, `--open` |
| `redline runners` | List or copy runner and log adapter templates. | `--copy`, `--out`, `--force` |
| `redline judges` | List or copy judge templates. | `--copy`, `--out`, `--force` |
| `redline audit` | Show and verify local audit events. | `--verify`, `--checkpoint`, `--out-checkpoint`, `--expect-last-hash` |
@@ -51,3 +54,19 @@ output and examples after installing.
Structural checks are deterministic and local. Green or neutral does not prove
semantic equivalence. Use `redline require`, `redline mark`, and optional judge
templates for factual, tone, hallucination, policy, or reasoning risks.
+
+## Import Help
+
+Use `redline import --detect` when you do not know an export's field names.
+Use `--auto-map --preview 3` when you want redline to try the best detected
+mapping before writing normalized logs. Source-specific recipes live in
+[docs/import-guides.md](import-guides.md).
+
+## Diff profiles
+
+- `--profile strict` is the default CI-oriented mode. Missing concrete details
+ such as numbers and likely entities are blocking regressions.
+- `--profile review` is a softer calibration mode for noisy exploratory logs.
+ Missing numbers and likely entities become reviewable `changed` cases while
+ hard structural losses, such as invalid JSON, missing JSON keys, empty
+ outputs, refusals, tables, lists, code blocks, and URLs, stay blocking.
diff --git a/docs/dogfood.md b/docs/dogfood.md
index 143dfb2..a1eade3 100644
--- a/docs/dogfood.md
+++ b/docs/dogfood.md
@@ -23,13 +23,13 @@ Estimated time: 10 minutes.
python -m pip install redline-ai
redline demo
redline cases .redline/demo/suite.json
-redline dashboard --reports-dir .redline/demo/reports --out .redline/dashboard.html
+redline app --reports-dir .redline/demo/reports --no-open --out .redline/app.html
redline runners
redline doctor
```
Expected result: the demo catches realistic support-agent regressions, the next
-steps are obvious, the dashboard links reports cleanly, and the first warning
+steps are obvious, the app links reports cleanly, and the first warning
explains exactly what to run next.
## Pass 2: Real Replay Setup
diff --git a/docs/import-guides.md b/docs/import-guides.md
new file mode 100644
index 0000000..616ffc5
--- /dev/null
+++ b/docs/import-guides.md
@@ -0,0 +1,120 @@
+# Import Guides
+
+redline only needs prompt-response JSONL. Your production system probably calls
+those fields something else. Start with detection, preview the mapping, then
+write the normalized baseline.
+
+```bash
+redline import raw-export.jsonl --detect
+redline import raw-export.jsonl --auto-map --preview 3
+redline import raw-export.jsonl --auto-map --out logs/baseline.jsonl
+redline import raw-export.jsonl --input-field request.prompt --output-field response.text --preview 3
+redline import raw-export.jsonl --input-field request.prompt --output-field response.text --out logs/baseline.jsonl
+```
+
+Redaction is enabled by default during import. It is best-effort pattern
+matching, not a privacy boundary; review normalized files before sharing or
+committing them.
+
+## Langfuse-Style Exports
+
+Use the built-in preset when your export has `input` and `output` fields.
+
+```bash
+redline import langfuse-export.jsonl --preset langfuse --preview 3
+redline import langfuse-export.jsonl --preset langfuse --out logs/baseline.jsonl
+```
+
+If your export nests generations under a trace object, detect first:
+
+```bash
+redline import langfuse-export.jsonl --detect
+```
+
+Then rerun import with the suggested field paths.
+
+## Helicone-Style Exports
+
+The preset expects `request.prompt` and `response.text`.
+
+```bash
+redline import helicone-export.jsonl --preset helicone --preview 3
+redline import helicone-export.jsonl --preset helicone --out logs/baseline.jsonl
+```
+
+For chat payloads, the prompt may be an array of messages. That is acceptable;
+redline stringifies structured prompts deterministically.
+
+## OpenAI Chat Traces
+
+The `openai-chat` preset maps chat messages to the prompt and the first choice
+message to the response.
+
+```bash
+redline import openai-chat.jsonl --preset openai-chat --preview 3
+redline import openai-chat.jsonl --preset openai-chat --out logs/baseline.jsonl
+```
+
+If your app stores `response.output_text`, override the response path:
+
+```bash
+redline import openai-chat.jsonl \
+ --input-field request.messages \
+ --output-field response.output_text \
+ --preview 3
+```
+
+## Datadog Or App Logs
+
+The `datadog` preset expects `attributes.input` and `attributes.output`.
+
+```bash
+redline import datadog.jsonl --preset datadog --preview 3
+redline import datadog.jsonl --preset datadog --out logs/baseline.jsonl
+```
+
+For custom app logs, common mappings look like:
+
+```bash
+redline import app.jsonl --input-field ticket.text --output-field assistant.reply --preview 3
+redline import app.jsonl --input-field prompt --output-field completion --preview 3
+redline import app.jsonl --input-field payload.user_question --output-field result.assistant_answer --preview 3
+```
+
+## Database Exports
+
+Export the rows you want as JSONL, one object per line:
+
+```json
+{"prompt":"Classify ticket INV-1042","response":"{\"owner\":\"billing\"}","created_at":"2026-06-14T10:00:00Z"}
+```
+
+Then import:
+
+```bash
+redline import db-export.jsonl \
+ --input-field prompt \
+ --output-field response \
+ --metadata-field created_at \
+ --preview 3
+
+redline import db-export.jsonl \
+ --input-field prompt \
+ --output-field response \
+ --metadata-field created_at \
+ --out logs/baseline.jsonl
+```
+
+## After Import
+
+```bash
+redline suite logs/baseline.jsonl --out redline-suite.json
+redline summary redline-suite.json
+redline cases redline-suite.json
+```
+
+If the suite looks thin, add pinned cases:
+
+```bash
+redline suite add redline-suite.json --prompt "..." --response "..."
+```
diff --git a/docs/launch.md b/docs/launch.md
index b8da4dc..33e082e 100644
--- a/docs/launch.md
+++ b/docs/launch.md
@@ -34,15 +34,15 @@ Generate launch artifacts from a clean release checkout:
```bash
bash scripts/release_check.sh
bash scripts/demo_gif.sh .redline/launch .redline/launch/redline-demo.gif
-bash scripts/build_release.sh /tmp/redline-dist-v0.2.1
+bash scripts/build_release.sh /tmp/redline-dist-v0.3.0
```
Required artifacts:
- `.redline/launch/redline-demo.gif` or `.redline/launch/redline-demo-transcript.txt`
- `.redline/dashboard.html` from the demo path
-- `/tmp/redline-dist-v0.2.1/redline_ai-0.2.1-py3-none-any.whl`
-- `/tmp/redline-dist-v0.2.1/redline_ai-0.2.1.tar.gz`
+- `/tmp/redline-dist-v0.3.0/redline_ai-0.3.0-py3-none-any.whl`
+- `/tmp/redline-dist-v0.3.0/redline_ai-0.3.0.tar.gz`
## Website Checklist
@@ -79,19 +79,21 @@ should sell the loop, not the package manager.
2. Tag the exact commit:
```bash
- git tag v0.2.1
- git push origin v0.2.1
+ git tag v0.3.0
+ git push origin v0.3.0
```
3. Upload the release distribution:
```bash
- python -m twine upload /tmp/redline-dist-v0.2.1/*
+ python -m twine upload \
+ /tmp/redline-dist-v0.3.0/redline_ai-*.whl \
+ /tmp/redline-dist-v0.3.0/redline_ai-*.tar.gz
```
-4. Create the GitHub release from `v0.2.1`.
+4. Create the GitHub release from `v0.3.0`.
5. Attach the demo GIF or link the public README asset:
- `https://gowtham0992.github.io/redline/assets/redline-product-demo.gif`.
+ `https://raw.githubusercontent.com/gowtham0992/redline/main/site/assets/redline-product-demo.gif`.
## Launch Post
@@ -101,7 +103,7 @@ Short post:
I built redline, a local-first tool that turns prompt-response logs into eval
suites automatically.
-You point it at existing JSONL logs, it clusters behavior, picks representative
+You point it at existing JSONL logs, groups behavior by deterministic signatures, picks representative
cases, and catches regressions when a prompt change drops structure, numbers,
URLs, entities, refusals, tables, code blocks, or required fields.
@@ -123,7 +125,7 @@ Longer post:
Most eval tools start with "write test cases." redline starts from the logs you
already have.
-It watches or imports prompt-response JSONL, clusters observed behavior, builds
+It watches or imports prompt-response JSONL, groups observed behavior by deterministic signatures, builds
a representative suite, and compares new prompt runs against the accepted
baseline. The default checks are deterministic and local: JSON validity, missing
keys, empty outputs, refusals, URLs, numbers, entities, code blocks, tables,
diff --git a/docs/mcp.md b/docs/mcp.md
index 1350bea..cf61b83 100644
--- a/docs/mcp.md
+++ b/docs/mcp.md
@@ -63,10 +63,13 @@ reports.
Available tools:
+- `redline_status`
- `redline_doctor`
- `redline_suite`
+- `redline_quick_check`
- `redline_redact`
-- `redline_import`
+- `redline_import` (supports `detect`, `auto_map`, and `preview` before writing)
+- `redline_import_presets`
- `redline_watch_stats`
- `redline_watch_snippets`
- `redline_prompts`
@@ -86,9 +89,9 @@ Available tools:
- `redline_audit` (including `verify` and checkpoint output for the local audit hash chain)
- `redline_sbom`
-`redline_diff` and `redline_eval` return the underlying redline exit code as
-structured data. Exit code `1` means redline found blocking regressions or
-missing outputs; the MCP tool still returns successfully because that is a
+`redline_quick_check`, `redline_diff`, and `redline_eval` return the underlying
+redline exit code as structured data. Exit code `1` means redline found blocking regressions
+or missing outputs; the MCP tool still returns successfully because that is a
product finding, not a protocol failure. Exit code `2` and above indicates a
setup or command error.
@@ -101,9 +104,9 @@ assistants can reason over reports without scraping terminal text.
The server also exposes MCP prompt templates for the workflows agents should
reach for most often:
-- `check_prompt_change`: run doctor, then eval a changed prompt file.
+- `check_prompt_change`: run status/doctor, then eval a changed prompt file.
- `build_suite_from_logs`: generate a suite, validate it, and summarize coverage.
-- `review_candidate_outputs`: diff candidate JSONL outputs and lead with blocking findings.
+- `review_candidate_outputs`: quick-check two JSONL logs or diff candidate outputs against a suite, then lead with blocking findings.
- `setup_redline_project`: guide first-time setup through runner selection,
prompt/log discovery, suite validation, CI scale checks, and optional judge setup.
@@ -120,6 +123,11 @@ Ask your assistant:
Run redline doctor in this repo and tell me the next setup step.
```
+```text
+Run redline status in this repo and tell me the exact next command, app command,
+first review case, and why that case matters.
+```
+
```text
Set up redline for this project. Use my existing logs if available, choose the
right runner adapter, and do not mutate the baseline.
@@ -130,6 +138,11 @@ Generate a redline suite from logs/baseline.jsonl, then diff it against
logs/candidate.jsonl. Summarize only regressions I should review.
```
+```text
+Quick-check logs/baseline.jsonl against logs/candidate.jsonl and tell me the
+blocking regressions before I wire a permanent suite.
+```
+
```text
Run redline eval with prompts/v2.txt and tell me whether this prompt change is
safe to ship. Do not accept or modify the baseline.
diff --git a/docs/methodology.md b/docs/methodology.md
new file mode 100644
index 0000000..d02a7c3
--- /dev/null
+++ b/docs/methodology.md
@@ -0,0 +1,129 @@
+# Methodology
+
+redline is a deterministic regression detector for prompt-response systems. It
+does not claim that a green run proves semantic safety. It turns logs into a
+reviewable suite, checks high-signal behavioral changes, and leaves factual,
+tone, policy, hallucination, and subtle reasoning judgment to explicit
+requirements, optional judges, or humans.
+
+This is a deliberate product boundary. A merge blocker should be fast,
+repeatable, local, and cheap. Deterministic checks catch the production failures
+that break parsers and workflows without asking a model to grade every run. Use
+judges as review evidence for ambiguous or domain-specific behavior, not as the
+only line of defense.
+
+## Input Model
+
+redline expects prompt-response observations as JSONL. The canonical fields are
+`prompt` and `response`, but `redline import` can normalize nested exports from
+application logs, observability tools, or model gateways.
+
+Import is redaction-first by default. The importer redacts common secrets and
+PII before extracting fields so copied metadata is treated with the same local
+privacy boundary as prompt and response text. Use `--no-redact` only for local
+fixtures you have already inspected.
+
+## Behavior Grouping
+
+redline does not run statistical, embedding, or semantic clustering by default.
+It groups observations by deterministic behavior signatures:
+
+- prompt intent, such as classify, summarize, transform, generate, or answer
+- response shape, such as JSON, table, code block, bullet list, numbered list,
+ refusal, empty output, or prose
+- response length bucket
+- JSON schema shape when the response is valid JSON
+
+The CLI command is named `redline cluster` for compatibility, but the public
+model is behavior-signature grouping. These groups are explainable and stable;
+they are not a claim that two prompts are semantically equivalent.
+
+## Case Selection
+
+Suite generation chooses cases in this order:
+
+1. One representative from each behavior group.
+2. High-risk groups with obvious failure patterns, such as invalid JSON,
+ malformed tables, empty outputs, or refusals.
+3. High-variance edges where same-shape outputs vary substantially in length.
+4. Prompt-diverse samples from large groups when the case budget allows.
+5. User-pinned cases from `redline suite add`.
+
+This strategy optimizes for useful regression coverage from existing logs, not
+perfect coverage of every product risk. If a scenario matters, pin it.
+
+Generated suites include a `methodology` block with the deterministic
+case-selection method and version, currently `behavior-signature-v1`, so
+reports can be audited against the algorithm that selected the suite. Diff
+and eval reports copy that block into JSON, Markdown, HTML, and dashboard
+surfaces so reviewers can see the trust method beside the result.
+The generated suite summary also carries `case_coverage` and
+`cluster_coverage` so downstream tools can display basic suite-health context
+without recomputing it.
+
+Suite summaries also count `stochastic_prompt_groups`: prompts that appear with
+more than one distinct baseline response. Those prompts may be genuinely
+multi-answer, or they may come from high-temperature sampling, model drift, or
+mixed production contexts. redline warns on them because natural baseline
+variance can look like a regression unless the suite is stabilized, switched to
+review mode, or guarded with explicit requirements.
+
+## Regression Signals
+
+The deterministic diff engine checks for high-signal changes:
+
+- lost JSON validity or missing JSON keys
+- lost table, list, or code-block structure
+- missing numbers, URLs, ticket IDs, and likely entities
+- empty outputs
+- newly refused safe-looking tasks
+- allow/deny polarity flips
+- large same-shape content drift
+- explicit include/exclude requirements
+
+These checks are intentionally conservative. They catch common prompt
+regressions quickly in CI, but they do not replace domain review.
+
+## Suite Readiness Score
+
+`redline summary` reports a suite readiness score from 0 to 100. The score is a
+suite-health signal, not a model-quality score. It combines:
+
+- behavior-group coverage
+- unique prompt-response pair coverage
+- explicit requirements or recorded judgments
+- owner coverage
+- high-risk group calibration
+- non-English calibration warnings
+- stochastic baseline warnings
+
+Use the score to decide what to improve before CI gating. A high score means the
+suite is better prepared for review and automation. It does not mean the
+candidate prompt is safe to ship.
+
+## Judges
+
+Optional judges are for ambiguous `changed` cases and semantic risks that
+deterministic checks cannot prove. A judge can be a local script, a local model,
+or a cloud LLM command you control. redline routes only the configured review
+surface and records judge output as evidence.
+
+Default redline behavior is local and deterministic. No cloud judge is called
+unless you configure one.
+
+## Calibration
+
+Use redline as a safety loop:
+
+1. Import or watch real logs.
+2. Generate a suite.
+3. Inspect `redline summary` and `redline cases`.
+4. Pin must-cover edge cases.
+5. Add requirements for details that must not disappear.
+6. Run `redline eval` or `redline diff` before prompt changes ship.
+7. Mark expected changes and accept new baselines after review.
+
+The strongest evidence for redline is not a score. It is a concrete report that
+says which production behavior changed and why it matters.
+
+For a tiny runnable boundary check, see [docs/calibration.md](calibration.md).
diff --git a/docs/real-log-dogfood.md b/docs/real-log-dogfood.md
new file mode 100644
index 0000000..68a19f4
--- /dev/null
+++ b/docs/real-log-dogfood.md
@@ -0,0 +1,133 @@
+# Real Log Dogfood
+
+Use this protocol when testing redline on real prompt/output logs from a team,
+agent, support workflow, extraction job, or public dataset. The goal is to learn
+whether redline catches useful regressions without leaking private data.
+
+## Rule Zero
+
+Do not send raw customer logs, secrets, API keys, or private prompts to another
+person. Keep the raw export local. Share only sanitized reports or short
+hand-redacted examples.
+
+## 1. Inspect the Export
+
+Open a few rows before importing:
+
+```bash
+python -m json.tool < first-row.json
+```
+
+Find the field that contains the user prompt or instruction, and the field that
+contains the model or agent response. Common examples:
+
+| Source shape | Input field | Output field |
+| --- | --- | --- |
+| redline JSONL | `prompt` | `response` |
+| Dolly-style public data | `instruction` | `response` |
+| OpenAI wrapper log | `request.messages` | `response.output_text` |
+| Langfuse/Helicone-style export | `input` | `output` |
+| Support traces | `ticket.text` | `assistant.reply` |
+
+For common exports, start with a preset and override fields only when your file
+differs:
+
+```bash
+redline import --list-presets
+redline import raw-export.jsonl --detect
+redline import raw-export.jsonl --preset langfuse --preview 3
+redline import raw-export.jsonl --preset langfuse --out .redline/dogfood/baseline.jsonl
+redline import raw-export.jsonl --preset helicone --out .redline/dogfood/baseline.jsonl
+redline import raw-export.jsonl --preset openai-chat --out .redline/dogfood/baseline.jsonl
+```
+
+## 2. Import With Redaction On
+
+`redline import` redacts common secrets and PII by default before writing the
+normalized baseline file.
+
+```bash
+redline import raw-export.jsonl \
+ --input-field instruction \
+ --output-field response \
+ --metadata-field category \
+ --preview 3
+
+redline import raw-export.jsonl \
+ --input-field instruction \
+ --output-field response \
+ --metadata-field category \
+ --out .redline/dogfood/baseline.jsonl
+```
+
+Use `--detect` when you do not know the export field names. Use `--preview`
+before writing when the export shape is unfamiliar. It prints mapped, redacted
+sample prompt-response rows without writing a file. The import command prints
+how many values were redacted. If the count is non-zero, inspect the normalized
+output before generating a suite.
+
+Use `--no-redact` only for local-only test fixtures that have already been
+reviewed.
+
+## 3. Generate a Suite
+
+```bash
+redline suite .redline/dogfood/baseline.jsonl \
+ --out .redline/dogfood/suite.json
+```
+
+Then inspect the selected cases:
+
+```bash
+redline cases .redline/dogfood/suite.json
+redline summary .redline/dogfood/suite.json
+```
+
+## 4. Compare a Candidate
+
+If you already have candidate outputs:
+
+```bash
+redline diff .redline/dogfood/suite.json .redline/dogfood/candidate.jsonl \
+ --compact \
+ --out-json .redline/dogfood/diff.json \
+ --out-html .redline/dogfood/diff.html \
+ --fail-on none
+```
+
+If you have a runner configured:
+
+```bash
+redline eval .redline/dogfood/suite.json \
+ --prompt prompts/v2.txt \
+ --compact \
+ --fail-on none
+```
+
+## 5. Report Back
+
+For early dogfood, collect this information:
+
+- Source type: public dataset, app logs, trace export, SDK capture, or manual JSONL.
+- Rows imported and redaction count.
+- Suite cases generated and behavior groups covered.
+- Blocking regressions found, if any.
+- False positives that felt noisy.
+- First command or error message that caused confusion.
+- Whether the result would have changed a shipping decision.
+
+Do not share raw logs. Share the redline report only after reviewing it for
+private content.
+
+## What Good Looks Like
+
+A useful dogfood run does not need to prove semantic correctness. It should show
+whether redline caught concrete structural drift:
+
+- JSON keys disappeared.
+- A table became prose.
+- A safe task became a refusal.
+- Required numbers, URLs, owners, or action items were dropped.
+- A pinned requirement failed.
+
+Those are the regressions redline is designed to catch quickly and locally.
diff --git a/docs/release.md b/docs/release.md
index 87b0edd..96808c9 100644
--- a/docs/release.md
+++ b/docs/release.md
@@ -14,7 +14,8 @@ changes; patch versions are reserved for compatible fixes.
- Work from a clean release branch that is pushed to origin.
- Before tagging, merge to `main` and rerun `bash scripts/certify_release.sh`.
-- Update the version in `pyproject.toml` and `redline/__init__.py`.
+- Update the version in `pyproject.toml`, `redline/__init__.py`, and
+ `server.json`.
- Update `CHANGELOG.md` with the user-visible changes.
- Run the packaged release gate:
@@ -51,7 +52,7 @@ new user:
python -m pip install redline-ai
redline demo
redline demo --public --compact
-redline dashboard --reports-dir .redline/demo/reports --out .redline/dashboard.html
+redline app --reports-dir .redline/demo/reports --no-open --out .redline/app.html
redline runners
redline init --runner stdio --copy-runner
redline doctor
@@ -59,7 +60,7 @@ redline doctor
Confirm the demo ends with actionable next steps, shows the mark/accept review
loop, and catches the intentional support-agent regressions. Confirm the
-dashboard opens as a self-contained local report index. Confirm the
+guided app opens as a self-contained local workflow. Confirm the
public-pattern proof works from the installed package without relying on
repo-local `examples/` paths.
@@ -78,7 +79,7 @@ To run the package gate, external-project Action smoke, release build, and
`twine check` as one certification pass:
```bash
-bash scripts/certify_release.sh /tmp/redline-certify-v0.2.1
+bash scripts/certify_release.sh /tmp/redline-certify-v0.3.0
```
The certification summary records the git commit, branch, and clean/dirty worktree state.
@@ -89,8 +90,8 @@ Release evidence can be traced back to the exact code that was tested.
After the release gate and public-alpha smoke both pass:
```bash
-git tag v0.2.1
-git push origin v0.2.1
+git tag v0.3.0
+git push origin v0.3.0
```
Use a new tag for every public release. Do not move an existing release tag
@@ -106,11 +107,11 @@ that was tagged. Do not upload an ignored local `dist/*`; it can contain stale
dogfood artifacts from earlier builds.
```bash
-bash scripts/build_release.sh /tmp/redline-dist-v0.2.1
-python -m twine upload /tmp/redline-dist-v0.2.1/redline_ai-*.whl /tmp/redline-dist-v0.2.1/redline_ai-*.tar.gz
+bash scripts/build_release.sh /tmp/redline-dist-v0.3.0
+python -m twine upload /tmp/redline-dist-v0.3.0/redline_ai-*.whl /tmp/redline-dist-v0.3.0/redline_ai-*.tar.gz
```
-`build_release.sh` also writes `/tmp/redline-dist-v0.2.1/redline-sbom.json` as
+`build_release.sh` also writes `/tmp/redline-dist-v0.3.0/redline-sbom.json` as
CycloneDX release evidence. Keep it with internal release records or attach it
to GitHub release artifacts when needed.
diff --git a/docs/repository.md b/docs/repository.md
index 6421d2a..0572efb 100644
--- a/docs/repository.md
+++ b/docs/repository.md
@@ -33,7 +33,7 @@ release needs a fix, cut a new patch tag.
Before tagging, run:
```bash
-bash scripts/certify_release.sh /tmp/redline-certify-v0.2.1
+bash scripts/certify_release.sh /tmp/redline-certify-v0.3.0
```
Build and upload PyPI artifacts from the exact tagged commit, using a fresh
@@ -65,7 +65,7 @@ Before posting publicly:
```bash
git status --short --branch
-bash scripts/certify_release.sh /tmp/redline-certify-v0.2.1
+bash scripts/certify_release.sh /tmp/redline-certify-v0.3.0
bash scripts/demo_gif.sh .redline/launch .redline/launch/redline-demo.gif
```
diff --git a/docs/runners.md b/docs/runners.md
index cb30b24..a6187a2 100644
--- a/docs/runners.md
+++ b/docs/runners.md
@@ -247,7 +247,7 @@ python runners/braintrust_suite_export.py redline-suite.json \
```
What it does: writes one JSONL row per redline case with `input`, `expected`,
-and redline metadata such as case ID, cluster, owner, and source line. Import
+and redline metadata such as case ID, behavior signature, owner, and source line. Import
that JSONL into Braintrust when you want Braintrust to mirror the same dataset
redline gates locally.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 2298d3d..9e99a95 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -32,14 +32,15 @@ The demo writes only local files under `.redline/demo`.
## Dashboard does not open
-`--open` uses Python's local browser integration. It can be a no-op on headless
-CI, SSH sessions, containers, and remote agents. Write the file explicitly:
+`redline app` uses Python's local browser integration by default. It can be a
+no-op on headless CI, SSH sessions, containers, and remote agents. Write the
+file explicitly:
```bash
-redline dashboard --reports-dir .redline/demo/reports --out .redline/dashboard.html
+redline app --reports-dir .redline/demo/reports --no-open --out .redline/app.html
```
-Then open `.redline/dashboard.html` from your local machine or upload it as a CI
+Then open `.redline/app.html` from your local machine or upload it as a CI
artifact.
## Suite file not found
@@ -75,7 +76,7 @@ The Action runs in CI against committed files. Make sure the suite or prompt
manifest exists in the repository and the action input points to it:
```yaml
-uses: gowtham0992/redline@v0.2.1
+uses: gowtham0992/redline@v0.3.0
with:
suite: redline-suite.json
```
diff --git a/examples/calibration_baseline.jsonl b/examples/calibration_baseline.jsonl
new file mode 100644
index 0000000..4a2c9f4
--- /dev/null
+++ b/examples/calibration_baseline.jsonl
@@ -0,0 +1,4 @@
+{"prompt":"Return JSON with keys owner and priority for ticket INV-1042.","response":"{\"owner\":\"billing\",\"priority\":\"high\"}"}
+{"prompt":"Answer a customer asking about refunds. Include the 30 day window and policy URL https://docs.example.test/refunds.","response":"Refunds are available within 30 days. See https://docs.example.test/refunds for the full policy."}
+{"prompt":"Tell Sam that the export is ready.","response":"Sam, the export is ready in the dashboard."}
+{"prompt":"Confirm the workspace status in one sentence.","response":"The workspace remains available."}
diff --git a/examples/calibration_candidate.jsonl b/examples/calibration_candidate.jsonl
new file mode 100644
index 0000000..7461a9b
--- /dev/null
+++ b/examples/calibration_candidate.jsonl
@@ -0,0 +1,4 @@
+{"prompt":"Return JSON with keys owner and priority for ticket INV-1042.","response":"owner: billing, priority: high"}
+{"prompt":"Answer a customer asking about refunds. Include the 30 day window and policy URL https://docs.example.test/refunds.","response":"Refunds are available. See the policy page."}
+{"prompt":"Tell Sam that the export is ready.","response":"Sorry Sam, unfortunately the export is ready in the dashboard."}
+{"prompt":"Confirm the workspace status in one sentence.","response":"The workspace remains available."}
diff --git a/pyproject.toml b/pyproject.toml
index 8109fbf..8a8e2e9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "redline-ai"
-version = "0.2.1"
+version = "0.3.0"
description = "Generate eval suites from prompt logs and catch LLM regressions locally."
readme = "README.md"
requires-python = ">=3.10"
diff --git a/redline-report.schema.json b/redline-report.schema.json
index 6c70357..04214a0 100644
--- a/redline-report.schema.json
+++ b/redline-report.schema.json
@@ -19,6 +19,31 @@
"enum": ["strict", "review"],
"description": "Diff profile used to classify missing numbers and entities."
},
+ "methodology": {
+ "type": "object",
+ "description": "Suite selection methodology copied from the suite, when available.",
+ "properties": {
+ "name": {"type": "string"},
+ "version": {"type": "string"},
+ "trust_scope": {"type": "string"}
+ },
+ "additionalProperties": true
+ },
+ "suite_summary": {
+ "type": "object",
+ "description": "Suite-health metadata copied from the suite used to generate this report.",
+ "properties": {
+ "records_seen": {"type": "integer", "minimum": 0},
+ "unique_prompt_response_pairs": {"type": "integer", "minimum": 0},
+ "clusters": {"type": "integer", "minimum": 0},
+ "cases": {"type": "integer", "minimum": 0},
+ "case_coverage": {"type": ["number", "null"], "minimum": 0, "maximum": 1},
+ "cluster_coverage": {"type": ["number", "null"], "minimum": 0, "maximum": 1},
+ "stochastic_prompt_groups": {"type": "integer", "minimum": 0},
+ "selection": {"type": "string"}
+ },
+ "additionalProperties": true
+ },
"summary": {
"type": "object",
"required": ["cases", "regression", "changed", "improved", "accepted", "ignored", "neutral", "missing"],
@@ -93,6 +118,10 @@
"baseline_response": {"type": "string"},
"candidate_response": {"type": ["string", "null"]},
"reasons": {"type": "array", "items": {"type": "string"}},
+ "impact": {
+ "type": "string",
+ "description": "Plain-English explanation of why this case matters to reviewers."
+ },
"confidence": {"type": "string", "enum": ["low", "medium", "high"]},
"signal": {
"type": "string",
diff --git a/redline-suite.schema.json b/redline-suite.schema.json
index e786323..c815c79 100644
--- a/redline-suite.schema.json
+++ b/redline-suite.schema.json
@@ -31,6 +31,17 @@
"type": "string",
"description": "JSONL field path used as the baseline response/output."
},
+ "methodology": {
+ "type": "object",
+ "description": "Deterministic grouping and case-selection method used to generate this suite.",
+ "properties": {
+ "name": {"type": "string"},
+ "version": {"type": "string"},
+ "trust_scope": {"type": "string"},
+ "case_selection": {"type": "array", "items": {"type": "string"}}
+ },
+ "additionalProperties": true
+ },
"summary": {
"type": "object",
"required": ["records_seen", "unique_prompt_response_pairs", "clusters", "cases", "selection"],
@@ -40,6 +51,8 @@
"duplicate_prompt_response_pairs": {"type": "integer", "minimum": 0},
"clusters": {"type": "integer", "minimum": 0},
"cases": {"type": "integer", "minimum": 0},
+ "case_coverage": {"type": ["number", "null"], "minimum": 0, "maximum": 1},
+ "cluster_coverage": {"type": ["number", "null"], "minimum": 0, "maximum": 1},
"max_cases": {"type": "integer", "minimum": 0},
"selection": {"type": "string", "enum": ["representative", "all"]},
"pinned_cases": {"type": "integer", "minimum": 0},
@@ -49,7 +62,8 @@
"high_variance_clusters": {"type": "integer", "minimum": 0},
"failure_pattern_clusters": {"type": "integer", "minimum": 0},
"prompt_diversity_cases": {"type": "integer", "minimum": 0},
- "non_ascii_records": {"type": "integer", "minimum": 0}
+ "non_ascii_records": {"type": "integer", "minimum": 0},
+ "stochastic_prompt_groups": {"type": "integer", "minimum": 0}
},
"additionalProperties": true
},
diff --git a/redline/__init__.py b/redline/__init__.py
index 5db1b9b..ea71969 100644
--- a/redline/__init__.py
+++ b/redline/__init__.py
@@ -3,4 +3,4 @@
__all__ = ["RedlineMiddleware", "__version__", "patch_anthropic", "patch_openai", "record", "watch"]
-__version__ = "0.2.1"
+__version__ = "0.3.0"
diff --git a/redline/benchmark.py b/redline/benchmark.py
index e479a93..1d7b9bd 100644
--- a/redline/benchmark.py
+++ b/redline/benchmark.py
@@ -186,7 +186,7 @@ def format_benchmark_report(report: dict[str, Any], *, command_name: str = "benc
"Mode: static estimate; no replay commands are executed",
f"{suite_label + ':':<22}{report['suite']}",
f"Cases: {report['cases']}",
- f"Behavioral clusters: {report['clusters']}",
+ f"Behavior groups: {report['clusters']}",
f"Records seen: {report['records_seen']}",
f"Workers: {report['workers']}",
f"Timeout per case: {_seconds(report['timeout_seconds'])}",
@@ -250,7 +250,7 @@ def format_benchmark_markdown(report: dict[str, Any], *, command_name: str = "be
"| --- | --- |",
f"| {'Prompt manifest' if report.get('is_prompt_manifest') else 'Suite'} | `{report['suite']}` |",
f"| Cases | {report['cases']} |",
- f"| Behavioral clusters | {report['clusters']} |",
+ f"| Behavior groups | {report['clusters']} |",
f"| Workers | {report['workers']} |",
f"| Timeout per case | {_seconds(report['timeout_seconds'])} |",
f"| Worst-case eval budget | {_duration(report['worst_case_seconds'])} |",
diff --git a/redline/cli.py b/redline/cli.py
index ce803f5..67f26f2 100644
--- a/redline/cli.py
+++ b/redline/cli.py
@@ -63,7 +63,16 @@
read_history,
should_fail_history,
)
-from .import_logs import import_jsonl_log
+from .import_logs import (
+ IMPORT_PRESETS,
+ detect_import_fields,
+ format_import_detection,
+ format_import_presets,
+ import_jsonl_log,
+ import_preset,
+ import_preset_rows,
+ preview_jsonl_import,
+)
from .io import append_jsonl, append_text, read_json, read_jsonl_records, write_json, write_jsonl, write_text
from .judge import apply_judge
from .judge_templates import (
@@ -81,7 +90,7 @@
format_prompt_manifest,
format_prompt_manifest_check,
)
-from .redact import DEFAULT_PLACEHOLDER, format_redaction_report, redact_jsonl, scan_jsonl_redactions
+from .redact import DEFAULT_PLACEHOLDER, REDACTION_BOUNDARY, format_redaction_report, redact_jsonl, scan_jsonl_redactions
from .reports import (
format_github_annotations,
format_html_report,
@@ -100,6 +109,7 @@
runner_adapters,
)
from .sbom import build_sbom, format_sbom_report
+from .status import build_project_status, format_project_status
from .summary import (
format_prompt_manifest_summary,
format_suite_summary,
@@ -140,6 +150,9 @@ def _root_help() -> str:
Start here:
redline demo
+ redline status
+ redline app --demo
+ redline quick-check path/to/baseline.jsonl path/to/candidate.jsonl --open
redline dashboard
redline init --runner stdio --copy-runner
redline runners
@@ -210,6 +223,15 @@ def build_parser() -> argparse.ArgumentParser:
doctor_parser.add_argument("--strict", action="store_true", help="exit non-zero when warnings are present")
doctor_parser.set_defaults(func=cmd_doctor)
+ status_parser = subparsers.add_parser("status", help="show project readiness and the next command to run")
+ status_parser.add_argument("--config", default=DEFAULT_CONFIG_PATH, help="config path to read")
+ status_parser.add_argument("--reports-dir", default=".redline/reports", help="directory containing redline JSON reports")
+ status_parser.add_argument("--history", default=".redline/history.jsonl", help="history JSONL path")
+ status_parser.add_argument("--checkpoint", default=".redline/audit-checkpoint.json", help="audit checkpoint JSON path")
+ status_parser.add_argument("--limit", type=int, default=20, help="recent reports/history entries to inspect; use 0 for all")
+ status_parser.add_argument("--json", action="store_true", help="print machine-readable status metadata")
+ status_parser.set_defaults(func=cmd_status)
+
sbom_parser = subparsers.add_parser("sbom", help="write CycloneDX SBOM release evidence")
sbom_parser.add_argument("--out", help="write SBOM JSON to this path")
sbom_parser.add_argument("--json", action="store_true", help="print machine-readable JSON")
@@ -238,9 +260,13 @@ def build_parser() -> argparse.ArgumentParser:
runners_parser.set_defaults(func=cmd_runners)
import_parser = subparsers.add_parser("import", help="normalize exported JSONL logs into redline format")
- import_parser.add_argument("path", help="source JSONL file to normalize")
- import_parser.add_argument("--input-field", default="prompt", help="source field path containing prompt text")
- import_parser.add_argument("--output-field", default="response", help="source field path containing response text")
+ import_parser.add_argument("path", nargs="?", help="source JSONL file to normalize")
+ import_parser.add_argument("--list-presets", action="store_true", help="list built-in import presets")
+ import_parser.add_argument("--detect", action="store_true", help="suggest prompt and response field mappings")
+ import_parser.add_argument("--auto-map", action="store_true", help="use the best detected prompt/response mapping")
+ import_parser.add_argument("--preset", choices=sorted(IMPORT_PRESETS), help="field mapping preset for common log exports")
+ import_parser.add_argument("--input-field", help="source field path containing prompt text")
+ import_parser.add_argument("--output-field", help="source field path containing response text")
import_parser.add_argument("--context-field", help="optional source field path appended to the prompt as Context")
import_parser.add_argument("--id-field", help="optional source field path copied to the redline id field")
import_parser.add_argument(
@@ -250,7 +276,10 @@ def build_parser() -> argparse.ArgumentParser:
help="source field path copied into metadata; repeat for multiple fields",
)
import_parser.add_argument("--limit", type=int, help="maximum records to import")
- import_parser.add_argument("--out", required=True, help="redline JSONL output path")
+ import_parser.add_argument("--preview", type=int, metavar="N", help="preview N mapped rows without writing output")
+ import_parser.add_argument("--out", help="redline JSONL output path")
+ import_parser.add_argument("--no-redact", action="store_true", help="write raw values without import redaction")
+ import_parser.add_argument("--redaction-placeholder", default=DEFAULT_PLACEHOLDER, help="replacement text for import redaction")
import_parser.add_argument("--json", action="store_true", help="print machine-readable JSON")
import_parser.set_defaults(func=cmd_import)
@@ -320,7 +349,7 @@ def build_parser() -> argparse.ArgumentParser:
audit_parser.add_argument("--json", action="store_true", help="print machine-readable JSON")
audit_parser.set_defaults(func=cmd_audit)
- cluster_parser = subparsers.add_parser("cluster", help="analyze behavioral clusters in a log")
+ cluster_parser = subparsers.add_parser("cluster", help="inspect deterministic behavior-signature groups in a log")
cluster_parser.add_argument("log", nargs="?", help="JSONL prompt-response log; defaults to watched log")
cluster_parser.add_argument("--config", default=DEFAULT_CONFIG_PATH, help="config path to read")
cluster_parser.add_argument("--input-field", help="JSONL input field")
@@ -391,6 +420,26 @@ def build_parser() -> argparse.ArgumentParser:
validate_parser.add_argument("--strict", action="store_true", help="exit non-zero when warnings are present")
validate_parser.set_defaults(func=cmd_validate)
+ quick_check_parser = subparsers.add_parser("quick-check", help="generate a temporary suite and diff candidate outputs")
+ quick_check_parser.add_argument("baseline", help="baseline prompt-response JSONL")
+ quick_check_parser.add_argument("candidate", help="candidate prompt-response JSONL")
+ quick_check_parser.add_argument("--input-field", default="prompt", help="JSONL prompt field path")
+ quick_check_parser.add_argument("--output-field", default="response", help="JSONL response field path")
+ quick_check_parser.add_argument("--out-dir", default=".redline/quick-check", help="directory for suite and reports")
+ quick_check_parser.add_argument("--max-cases", type=int, default=42, help="maximum representative suite cases")
+ quick_check_parser.add_argument("--all-cases", action="store_true", help="include every unique baseline record")
+ quick_check_parser.add_argument("--profile", choices=DIFF_PROFILES, default="strict", help="diff signal profile")
+ quick_check_parser.add_argument("--compact", action="store_true", help="print compact one-line-per-case output")
+ quick_check_parser.add_argument("--json", action="store_true", help="print machine-readable JSON")
+ quick_check_parser.add_argument("--open", action="store_true", help="open the HTML report in the default browser")
+ quick_check_parser.add_argument("--open-app", action="store_true", help="open the guided local app in the default browser")
+ quick_check_parser.add_argument(
+ "--fail-on",
+ default="regression,missing",
+ help="comma-separated statuses that produce exit code 1; use 'none' for report-only",
+ )
+ quick_check_parser.set_defaults(func=cmd_quick_check)
+
diff_parser = subparsers.add_parser("diff", help="compare candidate JSONL to a suite")
diff_parser.add_argument(
"paths",
@@ -456,10 +505,24 @@ def build_parser() -> argparse.ArgumentParser:
dashboard_parser.add_argument("--checkpoint", default=".redline/audit-checkpoint.json", help="audit checkpoint JSON path")
dashboard_parser.add_argument("--out", default=".redline/dashboard.html", help="dashboard HTML output path")
dashboard_parser.add_argument("--limit", type=int, default=20, help="recent reports/history entries to include; use 0 for all")
+ dashboard_parser.add_argument("--style", choices=("classic", "app"), default="classic", help="dashboard visual style")
dashboard_parser.add_argument("--open", action="store_true", help="open the dashboard in the default browser")
dashboard_parser.add_argument("--json", action="store_true", help="print machine-readable dashboard metadata")
dashboard_parser.set_defaults(func=cmd_dashboard)
+ app_parser = subparsers.add_parser("app", help="open the guided local redline app")
+ app_parser.add_argument("--reports-dir", default=".redline/reports", help="directory containing redline JSON reports")
+ app_parser.add_argument("--history", default=".redline/history.jsonl", help="history JSONL path")
+ app_parser.add_argument("--checkpoint", default=".redline/audit-checkpoint.json", help="audit checkpoint JSON path")
+ app_parser.add_argument("--out", default=".redline/app.html", help="app HTML output path")
+ app_parser.add_argument("--limit", type=int, default=20, help="recent reports/history entries to include; use 0 for all")
+ app_parser.add_argument("--demo", action="store_true", help="generate the public demo reports before opening the app")
+ app_parser.add_argument("--demo-out", default=".redline/demo", help="demo output directory for --demo")
+ app_parser.add_argument("--open", dest="open", action="store_true", default=True, help="open the local app in the default browser")
+ app_parser.add_argument("--no-open", dest="open", action="store_false", help="write the app HTML without opening a browser")
+ app_parser.add_argument("--json", action="store_true", help="print machine-readable app metadata")
+ app_parser.set_defaults(func=cmd_app)
+
eval_parser = subparsers.add_parser("eval", help="replay a suite with a local command")
eval_parser.add_argument("suite", nargs="?", help="suite JSON, or prompt manifest JSON from redline prompts")
eval_parser.add_argument("--config", default=DEFAULT_CONFIG_PATH, help="config path to read")
@@ -595,7 +658,7 @@ def cmd_init(args: argparse.Namespace) -> int:
print("Next:")
if not replay:
print("- Connect a runner: redline init --runner stdio --copy-runner --force")
- print(f"- Generate suite: redline suite path/to/log.jsonl --out {config['suite']}")
+ print(f"- Generate suite: {_shell_command('redline', 'suite', 'path/to/log.jsonl', '--out', config['suite'])}")
if replay:
print("- Run eval: redline eval")
print("- Check setup: redline doctor")
@@ -630,6 +693,36 @@ def cmd_doctor(args: argparse.Namespace) -> int:
return 0
+def cmd_status(args: argparse.Namespace) -> int:
+ if args.limit < 0:
+ raise ValueError("status --limit must be 0 or greater")
+ config = load_config(args.config)
+ suite_path = str(config.get("suite") or "redline-suite.json")
+ suite = None
+ suite_error = None
+ if Path(suite_path).exists():
+ try:
+ suite = _read_suite_or_manifest(suite_path)
+ except ValueError as exc:
+ suite_error = str(exc)
+ status = build_project_status(
+ config_path=args.config,
+ config=config,
+ suite=suite,
+ suite_error=suite_error,
+ suite_git_ignored=_is_git_ignored(suite_path),
+ reports_dir=args.reports_dir,
+ history_path=args.history,
+ checkpoint_path=args.checkpoint,
+ limit=args.limit,
+ )
+ if args.json:
+ print(json.dumps(status, indent=2, sort_keys=True))
+ else:
+ print(format_project_status(status), end="")
+ return 0
+
+
def cmd_sbom(args: argparse.Namespace) -> int:
sbom = build_sbom()
if args.out:
@@ -729,35 +822,202 @@ def cmd_judges(args: argparse.Namespace) -> int:
def cmd_import(args: argparse.Namespace) -> int:
+ if args.list_presets:
+ if args.json:
+ print(json.dumps({"presets": import_preset_rows()}, indent=2, sort_keys=True))
+ else:
+ print(format_import_presets(), end="")
+ return 0
+ if not args.path:
+ raise ValueError("import path is required unless --list-presets is used")
+ if args.detect:
+ result = detect_import_fields(args.path, limit=args.limit or 20)
+ if args.json:
+ print(json.dumps(result, indent=2, sort_keys=True))
+ else:
+ print(format_import_detection(result), end="")
+ return 0
+ auto_mapping = _auto_import_mapping(args) if args.auto_map else {}
+ preset_name = args.preset or str(auto_mapping.get("preset") or "")
+ preset = import_preset(preset_name) if preset_name else {}
+ input_field = args.input_field or str(preset.get("input_field") or "prompt")
+ output_field = args.output_field or str(preset.get("output_field") or "response")
+ if args.auto_map:
+ input_field = args.input_field or str(auto_mapping.get("input_field") or input_field)
+ output_field = args.output_field or str(auto_mapping.get("output_field") or output_field)
+ context_field = args.context_field if args.context_field is not None else _optional_preset_string(preset, "context_field")
+ id_field = args.id_field if args.id_field is not None else _optional_preset_string(preset, "id_field")
+ metadata_fields = _preset_string_list(preset, "metadata_fields")
+ metadata_fields.extend(args.metadata_field)
+ if args.preview is not None:
+ report = preview_jsonl_import(
+ args.path,
+ input_field=input_field,
+ output_field=output_field,
+ context_field=context_field,
+ id_field=id_field,
+ metadata_fields=metadata_fields,
+ limit=args.preview,
+ redact=not args.no_redact,
+ placeholder=args.redaction_placeholder,
+ )
+ report["preset"] = preset_name
+ report["auto_mapped"] = bool(args.auto_map)
+ if auto_mapping:
+ report["auto_mapping"] = auto_mapping
+ if args.json:
+ print(json.dumps(report, indent=2, sort_keys=True))
+ else:
+ _print_import_preview(report)
+ return 0
+ if not args.out:
+ raise ValueError("--out is required unless --list-presets or --preview is used")
report = import_jsonl_log(
args.path,
output=args.out,
- input_field=args.input_field,
- output_field=args.output_field,
- context_field=args.context_field,
- id_field=args.id_field,
- metadata_fields=args.metadata_field,
+ input_field=input_field,
+ output_field=output_field,
+ context_field=context_field,
+ id_field=id_field,
+ metadata_fields=metadata_fields,
limit=args.limit,
+ redact=not args.no_redact,
+ placeholder=args.redaction_placeholder,
)
+ report["preset"] = preset_name
+ report["auto_mapped"] = bool(args.auto_map)
+ if auto_mapping:
+ report["auto_mapping"] = auto_mapping
if args.json:
print(json.dumps(report, indent=2, sort_keys=True))
else:
print(f"Imported {report['records']} prompt-response pairs from {Path(str(report['source']))}.")
+ if report["auto_mapped"]:
+ _print_auto_mapping(report)
+ if report["preset"]:
+ print(f"Preset: {report['preset']}")
print(f"Mapped prompt: {report['input_field']}")
print(f"Mapped response: {report['output_field']}")
if report["context_field"]:
print(f"Appended context: {report['context_field']}")
if report["metadata_fields"]:
print(f"Copied metadata: {', '.join(report['metadata_fields'])}")
+ if report["redacted"]:
+ print(
+ "Redaction: "
+ f"best-effort common secrets/PII scanned; {report['redactions']} value(s) redacted"
+ )
+ print(f"Boundary: {REDACTION_BOUNDARY}")
+ patterns = report.get("redaction_patterns")
+ if isinstance(patterns, dict) and patterns:
+ print(f"Redaction hits: {', '.join(f'{key}={value}' for key, value in patterns.items())}")
+ else:
+ print("Redaction: disabled; review sensitive logs before sharing or committing")
print(f"Wrote {Path(str(report['output']))}.")
print()
print("Next:")
- print(f"- Generate suite: redline suite {Path(str(report['output']))} --out redline-suite.json")
+ print(
+ f"- Generate suite: "
+ f"{_shell_command('redline', 'suite', report['output'], '--out', 'redline-suite.json')}"
+ )
print("- Inspect cases: redline cases redline-suite.json")
print("- Compare candidate outputs: redline diff redline-suite.json path/to/candidate.jsonl")
return 0
+def _auto_import_mapping(args: argparse.Namespace) -> dict[str, object]:
+ detection = detect_import_fields(args.path, limit=args.limit or 20)
+ suggestions = detection.get("suggestions")
+ if not isinstance(suggestions, list) or not suggestions:
+ raise ValueError(
+ "auto-map could not find a prompt/response mapping; run "
+ "`redline import path/to/export.jsonl --detect` and pass fields manually"
+ )
+ best = suggestions[0]
+ if not isinstance(best, dict):
+ raise ValueError("auto-map produced an invalid mapping; pass --input-field and --output-field manually")
+ return {
+ "input_field": str(best.get("input_field") or ""),
+ "output_field": str(best.get("output_field") or ""),
+ "preset": str(best.get("preset") or ""),
+ "score": int(best.get("score") or 0),
+ "matches": int(best.get("matches") or 0),
+ "records_scanned": int(best.get("records_scanned") or 0),
+ }
+
+
+def _print_auto_mapping(report: Mapping[str, object]) -> None:
+ mapping = report.get("auto_mapping")
+ mapping = mapping if isinstance(mapping, Mapping) else {}
+ details = f"score {mapping.get('score', 0)}"
+ if mapping.get("preset"):
+ details += f"; preset {mapping['preset']}"
+ print(f"Auto-mapped: yes ({details})")
+
+
+def _print_import_preview(report: Mapping[str, object]) -> None:
+ print(f"Previewed {report['previewed']} prompt-response pairs from {Path(str(report['source']))}.")
+ if report.get("auto_mapped"):
+ _print_auto_mapping(report)
+ if report.get("preset"):
+ print(f"Preset: {report['preset']}")
+ print(f"Mapped prompt: {report['input_field']}")
+ print(f"Mapped response: {report['output_field']}")
+ if report["context_field"]:
+ print(f"Appended context: {report['context_field']}")
+ metadata_fields = report.get("metadata_fields")
+ if isinstance(metadata_fields, list) and metadata_fields:
+ print(f"Copied metadata: {', '.join(str(value) for value in metadata_fields)}")
+ if report["redacted"]:
+ print(
+ "Redaction: "
+ f"best-effort common secrets/PII scanned; {report['redactions']} value(s) redacted"
+ )
+ print(f"Boundary: {REDACTION_BOUNDARY}")
+ else:
+ print("Redaction: disabled; review sensitive logs before sharing or committing")
+ print()
+ print("Preview:")
+ rows = report.get("rows")
+ if isinstance(rows, list):
+ for index, row in enumerate(rows, start=1):
+ if not isinstance(row, Mapping):
+ continue
+ prompt = _truncate_preview(str(row.get("prompt") or ""))
+ response = _truncate_preview(str(row.get("response") or ""))
+ print(f"{index}. prompt: {prompt}")
+ print(f" response: {response}")
+ metadata = row.get("metadata")
+ if isinstance(metadata, Mapping) and metadata:
+ print(f" metadata: {json.dumps(metadata, sort_keys=True)}")
+ print()
+ print("No file written.")
+ print("Next:")
+ print(f"- Import when ready: {_shell_command('redline', 'import', report['source'], '--out', 'baseline.jsonl')}")
+
+
+def _truncate_preview(value: str, *, limit: int = 120) -> str:
+ compact = " ".join(value.split())
+ if len(compact) <= limit:
+ return compact
+ return compact[: limit - 3].rstrip() + "..."
+
+
+def _optional_preset_string(preset: Mapping[str, object], key: str) -> str | None:
+ value = preset.get(key)
+ if value is None:
+ return None
+ text = str(value).strip()
+ return text or None
+
+
+def _preset_string_list(preset: Mapping[str, object], key: str) -> list[str]:
+ value = preset.get(key)
+ if not isinstance(value, list):
+ return []
+ return [str(item) for item in value if str(item).strip()]
+
+
def cmd_watch(args: argparse.Namespace) -> int:
config = load_config(args.config)
input_field = str(_config_value(args.input_field, config, "input_field", "prompt"))
@@ -816,6 +1076,7 @@ def on_records(rows) -> None:
print(f"Skipped {result['skipped_duplicates']} duplicate records.")
if result.get("redactions"):
print(f"Redacted {result['redactions']} sensitive value(s).")
+ print(f"Boundary: {REDACTION_BOUNDARY}.")
print(f"{str(result['mode']).title()} {Path(str(result['output']))}.")
return 0
@@ -1008,11 +1269,16 @@ def cmd_suite(args: argparse.Namespace) -> int:
},
},
)
+ suite_health = suite_summary(suite)
+ readiness = suite_health.get("suite_readiness")
+ readiness = readiness if isinstance(readiness, dict) else {}
summary = suite["summary"]
print(f"Generated {summary['cases']} cases from {summary['records_seen']} records.")
if summary.get("duplicate_prompt_response_pairs"):
print(f"Skipped {summary['duplicate_prompt_response_pairs']} duplicate prompt-response pairs.")
- print(f"Detected {summary['clusters']} behavioral clusters.")
+ print(f"Detected {summary['clusters']} behavior-signature groups.")
+ print(f"Suite readiness: {readiness.get('label', 'unknown')} ({readiness.get('score', 0)}/100)")
+ print("Readiness scope: suite health, not model quality or candidate safety")
if summary.get("selection") != "all":
unique_pairs = int(summary.get("unique_prompt_response_pairs", summary["records_seen"]))
cases = int(summary["cases"])
@@ -1021,11 +1287,16 @@ def cmd_suite(args: argparse.Namespace) -> int:
f"Selected {cases} representative cases from {unique_pairs} unique prompt-response pairs. "
"Use --all-cases for exhaustive coverage or redline suite add for must-cover edge cases."
)
+ improvement_steps = suite_health.get("next_steps")
+ if isinstance(improvement_steps, list) and improvement_steps:
+ print("Improve suite:")
+ for step in improvement_steps[:2]:
+ print(f"- {step}")
print(f"Wrote {Path(output)}.")
print()
print("Next:")
- print(f"- Inspect cases: redline cases {Path(output)}")
- print(f"- Compare a candidate log: redline diff {Path(output)} path/to/candidate.jsonl")
+ print(f"- Inspect cases: {_shell_command('redline', 'cases', output)}")
+ print(f"- Compare a candidate log: {_shell_command('redline', 'diff', output, 'path/to/candidate.jsonl')}")
print("- Configure replay when ready: redline init --runner stdio --copy-runner")
return 0
@@ -1089,9 +1360,9 @@ def cmd_suite_add(args: argparse.Namespace) -> int:
print(f"Added {rules} requirement rule(s).")
print()
print("Next:")
- print(f"- Inspect case: redline case {Path(output)} {case['id']}")
- print(f"- Validate suite: redline validate {Path(output)}")
- print(f"- Compare candidate: redline diff {Path(output)} path/to/candidate.jsonl")
+ print(f"- Inspect case: {_shell_command('redline', 'case', output, case['id'])}")
+ print(f"- Validate suite: {_shell_command('redline', 'validate', output)}")
+ print(f"- Compare candidate: {_shell_command('redline', 'diff', output, 'path/to/candidate.jsonl')}")
return 0
@@ -1199,6 +1470,86 @@ def cmd_validate(args: argparse.Namespace) -> int:
return 0
+def cmd_quick_check(args: argparse.Namespace) -> int:
+ if args.all_cases and args.max_cases != 42:
+ raise ValueError("--all-cases cannot be combined with --max-cases")
+ baseline_records = read_jsonl_records(args.baseline, args.input_field, args.output_field)
+ candidate_records = read_jsonl_records(args.candidate, args.input_field, args.output_field)
+ suite = build_suite(
+ baseline_records,
+ source=args.baseline,
+ input_field=args.input_field,
+ output_field=args.output_field,
+ max_cases=args.max_cases,
+ all_cases=args.all_cases,
+ )
+ out_dir = Path(args.out_dir)
+ suite_path = out_dir / "suite.json"
+ report_json = out_dir / "diff.json"
+ report_md = out_dir / "diff.md"
+ report_html = out_dir / "diff.html"
+ app_html = out_dir / "app.html"
+ write_json(suite_path, suite)
+
+ result = compare_suite_to_candidate(suite, candidate_records, profile=args.profile)
+ result["suite"] = str(suite_path)
+ result["candidate"] = str(args.candidate)
+ result["artifacts"] = _artifact_paths(
+ {
+ "json": str(report_json),
+ "markdown": str(report_md),
+ "html": str(report_html),
+ "app": str(app_html),
+ }
+ )
+ write_json(report_json, result)
+ write_text(report_md, format_markdown_report(result, title="redline quick-check"))
+ write_text(report_html, format_html_report(result, title="redline quick-check"))
+ dashboard = build_dashboard(
+ reports_dir=out_dir,
+ history_path=out_dir / "history.jsonl",
+ checkpoint_path=out_dir / "audit-checkpoint.json",
+ )
+ write_text(app_html, format_dashboard_html(dashboard, title="redline quick-check app", output_path=app_html, style="app"))
+ if args.open:
+ webbrowser.open(report_html.resolve().as_uri())
+ if args.open_app:
+ webbrowser.open(app_html.resolve().as_uri())
+
+ fail_on = parse_fail_on(args.fail_on)
+ if args.json:
+ print(json.dumps(result, indent=2, sort_keys=True))
+ elif args.compact:
+ print(format_compact_report(result, title="redline quick-check"), end="")
+ else:
+ print(format_report(result, title="redline quick-check"), end="")
+ if not args.json:
+ summary = suite["summary"]
+ print()
+ print("Artifacts")
+ print(f"- Suite: {suite_path}")
+ print(f"- JSON report: {report_json}")
+ print(f"- Markdown: {report_md}")
+ print(f"- HTML report: {report_html}")
+ print(f"- App: {app_html}")
+ print()
+ print("Suite coverage")
+ print(f"- Cases: {summary['cases']}/{summary['unique_prompt_response_pairs']} unique prompt-response pairs")
+ covered_clusters = round(int(summary["clusters"]) * float(summary.get("cluster_coverage") or 0.0))
+ print(f"- Behavior groups: {covered_clusters}/{summary['clusters']} represented")
+ print()
+ print("Next:")
+ print(f"- Open HTML report: {report_html}")
+ print(f"- Open guided app: {app_html}")
+ print(f"- Inspect cases: {_shell_command('redline', 'cases', suite_path)}")
+ print(f"- Make this persistent: {_shell_command('redline', 'suite', args.baseline, '--out', 'redline-suite.json')}")
+ if args.open:
+ print("- Opened HTML report in the default browser.")
+ if args.open_app:
+ print("- Opened guided app in the default browser.")
+ return 1 if should_fail(result, fail_on) else 0
+
+
def cmd_diff(args: argparse.Namespace) -> int:
config = load_config(args.config)
suite_path, candidate_path = _suite_candidate_args(args.paths, config)
@@ -1287,15 +1638,50 @@ def cmd_history(args: argparse.Namespace) -> int:
def cmd_dashboard(args: argparse.Namespace) -> int:
+ return _write_dashboard_artifact(
+ args,
+ style=args.style,
+ title="redline dashboard",
+ noun="dashboard",
+ open_message="Opened dashboard in the default browser.",
+ )
+
+
+def cmd_app(args: argparse.Namespace) -> int:
+ if args.demo:
+ run_demo(args.demo_out, public=True)
+ if args.reports_dir == ".redline/reports":
+ args.reports_dir = str(Path(args.demo_out) / "reports")
+ if not args.json:
+ print(f"Generated public demo evidence in {Path(args.demo_out)}.")
+ return _write_dashboard_artifact(
+ args,
+ style="app",
+ title="redline app",
+ noun="app",
+ open_message="Opened redline app in the default browser.",
+ )
+
+
+def _write_dashboard_artifact(
+ args: argparse.Namespace,
+ *,
+ style: str,
+ title: str,
+ noun: str,
+ open_message: str,
+) -> int:
dashboard = build_dashboard(
reports_dir=args.reports_dir,
history_path=args.history,
checkpoint_path=args.checkpoint,
limit=args.limit,
)
- write_text(args.out, format_dashboard_html(dashboard, output_path=args.out))
- if args.open:
+ write_text(args.out, format_dashboard_html(dashboard, title=title, output_path=args.out, style=style))
+ opened = False
+ if args.open and not args.json:
webbrowser.open(Path(args.out).resolve().as_uri())
+ opened = True
if args.json:
print(json.dumps({**dashboard, "output": args.out}, indent=2, sort_keys=True))
else:
@@ -1306,10 +1692,16 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
print(f"Checkpoint: {'yes' if dashboard.get('checkpoint') else 'no'}")
print(f"Notices: {len(dashboard.get('notices', []))}")
print(f"Warnings: {len(dashboard.get('errors', []))}")
- if args.open:
- print("Opened dashboard in the default browser.")
+ if opened:
+ print(open_message)
else:
print(f"Open: {Path(args.out)}")
+ if noun == "app":
+ print()
+ print("Next:")
+ print("- Import logs: redline import path/to/export.jsonl --detect")
+ print("- Generate suite: redline suite .redline/logs/baseline.jsonl --out redline-suite.json")
+ print("- Run eval: redline eval --compact")
return 0
@@ -1525,14 +1917,14 @@ def cmd_mark(args: argparse.Namespace) -> int:
print(f"Marked {case_id} as {args.status} in {Path(output)}.")
print()
print("Next:")
- print(f"- Validate suite: redline validate {Path(output)}")
+ print(f"- Validate suite: {_shell_command('redline', 'validate', output)}")
if args.status == "expected":
print(
- f"- Promote reviewed output: redline accept {Path(output)} {case_id} "
- '--candidate path/to/candidate.jsonl --note "accepted prompt change"'
+ f"- Promote reviewed output: "
+ f"{_shell_command('redline', 'accept', output, case_id, '--candidate', 'path/to/candidate.jsonl', '--note', 'accepted prompt change')}"
)
else:
- print(f"- Re-run diff: redline diff {Path(output)} path/to/candidate.jsonl")
+ print(f"- Re-run diff: {_shell_command('redline', 'diff', output, 'path/to/candidate.jsonl')}")
return 0
@@ -1692,6 +2084,10 @@ def _emit_result(
print(format_compact_report(result, title=title), end="")
else:
print(format_report(result, title=title), end="")
+ if not args.json and out_html:
+ print()
+ print(f"Open HTML report: {Path(out_html)}")
+ print(f"Open app: redline app --reports-dir {shlex.quote(str(Path(out_html).parent))}")
exit_code = 1 if should_fail(result, fail_on) else 0
if audit_event is not None:
@@ -1886,7 +2282,7 @@ def _read_suite_or_manifest(path: str) -> dict[str, object]:
raise ValueError(
f"{path} looks like raw JSONL logs, but this command expects a redline suite JSON "
"or prompt manifest. Build a suite first: "
- f"redline suite {path} --out redline-suite.json"
+ f"{_shell_command('redline', 'suite', path, '--out', 'redline-suite.json')}"
) from exc
raise
@@ -2235,4 +2631,8 @@ def _prompts_regenerate_command(args: argparse.Namespace) -> str:
parts = ["redline", "prompts", str(args.path), "--suite-dir", str(args.suite_dir), "--out", str(args.out)]
for extension in args.ext:
parts.extend(["--ext", str(extension)])
- return " ".join(shlex.quote(part) for part in parts)
+ return _shell_command(*parts)
+
+
+def _shell_command(*parts: object) -> str:
+ return " ".join(shlex.quote(str(part)) for part in parts)
diff --git a/redline/clusters.py b/redline/clusters.py
index e3caa31..8335481 100644
--- a/redline/clusters.py
+++ b/redline/clusters.py
@@ -46,11 +46,11 @@ def format_cluster_report(suite: dict[str, Any]) -> str:
lines = [
"redline cluster",
"",
- f"Identified {report['clusters']} behavioral clusters from {report['unique_prompt_response_pairs']} unique pairs.",
+ f"Identified {report['clusters']} behavior-signature groups from {report['unique_prompt_response_pairs']} unique pairs.",
f"Records seen: {report['records_seen']} Duplicate pairs: {report['duplicate_prompt_response_pairs']}",
- f"High-risk clusters: {report['high_risk_clusters']}",
- f"High-variance clusters: {report['high_variance_clusters']}",
- f"Failure-pattern clusters: {report['failure_pattern_clusters']}",
+ f"High-risk groups: {report['high_risk_clusters']}",
+ f"High-variance groups: {report['high_variance_clusters']}",
+ f"Failure-pattern groups: {report['failure_pattern_clusters']}",
f"Suggested eval suite: {report['suggested_cases']} representative cases.",
f"Case coverage: {report['suggested_cases']}/{report['unique_prompt_response_pairs']} ({_percent(report['case_coverage'])})",
"",
diff --git a/redline/dashboard.py b/redline/dashboard.py
index d9e673f..1392e06 100644
--- a/redline/dashboard.py
+++ b/redline/dashboard.py
@@ -7,7 +7,7 @@
from typing import Any
from urllib.parse import quote
-from .diff import TRUST_SCOPE
+from .diff import TRUST_SCOPE, case_impact
from .history import history_trend, read_history
from .io import read_json
@@ -50,6 +50,7 @@ def format_dashboard_html(
*,
title: str = "redline dashboard",
output_path: str | Path | None = None,
+ style: str = "classic",
) -> str:
reports = dashboard.get("reports")
if not isinstance(reports, list):
@@ -78,6 +79,24 @@ def format_dashboard_html(
raw_trend = dashboard.get("trend")
trend = raw_trend if isinstance(raw_trend, dict) else history_trend(list(reversed(history)))
latest = reports[0] if reports and isinstance(reports[0], dict) else {}
+ if style == "app":
+ return _format_dashboard_app_html(
+ dashboard,
+ title=title,
+ output_path=output_path,
+ reports=reports,
+ history=history,
+ benchmarks=benchmarks,
+ errors=errors,
+ notices=notices,
+ owners=owners,
+ trust=trust,
+ checkpoint=checkpoint,
+ trend=trend,
+ latest=latest,
+ )
+ if style != "classic":
+ raise ValueError(f"unknown dashboard style: {style}")
return "\n".join(
[
"",
@@ -120,6 +139,786 @@ def format_dashboard_html(
)
+def _format_dashboard_app_html(
+ dashboard: dict[str, Any],
+ *,
+ title: str,
+ output_path: str | Path | None,
+ reports: list[Any],
+ history: list[Any],
+ benchmarks: list[Any],
+ errors: list[Any],
+ notices: list[Any],
+ owners: list[Any],
+ trust: dict[str, Any],
+ checkpoint: dict[str, Any],
+ trend: dict[str, Any],
+ latest: dict[str, Any],
+) -> str:
+ summary = _app_dict(latest.get("summary"))
+ suite_summary = _app_dict(latest.get("suite_summary"))
+ decision = _app_dict(latest.get("decision"))
+ active = _blocking_count(summary)
+ changed = _changed_count(summary)
+ status_class = "red" if active else "amber" if changed else "green"
+ status_text = "Regressions" if active else "Review" if changed else "Clear"
+ review_cases = _app_list(latest.get("review_cases"))
+ warnings = _app_list(latest.get("warnings"))
+ if notices:
+ warnings = [*(str(item.get("message") or "") for item in notices if isinstance(item, dict)), *warnings]
+ alert_count = active + len([warning for warning in warnings if str(warning).strip()])
+ return "\n".join(
+ [
+ "",
+ '',
+ "",
+ '',
+ '',
+ f"{_h(title)}",
+ "",
+ "",
+ "",
+ '
'
+ f'{_app_metric("Next command", str(next_item["stage"]), next_item["title"], "amber" if next_item["tone"] == "amber" else "red" if next_item["tone"] == "red" else "green")}'
+ f'{_app_metric("Reports", str(len(reports)), "local evidence files", "green" if reports else "amber")}'
+ f'{_app_metric("Blocking", str(blocking), "cases to inspect", "red" if blocking else "green")}'
+ f'{_app_metric("Automation", "copy-only", "dashboard never runs shell commands", "blue")}'
+ "
"
+ f'{_app_alert("blue", "Command center.", "Copy a command, run it in your terminal, then refresh this dashboard. redline stays local-first and does not execute shell commands from the browser.")}'
+ f'
{_app_icon("workflow")} Next best actionstate-aware
{_app_workflow_card(next_item, featured=True)}
'
+ f'
{cards}
'
+ ""
+ )
+
+
+def _app_workflow_items(
+ *,
+ dashboard: dict[str, Any],
+ latest: dict[str, Any],
+ reports: list[Any],
+ benchmarks: list[Any],
+ history: list[Any],
+ checkpoint: dict[str, Any],
+ review_cases: list[Any],
+) -> list[dict[str, str]]:
+ reports_dir = str(dashboard.get("reports_dir") or ".redline/reports")
+ history_path = str(dashboard.get("history_path") or ".redline/history.jsonl")
+ checkpoint_path = str(dashboard.get("checkpoint_path") or ".redline/audit-checkpoint.json")
+ latest_report = str(latest.get("path") or f"{reports_dir}/eval.json")
+ suite = _app_latest_suite(latest=latest, benchmarks=benchmarks)
+ first_case = next((item for item in review_cases if isinstance(item, dict)), {})
+ case_id = str(first_case.get("case_id") or first_case.get("suite_case_id") or "")
+ case_suite = str(first_case.get("suite") or suite)
+ blocking = _blocking_count(_app_dict(latest.get("summary")))
+ changed = _changed_count(_app_dict(latest.get("summary")))
+ has_reports = bool(reports)
+ has_history = bool(history)
+ has_benchmarks = bool(benchmarks)
+ has_checkpoint = bool(checkpoint)
+ return [
+ {
+ "stage": "0",
+ "title": "Prove the loop",
+ "body": "Run the bundled regression proof before connecting private logs.",
+ "command": "redline demo --public --compact",
+ "tone": "green" if has_reports else "amber",
+ },
+ {
+ "stage": "1",
+ "title": "Get logs in",
+ "body": "Detect fields first. Redaction is on by default when writing normalized JSONL.",
+ "command": "redline import path/to/export.jsonl --detect",
+ "tone": "green" if has_reports else "amber",
+ },
+ {
+ "stage": "2",
+ "title": "Preview import",
+ "body": "Check a few mapped rows before creating a baseline log.",
+ "command": "redline import path/to/export.jsonl --auto-map --preview 3",
+ "tone": "green" if has_reports else "amber",
+ },
+ {
+ "stage": "3",
+ "title": "Generate suite",
+ "body": "Turn normalized baseline logs into representative regression cases.",
+ "command": f"redline suite .redline/logs/baseline.jsonl --out {shell_quote(suite)}",
+ "tone": "green" if has_reports else "amber",
+ },
+ {
+ "stage": "4",
+ "title": "Run eval",
+ "body": "Replay your changed prompt or runner and write reports for the dashboard.",
+ "command": "redline eval --compact",
+ "tone": "green" if has_reports else "amber",
+ },
+ {
+ "stage": "5",
+ "title": "Review first case",
+ "body": "Inspect the exact baseline, candidate, and concrete regression reason.",
+ "command": f"redline case {shell_quote(case_suite)} {shell_quote(case_id)}",
+ "tone": "red" if blocking else "amber" if changed else "green",
+ },
+ {
+ "stage": "6",
+ "title": "Record trend",
+ "body": "Add this run to history so prompt quality is visible over time.",
+ "command": f"redline history {shell_quote(latest_report)} --label prompt-v2 --out {shell_quote(history_path)} --out-md .redline/history.md",
+ "tone": "green" if has_history else "amber",
+ },
+ {
+ "stage": "7",
+ "title": "Add runtime evidence",
+ "body": "Estimate worst-case eval time and attach local benchmark evidence.",
+ "command": f"redline budget {shell_quote(suite)} --measure-local --out-json {shell_quote(reports_dir + '/benchmark.json')} --out-md {shell_quote(reports_dir + '/benchmark.md')}",
+ "tone": "green" if has_benchmarks else "amber",
+ },
+ {
+ "stage": "8",
+ "title": "Verify audit trail",
+ "body": "Write a checkpoint for stronger evidence that local run history was not silently changed.",
+ "command": f"redline audit --verify --out-checkpoint {shell_quote(checkpoint_path)}",
+ "tone": "green" if has_checkpoint else "amber",
+ },
+ {
+ "stage": "9",
+ "title": "Refresh dashboard",
+ "body": "Regenerate the app dashboard after each run.",
+ "command": f"redline app --reports-dir {shell_quote(reports_dir)} --history {shell_quote(history_path)}",
+ "tone": "green",
+ },
+ ]
+
+
+def _app_latest_suite(*, latest: dict[str, Any], benchmarks: list[Any]) -> str:
+ suite = str(latest.get("suite") or "").strip()
+ if suite:
+ return suite
+ prompt_evals = latest.get("prompt_evals")
+ if isinstance(prompt_evals, list):
+ for item in prompt_evals:
+ if isinstance(item, dict) and str(item.get("suite") or "").strip():
+ return str(item.get("suite"))
+ for item in benchmarks:
+ if isinstance(item, dict) and str(item.get("suite") or "").strip() and str(item.get("suite")) != "-":
+ return str(item.get("suite"))
+ return "redline-suite.json"
+
+
+def _app_workflow_card(item: dict[str, str], *, featured: bool = False) -> str:
+ tone = item.get("tone") or "blue"
+ command = item.get("command") or ""
+ featured_class = " featured" if featured else ""
+ return (
+ f'
"
+ f'{_app_alert("blue", f"Trend: {direction}", str(trend.get("summary") or "Record history entries to see whether prompt quality is improving or regressing."))}'
+ f'
"
+ )
+
+
+def _app_warning_banner(warnings: list[Any], *, status_class: str) -> str:
+ if warnings:
+ text = " ".join(_preview(str(warning), 160) for warning in warnings[:2])
+ return _app_alert("amber", "Calibration warning.", text)
+ if status_class == "red":
+ return _app_alert("red", "Blocking regressions detected.", "Fix or mark expected changes before shipping.")
+ return _app_alert("green", "No blocking structural regressions in latest report.", "Review semantic risks separately.")
+
+
+def _app_review_rows(items: list[Any], *, empty: str = "No review cases found.") -> str:
+ rows = []
+ for item in items[:8]:
+ if not isinstance(item, dict):
+ continue
+ status = str(item.get("status") or item.get("kind") or "review")
+ case_id = str(item.get("case_id") or item.get("suite_case_id") or "-")
+ prompt = _preview(str(item.get("prompt") or item.get("reason") or "Review case"), 96)
+ reason = _preview(str(item.get("reason") or "; ".join(str(value) for value in item.get("reasons", []) if value) or ""), 120)
+ impact = _preview(str(item.get("impact") or ""), 140)
+ impact_html = f'
Why this matters: {_h(impact)}
' if impact else ""
+ tone = "red" if status in {"regression", "missing"} else "amber" if status == "changed" else "green"
+ rows.append(
+ '
' for label, value in rows)
+
+
+def _app_owner_rows(owners: list[Any]) -> str:
+ rows = []
+ for owner in owners[:8]:
+ if not isinstance(owner, dict):
+ continue
+ rows.append(
+ '
'
+ f'{_h(str(owner.get("owner") or "unowned"))}'
+ f'{_safe_int(owner.get("reviewable"))}'
+ "
'
+ for label, value, tone in rows
+ )
+ return f'
{_app_icon("breakdown")} {_h(title)}
{body}
'
+
+
+def _app_fix_card(review_cases: list[Any]) -> str:
+ first = next((item for item in review_cases if isinstance(item, dict)), {})
+ case_id = str(first.get("case_id") or first.get("suite_case_id") or "case id")
+ suite = str(first.get("suite") or "redline-suite.json")
+ command = f"redline case {shell_quote(suite)} {shell_quote(case_id)}" if case_id != "case id" else "redline cases redline-suite.json"
+ body = (
+ '
'
+ 'Recommended local loop'
+ f'1. Inspect: {_h(command)}'
+ '2. Fix prompt or runner output'
+ '3. Mark expected only when the change is intentional'
+ '
'
+ )
+ return f'
{_app_icon("fix")} Suggested fix
{body}
'
+
+
+def _app_report_chart(reports: list[Any]) -> str:
+ points = []
+ for index, report in enumerate(reversed([item for item in reports[:12] if isinstance(item, dict)])):
+ summary = _app_dict(report.get("summary"))
+ cases = max(_safe_int(summary.get("cases")), 1)
+ blocking = _blocking_count(summary)
+ pass_rate = max(0.0, min(1.0, (cases - blocking) / cases))
+ x = 24 + index * 74
+ y = 132 - int(pass_rate * 96)
+ points.append((x, y))
+ if not points:
+ return '
No report chart yet. Run redline diff or eval to populate trend evidence.
'
+ path = " ".join(f"{x},{y}" for x, y in points)
+ dots = "".join(f'' for x, y in points)
+ return (
+ '"
+ '
')
+ impact = _why_this_matters(item)
+ if impact:
+ lines.append(f'
Why this matters: {_h(impact)}
')
if isinstance(reasons, list) and reasons:
lines.append('
')
for reason in reasons:
diff --git a/redline/status.py b/redline/status.py
new file mode 100644
index 0000000..da6e260
--- /dev/null
+++ b/redline/status.py
@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+from pathlib import Path
+from shlex import quote
+from typing import Any
+
+from .dashboard import build_dashboard
+from .diff import TRUST_SCOPE
+from .doctor import doctor_report
+
+
+def build_project_status(
+ *,
+ config_path: str,
+ config: dict[str, Any],
+ suite: dict[str, Any] | None,
+ suite_error: str | None = None,
+ suite_git_ignored: bool = False,
+ reports_dir: str | Path = ".redline/reports",
+ history_path: str | Path = ".redline/history.jsonl",
+ checkpoint_path: str | Path = ".redline/audit-checkpoint.json",
+ limit: int = 20,
+) -> dict[str, Any]:
+ doctor = doctor_report(
+ config_path=config_path,
+ config=config,
+ suite=suite,
+ suite_error=suite_error,
+ suite_git_ignored=suite_git_ignored,
+ )
+ dashboard = build_dashboard(
+ reports_dir=reports_dir,
+ history_path=history_path,
+ checkpoint_path=checkpoint_path,
+ limit=limit,
+ )
+ latest = _first_dict(dashboard.get("reports"))
+ summary = _dict(latest.get("summary"))
+ review_cases = _list(latest.get("review_cases"))
+ first_review_case = _first_dict(review_cases)
+ blocking = _count(summary, "regression") + _count(summary, "missing")
+ changed = _count(summary, "changed")
+ suite_path = str(config.get("suite") or "redline-suite.json")
+ state, message, next_command = _state_and_next(
+ doctor=doctor,
+ latest=latest,
+ summary=summary,
+ review_cases=review_cases,
+ blocking=blocking,
+ changed=changed,
+ suite_path=suite_path,
+ reports_dir=str(reports_dir),
+ history_path=str(history_path),
+ )
+ return {
+ "version": "0.1",
+ "state": state,
+ "message": message,
+ "next_command": next_command,
+ "app_command": _app_command(
+ reports_dir=str(reports_dir),
+ history_path=str(history_path),
+ checkpoint_path=str(checkpoint_path),
+ ),
+ "config_path": config_path,
+ "suite_path": suite_path,
+ "reports_dir": str(reports_dir),
+ "history_path": str(history_path),
+ "checkpoint_path": str(checkpoint_path),
+ "doctor": {
+ "errors": int(doctor.get("errors") or 0),
+ "warnings": int(doctor.get("warnings") or 0),
+ "checks": doctor.get("checks") if isinstance(doctor.get("checks"), list) else [],
+ "next_steps": doctor.get("next_steps") if isinstance(doctor.get("next_steps"), list) else [],
+ },
+ "reports": len(_list(dashboard.get("reports"))),
+ "history": len(_list(dashboard.get("history"))),
+ "benchmarks": len(_list(dashboard.get("benchmarks"))),
+ "checkpoint": bool(dashboard.get("checkpoint")),
+ "latest_report": latest,
+ "first_review_case": _status_review_case(first_review_case, fallback_suite=suite_path),
+ "summary": summary,
+ "blocking": blocking,
+ "changed": changed,
+ "scope": TRUST_SCOPE,
+ }
+
+
+def format_project_status(status: dict[str, Any]) -> str:
+ latest = _dict(status.get("latest_report"))
+ summary = _dict(status.get("summary"))
+ doctor = _dict(status.get("doctor"))
+ checks = _list(doctor.get("checks"))
+ first_review_case = _dict(status.get("first_review_case"))
+ lines = [
+ "redline status",
+ "",
+ f"State: {str(status.get('state') or '').upper()} - {status.get('message')}",
+ f"Next: {status.get('next_command')}",
+ f"App: {status.get('app_command')}",
+ "",
+ "Evidence",
+ f"- Config: {_check_line(checks, 'config')}",
+ f"- Suite: {_check_line(checks, 'suite')}",
+ f"- Replay: {_check_line(checks, 'replay')}",
+ f"- Reports: {status.get('reports')} in {status.get('reports_dir')}",
+ f"- History: {status.get('history')} in {status.get('history_path')}",
+ f"- Benchmarks: {status.get('benchmarks')}",
+ f"- Audit checkpoint: {'yes' if status.get('checkpoint') else 'no'}",
+ f"- Doctor: errors={doctor.get('errors', 0)} warnings={doctor.get('warnings', 0)}",
+ ]
+ if latest:
+ lines.extend(
+ [
+ "",
+ "Latest report",
+ f"- File: {latest.get('path')}",
+ f"- Summary: {_summary_line(summary)}",
+ f"- Decision: {_decision_line(latest)}",
+ ]
+ )
+ if first_review_case:
+ lines.extend(
+ [
+ "",
+ "First review case",
+ f"- Case: {first_review_case.get('case_id')} ({first_review_case.get('status')})",
+ f"- Reason: {first_review_case.get('reason') or 'review report details'}",
+ f"- Impact: {first_review_case.get('impact') or 'review the case before shipping'}",
+ f"- Command: {first_review_case.get('command')}",
+ ]
+ )
+ else:
+ lines.extend(["", "Latest report", "- No redline report found yet."])
+ lines.extend(["", "Trust boundary", f"- {status.get('scope') or TRUST_SCOPE}"])
+ next_steps = _list(doctor.get("next_steps"))
+ if next_steps:
+ lines.extend(["", "Doctor next steps"])
+ lines.extend(f"- {step}" for step in next_steps)
+ return "\n".join(str(line) for line in lines) + "\n"
+
+
+def _state_and_next(
+ *,
+ doctor: dict[str, Any],
+ latest: dict[str, Any],
+ summary: dict[str, Any],
+ review_cases: list[Any],
+ blocking: int,
+ changed: int,
+ suite_path: str,
+ reports_dir: str,
+ history_path: str,
+) -> tuple[str, str, str]:
+ checks = {str(check.get("name") or ""): check for check in _list(doctor.get("checks")) if isinstance(check, dict)}
+ doctor_steps = [str(step) for step in _list(doctor.get("next_steps")) if str(step).strip()]
+ config = _dict(checks.get("config"))
+ suite = _dict(checks.get("suite"))
+ if config.get("status") == "warn":
+ return "setup", "project is not initialized", _first_step(doctor_steps, "redline init --runner stdio --copy-runner")
+ if suite.get("status") in {"warn", "error"}:
+ return "setup", "no usable suite baseline yet", _first_step(
+ doctor_steps,
+ f"redline suite path/to/log.jsonl --out {quote(suite_path)}",
+ )
+ if not latest:
+ return "ready", "suite exists; run the first eval or diff", "redline eval --compact"
+ first_case = _first_dict(review_cases)
+ case_command = _case_command(first_case, fallback_suite=suite_path)
+ if blocking:
+ return "blocked", f"latest report has {blocking} blocking case(s)", case_command
+ if changed:
+ return "review", f"latest report has {changed} changed case(s)", case_command
+ if int(doctor.get("errors") or 0):
+ return "setup", "doctor found setup errors", _first_step(doctor_steps, "redline doctor")
+ if not Path(history_path).exists():
+ report_path = str(latest.get("path") or f"{reports_dir}/eval.json")
+ return "record", "latest report is clear; record it in history", (
+ f"redline history {quote(report_path)} --label prompt-v2 --out {quote(history_path)}"
+ )
+ return "ready", "latest report has no blocking structural regressions", (
+ f"redline app --reports-dir {quote(reports_dir)} --history {quote(history_path)}"
+ )
+
+
+def _case_command(case: dict[str, Any], *, fallback_suite: str) -> str:
+ case_id = str(case.get("suite_case_id") or case.get("case_id") or "")
+ suite = str(case.get("suite") or fallback_suite)
+ return f"redline case {quote(suite)} {quote(case_id)}"
+
+
+def _app_command(*, reports_dir: str, history_path: str, checkpoint_path: str) -> str:
+ return (
+ f"redline app --reports-dir {quote(reports_dir)} "
+ f"--history {quote(history_path)} --checkpoint {quote(checkpoint_path)}"
+ )
+
+
+def _status_review_case(case: dict[str, Any], *, fallback_suite: str) -> dict[str, str]:
+ if not case:
+ return {}
+ command = _case_command(case, fallback_suite=fallback_suite)
+ return {
+ "case_id": str(case.get("case_id") or case.get("suite_case_id") or ""),
+ "status": str(case.get("status") or ""),
+ "reason": str(case.get("reason") or ""),
+ "impact": str(case.get("impact") or ""),
+ "command": command,
+ }
+
+
+def _summary_line(summary: dict[str, Any]) -> str:
+ if not summary:
+ return "none"
+ keys = ("cases", "regression", "missing", "changed", "neutral", "improved", "accepted", "ignored")
+ return " ".join(f"{key}={_count(summary, key)}" for key in keys if key in summary)
+
+
+def _decision_line(report: dict[str, Any]) -> str:
+ decision = _dict(report.get("decision"))
+ return str(decision.get("recommended_action") or "review report details")
+
+
+def _check_line(checks: list[Any], name: str) -> str:
+ for item in checks:
+ if isinstance(item, dict) and item.get("name") == name:
+ return f"{item.get('status')} - {item.get('message')}"
+ return "not checked"
+
+
+def _first_step(steps: list[str], fallback: str) -> str:
+ if not steps:
+ return fallback
+ step = steps[0]
+ if ": redline " in step:
+ return "redline " + step.split(": redline ", 1)[1]
+ return step
+
+
+def _first_dict(value: Any) -> dict[str, Any]:
+ items = _list(value)
+ first = items[0] if items else {}
+ return first if isinstance(first, dict) else {}
+
+
+def _dict(value: Any) -> dict[str, Any]:
+ return value if isinstance(value, dict) else {}
+
+
+def _list(value: Any) -> list[Any]:
+ return value if isinstance(value, list) else []
+
+
+def _count(summary: dict[str, Any], key: str) -> int:
+ try:
+ return int(summary.get(key) or 0)
+ except (TypeError, ValueError):
+ return 0
diff --git a/redline/suite.py b/redline/suite.py
index f27851e..c5ec2fa 100644
--- a/redline/suite.py
+++ b/redline/suite.py
@@ -16,6 +16,24 @@
ClusterInfo = dict[str, Any]
SUITE_SCHEMA_URL = "https://raw.githubusercontent.com/gowtham0992/redline/main/redline-suite.schema.json"
PROMPT_DIVERSITY_EDGE_TARGET = 8
+SELECTION_METHODOLOGY_VERSION = "behavior-signature-v1"
+SELECTION_METHODOLOGY = {
+ "name": "deterministic behavior-signature grouping",
+ "version": SELECTION_METHODOLOGY_VERSION,
+ "trust_scope": "structural checks only; review factual, tone, hallucination, policy, and reasoning risks separately",
+ "case_selection": [
+ "one representative per behavior-signature group",
+ "high-risk groups first when case budget is tight",
+ "high-variance edge cases when budget remains",
+ "prompt-diverse samples from large groups when budget remains",
+ ],
+}
+
+
+def _ratio(part: int, total: int) -> float | None:
+ if total <= 0:
+ return None
+ return part / total
def build_suite(
@@ -48,7 +66,9 @@ def build_suite(
if all_cases
else _select_representatives(grouped, max_cases, feature_cache, cluster_infos)
)
+ selected_clusters = {signatures[id(record)] for record, _ in selected}
non_ascii_records = sum(1 for record in unique_records if _has_non_ascii(record.prompt) or _has_non_ascii(record.response))
+ stochastic_prompt_groups = _stochastic_prompt_groups(unique_records)
cases = []
for index, (record, selection_reason) in enumerate(selected, 1):
signature = signatures[id(record)]
@@ -99,12 +119,15 @@ def build_suite(
"source": str(source),
"input_field": input_field,
"output_field": output_field,
+ "methodology": dict(SELECTION_METHODOLOGY),
"summary": {
"records_seen": len(records),
"unique_prompt_response_pairs": len(unique_records),
"duplicate_prompt_response_pairs": len(records) - len(unique_records),
"clusters": len(grouped),
"cases": len(cases),
+ "case_coverage": _ratio(len(cases), len(unique_records)),
+ "cluster_coverage": _ratio(len(selected_clusters), len(grouped)),
"max_cases": len(unique_records) if all_cases else max_cases,
"selection": "all" if all_cases else "representative",
"high_risk_clusters": _risk_count(cluster_infos, "high"),
@@ -113,6 +136,7 @@ def build_suite(
"failure_pattern_clusters": sum(1 for info in cluster_infos.values() if info["failure_patterns"]),
"prompt_diversity_cases": sum(1 for _, reason in selected if reason == "prompt_diversity_edge"),
"non_ascii_records": non_ascii_records,
+ "stochastic_prompt_groups": stochastic_prompt_groups,
"owned_cases": _owned_case_count(cases),
},
"clusters": clusters,
@@ -251,6 +275,13 @@ def _unique_prompt_response_records(records: list[LogRecord]) -> list[LogRecord]
return unique
+def _stochastic_prompt_groups(records: list[LogRecord]) -> int:
+ responses_by_prompt: dict[str, set[str]] = defaultdict(set)
+ for record in records:
+ responses_by_prompt[record.prompt].add(record.response)
+ return sum(1 for responses in responses_by_prompt.values() if len(responses) > 1)
+
+
def _has_non_ascii(value: str) -> bool:
return any(ord(char) > 127 for char in value)
diff --git a/redline/summary.py b/redline/summary.py
index e20774f..3f0e99e 100644
--- a/redline/summary.py
+++ b/redline/summary.py
@@ -59,6 +59,8 @@ def suite_summary(suite: dict[str, Any]) -> dict[str, Any]:
summary = suite.get("summary", {})
if not isinstance(summary, dict):
summary = {}
+ methodology = suite.get("methodology")
+ methodology = methodology if isinstance(methodology, dict) else {}
cases = suite.get("cases", [])
if not isinstance(cases, list):
cases = []
@@ -121,6 +123,8 @@ def suite_summary(suite: dict[str, Any]) -> dict[str, Any]:
"source": str(suite.get("source") or ""),
"created_at": str(suite.get("created_at") or ""),
"selection": str(summary.get("selection") or ""),
+ "methodology_version": str(methodology.get("version") or ""),
+ "methodology_name": str(methodology.get("name") or ""),
"records_seen": records_seen,
"unique_prompt_response_pairs": unique_pairs,
"duplicate_prompt_response_pairs": int(summary.get("duplicate_prompt_response_pairs", 0)),
@@ -150,10 +154,12 @@ def suite_summary(suite: dict[str, Any]) -> dict[str, Any]:
"high_variance_clusters": len(high_variance),
"failure_pattern_clusters": len(failure_patterns),
"non_ascii_records": int(summary.get("non_ascii_records", 0)),
+ "stochastic_prompt_groups": int(summary.get("stochastic_prompt_groups", 0)),
"judgments": dict(sorted(judgment_counts.items())),
"requirements": requirements_count,
"top_clusters": top_clusters,
}
+ result["suite_readiness"] = _suite_readiness(result)
result["next_steps"] = _summary_next_steps(result)
return result
@@ -240,6 +246,7 @@ def prompt_manifest_summary(
"high_variance_clusters",
"failure_pattern_clusters",
"non_ascii_records",
+ "stochastic_prompt_groups",
"requirements",
):
totals[key] += int(child_summary.get(key) or 0)
@@ -302,6 +309,7 @@ def prompt_manifest_summary(
"high_variance_clusters": totals["high_variance_clusters"],
"failure_pattern_clusters": totals["failure_pattern_clusters"],
"non_ascii_records": totals["non_ascii_records"],
+ "stochastic_prompt_groups": totals["stochastic_prompt_groups"],
"requirements": totals["requirements"],
"prompts": prompt_rows,
"missing_suites": missing_suites,
@@ -326,13 +334,16 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None
f"Source: {summary['source'] or ''}",
f"Created: {summary['created_at'] or ''}",
f"Selection: {summary['selection'] or ''}",
+ f"Methodology: {_methodology_label(summary)}",
f"Records seen: {summary['records_seen']}",
f"Unique pairs: {summary['unique_prompt_response_pairs']}",
f"Duplicate pairs: {summary['duplicate_prompt_response_pairs']}",
- f"Behavioral clusters: {summary['clusters']}",
- f"Cluster coverage: {summary['covered_clusters']}/{summary['clusters']} ({_percent(summary['cluster_coverage'])})",
+ f"Behavior groups: {summary['clusters']}",
+ f"Group coverage: {summary['covered_clusters']}/{summary['clusters']} ({_percent(summary['cluster_coverage'])})",
f"Representative cases: {summary['cases']}",
f"Case coverage: {summary['cases']}/{summary['unique_prompt_response_pairs']} ({_percent(summary['case_coverage'])})",
+ f"Suite readiness: {_format_readiness(summary['suite_readiness'])}",
+ "Readiness scope: suite health, not model quality or candidate safety",
f"Pinned cases: {summary['pinned_cases']}",
f"Owned cases: {summary['owned_cases']}/{summary['cases']}",
f"Owner rule coverage: {summary['owner_rule_cases']}/{summary['owned_cases']} ({_percent(summary['owner_rule_coverage'])})",
@@ -340,11 +351,12 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None
f"Approved baselines: {summary['approved_baselines']}/{summary['accepted_baselines']}",
f"Explicit guard coverage: {summary['explicit_guard_cases']}/{summary['cases']} ({_percent(summary['explicit_guard_coverage'])})",
f"Max cases: {summary['max_cases']}",
- f"High-risk clusters: {summary['high_risk_clusters']}",
- f"Medium-risk clusters: {summary['medium_risk_clusters']}",
- f"High-variance clusters: {summary['high_variance_clusters']}",
- f"Failure-pattern clusters: {summary['failure_pattern_clusters']:>2}",
+ f"High-risk groups: {summary['high_risk_clusters']}",
+ f"Medium-risk groups: {summary['medium_risk_clusters']}",
+ f"High-variance groups: {summary['high_variance_clusters']}",
+ f"Failure-pattern groups: {summary['failure_pattern_clusters']:>2}",
f"Non-ASCII records: {summary['non_ascii_records']}",
+ f"Stochastic prompts: {summary['stochastic_prompt_groups']}",
f"Cases with requirements: {summary['requirements']:>2}",
"",
]
@@ -365,7 +377,7 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None
top_clusters = summary["top_clusters"]
if top_clusters:
- lines.append("Top clusters:")
+ lines.append("Top groups:")
for cluster in top_clusters:
marker = " high-variance" if cluster["high_variance"] else ""
flags = cluster["failure_patterns"]
@@ -374,6 +386,15 @@ def format_suite_summary(suite: dict[str, Any], *, suite_path: str | None = None
lines.append(f" {cluster['size']:>4} {cluster['behavior']}{marker}")
lines.append("")
+ readiness = summary.get("suite_readiness")
+ if isinstance(readiness, dict):
+ reasons = readiness.get("reasons")
+ if isinstance(reasons, list) and reasons:
+ lines.append("Readiness signals:")
+ for reason in reasons:
+ lines.append(f"- {reason}")
+ lines.append("")
+
next_steps = summary["next_steps"]
if next_steps:
lines.append("Next:")
@@ -398,8 +419,8 @@ def format_prompt_manifest_summary(report: dict[str, Any]) -> str:
f"Invalid suites: {report['invalid_suite_count']}",
f"Records seen: {report['records_seen']}",
f"Unique pairs: {report['unique_prompt_response_pairs']}",
- f"Behavioral clusters: {report['clusters']}",
- f"Cluster coverage: {report['covered_clusters']}/{report['clusters']} ({_percent(report['cluster_coverage'])})",
+ f"Behavior groups: {report['clusters']}",
+ f"Group coverage: {report['covered_clusters']}/{report['clusters']} ({_percent(report['cluster_coverage'])})",
f"Representative cases: {report['cases']}",
f"Case coverage: {report['cases']}/{report['unique_prompt_response_pairs']} ({_percent(report['case_coverage'])})",
f"Pinned cases: {report['pinned_cases']}",
@@ -408,9 +429,10 @@ def format_prompt_manifest_summary(report: dict[str, Any]) -> str:
f"Accepted baselines: {report['accepted_baselines']}",
f"Approved baselines: {report['approved_baselines']}/{report['accepted_baselines']}",
f"Explicit guard coverage: {report['explicit_guard_cases']}/{report['cases']} ({_percent(report['explicit_guard_coverage'])})",
- f"High-risk clusters: {report['high_risk_clusters']}",
- f"Medium-risk clusters: {report['medium_risk_clusters']}",
+ f"High-risk groups: {report['high_risk_clusters']}",
+ f"Medium-risk groups: {report['medium_risk_clusters']}",
f"Non-ASCII records: {report['non_ascii_records']}",
+ f"Stochastic prompts: {report['stochastic_prompt_groups']}",
f"Cases with requirements: {report['requirements']:>2}",
"",
]
@@ -477,6 +499,10 @@ def _summary_next_steps(summary: dict[str, Any], *, suite_path: str | None = Non
"Review non-English cases manually or with a domain judge; structural checks still work, "
"but entity/refusal heuristics are English-oriented."
)
+ if int(summary.get("stochastic_prompt_groups") or 0):
+ steps.append(
+ "Stabilize repeated-prompt baselines or use review mode; otherwise natural sampling variance can look like regressions."
+ )
if int(summary["cases"]) and not summary["judgments"]:
steps.append("After the first eval, mark expected or ignored changes to train the suite.")
if int(summary["cases"]) == 0:
@@ -484,6 +510,99 @@ def _summary_next_steps(summary: dict[str, Any], *, suite_path: str | None = Non
return steps
+def _suite_readiness(summary: dict[str, Any]) -> dict[str, Any]:
+ cases = int(summary.get("cases") or 0)
+ if cases == 0:
+ return {
+ "score": 0,
+ "label": "empty",
+ "reasons": ["no suite cases are available yet"],
+ }
+
+ score = 0.0
+ score += 35.0 * float(summary.get("cluster_coverage") or 0.0)
+ score += 20.0 * float(summary.get("case_coverage") or 0.0)
+ score += 20.0 * float(summary.get("explicit_guard_coverage") or 0.0)
+ owner_coverage = _ratio(int(summary.get("owned_cases") or 0), cases) or 0.0
+ score += 10.0 * owner_coverage
+ if int(summary.get("high_risk_clusters") or 0) == 0 or int(summary.get("requirements") or 0):
+ score += 10.0
+ if int(summary.get("non_ascii_records") or 0) == 0:
+ score += 5.0
+
+ rounded = int(round(min(100.0, max(0.0, score))))
+ if rounded >= 80:
+ label = "strong"
+ elif rounded >= 55:
+ label = "usable"
+ else:
+ label = "needs_work"
+
+ return {
+ "score": rounded,
+ "label": label,
+ "reasons": _readiness_reasons(summary, owner_coverage=owner_coverage),
+ }
+
+
+def _readiness_reasons(summary: dict[str, Any], *, owner_coverage: float) -> list[str]:
+ reasons = []
+ cluster_coverage = float(summary.get("cluster_coverage") or 0.0)
+ case_coverage = float(summary.get("case_coverage") or 0.0)
+ explicit_guard_coverage = float(summary.get("explicit_guard_coverage") or 0.0)
+
+ if cluster_coverage >= 1.0:
+ reasons.append("all detected behavior-signature groups have at least one selected case")
+ else:
+ reasons.append("some behavior-signature groups are not represented in the suite")
+
+ if case_coverage >= 0.8:
+ reasons.append("case budget covers most unique prompt-response pairs")
+ else:
+ reasons.append("case budget is sampling a small share of unique prompt-response pairs")
+
+ if explicit_guard_coverage >= 0.5:
+ reasons.append("many cases have requirements or recorded judgments")
+ elif explicit_guard_coverage > 0:
+ reasons.append("some cases have requirements or recorded judgments")
+ else:
+ reasons.append("no cases have explicit requirements or recorded judgments yet")
+
+ if owner_coverage >= 1.0:
+ reasons.append("all cases have owners")
+ elif owner_coverage > 0:
+ reasons.append("some cases still need owners")
+ else:
+ reasons.append("no cases have owners yet")
+
+ if int(summary.get("high_risk_clusters") or 0) and int(summary.get("requirements") or 0) == 0:
+ reasons.append("high-risk groups need explicit requirements or a judge before CI gating")
+ if int(summary.get("non_ascii_records") or 0):
+ reasons.append("non-ASCII records need extra review because entity/refusal heuristics are English-oriented")
+ if int(summary.get("stochastic_prompt_groups") or 0):
+ reasons.append("repeated prompts with multiple baseline responses need stochastic baseline review")
+
+ return reasons
+
+
+def _format_readiness(value: object) -> str:
+ if not isinstance(value, dict):
+ return ""
+ score = int(value.get("score") or 0)
+ label = str(value.get("label") or "unknown").replace("_", " ")
+ return f"{score}/100 ({label})"
+
+
+def _methodology_label(summary: dict[str, Any]) -> str:
+ name = str(summary.get("methodology_name") or "").strip()
+ version = str(summary.get("methodology_version") or "").strip()
+ if name and version:
+ return f"{name} ({version})"
+ if version:
+ return version
+ return ""
+
+
def _manifest_summary_status(summary: dict[str, Any]) -> str:
if int(summary["invalid_suite_count"]):
return "invalid"
diff --git a/redline/validate.py b/redline/validate.py
index 92afea5..582f261 100644
--- a/redline/validate.py
+++ b/redline/validate.py
@@ -26,10 +26,13 @@
"shape",
"length_bucket",
)
+_CURRENT_SUITE_VERSION = "0.1"
+_SUPPORTED_SUITE_VERSIONS = {_CURRENT_SUITE_VERSION}
def validate_suite(suite: dict[str, Any], *, suite_path: str = "") -> dict[str, Any]:
items: list[dict[str, str]] = []
+ _validate_suite_metadata(items, suite)
cases = suite.get("cases")
if not isinstance(cases, list):
_add(items, "error", "cases", "expected a list of suite cases")
@@ -93,7 +96,7 @@ def validate_suite(suite: dict[str, Any], *, suite_path: str = "") -> dict[str,
error_count = _count(items, "error")
warning_count = _count(items, "warning")
return {
- "version": "0.1",
+ "version": _CURRENT_SUITE_VERSION,
"suite": suite_path,
"valid": error_count == 0,
"errors": error_count,
@@ -107,6 +110,33 @@ def validate_suite(suite: dict[str, Any], *, suite_path: str = "") -> dict[str,
}
+def _validate_suite_metadata(items: list[dict[str, str]], suite: dict[str, Any]) -> None:
+ version = suite.get("version")
+ if not isinstance(version, str) or not version.strip():
+ _add(
+ items,
+ "warning",
+ "version",
+ "missing suite version; treating as legacy and recommending regeneration with current redline",
+ )
+ elif version.strip() not in _SUPPORTED_SUITE_VERSIONS:
+ supported = ", ".join(sorted(_SUPPORTED_SUITE_VERSIONS))
+ _add(
+ items,
+ "error",
+ "version",
+ f"unsupported suite version {version.strip()}; supported versions: {supported}",
+ )
+
+ schema = suite.get("$schema")
+ if not isinstance(schema, str) or not schema.strip():
+ _add(items, "warning", "$schema", "missing suite JSON schema reference")
+
+ methodology = suite.get("methodology")
+ if not isinstance(methodology, dict):
+ _add(items, "warning", "methodology", "missing suite selection methodology metadata")
+
+
def validate_prompt_manifest(
manifest: dict[str, Any],
*,
@@ -281,6 +311,8 @@ def _next_steps(items: list[dict[str, str]], *, suite_path: str, source: object)
steps.append(f"Use a supported judgment status, then rerun: redline validate {suite_arg}")
if any(item["level"] == "error" and item["path"] == "cases" for item in items):
steps.append(f"Fix suite JSON shape, then rerun: redline validate {suite_arg}")
+ if any(item["level"] == "error" and item["path"] == "version" for item in items):
+ steps.append(f"Regenerate or migrate unsupported suite schema: redline suite {source_arg} --out {suite_arg}")
if any(
item["level"] == "warning" and "duplicate prompt-response pair" in item["message"]
@@ -295,6 +327,7 @@ def _next_steps(items: list[dict[str, str]], *, suite_path: str, source: object)
item["path"].endswith(".content_hash")
or ".features." in item["path"]
or item["path"] == "summary"
+ or item["path"] in {"version", "$schema", "methodology"}
)
for item in items
):
diff --git a/scripts/demo_gif.sh b/scripts/demo_gif.sh
index 6a55643..cabd778 100755
--- a/scripts/demo_gif.sh
+++ b/scripts/demo_gif.sh
@@ -27,7 +27,7 @@ Sleep 5s
Type "python -m redline history ${run_dir}/demo/reports/public_diff.json --label public-demo --out ${run_dir}/history.jsonl --out-md ${run_dir}/history.md"
Enter
Sleep 2s
-Type "python -m redline dashboard --reports-dir ${run_dir}/demo/reports --history ${run_dir}/history.jsonl --out ${run_dir}/dashboard.html"
+Type "python -m redline app --reports-dir ${run_dir}/demo/reports --history ${run_dir}/history.jsonl --no-open --out ${run_dir}/app.html"
Enter
Sleep 2s
TAPE
diff --git a/scripts/demo_terminal.sh b/scripts/demo_terminal.sh
index 23a566b..40b90d6 100755
--- a/scripts/demo_terminal.sh
+++ b/scripts/demo_terminal.sh
@@ -23,10 +23,11 @@ python -m redline history "$demo_dir/reports/public_diff.json" \
--out "$history_path" \
--out-md "$history_markdown_path"
-printf '\n$ redline dashboard --reports-dir %s --history %s --out %s\n' "$demo_dir/reports" "$history_path" "$dashboard_path"
-python -m redline dashboard \
+printf '\n$ redline app --reports-dir %s --history %s --no-open --out %s\n' "$demo_dir/reports" "$history_path" "$dashboard_path"
+python -m redline app \
--reports-dir "$demo_dir/reports" \
--history "$history_path" \
+ --no-open \
--out "$dashboard_path"
printf '\nArtifacts:\n'
diff --git a/scripts/release_check.sh b/scripts/release_check.sh
index dad0da9..92ac13a 100755
--- a/scripts/release_check.sh
+++ b/scripts/release_check.sh
@@ -121,12 +121,13 @@ printf '\n$ %s -m pip install --no-deps --force-reinstall %s\n' "$venv_dir/bin/p
--fail-on none
test -s compare.html
- printf '\n$ redline dashboard --reports-dir .redline/demo/reports --history history.jsonl --out dashboard.html\n'
- "$venv_dir/bin/redline" dashboard \
+ printf '\n$ redline app --reports-dir .redline/demo/reports --history history.jsonl --no-open --out app.html\n'
+ "$venv_dir/bin/redline" app \
--reports-dir .redline/demo/reports \
--history history.jsonl \
- --out dashboard.html
- test -s dashboard.html
+ --no-open \
+ --out app.html
+ test -s app.html
printf '\n$ redline suite .redline/demo/baseline.jsonl --out all-suite.json --all-cases\n'
"$venv_dir/bin/redline" suite .redline/demo/baseline.jsonl --out all-suite.json --all-cases
diff --git a/server.json b/server.json
index 711721e..042d862 100644
--- a/server.json
+++ b/server.json
@@ -8,19 +8,19 @@
"url": "https://github.com/gowtham0992/redline",
"source": "github"
},
- "version": "0.2.1",
+ "version": "0.3.0",
"packages": [
{
"registryType": "pypi",
"registryBaseUrl": "https://pypi.org",
"identifier": "redline-ai",
- "version": "0.2.1",
+ "version": "0.3.0",
"runtimeHint": "uvx",
"runtimeArguments": [
{
"type": "named",
"name": "--from",
- "value": "redline-ai==0.2.1",
+ "value": "redline-ai==0.3.0",
"description": "Install the redline-ai package version that matches this registry entry."
},
{
diff --git a/site/assets/redline-dashboard-proof.png b/site/assets/redline-dashboard-proof.png
index 0a1b48b..0f8e509 100644
Binary files a/site/assets/redline-dashboard-proof.png and b/site/assets/redline-dashboard-proof.png differ
diff --git a/site/assets/redline-preview.png b/site/assets/redline-preview.png
index cac96bd..0f8e509 100644
Binary files a/site/assets/redline-preview.png and b/site/assets/redline-preview.png differ
diff --git a/site/assets/redline-report-proof.png b/site/assets/redline-report-proof.png
index e687dc2..3e6a8ca 100644
Binary files a/site/assets/redline-report-proof.png and b/site/assets/redline-report-proof.png differ
diff --git a/site/index.html b/site/index.html
index fb917d9..11ed861 100644
--- a/site/index.html
+++ b/site/index.html
@@ -3,13 +3,13 @@
- redline - Local-first prompt regression diffs
+ redline - Catch prompt regressions before they ship
-
+
@@ -29,68 +29,51 @@
MCPCertifyDocs
- GitHub
+ Star on GitHub
-
+
-
Automatic eval suites from logs you already have
-
redline
+
Local-first prompt regression diffs
+
Catch prompt regressions before they ship.
- Local-first prompt regression diffs for AI engineers. Point redline
- at real prompt-response logs, change your prompt, then catch broken
- JSON, missing URLs, lost tables, refusals, and other structural
- regressions before they ship.
+ Point redline at the prompt-response logs you already have, change
+ your prompt, then catch broken JSON, missing URLs, lost tables, and
+ silent refusals before your users do.
-
- 0manual test cases to start
- 0cloud calls by default
- 1command to see the proof
-
- The public demo is synthetic and safe to share, but it behaves like
- a real prompt change: the candidate gets shorter and silently drops
- required structure, dates, owners, URLs, and refusal behavior.
-
REGRESSION case_006: candidate newly refuses
REGRESSION case_010: candidate lost bullet list structure
-Next: inspect the HTML report, mark intentional changes, accept reviewed outputs.
+10 blocking regressions - fix before you ship.
+
+$ redline diff dolly-suite-100.json dolly-candidate-100.jsonl --compact
+redline diff: cases=100 regression=51 changed=27 missing=0 neutral=22
+Diagnosis: candidate got shorter, lost required structure, dropped concrete details, returned empty outputs, and changed content substantially.
+
+
+
+
+
First-run proof
+
One command for the demo. One public dataset for proof.
+
+ The public demo is synthetic and safe to share, but it behaves like
+ a real prompt change: the candidate gets shorter and silently drops
+ required structure, dates, owners, URLs, and refusal behavior. The
+ larger internet dogfood run imports 100 rows from Databricks Dolly
+ 15k and pushes them through the same suite, diff, history,
+ benchmark, app, and dashboard surfaces.
+
reason: candidate lost valid JSON format; required keys are no longer parseable.
+
+
+
+
+
+ 100 public prompt-response rows
+
Imported from Databricks Dolly 15k with redline's JSONL importer and kept as local-only dogfood evidence.
+
+
+ 20 behavior groups covered
+
The generated suite covered 100/100 cases and 20/20 deterministic behavior groups.
+
+
+ 51 blocking regressions found
+
The report diagnosed shorter answers, lost structure, dropped concrete details, empty outputs, and substantial content drift.
+
+
+ 0 dashboard warnings
+
The local app loaded 1 report, 1 benchmark, and 1 history entry with no sidecar-artifact noise.
+
+
+
+
+
- Local dashboard with ship readiness, history, benchmark evidence, and report links.
+ Local app from the 100-row Dolly dogfood run: reports, benchmark evidence, history, and ship readiness.
- HTML report with concrete reasons and side-by-side baseline versus candidate output.
+ HTML report from the same run with the diagnosis, coverage, methodology, and concrete reasons.
-
Why it exists
+
Product promise
You already have the eval data. redline turns it into a gate.
- Zero manual test writing
-
Build suites from JSONL prompt-response logs instead of inventing cases by hand.
+ Zero manual cases to start
+
Build suites from JSONL prompt-response logs instead of inventing eval cases by hand.
- Deterministic by default
-
No cloud account is required for the core loop; CI-friendly checks run locally, cost nothing, and do not depend on hosted judges.
+ No cloud calls by default
+
The core checks run locally with zero runtime dependencies, no API keys, and reproducible CI output.
- Reports people can review
-
Write JSON, Markdown, JUnit, and self-contained HTML reports for local review, CI artifacts, and GitHub summaries.
+ Clear failure reasons
+
Reports name the broken behavior: lost JSON, missing keys, dropped URLs, new refusals, empty output, or structure drift.
Review loop included
-
Mark intentional changes, accept new baselines, and keep the suite moving with your prompt.
+
Mark intentional changes, accept reviewed baselines, and keep the suite evolving with your prompt.
@@ -163,6 +236,7 @@
From logs to a shipping gate in four commands.
01
Collect behavior
+
Watch an app, import JSONL, or adapt exports from your logging stack.
redline watch --log logs/prompts.jsonl
@@ -170,13 +244,15 @@
Collect behavior
02
Generate the suite
- redline suite .redline/logs/prompts.jsonl --out redline-suite.json
+
Select representative behavior from real prompt-output data.
+ redline suite logs/prompts.jsonl --out redline-suite.json
03
-
Evaluate a prompt change
+
Evaluate a change
+
Replay the suite against a new prompt file or candidate log.
redline eval --prompt prompts/v2.txt
@@ -184,6 +260,7 @@
Evaluate a prompt change
04
Review and accept
+
Pin requirements, mark expected changes, and promote reviewed outputs.
redline mark ...redline accept ...
@@ -250,8 +327,8 @@
Replay any app
Local dashboard
-
Publish self-contained HTML reports as CI artifacts or browse them locally.
- 
- 
+ 
- 
-