diff --git a/docs/docs/pages/examples/multimodal/multimodal-files.mdx b/docs/docs/pages/examples/multimodal/multimodal-files.mdx
index 33c3607f..ea5e7a11 100644
--- a/docs/docs/pages/examples/multimodal/multimodal-files.mdx
+++ b/docs/docs/pages/examples/multimodal/multimodal-files.mdx
@@ -10,9 +10,36 @@ import { LanguageTabs } from "../../../components/LanguageTabs";
This page demonstrates how to write Scenario tests where the user provides **files** (PDF, CSV, etc.) as part of the conversation and the agent must parse and respond appropriately.
:::tip
-The focus here is on **testing** your file-handling agent, not building it. Your agent implementation can use any framework (LangChain, Agno, custom code, etc.) — Scenario tests are framework-agnostic.
+The focus here is on **testing** your file-handling agent, not building it. Your agent implementation can use any framework (LangChain, Agno, custom code, etc.) — Scenario tests are framework-agnostic. If you want to see an example of how it's built, check out the [multimodal-ai repository](https://github.com/langwatch/multimodal-ai).
:::
+## Understanding Test Fixtures
+
+Before diving into code, let's talk about **test fixtures** — the sample files you'll use to test your agent.
+
+### What Are Test Fixtures?
+
+Test fixtures are pre-prepared files (PDFs, CSVs, images, etc.) that serve as controlled inputs for your tests. Think of them as the "props" in your testing stage. Just like actors rehearse with the same props each time, your agent should be tested against consistent, well-defined files.
+
+### Why Fixtures Matter
+
+1. **Reproducibility**: Tests run the same way every time with the same inputs
+2. **Coverage**: Different fixtures test different scenarios (edge cases, happy paths, error conditions)
+3. **Documentation**: Fixtures serve as examples of what your agent should handle
+4. **Debugging**: When a test fails, you can inspect the exact file that caused the issue
+
+### Organizing Your Test Fixtures
+
+**You should create a dedicated `fixtures` folder** in your test directory to store all your test files. Organize them by file type (e.g., `fixtures/pdfs/`, `fixtures/csvs/`, `fixtures/images/`).
+
+**Key principles:**
+
+- **One folder per file type**: Separate PDFs, CSVs, images, etc.
+- **Descriptive names**: The filename should indicate what scenario it tests (e.g., `financial-report-2024-q1.pdf`, `employee-database-small.csv`)
+- **Multiple variants**: Have several examples for each scenario type (e.g., multiple financial reports, different-sized datasets)
+- **Include edge cases**: Empty files, corrupted files, unusually large files
+- **Version fixtures**: Keep different versions if testing historical behavior
+
## Adding Files to Scenario Messages
Files are included in scenario messages using the OpenAI `ChatCompletionMessageParam` format. You can pass file content as base64-encoded data using the `file` type with `file_data`:
@@ -201,7 +228,7 @@ describe("PDF Analysis", () => {
],
}),
scenario.agent(),
- scenario.succeed("Agent successfully summarized the PDF document."),
+ scenario.judge(),
],
});
@@ -257,7 +284,7 @@ async def test_pdf_summarization():
],
}),
scenario.agent(),
- scenario.succeed("Agent successfully summarized the PDF document."),
+ scenario.judge(),
],
)
@@ -318,7 +345,7 @@ describe("CSV Analysis", () => {
],
}),
scenario.agent(),
- scenario.succeed("Agent successfully analyzed the employee database."),
+ scenario.judge(),
],
});
@@ -376,7 +403,7 @@ async def test_csv_employee_analysis():
],
}),
scenario.agent(),
- scenario.succeed("Agent successfully analyzed the employee database."),
+ scenario.judge(),
],
)
@@ -386,6 +413,401 @@ async def test_csv_employee_analysis():
+## Scaling Your Tests: Multiple Files
+
+As your test suite grows, you'll want to test your agent against **multiple files for the same scenario**. This ensures your agent works consistently across different variations of similar content.
+
+### Pattern 1: Looping Over Multiple Fixtures
+
+When you have several files that test the same capability (e.g., multiple financial reports), you can loop over them to run the same test scenario with each file. This is excellent for comprehensive coverage.
+
+
+
+
+```typescript
+import * as fs from "fs";
+import * as path from "path";
+import scenario from "@langwatch/scenario";
+import { describe, it, expect } from "vitest";
+
+const FIXTURES_DIR = path.join(__dirname, "fixtures", "pdfs");
+
+// Define multiple PDF fixtures for the same scenario type
+const FINANCIAL_REPORTS = [
+ "financial-report-2024-q1.pdf",
+ "financial-report-2024-q2.pdf",
+ "financial-report-2024-q3.pdf",
+ "financial-report-2024-q4.pdf",
+];
+
+describe("PDF Financial Analysis - Multiple Reports", () => {
+ // Loop over each financial report
+ FINANCIAL_REPORTS.forEach((filename) => {
+ it(`should extract revenue data from ${filename}`, async () => {
+ const pdfPath = path.join(FIXTURES_DIR, filename);
+ const fileContent = fs.readFileSync(pdfPath);
+ const base64Data = fileContent.toString("base64");
+
+ const result = await scenario.run({
+ name: `Financial Analysis - ${filename}`,
+ description: `Test revenue extraction from ${filename}`,
+ agents: [
+ yourAgentAdapter,
+ scenario.userSimulatorAgent(),
+ scenario.judgeAgent({
+ criteria: [
+ "Agent correctly identifies total revenue figures",
+ "Agent mentions the reporting period",
+ "Agent provides key financial metrics",
+ ],
+ }),
+ ],
+ script: [
+ scenario.message({
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: "What was the total revenue in this financial report?",
+ },
+ {
+ type: "file",
+ file: {
+ filename,
+ file_data: `data:application/pdf;base64,${base64Data}`,
+ },
+ },
+ ],
+ }),
+ scenario.agent(),
+ scenario.judge(),
+ ],
+ });
+
+ expect(result.success).toBe(true);
+ });
+ });
+});
+```
+
+
+
+
+```python
+import base64
+from pathlib import Path
+import pytest
+import scenario
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pdfs"
+
+# Define multiple PDF fixtures for the same scenario type
+FINANCIAL_REPORTS = [
+ "financial-report-2024-q1.pdf",
+ "financial-report-2024-q2.pdf",
+ "financial-report-2024-q3.pdf",
+ "financial-report-2024-q4.pdf",
+]
+
+
+# Loop over each financial report using pytest parametrize
+@pytest.mark.asyncio
+@pytest.mark.parametrize("filename", FINANCIAL_REPORTS)
+async def test_extract_revenue_from_multiple_reports(filename: str):
+ """Test revenue extraction across multiple quarterly reports."""
+
+ pdf_path = FIXTURES_DIR / filename
+ file_content = pdf_path.read_bytes()
+ base64_data = base64.b64encode(file_content).decode()
+
+ result = await scenario.run(
+ name=f"Financial Analysis - {filename}",
+ description=f"Test revenue extraction from {filename}",
+ agents=[
+ YourAgentAdapter(),
+ scenario.UserSimulatorAgent(),
+ scenario.JudgeAgent(
+ criteria=[
+ "Agent correctly identifies total revenue figures",
+ "Agent mentions the reporting period",
+ "Agent provides key financial metrics",
+ ]
+ ),
+ ],
+ script=[
+ scenario.message({
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What was the total revenue in this financial report?",
+ },
+ {
+ "type": "file",
+ "file": {
+ "filename": filename,
+ "file_data": f"data:application/pdf;base64,{base64_data}",
+ },
+ },
+ ],
+ }),
+ scenario.agent(),
+ scenario.judge(),
+ ],
+ )
+
+ assert result.success, f"Scenario failed for {filename}: {result.reasoning}"
+```
+
+
+
+
+**Benefits of looping:**
+- Tests all your fixtures automatically — add a new PDF and it's instantly tested
+- Identifies which specific files cause failures
+- Ensures consistent behavior across variations
+
+### Pattern 2: Random Selection for Diverse Testing
+
+Sometimes you want to test against **one random file** from a collection to keep test runs fast while still ensuring variety over time. This is useful for large fixture sets or in CI/CD pipelines.
+
+
+
+
+```typescript
+import * as fs from "fs";
+import * as path from "path";
+import scenario from "@langwatch/scenario";
+import { describe, it, expect } from "vitest";
+
+const FIXTURES_DIR = path.join(__dirname, "fixtures", "pdfs");
+
+const LEGAL_CONTRACTS = [
+ "employment-contract-2023.pdf",
+ "employment-contract-2024.pdf",
+ "vendor-agreement.pdf",
+ "service-level-agreement.pdf",
+ "non-disclosure-agreement.pdf",
+];
+
+describe("Legal Contract Analysis", () => {
+ it("should identify key terms in a random contract", async () => {
+ // Pick a random contract from the collection
+ const randomContract =
+ LEGAL_CONTRACTS[Math.floor(Math.random() * LEGAL_CONTRACTS.length)];
+ const pdfPath = path.join(FIXTURES_DIR, randomContract);
+
+ console.log(`Testing with: ${randomContract}`); // Helps with debugging
+
+ const result = await scenario.run({
+ name: `Legal Analysis - ${randomContract}`,
+ description:
+ "Test that agent can identify key contractual terms from legal documents",
+ agents: [
+ yourAgentAdapter,
+ scenario.userSimulatorAgent(),
+ scenario.judgeAgent({
+ criteria: [
+ "Agent identifies the type of legal document",
+ "Agent extracts key dates or terms",
+ "Agent mentions parties involved",
+ ],
+ }),
+ ],
+ script: [
+ scenario.message({
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: "Please review this contract and tell me the key terms.",
+ },
+ {
+ type: "file",
+ file: {
+ filename: randomContract,
+ file_data: `data:application/pdf;base64,${fs.readFileSync(pdfPath).toString("base64")}`,
+ },
+ },
+ ],
+ }),
+ scenario.agent(),
+ scenario.judge(),
+ ],
+ });
+
+ expect(result.success).toBe(true);
+ });
+});
+```
+
+
+
+
+```python
+import base64
+import random
+from pathlib import Path
+import pytest
+import scenario
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures" / "pdfs"
+
+LEGAL_CONTRACTS = [
+ "employment-contract-2023.pdf",
+ "employment-contract-2024.pdf",
+ "vendor-agreement.pdf",
+ "service-level-agreement.pdf",
+ "non-disclosure-agreement.pdf",
+]
+
+
+@pytest.mark.asyncio
+async def test_identify_key_terms_random_contract():
+ """Test contract analysis with a randomly selected legal document."""
+
+ # Pick a random contract from the collection
+ random_contract = random.choice(LEGAL_CONTRACTS)
+ pdf_path = FIXTURES_DIR / random_contract
+
+ print(f"Testing with: {random_contract}") # Helps with debugging
+
+ result = await scenario.run(
+ name=f"Legal Analysis - {random_contract}",
+ description="Test that agent can identify key contractual terms from legal documents",
+ agents=[
+ YourAgentAdapter(),
+ scenario.UserSimulatorAgent(),
+ scenario.JudgeAgent(
+ criteria=[
+ "Agent identifies the type of legal document",
+ "Agent extracts key dates or terms",
+ "Agent mentions parties involved",
+ ]
+ ),
+ ],
+ script=[
+ scenario.message({
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Please review this contract and tell me the key terms.",
+ },
+ {
+ "type": "file",
+ "file": {
+ "filename": random_contract,
+ "file_data": f"data:application/pdf;base64,{base64.b64encode(pdf_path.read_bytes()).decode()}",
+ },
+ },
+ ],
+ }),
+ scenario.agent(),
+ scenario.judge(),
+ ],
+ )
+
+ assert result.success, f"Scenario failed for {random_contract}: {result.reasoning}"
+```
+
+
+
+
+**Benefits of random selection:**
+- Faster test runs (only one file per test)
+- Still ensures variety across multiple test executions
+- Good for CI/CD where you want quick feedback
+
+**Pro tip:** For comprehensive testing, use **looping in your full test suite** and **random selection in quick smoke tests** or during development.
+
+## Advanced: Cross-File Comparison and Multi-File Scenarios
+
+Real users often need to work with **multiple files simultaneously** — comparing reports, aggregating data across documents, or validating consistency between sources. Testing these scenarios ensures your agent can synthesize information across multiple files.
+
+### Why Test Multi-File Scenarios?
+
+1. **Context Management**: Validates that your agent can track information from multiple sources without confusion
+2. **Information Synthesis**: Tests the agent's ability to combine and compare data across documents
+3. **Real-World Relevance**: Users commonly upload multiple related files (e.g., "Compare Q1 and Q2 results")
+4. **Complex Reasoning**: Requires higher-level analysis than single-file processing
+
+### Providing Multiple Files in a Single Message
+
+To send multiple files at once, simply include multiple `file` objects in the content array:
+
+
+
+
+```typescript
+scenario.message({
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: "Compare these two quarterly reports and tell me which had higher revenue."
+ },
+ {
+ type: "file",
+ file: {
+ filename: "q1-report.pdf",
+ file_data: `data:application/pdf;base64,${fs.readFileSync(q1Path).toString("base64")}`,
+ },
+ },
+ {
+ type: "file",
+ file: {
+ filename: "q2-report.pdf",
+ file_data: `data:application/pdf;base64,${fs.readFileSync(q2Path).toString("base64")}`,
+ },
+ },
+ ],
+});
+```
+
+
+
+
+```python
+scenario.message({
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Compare these two quarterly reports and tell me which had higher revenue.",
+ },
+ {
+ "type": "file",
+ "file": {
+ "filename": "q1-report.pdf",
+ "file_data": f"data:application/pdf;base64,{base64.b64encode(q1_path.read_bytes()).decode()}",
+ },
+ },
+ {
+ "type": "file",
+ "file": {
+ "filename": "q2-report.pdf",
+ "file_data": f"data:application/pdf;base64,{base64.b64encode(q2_path.read_bytes()).decode()}",
+ },
+ },
+ ],
+})
+```
+
+
+
+
+**Other multi-file scenarios to test:**
+
+- **Aggregation**: "Here are 3 invoices — what's the total amount across all of them?"
+- **Cross-validation**: "Analyze this CSV and this PDF together — do the numbers match?"
+- **Data enrichment**: "Use this price list CSV to calculate totals for the items in this invoice PDF"
+
+Use `scenario.judgeAgent()` with criteria like:
+- "Agent correctly identifies which report had higher revenue"
+- "Agent mentions specific numbers from both files"
+- "Agent doesn't confuse data between the two documents"
+
## Real-World Example
For a complete, production-ready example, see the [langwatch/multimodal-ai](https://github.com/langwatch/multimodal-ai) repository. It includes:
diff --git a/docs/scripts/replace_scenario_metadata.py b/docs/scripts/replace_scenario_metadata.py
new file mode 100644
index 00000000..27c4a970
--- /dev/null
+++ b/docs/scripts/replace_scenario_metadata.py
@@ -0,0 +1,370 @@
+"""
+Replace Scenario docs meta descriptions using a CSV mapping.
+
+Default CSV: c:\\Users\\aryan\\Downloads\\Langwatch_MetaDesc.csv
+Columns used:
+- Meta description (OLD): text to find
+- Meta NEW: replacement text
+- URL: kept for reporting
+
+Usage:
+ # dry run (recommended first)
+ python docs/scripts/replace_scenario_metadata.py
+
+ # apply changes
+ python docs/scripts/replace_scenario_metadata.py --apply
+"""
+import argparse
+import csv
+import json
+import re
+import sys
+from functools import lru_cache
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, List, Tuple
+from urllib.parse import urlparse
+
+DEFAULT_CSV = r"c:\Users\aryan\Downloads\Langwatch_MetaDesc.csv"
+# Default to the Scenario docs directory (docs/docs) relative to this file.
+DEFAULT_ROOT = Path(__file__).resolve().parents[1] / "docs"
+# Only apply rows whose URL starts with this prefix.
+DEFAULT_URL_PREFIX = "https://scenario.langwatch.ai"
+
+# Text-based extensions to scan (tuned for the docs site)
+DEFAULT_EXTS = {".md", ".mdx"}
+
+# Directories to skip while scanning
+SKIP_DIRS = {
+ "node_modules",
+ ".git",
+ ".next",
+ ".turbo",
+ "dist",
+ "build",
+ ".cache",
+}
+
+
+@dataclass
+class Mapping:
+ new: str
+ url: str
+
+
+def resolve_csv_path(raw: Path) -> Path:
+ """
+ Allow Windows-style paths when running inside WSL.
+ If the given path does not exist, try converting `C:\\foo\\bar` to `/mnt/c/foo/bar`.
+ """
+ if raw.exists():
+ return raw
+ m = re.match(r"^([a-zA-Z]):\\\\?(.*)$", str(raw))
+ if m:
+ drive = m.group(1).lower()
+ rest = m.group(2).replace("\\", "/")
+ alt = Path(f"/mnt/{drive}/{rest}")
+ if alt.exists():
+ return alt
+ return raw
+
+
+def load_mapping(csv_path: Path, url_prefix: str) -> Dict[str, Mapping]:
+ if not csv_path.exists():
+ raise FileNotFoundError(f"CSV not found: {csv_path}")
+
+ with csv_path.open(newline="", encoding="utf-8") as f:
+ reader = csv.DictReader(f)
+ required = {"Meta description (OLD)", "Meta NEW"}
+ missing = required - set(reader.fieldnames or [])
+ if missing:
+ raise ValueError(f"Missing required columns: {', '.join(sorted(missing))}")
+
+ mapping: Dict[str, Mapping] = {}
+ conflicts: List[Tuple[str, str, str, int]] = []
+ skipped_prefix: int = 0
+ for idx, row in enumerate(reader, start=2): # header is line 1
+ old = (row.get("Meta description (OLD)", "") or "").strip()
+ new = (row.get("Meta NEW", "") or "").strip()
+ url = (row.get("URL", "") or "").strip()
+ if not old or not new:
+ continue
+ if url_prefix and not url.startswith(url_prefix):
+ skipped_prefix += 1
+ continue
+ if old in mapping and mapping[old].new != new:
+ # keep the first occurrence, record conflict for reporting
+ conflicts.append((old, mapping[old].new, new, idx))
+ continue
+ mapping[old] = Mapping(new=new, url=url)
+
+ if conflicts:
+ print("Detected conflicting rows (kept the first occurrence for each):")
+ for old, kept, skipped, idx in conflicts:
+ print(f"- Row {idx}: {old!r} -> {skipped!r} (kept existing: {kept!r})")
+ if skipped_prefix:
+ print(
+ f"Skipped {skipped_prefix} row(s) whose URL did not start with prefix: {url_prefix}"
+ )
+
+ return mapping
+
+
+def iter_text_files(root: Path, exts: Iterable[str]) -> Iterable[Path]:
+ for path in root.rglob("*"):
+ if path.is_dir():
+ if path.name in SKIP_DIRS:
+ # Skip entire subtree
+ continue
+ continue
+ if path.suffix.lower() in exts:
+ yield path
+
+
+def apply_replacements(
+ path: Path, mapping: Dict[str, Mapping], apply: bool
+) -> Tuple[bool, Dict[str, int]]:
+ text = path.read_text(encoding="utf-8")
+ replaced = False
+ counts: Dict[str, int] = {}
+ new_text = text
+
+ for old, m in mapping.items():
+ occurrences = len(re.findall(re.escape(old), new_text))
+ if occurrences:
+ counts[old] = occurrences
+ if apply:
+ new_text = re.sub(re.escape(old), m.new, new_text)
+ replaced = True
+
+ if apply and replaced and new_text != text:
+ path.write_text(new_text, encoding="utf-8")
+
+ return replaced, counts
+
+
+@lru_cache(maxsize=512)
+def resolve_doc_path(root: Path, url: str) -> Path | None:
+ """
+ Map a doc URL to a local file path under docs/docs/pages.
+ Supports:
+ - Stripping domain, handling trailing slashes.
+ - Removing trailing index.html or .html.
+ - Trying .mdx, .md, and directory index files.
+ """
+ parsed = urlparse(url)
+ path = parsed.path or "/"
+ path = path.strip("/")
+ # Remove trailing index.html or .html
+ if path.endswith("index.html"):
+ path = path[: -len("index.html")].rstrip("/")
+ elif path.endswith(".html"):
+ path = path[: -len(".html")]
+ # Default to index
+ if not path:
+ path = "index"
+
+ base = root / "pages"
+ candidates = [
+ base / f"{path}.mdx",
+ base / f"{path}.md",
+ base / path / "index.mdx",
+ base / path / "index.md",
+ ]
+ for candidate in candidates:
+ if candidate.exists():
+ return candidate
+ return None
+
+
+def update_frontmatter_description(content: str, new_desc: str) -> str:
+ """
+ Replace or add description in YAML frontmatter.
+ If no frontmatter is present, frontmatter will be added.
+ """
+ if content.startswith("---"):
+ parts = content.split("---", 2)
+ if len(parts) >= 3:
+ _, fm_body, rest = parts[0], parts[1], parts[2]
+ lines = fm_body.strip("\n").splitlines()
+ out_lines = []
+ found = False
+ for line in lines:
+ if re.match(r"^description\s*:", line):
+ out_lines.append(f'description: {new_desc}')
+ found = True
+ else:
+ out_lines.append(line)
+ if not found:
+ out_lines.append(f'description: {new_desc}')
+ fm_new = "\n".join(out_lines)
+ return "---\n" + fm_new + "\n---" + rest
+ # No frontmatter; add one
+ return f"---\ndescription: {new_desc}\n---\n{content}"
+
+
+def apply_url_targeted_replacements(
+ root: Path, mapping: Dict[str, Mapping], apply: bool
+) -> Tuple[List[Tuple[Path, Mapping]], List[Tuple[str, str]]]:
+ """
+ Replace frontmatter descriptions based on URL-to-file mapping.
+ Returns:
+ - list of (path, mapping) that were matched (or would be changed)
+ - list of (url, reason) for misses
+ """
+ hits: List[Tuple[Path, Mapping]] = []
+ misses: List[Tuple[str, str]] = []
+
+ for old, m in mapping.items():
+ target_path = resolve_doc_path(root, m.url)
+ if not target_path:
+ misses.append((m.url, "no local file for URL"))
+ continue
+ try:
+ content = target_path.read_text(encoding="utf-8")
+ except Exception as exc: # pragma: no cover - IO guard
+ misses.append((m.url, f"read error: {exc}"))
+ continue
+ new_content = update_frontmatter_description(content, m.new)
+ if new_content != content:
+ hits.append((target_path, m))
+ if apply:
+ target_path.write_text(new_content, encoding="utf-8")
+ else:
+ # Already matches desired state
+ hits.append((target_path, m))
+ return hits, misses
+
+
+def summarize(
+ text_scan_results: List[Tuple[Path, Dict[str, int]]],
+ url_hits: List[Tuple[Path, Mapping]],
+ url_misses: List[Tuple[str, str]],
+ mapping: Dict[str, Mapping],
+ report_path: Path,
+):
+ total_text_files = len(text_scan_results)
+ total_text_hits = sum(sum(counts.values()) for _, counts in text_scan_results)
+ print(f"Text scan - files with matches: {total_text_files}")
+ print(f"Text scan - total occurrences: {total_text_hits}")
+ for path, counts in sorted(text_scan_results, key=lambda x: str(x[0])):
+ print(f"- {path}")
+ for old, count in counts.items():
+ info = mapping.get(old)
+ url = info.url if info else ""
+ print(f" * {count}x '{old}' -> '{info.new if info else ''}' (URL: {url})")
+
+ print("\nFrontmatter updates (URL-targeted):")
+ for path, m in sorted(url_hits, key=lambda x: str(x[0])):
+ print(f"- {path} <- {m.url}")
+
+ if url_misses:
+ print("\nSkipped URLs (no local file):")
+ for url, reason in url_misses:
+ print(f"- {url} ({reason})")
+
+ report = {
+ "text_scan": {
+ "files_with_matches": total_text_files,
+ "total_occurrences": total_text_hits,
+ "files": [
+ {
+ "path": str(path),
+ "occurrences": counts,
+ "urls": {old: mapping[old].url for old in counts if old in mapping},
+ }
+ for path, counts in text_scan_results
+ ],
+ },
+ "frontmatter_updates": {
+ "hits": [{"path": str(p), "url": m.url, "new": m.new} for p, m in url_hits],
+ "misses": [{"url": url, "reason": reason} for url, reason in url_misses],
+ },
+ }
+ print("\nJSON summary:")
+ print(json.dumps(report, indent=2))
+ if report_path:
+ report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
+ print(f"\nReport written to: {report_path}")
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Replace Scenario docs meta descriptions using a CSV mapping."
+ )
+ parser.add_argument(
+ "--csv",
+ type=Path,
+ default=Path(DEFAULT_CSV),
+ help="Path to CSV file with columns: Meta description (OLD), Meta NEW, URL.",
+ )
+ parser.add_argument(
+ "--url-prefix",
+ type=str,
+ default=DEFAULT_URL_PREFIX,
+ help="Only apply rows whose URL starts with this prefix (default: scenario docs).",
+ )
+ parser.add_argument(
+ "--root",
+ type=Path,
+ default=DEFAULT_ROOT,
+ help="Root directory to scan (defaults to docs/docs).",
+ )
+ parser.add_argument(
+ "--exts",
+ type=str,
+ default=",".join(sorted(DEFAULT_EXTS)),
+ help="Comma-separated list of file extensions to scan.",
+ )
+ parser.add_argument(
+ "--apply",
+ action="store_true",
+ help="Write changes. If omitted, runs in dry-run mode.",
+ )
+ parser.add_argument(
+ "--report",
+ type=Path,
+ default=Path("scenario_meta_report.json"),
+ help="Path to write JSON summary (default: scenario_meta_report.json).",
+ )
+ args = parser.parse_args()
+
+ exts = {ext.strip().lower() for ext in args.exts.split(",") if ext.strip()}
+ csv_path = resolve_csv_path(args.csv)
+ mapping = load_mapping(csv_path, args.url_prefix)
+ print(
+ f"Loaded {len(mapping)} mappings from {csv_path} with URL prefix {args.url_prefix}"
+ )
+
+ root = args.root
+ if not root.exists():
+ raise FileNotFoundError(f"Root directory not found: {root}")
+
+ # Legacy text scan (kept for compatibility; often zero for Scenario docs)
+ text_scan_results: List[Tuple[Path, Dict[str, int]]] = []
+ files_scanned = 0
+
+ for file_path in iter_text_files(root, exts):
+ files_scanned += 1
+ _, counts = apply_replacements(file_path, mapping, apply=args.apply)
+ if counts:
+ text_scan_results.append((file_path, counts))
+
+ # URL-targeted frontmatter description updates
+ url_hits, url_misses = apply_url_targeted_replacements(root, mapping, apply=args.apply)
+
+ mode = "APPLY" if args.apply else "DRY-RUN"
+ print(f"\nMode: {mode}")
+ print(f"Root: {root}")
+ print(f"Files scanned: {files_scanned}")
+ summarize(text_scan_results, url_hits, url_misses, mapping, args.report)
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except Exception as exc: # pragma: no cover - CLI helper
+ print(f"ERROR: {exc}", file=sys.stderr)
+ sys.exit(1)
+
+