diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index eeeb42f..dae865a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,35 +1,26 @@ { "name": "quantecon", + "owner": { + "name": "QuantEcon" + }, "description": "QuantEcon's shared Claude Code skills and tools", "version": "0.2.0", "plugins": [ { "name": "qe", - "source": { - "source": "github", - "repo": "QuantEcon/skills", - "path": "qe" - }, + "source": "./qe", "version": "0.1.0", "description": "QuantEcon's author-facing base skills — style checking and lecture editing support" }, { "name": "benchmark", - "source": { - "source": "github", - "repo": "QuantEcon/skills", - "path": "benchmark" - }, - "version": "0.1.0", + "source": "./benchmark", + "version": "0.3.0", "description": "Benchmarking and acceleration-evaluation tools for QuantEcon lecture code" }, { "name": "audit", - "source": { - "source": "github", - "repo": "QuantEcon/skills", - "path": "audit" - }, + "source": "./audit", "version": "0.1.0", "description": "Bulk, read-only audits of a QuantEcon repository — issue triage, PR review, technical debt, translation parity — each producing an evidence-cited report bundle" } diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f4c9c5e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# Normalise line endings in the repository so an editor's default on one +# platform cannot turn the next edit of a file into a whole-file diff that +# buries the real change. Text is stored LF in git and checked out LF. +* text=auto eol=lf + +# Binary-ish assets git should never touch. +*.png binary +*.jpg binary +*.pdf binary diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8c3cb0a..16e4c59 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -15,3 +15,19 @@ jobs: python-version: "3.12" - name: Validate manifests and skill frontmatter run: python scripts/validate.py + + # The benchmark plugin's claim is that no score is ever written by hand: + # every scorecard is a deterministic function of its evidence.json. That + # only stays true if it is checked. A non-empty diff here means either a + # scorecard was hand-edited, or a rubric change moved a published verdict + # without the baselines being regenerated — in the second case the fix is + # to re-run these two commands and commit, so the verdict move lands in + # the diff where a reviewer sees it. Stdlib only; no install step needed. + - name: Scorecards reproduce from evidence + working-directory: benchmark + run: | + python scripts/scoring/score.py references/examples/ge_arrow + python scripts/scoring/score.py references/examples/markov_asset + python scripts/scoring/score.py references/fixtures/rubric_v2 + git diff --exit-code -- 'references/examples/*/results/scorecard.json' \ + 'references/fixtures/*/results/scorecard.json' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..88c478e --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.py[cod] +.DS_Store + +# Generated per-run by each example's run_all.py — machine-local, not part of +# the committed regression baseline (the committed provenance is evidence.json) +benchmark/references/examples/*/results/as_used.json +benchmark/references/examples/*/results/cold_start.json +benchmark/references/examples/*/results/env.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..bab04ca --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,60 @@ +# AGENTS.md + +Guidance for AI coding agents and human contributors working in `QuantEcon/skills`. This is the canonical instructions file; tool-specific files point here — Claude Code reads [CLAUDE.md](CLAUDE.md), which imports it. + +## What this repository is + +A [Claude Code plugin marketplace](https://code.claude.com/docs/en/plugin-marketplaces) holding QuantEcon's shared agent skills and the deterministic scripts they drive. The goal is to share institutional knowledge — the checks, rubrics and procedures experienced maintainers already apply by hand — so the same work produces consistent results wherever it runs. Orientation is in [README.md](README.md); what has actually shipped is in [CATALOG.md](CATALOG.md); work in flight is in the per-plugin tracking issues. + +**The repo is early, and its conventions are deliberately loose.** Where a doc describes a report shape, a phase division, a naming form or a directory layout, read it as what an existing skill does rather than as a contract a new one must satisfy — see [CATALOG.md § Principles](CATALOG.md#principles). The few things that genuinely must hold are stated plainly, with their reasons, and they are all about keeping a skill's output checkable by someone who will not re-run it. + +## Single source of truth + +The governing principle for everything here — [@jstac](https://github.com/jstac), by email, July 2026: + +> For example, skills point to existing documentation in the manual wherever possible, instead of repeating what the manual says. + +Content lives in exactly one place; everywhere else links to it. Restated copies drift, and a drifted copy is worse than no copy — a reader who finds two versions cannot tell which is current, and neither can an agent. + +What this means in practice: + +- **Skills point outward.** When a skill needs a rule, a convention, or a procedure that already exists in the [QuantEcon manual](https://manual.quantecon.org) or in `QuantEcon/style-guide`, it cites and links to it. A `SKILL.md` carries only what the skill itself adds: the procedure it runs, the judgement it applies, the output it produces. +- **Rule text is authored upstream only.** `qe/references/rules/` is a *rendered consumer* of `QuantEcon/style-guide`, kept aligned by a render target and a CI drift check — never hand-edited here. See [qe/references/rules/README.md](qe/references/rules/README.md). +- **Numbers drift fastest** — weights, thresholds, verdict bands, versions. The rubric's weights are stated in [`benchmark/references/EVALUATION_FRAMEWORK.md`](benchmark/references/EVALUATION_FRAMEWORK.md) and implemented once in `benchmark/scripts/scoring/rubric.py`; anywhere else they come up, quote with a pointer rather than re-tabulating. +- **Every topic has an owning doc** (see the map below). Before adding a section, work out which file owns the topic, put it there, and link from wherever else it comes up. +- **Across boundaries, link — don't copy.** An installed plugin ships only its own directory, so a reference to a repo-level file or another plugin is an absolute GitHub URL, never a duplicated paragraph ([developing-skills § Conventions](docs/developing-skills.md#conventions)). +- **The one deliberate exception**: a `SKILL.md` frontmatter `description` must stand alone, because it is what natural-language invocation matches against. Restate what the skill does in that one sentence; the details stay behind the link. + +Before adding a paragraph, check whether it already exists. If it does, link to it. If it already exists twice, collapsing the two into one plus a pointer is a fix, not scope creep. + +## Where things are documented + +| Topic | Canonical location | +|---|---| +| What the marketplace is, installation (local, lecture repos, CI) | [README.md](README.md) | +| Using the skills: setup, invocation, what to expect | [docs/using-skills.md](docs/using-skills.md) | +| Contributing: layout, conventions, dev loop, local testing, versioning, PR flow | [docs/developing-skills.md](docs/developing-skills.md) | +| Running an evaluation by hand, end to end | [docs/tutorial-run-an-evaluation.md](docs/tutorial-run-an-evaluation.md) | +| The benchmark skill: modes, report format, manual pipeline | [benchmark/README.md](benchmark/README.md) | +| Rubric: dimensions, weights, anchors, verdict bands | [benchmark/references/EVALUATION_FRAMEWORK.md](benchmark/references/EVALUATION_FRAMEWORK.md) | +| Style rule text and schema | `QuantEcon/style-guide` (upstream — never authored in this repo) | +| What has shipped, and the principles behind it | [CATALOG.md](CATALOG.md) | +| Parked ideas, not committed to | [FUTURE-IDEAS.md](FUTURE-IDEAS.md) | +| Work in flight, per plugin | issues [#3](https://github.com/QuantEcon/skills/issues/3) (`qe`), [#4](https://github.com/QuantEcon/skills/issues/4) (`benchmark`), [#12](https://github.com/QuantEcon/skills/issues/12) (`audit`) | + +## Working in this repo + +- **Validate before committing**: `python scripts/validate.py`. A malformed manifest breaks installation silently in every consuming repo, so CI runs the same check. +- **Test from a real consuming project**, not from inside this repo — path-resolution bugs only surface when a plugin runs from an install location. Both tiers are in [developing-skills § Testing locally](docs/developing-skills.md#testing-locally). +- **The product principles** — report first, fix on request; deterministic before LLM; cited claims and computed scores; scaffolding as advice rather than instruction — are stated once in [CATALOG.md § Principles](CATALOG.md#principles) and elaborated in [developing-skills § Conventions](docs/developing-skills.md#conventions). Follow them; don't restate them in new files. +- **A new skill starts as an issue, not a doc entry.** CATALOG.md lists what has merged, so it stays true; the plan for something unbuilt belongs in its plugin's tracking issue, where it can change without anyone mistaking it for a description of the repo. +- **Commit subjects** name the area, then the change: `Docs: hands-on evaluation tutorial…`, `Rubric v2: enforced couplings…`. The repo squash-merges, so stacked branches need `git rebase --onto origin/main ` once the base PR lands. +- **`NEXT-SESSION.md` is scratch**: deliberately uncommitted working notes. Don't commit it and don't cite it as documentation. + +## Writing to GitHub + +Issue bodies, PR descriptions, and comments render differently from committed Markdown — write for the renderer: + +- **Don't hard-wrap paragraphs.** GitHub turns a single newline into a line break, so source wrapped at 80 characters renders as ragged mid-sentence breaks. One unbroken line per paragraph, blank line between paragraphs. (Committed `.md` files are the opposite case, where wrapping is fine.) +- **Don't put prose in fenced code blocks.** A fence renders as a fixed-width scrolling box that crops the readable width. Use tables and lists for explanation; reserve fences for code and commands meant to be copied. +- **Don't precede a cross-repo reference with a closing keyword.** `Fixes QuantEcon/style-guide#6` auto-closes that upstream issue when the commit lands on `main`. Write `See …`, `Mirrors the change in …`, or `Ports the fix from …` instead. diff --git a/CATALOG.md b/CATALOG.md index ff6caf1..4b55dc4 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,47 +1,22 @@ -# Skill catalog — active plan +# Skill catalog -Current focus, validated against ~630 merged PRs across the four main lecture repos (2026-07-21). Parked ideas live in [FUTURE-IDEAS.md](FUTURE-IDEAS.md). +What this marketplace installs today. A skill appears here once it is merged — anything not listed does not exist yet, however firmly it has been discussed. Work in flight lives in the tracking issue for its plugin; ideas nobody has committed to live in [FUTURE-IDEAS.md](FUTURE-IDEAS.md). -## Principles - -- **Few, high-frequency skills** over many niche ones; every skill is validated against actual PR history. -- **Focal point is PR management**: consistent review results, plus the same checks run by authors on a working copy before opening a PR. -- **Report first, fix on request.** Skills produce a structured report and offer fixes; they never silently edit. Safe in CI, authors stay in control. - -## 1. Style skill family — flagship - -Check a lecture (working copy or PR diff) against the QuantEcon style guide; report violations by rule ID; offer fixes. - -**Evidence:** the largest recurring PR theme — ~25+ human style PRs across all repos plus the `action-style-guide` campaign in lecture-python-advanced; the ~30 RA "editorial suggestions" PRs in lecture-python-intro overlap almost 1:1 with the rule set, so this skill absorbs that workflow too. Weakest audit categories (Figures 7.4/10 corpus-wide; Math 5.6 in adv) show where it pays first. - -**Naming and structure (decided 2026-07-21):** the author-facing surface lives in a **`qe` plugin** — one memorable prefix for the author network — while `benchmark` remains its own specialist plugin. An umbrella **`/qe:check-style`** runs the full check (optional trailing category words, e.g. `/qe:check-style aiyagari figures math`), and **thin per-category sub-skills** (`/qe:check-writing`, `/qe:check-math`, `/qe:check-code`, `/qe:check-figures`, `/qe:check-jax`, `/qe:check-refs`) give autocomplete discoverability and precise natural-language auto-triggering. Sub-skills are ~10-line pointers; all rule content lives once at plugin level (`qe/references/rules/`, `qe/scripts/`), so there is no duplicated rule text to keep in sync. Future author-facing base skills (see [FUTURE-IDEAS.md](FUTURE-IDEAS.md)) join the `qe` plugin. - -**Rules architecture (two layers):** - -1. **Canonical source:** `QuantEcon/style-guide` — the machine-readable rules DB. Its Phase 0 (schema, `build/validate.py` + CI, fixtures convention, governance) landed 2026-06-11; Phase 1 (transcribing the ~55 catalogued rules) restarts with this skill work supplying the priority order and the labour. Rule text is authored **only** there (the programme's no-drift exit criterion); the manual's `styleguide/` pages and this plugin both become rendered consumers. Proposal: [project-style-guide#6](https://github.com/QuantEcon/project-style-guide/issues/6). -2. **Rendered snapshot (in this plugin):** `qe/references/rules/` holds a rendered, drift-checked copy (`render-skill` target upstream + `scripts/sync-rules.py` + CI check here) in the style-guide schema (`qe-*` IDs, `mode: mechanical|hybrid|llm`, `severity`, `build_risk`, `auto_fix`, `exclusions`). Open decision (issue #6): `style-guide` is private (D8) while this marketplace is public — vendor full rules (revisit D8), fetch at runtime via `gh`, or render summaries only. +| Plugin | Skills | State | Tracking | +|---|---|---|---| +| **`qe`** — author-facing style checks | `/qe:check-style`, plus six per-category siblings: `check-writing`, `check-math`, `check-code`, `check-figures`, `check-jax`, `check-refs` | Scaffolding. The rendered rule snapshot and the deterministic preflight have not landed, and the skills report that they are not yet operational when run. | [#3](https://github.com/QuantEcon/skills/issues/3) | +| **`benchmark`** — evaluating accelerated lecture implementations | `/benchmark:review-acceleration` | Operational for workspace runs: rubric v2, a deterministic scoring engine, and two complete worked evaluations as regression baselines. | [#4](https://github.com/QuantEcon/skills/issues/4) | +| **`audit`** — bulk, read-only repository audits | `/audit:issues` | Runbook landed, executed once end to end against a real repo. | [#12](https://github.com/QuantEcon/skills/issues/12) | -**Check machinery (inside the skill, regardless of naming):** +Installation and setup are in [README.md](README.md); what it is like to run one is in [docs/using-skills.md](docs/using-skills.md). -- **Deterministic preflight scripts first** — the three `build_risk` rules (`align` in `$$`, tick-count nesting, floats inside exercise/solution/`prf:`) plus the ~24 mechanical rules (legacy `np.random.*`, `time.time()`, `\mathcal N`, `pip install jax`, `PRNGKey`, transpose notation with derivative carve-outs, …). Must be MyST-context-aware (narrative vs math env vs code cell) — plain grep produces false positives; the style-guide repo's `tests/fixtures/` layout is the ready-made zero-FP test harness. -- **LLM passes per category** for hybrid/judgement rules (heading case with proper-noun allow-list, one-sentence-paragraph splitting, caption quality, JAX anti-patterns, …). -- **Never auto-fix `build_risk`/RNG rules** — changing an RNG stream changes published figures; report + guided fix only. -- The "should this lecture use JAX at all" gate stays thin here — that judgement belongs to `review-acceleration` (the boundary QuantEcon.manual#104 draws). - -This delivers the planned `qestyle-linter` + `qestyle` pair as Claude Code skills — consistent with programme decision D2 ("CLI engine, Claude Code as the interface") and the planned split-and-retire of `action-style-guide`. - -## 2. `/benchmark:review-acceleration` — finish - -Blocked on the eight evaluation scripts (@xuanguang-li agreed 2026-07-07 to package and send). **Unblocked now:** the full rubric (7 weighted dimensions, verdict bands, HIGH/LOW calibration cases) is specified in the lecture-python.myst#717 thread and can move into `references/rubric.md` ahead of the scripts. Validation cases: lecture-python.myst#717 and #654. - -## 3. `audit` — the bulk-audit family - -Portfolio-wide, read-only sweeps of one repository, each delivering a report bundle. Four skills: **`/audit:issues`** (every issue, open and closed — landed), then **`/audit:prs`** (every open PR: does it solve a real issue, is it mergeable, what should the review say), **`/audit:tech-debt`** (a codebase's debt plus a filing-ready issue catalog), **`/audit:translations`** (parity between a source series and its translation). - -**Evidence:** unlike the style family, the case here is not per-repo frequency — a tracker audit is a once-or-twice-a-year event for any one repo. It is *breadth*: the org has ~245 non-archived repos, and three of the four procedures have already been executed by hand — the issue triage this plugin ships, the `quantecon-py` technical-debt report, and the zh-cn translation work. Each hand run cost hours of re-derivation because the method lived in a pasted prompt. +## Principles -**Structure (proposed 2026-07-26):** four sibling skills, no umbrella — unlike `qe`'s categories, these are distinct procedures over distinct inputs, so the `qe` pattern to reuse is plugin-level sharing, not the umbrella. The method is authored once in `audit/references/` (`doctrine.md`, `quantecon-context.md`, `deliverables.md`) and each `SKILL.md` carries only its own subject matter. Membership test: bulk **and** read-only **and** report-bundle output — which keeps single-item review (`/benchmark:review-acceleration`) outside the family and stops the plugin becoming a general runbook dump. +The point of the marketplace is to **share institutional knowledge** — the checks, rubrics and procedures that experienced maintainers already apply by hand — so the same work produces more consistent results wherever it is run, across roughly 245 non-archived repos. Everything below serves that. -**Why read-only is structural, not cautious:** the boundary mirrors the org's own automation split, where the family line *is* the permission line. It makes the family safe to point at any repo and safe to run headlessly, and it keeps a report honest — an audit that half-applied its findings would describe a repo that no longer exists. Acting on a bundle (filing the catalog, posting drafted comments, `qe gh labels sync`) is a separate human-invoked step. +- **Few, high-frequency skills** over many niche ones, each validated against actual PR history. The 2026-07-21 analysis of ~630 merged PRs across the four main lecture repos is the evidence base: style was the largest recurring theme by a wide margin, which is why it is the flagship. A skill justified by breadth rather than frequency, as the audit family is, should say so. +- **Report first, fix on request.** Skills produce a structured report and offer fixes; they never silently edit. Safe to run in CI, and authors stay in control. +- **Cited claims; computed scores.** Every finding carries a citation — a rule ID plus `file:line`, or a number plus its source. Skills whose output is a findings list need nothing more. Skills that aggregate judgements into a scored verdict use the evidence-file pattern from the benchmark plugin: judgement recorded as cited answers, every score computed by a deterministic engine, never typed by hand (see [docs/developing-skills.md](docs/developing-skills.md)). +- **Scaffolding is advice, not instruction.** Report shapes, phase divisions, naming forms and directory conventions are described as what an existing skill does, not as contracts a new one has to satisfy. Three plugins is not enough to know which of them generalise, and a rule invented from one worked example mostly succeeds at forcing the next skill into the first one's shape. A skill can be a single `SKILL.md`. Where something genuinely must hold — read-only boundaries, cited claims, a stated coverage of what was and was not checked — say so plainly and give the reason; everything else can converge later, once there is something to generalise from. -**Long-run machinery:** every skill runs the same five phases (snapshot → verify → relate → write → self-audit), each checkpointed to disk so a lost session resumes rather than restarts, and all reading one frozen snapshot so the report describes a single point in time. Phase 1 is deterministic (`audit/scripts/fetch_tracker.py`), which also makes the coverage self-audit mechanical rather than narrated. +Note what the last two have in common: the rules stated firmly are the ones that keep output *checkable by someone who will not re-run it*. That is the test worth applying before writing any new rule down. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..61f7cfb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +# CLAUDE.md + +Repository guidance for Claude Code lives in [AGENTS.md](AGENTS.md) — the canonical, tool-agnostic instructions (what this marketplace is, the **single-source-of-truth principle** that governs where content lives, the doc map, and the working conventions). + +@AGENTS.md diff --git a/FUTURE-IDEAS.md b/FUTURE-IDEAS.md index d027744..c62a636 100644 --- a/FUTURE-IDEAS.md +++ b/FUTURE-IDEAS.md @@ -1,6 +1,6 @@ # Future skill ideas -Documented so they aren't lost; deliberately **not** in the active plan. Current focus: the style skill family (see [CATALOG.md](CATALOG.md)). Each idea below is evidence-backed by the 2026-07 PR-history analysis (~630 merged PRs across lecture-python.myst, lecture-python-intro, lecture-python-advanced.myst, lecture-dp). +Documented so they aren't lost; deliberately **not** committed to. What has actually shipped is in [CATALOG.md](CATALOG.md), and work in flight is in the per-plugin tracking issues. Each idea below is evidence-backed by the 2026-07 PR-history analysis (~630 merged PRs across lecture-python.myst, lecture-python-intro, lecture-python-advanced.myst, lecture-dp). ## Parked candidates @@ -28,6 +28,10 @@ meta#338 / data-lectures#15 playbook: data PR first, byte-compare sha256 gate, r Parse a review issue's checklist into a diff against one lecture — lecture-python-intro's dominant RA workflow (~30 PRs). Largely absorbed by the style skill's fix mode; revisit only if a gap remains after it ships. +### A naming convention, once there is something to generalise from + +Two shapes are in use — the verb in the skill (`/qe:check-style`) and the verb in the plugin (`/audit:issues`) — and [developing-skills.md](docs/developing-skills.md) deliberately declines to rank them. Worth revisiting once enough plugins exist to show which reads better in practice, and specifically whether mixing both shapes inside one plugin actually causes trouble or only looks untidy. Writing the rule now would mean generalising from three plugins and no usage. + ## How skills could serve the benchmarking programme (note for meta#335) Benchmark *data capture* stays in the programme repos (`QuantEcon/benchmarks` — workstream A; `QuantEcon/tool-lecture-benchmark` — workstream C). Skills are natural *consumers and interfaces* of that data: diff --git a/README.md b/README.md index 5153ce4..8738c77 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,26 @@ Each plugin bundles one area of work — a skill (the instructions Claude follow ## Plugins -| Plugin | Skills | Status | Tracking | -|---|---|---|---| -| `qe` | `/qe:check-style` (+ `check-writing`, `check-math`, `check-code`, `check-figures`, `check-jax`, `check-refs`) | scaffolding | [CATALOG.md](CATALOG.md), work plan in `project-style-guide` | -| `benchmark` | `/benchmark:review-acceleration` | under construction | [meta#335](https://github.com/QuantEcon/meta/issues/335) | -| `audit` | `/audit:issues` (+ planned `prs`, `tech-debt`, `translations`) | first runbook landed | [CATALOG.md](CATALOG.md) §3 | +| Plugin | For | Covers | +|---|---|---| +| `qe` | Authors and RAs writing lectures | Style checks against the QuantEcon style guide, while editing and before opening a PR | +| `benchmark` | Maintainers reviewing accelerated implementations | Measured, rubric-scored evaluation of a conversion | +| `audit` | Maintainers sweeping a whole repository | Bulk, read-only audits — every issue, every PR, a codebase, a translated series — each producing a written report | -The `qe` plugin is the author-facing surface: one memorable prefix for the skills authors use while editing lectures and preparing PRs. `check-style` is the umbrella (whole lecture, optional category filter, e.g. `/qe:check-style lectures/aiyagari.md figures math`); the per-category sub-skills run the same shared rules individually. `benchmark` is a specialist family for maintainers evaluating accelerated implementations. `audit` is the maintainer-facing family for bulk, read-only sweeps of a whole repository — every issue, every PR, a whole codebase, a whole translated series — each delivering a report bundle rather than a chat answer. See [CATALOG.md](CATALOG.md) for the plan and [FUTURE-IDEAS.md](FUTURE-IDEAS.md) for parked candidates. +`qe` is the author-facing surface — one memorable prefix for everyday work. `check-style` is the umbrella (whole lecture, optional category filter, e.g. `/qe:check-style lectures/aiyagari.md figures math`) and the per-category sub-skills run the same shared rules individually. `benchmark` and `audit` are specialist toolkits, installed by the maintainers who need them. + +**Which skills exist right now, and what state each is in, is in [CATALOG.md](CATALOG.md)** — that list is kept current with what has merged, so this page does not repeat it. Ideas nobody has committed to are in [FUTURE-IDEAS.md](FUTURE-IDEAS.md). + +## Documentation + +| Guide | For | +|---|---| +| [AGENTS.md](AGENTS.md) | AI agents and contributors: canonical repo instructions — the single-source-of-truth principle, doc map, working conventions | +| [docs/using-skills.md](docs/using-skills.md) | Authors/reviewers: setup, invoking skills, what to expect | +| [docs/developing-skills.md](docs/developing-skills.md) | Contributors: layout, conventions, dev loop, testing locally, versioning, PR flow | +| [docs/tutorial-run-an-evaluation.md](docs/tutorial-run-an-evaluation.md) | Tutorial: the evaluation procedure by hand, with the ge_arrow validation run as the checkable example | +| [benchmark/README.md](benchmark/README.md) | The evaluation skill: review mode, triage mode, report format, manual pipeline | +| [audit/README.md](audit/README.md) | The audit family: what belongs in it, the shared method, running one | ## Installation @@ -75,7 +88,7 @@ benchmark/ # specialist plugin audit/ # maintainer-facing bulk-audit plugin .claude-plugin/plugin.json skills/issues/SKILL.md # one skill per audit subject - references/ # shared doctrine, org context, bundle contract + references/ # shared doctrine, org context, reporting guidance scripts/ # deterministic snapshot + coverage machinery ``` diff --git a/benchmark/.claude-plugin/plugin.json b/benchmark/.claude-plugin/plugin.json index 9c55a54..7691505 100644 --- a/benchmark/.claude-plugin/plugin.json +++ b/benchmark/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "benchmark", "description": "Benchmarking and acceleration-evaluation tools for QuantEcon lecture code", - "version": "0.1.0", + "version": "0.3.0", "author": { "name": "QuantEcon" } } diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..040b5ba --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,79 @@ +# benchmark plugin + +Evaluation tooling for QuantEcon lecture code rewrites — the question it answers is never "is JAX faster?" but **"does this implementation earn its place in this lecture?"** Lectures are teaching materials first and programs second; the plugin's rubric weights readability (0.25) above efficiency (0.15) on purpose. + +One skill, two modes: + +| Mode | Question | Needs | Produces | +|---|---|---|---| +| **Review** | Did this conversion PR improve the lecture? | baseline + candidate implementations | A scored report with a merge recommendation | +| **Triage** | Is this lecture worth converting at all? | the existing lecture only | A predicted verdict band with the binding constraint named | + +Status: evaluation system landed (v0.2.0); skill wiring tracked in [skills#4](https://github.com/QuantEcon/skills/issues/4). The system was developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). + +## Using the skill + +``` +/benchmark:review-acceleration # review mode +/benchmark:review-acceleration should we convert ? # triage mode +``` + +### Review mode — what you get + +The skill follows the measure → record-evidence → score contract ([scripts/README.md](scripts/README.md)): it extracts both implementations verbatim from the lecture's code cells, adapts the measurement templates, runs them, fills `evidence.json` with cited answers, and lets the engine compute the verdict — **no score is ever typed by hand**. The session shows the engine's derivation table (every score with the measured number and threshold band that produced it), and the final report follows the worked examples' format: + +1. **TL;DR** — weighted score, verdict band, the decisive facts in one paragraph +2. **Dimension table** — weight / score / weighted contribution / one-line driver each +3. **What changed** — before/after implementation shape +4. **Evidence by dimension** — `max|Δ|` in both dtype regimes, prerequisite-concept and docstring deltas, the as-used vs warm timing table, crossover-n, recompile audit +5. **Recommendation** — a must-fix list where each item is tagged with the dimension it lifts, plus where the score lands after fixes + +See [references/examples/ge_arrow/ge_arrow_REPORT.md](references/examples/ge_arrow/ge_arrow_REPORT.md) (2.85/5, no-conversion; candidate band mixed/wash) and [references/examples/markov_asset/markov_asset_REPORT.md](references/examples/markov_asset/markov_asset_REPORT.md) (2.25/5, no-conversion + gated net regression) for complete real reports. Verdict bands, the v2 verdict gates / no-conversion rule / sensitivity stamp, weights, and scoring anchors: [references/EVALUATION_FRAMEWORK.md](references/EVALUATION_FRAMEWORK.md) §1–2. + +**The one rule to remember:** warm-only speedups are never the headline. The ge_arrow case measured 1.4–4.8× faster warm and **45× slower as-used** — the as-used number (fresh process, actual problem sizes, compile time included) decides the efficiency score. + +### Triage mode — before any code is written + +Four checks, using only the existing lecture: + +1. **Baseline as-used total** — replay the lecture's real call sequence (the NumPy half of an `as_used_total.py` template). This bounds the entire possible win: a lecture whose compute totals 30ms has nothing to give. +2. **Workload-pattern match** — against the two calibrated poles: **aiyagari-shaped** (large fixed-shape arrays, many re-solves, stable static args → measured ~24× as-used win) vs **ge_arrow-shaped** (tiny models, fresh static args per call → measured ~45× as-used loss). +3. **Crossover comparison** — the lecture's problem sizes vs the warm crossover-n from the scaling data. +4. **Readability-cost forecast** — which concepts the conversion would force on readers (static args, `lax` carries, checkify, the float32/x64 distinction), against the prerequisite-concept bands. + +Then the decision rule that falls out of the rubric weights: efficiency (0.15) can gain at most +0.30 weighted (band 3→5), while readability (0.25) losing two bands costs −0.50 — **a conversion that costs meaningful readability cannot break even on speed alone**; it must also win on logic & design and ergonomics, and those structural wins are usually achievable in plain NumPy. + +**Validation (2026-07-21):** triage applied blind (baseline-side data only) to the three known cases reproduces every known verdict: + +| Case | Baseline total (triage-time) | Pattern | Triage says | Full evaluation says | +|---|---|---|---|---| +| ge_arrow | 0.028 s | n=2/3, fresh static args | don't convert | no-conversion; candidate quality 2.85/5, mixed/wash | +| markov_asset | 0.087 s | n=5/25, LAPACK-bound | don't convert | no-conversion; candidate quality 2.25/5, net regression | +| aiyagari pattern | 54.3 s | 200×7 fixed, 20 re-solves | convert | 23.8× as-used win | + +The baseline totals above are the **triage-time** measurements taken on 2026-07-21, kept as a record of what that blind run saw. They are not the numbers the no-conversion gate reads: that value is each lecture's `baseline_as_used_seconds` in its own `evidence.json`, re-measured at evaluation time. Both are the same quantity, so expect them to differ by run rather than to agree. + +Scope limit, confirmed by the same test: triage predicts whether the prize is worth pursuing — it cannot predict conversion-quality outcomes (markov_asset's masked `err.throw()` defect was a property of the PR, invisible to triage). Note also that this validation is **in-sample** — the three cases are the ones the thresholds were calibrated on; out-of-sample validation accumulates as fresh lectures are triaged. + +## Manual usage (no skill) + +The full recipe is in [scripts/README.md](scripts/README.md) ("Evaluating a new lecture"); quickstart from this directory: + +```bash +conda activate quantecon +python references/examples//scripts/run_all.py # measure + provenance stamp +# fill references/examples//evidence.json (numbers + cited yes/no answers) +python scripts/scoring/score.py references/examples/ +``` + +Sanity anchors: re-running either worked example must reproduce **2.85** / **2.25** (both now carrying the v2 **no-conversion** verdict; ge_arrow stamps *fragile*, markov_asset *robust-at-floor* — its verdict is already bottom-band, so nothing could move it downward). The synthetic fixtures in [`references/fixtures/`](references/fixtures/) reproduce alongside them; CI regenerates all three and fails on any diff. A step-by-step walkthrough of the whole procedure — with the ge_arrow reproduction as a checkable example — is [docs/tutorial-run-an-evaluation.md](https://github.com/QuantEcon/skills/blob/main/docs/tutorial-run-an-evaluation.md). + +## Map + +| Path | What | +|---|---| +| [skills/review-acceleration/](skills/review-acceleration/SKILL.md) | The skill (procedure, both modes) | +| [scripts/README.md](scripts/README.md) | Deterministic engine (`scripts/scoring/`): rubric, scorer, evidence template, provenance stamp | +| [scripts/calibration/](scripts/calibration/bellman_bench.py) | The shared HIGH-efficiency anchor (~24× ⇒ score 5) | +| [references/EVALUATION_FRAMEWORK.md](references/EVALUATION_FRAMEWORK.md) | The standard in prose — weights, anchors, checklists, verdict bands | +| [references/examples/](references/examples/README.md) | Two complete worked evaluations + the logic-check/provenance audit; the regression baseline | diff --git a/benchmark/references/EVALUATION_FRAMEWORK.md b/benchmark/references/EVALUATION_FRAMEWORK.md new file mode 100644 index 0000000..bb40332 --- /dev/null +++ b/benchmark/references/EVALUATION_FRAMEWORK.md @@ -0,0 +1,312 @@ +# A Quantitative Evaluation System for JAX Rewrites of QuantEcon Lectures + +This document defines a **reusable, quantitative system** for deciding whether rewriting a QuantEcon lecture's code (e.g. converting NumPy → JAX) actually *improves the lecture*. It was designed against the first such change, `lectures/ge_arrow.md` (branch `update_ge_arrow` vs `main`), and is applied to it in [`ge_arrow_REPORT.md`](examples/ge_arrow/ge_arrow_REPORT.md). + +The guiding principle: **these are teaching lectures first and programs second.** A rewrite that makes the code faster or more "modern" but harder for a learner to read, or that silently changes the numbers, is not automatically an improvement. The system therefore weights pedagogy heavily and never treats "uses JAX" as a goal in itself — JAX must *earn* its place on each lecture. + +--- + +## 1. The seven dimensions + +| # | Dimension | Weight | What it answers | +|---|-----------|:---:|---| +| 1 | Correctness & numerical fidelity | 0.20 | Does the new code compute the *same economics*, at the *same precision*? | +| 2 | Readability & pedagogical clarity | 0.25 | Can a learner follow it? Does the code mirror the math? | +| 3 | Computational efficiency (as used) | 0.15 | Is it faster *in the regime the lecture actually runs*? | +| 4 | Logic & design | 0.15 | Are functions natural, pure, non-repetitive, bug-free? | +| 5 | Coding style & idiom | 0.10 | Idiomatic Python/JAX and consistent with house style? | +| 6 | API ergonomics & reusability | 0.10 | How easy is the object to call, compose, and reuse? | +| 7 | Maintainability & robustness | 0.05 | How easy to test, debug, and safely extend later? | + +Weights sum to 1.0. **Readability (0.25) outranks efficiency (0.15)** on purpose: the audience is learners, and most lecture models are tiny. Adjust the weights per lecture family if needed (e.g. a "performance" lecture could raise dimension 3), but record any change. + +Each dimension is scored **1–5** against the anchors below, then combined: + +``` +weighted_total = Σ weight_d × score_d (range 1–5) +``` + +### Interpreting the total + +| Total | Meaning | +|---|---| +| **≥ 4.0** | Clear improvement — merge. | +| **3.0 – 3.9** | Net positive but with fixable regressions — merge after addressing them. | +| **2.5 – 2.9** | Mixed / wash — improvements offset by real regressions; revisit before merging. | +| **< 2.5** | Net regression — do not merge as-is. | + +The band is the *starting point* of the verdict, not the whole of it. Three v2 rules (from the 2026-07-21 design review; all enforced in `score_all`, never by reviewer discipline) modify what the scorecard reports: + +- **Verdict gates.** Correctness 1 caps the verdict at *net regression*; correctness 2 caps it at *mixed/wash* — a candidate whose numbers are wrong cannot weighted-average its way into a merge band, whatever its polish. The logic&design correctness-bug cap is likewise **derived** from the correctness evidence (`builds` false, or x64 divergence — either one, unconditionally) rather than trusted to a hand-set boolean. +- **No-conversion.** When the efficiency evidence shows the triage don't-convert profile — baseline as-used total under the **1 s materiality floor** *and* the candidate slower as-used — the verdict says *no-conversion* instead of scoring the polish, with the banded candidate quality recorded alongside. The 1 s floor is a **policy choice, anchored not derived**: the validated don't-convert baselines sit two orders of magnitude below it and the convert case (aiyagari pattern, ~54 s) two above, so its exact placement inside that gap is not load-bearing. The measured baselines are deliberately not restated here — the value the gate reads is each lecture's own `baseline_as_used_seconds` in `evidence.json`. +- **Sensitivity stamp.** `score.py` perturbs every scored input one at a time (booleans flipped, integer counts ±1, measured floats ±10%) and stamps the scorecard **robust**, **robust-at-floor**, or **fragile** with the deciding flips listed — the total is reported at the precision the instrument actually supports. *robust-at-floor* is issued when nothing moved the outcome **and** the verdict is already in the bottom band: there no single input can make it worse, so only upward moves were available to the search and the stability is partly the band's geometry rather than the evidence's strength. It is not the same claim as *robust*, and reports must not collapse the two. + + Known limit (unresolved): the perturbation walk flips measured observations (`builds`, `matches_under_x64`) and adjudicated judgements (checklist criteria) alike, so *fragile* answers "would a different observation change this?" and "would a different opinion change this?" in one word. Splitting the two is a v3 change; until then, read the deciding-flip list rather than the stamp alone. + +The headline as-used metric is a **median of ≥ 3 fresh-process runs** per side (`run_all.py` repeats the as-used steps); when the runs alone span more than one efficiency band, the scorecard carries a *contested band* annotation. + +A score is only as good as its evidence. **Every dimension must cite at least one measured number or a concrete code excerpt** (see §3). The scripts in [each example's `scripts/`](examples/) produce the numbers automatically. + +--- + +## 2. Scoring anchors + worked high/low examples + +For each dimension we give (a) the metric(s) that quantify it, (b) the 1–5 anchors, and (c) a **HIGH-scoring** and **LOW-scoring** example so reviewers agree on what "good" looks like. + +**Dimensions 1, 2, 3, 6 carry numeric score thresholds** (a measured number maps directly to 1–5); dimensions 4, 5, 7 are structural and scored against criteria + cited evidence. The numeric thresholds were calibrated against two real, *measured* end points: a HIGH case (the aiyagari Bellman pattern, 25× faster as-used) and a LOW case (the full `ge_arrow` lecture, 45× slower as-used). + +**Every example below is real code already in `lecture-python.myst`**, cited by `file:line`, not a hypothetical. The HIGH examples are mostly drawn from lectures the QuantEcon team has already converted well to JAX — `aiyagari.md`, `lake_model.md` — and the LOW examples from `ge_arrow.md` and the older class-based `odu.py`. (Line numbers are as of branch `update_ge_arrow` / `main` at the time of writing; search the cited symbol if they drift.) + +### How a score is computed (no hand-typed numbers) + +The rubric is machine-encoded in [`scoring/rubric.py`](../scripts/scoring/rubric.py) so that **a score is a deterministic function of evidence**, applied identically to every lecture. The workflow — and the contract an AI skill follows — is: + +1. **Measure** (per lecture, objective): run `/scripts/run_all.py` → `/results/*.json`. +2. **Record evidence** (per lecture): fill `/evidence.json` (schema: [`scoring/EVIDENCE_TEMPLATE.json`](../scripts/scoring/EVIDENCE_TEMPLATE.json)) — copy the measured numbers into the quantitative slots (noting the source file) and answer each structural checklist item true/false **with a citation to the diff**. This file, plus the measured results, is the *only* per-lecture input. +3. **Score** (shared, mechanical): `python scripts/scoring/score.py references/examples/` applies `rubric.py` and writes `/results/scorecard.json`, printing the derivation of every score. No score is ever written by hand; to change one you change a measured metric, a checklist answer, or the standard itself. + +**Quantitative dimensions (1, 2, 3, 6)** map a measured number to 1–5 via the threshold tables in the sections below (calibrated against two measured end points: the aiyagari Bellman pattern at 25× faster as-used, and the full `ge_arrow` lecture at 45× slower). + +**Structural dimensions (4, 5, 7)** each have a fixed **4-item yes/no checklist**; the score is **`1 + (number of criteria met)`**, plus a small number of documented override caps. The checklists are: + +| Dim | Criterion 1 | Criterion 2 | Criterion 3 | Criterion 4 | Override | +|---|---|---|---|---|---| +| **4 Logic & design** | pure / no order-dependence | no global state | good algorithmic choices | fixes prior bugs | introduces a correctness bug → **cap 3** | +| **5 Style & idiom** | vectorised where natural | correct control-flow primitive | no anti-idiomatic constructs | clean call sites & naming | — | +| **7 Maintainability** | pure / unit-testable | dtype/precision-safe | no footgun for editors | robust (no brittle conditions) | — | + +Worked check (from the two committed `evidence.json` files): `ge_arrow` style meets only *clean call sites* → 1+1 = **2**; `markov_asset` logic meets all four but introduces a build-breaking bug → 1+4 = 5, capped to **3**. + +--- + +### Dimension 1 — Correctness & numerical fidelity · weight 0.20 + +**Metrics** (from `check_equivalence.py`): +- `all_equivalent` — do all equilibrium objects (Q, R, A, V, α, ψ, J) match the original across every example economy? +- `max_abs_err` — largest absolute deviation from the original numbers. +- default dtype / precision (float32 vs float64). + +**Anchors (numeric — keyed to `max|Δ|` vs the original, as the lecture ships)** + +| Score | as-shipped `max\|Δ\|` | precision | +|:---:|:---:|---| +| 5 | ≤ 1e-10 | float64 preserved; any diff explained | +| 4 | ≤ 1e-8 | preserved | +| 3 | logic matches under x64 (≤1e-10) **but ships float32** → 1e-5…1e-3 drift, unflagged | +| 2 | 1e-3 … 1e-1 on some object, or instability in edge cases | +| 1 | > 1e-1 / NaN where equality is expected — wrong economics | + +The `max|Δ|` bands above apply only when the logic agrees. `matches_under_x64` records the equivalence check re-run with `JAX_ENABLE_X64=1`, and TRUE means the economics agree once precision is removed from the question — residuals at x64 noise (~1e-14 to ~1e-11) are recorded TRUE. **FALSE therefore asserts that the economics genuinely differ, and scores 1 on its own, whatever the shipped `max|Δ|`.** Conditioning that cap on a visible shipped delta as well (as v2.0 did) made the guard weakest exactly where the defect is hardest to see: a candidate whose logic diverges but whose float32 output happens to agree closely is the *wrong economics masked by low precision* case this dimension exists to catch. Agreement as shipped is luck, not correctness. + +> **HIGH (5):** `lectures/aiyagari.md:72` opens the JAX section with +> ```python +> jax.config.update("jax_enable_x64", True) +> ``` +> so its linear solves and value iteration run in double precision — published capital/interest numbers match a NumPy reference to machine epsilon. (`lectures/newton_method.md` does the same.) +> +> **LOW (3, the `ge_arrow` case):** `lectures/ge_arrow.md` has **no** such line. Under float64 the rewrite matches the original to `1.4e-14`, proving the logic is identical — but as shipped it runs in float32, so example 2's printed `α`, `ψ`, `J` move by `1.7e-4`. Correct math, quietly degraded precision. *(Reproduce: run `check_equivalence.py` with and without `JAX_ENABLE_X64=1`.)* + +--- + +### Dimension 2 — Readability & pedagogical clarity · weight 0.25 + +**Metrics** (from `static_metrics.py`) + reviewer reading: +- `n_prerequisite_concepts` — distinct ideas a reader must already know. +- `docstring_coverage` — fraction of defs with a docstring. +- `code_lines`, `n_defs`, closure-nesting depth. +- **Math-to-code distance** (judgement): does a code line look like the equation it implements? + +**Anchors (numeric — keyed to Δ prerequisite-concepts vs the original and to docstring coverage; both from `static_metrics.py`)** + +| Score | new prerequisite concepts | docstring coverage | & | +|:---:|:---:|:---:|---| +| 5 | +0 | ≥ 0.80 | code lines read like the math | +| 4 | +1–2 | ≥ 0.75 | still transparent | +| 3 | +3–4 | 0.60–0.75 | readable if you know the framework | +| 2 | +5–6 | < 0.60 | core formula obscured by plumbing | +| 1 | +7 or more | — | learner can't map a cell to its economics | + +*(Use the worse of the two columns; the "&" column is the tie-breaker.)* + +> **HIGH (5):** `lectures/aiyagari.md:288-300` builds the Bellman right-hand side with broadcasting that visibly mirrors the math $r(a,z,a') + \beta\,E\,v$, and a single branchless feasibility test: +> ```python +> a = jnp.reshape(a_grid, (a_size, 1, 1)) # a[i] -> a[i, j, ip] +> z = jnp.reshape(z_grid, (1, z_size, 1)) # z[j] -> z[i, j, ip] +> ap = jnp.reshape(a_grid, (1, 1, a_size)) # ap[ip] -> ap[i, j, ip] +> c = w * z + (1 + r) * a - ap +> ... +> return jnp.where(c > 0, u(c) + β * EV, -jnp.inf) +> ``` +> A reader sees the budget constraint and the Bellman equation directly. +> +> **LOW (2, the `ge_arrow` case):** `lectures/ge_arrow.md:938-959` expands the one-line kernel $Q_{ij}=\beta\,(y_j/y_i)^{-\gamma}P_{ij}$ into two nested `jax.lax.fori_loop`s with `q.at[j].set(...)` carries: +> ```python +> def body_fun_i(i, Q): +> def body_fun_j(j, q): +> ratio = u_prime(c[j]) / u_prime(c[i]) +> return q.at[j].set(β * ratio * P[i, j]) +> q = jax.lax.fori_loop(0, n, body_fun_j, jnp.zeros((n,))) +> return Q.at[i, :].set(q) +> Q = jax.lax.fori_loop(0, n, body_fun_i, jnp.zeros((n, n))) +> ``` +> Prerequisite concepts rise **7 → 13**, docstring coverage falls **0.90 → 0.55**, and the simple "ratio of marginal utilities" idea is buried under functional-update plumbing. +> +> **Also LOW (2), pre-JAX style:** the Bellman operator in `lectures/_static/lecture_specific/odu/odu.py:114-123` loops `for i in range(N)` over flattened grid points doing a `fixed_quad` integral per cell — the value-iteration math is hard to see through the Python scaffolding. (This is the kind of code a *good* JAX rewrite should improve; contrast with the aiyagari HIGH example above.) + +--- + +### Dimension 3 — Computational efficiency (as actually used) · weight 0.15 + +**Crucial rule:** measure efficiency **in the regime the lecture runs**, not a hypothetical large-scale one. For JAX that means *including* trace+compile time whenever the lecture hits a new shape or `static_argnames` value, because each of those triggers a recompile. + +**Metrics** (from `benchmark.py`, `cold_start.py`, `sweep_bench.py`): +- as-used latency (cold, lecture problem size); +- warm/amortized latency; +- scaling curve + the crossover `n` where JAX overtakes NumPy; +- recompile cost per distinct static-arg value. + +**The metric that decides the score** is the **as-used speedup** + +``` +as_used_speedup = (total NumPy wall time) / (total JAX wall time) +``` + +measured over the lecture's *actual* sequence of solver calls, at its *actual* problem sizes, in a fresh interpreter (so JAX's compiles count). `>1` = JAX faster, `<1` = JAX slower. + +**Anchors (numeric)** + +| Score | as-used speedup | meaning | +|:---:|:---:|---| +| 5 | **≥ 3×** | materially faster as the lecture runs it | +| 4 | **1.3× – 3×** | clearly faster | +| 3 | **0.8× – 1.3×** | wash; JAX only wins warm / at sizes the lecture never reaches | +| 2 | **< 0.8×** | measurably slower as used; stated goal not met, but correct & fixable | +| 1 | < 0.8× **and** worse (wrong/unstable, or no fix path) | slower with no redemption | + +> **HIGH (5) — MEASURED.** `aiyagari.md` is JAX on both branches (no NumPy baseline in-repo), so we benchmarked *its computational pattern* — the vectorised Bellman of `aiyagari.md:288-300` solved by value-function iteration on a `200×7` grid, then re-solved 20× as an equilibrium loop would. This is a **shared calibration of the efficiency threshold, not a per-lecture script** (`../scripts/calibration/bellman_bench.py`, results in `../scripts/calibration/bellman_bench.json`): +> +> | | NumPy | JAX | speedup | +> |---|--:|--:|--:| +> | one solve (397 VFI iters), warm | 1664 ms | 69 ms | **24×** | +> | equilibrium loop, R=20 (as-used, incl. compile) | 29.3 s | 1.16 s | **25×** | +> +> Results agree to `1.1e-14`. Large array + many fixed-shape re-solves → the one-time compile is amortised and JAX wins by ~25×. *(Representative single-CPU medians; ±~15% run-to-run — the decisive fact is the order of magnitude.)* **as-used speedup ≈25 ≥ 3 → score 5.** +> +> **LOW (2) — MEASURED.** Replaying the *entire* `ge_arrow` solver sequence (all examples + the λ-sweep + finite/`T=10000`) once in a fresh process (`scripts/as_used_total.py`): +> +> | NumPy total | JAX total | as-used speedup | +> |--:|--:|--:| +> | **0.035 s** | **1.56 s** | **0.022× (≈45× slower)** | +> +> Every economy is 2×2/3×3 and each call uses fresh static args (`s0_idx`, `T`) → a fresh compile each time (first solve 286 ms, recompile 133 ms, λ-sweep 300 ms cold). JAX *would* win warm at `n ≳ 25` (see `benchmark.py` scaling), but the lecture's economics fix the size tiny. **as-used 0.022 < 0.8 → score 2** (correct and fixable, so not a 1). + +--- + +### Dimension 4 — Logic & design · weight 0.15 + +**Metrics**: `explicit_loops`, repetition/DRY review, statefulness, latent bugs. + +**Anchors** + +| Score | Criterion | +|:---:|---| +| 5 | Pure, single-responsibility functions; no repetition; no order-dependence; no global reliance; fixes prior bugs. | +| 4 | Mostly clean; minor redundancy. | +| 3 | Works but has some duplication or awkward coupling. | +| 2 | Order-dependent mutation, duplicated computation, or reliance on globals. | +| 1 | Tangled control flow or logic that is hard to reason about / buggy. | + +> **HIGH (5):** `lectures/lake_model.md:216-275` declares parameters as a frozen `LakeModel(NamedTuple)` with defaults, then computes everything with *pure* jitted functions that take the model as an argument: +> ```python +> class LakeModel(NamedTuple): +> λ: float = 0.283; α: float = 0.013; b: float = 0.0124; d: float = 0.00822 +> +> @jax.jit +> def compute_matrices(model: LakeModel): +> λ, α, b, d = model.λ, model.α, model.b, model.d +> ... +> ``` +> No instance is mutated, no call ordering matters, no globals. (The `ge_arrow` rewrite adopts this same pattern — its strongest aspect.) +> +> **LOW (2, the `ge_arrow` *original*):** `wealth_distribution(s0)` → `continuation_wealths()` → `value_functionss()` must be called **in that order** because each mutates `self`; `risk_free_rate` recomputes `sum(Q)` instead of reusing `PRF`; `pricing_kernel` references the **module-level** `P`; and the public method is misspelled `value_functionss`. The rewrite fixing these is exactly why it scores well here. + +--- + +### Dimension 5 — Coding style & idiom · weight 0.10 + +**Metrics**: PEP 8 / project-style conformance, and — for JAX — whether the code uses *idiomatic* JAX (vectorisation, `vmap`, `where`) rather than mechanically porting Python loops. + +**Anchors** + +| Score | Criterion | +|:---:|---| +| 5 | Idiomatic in both languages; vectorised where natural; consistent naming. | +| 4 | Idiomatic with minor nits. | +| 3 | Correct but mixes idioms or ports loops literally where vectorisation fits. | +| 2 | Anti-idiomatic constructs that a JAX reviewer would flag. | +| 1 | Fights the framework throughout. | + +> **HIGH (5):** `lectures/aiyagari.md:300` uses branchless `jnp.where(c > 0, u(c) + β * EV, -jnp.inf)` to impose feasibility, and `lectures/lake_model.md: 261` iterates a time series with `jax.lax.scan` (the idiomatic carry/collect primitive) instead of hand-rolled index updates. +> +> **LOW (2, the `ge_arrow` case):** nested `fori_loop` scalar scatter for the pricing kernel (vectorisation was a one-liner), and `jax.lax.cond(T==0, …)` that **traces both branches every call** where `T` is already static and a plain Python `if` would do. Only 1 of 4 idiom criteria is met (clean call sites), so the checklist gives 1+1 = 2. + +--- + +### Dimension 6 — API ergonomics & reusability · weight 0.10 + +**Metrics**: `statements_for_one_result` (calls needed to obtain α, ψ, J); composability (jit/vmap-friendly?); immutability. + +**Anchors (numeric — keyed to `statements_for_one_result`, i.e. the calls a user must write to obtain α, ψ, J for one economy)** + +| Score | statements | & | +|:---:|:---:|---| +| 5 | 1 | immutable result, trivially `jit`/`vmap`-composable | +| 4 | ≤ 2 | one object + minor setup | +| 3 | 3 | order-independent | +| 2 | ≥ 3 | **ordered, side-effecting** calls (wrong order → silent garbage) | +| 1 | — | fragile protocol, easy to misuse silently | + +> **HIGH (5):** `lectures/lake_model.md` — `model = LakeModel()` then `compute_matrices(model)` / `simulate_path(...)`; the model is an immutable argument passed to stateless functions, trivially `vmap`-able over parameters. The `ge_arrow` rewrite matches this: `m = compute_rc_model(s, P, ys, s0_idx=1, T=10)` returns one immutable bundle (`m.Q, m.α, m.ψ, m.J, …`), `statements_for_one_result = 1`. +> +> **LOW (2):** `odu.py`'s `SearchProblem` and the `ge_arrow` *original* both require *build object → call mutating methods in the correct order*. For `ge_arrow` that is `wealth_distribution → continuation_wealths → value_functionss`; `statements_for_one_result = 4`, and calling them out of order silently gives wrong/garbage results. + +--- + +### Dimension 7 — Maintainability & robustness · weight 0.05 + +**Metrics**: testability (pure vs stateful), debuggability (can you step through it?), and "footguns" left for future editors. + +**Anchors** + +| Score | Criterion | +|:---:|---| +| 5 | Pure & easily unit-tested; no silent traps; easy to extend. | +| 4 | Testable; small caveats. | +| 3 | Testable but harder to debug, or leaves a minor trap. | +| 2 | Hard to debug or carries a silent correctness trap (e.g. dtype). | +| 1 | Brittle; changes likely to break silently. | + +> **HIGH (5):** `lectures/aiyagari.md` pairs pure jitted functions with the explicit `jax.config.update("jax_enable_x64", True)` at `:72`, so a future editor reusing the functions gets full precision by default and can unit-test each `@jax.jit` function in isolation. +> +> **LOW (3, the `ge_arrow` case):** purity *helps* testing, but `jit` + `static_argnames` + 3-deep closures make stepping hard, and the float32 default is a silent trap for the next person who reuses the function. + +--- + +## 3. How to run the system + +```bash +conda activate quantecon # jax 0.4.x, numpy 2.x, quantecon +python references/examples//scripts/run_all.py # measure → /results/*.json, + # then apply the shared rubric +python scripts/scoring/score.py references/examples/ # (re)compute the scorecard alone +``` + +`run_all.py` runs the measurement scripts (e.g. `check_equivalence.py`, `static_metrics.py`, `benchmark.py`, `as_used_total.py`, and lecture-specific ones) and finishes by invoking `scripts/scoring/score.py`, which reads `/evidence.json` and writes `/results/scorecard.json`. + +**To evaluate a *different* lecture** see the "Evaluate a new lecture" recipe in [`README.md`](../scripts/README.md): scaffold `/`, drop in `model_old.py` / `model_new.py`, adapt the measurement scripts, fill `evidence.json`, and run the two commands above. The framework, weights, thresholds, and checklists are lecture-independent; only the inputs change. + +## 4. Limitations / honesty notes + +- Benchmarks are **CPU-only** (`jax.devices() == [CpuDevice]`). On GPU/TPU the crossover `n` shifts left and JAX's warm advantage grows — but the lecture's models are still tiny, so the as-used verdict is unlikely to change. +- Dimension scores 2/5/6 are partly judgement; the rubric anchors and the cited metrics make them auditable, not arbitrary. +- `concept_token_hits` from `static_metrics.py` is a raw frequency and is *informational only*; `n_prerequisite_concepts` is the readability metric that feeds scoring. diff --git a/benchmark/references/examples/README.md b/benchmark/references/examples/README.md new file mode 100644 index 0000000..92a3e24 --- /dev/null +++ b/benchmark/references/examples/README.md @@ -0,0 +1,115 @@ +# Reference examples — logic check and provenance + +The two evaluations in this directory are the **canonical reference cases** for `/benchmark:review-acceleration`: they calibrate the rubric's thresholds, serve as the worked demonstrations of the method, and act as the **regression baseline** — any change to the scoring engine or the measurement templates must reproduce their scorecards. This document explains each example in detail and records the line-by-line logic review performed on 2026-07-21 (all scripts, all data files), so their accuracy is auditable rather than asserted. + +**Review verdict: logic sound, methodology fair, every traceable number consistent with its source.** Known caveats are listed at the end — none changes either verdict. + +--- + +## How an evaluation fits together + +Each example directory is one self-contained evaluation of a lecture rewrite (NumPy `main` → JAX PR branch): + +| Piece | Role | +|---|---| +| `scripts/model_old.py`, `model_new.py` | Faithful extractions of the two implementations. Any deviation from the lecture source is disclosed in the module docstring and is itself an evaluation finding | +| `scripts/*` (measurement) | Produce objective numbers into `results/*.json` | +| `evidence.json` | The **only place judgement is recorded**: measured numbers copied into quantitative slots (with source), structural checklist items answered true/false with citations to the diff | +| `results/scorecard.json` | Computed by `../../scripts/scoring/score.py` from the evidence — **no score is ever typed by hand** | +| `_REPORT.md` | The human-readable verdict, written from scorecard + evidence | + +The measurement standard throughout: **as-used, fresh-process** — replay the lecture's actual call sequence at its actual problem sizes in a new interpreter, so JAX trace/compile time counts; warm numbers are reported alongside, never alone. + +--- + +## Example 1 — `ge_arrow` (lecture-python.myst#717) → **2.85/5, mixed/wash** + +### The economics + +The lecture solves a recursive competitive equilibrium with one-period Arrow securities. For each example economy `(s, P, ys)` it computes: the pricing kernel `Q` (β × marginal-utility ratio × transition probability), the resolvent `V = (I−Q)⁻¹` (or partial sums for finite `T`), wealth-distribution shares `α` from the initial-state row of `V`, continuation wealths `ψ`, bond price `PRF`, risk-free rate `R`, and value functions `J`. All economies are tiny: 2×2 or 3×3. + +### The two implementations + +- **Old (NumPy):** a mutable class; the lecture calls `wealth_distribution(s0) → continuation_wealths() → value_functionss()` **in that order** (each mutates `self`; the typo'd method name is real). Disclosed extraction deviation: the lecture's `pricing_kernel`/`continuation_wealths` referenced module-level globals `P, n, K`; the extraction uses `self.*` so the module is self-contained — recorded as a logic finding against the original, not silently repaired behaviour. +- **New (JAX):** one `NamedTuple` + a single jitted factory `compute_rc_model(s, P, ys, s0_idx, T)` with `static_argnames=("T", "s0_idx")` — every distinct `(s0_idx, T, shape)` triggers a fresh compile. Kept verbatim, **including** the patterns the evaluation criticizes (nested `fori_loop` scalar-scatter pricing kernel; `lax.cond` on the static `T`). + +### The measurements and where each evidence number comes from + +| Evidence slot | Value | Source | Logic-check notes | +|---|---|---|---| +| `max_delta_shipped` | 1.7e-4 | `check_equivalence.py` → `results/equivalence.json` | 11 cases: 4 economies × initial states + finite T=10. Compares Q, R, A, V[-1], α, ψ, J. Verified worst-case in the results file: 1.679e-4 (ex2, float32-as-shipped) | +| `matches_under_x64` | true | same script re-run with `JAX_ENABLE_X64=1` | max\|Δ\| ≈ 1.4e-14 under x64 → logic identical, drift is purely float32. *Caveat m3: the x64 run overwrites the same results file; regime not stamped* | +| `delta_prereq_concepts` | +6 (7→13) | `static_metrics.py` → `results/static_metrics.json` | **Hand-curated lists** (see caveat M1), disclosed as such in the script | +| `docstring_cov_new` | 0.55 (from 0.90) | same | AST-measured (objective) | +| `as_used_speedup` | **0.022× (≈45× slower)** | `as_used_total.py` (two fresh processes) → `results/as_used.json` *(generated per-run, not committed — the committed provenance is `evidence.json`)* | The replayed sequence mirrors the lecture exactly: ex1–ex3 × 2 initial states, the λ-sweep (NumPy: 100-iteration Python loop; JAX: one jitted `fori_loop` sweep, *as each lecture version does it*), ex4 × 3 states, finite T=10 × 2, T=10000 × 1. `block_until_ready` on every JAX call | +| `statements_for_one_result` | 1 (from 4) | `static_metrics.py` | Hand-asserted constant (caveat M1); the 4 is the ordered old protocol, the 1 is the single factory call | +| structural checklists | 1–4 criteria each | `evidence.json` citations | e.g. style meets only clean-call-sites → 1+1 = 2 | + +Supporting (context, not scored directly): `benchmark.py` (cold ≈ 300 ms incl. compile at n=2; warm crossover at n≈10–25; scaling to n=400), `cold_start.py` (first call + the 133 ms recompile at a new `s0_idx`), `sweep_bench.py` (the JAX-favourable bound: the sweep is the one repeated workload). + +### Why 2.85 + +Weighted: correctness 3 (logic identical under x64 but ships float32 with unflagged 4th–5th-significant-figure drift) + readability 2 (kernel one-liner becomes nested `fori_loop` plumbing) + efficiency 2 (45× slower as-used; warm wins never materialize at n≤3) + logic 4 (fixes the ordered-mutation/global-state/typo defects, minus the unvectorised O(n²) kernel) + style 2 + ergonomics 5 (one immutable call) + maintainability 3. The verdict captures the case's essence: **a structurally better rewrite that is slower and harder to read in the regime the lecture actually runs.** + +--- + +## Example 2 — `markov_asset` (lecture-python.myst#654) → **2.25/5, net regression** + +### The economics + +Lucas-tree, consol, and call-option pricing over a Markov chain (default: 25-state Tauchen). Core operations are `eigvals` (stability check) and dense `solve` — O(n³) LAPACK work in both libraries, at n = 5 and 25. + +### The two implementations + +- **Old (NumPy):** mutable `AssetPriceModel` class + four pricing functions. Disclosed extraction deviation: the lecture's `__init__` reads a module-level `n=25`; the extraction defines it at module level too (global reliance recorded as a finding, mirroring ge_arrow). +- **New (JAX):** two `NamedTuple`s + factories; `checkify` for the stability assertion under `jit`; `lax.while_loop`/`fori_loop` solvers. Kept verbatim **including the build-breaking bug**: `call_option` contains a stray `err.throw()` referencing a name never bound in that scope (marked `# <-- VERBATIM from lecture` in `model_new.py`). Whether it runs is part of what is evaluated. + +### The bug and the near-critical precision finding + +- `smoke_test.py` demonstrates the failure in a clean namespace: `call_option_jit(...)` → `NameError: name 'err' is not defined` → correctness 1 by the does-not-build override, under the system's fresh-process measurement regime. **Corrected 2026-07-21:** the notebook itself executes in cell order — earlier cells bind a global `err` that the stray `err.throw()` silently resolves to, which means the checkify stability validation is *never actually performed* in the shipped lecture (a masked failure rather than a crash); a reader copying the function into a clean namespace hits the `NameError`. See the REPORT's erratum. +- `check_equivalence.py` additionally compares a **bug-patched copy** (verified line-identical to shipped logic minus the stray `err.throw()`) to establish that the *intended* logic is right: under x64 every asset matches NumPy to ≈1e-11. As shipped (float32) drift reaches **1.02e-2** on the exercise model — and that model's spectral radius (1.0618) sits **0.002 below** the stability bound 1/β = 1.0638, so float32 is not merely imprecise but close to flipping the stability check itself. Regimes are stored separately (`equivalence_x64_{True,False}.json`) with the x64 flag stamped in `_meta` — the pattern ge_arrow's template should adopt (caveat m3). + +### The measurements + +| Evidence slot | Value | Source | Logic-check notes | +|---|---|---|---| +| `builds` | false | `smoke_test.py`, recorded in `equivalence_x64_False.json` | Decisive: overrides the Δ bands → correctness 1 | +| `max_delta_shipped` | 1.02e-2 | `equivalence_x64_False.json` | Verified in file: 1.016e-2 | +| `matches_under_x64` | true | `equivalence_x64_True.json` | ≈1e-11 on all working assets | +| `delta_prereq_concepts` | +5 (8→13) | `static_metrics.py` | All five additions are checkify/JAX-structural (hand-curated; caveat M1) | +| `docstring_cov_new` | 0.75 (from 0.86) | same | AST-measured | +| `as_used_speedup` | 0.17× (≈6× slower) | `as_used_total.py` → `results/as_used.json` *(generated per-run, not committed)* | JAX side uses the patched `call_option` **so an end-to-end timing exists at all** — disclosed in the docstring and in the output record (`"mode": "jax_patched"`). Sequence mirrors the lecture: γ-sweep ×5, consol+call at β=0.9, exercise model (tree, consol, call, finite k=5,25) | +| `statements_for_one_result` | 3 (from 2) | `static_metrics.py` | The `(err, val)` unpack + `err.throw()` ceremony | +| logic checklist | 4/4 met, **capped at 3** | evidence citations | The cap (introduces a correctness bug) is the rubric's override working as designed | + +`benchmark.py` (scaling) provides the context for efficiency: the workload is LAPACK-bound in both libraries, so JAX is slower at n=5 and n=25 and only edges ahead (~1.2–1.4×) at n≥250 — sizes this lecture never uses. + +### Why 2.25 + +correctness 1 (does not build) + readability 2 + efficiency 2 + logic 3 (capped) + style 4 (the `lax` loops are the *correct* primitives here — a better JAX use than ge_arrow's) + ergonomics 3 + maintainability 2. The report's must-fix list is concrete: delete the stray `err.throw()`, enable x64 (genuinely required given the stability margin); those two fixes alone lift the total past the 2.5 line. + +--- + +## The shared HIGH anchor — `../../scripts/calibration/bellman_bench.py` + +`aiyagari.md` is JAX on both branches, so the HIGH end of the efficiency scale is calibrated on its *computational pattern*: the vectorised Bellman operator on a 200×7 grid, solved by VFI (~397 iterations), then re-solved 20× as an equilibrium loop would — the regime JAX is built for. Fairness properties verified: x64 enabled; the NumPy baseline uses the same broadcast/vectorised algorithm (not a strawman); implementations agree to ~1e-14; cold timing uses `_clear_cache()`; the equilibrium loop includes exactly one compile. Result: **~25× faster as-used** → pins "≥3× → score 5", with ge_arrow's 0.022× anchoring the LOW end. + +--- + +## Verification performed (2026-07-21) + +1. **Line-by-line review** of every script in the package (models, measurements, orchestration, scoring engine, calibration). +2. **Scorecard reproduction:** `score.py` regenerates both committed scorecards **byte-identically** from `evidence.json` alone. +3. **Evidence↔results cross-check (scripted):** every quantitative evidence slot matches its results-file source (Δprereq, docstring coverage, max\|Δ\| shipped, crash record, statements). +4. **Rubric edge audit:** brute force over all 5⁷ score combinations found FP band-edge misclassifications (797 cases), fixed by computing the verdict from the rounded total; neither reference case was affected. +5. **Fairness audit:** `block_until_ready` on every JAX timing; fresh processes for as-used; medians over repeats in the warm/scaling benchmarks (**correction:** the as-used totals themselves are single passes per side — a known limitation, tracked for v2); identical call sequences per side at the level the replay scripts encode (**correction:** later review found both replays deviate from the lecture in construction patterns — see the design-review documents in `reviews/`); disclosed patches only where timing is otherwise impossible. + +## Known caveats (recorded, deliberate, or pending upstream) + +| ID | Caveat | Status | +|---|---|---| +| **M1** | `n_prerequisite_concepts` (readability driver, weight 0.25) and `statements_for_one_result` (ergonomics) are **hand-curated judgements encoded in the measurement scripts**, disclosed as such — not AST measurements. When the skill adapts templates to a new lecture it authors these lists, so they need the same citation discipline as the structural checklists. Proposed: move them into `evidence.json` as cited judgement slots. | Raised with the system's author (@xuanguang-li) on skills PR #5 | +| m3 | ge_arrow's `equivalence.json` does not stamp the x64 regime and is overwritten between regimes; markov_asset's split-file pattern (`equivalence_x64_{bool}.json` + `_meta`) is the better template | Adopt on next template revision | +| n6 | `sweep_bench` asymmetry: the old sweep computes only α (constructor + `wealth_distribution`), while the new one-call API forcibly computes everything — faithful to each API as used, but part of the measured sweep disadvantage is API-induced | Documented; by design (the API's cost is real) | +| — | markov_asset's as-used timing requires the bug-patched `call_option`; the shipped code cannot complete in a clean namespace (in notebook order it runs with the validation silently masked — see the REPORT erratum) | Disclosed in script, output record, and report | +| — | Benchmarks are CPU-only; timings vary ±~15% run-to-run — the rubric keys on orders of magnitude | Framework limitation note | diff --git a/benchmark/references/examples/ge_arrow/evidence.json b/benchmark/references/examples/ge_arrow/evidence.json new file mode 100644 index 0000000..96bf7b0 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/evidence.json @@ -0,0 +1,82 @@ +{ + "lecture": "ge_arrow", + "branch": "update_ge_arrow", + "source_pr": "QuantEcon/lecture-python.myst#717", + "refs": { + "base": "8cfba4c90ebc08d3e51718ee65246ac249305ce0", + "head": "8c2d0d798d6ad7fb669351858f0e0d8e07659f6b" + }, + "_how": "Quantitative values are copied from results/*.json (source noted). Structural criteria are yes/no facts about the diff, each with a citation. Scores are computed by scoring/rubric.py — run: python scripts/scoring/score.py references/examples/ (from the plugin root).", + + "quantitative": { + "correctness": { + "builds": true, + "max_delta_shipped": 1.7e-4, + "matches_under_x64": true, + "source": "results/equivalence.json: float32 run worst=1.7e-4 (ex2); x64 run worst=1.4e-14" + }, + "readability": { + "delta_prereq_concepts": 6, + "docstring_cov_new": 0.55, + "source": "results/static_metrics.json: prereq 7->13 (+6), docstring_coverage 0.90->0.55" + }, + "efficiency": { + "as_used_speedup": 0.022, + "as_used_runs": [], + "baseline_as_used_seconds": 0.035, + "correct_or_fixable": true, + "source": "as_used_total.py: numpy 0.035s / jax 1.56s = 0.022x; fixable by not making s0_idx/T static. Single-pass v1 measurement (v2 standard: median of >=3 fresh-process runs)" + }, + "ergonomics": { + "statements_for_one_result": 1, + "fragile_protocol": false, + "source": "results/static_metrics.json: compute_rc_model(...) returns one immutable NamedTuple" + } + }, + + "structural": { + "logic_design": { + "criteria": { + "pure_no_order_dependence": true, + "no_global_state": true, + "good_algorithmic_choices": false, + "fixes_prior_bugs": true + }, + "introduces_correctness_bug": false, + "citations": { + "pure_no_order_dependence": "single pure compute_rc_model; original needed 3 ordered stateful methods", + "no_global_state": "removes original's reliance on module-level P,n,K", + "good_algorithmic_choices": "FALSE: pricing kernel is an O(n^2) nested fori_loop scatter (ge_arrow.md:938-959) instead of a vectorised outer product", + "fixes_prior_bugs": "fixes value_functionss typo and de-duplicates R=sum(Q)" + } + }, + "style_idiom": { + "criteria": { + "vectorised_where_natural": false, + "correct_control_flow_primitive": false, + "no_anti_idiomatic_constructs": false, + "clean_call_sites_and_naming": true + }, + "citations": { + "vectorised_where_natural": "FALSE: kernel written as scalar loops", + "correct_control_flow_primitive": "FALSE: fori_loop used where broadcasting fits", + "no_anti_idiomatic_constructs": "FALSE: jax.lax.cond(T==0,...) traces both branches though T is static", + "clean_call_sites_and_naming": "TRUE: NamedTuple result, m.Q/m.alpha/... read cleanly" + } + }, + "maintainability": { + "criteria": { + "pure_unit_testable": true, + "dtype_precision_safe": false, + "no_footgun_for_editors": true, + "robust_no_brittle_conditions": false + }, + "citations": { + "pure_unit_testable": "pure function, trivially testable", + "dtype_precision_safe": "FALSE: float32 default, no jax_enable_x64", + "no_footgun_for_editors": "TRUE: simple call/return, no error-protocol", + "robust_no_brittle_conditions": "FALSE: jit+static_argnames+3-deep closures hinder step-debugging" + } + } + } +} diff --git a/benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md b/benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md new file mode 100644 index 0000000..094db48 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md @@ -0,0 +1,113 @@ +# Evaluation Report — `ge_arrow.md`: NumPy (`main`) → JAX (`update_ge_arrow`) + +Applies the system in [`../../EVALUATION_FRAMEWORK.md`](../../EVALUATION_FRAMEWORK.md) to the only code change on branch `update_ge_arrow`. All numbers below are reproduced by `scripts/run_all.py` (CPU, jax 0.4.35, numpy 2.1.3) into `results/`. Every dimension score is **computed from [`evidence.json`](evidence.json) by the shared rubric** (`../../../scripts/scoring/rubric.py`) — see `results/scorecard.json` for the derivation. + +> **Rubric v2 note (2026-07-22).** Re-scored under rubric v2 (verdict gates, no-conversion, sensitivity stamp — see `reviews/`): the total is unchanged at **2.85/5**, but the headline verdict is now **no-conversion** — the baseline as-used total (0.035 s) is under the 1 s materiality floor and the candidate is slower as-used (0.022×), so this lecture should not be converted regardless of the candidate's polish (candidate band for the record: mixed/wash). Sensitivity stamp: **fragile** — flipping `good_algorithmic_choices` alone moves the candidate band to 3.00/net-positive, and flipping either correctness boolean drops it to a gated net regression. Derivation: `results/scorecard.json`. + +## TL;DR — weighted score **2.85 / 5** → *net mixed, slightly negative for this lecture* + +The rewrite is **better software** (one-call pure API, real bug fixes) but a **worse lecture** on the two axes that matter most here: it is harder to read and — contrary to the stated motivation — **slower in every regime this lecture actually runs**, while silently dropping numerical precision. + +| Dimension | Wt | Score | Weighted | how the score arises | +|---|:--:|:--:|:--:|---| +| Correctness & numerical fidelity | 0.20 | 3 | 0.60 | ships float32 → drift 1.7e-4 ∈ (1e-8,1e-3] | +| Readability & pedagogical clarity | 0.25 | 2 | 0.50 | Δprereq +6→2, docstrings 0.55→2 (worse-of-two) | +| Computational efficiency (as used) | 0.15 | 2 | 0.30 | as-used speedup 0.022× < 0.8 | +| Logic & design | 0.15 | 4 | 0.60 | 3/4 criteria met | +| Coding style & idiom | 0.10 | 2 | 0.20 | 1/4 criteria met (only clean call sites) | +| API ergonomics & reusability | 0.10 | 5 | 0.50 | 1 statement per result | +| Maintainability & robustness | 0.05 | 3 | 0.15 | 2/4 criteria met | +| **Total** | **1.00** | | **2.85** | | + +--- + +## What changed + +| | Original (`main`) | Rewrite (`update_ge_arrow`) | +|---|---|---| +| Library | NumPy | JAX (`jnp`, `lax`, `jit`) | +| Container | mutable `class` with methods | immutable `NamedTuple` of results | +| Entry point | build object + 3 ordered method calls | one `@jit` function `compute_rc_model` | +| Loops | Python `for` (×6) | `jax.lax.fori_loop` / `lax.cond` (0 Python loops) | +| Infinite-horizon flag | `T=None` | `T=0` | +| Notable | typo `value_functionss`; uses global `P,n,K` | fixes both | + +--- + +## Evidence by dimension + +### 1 · Correctness & numerical fidelity → **3/5** +`check_equivalence.py` over all 11 example/initial-state combinations: + +- **Under float64:** every object matches, `max|Δ| = 1.4e-14` → the rewrite's *logic is identical*. ✅ +- **As the lecture actually runs (float32 default, no `jax_enable_x64`):** `ex2` deviates by `1.7e-4`; several others ~`1e-4`. The published tables move in the 4th–5th significant figure. ❌ unflagged precision loss. + +→ Correct economics, silently reduced precision. Score capped at 3. + +### 2 · Readability & pedagogical clarity → **2/5** +`static_metrics.py`: + +| metric | old | new | +|---|--:|--:| +| prerequisite concepts | **7** | **13** | +| docstring coverage | **0.90** | **0.55** | +| code lines (model def) | 119 | 161 | +| sub-definitions | 10 | 22 | +| Python loops a reader parses | 6 | 0 (replaced by `fori_loop` closures) | + +The pricing kernel — mathematically just $Q_{ij}=\beta(y_j/y_i)^{-\gamma}P_{ij}$ — becomes two nested `fori_loop`s with `.at[j].set(...)` carries. For a lecture whose economies are 2×2, this is pure cognitive overhead. **Biggest single driver of the negative verdict** (and the heaviest-weighted dimension). + +### 3 · Computational efficiency (as used) → **2/5** +This was the stated motivation, so it matters that it is **not achieved here.** + +**Headline metric — replaying the *entire* lecture solver sequence once in a fresh process (`as_used_total.py`):** + +| NumPy total | JAX total | **as-used speedup** | +|--:|--:|--:| +| **0.035 s** | **1.56 s** | **0.022× — i.e. ~45× slower** | + +Per-regime detail explaining why: + +| Regime (n=2 unless noted) | NumPy | JAX | Result | +|---|--:|--:|---| +| First solve (cold, incl. compile) | 6.2 ms | 286 ms | **46× slower** | +| Recompile per new `s0_idx`/`T` | — | 133 ms | each distinct call recompiles | +| Warm repeat | 0.032 ms | 0.022 ms | 1.4× faster *(never used)* | +| λ-sweep (100 pts), as run once | 1.8 ms | 300 ms cold | **170× slower** | +| λ-sweep warm | — | 0.37 ms | 4.8× faster *(never realized)* | + +Scaling crossover (`benchmark.py`): NumPy and JAX-warm are even near **n≈10**; JAX wins **2–6×** for `n = 25…200`. **The lecture never exceeds n=3.** For calibration, the same machinery on the large, repeatedly-solved aiyagari pattern (shared `../../../scripts/calibration/bellman_bench.py`) is **25× faster** — a score-5 case. `ge_arrow`'s `0.022×` maps to **score 2** (< 0.8×, but correct and fixable). + +### 4 · Logic & design → **4/5** +Genuine improvements, all verified in the diff: +- removes order-dependent stateful methods (old required `wealth_distribution → continuation_wealths → value_functionss`); +- removes reliance on module-level `P, n, K` (a latent bug in the original); +- fixes the `value_functionss` typo; +- de-duplicates (`R` no longer recomputes `sum(Q)`); returns one result object. + +Minus one point: the pricing kernel is ported as an `O(n²)` scalar loop instead of a vectorised outer product. + +### 5 · Coding style & idiom → **2/5** +Only 1 of the 4 idiom criteria is met (clean call sites / `NamedTuple` naming). The three computational-idiom criteria fail: not vectorised where natural (the nested-`fori_loop` pricing kernel), wrong control-flow primitive (`fori_loop` where broadcasting fits), and an anti-idiomatic `jax.lax.cond(T==0, …)` that **traces both branches** although `T` is already a static argument (a plain `if` would compile only the needed branch). + +### 6 · API ergonomics & reusability → **5/5** +`statements_for_one_result`: **4 → 1**. `compute_rc_model(s, P, ys, s0_idx=1, T=10)` returns an immutable bundle; fully `jit`/`vmap`-composable. Clear win. + +### 7 · Maintainability & robustness → **3/5** +Purity aids unit testing, but `jit` + `static_argnames` + 3-deep closures hinder step-debugging, and the float32 default is a silent trap for future reuse. + +--- + +## Recommendation + +The conversion is **not yet a net improvement for this particular lecture.** Two paths: + +**A. Keep NumPy for `ge_arrow`.** The models are 2×2/3×3; NumPy is faster as-used, more readable, and matches the published numbers. Reserve JAX for lectures with large, repeated, fixed-shape computation. + +**B. If JAX is kept, fix these before re-scoring** (each maps to a dimension): +1. **Vectorise the pricing kernel** → `Q = β*(y[None,:]/y[:,None])**(-γ)*P` *(D2 readability, D3 efficiency, D5 idiom).* +2. **Enable float64**: `jax.config.update("jax_enable_x64", True)` so published numbers are preserved *(D1, D7).* +3. **Reduce recompiles**: avoid making `s0_idx`/`T` static, or vectorise over `s0_idx`, so the lecture doesn't pay a fresh compile per call *(D3).* +4. **Restore docstrings** on the nested helpers; replace `lax.cond` on a static `T` with a Python `if` *(D2, D5).* + +Re-running `run_all.py` after these fixes would likely lift readability to ~3, efficiency to ~3, and the total above the 3.0 "merge after fixes" line. diff --git a/benchmark/references/examples/ge_arrow/results/benchmark.json b/benchmark/references/examples/ge_arrow/results/benchmark.json new file mode 100644 index 0000000..7a0c892 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/results/benchmark.json @@ -0,0 +1,80 @@ +{ + "as_used_latency": { + "n": 2, + "numpy_s": 3.5200000638724305e-05, + "jax_cold_s": 0.00013939999917056412, + "slowdown_cold": 3.9602271773031466 + }, + "warm": [ + { + "n": 2, + "numpy_s": 2.749999985098839e-05, + "jax_warm_s": 2.263999995193444e-05, + "speedup_warm": 1.214664306951056 + }, + { + "n": 3, + "numpy_s": 3.173999975842889e-05, + "jax_warm_s": 2.3220000002766028e-05, + "speedup_warm": 1.3669250540330722 + } + ], + "scaling": [ + { + "n": 2, + "numpy_s": 2.9899998480686918e-05, + "jax_warm_s": 2.2899999748915434e-05, + "speedup_warm": 1.3056768038655988 + }, + { + "n": 3, + "numpy_s": 3.06999991153134e-05, + "jax_warm_s": 2.4400000256719068e-05, + "speedup_warm": 1.258196671815997 + }, + { + "n": 5, + "numpy_s": 4.040000021632295e-05, + "jax_warm_s": 3.079999987676274e-05, + "speedup_warm": 1.3116883239601242 + }, + { + "n": 10, + "numpy_s": 8.999999954539817e-05, + "jax_warm_s": 0.00013129999933880754, + "speedup_warm": 0.6854531606901343 + }, + { + "n": 25, + "numpy_s": 0.00030280000100901816, + "jax_warm_s": 9.940000018104911e-05, + "speedup_warm": 3.046277670598514 + }, + { + "n": 50, + "numpy_s": 0.0015798999993421603, + "jax_warm_s": 0.0004999000011594035, + "speedup_warm": 3.160432077771443 + }, + { + "n": 100, + "numpy_s": 0.004930600000079721, + "jax_warm_s": 0.0008147999997163424, + "speedup_warm": 6.0513009349487215 + }, + { + "n": 200, + "numpy_s": 0.019143900000926806, + "jax_warm_s": 0.003638100000898703, + "speedup_warm": 5.262059865368675 + }, + { + "n": 400, + "numpy_s": 0.07548590000078548, + "jax_warm_s": 0.013205000001107692, + "speedup_warm": 5.716463460390261 + } + ], + "jax_x64": false, + "device": "TFRT_CPU_0" +} \ No newline at end of file diff --git a/benchmark/references/examples/ge_arrow/results/equivalence.json b/benchmark/references/examples/ge_arrow/results/equivalence.json new file mode 100644 index 0000000..2516eb5 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/results/equivalence.json @@ -0,0 +1,370 @@ +{ + "ex1_s0": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "R": { + "match": true, + "max_abs_err": 9.731370598231592e-09 + }, + "A": { + "match": true, + "max_abs_err": 3.242492679333964e-05 + }, + "V": { + "match": true, + "max_abs_err": 3.242492679333964e-05 + }, + "\u03b1": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "\u03c8": { + "match": true, + "max_abs_err": 2.0503966879914515e-07 + }, + "J": { + "match": true, + "max_abs_err": 9.86491419041613e-05 + } + } + }, + "ex1_s1": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "R": { + "match": true, + "max_abs_err": 9.731370598231592e-09 + }, + "A": { + "match": true, + "max_abs_err": 3.242492679333964e-05 + }, + "V": { + "match": true, + "max_abs_err": 3.242492679333964e-05 + }, + "\u03b1": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "\u03c8": { + "match": true, + "max_abs_err": 2.384185791015625e-07 + }, + "J": { + "match": true, + "max_abs_err": 9.86491419041613e-05 + } + } + }, + "ex2_s0": { + "ok": false, + "objects": { + "Q": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "R": { + "match": true, + "max_abs_err": 3.614568022669573e-08 + }, + "A": { + "match": true, + "max_abs_err": 2.892624914352382e-05 + }, + "V": { + "match": true, + "max_abs_err": 9.536743185378782e-06 + }, + "\u03b1": { + "match": true, + "max_abs_err": 4.1358226177123925e-08 + }, + "\u03c8": { + "match": false, + "max_abs_err": 6.232608246392601e-06 + }, + "J": { + "match": true, + "max_abs_err": 0.00016027740922197609 + } + } + }, + "ex2_s1": { + "ok": false, + "objects": { + "Q": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "R": { + "match": true, + "max_abs_err": 3.614568022669573e-08 + }, + "A": { + "match": true, + "max_abs_err": 2.892624914352382e-05 + }, + "V": { + "match": true, + "max_abs_err": 9.536743185378782e-06 + }, + "\u03b1": { + "match": true, + "max_abs_err": 1.4703028572427002e-08 + }, + "\u03c8": { + "match": false, + "max_abs_err": 2.567872077641198e-06 + }, + "J": { + "match": true, + "max_abs_err": 0.00016790680376743694 + } + } + }, + "ex3_s0": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 3.051757813121725e-08 + }, + "R": { + "match": true, + "max_abs_err": 9.731370598231592e-09 + }, + "A": { + "match": true, + "max_abs_err": 4.9591064495757564e-05 + }, + "V": { + "match": true, + "max_abs_err": 4.9591064495757564e-05 + }, + "\u03b1": { + "match": true, + "max_abs_err": 2.114577202227963e-08 + }, + "\u03c8": { + "match": true, + "max_abs_err": 2.540135861650583e-07 + }, + "J": { + "match": true, + "max_abs_err": 0.00010068437867971625 + } + } + }, + "ex3_s1": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 3.051757813121725e-08 + }, + "R": { + "match": true, + "max_abs_err": 9.731370598231592e-09 + }, + "A": { + "match": true, + "max_abs_err": 4.9591064495757564e-05 + }, + "V": { + "match": true, + "max_abs_err": 4.9591064495757564e-05 + }, + "\u03b1": { + "match": true, + "max_abs_err": 0.0 + }, + "\u03c8": { + "match": true, + "max_abs_err": 1.559500684145121e-08 + }, + "J": { + "match": true, + "max_abs_err": 9.918212899151513e-05 + } + } + }, + "ex4_s0": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 3.051757813121725e-08 + }, + "R": { + "match": true, + "max_abs_err": 1.0327444655011675e-07 + }, + "A": { + "match": true, + "max_abs_err": 1.0270032113623984e-07 + }, + "V": { + "match": true, + "max_abs_err": 1.6882475395441077e-07 + }, + "\u03b1": { + "match": true, + "max_abs_err": 2.268358323398445e-08 + }, + "\u03c8": { + "match": true, + "max_abs_err": 6.386865369911732e-08 + }, + "J": { + "match": true, + "max_abs_err": 6.586516674289555e-07 + } + } + }, + "ex4_s1": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 3.051757813121725e-08 + }, + "R": { + "match": true, + "max_abs_err": 1.0327444655011675e-07 + }, + "A": { + "match": true, + "max_abs_err": 1.0270032113623984e-07 + }, + "V": { + "match": true, + "max_abs_err": 1.6882475395441077e-07 + }, + "\u03b1": { + "match": true, + "max_abs_err": 2.810550470133677e-08 + }, + "\u03c8": { + "match": true, + "max_abs_err": 9.368192638711417e-08 + }, + "J": { + "match": true, + "max_abs_err": 3.7879615977232106e-07 + } + } + }, + "ex4_s2": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 3.051757813121725e-08 + }, + "R": { + "match": true, + "max_abs_err": 1.0327444655011675e-07 + }, + "A": { + "match": true, + "max_abs_err": 1.0270032113623984e-07 + }, + "V": { + "match": true, + "max_abs_err": 1.6882475395441077e-07 + }, + "\u03b1": { + "match": true, + "max_abs_err": 8.132821038842053e-08 + }, + "\u03c8": { + "match": true, + "max_abs_err": 2.5575955336920586e-07 + }, + "J": { + "match": true, + "max_abs_err": 8.124164505574072e-07 + } + } + }, + "ex1_finite_T10_s0": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "R": { + "match": true, + "max_abs_err": 9.731370598231592e-09 + }, + "A": { + "match": true, + "max_abs_err": 8.784250757898349e-07 + }, + "V": { + "match": true, + "max_abs_err": 8.784250757898349e-07 + }, + "\u03b1": { + "match": true, + "max_abs_err": 3.371615131531058e-08 + }, + "\u03c8": { + "match": true, + "max_abs_err": 2.9802322432104233e-07 + }, + "J": { + "match": true, + "max_abs_err": 1.6151093653604676e-06 + } + } + }, + "ex1_finite_T10_s1": { + "ok": true, + "objects": { + "Q": { + "match": true, + "max_abs_err": 9.536743172944284e-09 + }, + "R": { + "match": true, + "max_abs_err": 9.731370598231592e-09 + }, + "A": { + "match": true, + "max_abs_err": 8.784250757898349e-07 + }, + "V": { + "match": true, + "max_abs_err": 8.784250757898349e-07 + }, + "\u03b1": { + "match": true, + "max_abs_err": 3.371615131531058e-08 + }, + "\u03c8": { + "match": true, + "max_abs_err": 2.728934869189459e-07 + }, + "J": { + "match": true, + "max_abs_err": 1.6151093653604676e-06 + } + } + }, + "_summary": { + "all_equivalent": false, + "atol": 1e-06, + "rtol": 1e-05 + } +} \ No newline at end of file diff --git a/benchmark/references/examples/ge_arrow/results/scorecard.json b/benchmark/references/examples/ge_arrow/results/scorecard.json new file mode 100644 index 0000000..b167695 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/results/scorecard.json @@ -0,0 +1,126 @@ +{ + "lecture": "ge_arrow", + "branch": "update_ge_arrow", + "weighted_total_out_of_5": 2.85, + "verdict": "no-conversion — the baseline as-used total 0.035 s is under the 1 s materiality floor and the candidate is slower as-used (0.022×): this lecture should not be converted, whatever the candidate's polish. Candidate quality for the record: 2.85/5, mixed/wash", + "band_verdict": "mixed / wash — improvements offset by real regressions; revisit before merging", + "verdict_gate": null, + "no_conversion": true, + "sensitivity": { + "stamp": "fragile", + "stamp_note": "", + "perturbations_tested": 29, + "deciding_flips": [ + { + "input": "quantitative.correctness.builds", + "from": true, + "to": false, + "total": 2.3, + "outcome": "no-conversion; net regression — do not merge as-is" + }, + { + "input": "quantitative.correctness.matches_under_x64", + "from": true, + "to": false, + "total": 2.3, + "outcome": "no-conversion; net regression — do not merge as-is" + }, + { + "input": "structural.logic_design.criteria.good_algorithmic_choices", + "from": false, + "to": true, + "total": 3.0, + "outcome": "no-conversion; net positive with fixable regressions — merge after addressing them" + } + ], + "perturbations_skipped": [] + }, + "dimensions": [ + { + "dim": "correctness", + "title": "Correctness & numerical fidelity", + "kind": "quantitative", + "weight": 0.2, + "score": 3, + "weighted": 0.6, + "reason": "logic matches under x64 but ships float32 → drift 1.7e-04 (1e-8,1e-3] → 3", + "citations": "results/equivalence.json: float32 run worst=1.7e-4 (ex2); x64 run worst=1.4e-14" + }, + { + "dim": "readability", + "title": "Readability & pedagogical clarity", + "kind": "quantitative", + "weight": 0.25, + "score": 2, + "weighted": 0.5, + "reason": "Δprereq=+6→2, docstrings=0.55→2; worse-of-two → 2", + "citations": "results/static_metrics.json: prereq 7->13 (+6), docstring_coverage 0.90->0.55" + }, + { + "dim": "efficiency", + "title": "Computational efficiency (as used)", + "kind": "quantitative", + "weight": 0.15, + "score": 2, + "weighted": 0.3, + "reason": "as-used speedup 0.022× < 0.8 (slower) but correct/fixable → 2 [single-run measurement; the v2 standard is a median of ≥3 fresh-process runs — see as_used_runs in the evidence template]", + "citations": "as_used_total.py: numpy 0.035s / jax 1.56s = 0.022x; fixable by not making s0_idx/T static. Single-pass v1 measurement (v2 standard: median of >=3 fresh-process runs)" + }, + { + "dim": "logic_design", + "title": "Logic & design", + "kind": "structural", + "weight": 0.15, + "score": 4, + "weighted": 0.6, + "reason": "3/4 criteria met [pure_no_order_dependence, no_global_state, fixes_prior_bugs] → 1+3=4", + "citations": { + "pure_no_order_dependence": "single pure compute_rc_model; original needed 3 ordered stateful methods", + "no_global_state": "removes original's reliance on module-level P,n,K", + "good_algorithmic_choices": "FALSE: pricing kernel is an O(n^2) nested fori_loop scatter (ge_arrow.md:938-959) instead of a vectorised outer product", + "fixes_prior_bugs": "fixes value_functionss typo and de-duplicates R=sum(Q)" + } + }, + { + "dim": "style_idiom", + "title": "Coding style & idiom", + "kind": "structural", + "weight": 0.1, + "score": 2, + "weighted": 0.2, + "reason": "1/4 criteria met [clean_call_sites_and_naming] → 1+1=2", + "citations": { + "vectorised_where_natural": "FALSE: kernel written as scalar loops", + "correct_control_flow_primitive": "FALSE: fori_loop used where broadcasting fits", + "no_anti_idiomatic_constructs": "FALSE: jax.lax.cond(T==0,...) traces both branches though T is static", + "clean_call_sites_and_naming": "TRUE: NamedTuple result, m.Q/m.alpha/... read cleanly" + } + }, + { + "dim": "ergonomics", + "title": "API ergonomics & reusability", + "kind": "quantitative", + "weight": 0.1, + "score": 5, + "weighted": 0.5, + "reason": "1 statement(s) to obtain one result → 5", + "citations": "results/static_metrics.json: compute_rc_model(...) returns one immutable NamedTuple" + }, + { + "dim": "maintainability", + "title": "Maintainability & robustness", + "kind": "structural", + "weight": 0.05, + "score": 3, + "weighted": 0.15, + "reason": "2/4 criteria met [pure_unit_testable, no_footgun_for_editors] → 1+2=3", + "citations": { + "pure_unit_testable": "pure function, trivially testable", + "dtype_precision_safe": "FALSE: float32 default, no jax_enable_x64", + "no_footgun_for_editors": "TRUE: simple call/return, no error-protocol", + "robust_no_brittle_conditions": "FALSE: jit+static_argnames+3-deep closures hinder step-debugging" + } + } + ], + "_note": "Scores are computed by scripts/scoring/rubric.py from ge_arrow/evidence.json; do not edit by hand." +} \ No newline at end of file diff --git a/benchmark/references/examples/ge_arrow/results/static_metrics.json b/benchmark/references/examples/ge_arrow/results/static_metrics.json new file mode 100644 index 0000000..b511e44 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/results/static_metrics.json @@ -0,0 +1,46 @@ +{ + "old": { + "code_lines": 119, + "n_defs": 10, + "docstring_coverage": 0.9, + "max_nesting_depth": 3, + "explicit_loops": 6, + "concept_token_hits": 105, + "n_prerequisite_concepts": 7, + "prerequisite_concepts": [ + "Python class / OOP", + "__init__ constructor", + "instance state (self.)", + "NumPy arrays & slicing", + "matrix @ / .dot", + "np.linalg.inv", + "Python for-loops" + ], + "statements_for_one_result": 4 + }, + "new": { + "code_lines": 161, + "n_defs": 22, + "docstring_coverage": 0.55, + "max_nesting_depth": 3, + "explicit_loops": 0, + "concept_token_hits": 65, + "n_prerequisite_concepts": 13, + "prerequisite_concepts": [ + "Python class / OOP (NamedTuple)", + "immutable NamedTuple", + "typing annotations", + "functools.partial", + "jax.jit & tracing", + "static_argnames & recompilation", + "functional purity (no in-place)", + "jnp vs np", + "jax.lax.fori_loop (carry)", + "jax.lax.cond", + "functional array update .at[].set()", + "nested closures as sub-fns", + "float32 default / x64 flag" + ], + "statements_for_one_result": 1 + } +} \ No newline at end of file diff --git a/benchmark/references/examples/ge_arrow/results/sweep.json b/benchmark/references/examples/ge_arrow/results/sweep.json new file mode 100644 index 0000000..fd7a846 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/results/sweep.json @@ -0,0 +1,7 @@ +{ + "old_python_loop_s": 0.0017661000001680804, + "jax_sweep_cold_s": 0.31091550000019197, + "jax_sweep_warm_s": 0.00041553333297391265, + "speedup_warm": 4.250200549564473, + "speedup_cold": 0.005680321502681564 +} \ No newline at end of file diff --git a/benchmark/references/examples/ge_arrow/scripts/as_used_total.py b/benchmark/references/examples/ge_arrow/scripts/as_used_total.py new file mode 100644 index 0000000..de56390 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/as_used_total.py @@ -0,0 +1,108 @@ +""" +"As-used" total latency for the WHOLE ge_arrow lecture. + +We replay the exact sequence of solver calls the lecture's code cells make and +time it end-to-end, once, in a fresh interpreter -- i.e. what a reader actually +waits for. For JAX every distinct (shape, s0_idx, T) triggers its own +trace+compile, so this captures the real compile burden; NumPy just runs. + +Run as two separate processes: + python as_used_total.py numpy + python as_used_total.py jax +Each prints a JSON line; run_all.py aggregates them into results/as_used.json. +""" +import json +import sys +import time +import numpy as np + +mode = sys.argv[1] if len(sys.argv) > 1 else "numpy" + + +def economies(): + out = {} + s = np.array([0., 1.]); P = np.array([[.5, .5], [.5, .5]]) + ys = np.empty((2, 2)); ys[:, 0] = 1 - s; ys[:, 1] = s + out["ex1"] = (s, P, ys.copy()) + s = np.array([1., 2.]); ys = np.empty((2, 2)); ys[:, 0] = 1.5; ys[:, 1] = s + out["ex2"] = (s, P, ys.copy()) + λ = 0.9; P3 = np.array([[1 - λ, λ], [0., 1.]]) + ys = np.empty((2, 2)); ys[:, 0] = [1, 0]; ys[:, 1] = [0, 1] + out["ex3"] = (s, P3, ys.copy()) + s4 = np.array([1., 2., 3.]); λ = μ = .9; δ = .05 + P4 = np.array([[1 - λ, λ, 0], [μ/2, μ, μ/2], [(1-δ)/2, (1-δ)/2, δ]]) + ys4 = np.empty((3, 2)); ys4[:, 0] = [.25, .75, .2]; ys4[:, 1] = [1.25, .25, .2] + out["ex4"] = (s4, P4, ys4.copy()) + return out + + +if mode == "numpy": + import model_old as old + E = economies() + t0 = time.perf_counter() + # ex1, ex2, ex3 : two initial states each + for key in ["ex1", "ex2", "ex3"]: + s, P, ys = E[key] + for s0 in (0, 1): + m = old.RecurCompetitive(s, P, ys) + m.wealth_distribution(s0); m.continuation_wealths(); m.value_functionss() + # ex3 lambda sweep (100 points, python loop) + s, _, ys = E["ex3"] + for λ in np.linspace(0, 0.99, 100): + P = np.array([[1 - λ, λ], [0., 1.]]) + m = old.RecurCompetitive(s, P, ys) + m.wealth_distribution(0); m.wealth_distribution(1) + # ex4 : three initial states + s, P, ys = E["ex4"] + for s0 in (0, 1, 2): + m = old.RecurCompetitive(s, P, ys) + m.wealth_distribution(s0); m.continuation_wealths(); m.value_functionss() + # finite T=10 (ex1) two states, and T=10000 convergence check + s, P, ys = E["ex1"] + for s0 in (0, 1): + m = old.RecurCompetitive(s, P, ys, T=10) + m.wealth_distribution(s0); m.continuation_wealths(); m.value_functionss() + m = old.RecurCompetitive(s, P, ys, T=10000) + m.wealth_distribution(1); m.continuation_wealths(); m.value_functionss() + print(json.dumps({"mode": "numpy", "total_s": time.perf_counter() - t0})) + +else: + import jax + import jax.numpy as jnp + import model_new as new + E = economies() + + def J(s, P, ys, s0, T=0): + m = new.compute_rc_model(jnp.asarray(s), jnp.asarray(P), + jnp.asarray(ys), s0_idx=s0, T=T) + jax.block_until_ready(m.J) + + t0 = time.perf_counter() + for key in ["ex1", "ex2", "ex3"]: + s, P, ys = E[key] + for s0 in (0, 1): + J(s, P, ys, s0) + # lambda sweep as the lecture does it: one jitted fori_loop + s, _, ys = E["ex3"]; sj = jnp.asarray(s); ysj = jnp.asarray(ys) + λj = jnp.linspace(0, 0.99, 100) + + @jax.jit + def sweep(): + def body(i, carry): + a0, a1 = carry + λ = λj[i]; P = jnp.array([[1 - λ, λ], [0., 1.]]) + m0 = new.compute_rc_model(sj, P, ysj, s0_idx=0) + m1 = new.compute_rc_model(sj, P, ysj, s0_idx=1) + return a0.at[i].set(m0.α), a1.at[i].set(m1.α) + return jax.lax.fori_loop(0, 100, body, + (jnp.empty((100, 2)), jnp.empty((100, 2)))) + jax.block_until_ready(sweep()) + + s, P, ys = E["ex4"] + for s0 in (0, 1, 2): + J(s, P, ys, s0) + s, P, ys = E["ex1"] + for s0 in (0, 1): + J(s, P, ys, s0, T=10) + J(s, P, ys, 1, T=10000) + print(json.dumps({"mode": "jax", "total_s": time.perf_counter() - t0})) diff --git a/benchmark/references/examples/ge_arrow/scripts/benchmark.py b/benchmark/references/examples/ge_arrow/scripts/benchmark.py new file mode 100644 index 0000000..8f1070a --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/benchmark.py @@ -0,0 +1,144 @@ +""" +Performance benchmark: OLD (NumPy) vs NEW (JAX) Arrow-securities model. + +We measure three regimes that matter for a *lecture*: + +1. "As-used latency" -- the time a learner actually waits for one result the + first time a given (shape, static-arg) combination is requested. For JAX this + INCLUDES tracing + XLA compilation, because every new `s0_idx`/`T`/shape + triggers a fresh compile (they are static_argnames / shape-dependent). + +2. "Warm / amortized" -- repeated calls once compilation is cached. This is the + regime JAX is designed to win, relevant only if the function is called many + times at a fixed shape. + +3. "Scaling" -- how warm runtime grows with the number of Markov states + n, to show the problem size at which JAX's vectorised/compiled execution + overtakes NumPy. + +Output: results/benchmark.json + stdout table. +""" + +import json +import os +import time +import statistics +import numpy as np +import jax +import jax.numpy as jnp + +import model_old as old +import model_new as new + +RESULTS = os.path.join(os.path.dirname(__file__), "..", "results") +os.makedirs(RESULTS, exist_ok=True) + + +def make_economy(n, K=2, seed=0): + rng = np.random.default_rng(seed) + P = rng.random((n, n)); P /= P.sum(axis=1, keepdims=True) + ys = rng.random((n, K)) + 0.5 + s = np.arange(n) + return s, P, ys + + +def time_it(fn, repeat=7, number=1): + """Return median seconds per call over `repeat` trials of `number` calls.""" + samples = [] + for _ in range(repeat): + t0 = time.perf_counter() + for _ in range(number): + fn() + t1 = time.perf_counter() + samples.append((t1 - t0) / number) + return statistics.median(samples) + + +def old_call(s, P, ys, s0_idx=0, T=None): + m = old.RecurCompetitive(s, P, ys, T=T) + m.wealth_distribution(s0_idx) + m.continuation_wealths() + m.value_functionss() + return m + + +def new_call(s, P, ys, s0_idx=0, T=0): + m = new.compute_rc_model(s, P, ys, s0_idx=s0_idx, T=T) + # force completion of async dispatch + jax.block_until_ready(m.J) + return m + + +def bench_as_used(): + """Latency of a single fresh result at lecture size n=2 (cold for JAX).""" + s, P, ys = make_economy(2) + sj, Pj, ysj = jnp.asarray(s, float), jnp.asarray(P), jnp.asarray(ys) + + # NumPy: just one call + t_old = time_it(lambda: old_call(s, P, ys), repeat=11) + + # JAX cold: clear cache so compilation is included, fresh each trial + def jax_cold(): + new.compute_rc_model._clear_cache() + m = new.compute_rc_model(sj, Pj, ysj, s0_idx=0, T=0) + jax.block_until_ready(m.J) + t_new_cold = time_it(jax_cold, repeat=7) + + return {"n": 2, "numpy_s": t_old, "jax_cold_s": t_new_cold, + "slowdown_cold": t_new_cold / t_old} + + +def bench_warm(n=2): + s, P, ys = make_economy(n) + sj, Pj, ysj = jnp.asarray(s, float), jnp.asarray(P), jnp.asarray(ys) + # warm up compilation + new_call(sj, Pj, ysj) + t_old = time_it(lambda: old_call(s, P, ys), repeat=11, number=5) + t_new = time_it(lambda: new_call(sj, Pj, ysj), repeat=11, number=5) + return {"n": n, "numpy_s": t_old, "jax_warm_s": t_new, + "speedup_warm": t_old / t_new} + + +def bench_scaling(): + rows = [] + for n in [2, 3, 5, 10, 25, 50, 100, 200, 400]: + s, P, ys = make_economy(n) + sj, Pj, ysj = jnp.asarray(s, float), jnp.asarray(P), jnp.asarray(ys) + new_call(sj, Pj, ysj) # warm + rep = 7 if n <= 100 else 5 + t_old = time_it(lambda: old_call(s, P, ys), repeat=rep) + t_new = time_it(lambda: new_call(sj, Pj, ysj), repeat=rep) + rows.append({"n": n, "numpy_s": t_old, "jax_warm_s": t_new, + "speedup_warm": t_old / t_new}) + print(f" n={n:4d} numpy={t_old*1e3:9.3f} ms " + f"jax_warm={t_new*1e3:9.3f} ms speedup={t_old/t_new:6.2f}x") + return rows + + +def main(): + print("== As-used latency (n=2, JAX cold incl. compile) ==") + as_used = bench_as_used() + print(f" numpy = {as_used['numpy_s']*1e3:9.3f} ms") + print(f" jax cold = {as_used['jax_cold_s']*1e3:9.3f} ms" + f" ({as_used['slowdown_cold']:.0f}x slower than numpy)") + + print("\n== Warm / amortized (compile cached) ==") + warm = [bench_warm(2), bench_warm(3)] + for w in warm: + print(f" n={w['n']}: numpy={w['numpy_s']*1e3:.4f} ms " + f"jax_warm={w['jax_warm_s']*1e3:.4f} ms " + f"speedup={w['speedup_warm']:.2f}x") + + print("\n== Scaling (warm) ==") + scaling = bench_scaling() + + out = {"as_used_latency": as_used, "warm": warm, "scaling": scaling, + "jax_x64": jax.config.read("jax_enable_x64"), + "device": str(jax.devices()[0])} + with open(os.path.join(RESULTS, "benchmark.json"), "w") as f: + json.dump(out, f, indent=2) + print("\nwrote results/benchmark.json") + + +if __name__ == "__main__": + main() diff --git a/benchmark/references/examples/ge_arrow/scripts/check_equivalence.py b/benchmark/references/examples/ge_arrow/scripts/check_equivalence.py new file mode 100644 index 0000000..250b9c7 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/check_equivalence.py @@ -0,0 +1,126 @@ +""" +Numerical-equivalence check between the OLD (NumPy) and NEW (JAX) versions of +the Arrow-securities model. + +For every example economy that appears in the lecture we build both models and +compare the equilibrium objects (Q, R, A, V, α, ψ, J). This answers the most +basic evaluation question: *does the rewrite still compute the same economics?* + +Output: results/equivalence.json (default dtype) or results/equivalence_x64.json +(when run with JAX_ENABLE_X64=1), plus a human-readable summary on stdout — one +file per precision regime, so the x64 run never clobbers the as-shipped run. +""" + +import json +import os +import numpy as np +import jax +import jax.numpy as jnp + +import model_old as old +import model_new as new + +RESULTS = os.path.join(os.path.dirname(__file__), "..", "results") +os.makedirs(RESULTS, exist_ok=True) + +# Objects compared. The OLD API requires calling stateful methods in order, so +# we wrap a helper that reproduces what the lecture does. +ATOL = 1e-6 +RTOL = 1e-5 + + +def run_old(s, P, ys, s0_idx=0, T=None): + m = old.RecurCompetitive(np.asarray(s), np.asarray(P), np.asarray(ys), T=T) + α = m.wealth_distribution(s0_idx) + ψ = m.continuation_wealths() + J = m.value_functionss() + return { + "Q": np.asarray(m.Q), "R": np.asarray(m.R), "A": np.asarray(m.A), + "V": np.asarray(m.V[-1]), "α": np.asarray(α), + "ψ": np.asarray(ψ if T is None else ψ[-1]), + "J": np.asarray(J if T is None else J[-1]), + } + + +def run_new(s, P, ys, s0_idx=0, T=None): + Tn = 0 if T is None else T + m = new.compute_rc_model(jnp.asarray(s, dtype=float), + jnp.asarray(P, dtype=float), + jnp.asarray(ys, dtype=float), + s0_idx=s0_idx, T=Tn) + return { + "Q": np.asarray(m.Q), "R": np.asarray(m.R), "A": np.asarray(m.A), + "V": np.asarray(m.V[-1]), "α": np.asarray(m.α), + "ψ": np.asarray(m.ψ if T is None else m.ψ[-1]), + "J": np.asarray(m.J if T is None else m.J[-1]), + } + + +def examples(): + # Example 1 + s = [0, 1]; P = [[.5, .5], [.5, .5]] + ys = np.empty((2, 2)); ys[:, 0] = 1 - np.array(s); ys[:, 1] = s + yield "ex1_s0", dict(s=s, P=P, ys=ys.copy(), s0_idx=0) + yield "ex1_s1", dict(s=s, P=P, ys=ys.copy(), s0_idx=1) + + # Example 2 + s = [1, 2]; P = [[.5, .5], [.5, .5]] + ys = np.empty((2, 2)); ys[:, 0] = 1.5; ys[:, 1] = s + yield "ex2_s0", dict(s=s, P=P, ys=ys.copy(), s0_idx=0) + yield "ex2_s1", dict(s=s, P=P, ys=ys.copy(), s0_idx=1) + + # Example 3 + s = [1, 2]; λ = 0.9; P = [[1 - λ, λ], [0, 1]] + ys = np.empty((2, 2)); ys[:, 0] = [1, 0]; ys[:, 1] = [0, 1] + yield "ex3_s0", dict(s=s, P=P, ys=ys.copy(), s0_idx=0) + yield "ex3_s1", dict(s=s, P=P, ys=ys.copy(), s0_idx=1) + + # Example 4 + s = [1, 2, 3]; λ = μ = .9; δ = .05 + P = [[1 - λ, λ, 0], [μ / 2, μ, μ / 2], [(1 - δ) / 2, (1 - δ) / 2, δ]] + ys = np.empty((3, 2)); ys[:, 0] = [.25, .75, .2]; ys[:, 1] = [1.25, .25, .2] + for i in range(3): + yield f"ex4_s{i}", dict(s=s, P=P, ys=ys.copy(), s0_idx=i) + + # Finite horizon (Example 1, T=10) + s = [0, 1]; P = [[.5, .5], [.5, .5]] + ys = np.empty((2, 2)); ys[:, 0] = 1 - np.array(s); ys[:, 1] = s + yield "ex1_finite_T10_s0", dict(s=s, P=P, ys=ys.copy(), s0_idx=0, T=10) + yield "ex1_finite_T10_s1", dict(s=s, P=P, ys=ys.copy(), s0_idx=1, T=10) + + +def main(): + report = {} + all_ok = True + for name, kw in examples(): + o = run_old(**kw) + nw = run_new(**kw) + per_obj = {} + ok = True + for key in o: + a, b = o[key], nw[key] + try: + close = bool(np.allclose(a, b, atol=ATOL, rtol=RTOL)) + maxerr = float(np.max(np.abs(a - b))) + except Exception as e: # shape mismatch etc. + close, maxerr = False, float("nan") + per_obj[key] = {"match": close, "max_abs_err": maxerr} + ok = ok and close + report[name] = {"ok": ok, "objects": per_obj} + all_ok = all_ok and ok + flag = "OK " if ok else "FAIL" + worst = max((v["max_abs_err"] for v in per_obj.values() + if v["max_abs_err"] == v["max_abs_err"]), default=float("nan")) + print(f"[{flag}] {name:22s} max|Δ| = {worst:.2e}") + + x64 = bool(jax.config.jax_enable_x64) + report["_summary"] = {"all_equivalent": all_ok, "atol": ATOL, "rtol": RTOL, + "jax_enable_x64": x64} + fname = "equivalence_x64.json" if x64 else "equivalence.json" + with open(os.path.join(RESULTS, fname), "w") as f: + json.dump(report, f, indent=2) + print("\nALL EQUIVALENT:", all_ok) + + +if __name__ == "__main__": + main() diff --git a/benchmark/references/examples/ge_arrow/scripts/cold_start.py b/benchmark/references/examples/ge_arrow/scripts/cold_start.py new file mode 100644 index 0000000..6ab2de3 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/cold_start.py @@ -0,0 +1,46 @@ +""" +Measure GENUINE cold-start latency in a fresh interpreter. + +Run as a one-shot process: import time is excluded, but the first +trace+compile of `compute_rc_model` for the n=2 lecture economy is included. +This is what a reader waits for the first time a JAX cell executes, and again +each time a new `s0_idx` / `T` / shape is requested (static args -> recompile). + +Prints a single JSON line consumed by run_all.py. +""" +import json +import sys +import time + +mode = sys.argv[1] if len(sys.argv) > 1 else "jax_first" + +import numpy as np + +if mode == "numpy": + import model_old as old + s = np.array([0, 1]); P = np.array([[.5, .5], [.5, .5]]) + ys = np.empty((2, 2)); ys[:, 0] = 1 - s; ys[:, 1] = s + t0 = time.perf_counter() + m = old.RecurCompetitive(s, P, ys) + m.wealth_distribution(0); m.continuation_wealths(); m.value_functionss() + t1 = time.perf_counter() + print(json.dumps({"mode": mode, "first_call_s": t1 - t0})) + +else: + import jax + import jax.numpy as jnp + import model_new as new + s = jnp.array([0., 1.]); P = jnp.array([[.5, .5], [.5, .5]]) + ys = jnp.array([[1., 0.], [0., 1.]]) + t0 = time.perf_counter() + m = new.compute_rc_model(s, P, ys, s0_idx=0, T=0) + jax.block_until_ready(m.J) + t1 = time.perf_counter() + # second call at a NEW static arg (s0_idx=1) -> recompiles + t2 = time.perf_counter() + m2 = new.compute_rc_model(s, P, ys, s0_idx=1, T=0) + jax.block_until_ready(m2.J) + t3 = time.perf_counter() + print(json.dumps({"mode": mode, + "first_call_s": t1 - t0, + "recompile_new_s0idx_s": t3 - t2})) diff --git a/benchmark/references/examples/ge_arrow/scripts/model_new.py b/benchmark/references/examples/ge_arrow/scripts/model_new.py new file mode 100644 index 0000000..9cc3edd --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/model_new.py @@ -0,0 +1,221 @@ +""" +Verbatim extraction of the NEW (branch `update_ge_arrow`) JAX implementation +of the Arrow-securities competitive-equilibrium model from +`lectures/ge_arrow.md`. + +Copied as-is from the lecture so that benchmarks and equivalence checks run the +exact code under evaluation. +""" + +import jax +import jax.numpy as jnp +import numpy as np +from typing import NamedTuple +from functools import partial + + +class RecurCompetitive(NamedTuple): + """ + A class that represents a recursive competitive economy + with one-period Arrow securities. + """ + s: jax.Array # state vector + P: jax.Array # transition matrix + ys: jax.Array # endowments ys = [y1, y2, .., yT] + y: jax.Array # total endowment under each state + n: int # number of states + K: int # number of agents + γ: float # risk aversion + β: float # discount rate + T: float # time horizon, 0 if infinite + Q: jax.Array # pricing kernel + V: jax.Array # resolvent / partial-sum matrices + PRF: jax.Array # price of risk-free bond + R: jax.Array # risk-free rate + A: jax.Array # natural debt limit + α: jax.Array # wealth distribution + ψ: jax.Array # continuation value + J: jax.Array # optimal value + + +@partial(jax.jit, static_argnames=("T", "s0_idx")) +def compute_rc_model(s, P, ys, s0_idx=0, γ=0.5, β=0.98, T=0): + """Complete equilibrium objects under the endogenous pricing kernel. + + Args + ---- + s : array-like + Markov states. + P : array-like + Transition matrix. + ys : array-like + Endowment matrix; rows index states, columns index agents. + s0_idx : int, optional + Index of the initial zero-asset-holding state. + γ : float, optional + Risk aversion parameter. + β : float, optional + Discount factor. + T : int, optional + Number of periods; 0 means an infinite-horizon economy. + + Returns + ------- + RecurCompetitive + Instance containing all parameters and computed equilibrium results. + """ + n, K = ys.shape + y = jnp.sum(ys, axis=1) + + def u(c): + "CRRA utility evaluated elementwise." + return c ** (1 - γ) / (1 - γ) + + def u_prime(c): + "Marginal utility for the CRRA specification." + return c ** (-γ) + + def pricing_kernel(c): + "Build the Arrow-security pricing kernel matrix." + + Q = jnp.empty((n, n)) + # fori_loop iterates over each state i while carrying the partially + # filled matrix Q as the loop carry. + def body_fun_i(i, Q): + # fills row i entry-by-entry. + def body_fun_j(j, q): + ratio = u_prime(c[j]) / u_prime(c[i]) + # Return a (n,) array + return q.at[j].set(β * ratio * P[i, j]) + + q = jax.lax.fori_loop( + 0, n, body_fun_j, jnp.zeros((n,)) + ) + return Q.at[i, :].set(q) + + Q = jax.lax.fori_loop( + 0, n, body_fun_i, jnp.zeros((n, n)) + ) + return Q + + def resolvent_operator(Q): + "Compute the resolvent or finite partial sums of Q depending on T." + + def infinite_period(): + # If T=0, V.shape = (1, n, n) + V = jnp.zeros((T+1, n, n)) + V = V.at[0].set(jnp.linalg.inv(jnp.eye(n) - Q)) + return V + + # V = [I + Q + Q^2 + ... + Q^T] (finite case) + def finite_period(): + V = jnp.zeros((T+1, n, n)) + V = V.at[0].set(jnp.eye(n)) + + Qt = jnp.eye(n) + + # Loop body_fun advances the Q power and accumulates the geometric sum. + def body_fun(t, carry): + Qt, V = carry + Qt = Qt @ Q + V = V.at[t].set(V[t-1] + Qt) + return Qt, V + + _, V = jax.lax.fori_loop(1, T+1, body_fun, (Qt, V)) + return V + + V = jax.lax.cond(T==0, infinite_period, finite_period) + + return V + + def natural_debt_limit(ys, V): + "Compute natural debt limits from the terminal resolvent block." + return V[-1] @ ys + + def wealth_distribution(V, ys, y, s0_idx): + "Recover equilibrium wealth shares α from the initial state row." + + # row of V corresponding to s0 + Vs0 = V[-1, s0_idx, :] + α = Vs0 @ ys / (Vs0 @ y) + + return α + + def continuation_wealths(V, α): + "Back out continuation wealths for each agent." + diff = jnp.empty((n, K)) + + # Loop scatters each agent's state-dependent surplus into the column k. + def body_fun(k, diff): + return diff.at[:, k].set(α[k] * y - ys[:, k]) + + # Applies body_fun sequentially while threading diff. + diff = jax.lax.fori_loop(0, K, body_fun, diff) + + ψ = V @ diff + + return ψ + + def price_risk_free_bond(Q): + "Given Q, compute price of one-period risk-free bond" + return jnp.sum(Q, axis=1) + + def risk_free_rate(Q): + "Given Q, compute one-period gross risk-free interest rate R" + return jnp.reciprocal(price_risk_free_bond(Q)) + + def value_functions(α, y): + "Assemble lifetime value functions for each agent." + + # compute (I - βP)^(-1) in infinite case + def infinite_period(): + # If T=0, V.shape = (1, n, n) + P_seq = jnp.empty((T+1, n, n)) + P_seq = P_seq.at[0].set( + jnp.linalg.inv(jnp.eye(n) - β * P) + ) + return P_seq + # and (I + βP + ... + β^T P^T) in finite case + + def finite_period(): + P_seq = jnp.empty((T+1, n, n)) + P_seq = P_seq.at[0].set(jnp.eye(n)) + + Pt = jnp.eye(n) + + def body_fun(t, carry): + Pt, P_seq = carry + Pt = Pt @ P + P_seq = P_seq.at[t].set(P_seq[t-1] + Pt * β ** t) + return Pt, P_seq + + _, P_seq = jax.lax.fori_loop( + 1, T+1, body_fun, (Pt, P_seq) + ) + return P_seq + + P_seq = jax.lax.cond(T==0, infinite_period, finite_period) + + # compute the matrix [u(α_1 y), ..., u(α_K, y)] + def body_fun(k, flow): + return flow.at[:, k].set(u(α[k] * y)) + + flow = jax.lax.fori_loop(0, K, body_fun, jnp.empty((n, K))) + + J = P_seq @ flow + + return J + + Q = pricing_kernel(y) + V = resolvent_operator(Q) + A = natural_debt_limit(ys, V) + α = wealth_distribution(V, ys, y, s0_idx) + ψ = continuation_wealths(V, α) + PRF = price_risk_free_bond(Q) + R = risk_free_rate(Q) + J = value_functions(α, y) + + return RecurCompetitive( + s=s, P=P, ys=ys, y=y, n=n, K=K, γ=γ, β=β, T=T, + Q=Q, V=V, A=A, α=α, ψ=ψ, PRF=PRF, R=R, J=J + ) diff --git a/benchmark/references/examples/ge_arrow/scripts/model_old.py b/benchmark/references/examples/ge_arrow/scripts/model_old.py new file mode 100644 index 0000000..6e81081 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/model_old.py @@ -0,0 +1,185 @@ +""" +Faithful, runnable extraction of the ORIGINAL (main branch) NumPy implementation +of the Arrow-securities competitive-equilibrium model from +`lectures/ge_arrow.md`. + +Notes on fidelity +----------------- +* The class is copied as-is from the `main` branch, with ONE class of change: + the original methods `pricing_kernel` and `continuation_wealths` referenced + the *module-level globals* `P`, `n`, `K` instead of `self.P`, `self.n`, + `self.K`. In the lecture those globals happen to exist, so the code runs, but + it is a latent bug / reliance on global state. To make this module + self-contained and correct we replaced those references with `self.*`. This + is recorded as an evaluation finding (see EVALUATION_FRAMEWORK.md, "Logic"). +* The method name `value_functionss` (double 's') is preserved verbatim, because + it is part of what we are evaluating (a typo in the public API). +* Cosmetic-only: arithmetic spacing was normalised in a few places + (`T+1` → `T + 1`, `t-1` → `t - 1`). Found by diffing this file against a + fresh extraction from the lecture at base 8cfba4c (validation run + 2026-07-22); semantics identical, disclosed per the v2 verbatim rule. +""" + +import numpy as np + + +class RecurCompetitive: + """ + A class that represents a recursive competitive economy + with one-period Arrow securities. + """ + + def __init__(self, + s, # state vector + P, # transition matrix + ys, # endowments ys = [y1, y2, .., yI] + γ=0.5, # risk aversion + β=0.98, # discount rate + T=None): # time horizon, none if infinite + + # preference parameters + self.γ = γ + self.β = β + + # variables dependent on state + self.s = s + self.P = P + self.ys = ys + self.y = np.sum(ys, 1) + + # dimensions + self.n, self.K = ys.shape + + # compute pricing kernel + self.Q = self.pricing_kernel() + + # compute price of risk-free one-period bond + self.PRF = self.price_risk_free_bond() + + # compute risk-free rate + self.R = self.risk_free_rate() + + # V = [I - Q]^{-1} (infinite case) + if T is None: + self.T = None + self.V = np.empty((1, self.n, self.n)) + self.V[0] = np.linalg.inv(np.eye(self.n) - self.Q) + # V = [I + Q + Q^2 + ... + Q^T] (finite case) + else: + self.T = T + self.V = np.empty((T + 1, self.n, self.n)) + self.V[0] = np.eye(self.n) + + Qt = np.eye(self.n) + for t in range(1, T + 1): + Qt = Qt.dot(self.Q) + self.V[t] = self.V[t - 1] + Qt + + # natural debt limit + self.A = self.V[-1] @ ys + + def u(self, c): + "The CRRA utility" + + return c ** (1 - self.γ) / (1 - self.γ) + + def u_prime(self, c): + "The first derivative of CRRA utility" + + return c ** (-self.γ) + + def pricing_kernel(self): + "Compute the pricing kernel matrix Q" + + c = self.y + + n = self.n + Q = np.empty((n, n)) + for i in range(n): + for j in range(n): + ratio = self.u_prime(c[j]) / self.u_prime(c[i]) + Q[i, j] = self.β * ratio * self.P[i, j] + + self.Q = Q + + return Q + + def wealth_distribution(self, s0_idx): + "Solve for wealth distribution α" + + # set initial state + self.s0_idx = s0_idx + + # simplify notations + n = self.n + Q = self.Q + y, ys = self.y, self.ys + + # row of V corresponding to s0 + Vs0 = self.V[-1, s0_idx, :] + α = Vs0 @ self.ys / (Vs0 @ self.y) + + self.α = α + + return α + + def continuation_wealths(self): + "Given α, compute the continuation wealths ψ" + + diff = np.empty((self.n, self.K)) + for k in range(self.K): + diff[:, k] = self.α[k] * self.y - self.ys[:, k] + + ψ = self.V @ diff + self.ψ = ψ + + return ψ + + def price_risk_free_bond(self): + "Give Q, compute price of one-period risk free bond" + + PRF = np.sum(self.Q, axis=1) + self.PRF = PRF + + return PRF + + def risk_free_rate(self): + "Given Q, compute one-period gross risk-free interest rate R" + + R = np.sum(self.Q, axis=1) + R = np.reciprocal(R) + self.R = R + + return R + + def value_functionss(self): + "Given α, compute the optimal value functions J in equilibrium" + + n, T = self.n, self.T + β = self.β + P = self.P + + # compute (I - βP)^(-1) in infinite case + if T is None: + P_seq = np.empty((1, n, n)) + P_seq[0] = np.linalg.inv(np.eye(n) - β * P) + # and (I + βP + ... + β^T P^T) in finite case + else: + P_seq = np.empty((T + 1, n, n)) + P_seq[0] = np.eye(n) + + Pt = np.eye(n) + for t in range(1, T + 1): + Pt = Pt.dot(P) + P_seq[t] = P_seq[t - 1] + Pt * β ** t + + # compute the matrix [u(α_1 y), ..., u(α_K, y)] + flow = np.empty((n, self.K)) + for k in range(self.K): + flow[:, k] = self.u(self.α[k] * self.y) + + J = P_seq @ flow + + self.J = J + + return J diff --git a/benchmark/references/examples/ge_arrow/scripts/run_all.py b/benchmark/references/examples/ge_arrow/scripts/run_all.py new file mode 100644 index 0000000..e36add2 --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/run_all.py @@ -0,0 +1,118 @@ +""" +Run the whole evaluation pipeline and regenerate everything in ../results/. + +Usage: + python run_all.py + +Requires the `quantecon` conda env (jax 0.4.x, numpy 2.x). See README.md. +""" +import json +import subprocess +import sys +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +PY = sys.executable +K_AS_USED = 3 # fresh-process repeats per as-used side (v2: median, not a single pass) + + +def median(xs): + s = sorted(xs) + n = len(s) + return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2.0 + +# (title, command, results-file to aggregate the script's JSON line into or None) +STEPS = [ + ("Numerical equivalence", "check_equivalence.py", None), + ("Static metrics", "static_metrics.py", None), + ("Performance benchmark", "benchmark.py", None), + ("Cold-start (numpy)", "cold_start.py numpy", "cold_start.json"), + ("Cold-start (jax)", "cold_start.py jax_first", "cold_start.json"), + ("Lambda-sweep benchmark", "sweep_bench.py", None), + ("As-used total (numpy)", "as_used_total.py numpy", "as_used.json"), + ("As-used total (jax)", "as_used_total.py jax", "as_used.json"), + # HIGH-end efficiency calibration is shared, not lecture-specific: + # see scripts/calibration/bellman_bench.py +] + +collected = {} +failed = [] +for title, cmd, agg in STEPS: + print("\n" + "=" * 70) + print("==", title) + print("=" * 70) + parts = cmd.split() + argv = [PY, os.path.join(HERE, parts[0])] + parts[1:] + if agg is None: + p = subprocess.run(argv, cwd=HERE, check=False) + if p.returncode: + failed.append(title) + continue + # Persist the script's JSON line (these run as fresh processes and only + # print their result; the headline as-used metric must not live on the + # console alone). As-used steps repeat K_AS_USED times so the headline + # metric is a median of fresh-process runs, never a single pass. + reps = K_AS_USED if agg == "as_used.json" else 1 + for _ in range(reps): + p = subprocess.run(argv, cwd=HERE, check=False, capture_output=True, + text=True) + sys.stdout.write(p.stdout) + if p.stderr: + sys.stderr.write(p.stderr) + if p.returncode: + failed.append(title) + break + for line in reversed(p.stdout.strip().splitlines()): + try: + rec = json.loads(line) + except ValueError: + continue + if not isinstance(rec, dict): + continue # a stray scalar/list line is not a result record + bucket = collected.setdefault(agg, {}) + key = rec.get("mode", "?") + if agg == "as_used.json": + bucket.setdefault(key, dict(rec, runs=[]))["runs"].append( + rec["total_s"]) + else: + if key in bucket: + print(f"WARNING: duplicate mode {key!r} for {agg}", + file=sys.stderr) + bucket[key] = rec + break + +RES = os.path.join(os.path.dirname(HERE), "results") +os.makedirs(RES, exist_ok=True) +for fname, recs in collected.items(): + modes = list(recs) + if fname == "as_used.json" and "numpy" in recs and len(modes) == 2: + other = next(m for m in modes if m != "numpy") + a, b = recs["numpy"], recs[other] + if a.get("runs") and b.get("runs"): + a["total_s"] = median(a["runs"]) + b["total_s"] = median(b["runs"]) + recs["as_used_speedup"] = a["total_s"] / b["total_s"] + recs["as_used_speedup_runs"] = [x / y + for x, y in zip(a["runs"], b["runs"])] + recs["baseline_as_used_seconds"] = a["total_s"] + with open(os.path.join(RES, fname), "w", encoding="utf-8") as f: + json.dump(recs, f, indent=2) + print(f"wrote results/{fname}") + +LEC_DIR = os.path.dirname(HERE) # this example's folder +# Shared engine location: the installed plugin root when the skill drives an +# evaluation from a user workspace; falls back to this repo's layout. +PLUGIN = (os.environ.get("CLAUDE_PLUGIN_ROOT") + or os.path.dirname(os.path.dirname(os.path.dirname(LEC_DIR)))) + +# Provenance stamp (shared: scripts/scoring/env_stamp.py — the seed of the +# QuantEcon/meta#335 result + environment-descriptor schema). Failed step +# titles are recorded so a partial run cannot claim full provenance. +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "env_stamp.py"), + LEC_DIR] + failed, check=False) + +# Scoring is shared across lectures: fill ../evidence.json from the results +# above, then apply the common rubric (scripts/scoring/rubric.py) via the engine. +print("\n" + "=" * 70 + "\n== Scorecard (shared rubric)\n" + "=" * 70) +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "score.py"), + LEC_DIR], check=False) diff --git a/benchmark/references/examples/ge_arrow/scripts/static_metrics.py b/benchmark/references/examples/ge_arrow/scripts/static_metrics.py new file mode 100644 index 0000000..170425c --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/static_metrics.py @@ -0,0 +1,143 @@ +""" +Static code metrics for the OLD and NEW implementations. + +Computes objective, reproducible numbers used by several rubric dimensions: + * code size (non-blank, non-comment lines in the model definition) + * number of def/functions and maximum lexical nesting depth + * docstring coverage + * "concept surface" -- count of advanced-API tokens a reader must understand + * number of explicit loops + * call-site ergonomics -- how many statements the lecture needs to obtain one + full set of results (α, ψ, J) + +Source of truth = the two extracted modules (model_old.py / model_new.py), +which are verbatim copies of the lecture code (see their headers). + +Output: results/static_metrics.json + stdout table. +""" +import ast +import json +import os +import re + +HERE = os.path.dirname(__file__) +RESULTS = os.path.join(HERE, "..", "results") +os.makedirs(RESULTS, exist_ok=True) + +# Tokens that represent a *concept a reader must already understand* to follow +# the code. NumPy-side and JAX-side, scored symmetrically. +CONCEPTS = { + "old": [ + r"\bclass\b", r"def __init__", r"self\.", r"@", r"for .+ in ", + r"np\.linalg\.inv", r"np\.empty", r"\.dot\(", + ], + "new": [ + r"NamedTuple", r"@partial", r"jax\.jit", r"static_argnames", + r"jax\.lax\.fori_loop", r"jax\.lax\.cond", r"\.at\[", r"\.set\(", + r"jnp\.", r"def body_fun", r"carry", + ], +} + +# Distinct prerequisite *ideas* (deduplicated, hand-curated from the token hits). +PREREQS = { + "old": ["Python class / OOP", "__init__ constructor", "instance state (self.)", + "NumPy arrays & slicing", "matrix @ / .dot", "np.linalg.inv", + "Python for-loops"], + "new": ["Python class / OOP (NamedTuple)", "immutable NamedTuple", + "typing annotations", "functools.partial", "jax.jit & tracing", + "static_argnames & recompilation", "functional purity (no in-place)", + "jnp vs np", "jax.lax.fori_loop (carry)", "jax.lax.cond", + "functional array update .at[].set()", "nested closures as sub-fns", + "float32 default / x64 flag"], +} + + +def code_lines(src): + n = 0 + for line in src.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + n += 1 + return n + + +def max_nesting(tree): + """Maximum nesting depth of def/for/if/with within function bodies.""" + best = 0 + + def walk(node, depth): + nonlocal best + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.For, ast.While, + ast.If, ast.With)): + best = max(best, depth + 1) + walk(child, depth + 1) + else: + walk(child, depth) + + for node in tree.body: + walk(node, 0) + return best + + +def count_defs_and_docs(tree): + ndef, ndoc = 0, 0 + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, + ast.ClassDef)): + ndef += 1 + if ast.get_docstring(node): + ndoc += 1 + return ndef, ndoc + + +def count_loops(tree): + return sum(isinstance(n, (ast.For, ast.While)) for n in ast.walk(tree)) + + +def concept_hits(src, patterns): + return sum(len(re.findall(p, src)) for p in patterns) + + +def analyze(path, key): + src = open(path, encoding="utf-8").read() + tree = ast.parse(src) + ndef, ndoc = count_defs_and_docs(tree) + return { + "code_lines": code_lines(src), + "n_defs": ndef, + "docstring_coverage": round(ndoc / ndef, 2) if ndef else 0, + "max_nesting_depth": max_nesting(tree), + "explicit_loops": count_loops(tree), + "concept_token_hits": concept_hits(src, CONCEPTS[key]), + "n_prerequisite_concepts": len(PREREQS[key]), + "prerequisite_concepts": PREREQS[key], + } + + +def main(): + old = analyze(os.path.join(HERE, "model_old.py"), "old") + new = analyze(os.path.join(HERE, "model_new.py"), "new") + + # Call-site ergonomics: statements needed to get α, ψ, J for one economy. + # OLD: construct + 3 stateful method calls in order. NEW: 1 call. + old["statements_for_one_result"] = 4 + new["statements_for_one_result"] = 1 + + out = {"old": old, "new": new} + keys = ["code_lines", "n_defs", "docstring_coverage", "max_nesting_depth", + "explicit_loops", "concept_token_hits", "n_prerequisite_concepts", + "statements_for_one_result"] + print(f"{'metric':30s} {'OLD(numpy)':>12s} {'NEW(jax)':>12s}") + print("-" * 56) + for k in keys: + print(f"{k:30s} {str(old[k]):>12s} {str(new[k]):>12s}") + + with open(os.path.join(RESULTS, "static_metrics.json"), "w") as f: + json.dump(out, f, indent=2) + print("\nwrote results/static_metrics.json") + + +if __name__ == "__main__": + main() diff --git a/benchmark/references/examples/ge_arrow/scripts/sweep_bench.py b/benchmark/references/examples/ge_arrow/scripts/sweep_bench.py new file mode 100644 index 0000000..b4e810e --- /dev/null +++ b/benchmark/references/examples/ge_arrow/scripts/sweep_bench.py @@ -0,0 +1,98 @@ +""" +Benchmark the one workload in the lecture that repeats the solver many times: +Example 3's sweep over 100 values of the transition probability lambda. + +OLD: a Python for-loop building 100 NumPy models. +NEW: a single jitted `fori_loop` (compile once, run 100 iterations). + +This is the scenario most favourable to the JAX rewrite, so it bounds the +upside. Prints JSON consumed by run_all.py. +""" +import json +import os +import time +import statistics +import numpy as np +import jax +import jax.numpy as jnp + +import model_old as old +import model_new as new + +RESULTS = os.path.join(os.path.dirname(__file__), "..", "results") + +s_np = np.array([1, 2]) +ys_np = np.empty((2, 2)); ys_np[:, 0] = [1, 0]; ys_np[:, 1] = [0, 1] +λ_seq = np.linspace(0, 0.99, 100) + + +def old_sweep(): + αs0 = np.empty((100, 2)); αs1 = np.empty((100, 2)) + for i, λ in enumerate(λ_seq): + P = np.array([[1 - λ, λ], [0, 1]]) + m = old.RecurCompetitive(s_np, P, ys_np) + αs0[i] = m.wealth_distribution(0) + αs1[i] = m.wealth_distribution(1) + return αs0, αs1 + + +s_j = jnp.asarray(s_np, float) +ys_j = jnp.asarray(ys_np) +λ_j = jnp.asarray(λ_seq) + + +@jax.jit +def new_sweep(): + def body(i, carry): + a0, a1 = carry + λ = λ_j[i] + P = jnp.array([[1 - λ, λ], [0., 1.]]) + m0 = new.compute_rc_model(s_j, P, ys_j, s0_idx=0) + m1 = new.compute_rc_model(s_j, P, ys_j, s0_idx=1) + return a0.at[i].set(m0.α), a1.at[i].set(m1.α) + a0 = jnp.empty((100, 2)); a1 = jnp.empty((100, 2)) + return jax.lax.fori_loop(0, 100, body, (a0, a1)) + + +def med(fn, repeat, number=1): + xs = [] + for _ in range(repeat): + t0 = time.perf_counter() + for _ in range(number): + r = fn() + xs.append((time.perf_counter() - t0) / number) + return statistics.median(xs), r + + +def main(): + t_old, _ = med(old_sweep, repeat=7) + + # cold (includes compile of the whole sweep) + new.compute_rc_model._clear_cache() + t0 = time.perf_counter() + r = new_sweep(); jax.block_until_ready(r) + t_new_cold = time.perf_counter() - t0 + + # warm + def warm(): + r = new_sweep(); jax.block_until_ready(r); return r + t_new_warm, _ = med(warm, repeat=11, number=3) + + out = { + "old_python_loop_s": t_old, + "jax_sweep_cold_s": t_new_cold, + "jax_sweep_warm_s": t_new_warm, + "speedup_warm": t_old / t_new_warm, + "speedup_cold": t_old / t_new_cold, + } + print(f" old python loop = {t_old*1e3:8.3f} ms") + print(f" jax sweep (cold) = {t_new_cold*1e3:8.3f} ms " + f"({out['speedup_cold']:.2f}x vs numpy)") + print(f" jax sweep (warm) = {t_new_warm*1e3:8.3f} ms " + f"({out['speedup_warm']:.2f}x vs numpy)") + with open(os.path.join(RESULTS, "sweep.json"), "w") as f: + json.dump(out, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/benchmark/references/examples/markov_asset/evidence.json b/benchmark/references/examples/markov_asset/evidence.json new file mode 100644 index 0000000..bf47b8f --- /dev/null +++ b/benchmark/references/examples/markov_asset/evidence.json @@ -0,0 +1,83 @@ +{ + "lecture": "markov_asset", + "branch": "update_markov_asset", + "source_pr": "QuantEcon/lecture-python.myst#654", + "refs": { + "base": "8cfba4c90ebc08d3e51718ee65246ac249305ce0", + "head": "533f572d6d5abf3660b2cae93d667d9df9c931e0" + }, + "_how": "Quantitative values are copied from results/*.json (source noted). Structural criteria are yes/no facts about the diff, each with a citation. Scores are computed by scoring/rubric.py — run: python scripts/scoring/score.py references/examples/ (from the plugin root).", + + "quantitative": { + "correctness": { + "builds": false, + "max_delta_shipped": 1.02e-2, + "matches_under_x64": true, + "source": "smoke_test.py: call_option_jit raises NameError ('err' undefined) => does not build. equivalence_x64_False.json worst=1.02e-2; equivalence_x64_True.json worst~1e-11" + }, + "readability": { + "delta_prereq_concepts": 5, + "docstring_cov_new": 0.75, + "source": "results/static_metrics.json: prereq 8->13 (+5), docstring_coverage 0.86->0.75" + }, + "efficiency": { + "as_used_speedup": 0.17, + "as_used_runs": [], + "baseline_as_used_seconds": 0.18, + "correct_or_fixable": true, + "source": "as_used_total.py (bug-patched to run): numpy 0.18s / jax 1.05s = 0.17x; scaling.json: JAX only wins at n>=250, lecture uses n=5,25. Single-pass v1 measurement (v2 standard: median of >=3 fresh-process runs)" + }, + "ergonomics": { + "statements_for_one_result": 3, + "fragile_protocol": true, + "source": "results/static_metrics.json: `err, v = f_jit(ap); err.throw()`; checkify (err,val) contract is easy to misuse (the shipped bug proves it)" + } + }, + + "structural": { + "logic_design": { + "criteria": { + "pure_no_order_dependence": true, + "no_global_state": true, + "good_algorithmic_choices": true, + "fixes_prior_bugs": true + }, + "introduces_correctness_bug": true, + "citations": { + "pure_no_order_dependence": "immutable NamedTuples + factory functions replace the mutable class", + "no_global_state": "create_ap_model uses a local n; original __init__ referenced module-global n", + "good_algorithmic_choices": "M = P*G**(-gamma) vectorised; precomputes G", + "fixes_prior_bugs": "removes global-n reliance; extracts pure test_stability", + "introduces_correctness_bug": "TRUE (override->cap 3): stray err.throw() in call_option (undefined err) is a build-breaking defect" + } + }, + "style_idiom": { + "criteria": { + "vectorised_where_natural": true, + "correct_control_flow_primitive": true, + "no_anti_idiomatic_constructs": true, + "clean_call_sites_and_naming": false + }, + "citations": { + "vectorised_where_natural": "TRUE: M=P*G**(-gamma)", + "correct_control_flow_primitive": "TRUE: lax.while_loop (infinite option) and fori_loop (finite) are the right primitives for genuinely iterative solvers", + "no_anti_idiomatic_constructs": "TRUE: checkify is the idiomatic jit-safe runtime check", + "clean_call_sites_and_naming": "FALSE: every call site carries (err,val) unpack + err.throw() ceremony" + } + }, + "maintainability": { + "criteria": { + "pure_unit_testable": true, + "dtype_precision_safe": false, + "no_footgun_for_editors": false, + "robust_no_brittle_conditions": false + }, + "citations": { + "pure_unit_testable": "pure NamedTuples + functions", + "dtype_precision_safe": "FALSE: float32 default, no jax_enable_x64", + "no_footgun_for_editors": "FALSE: checkify (err,val) return-contract already produced the shipped NameError", + "robust_no_brittle_conditions": "FALSE: exercise spectral radius 1.0618 vs 1/beta 1.0638 (margin 0.002) is fragile under float32" + } + } + } +} diff --git a/benchmark/references/examples/markov_asset/markov_asset_REPORT.md b/benchmark/references/examples/markov_asset/markov_asset_REPORT.md new file mode 100644 index 0000000..cee9acc --- /dev/null +++ b/benchmark/references/examples/markov_asset/markov_asset_REPORT.md @@ -0,0 +1,98 @@ +# Evaluation Report — `markov_asset.md`: NumPy (`main`) → JAX (`update_markov_asset`) + +> **Erratum (2026-07-21).** This report states that "the lecture does not build as shipped." Design review later established this is false *as worded*: executing the notebook's cells **in order** completes cleanly (and the PR's preview CI passes), because earlier cells bind a global `err` immediately before `call_option_jit` is first traced, so the stray `err.throw()` inside `call_option` resolves to that stale, already-checked object and silently does nothing. The defect is real and arguably worse than a crash: the stale-global masking means the checkify stability validation inside `call_option` is **never actually performed** in the shipped lecture — on the model whose spectral radius sits 0.002 below the stability bound — and any reader copying the function into a clean namespace hits the `NameError` this report describes. The evaluation's `builds: false` refers to fresh-process/clean-namespace execution of the extracted implementation (the system's declared measurement regime); the verdict and must-fix list stand. Nothing has been posted to [lecture-python.myst#654](https://github.com/QuantEcon/lecture-python.myst/pull/654) — the PR will receive one authoritative evaluation once the rubric-v2 revision lands and the skill has run the full protocol. See the design-review documents in `reviews/`. + +Applies the system in [`../../EVALUATION_FRAMEWORK.md`](../../EVALUATION_FRAMEWORK.md) to the NumPy→JAX conversion of `markov_asset.md`. All numbers are reproduced by `scripts/run_all.py` (CPU, jax 0.4.35, numpy 2.1.3, quantecon) into `results/`. Every dimension score is **computed from [`evidence.json`](evidence.json) by the shared rubric** (`../../../scripts/scoring/rubric.py`) — see `results/scorecard.json` for the derivation. + +> **Rubric v2 note (2026-07-22).** Re-scored under rubric v2 (verdict gates, no-conversion, sensitivity stamp — see `reviews/`): the total is unchanged at **2.25/5** and the candidate band remains *net regression* — now **gated**: correctness 1 caps the verdict independent of the weighted sum, and the logic&design bug-cap is derived from the correctness evidence rather than a hand-set boolean. The headline verdict is now **no-conversion** (baseline as-used 0.18 s, 0.17× slowdown). Sensitivity stamp: **robust** across all 29 single-input perturbations — the design review's one-concept band flip (2.25→2.50) no longer changes the verdict because the correctness gate holds it at net regression. Derivation: `results/scorecard.json`. + +## TL;DR — weighted score **2.25 / 5** → *net regression as shipped: do not merge until fixed* + +The conversion **does not build**: the shipped `call_option` raises `NameError: name 'err' is not defined`, so the call-option and exercise cells crash. That single defect dominates the score. Once the bug is patched, the change is roughly *neutral* — genuinely better structure (immutable `NamedTuple` + factory functions, pure `test_stability`, idiomatic `lax` loops) offset by a readability drop from the `checkify` plumbing, ~6× slower as-used runtime at these tiny state spaces, and float32 precision drift up to `1e-2`. + +| Dimension | Wt | Score | Weighted | how the score arises | +|---|:--:|:--:|:--:|---| +| Correctness & numerical fidelity | 0.20 | 1 | 0.20 | does not build (overrides Δ bands) | +| Readability & pedagogical clarity | 0.25 | 2 | 0.50 | Δprereq +5→2, docstrings 0.75→4 (worse-of-two) | +| Computational efficiency (as used) | 0.15 | 2 | 0.30 | as-used speedup 0.17× < 0.8 | +| Logic & design | 0.15 | 3 | 0.45 | 4/4 criteria met, **capped at 3** (introduces a bug) | +| Coding style & idiom | 0.10 | 4 | 0.40 | 3/4 criteria met (idiomatic primitives) | +| API ergonomics & reusability | 0.10 | 3 | 0.30 | 3 statements + fragile protocol | +| Maintainability & robustness | 0.05 | 2 | 0.10 | 1/4 criteria met | +| **Total** | **1.00** | | **2.25** | | + +--- + +## What changed + +| | Original (`main`) | Rewrite (`update_markov_asset`) | +|---|---|---| +| Library | NumPy | JAX (`jnp`, `lax`, `jit`, `experimental.checkify`) | +| Model container | mutable `AssetPriceModel` class (`__init__`) | `MarkovChain` + `AssetPriceModel` `NamedTuple`s + `create_*` factories | +| Growth function | stores callable `g`, computes `g(y)` on demand | precomputes and stores vector `G = g(state_values)` | +| Stability check | `self.test_stability` raises `ValueError` | module `test_stability` via `checkify.check` | +| Iterative solvers | Python `while` / `for` loops | `jax.lax.while_loop` / `jax.lax.fori_loop` | +| Call protocol | `v = tree_price(ap)` | `err, v = tree_price_jit(ap); err.throw()` | +| Prose | — | many title-case → sentence-case + link/figure edits (non-code) | + +--- + +## Evidence by dimension + +### 1 · Correctness & numerical fidelity → **1/5** +Three findings, from `smoke_test.py` and `check_equivalence.py`: + +- **Build-breaking bug (decisive).** The shipped `call_option` contains a stray `err.throw()` referencing a name that is never bound in the function (it calls `p = consol_price(ap, ζ)`, which returns only `p`). `call_option_jit(...)` raises `NameError: name 'err' is not defined`. The lecture cells "consol price and call option value" and Exercise 1 both call it → **the lecture does not build as shipped.** +- **Logic otherwise equivalent.** Under `JAX_ENABLE_X64=1`, every *working* asset (`tree_price`, `consol_price`, `finite_call_option`, and a bug-patched `call_option`) matches NumPy to `max|Δ| ≈ 1e-11` across the default, β=0.9, and exercise models. +- **Silent precision drift.** As shipped (float32, no x64), drift reaches `1.02e-2` on the exercise model — amplified because its spectral radius `1.0618` sits a hair below the stability bound `1/β = 1.0638` (margin `0.002`), so float32 rounding is both inaccurate and close to flipping the stability check. + +A lecture that crashes on build is the worst correctness outcome, hence score 1. + +### 2 · Readability & pedagogical clarity → **2/5** +`static_metrics.py`: + +| metric | old | new | +|---|--:|--:| +| prerequisite concepts | **8** | **13** (+5) | +| docstring coverage | 0.86 | 0.75 | +| code lines (model+funcs) | 74 | 101 (+36%) | +| statements to price one asset | 2 | 3 | + +The five new prerequisites are all `checkify`-related or JAX-structural: `checkify.check`, the `(err, value)` return contract, `err.throw()`, two `NamedTuple`s + two factory functions, and the float32/x64 distinction. Every asset call site now carries boilerplate — `err, v = tree_price_jit(ap); err.throw()` — that has nothing to do with the economics. +5 prerequisites lands in the score-2 band. + +### 3 · Computational efficiency (as used) → **2/5** +Replaying the whole lecture's asset-pricing sequence once in a fresh process (`as_used_total.py`, JAX side bug-patched so it can complete): + +| NumPy total | JAX total | as-used speedup | +|--:|--:|--:| +| **~0.15–0.18 s** | **~0.7–1.1 s** | **~0.17× (≈5–6× slower)** | + +*(Representative single-CPU medians; ±~15% run-to-run.)* + +Scaling (`benchmark.py`, warm): the core operations are LAPACK `eigvals` + dense `solve` (`O(n³)`) in both libraries, so JAX has little to exploit on CPU — it is *slower* at the lecture's `n = 5` and `n = 25`, and only edges ahead (1.2–1.4×) at `n ≥ 250`, which this lecture never uses. Stated speed goal not met. + +### 4 · Logic & design → **3/5** +Real improvements: the mutable class becomes two immutable `NamedTuple`s built by clear `create_ap_model` / `create_customized_ap_model` factories; `test_stability` is a pure function; the original's reliance on a module-global `n` in `__init__` is gone. Offset by the introduced logic defect (the stray `err.throw()`) and the extra `checkify` indirection. + +### 5 · Coding style & idiom → **4/5** +3 of the 4 idiom criteria are met: `M = P * G**(-γ)` is properly vectorised, `jax.lax.while_loop` (infinite-horizon option) and `fori_loop` (finite horizon) are the *correct* primitives for these genuinely iterative solvers (a better use of JAX than `ge_arrow`'s loop-ported pricing kernel), and `checkify` is the idiomatic way to keep a runtime assertion under `jit`. The one failing criterion is clean call sites: every usage carries `(err, val)` unpack + `err.throw()` ceremony. + +### 6 · API ergonomics & reusability → **3/5** +Factory functions and immutable models compose well (easily `vmap`-able over γ). But the `checkify` call protocol — unpack `(err, val)` then remember `err.throw()` — is error-prone, and the shipped code itself demonstrates the failure mode. `statements_for_one_result`: 2 → 3. + +### 7 · Maintainability & robustness → **2/5** +Purity aids testing, but two silent traps remain for future editors: float32 by default (worsened by the `0.002` stability margin) and the `checkify` return contract that already produced one shipped `NameError`. + +--- + +## Recommendation + +**Must-fix before merge (correctness):** +1. Delete the stray `err.throw()` in `call_option` (it belongs only at the *call site* after `call_option_jit`). *(D1)* +2. Add `jax.config.update("jax_enable_x64", True)` — the near-critical spectral radius makes float32 genuinely risky here, not just imprecise. *(D1, D7)* + +**Then, to lift the score toward "merge":** +3. Reduce `checkify` boilerplate at call sites (e.g. a small helper that unpacks and throws) to recover readability. *(D2, D6)* +4. Given the `O(n³)` LAPACK-bound workload at `n ≤ 25`, consider whether JAX earns its place in *this* lecture at all, or whether it is better reserved for the large, repeatedly-solved models where it wins (cf. the aiyagari calibration in `EVALUATION_FRAMEWORK.md`). *(D3)* + +After fixes 1–2 the correctness score rises from 1 to ~3 and the total clears the 2.5 "wash" line; fixes 3–4 would push it toward the 3.0 "merge after addressing" threshold — the same profile as the `ge_arrow` conversion. diff --git a/benchmark/references/examples/markov_asset/results/equivalence_x64_False.json b/benchmark/references/examples/markov_asset/results/equivalence_x64_False.json new file mode 100644 index 0000000..734800f --- /dev/null +++ b/benchmark/references/examples/markov_asset/results/equivalence_x64_False.json @@ -0,0 +1,65 @@ +{ + "tree_price_default_sweep": { + "\u03b3=1.2": { + "match": true, + "max_abs_err": 6.743074095538759e-06 + }, + "\u03b3=1.4": { + "match": true, + "max_abs_err": 1.3713897232037198e-05 + }, + "\u03b3=1.6": { + "match": true, + "max_abs_err": 7.330243498415712e-06 + }, + "\u03b3=1.8": { + "match": true, + "max_abs_err": 2.2134805320206397e-05 + }, + "\u03b3=2.0": { + "match": true, + "max_abs_err": 0.00020266582063754868 + } + }, + "consol_price_beta0.9": { + "match": true, + "max_abs_err": 0.00011387733371748254 + }, + "call_option_beta0.9": { + "shipped": { + "runs": false, + "error": "NameError: name 'err' is not defined" + }, + "patched_vs_numpy": { + "match": true, + "max_abs_err": 0.00011387733371748254 + } + }, + "exercise_model": { + "tree_price": { + "match": true, + "max_abs_err": 1.8592420875762627e-05 + }, + "consol_price": { + "match": true, + "max_abs_err": 0.0101597865821077 + }, + "finite_call_k5": { + "match": true, + "max_abs_err": 0.0101597865821077 + }, + "finite_call_k25": { + "match": true, + "max_abs_err": 0.0101597865821077 + }, + "call_option_patched": { + "match": true, + "max_abs_err": 0.0101597865821077 + } + }, + "_meta": { + "jax_enable_x64": false, + "atol": 1e-05, + "rtol": 0.0001 + } +} \ No newline at end of file diff --git a/benchmark/references/examples/markov_asset/results/equivalence_x64_True.json b/benchmark/references/examples/markov_asset/results/equivalence_x64_True.json new file mode 100644 index 0000000..6e2b822 --- /dev/null +++ b/benchmark/references/examples/markov_asset/results/equivalence_x64_True.json @@ -0,0 +1,65 @@ +{ + "tree_price_default_sweep": { + "\u03b3=1.2": { + "match": true, + "max_abs_err": 3.552713678800501e-15 + }, + "\u03b3=1.4": { + "match": true, + "max_abs_err": 2.1316282072803006e-14 + }, + "\u03b3=1.6": { + "match": true, + "max_abs_err": 1.4210854715202004e-14 + }, + "\u03b3=1.8": { + "match": true, + "max_abs_err": 2.1316282072803006e-14 + }, + "\u03b3=2.0": { + "match": true, + "max_abs_err": 3.552713678800501e-15 + } + }, + "consol_price_beta0.9": { + "match": true, + "max_abs_err": 1.4210854715202004e-14 + }, + "call_option_beta0.9": { + "shipped": { + "runs": false, + "error": "NameError: name 'err' is not defined" + }, + "patched_vs_numpy": { + "match": true, + "max_abs_err": 1.4210854715202004e-14 + } + }, + "exercise_model": { + "tree_price": { + "match": true, + "max_abs_err": 1.7763568394002505e-14 + }, + "consol_price": { + "match": true, + "max_abs_err": 9.777068044058979e-12 + }, + "finite_call_k5": { + "match": true, + "max_abs_err": 9.777068044058979e-12 + }, + "finite_call_k25": { + "match": true, + "max_abs_err": 9.777068044058979e-12 + }, + "call_option_patched": { + "match": true, + "max_abs_err": 9.777068044058979e-12 + } + }, + "_meta": { + "jax_enable_x64": true, + "atol": 1e-05, + "rtol": 0.0001 + } +} \ No newline at end of file diff --git a/benchmark/references/examples/markov_asset/results/scaling.json b/benchmark/references/examples/markov_asset/results/scaling.json new file mode 100644 index 0000000..f4a8a9b --- /dev/null +++ b/benchmark/references/examples/markov_asset/results/scaling.json @@ -0,0 +1,48 @@ +{ + "scaling": [ + { + "n": 5, + "numpy_s": 5.759997293353081e-05, + "jax_warm_s": 5.229999078437686e-05, + "speedup_warm": 1.1013381086624812 + }, + { + "n": 25, + "numpy_s": 0.0001506999833509326, + "jax_warm_s": 0.00017959997057914734, + "speedup_warm": 0.8390869044408952 + }, + { + "n": 50, + "numpy_s": 0.00048099999548867345, + "jax_warm_s": 0.0006284999544732273, + "speedup_warm": 0.7653142885138634 + }, + { + "n": 100, + "numpy_s": 0.002140600001439452, + "jax_warm_s": 0.004362899984698743, + "speedup_warm": 0.4906369637046035 + }, + { + "n": 250, + "numpy_s": 0.054126600036397576, + "jax_warm_s": 0.08901989995501935, + "speedup_warm": 0.6080280933111256 + }, + { + "n": 500, + "numpy_s": 0.6400167000247166, + "jax_warm_s": 0.513420999981463, + "speedup_warm": 1.246572890567048 + }, + { + "n": 1000, + "numpy_s": 2.8287528000073507, + "jax_warm_s": 1.7714457999682054, + "speedup_warm": 1.5968610499164706 + } + ], + "device": "TFRT_CPU_0", + "jax_x64": false +} \ No newline at end of file diff --git a/benchmark/references/examples/markov_asset/results/scorecard.json b/benchmark/references/examples/markov_asset/results/scorecard.json new file mode 100644 index 0000000..c990c9d --- /dev/null +++ b/benchmark/references/examples/markov_asset/results/scorecard.json @@ -0,0 +1,105 @@ +{ + "lecture": "markov_asset", + "branch": "update_markov_asset", + "weighted_total_out_of_5": 2.25, + "verdict": "no-conversion — the baseline as-used total 0.18 s is under the 1 s materiality floor and the candidate is slower as-used (0.17×): this lecture should not be converted, whatever the candidate's polish. Candidate quality for the record: 2.25/5, net regression", + "band_verdict": "net regression — do not merge as-is", + "verdict_gate": null, + "no_conversion": true, + "sensitivity": { + "stamp": "robust-at-floor", + "stamp_note": "no perturbation changed the outcome, but the verdict is already in the bottom band, where no single input can make it worse — only upward moves were available to this search, so the stability is partly structural, not purely evidential", + "perturbations_tested": 29, + "deciding_flips": [], + "perturbations_skipped": [] + }, + "dimensions": [ + { + "dim": "correctness", + "title": "Correctness & numerical fidelity", + "kind": "quantitative", + "weight": 0.2, + "score": 1, + "weighted": 0.2, + "reason": "does not build as shipped → 1 (overrides Δ bands)", + "citations": "smoke_test.py: call_option_jit raises NameError ('err' undefined) => does not build. equivalence_x64_False.json worst=1.02e-2; equivalence_x64_True.json worst~1e-11" + }, + { + "dim": "readability", + "title": "Readability & pedagogical clarity", + "kind": "quantitative", + "weight": 0.25, + "score": 2, + "weighted": 0.5, + "reason": "Δprereq=+5→2, docstrings=0.75→4; worse-of-two → 2", + "citations": "results/static_metrics.json: prereq 8->13 (+5), docstring_coverage 0.86->0.75" + }, + { + "dim": "efficiency", + "title": "Computational efficiency (as used)", + "kind": "quantitative", + "weight": 0.15, + "score": 2, + "weighted": 0.3, + "reason": "as-used speedup 0.17× < 0.8 (slower) but correct/fixable → 2 [single-run measurement; the v2 standard is a median of ≥3 fresh-process runs — see as_used_runs in the evidence template]", + "citations": "as_used_total.py (bug-patched to run): numpy 0.18s / jax 1.05s = 0.17x; scaling.json: JAX only wins at n>=250, lecture uses n=5,25. Single-pass v1 measurement (v2 standard: median of >=3 fresh-process runs)" + }, + { + "dim": "logic_design", + "title": "Logic & design", + "kind": "structural", + "weight": 0.15, + "score": 3, + "weighted": 0.45, + "reason": "4/4 criteria met [pure_no_order_dependence, no_global_state, good_algorithmic_choices, fixes_prior_bugs] → 1+4=5 (capped at 3: introduces a correctness bug)", + "citations": { + "pure_no_order_dependence": "immutable NamedTuples + factory functions replace the mutable class", + "no_global_state": "create_ap_model uses a local n; original __init__ referenced module-global n", + "good_algorithmic_choices": "M = P*G**(-gamma) vectorised; precomputes G", + "fixes_prior_bugs": "removes global-n reliance; extracts pure test_stability", + "introduces_correctness_bug": "TRUE (override->cap 3): stray err.throw() in call_option (undefined err) is a build-breaking defect" + } + }, + { + "dim": "style_idiom", + "title": "Coding style & idiom", + "kind": "structural", + "weight": 0.1, + "score": 4, + "weighted": 0.4, + "reason": "3/4 criteria met [vectorised_where_natural, correct_control_flow_primitive, no_anti_idiomatic_constructs] → 1+3=4", + "citations": { + "vectorised_where_natural": "TRUE: M=P*G**(-gamma)", + "correct_control_flow_primitive": "TRUE: lax.while_loop (infinite option) and fori_loop (finite) are the right primitives for genuinely iterative solvers", + "no_anti_idiomatic_constructs": "TRUE: checkify is the idiomatic jit-safe runtime check", + "clean_call_sites_and_naming": "FALSE: every call site carries (err,val) unpack + err.throw() ceremony" + } + }, + { + "dim": "ergonomics", + "title": "API ergonomics & reusability", + "kind": "quantitative", + "weight": 0.1, + "score": 3, + "weighted": 0.3, + "reason": "3 statement(s) + fragile protocol → 3", + "citations": "results/static_metrics.json: `err, v = f_jit(ap); err.throw()`; checkify (err,val) contract is easy to misuse (the shipped bug proves it)" + }, + { + "dim": "maintainability", + "title": "Maintainability & robustness", + "kind": "structural", + "weight": 0.05, + "score": 2, + "weighted": 0.1, + "reason": "1/4 criteria met [pure_unit_testable] → 1+1=2", + "citations": { + "pure_unit_testable": "pure NamedTuples + functions", + "dtype_precision_safe": "FALSE: float32 default, no jax_enable_x64", + "no_footgun_for_editors": "FALSE: checkify (err,val) return-contract already produced the shipped NameError", + "robust_no_brittle_conditions": "FALSE: exercise spectral radius 1.0618 vs 1/beta 1.0638 (margin 0.002) is fragile under float32" + } + } + ], + "_note": "Scores are computed by scripts/scoring/rubric.py from markov_asset/evidence.json; do not edit by hand." +} \ No newline at end of file diff --git a/benchmark/references/examples/markov_asset/results/static_metrics.json b/benchmark/references/examples/markov_asset/results/static_metrics.json new file mode 100644 index 0000000..75cfca2 --- /dev/null +++ b/benchmark/references/examples/markov_asset/results/static_metrics.json @@ -0,0 +1,45 @@ +{ + "old": { + "code_lines": 74, + "n_defs": 7, + "docstring_coverage": 0.86, + "max_nesting_depth": 2, + "explicit_loops": 2, + "n_prerequisite_concepts": 8, + "prerequisite_concepts": [ + "Python class / OOP", + "__init__ constructor", + "instance state (self.)", + "NumPy arrays", + "matrix @ / solve", + "np.linalg eigvals/solve", + "Python while/for loops", + "raise/except for errors" + ], + "statements_for_one_result": 2 + }, + "new": { + "code_lines": 101, + "n_defs": 12, + "docstring_coverage": 0.75, + "max_nesting_depth": 1, + "explicit_loops": 0, + "n_prerequisite_concepts": 13, + "prerequisite_concepts": [ + "NamedTuple (2 of them)", + "typing annotations", + "factory functions", + "jnp vs np", + "jax.jit & tracing", + "float32 default / x64 flag", + "jax.experimental.checkify", + "checkify.check contract", + "checkified call returns (err, val) tuple", + "err.throw()", + "jax.lax.while_loop (cond/body/carry)", + "jax.lax.fori_loop", + "functional array update .at[].set()" + ], + "statements_for_one_result": 3 + } +} \ No newline at end of file diff --git a/benchmark/references/examples/markov_asset/scripts/as_used_total.py b/benchmark/references/examples/markov_asset/scripts/as_used_total.py new file mode 100644 index 0000000..a745e18 --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/as_used_total.py @@ -0,0 +1,91 @@ +""" +"As-used" total latency for the WHOLE markov_asset lecture asset-pricing code. + +Replays the sequence of solver calls the lecture cells make, once, in a fresh +interpreter (so JAX compiles + checkify overhead count). NOTE: the shipped NEW +`call_option` crashes (NameError `err`); to obtain a runnable end-to-end timing +we substitute a bug-patched call_option for the JAX side, and record that the +*shipped* code would not complete at all. + +Run: python as_used_total.py numpy | python as_used_total.py jax +""" +import json +import sys +import time +import numpy as np + +mode = sys.argv[1] if len(sys.argv) > 1 else "numpy" + + +def exercise_arrays(): + n = 5 + P = np.full((n, n), 0.0125) + P[range(n), range(n)] += 1 - P.sum(1) + s = np.array([0.95, 0.975, 1.0, 1.025, 1.05]) + return P, s + + +if mode == "numpy": + import quantecon as qe + import model_old as old + P5, s5 = exercise_arrays() + t0 = time.perf_counter() + # tree_price gamma sweep (default model, 5 values) + for γ in [1.2, 1.4, 1.6, 1.8, 2.0]: + old.tree_price(old.AssetPriceModel(γ=γ)) + # consol + call, beta=0.9 model + ap9 = old.AssetPriceModel(β=0.9) + old.consol_price(ap9, 1.0) + old.call_option(ap9, 1.0, 40.0) + # exercise model: tree, consol, call, finite (k=5,25) + apm = old.AssetPriceModel(β=0.94, mc=qe.MarkovChain(P5, state_values=s5), + γ=2.0, g=lambda x: x) + old.tree_price(apm); old.consol_price(apm, 1.0) + old.call_option(apm, 1.0, 150.0) + for k in (5, 25): + old.finite_horizon_call_option(apm, 1.0, 150.0, k) + print(json.dumps({"mode": "numpy", "total_s": time.perf_counter() - t0})) + +else: + import jax + import jax.numpy as jnp + import model_new as new + + # bug-patched call option so the JAX pipeline can complete end-to-end + def call_fixed(ap, ζ, p_s, ϵ=1e-7): + β, γ, P, G = ap.β, ap.γ, ap.mc.P, ap.G + M = P * G ** (- γ) + new.test_stability(M, β) + p = new.consol_price(ap, ζ) + n = M.shape[0] + + def step(st): + w, _ = st + wn = jnp.maximum(β * M @ w, p - p_s) + return (wn, jnp.amax(jnp.abs(w - wn))) + + def cond(st): + _, e = st + return e > ϵ + fw, _ = jax.lax.while_loop(cond, step, (jnp.zeros(n), ϵ + 1)) + return fw + call_fixed_jit = jax.jit(new.checkify.checkify(call_fixed)) + + P5, s5 = exercise_arrays() + t0 = time.perf_counter() + for γ in [1.2, 1.4, 1.6, 1.8, 2.0]: + ap = new.create_customized_ap_model(new.create_ap_model().mc, γ=γ) + e, v = new.tree_price_jit(ap); e.throw() + ap9 = new.create_ap_model(β=0.9) + e, p = new.consol_price_jit(ap9, 1.0); e.throw() + e, w = call_fixed_jit(ap9, 1.0, 40.0); e.throw() + mc_n = new.MarkovChain(P=jnp.array(P5), state_values=jnp.array(s5)) + apm = new.create_customized_ap_model(mc=mc_n, g=lambda x: x, β=0.94, γ=2.0) + e, v = new.tree_price_jit(apm); e.throw() + e, p = new.consol_price_jit(apm, 1.0); e.throw() + e, w = call_fixed_jit(apm, 1.0, 150.0); e.throw() + for k in (5, 25): + e, w = new.finite_call_option_jit(apm, 1.0, 150.0, k); e.throw() + jax.block_until_ready(w) + print(json.dumps({"mode": "jax_patched", "total_s": time.perf_counter() - t0, + "note": "shipped call_option crashes; patched here to time"})) diff --git a/benchmark/references/examples/markov_asset/scripts/benchmark.py b/benchmark/references/examples/markov_asset/scripts/benchmark.py new file mode 100644 index 0000000..95d0582 --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/benchmark.py @@ -0,0 +1,83 @@ +""" +Performance benchmark: OLD (NumPy) vs NEW (JAX) markov_asset. + +Two parts: + 1. as-used total -- replay the lecture's asset-pricing calls once in a fresh + process (run via `as_used_total.py`; this file focuses on scaling). + 2. scaling -- tree_price + consol_price warm runtime as the state space n + grows, to find where JAX's compiled O(n^3) solve/eigvals overtakes NumPy. + +The core ops (eigvals, dense solve) are O(n^3) LAPACK either way; the lecture +uses n=5 and n=25. Uses the bug-patched call_option is not needed here (we time +tree/consol only). + +Output: results/scaling.json + stdout. +""" +import json +import os +import time +import statistics +import numpy as np +import jax +import jax.numpy as jnp + +import model_old as old +import model_new as new + +RESULTS = os.path.join(os.path.dirname(__file__), "..", "results") +os.makedirs(RESULTS, exist_ok=True) + + +def make(n, seed=0): + rng = np.random.default_rng(seed) + P = rng.random((n, n)); P /= P.sum(1, keepdims=True) + sv = np.linspace(-0.1, 0.1, n) # small so spectral radius stays < 1/β + return P, sv + + +def med(fn, repeat, number=1): + xs = [] + for _ in range(repeat): + t0 = time.perf_counter() + for _ in range(number): + r = fn() + xs.append((time.perf_counter() - t0) / number) + return statistics.median(xs) + + +def main(): + rows = [] + for n in [5, 25, 50, 100, 250, 500, 1000]: + P, sv = make(n) + # OLD + import quantecon as qe + mc = qe.MarkovChain(P, state_values=sv) + ap_o = old.AssetPriceModel(β=0.96, mc=mc, γ=2.0, g=np.exp) + # NEW + mc_n = new.MarkovChain(P=jnp.asarray(P), state_values=jnp.asarray(sv)) + ap_n = new.create_customized_ap_model(mc=mc_n, g=jnp.exp, β=0.96, γ=2.0) + + def old_call(): + old.tree_price(ap_o); old.consol_price(ap_o, 1.0) + + def new_call(): + e, v = new.tree_price_jit(ap_n) + e2, p = new.consol_price_jit(ap_n, 1.0) + jax.block_until_ready((v, p)) + + new_call() # warm + rep = 7 if n <= 250 else 5 + t_o = med(old_call, rep) + t_n = med(new_call, rep) + rows.append({"n": n, "numpy_s": t_o, "jax_warm_s": t_n, + "speedup_warm": t_o / t_n}) + print(f" n={n:5d} numpy={t_o*1e3:9.3f} ms jax_warm={t_n*1e3:9.3f} ms" + f" speedup={t_o/t_n:6.2f}x") + + with open(os.path.join(RESULTS, "scaling.json"), "w") as f: + json.dump({"scaling": rows, "device": str(jax.devices()[0]), + "jax_x64": jax.config.read("jax_enable_x64")}, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/benchmark/references/examples/markov_asset/scripts/check_equivalence.py b/benchmark/references/examples/markov_asset/scripts/check_equivalence.py new file mode 100644 index 0000000..0761a09 --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/check_equivalence.py @@ -0,0 +1,149 @@ +""" +Numerical-equivalence check: OLD (NumPy) vs NEW (JAX) markov_asset code, +using the SAME model/parameter combinations the lecture actually runs: + + * tree_price -- default model (β=0.96), risk-aversion sweep γ∈{1.2..2.0} + * consol_price -- β=0.9 model, ζ=1.0 + * call_option -- β=0.9 model, ζ=1.0, strike=40 (SHIPPED VERSION CRASHES: + NameError `err`; we also compare a bug-patched copy) + * exercise model (n=5 custom chain, β=0.94, γ=2, g=identity): + tree_price, consol_price, finite_call_option (k=5,25), + call_option (patched) + +Run with JAX_ENABLE_X64=1 to separate genuine logic differences from the +float32-vs-float64 gap (the lecture ships float32). + +Output: results/equivalence_x64_.json + stdout. +""" +import json +import os +import numpy as np +import jax +import jax.numpy as jnp +import quantecon as qe + +import model_old as old +import model_new as new + +RESULTS = os.path.join(os.path.dirname(__file__), "..", "results") +os.makedirs(RESULTS, exist_ok=True) +ATOL, RTOL = 1e-5, 1e-4 + + +def new_call_option_fixed(ap, ζ, p_s, ϵ=1e-7): + "NEW call_option with the stray err.throw() removed." + β, γ, P, G = ap.β, ap.γ, ap.mc.P, ap.G + M = P * G ** (- γ) + new.test_stability(M, β) + p = new.consol_price(ap, ζ) + n = M.shape[0] + w = jnp.zeros(n) + + def step(state): + w, _ = state + w_new = jnp.maximum(β * M @ w, p - p_s) + return (w_new, jnp.amax(jnp.abs(w - w_new))) + + def cond(state): + _, e = state + return e > ϵ + final_w, _ = jax.lax.while_loop(cond, step, (w, ϵ + 1)) + return final_w + + +new_call_fixed_jit = jax.jit(new.checkify.checkify(new_call_option_fixed)) + + +def cmp(a, b): + a, b = np.asarray(a), np.asarray(b) + try: + return {"match": bool(np.allclose(a, b, atol=ATOL, rtol=RTOL)), + "max_abs_err": float(np.max(np.abs(a - b)))} + except Exception: + return {"match": False, "max_abs_err": float("nan")} + + +def exercise_models(): + n = 5 + P = np.full((n, n), 0.0125) + P[range(n), range(n)] += 1 - P.sum(1) + s = np.array([0.95, 0.975, 1.0, 1.025, 1.05]) + ap_o = old.AssetPriceModel(β=0.94, mc=qe.MarkovChain(P, state_values=s), + γ=2.0, g=lambda x: x) + mc_n = new.MarkovChain(P=jnp.array(P), state_values=jnp.array(s)) + ap_n = new.create_customized_ap_model(mc=mc_n, g=lambda x: x, β=0.94, γ=2.0) + return ap_o, ap_n + + +def main(): + r = {} + + # A. tree_price, default model, gamma sweep + tree = {} + for γ in [1.2, 1.4, 1.6, 1.8, 2.0]: + v_o = old.tree_price(old.AssetPriceModel(γ=γ)) + ap_n = new.create_customized_ap_model(new.create_ap_model().mc, γ=γ) + err, v_n = new.tree_price_jit(ap_n); err.throw() + tree[f"γ={γ}"] = cmp(v_o, v_n) + r["tree_price_default_sweep"] = tree + + # B. consol_price, β=0.9 model + ap_o9 = old.AssetPriceModel(β=0.9) + ap_n9 = new.create_ap_model(β=0.9) + err, p_n = new.consol_price_jit(ap_n9, 1.0); err.throw() + r["consol_price_beta0.9"] = cmp(old.consol_price(ap_o9, 1.0), p_n) + + # C. call_option, β=0.9 model (shipped crashes -> compare patched) + try: + new.call_option_jit(ap_n9, 1.0, 40.0) + shipped = {"runs": True, "error": None} + except Exception as e: + shipped = {"runs": False, "error": f"{type(e).__name__}: {e}"} + err, w_fix = new_call_fixed_jit(ap_n9, 1.0, 40.0); err.throw() + r["call_option_beta0.9"] = { + "shipped": shipped, + "patched_vs_numpy": cmp(old.call_option(ap_o9, 1.0, 40.0), w_fix)} + + # D/E. exercise model (n=5, β=0.94) + ap_o, ap_n = exercise_models() + err, v_n = new.tree_price_jit(ap_n); err.throw() + err, p_n = new.consol_price_jit(ap_n, 1.0); err.throw() + ex = {"tree_price": cmp(old.tree_price(ap_o), v_n), + "consol_price": cmp(old.consol_price(ap_o, 1.0), p_n)} + for k in (5, 25): + err, w_n = new.finite_call_option_jit(ap_n, 1.0, 150.0, k); err.throw() + ex[f"finite_call_k{k}"] = cmp( + old.finite_horizon_call_option(ap_o, 1.0, 150.0, k), w_n) + err, w_fix = new_call_fixed_jit(ap_n, 1.0, 150.0); err.throw() + ex["call_option_patched"] = cmp(old.call_option(ap_o, 1.0, 150.0), w_fix) + r["exercise_model"] = ex + + x64 = jax.config.read("jax_enable_x64") + r["_meta"] = {"jax_enable_x64": x64, "atol": ATOL, "rtol": RTOL} + + # print + print(f"jax_enable_x64 = {x64}") + worst = 0.0 + def show(name, d): + nonlocal worst + worst = max(worst, d["max_abs_err"]) + print(f" {name:34s} match={d['match']!s:5s} max|Δ|={d['max_abs_err']:.2e}") + print(" tree_price (default sweep):") + for k, d in r["tree_price_default_sweep"].items(): + show(" " + k, d) + show("consol_price (β=0.9)", r["consol_price_beta0.9"]) + print(f" call_option (β=0.9) SHIPPED runs = {shipped['runs']}" + + ("" if shipped["runs"] else f" <-- {shipped['error']}")) + show("call_option (β=0.9) patched", r["call_option_beta0.9"]["patched_vs_numpy"]) + print(" exercise model (n=5, β=0.94):") + for k in ("tree_price", "consol_price", "finite_call_k5", + "finite_call_k25", "call_option_patched"): + show(" " + k, r["exercise_model"][k]) + print(f"\n worst max|Δ| across working assets = {worst:.2e}") + + with open(os.path.join(RESULTS, f"equivalence_x64_{x64}.json"), "w") as f: + json.dump(r, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/benchmark/references/examples/markov_asset/scripts/model_new.py b/benchmark/references/examples/markov_asset/scripts/model_new.py new file mode 100644 index 0000000..2ef194d --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/model_new.py @@ -0,0 +1,134 @@ +""" +Verbatim extraction of the NEW (branch `update_markov_asset`) JAX implementation +of the Markov asset-pricing code from `lectures/markov_asset.md`. + +Copied AS-IS from the lecture so benchmarks and equivalence checks exercise the +exact code under evaluation -- INCLUDING the `err.throw()` line inside +`call_option`, which references a name that is never bound in that scope. We +keep it verbatim because whether it runs is part of what we are evaluating. +""" +import jax +import jax.numpy as jnp +import quantecon as qe +from jax.numpy.linalg import eigvals, solve +from jax.experimental import checkify +from typing import NamedTuple + + +class MarkovChain(NamedTuple): + "Stores the primitives of a Markov chain." + P: jax.Array + state_values: jax.Array + + +class AssetPriceModel(NamedTuple): + "Stores the primitives of the asset pricing model." + mc: MarkovChain + G: jax.Array + β: float + γ: float + + +def create_ap_model(g=jnp.exp, β=0.96, γ=2.0): + "Create an AssetPriceModel class using standard Markov chain." + n, ρ, σ = 25, 0.9, 0.02 + qe_mc = qe.tauchen(n, ρ, σ) + P = jnp.array(qe_mc.P) + state_values = jnp.array(qe_mc.state_values) + G = g(state_values) + mc = MarkovChain(P=P, state_values=state_values) + return AssetPriceModel(mc=mc, G=G, β=β, γ=γ) + + +def create_customized_ap_model(mc: MarkovChain, g=jnp.exp, β=0.96, γ=2.0): + "Create an AssetPriceModel class using a customized Markov chain." + G = g(mc.state_values) + return AssetPriceModel(mc=mc, G=G, β=β, γ=γ) + + +def test_stability(Q, β): + "Stability test for a given matrix Q." + sr = jnp.max(jnp.abs(eigvals(Q))) + checkify.check( + sr < 1 / β, + "Spectral radius condition failed with radius = {sr}", sr=sr + ) + return sr + + +def tree_price(ap): + "Computes the price-dividend ratio of the Lucas tree." + β, γ, P, G = ap.β, ap.γ, ap.mc.P, ap.G + J = P * G ** (1 - γ) + test_stability(J, β) + n = J.shape[0] + I = jnp.identity(n) + Ones = jnp.ones(n) + v = solve(I - β * J, β * J @ Ones) + return v + + +tree_price_jit = jax.jit(checkify.checkify(tree_price)) + + +def consol_price(ap, ζ): + "Computes price of a consol bond with payoff ζ." + β, γ, P, G = ap.β, ap.γ, ap.mc.P, ap.G + M = P * G ** (- γ) + test_stability(M, β) + n = M.shape[0] + I = jnp.identity(n) + Ones = jnp.ones(n) + p = solve(I - β * M, β * ζ * M @ Ones) + return p + + +consol_price_jit = jax.jit(checkify.checkify(consol_price)) + + +def call_option(ap, ζ, p_s, ϵ=1e-7): + "Computes price of a call option on a consol bond." + β, γ, P, G = ap.β, ap.γ, ap.mc.P, ap.G + M = P * G ** (- γ) + test_stability(M, β) + # Compute option price + p = consol_price(ap, ζ) + err.throw() # <-- VERBATIM from lecture: `err` undefined + n = M.shape[0] + w = jnp.zeros(n) + error = ϵ + 1 + + def step(state): + w, _ = state + w_new = jnp.maximum(β * M @ w, p - p_s) + error_new = jnp.amax(jnp.abs(w - w_new)) + return (w_new, error_new) + + def cond(state): + _, error = state + return error > ϵ + + final_w, _ = jax.lax.while_loop(cond, step, (w, error)) + return final_w + + +call_option_jit = jax.jit(checkify.checkify(call_option)) + + +def finite_call_option(ap, ζ, p_s, k): + "Computes k period option value." + β, γ, P, G = ap.β, ap.γ, ap.mc.P, ap.G + M = P * G ** (- γ) + test_stability(M, β) + p = consol_price(ap, ζ) + n = M.shape[0] + + def step(i, w): + w = jnp.maximum(β * M @ w, p - p_s) + return w + + w = jax.lax.fori_loop(0, k, step, jnp.zeros(n)) + return w + + +finite_call_option_jit = jax.jit(checkify.checkify(finite_call_option)) diff --git a/benchmark/references/examples/markov_asset/scripts/model_old.py b/benchmark/references/examples/markov_asset/scripts/model_old.py new file mode 100644 index 0000000..8a8f828 --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/model_old.py @@ -0,0 +1,90 @@ +""" +Verbatim-faithful extraction of the ORIGINAL (main branch) NumPy implementation +of the Markov asset-pricing code from `lectures/markov_asset.md`. + +Copied from `main`. The only adaptation: the original `AssetPriceModel.__init__` +references a *module-level* `n` when building the default Markov chain +(`qe.tauchen(n, ...)`); in the lecture `n=25` is defined in an earlier cell, so +we define it here too. (This global reliance is itself an evaluation finding, +mirroring the ge_arrow case.) +""" +import numpy as np +import quantecon as qe +from numpy.linalg import eigvals, solve + +n = 25 # module-level default state-space size (as in the lecture) + + +class AssetPriceModel: + """ + A class that stores the primitives of the asset pricing model. + """ + def __init__(self, β=0.96, mc=None, γ=2.0, g=np.exp): + self.β, self.γ = β, γ + self.g = g + + # A default process for the Markov chain + if mc is None: + self.ρ = 0.9 + self.σ = 0.02 + self.mc = qe.tauchen(n, self.ρ, self.σ) + else: + self.mc = mc + + self.n = self.mc.P.shape[0] + + def test_stability(self, Q): + "Stability test for a given matrix Q." + sr = np.max(np.abs(eigvals(Q))) + if not sr < 1 / self.β: + msg = f"Spectral radius condition failed with radius = {sr}" + raise ValueError(msg) + + +def tree_price(ap): + "Computes the price-dividend ratio of the Lucas tree." + β, γ, P, y = ap.β, ap.γ, ap.mc.P, ap.mc.state_values + J = P * ap.g(y) ** (1 - γ) + ap.test_stability(J) + I = np.identity(ap.n) + Ones = np.ones(ap.n) + v = solve(I - β * J, β * J @ Ones) + return v + + +def consol_price(ap, ζ): + "Computes price of a consol bond with payoff ζ." + β, γ, P, y = ap.β, ap.γ, ap.mc.P, ap.mc.state_values + M = P * ap.g(y) ** (- γ) + ap.test_stability(M) + I = np.identity(ap.n) + Ones = np.ones(ap.n) + p = solve(I - β * M, β * ζ * M @ Ones) + return p + + +def call_option(ap, ζ, p_s, ϵ=1e-7): + "Computes price of a call option on a consol bond." + β, γ, P, y = ap.β, ap.γ, ap.mc.P, ap.mc.state_values + M = P * ap.g(y) ** (- γ) + ap.test_stability(M) + p = consol_price(ap, ζ) + w = np.zeros(ap.n) + error = ϵ + 1 + while error > ϵ: + w_new = np.maximum(β * M @ w, p - p_s) + error = np.amax(np.abs(w - w_new)) + w = w_new + return w + + +def finite_horizon_call_option(ap, ζ, p_s, k): + "Computes k period option value." + β, γ, P, y = ap.β, ap.γ, ap.mc.P, ap.mc.state_values + M = P * ap.g(y) ** (- γ) + ap.test_stability(M) + p = consol_price(ap, ζ) + w = np.zeros(ap.n) + for i in range(k): + w = np.maximum(β * M @ w, p - p_s) + return w diff --git a/benchmark/references/examples/markov_asset/scripts/run_all.py b/benchmark/references/examples/markov_asset/scripts/run_all.py new file mode 100644 index 0000000..633e8e7 --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/run_all.py @@ -0,0 +1,111 @@ +""" +Run the whole markov_asset evaluation pipeline; regenerate ../results/. +Requires the `quantecon` conda env (jax 0.4.x, numpy 2.x, quantecon). +""" +import json +import subprocess +import sys +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +PY = sys.executable +K_AS_USED = 3 # fresh-process repeats per as-used side (v2: median, not a single pass) + + +def median(xs): + s = sorted(xs) + n = len(s) + return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2.0 + +# (title, command, results-file to aggregate the script's JSON line into or None) +STEPS = [ + ("Smoke test (does each version run?)", "smoke_test.py", None), + ("Numerical equivalence (float32, as shipped)", "check_equivalence.py", None), + ("Static metrics", "static_metrics.py", None), + ("Scaling benchmark", "benchmark.py", None), + ("As-used total (numpy)", "as_used_total.py numpy", "as_used.json"), + ("As-used total (jax, bug-patched)", "as_used_total.py jax", "as_used.json"), +] + +collected = {} +failed = [] +for title, cmd, agg in STEPS: + print("\n" + "=" * 70 + f"\n== {title}\n" + "=" * 70) + parts = cmd.split() + argv = [PY, os.path.join(HERE, parts[0])] + parts[1:] + if agg is None: + p = subprocess.run(argv, cwd=HERE, check=False) + if p.returncode: + failed.append(title) + continue + # Persist the script's JSON line (these run as fresh processes and only + # print their result; the headline as-used metric must not live on the + # console alone). As-used steps repeat K_AS_USED times so the headline + # metric is a median of fresh-process runs, never a single pass. + reps = K_AS_USED if agg == "as_used.json" else 1 + for _ in range(reps): + p = subprocess.run(argv, cwd=HERE, check=False, capture_output=True, + text=True) + sys.stdout.write(p.stdout) + if p.stderr: + sys.stderr.write(p.stderr) + if p.returncode: + failed.append(title) + break + for line in reversed(p.stdout.strip().splitlines()): + try: + rec = json.loads(line) + except ValueError: + continue + if not isinstance(rec, dict): + continue # a stray scalar/list line is not a result record + bucket = collected.setdefault(agg, {}) + key = rec.get("mode", "?") + if agg == "as_used.json": + bucket.setdefault(key, dict(rec, runs=[]))["runs"].append( + rec["total_s"]) + else: + if key in bucket: + print(f"WARNING: duplicate mode {key!r} for {agg}", + file=sys.stderr) + bucket[key] = rec + break + +RES = os.path.join(os.path.dirname(HERE), "results") +os.makedirs(RES, exist_ok=True) +for fname, recs in collected.items(): + modes = list(recs) + if fname == "as_used.json" and "numpy" in recs and len(modes) == 2: + other = next(m for m in modes if m != "numpy") + a, b = recs["numpy"], recs[other] + if a.get("runs") and b.get("runs"): + a["total_s"] = median(a["runs"]) + b["total_s"] = median(b["runs"]) + recs["as_used_speedup"] = a["total_s"] / b["total_s"] + recs["as_used_speedup_runs"] = [x / y + for x, y in zip(a["runs"], b["runs"])] + recs["baseline_as_used_seconds"] = a["total_s"] + with open(os.path.join(RES, fname), "w", encoding="utf-8") as f: + json.dump(recs, f, indent=2) + print(f"wrote results/{fname}") + +LEC_DIR = os.path.dirname(HERE) # this example's folder +# Shared engine location: the installed plugin root when the skill drives an +# evaluation from a user workspace; falls back to this repo's layout. +PLUGIN = (os.environ.get("CLAUDE_PLUGIN_ROOT") + or os.path.dirname(os.path.dirname(os.path.dirname(LEC_DIR)))) + +# Provenance stamp (shared: scripts/scoring/env_stamp.py — the seed of the +# QuantEcon/meta#335 result + environment-descriptor schema). Failed step +# titles are recorded so a partial run cannot claim full provenance. +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "env_stamp.py"), + LEC_DIR] + failed, check=False) + +# Scoring is shared across lectures: fill ../evidence.json from the results +# above, then apply the common rubric (scripts/scoring/rubric.py) via the engine. +print("\n" + "=" * 70 + "\n== Scorecard (shared rubric)\n" + "=" * 70) +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "score.py"), + LEC_DIR], check=False) + +print("\nNote: to also get the x64 equivalence numbers, run:") +print(" JAX_ENABLE_X64=1 python check_equivalence.py") diff --git a/benchmark/references/examples/markov_asset/scripts/smoke_test.py b/benchmark/references/examples/markov_asset/scripts/smoke_test.py new file mode 100644 index 0000000..20c4bc6 --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/smoke_test.py @@ -0,0 +1,61 @@ +""" +Smoke test: does each version's four priced assets actually run, exactly as the +lecture calls them? Prints PASS/ERROR per function. +""" +import traceback +import jax +import jax.numpy as jnp +import numpy as np + +import model_old as old +import model_new as new + + +def try_old(): + print("=== OLD (numpy) ===") + ap = old.AssetPriceModel(β=0.9) + for name, fn in [ + ("tree_price", lambda: old.tree_price(old.AssetPriceModel())), + ("consol_price", lambda: old.consol_price(ap, 1.0)), + ("call_option", lambda: old.call_option(ap, 1.0, 40.0)), + ("finite_horizon_call_option", + lambda: old.finite_horizon_call_option(ap, 1.0, 40.0, 5)), + ]: + try: + fn(); print(f" PASS {name}") + except Exception as e: + print(f" ERROR {name}: {type(e).__name__}: {e}") + + +def try_new(): + print("=== NEW (jax), called exactly as the lecture does ===") + ap = new.create_ap_model(β=0.9) + # tree_price + try: + err, v = new.tree_price_jit(new.create_ap_model()); err.throw() + print(" PASS tree_price_jit") + except Exception as e: + print(f" ERROR tree_price_jit: {type(e).__name__}: {e}") + # consol_price + try: + err, p = new.consol_price_jit(ap, 1.0); err.throw() + print(" PASS consol_price_jit") + except Exception as e: + print(f" ERROR consol_price_jit: {type(e).__name__}: {e}") + # call_option + try: + err, w = new.call_option_jit(ap, 1.0, 40.0); err.throw() + print(" PASS call_option_jit") + except Exception as e: + print(f" ERROR call_option_jit: {type(e).__name__}: {e}") + # finite_call_option + try: + err, w = new.finite_call_option_jit(ap, 1.0, 40.0, 5); err.throw() + print(" PASS finite_call_option_jit") + except Exception as e: + print(f" ERROR finite_call_option_jit: {type(e).__name__}: {e}") + + +if __name__ == "__main__": + try_old() + try_new() diff --git a/benchmark/references/examples/markov_asset/scripts/static_metrics.py b/benchmark/references/examples/markov_asset/scripts/static_metrics.py new file mode 100644 index 0000000..c92287d --- /dev/null +++ b/benchmark/references/examples/markov_asset/scripts/static_metrics.py @@ -0,0 +1,94 @@ +""" +Static code metrics for the OLD and NEW markov_asset implementations +(same methodology as the ge_arrow evaluation). + +Output: results/static_metrics.json + stdout. +""" +import ast +import json +import os +import re + +HERE = os.path.dirname(__file__) +RESULTS = os.path.join(HERE, "..", "results") +os.makedirs(RESULTS, exist_ok=True) + +PREREQS = { + "old": ["Python class / OOP", "__init__ constructor", "instance state (self.)", + "NumPy arrays", "matrix @ / solve", "np.linalg eigvals/solve", + "Python while/for loops", "raise/except for errors"], + "new": ["NamedTuple (2 of them)", "typing annotations", "factory functions", + "jnp vs np", "jax.jit & tracing", "float32 default / x64 flag", + "jax.experimental.checkify", "checkify.check contract", + "checkified call returns (err, val) tuple", "err.throw()", + "jax.lax.while_loop (cond/body/carry)", "jax.lax.fori_loop", + "functional array update .at[].set()"], +} + + +def code_lines(src): + return sum(1 for ln in src.splitlines() + if ln.strip() and not ln.strip().startswith("#")) + + +def max_nesting(tree): + best = 0 + def walk(node, d): + nonlocal best + for c in ast.iter_child_nodes(node): + if isinstance(c, (ast.FunctionDef, ast.For, ast.While, ast.If, ast.With)): + best = max(best, d + 1); walk(c, d + 1) + else: + walk(c, d) + for n in tree.body: + walk(n, 0) + return best + + +def defs_docs(tree): + nd = ndoc = 0 + for n in ast.walk(tree): + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + nd += 1 + if ast.get_docstring(n): + ndoc += 1 + return nd, ndoc + + +def loops(tree): + return sum(isinstance(n, (ast.For, ast.While)) for n in ast.walk(tree)) + + +def analyze(path, key): + src = open(path, encoding="utf-8").read() + t = ast.parse(src) + nd, ndoc = defs_docs(t) + return {"code_lines": code_lines(src), "n_defs": nd, + "docstring_coverage": round(ndoc / nd, 2) if nd else 0, + "max_nesting_depth": max_nesting(t), "explicit_loops": loops(t), + "n_prerequisite_concepts": len(PREREQS[key]), + "prerequisite_concepts": PREREQS[key]} + + +def main(): + old = analyze(os.path.join(HERE, "model_old.py"), "old") + new = analyze(os.path.join(HERE, "model_new.py"), "new") + # calls to obtain one priced asset (e.g. call option) + handle result + # (key name matches EVIDENCE_TEMPLATE.json / the ge_arrow template) + # OLD: build model, call function -> 2 + # NEW: build model, call *_jit -> (err,val), err.throw() -> 3 + old["statements_for_one_result"] = 2 + new["statements_for_one_result"] = 3 + out = {"old": old, "new": new} + keys = ["code_lines", "n_defs", "docstring_coverage", "max_nesting_depth", + "explicit_loops", "n_prerequisite_concepts", "statements_for_one_result"] + print(f"{'metric':30s} {'OLD':>10s} {'NEW':>10s}") + print("-" * 52) + for k in keys: + print(f"{k:30s} {str(old[k]):>10s} {str(new[k]):>10s}") + with open(os.path.join(RESULTS, "static_metrics.json"), "w") as f: + json.dump(out, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/benchmark/references/fixtures/README.md b/benchmark/references/fixtures/README.md new file mode 100644 index 0000000..0997a33 --- /dev/null +++ b/benchmark/references/fixtures/README.md @@ -0,0 +1,35 @@ +# Rubric fixtures + +Synthetic `evidence.json` files whose only job is to drive rubric code paths that the worked examples in [`../examples/`](../examples/) do not reach. They are **not** evaluations: no lecture, no measurement, no verdict anyone should cite. + +The distinction matters. An example is a record of what was measured about a real PR; a fixture is a test input chosen to make a specific branch execute. Reading a fixture's numbers as evidence about JAX conversion would be a category error, which is why every `source` string in one begins `SYNTHETIC:`. + +Like the examples, each fixture's `results/scorecard.json` is committed and regenerated by CI, so a change in scoring behaviour shows up as a diff rather than as silence. + +| Fixture | Pins | +|---|---| +| [`rubric_v2/`](rubric_v2/) | The five v2 paths the worked examples leave untested (below) | + +## `rubric_v2` + +Both worked examples conclude *no-conversion* on sub-second baselines, take the single-run efficiency fallback, and hand-set the correctness-bug flag. That leaves the newest scoring behaviour — everything added in rubric v2 — running only when someone probes it by hand. This fixture makes each path execute on every CI run: + +1. **`matches_under_x64` caps unconditionally.** Divergent logic with a shipped `max|Δ|` of 1e-12 — agreement at float32 that does not redeem the logic. Correctness scores **1**, not the 5 the delta bands alone would give. +2. **The bug cap is derived, not trusted.** `logic_design` meets 4/4 criteria and leaves `introduces_correctness_bug` FALSE. The cap must still fire from the correctness evidence, pulling 5 → 3 with the *derived* reason string. +3. **`as_used_runs` median path.** Three fresh-process runs rather than the single-run fallback. +4. **Contested-band annotation.** The runs straddle the 1.3× band edge, so the per-run scores disagree and the annotation reports it. +5. **Verdict gate with the ungated total shown.** Polish is deliberately near-perfect, so the ungated 3.60 would band as *net positive*; the correctness gate gives *net regression* and says what it overrode. + +The baseline is deliberately set above the 1 s materiality floor so the **no-conversion** verdict does not fire and mask paths 1–5 — that verdict is already covered by both worked examples. + +The scoring rules themselves are defined once in [`../EVALUATION_FRAMEWORK.md`](../EVALUATION_FRAMEWORK.md) §1–2 and implemented in `../../scripts/scoring/rubric.py`; this file says only which of them the fixture pins. + +## Changing a fixture + +The specific values are the test. If a rubric change makes a fixture's scorecard move, that is the check working — confirm the new behaviour is what you intended, then regenerate and commit: + +```bash +python scripts/scoring/score.py references/fixtures/rubric_v2 +``` + +Adjusting a fixture's inputs to restore an old scorecard defeats the point. If a path stops being reachable at all, delete the fixture and say why in the commit, rather than leaving one that no longer tests anything. diff --git a/benchmark/references/fixtures/rubric_v2/evidence.json b/benchmark/references/fixtures/rubric_v2/evidence.json new file mode 100644 index 0000000..a9ea270 --- /dev/null +++ b/benchmark/references/fixtures/rubric_v2/evidence.json @@ -0,0 +1,77 @@ +{ + "lecture": "rubric_v2", + "branch": "n/a — synthetic fixture, not a real lecture", + "_how": "SYNTHETIC. Every number here is chosen to drive a rubric v2 code path, not measured from a lecture. See README.md in this directory for what each value pins and why. Do not cite these values as evidence about any lecture, and do not 'correct' them toward realism — the specific values are the test.", + + "quantitative": { + "correctness": { + "builds": true, + "max_delta_shipped": 1e-12, + "matches_under_x64": false, + "source": "SYNTHETIC: divergent logic whose shipped float32 output happens to agree to 1e-12 — the 'wrong economics masked by low precision' case. Pins that matches_under_x64 caps on its own, without needing a visible shipped delta." + }, + "readability": { + "delta_prereq_concepts": 0, + "docstring_cov_new": 0.95, + "source": "SYNTHETIC: deliberately excellent, so the verdict is driven by the correctness gate rather than by a low total." + }, + "efficiency": { + "as_used_speedup": 1.28, + "as_used_runs": [1.15, 1.28, 1.42], + "baseline_as_used_seconds": 12.0, + "correct_or_fixable": true, + "source": "SYNTHETIC: three fresh-process runs straddling the 1.3x band edge (1.15->3, 1.28->3, 1.42->4), so the median scores 3 and the CONTESTED BAND annotation fires. baseline 12.0s is deliberately above the 1s materiality floor so no-conversion does NOT mask the band/gate paths under test." + }, + "ergonomics": { + "statements_for_one_result": 1, + "fragile_protocol": false, + "source": "SYNTHETIC: deliberately excellent, as for readability." + } + }, + + "structural": { + "logic_design": { + "criteria": { + "pure_no_order_dependence": true, + "no_global_state": true, + "good_algorithmic_choices": true, + "fixes_prior_bugs": true + }, + "introduces_correctness_bug": false, + "citations": { + "pure_no_order_dependence": "SYNTHETIC: all four criteria met and the bug flag left FALSE on purpose — the derived cap must pull this dimension from 5 down to 3 using the correctness evidence alone.", + "no_global_state": "SYNTHETIC: see above.", + "good_algorithmic_choices": "SYNTHETIC: see above.", + "fixes_prior_bugs": "SYNTHETIC: see above." + } + }, + "style_idiom": { + "criteria": { + "vectorised_where_natural": true, + "correct_control_flow_primitive": true, + "no_anti_idiomatic_constructs": true, + "clean_call_sites_and_naming": true + }, + "citations": { + "vectorised_where_natural": "SYNTHETIC: deliberately perfect polish, so the verdict cannot be explained by weak structural scores.", + "correct_control_flow_primitive": "SYNTHETIC: see above.", + "no_anti_idiomatic_constructs": "SYNTHETIC: see above.", + "clean_call_sites_and_naming": "SYNTHETIC: see above." + } + }, + "maintainability": { + "criteria": { + "pure_unit_testable": true, + "dtype_precision_safe": true, + "no_footgun_for_editors": true, + "robust_no_brittle_conditions": true + }, + "citations": { + "pure_unit_testable": "SYNTHETIC: deliberately perfect polish, as for style_idiom.", + "dtype_precision_safe": "SYNTHETIC: see above.", + "no_footgun_for_editors": "SYNTHETIC: see above.", + "robust_no_brittle_conditions": "SYNTHETIC: see above." + } + } + } +} diff --git a/benchmark/references/fixtures/rubric_v2/results/scorecard.json b/benchmark/references/fixtures/rubric_v2/results/scorecard.json new file mode 100644 index 0000000..5b315da --- /dev/null +++ b/benchmark/references/fixtures/rubric_v2/results/scorecard.json @@ -0,0 +1,112 @@ +{ + "lecture": "rubric_v2", + "branch": "n/a — synthetic fixture, not a real lecture", + "weighted_total_out_of_5": 3.6, + "verdict": "net regression — do not merge as-is [gated: correctness 1 caps the verdict at net regression; the ungated total 3.60 would band as net positive]", + "band_verdict": "net regression — do not merge as-is", + "verdict_gate": "correctness 1 caps the verdict at net regression", + "no_conversion": false, + "sensitivity": { + "stamp": "fragile", + "stamp_note": "", + "perturbations_tested": 29, + "deciding_flips": [ + { + "input": "quantitative.correctness.matches_under_x64", + "from": false, + "to": true, + "total": 4.7, + "outcome": "clear improvement — merge" + } + ], + "perturbations_skipped": [] + }, + "dimensions": [ + { + "dim": "correctness", + "title": "Correctness & numerical fidelity", + "kind": "quantitative", + "weight": 0.2, + "score": 1, + "weighted": 0.2, + "reason": "logic diverges under x64 → wrong economics → 1 (shipped max|Δ|=1.0e-12; agreement as shipped does not redeem divergent logic)", + "citations": "SYNTHETIC: divergent logic whose shipped float32 output happens to agree to 1e-12 — the 'wrong economics masked by low precision' case. Pins that matches_under_x64 caps on its own, without needing a visible shipped delta." + }, + { + "dim": "readability", + "title": "Readability & pedagogical clarity", + "kind": "quantitative", + "weight": 0.25, + "score": 5, + "weighted": 1.25, + "reason": "Δprereq=+0→5, docstrings=0.95→5; worse-of-two → 5", + "citations": "SYNTHETIC: deliberately excellent, so the verdict is driven by the correctness gate rather than by a low total." + }, + { + "dim": "efficiency", + "title": "Computational efficiency (as used)", + "kind": "quantitative", + "weight": 0.15, + "score": 3, + "weighted": 0.45, + "reason": "as-used speedup 1.28× ∈ [0.8,1.3) → 3 (wash) [median of 3 fresh-process runs, spread 1.15–1.42×; CONTESTED BAND: runs alone would score [3, 4]]", + "citations": "SYNTHETIC: three fresh-process runs straddling the 1.3x band edge (1.15->3, 1.28->3, 1.42->4), so the median scores 3 and the CONTESTED BAND annotation fires. baseline 12.0s is deliberately above the 1s materiality floor so no-conversion does NOT mask the band/gate paths under test." + }, + { + "dim": "logic_design", + "title": "Logic & design", + "kind": "structural", + "weight": 0.15, + "score": 3, + "weighted": 0.45, + "reason": "4/4 criteria met [pure_no_order_dependence, no_global_state, good_algorithmic_choices, fixes_prior_bugs] → 1+4=5 (capped at 3: correctness-bug cap derived from correctness evidence: logic diverges under x64)", + "citations": { + "pure_no_order_dependence": "SYNTHETIC: all four criteria met and the bug flag left FALSE on purpose — the derived cap must pull this dimension from 5 down to 3 using the correctness evidence alone.", + "no_global_state": "SYNTHETIC: see above.", + "good_algorithmic_choices": "SYNTHETIC: see above.", + "fixes_prior_bugs": "SYNTHETIC: see above." + } + }, + { + "dim": "style_idiom", + "title": "Coding style & idiom", + "kind": "structural", + "weight": 0.1, + "score": 5, + "weighted": 0.5, + "reason": "4/4 criteria met [vectorised_where_natural, correct_control_flow_primitive, no_anti_idiomatic_constructs, clean_call_sites_and_naming] → 1+4=5", + "citations": { + "vectorised_where_natural": "SYNTHETIC: deliberately perfect polish, so the verdict cannot be explained by weak structural scores.", + "correct_control_flow_primitive": "SYNTHETIC: see above.", + "no_anti_idiomatic_constructs": "SYNTHETIC: see above.", + "clean_call_sites_and_naming": "SYNTHETIC: see above." + } + }, + { + "dim": "ergonomics", + "title": "API ergonomics & reusability", + "kind": "quantitative", + "weight": 0.1, + "score": 5, + "weighted": 0.5, + "reason": "1 statement(s) to obtain one result → 5", + "citations": "SYNTHETIC: deliberately excellent, as for readability." + }, + { + "dim": "maintainability", + "title": "Maintainability & robustness", + "kind": "structural", + "weight": 0.05, + "score": 5, + "weighted": 0.25, + "reason": "4/4 criteria met [pure_unit_testable, dtype_precision_safe, no_footgun_for_editors, robust_no_brittle_conditions] → 1+4=5", + "citations": { + "pure_unit_testable": "SYNTHETIC: deliberately perfect polish, as for style_idiom.", + "dtype_precision_safe": "SYNTHETIC: see above.", + "no_footgun_for_editors": "SYNTHETIC: see above.", + "robust_no_brittle_conditions": "SYNTHETIC: see above." + } + } + ], + "_note": "Scores are computed by scripts/scoring/rubric.py from rubric_v2/evidence.json; do not edit by hand." +} \ No newline at end of file diff --git a/benchmark/scripts/README.md b/benchmark/scripts/README.md index 9a00113..defb300 100644 --- a/benchmark/scripts/README.md +++ b/benchmark/scripts/README.md @@ -1,16 +1,46 @@ # benchmark plugin — scripts -Supporting scripts for `/benchmark:review-acceleration`. +The deterministic core of `/benchmark:review-acceleration`: the shared scoring engine and the efficiency calibration. Developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). -**Pending:** these are being collected from the evaluation work on [QuantEcon/lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717), where they were developed and validated against `ge_arrow.md` (and the `aiyagari.md` Bellman pattern as the HIGH calibration case): +## Layout -- `check_equivalence.py` — diff all published objects between implementations, under default dtype and `jax_enable_x64` -- `static_metrics.py` — prerequisite-concept count, docstring coverage, code size metrics -- `benchmark.py` — scaling curves and crossover-n between implementations -- `cold_start.py` — cold-start / compile-time measurement -- `sweep_bench.py` — parameter-sweep timing (cold and warm) -- `as_used_total.py` — the headline metric: full lecture solver sequence replayed in a fresh process -- `bellman_bench.py` — the aiyagari-pattern calibration benchmark -- `run_all.py` — orchestrator +``` +scoring/ + rubric.py the standard as code: evidence → score, deterministically + score.py engine/CLI: /evidence.json → results/scorecard.json + env_stamp.py provenance stamp: /results/env.json (+ failed steps) + EVIDENCE_TEMPLATE.json the judgement contract a new evaluation fills in +calibration/ + bellman_bench.py shared aiyagari Bellman benchmark — pins the "25× as-used + bellman_bench.json = score 5" efficiency anchor +``` -As they land they will be generalised from `ge_arrow`-specific code to take a lecture/implementation pair as input. +All commands below run from the plugin root (`benchmark/` in this repo). + +The rubric in prose — dimensions, weights, anchors, checklists, verdict bands, worked HIGH/LOW examples — is [`../references/EVALUATION_FRAMEWORK.md`](../references/EVALUATION_FRAMEWORK.md). Two complete worked evaluations (measurement scripts, results, evidence, reports) live in [`../references/examples/`](../references/examples/) and double as the regression baseline the skill must reproduce. + +## How scoring works + +Scores are **never typed by hand** — each is a deterministic function of evidence: + +1. **Measure** — `python references/examples//scripts/run_all.py` runs the per-lecture measurement scripts and writes `results/*.json` plus a provenance stamp (`results/env.json`: Python/platform/library versions and any failed steps — the seed of the QuantEcon/meta#335 shared result schema; generated per-run, not committed). The as-used steps repeat 3× per side in fresh processes; the headline speedup is a **median**, with per-run values kept for the contested-band check. +2. **Record evidence** — fill `/evidence.json` (copy `scoring/EVIDENCE_TEMPLATE.json`): measured numbers into the quantitative slots with their source, and each structural checklist item answered true/false **with a citation to the diff**. +3. **Score** — `python scripts/scoring/score.py references/examples/` applies `rubric.py` and writes `results/scorecard.json`, printing the derivation of every score, the final verdict (after the v2 correctness gates and the no-conversion rule), and the one-flip **sensitivity stamp** (robust/fragile with deciding flips). + +## Evaluating a new lecture + +Per-lecture measurement scripts are **adapted templates, not a fixed harness** — copy an existing example and adapt (this is the step the skill automates): + +```bash +conda activate quantecon # jax 0.4.x, numpy 2.x, quantecon +mkdir -p references/examples//{scripts,results} +cp scripts/scoring/EVIDENCE_TEMPLATE.json references/examples//evidence.json +# drop in model_old.py (from main) and model_new.py (from the PR branch), +# adapt check_equivalence / static_metrics / benchmark / as_used_total from an +# existing example, wire them into run_all.py, then: +python references/examples//scripts/run_all.py +python scripts/scoring/score.py references/examples/ +# write _REPORT.md from the scorecard + evidence +``` + +Benchmarks are CPU-only; timings vary ±~15% run-to-run, so the rubric keys on orders of magnitude, not exact milliseconds. diff --git a/benchmark/scripts/calibration/bellman_bench.json b/benchmark/scripts/calibration/bellman_bench.json new file mode 100644 index 0000000..5a1218f --- /dev/null +++ b/benchmark/scripts/calibration/bellman_bench.json @@ -0,0 +1,17 @@ +{ + "grid": [ + 200, + 7 + ], + "vfi_iters": 397, + "agree_max_abs": 1.0658141036401503e-14, + "single_numpy_s": 2.9554021999938414, + "single_jax_cold_s": 0.1747544000390917, + "single_jax_warm_s": 0.11176240001805127, + "single_speedup_cold": 16.911746996543332, + "single_speedup_warm": 26.443617885053474, + "eq_loop_R": 20, + "eq_numpy_s": 54.29379700002028, + "eq_jax_total_s": 2.2794162000063807, + "eq_as_used_speedup": 23.819167820193737 +} \ No newline at end of file diff --git a/benchmark/scripts/calibration/bellman_bench.py b/benchmark/scripts/calibration/bellman_bench.py new file mode 100644 index 0000000..cac2cae --- /dev/null +++ b/benchmark/scripts/calibration/bellman_bench.py @@ -0,0 +1,180 @@ +""" +Measured backing for the HIGH-efficiency anchor (score 5). + +`aiyagari.md` is JAX on both branches, so there is no numpy baseline inside the +repo. Instead we re-implement *its computational pattern* -- the vectorised +Bellman operator of `aiyagari.md:288-300` solved by value-function iteration on +an `a_size` x `z_size` grid -- in BOTH NumPy and JAX, and time it the way the +lecture actually uses it: + + * one household solve = VFI to convergence (hundreds of jitted iterations); + * the equilibrium loop re-solves the household problem many times at a FIXED + shape (here R=20), so JAX's one-time compile is amortised. + +This is the regime where JAX is supposed to win, so the measured speedup here +calibrates what "score 5" means for the shared efficiency threshold in +scoring/rubric.py. It is NOT tied to any single lecture; the low-end +calibration is any tiny-model lecture's own as_used_total.py (e.g. ge_arrow, +~45x slower). Run it once (or when hardware changes) to re-check the anchor. + +Output: bellman_bench.json (beside this script) + stdout. +""" +import json +import os +import time +import numpy as np +import jax +import jax.numpy as jnp +from functools import partial + +jax.config.update("jax_enable_x64", True) + +RESULTS = os.path.dirname(__file__) # write beside this script + +A_SIZE, Z_SIZE = 200, 7 +β, γ, r, w = 0.96, 2.0, 0.03, 1.0 +TOL = 1e-7 + +rng = np.random.default_rng(0) +a_grid_np = np.linspace(1e-4, 20.0, A_SIZE) +z_grid_np = np.linspace(0.5, 1.5, Z_SIZE) +Π_np = rng.random((Z_SIZE, Z_SIZE)); Π_np /= Π_np.sum(1, keepdims=True) + + +# ----------------------------- NumPy ----------------------------- +def u_np(c): + return c ** (1 - γ) / (1 - γ) + + +def bellman_np(v, a_grid, z_grid, Π): + a = a_grid.reshape(A_SIZE, 1, 1) + z = z_grid.reshape(1, Z_SIZE, 1) + ap = a_grid.reshape(1, 1, A_SIZE) + c = w * z + (1 + r) * a - ap + vv = v.reshape(1, 1, A_SIZE, Z_SIZE) + PP = Π.reshape(1, Z_SIZE, 1, Z_SIZE) + EV = np.sum(vv * PP, axis=-1) + B = np.where(c > 0, u_np(c) + β * EV, -np.inf) + return np.max(B, axis=-1) + + +def solve_np(): + v = np.zeros((A_SIZE, Z_SIZE)) + err, it = 1.0, 0 + while err > TOL and it < 2000: + v_new = bellman_np(v, a_grid_np, z_grid_np, Π_np) + err = np.max(np.abs(v_new - v)) + v = v_new + it += 1 + return v, it + + +# ----------------------------- JAX ----------------------------- +a_grid_j = jnp.asarray(a_grid_np) +z_grid_j = jnp.asarray(z_grid_np) +Π_j = jnp.asarray(Π_np) + + +def u_j(c): + return c ** (1 - γ) / (1 - γ) + + +@jax.jit +def bellman_j(v): + a = a_grid_j.reshape(A_SIZE, 1, 1) + z = z_grid_j.reshape(1, Z_SIZE, 1) + ap = a_grid_j.reshape(1, 1, A_SIZE) + c = w * z + (1 + r) * a - ap + vv = v.reshape(1, 1, A_SIZE, Z_SIZE) + PP = Π_j.reshape(1, Z_SIZE, 1, Z_SIZE) + EV = jnp.sum(vv * PP, axis=-1) + B = jnp.where(c > 0, u_j(c) + β * EV, -jnp.inf) + return jnp.max(B, axis=-1) + + +@jax.jit +def solve_j(): + def cond(state): + v, err, it = state + return (err > TOL) & (it < 2000) + + def body(state): + v, err, it = state + v_new = bellman_j(v) + return v_new, jnp.max(jnp.abs(v_new - v)), it + 1 + + v0 = jnp.zeros((A_SIZE, Z_SIZE)) + v, err, it = jax.lax.while_loop(cond, body, (v0, 1.0, 0)) + return v, it + + +def med(fn, repeat): + xs = [] + for _ in range(repeat): + t0 = time.perf_counter() + r = fn() + xs.append(time.perf_counter() - t0) + xs.sort() + return xs[len(xs) // 2], r + + +def main(): + # correctness: numpy and jax agree + v_np, it_np = solve_np() + v_j, it_j = solve_j(); jax.block_until_ready(v_j) + max_diff = float(np.max(np.abs(np.asarray(v_j) - v_np))) + + # single solve, numpy + t_np, _ = med(solve_np, 5) + + # single solve, jax COLD (fresh compile) + solve_j._clear_cache(); bellman_j._clear_cache() + t0 = time.perf_counter() + r = solve_j(); jax.block_until_ready(r) + t_j_cold = time.perf_counter() - t0 + + # single solve, jax WARM + def warm(): + r = solve_j(); jax.block_until_ready(r); return r + t_j_warm, _ = med(warm, 9) + + # equilibrium loop: R=20 re-solves at fixed shape + R = 20 + t0 = time.perf_counter() + for _ in range(R): + solve_np() + eq_np = time.perf_counter() - t0 + + solve_j._clear_cache(); bellman_j._clear_cache() + t0 = time.perf_counter() + for _ in range(R): + r = solve_j(); jax.block_until_ready(r) + eq_j = time.perf_counter() - t0 # includes 1 compile + 20 warm solves + + out = { + "grid": [A_SIZE, Z_SIZE], "vfi_iters": int(it_np), + "agree_max_abs": max_diff, + "single_numpy_s": t_np, + "single_jax_cold_s": t_j_cold, + "single_jax_warm_s": t_j_warm, + "single_speedup_cold": t_np / t_j_cold, + "single_speedup_warm": t_np / t_j_warm, + "eq_loop_R": R, + "eq_numpy_s": eq_np, + "eq_jax_total_s": eq_j, + "eq_as_used_speedup": eq_np / eq_j, + } + print(f"grid {A_SIZE}x{Z_SIZE}, VFI {it_np} iters, agree to {max_diff:.1e}") + print(f"single solve : numpy {t_np*1e3:8.2f} ms | " + f"jax cold {t_j_cold*1e3:8.2f} ms | jax warm {t_j_warm*1e3:8.2f} ms") + print(f" : warm speedup = {out['single_speedup_warm']:.1f}x " + f"cold speedup = {out['single_speedup_cold']:.2f}x") + print(f"equilibrium loop (R={R}, the as-used pattern):") + print(f" : numpy {eq_np*1e3:8.1f} ms | jax total {eq_j*1e3:8.1f} ms" + f" -> as-used speedup = {out['eq_as_used_speedup']:.1f}x") + with open(os.path.join(RESULTS, "bellman_bench.json"), "w") as f: + json.dump(out, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/benchmark/scripts/scoring/EVIDENCE_TEMPLATE.json b/benchmark/scripts/scoring/EVIDENCE_TEMPLATE.json new file mode 100644 index 0000000..a5c8acf --- /dev/null +++ b/benchmark/scripts/scoring/EVIDENCE_TEMPLATE.json @@ -0,0 +1,62 @@ +{ + "lecture": "", + "branch": "", + "_how": "Fill quantitative values from this lecture's results/*.json (note the source). Answer each structural criterion true/false with a citation to the diff. Then run: python scripts/scoring/score.py references/examples/ (from the plugin root). Never type a score — rubric.py computes it. score.py validates this file before scoring and refuses to run until the placeholders below are replaced: baseline_as_used_seconds is null here on purpose (it gates the no-conversion verdict, so it must be measured, not defaulted), and every criterion you set true needs an entry in that dimension's citations.", + + "quantitative": { + "correctness": { + "builds": true, + "max_delta_shipped": 0.0, + "matches_under_x64": true, + "source": "results/equivalence*.json — worst max|Δ| as shipped (float32) and under x64; builds=false if any cell raises" + }, + "readability": { + "delta_prereq_concepts": 0, + "docstring_cov_new": 1.0, + "source": "results/static_metrics.json — (new prereq count - old); new docstring_coverage" + }, + "efficiency": { + "as_used_speedup": 1.0, + "as_used_runs": [], + "baseline_as_used_seconds": null, + "correct_or_fixable": true, + "source": "as_used_total.py via run_all.py — baseline_total / candidate_total over the lecture's real call sequence, median of >=3 fresh-process runs per side. as_used_runs: the per-run speedups (drives the contested-band annotation); leave it [] only to state deliberately that this evaluation scored a single run — omitting the key is rejected. baseline_as_used_seconds: median baseline total (drives the no-conversion floor); must be filled, null is rejected" + }, + "ergonomics": { + "statements_for_one_result": 1, + "fragile_protocol": false, + "source": "results/static_metrics.json — calls needed to obtain one result; fragile if the call protocol is easy to misuse" + } + }, + + "structural": { + "logic_design": { + "criteria": { + "pure_no_order_dependence": false, + "no_global_state": false, + "good_algorithmic_choices": false, + "fixes_prior_bugs": false + }, + "introduces_correctness_bug": false, + "citations": {} + }, + "style_idiom": { + "criteria": { + "vectorised_where_natural": false, + "correct_control_flow_primitive": false, + "no_anti_idiomatic_constructs": false, + "clean_call_sites_and_naming": false + }, + "citations": {} + }, + "maintainability": { + "criteria": { + "pure_unit_testable": false, + "dtype_precision_safe": false, + "no_footgun_for_editors": false, + "robust_no_brittle_conditions": false + }, + "citations": {} + } + } +} diff --git a/benchmark/scripts/scoring/env_stamp.py b/benchmark/scripts/scoring/env_stamp.py new file mode 100644 index 0000000..f886a21 --- /dev/null +++ b/benchmark/scripts/scoring/env_stamp.py @@ -0,0 +1,43 @@ +"""Write /results/env.json — the provenance stamp for a measurement run. + +Usage: + python scripts/scoring/env_stamp.py [failed-step-title ...] + +Seed of the QuantEcon/meta#335 shared result + environment-descriptor schema: +python/platform plus the versions of the measurement-relevant libraries. +Any extra arguments are recorded as `steps_failed`, so a partial run is +self-describing instead of silently claiming full provenance for results +files an earlier environment produced. + +Invoked by each example's run_all.py with the interpreter that ran the +measurements (the stamp must describe the measurement environment). +""" +import json +import os +import platform +import sys +from importlib import metadata + + +def main(lecture_dir, failed): + info = {"python": sys.version.split()[0], + "platform": platform.platform(), "machine": platform.machine()} + for pkg in ("numpy", "jax", "jaxlib", "quantecon"): + try: + info[pkg] = metadata.version(pkg) + except metadata.PackageNotFoundError: + pass + if failed: + info["steps_failed"] = failed + res = os.path.join(os.path.abspath(lecture_dir), "results") + os.makedirs(res, exist_ok=True) + with open(os.path.join(res, "env.json"), "w", encoding="utf-8") as f: + json.dump(info, f, indent=2) + suffix = f" (steps_failed: {len(failed)})" if failed else "" + print(f"wrote {os.path.join(res, 'env.json')}{suffix}") + + +if __name__ == "__main__": + if len(sys.argv) < 2: + sys.exit("usage: python scripts/scoring/env_stamp.py [failed-step-title ...]") + main(sys.argv[1], sys.argv[2:]) diff --git a/benchmark/scripts/scoring/rubric.py b/benchmark/scripts/scoring/rubric.py new file mode 100644 index 0000000..d3c7743 --- /dev/null +++ b/benchmark/scripts/scoring/rubric.py @@ -0,0 +1,430 @@ +""" +THE SCORING STANDARD (single source of truth, shared by every lecture). + +This module encodes the rubric described in prose in +../../references/EVALUATION_FRAMEWORK.md +so that a score is a *deterministic function of evidence*, never a hand-typed +number. Nothing here is lecture-specific: the same rubric is applied to every +lecture. Only the per-lecture `evidence.json` changes. + +Two kinds of dimension: + + * QUANTITATIVE (correctness, readability, efficiency, ergonomics) — the score + comes from a measured metric via an explicit threshold table. + * STRUCTURAL (logic&design, style&idiom, maintainability) — the score comes + from a fixed 4-item yes/no checklist: score = 1 + (#criteria met), with a + small number of documented override caps. Every checklist answer in + evidence.json must carry a citation. + +Each scorer returns (score:int, reason:str) so the derivation is auditable. + +v2 (2026-07, from the three-way design review): the logic&design bug-cap is +derived from the correctness evidence inside `score_all`; correctness 1/2 +gates the verdict band; a "no-conversion" verdict fires on the triage +don't-convert profile; the as-used metric accepts a median-of-runs list. +""" + +# ---- weights (sum to 1.0); rationale in EVALUATION_FRAMEWORK.md ------------- +WEIGHTS = { + "correctness": 0.20, + "readability": 0.25, + "efficiency": 0.15, + "logic_design": 0.15, + "style_idiom": 0.10, + "ergonomics": 0.10, + "maintainability": 0.05, +} + +TITLES = { + "correctness": "Correctness & numerical fidelity", + "readability": "Readability & pedagogical clarity", + "efficiency": "Computational efficiency (as used)", + "logic_design": "Logic & design", + "style_idiom": "Coding style & idiom", + "ergonomics": "API ergonomics & reusability", + "maintainability": "Maintainability & robustness", +} + +KIND = { + "correctness": "quantitative", "readability": "quantitative", + "efficiency": "quantitative", "ergonomics": "quantitative", + "logic_design": "structural", "style_idiom": "structural", + "maintainability": "structural", +} + +# ---- fixed 4-item checklists for the structural dimensions ------------------ +# Each key is a criterion phrased so that TRUE = good. Order is the display order. +CHECKLISTS = { + "logic_design": [ + "pure_no_order_dependence", # pure functions; no ordered stateful calls + "no_global_state", # no reliance on module/global variables + "good_algorithmic_choices", # vectorised where natural; no needless recompute + "fixes_prior_bugs", # removes a real bug/smell from the original + ], + "style_idiom": [ + "vectorised_where_natural", # broadcast/einsum, not scalar loops + "correct_control_flow_primitive", # scan/while_loop/fori_loop used aptly + "no_anti_idiomatic_constructs", # no cond-on-static, loop-where-vectorise + "clean_call_sites_and_naming", # call sites read cleanly; consistent naming + ], + "maintainability": [ + "pure_unit_testable", # easy to unit-test in isolation + "dtype_precision_safe", # x64/float64 enabled; no silent dtype trap + "no_footgun_for_editors", # call protocol hard to misuse + "robust_no_brittle_conditions", # no reliance on near-critical/low-precision edges + ], +} + + +# ---- total → verdict bands ------------------------------------------------- +# Ordered worst → best; verdict gating compares indices into this list. +BANDS = [ + "net regression — do not merge as-is", + "mixed / wash — improvements offset by real regressions; revisit before merging", + "net positive with fixable regressions — merge after addressing them", + "clear improvement — merge", +] +SHORT_BANDS = ["net regression", "mixed/wash", "net positive", "clear improvement"] + + +def band_index(total): + if total >= 4.0: + return 3 + if total >= 3.0: + return 2 + if total >= 2.5: + return 1 + return 0 + + +def verdict(total): + return BANDS[band_index(total)] + + +# Policy floor (a policy choice, anchored not derived): a lecture whose whole +# baseline replay finishes in under this many seconds has no as-used time worth +# buying, so a candidate that is *also slower* as-used earns the +# "no-conversion" verdict regardless of its polish. Reconciles review mode +# with triage — both blind-validated don't-convert baselines sit two orders of +# magnitude below this floor, the convert case (aiyagari pattern, ~54 s) two +# above it, so the exact placement inside that gap is not load-bearing. The +# measured baselines are not restated here: each lecture's own +# `baseline_as_used_seconds` in evidence.json is the value this gate reads. +NO_CONVERSION_BASELINE_S = 1.0 + + +def _median(xs): + s = sorted(xs) + n = len(s) + return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2.0 + + +# ===================== QUANTITATIVE SCORERS =============================== +def score_correctness(builds, max_delta_shipped, matches_under_x64): + """max|Δ| vs the original *as the lecture ships* (float32 unless x64 set). + + `matches_under_x64` is the equivalence check re-run with JAX_ENABLE_X64=1: + TRUE means the logic agrees once precision is removed from the question + (residuals at x64 noise, ~1e-14 to ~1e-11, are recorded TRUE — see + references/examples/README.md). So FALSE asserts that the economics + genuinely differ, and it caps on its own. It deliberately does *not* + require the shipped drift to be large as well: a candidate whose logic + diverges but whose float32 output happens to agree closely is the exact + "wrong economics masked by low precision" case §1 names, and conditioning + the cap on a visible shipped delta made the guard weakest precisely where + the defect is hardest to see. + """ + if not builds: + return 1, "does not build as shipped → 1 (overrides Δ bands)" + d = max_delta_shipped + if not matches_under_x64: + return 1, (f"logic diverges under x64 → wrong economics → 1 " + f"(shipped max|Δ|={d:.1e}; agreement as shipped does not " + f"redeem divergent logic)") + if d <= 1e-10: + return 5, f"max|Δ|={d:.1e} ≤ 1e-10 → 5" + if d <= 1e-8: + return 4, f"max|Δ|={d:.1e} ≤ 1e-8 → 4" + if d <= 1e-3: + return 3, f"logic matches under x64 but ships float32 → drift {d:.1e} (1e-8,1e-3] → 3" + if d <= 1e-1: + return 2, f"material drift max|Δ|={d:.1e} (1e-3,1e-1] → 2" + return 1, f"max|Δ|={d:.1e} > 1e-1 → 1" + + +def score_readability(delta_prereq_concepts, docstring_cov_new): + dp = delta_prereq_concepts + sp = 5 if dp <= 0 else 4 if dp <= 2 else 3 if dp <= 4 else 2 if dp <= 6 else 1 + c = docstring_cov_new + sd = 5 if c >= 0.80 else 4 if c >= 0.75 else 3 if c >= 0.60 else 2 + s = min(sp, sd) + return s, (f"Δprereq={dp:+d}→{sp}, docstrings={c:.2f}→{sd}; " + f"worse-of-two → {s}") + + +def score_efficiency(as_used_speedup, correct_or_fixable): + s = as_used_speedup + if s >= 3: + return 5, f"as-used speedup {s:.3g}× ≥ 3 → 5" + if s >= 1.3: + return 4, f"as-used speedup {s:.3g}× ∈ [1.3,3) → 4" + if s >= 0.8: + return 3, f"as-used speedup {s:.3g}× ∈ [0.8,1.3) → 3 (wash)" + if correct_or_fixable: + return 2, f"as-used speedup {s:.3g}× < 0.8 (slower) but correct/fixable → 2" + return 1, f"as-used speedup {s:.3g}× < 0.8 and wrong/unfixable → 1" + + +def score_ergonomics(statements_for_one_result, fragile_protocol): + n = statements_for_one_result + base = 5 if n <= 1 else 4 if n == 2 else 3 if n == 3 else 2 + if fragile_protocol and base > 3: + return 3, f"{n} statement(s)→{base}, but fragile protocol caps at 3" + if fragile_protocol: + return base, f"{n} statement(s) + fragile protocol → {base}" + return base, f"{n} statement(s) to obtain one result → {base}" + + +# ===================== STRUCTURAL SCORER ================================== +def score_structural(dim, criteria, overrides=None): + """score = 1 + (#criteria met), with documented override caps.""" + overrides = overrides or {} + keys = CHECKLISTS[dim] + met = [k for k in keys if criteria.get(k)] + score = 1 + len(met) + note = "" + if dim == "logic_design" and overrides.get("introduces_correctness_bug"): + derived = overrides.get("_bug_derived_from") + why = (f"correctness-bug cap derived from correctness evidence: {derived}" + if derived else "introduces a correctness bug") + if score > 3: + note = f" (capped at 3: {why})" + score = min(score, 3) + reason = (f"{len(met)}/4 criteria met [{', '.join(met) or 'none'}] " + f"→ 1+{len(met)}={1+len(met)}{note}") + return score, reason + + +# Scored inputs that `score_all` reads with `.get()` and therefore tolerates +# silently: omit one and a verdict weakens with nothing said. Every other +# scored input is read with `ev[...]`, which already fails loudly. Listing them +# here makes the tolerance a validation rule rather than an accident of which +# accessor a line happened to use — the same failure class the derived +# correctness-bug cap was written to close. +_REQUIRED_INPUTS = { + "efficiency": [ + ("baseline_as_used_seconds", + "drives the no-conversion materiality floor; without it that verdict " + "silently never fires"), + ("as_used_runs", + "the K-repeat as-used measurement; set it to [] to declare a " + "deliberate single-run score, but do not omit it"), + ], +} + + +def check_required_inputs(evidence): + """Return a list of scored inputs that are missing or null. + + `as_used_runs: []` is accepted — an empty list is a visible statement that + this evaluation scored a single run, which is a choice a reader can weigh. + An absent key is not: it is indistinguishable from forgetting. + """ + problems = [] + merged = {} + merged.update(evidence.get("quantitative", {})) + merged.update(evidence.get("structural", {})) + + for dim in WEIGHTS: + if not isinstance(merged.get(dim), dict): + problems.append(f"{dim}: dimension missing from evidence") + + for dim, fields in _REQUIRED_INPUTS.items(): + ev = merged.get(dim) + if not isinstance(ev, dict): + continue + for key, why in fields: + if key not in ev: + problems.append(f"{dim}.{key}: missing — {why}") + elif ev[key] is None: + problems.append(f"{dim}.{key}: null — {why}") + + eff = merged.get("efficiency") + if isinstance(eff, dict) and not eff.get("as_used_runs") \ + and eff.get("as_used_speedup") is None: + problems.append("efficiency.as_used_speedup: missing — required when " + "as_used_runs is empty (the single-run fallback)") + return problems + + +def validate_evidence(evidence): + """Every authoring contract this rubric depends on, checked in one pass. + + Returns a list of problems (empty when the evidence is well-formed). Run + once on authored evidence before scoring — never inside a scorer, so + `score_all` stays a pure function of evidence. + """ + return check_required_inputs(evidence) + check_citations(evidence) + + +def check_citations(evidence): + """Enforce the cited-judgement contract on *authored* evidence. + + A structural score is `1 + #criteria met`, so a criterion marked TRUE is + the thing that moves the score up: it must say what it is claiming on the + basis of. Returns a list of violations (empty when the contract holds). + + Deliberately a separate pass over the authored evidence rather than a check + inside `score_structural`, for two reasons: `score_all` stays a pure + function of evidence — which is what makes the perturbation search in + score.py meaningful — and the sensitivity loop, which scores hundreds of + mutated copies, does not re-run authoring checks and silently drop the + perturbations that trip them. + + Criteria marked FALSE are not required to carry one: a false "not met" can + only understate a score, which is the conservative direction. A manual + `introduces_correctness_bug` override *is* required to cite, because it + moves a verdict and is the hand-set flag the derived cap exists to distrust + (a cap derived inside `score_all` never appears in authored evidence). + """ + problems = [] + structural = evidence.get("structural", {}) + for dim, keys in CHECKLISTS.items(): + ev = structural.get(dim) + if not isinstance(ev, dict): + continue + criteria = ev.get("criteria", {}) + citations = ev.get("citations", {}) + if not isinstance(citations, dict): + problems.append(f"{dim}: `citations` must be an object mapping " + f"criterion → evidence") + continue + for key in keys: + if criteria.get(key) and not str(citations.get(key, "")).strip(): + problems.append(f"{dim}.{key}: marked met with no citation") + if ev.get("introduces_correctness_bug") and \ + not str(citations.get("introduces_correctness_bug", "")).strip(): + problems.append(f"{dim}.introduces_correctness_bug: override set " + f"with no citation") + return problems + + +# ===================== DRIVER ============================================ +def score_dimension(dim, ev): + """Dispatch one dimension's evidence dict `ev` to the right scorer.""" + if dim == "correctness": + return score_correctness(ev["builds"], ev["max_delta_shipped"], + ev["matches_under_x64"]) + if dim == "readability": + return score_readability(ev["delta_prereq_concepts"], + ev["docstring_cov_new"]) + if dim == "efficiency": + runs = ev.get("as_used_runs") + if runs: + sp = _median(runs) + s, reason = score_efficiency(sp, ev["correct_or_fixable"]) + per_run = {score_efficiency(r, ev["correct_or_fixable"])[0] + for r in runs} + if len(per_run) > 1: + reason += (f" [median of {len(runs)} fresh-process runs, spread " + f"{min(runs):.3g}–{max(runs):.3g}×; CONTESTED BAND: " + f"runs alone would score {sorted(per_run)}]") + else: + reason += (f" [median of {len(runs)} fresh-process runs, spread " + f"{min(runs):.3g}–{max(runs):.3g}× within one band]") + return s, reason + s, reason = score_efficiency(ev["as_used_speedup"], + ev["correct_or_fixable"]) + return s, reason + (" [single-run measurement; the v2 standard is a " + "median of ≥3 fresh-process runs — see " + "as_used_runs in the evidence template]") + if dim == "ergonomics": + return score_ergonomics(ev["statements_for_one_result"], + ev["fragile_protocol"]) + # structural + return score_structural(dim, ev["criteria"], + {k: v for k, v in ev.items() if k != "criteria"}) + + +def score_all(evidence): + """Return the full breakdown for a lecture's `evidence` dict. + + `evidence` has keys "quantitative" and "structural", each mapping a + dimension id to its evidence dict (see EVIDENCE_TEMPLATE.json). + """ + merged = {} + merged.update(evidence.get("quantitative", {})) + merged.update(evidence.get("structural", {})) + + # ---- derived safety coupling: the logic&design correctness-bug cap + # follows from the correctness evidence itself, so one forgotten boolean + # can no longer leave a non-building (or x64-divergent) candidate uncapped. + corr = merged.get("correctness", {}) + derived = None + if not corr.get("builds", True): + derived = "does not build as shipped" + elif not corr.get("matches_under_x64", True): + # Unconditional, matching score_correctness: divergent logic is a + # correctness bug whether or not the shipped float32 output hides it. + derived = "logic diverges under x64" + if derived and "logic_design" in merged \ + and not merged["logic_design"].get("introduces_correctness_bug"): + ld = dict(merged["logic_design"]) + ld["introduces_correctness_bug"] = True + ld["_bug_derived_from"] = derived + merged["logic_design"] = ld + + rows, total, scores = [], 0.0, {} + for dim in WEIGHTS: + ev = merged[dim] + s, reason = score_dimension(dim, ev) + w = WEIGHTS[dim] + total += w * s + scores[dim] = s + rows.append({"dim": dim, "title": TITLES[dim], "kind": KIND[dim], + "weight": w, "score": s, "weighted": round(w * s, 3), + "reason": reason, + "citations": ev.get("citations", ev.get("source"))}) + # Verdict on the rounded total so the band always agrees with the number + # shown: raw FP sums can land at e.g. 2.4999999999999996 for combinations + # that are exactly 2.50 in exact arithmetic. + total = round(total, 2) + + # ---- verdict gates: a candidate whose correctness is broken cannot + # weighted-average its way into a merge band, whatever its polish. + base_idx = band_index(total) + final_idx, gate = base_idx, None + if scores["correctness"] == 1 and base_idx > 0: + final_idx = 0 + gate = "correctness 1 caps the verdict at net regression" + elif scores["correctness"] == 2 and base_idx > 1: + final_idx = 1 + gate = "correctness 2 caps the verdict at mixed/wash" + + # ---- no-conversion: when the efficiency evidence shows the triage + # don't-convert profile, the verdict says so instead of scoring the polish. + eff = merged.get("efficiency", {}) + runs = eff.get("as_used_runs") + sp = _median(runs) if runs else eff.get("as_used_speedup") + base_s = eff.get("baseline_as_used_seconds") + no_conv = bool(base_s is not None and base_s < NO_CONVERSION_BASELINE_S + and sp is not None and sp < 1.0) + + v = BANDS[final_idx] + if gate: + v += (f" [gated: {gate}; the ungated total {total:.2f} would band as " + f"{SHORT_BANDS[base_idx]}]") + if no_conv: + v = (f"no-conversion — the baseline as-used total {base_s:.3g} s is under " + f"the {NO_CONVERSION_BASELINE_S:g} s materiality floor and the " + f"candidate is slower as-used ({sp:.3g}×): this lecture should not " + f"be converted, whatever the candidate's polish. Candidate quality " + f"for the record: {total:.2f}/5, {SHORT_BANDS[final_idx]}" + + (f" [{gate}]" if gate else "")) + return {"rows": rows, "total": total, "verdict": v, + "band_verdict": BANDS[final_idx], "verdict_gate": gate, + "no_conversion": no_conv, + # Final band position (post-gate). At 0 the verdict is at the floor + # and no single input can make it worse — which the sensitivity + # stamp has to account for before calling the outcome stable. + "band_index": final_idx} diff --git a/benchmark/scripts/scoring/score.py b/benchmark/scripts/scoring/score.py new file mode 100644 index 0000000..8dbf01e --- /dev/null +++ b/benchmark/scripts/scoring/score.py @@ -0,0 +1,167 @@ +""" +Scoring engine / CLI. Applies the shared rubric (rubric.py) to one lecture's +evidence and writes an auditable scorecard. + +Usage: + python scripts/scoring/score.py + # e.g. python scripts/scoring/score.py references/examples/ge_arrow + +It reads /evidence.json (inputs + citations; filled from results/) +and writes /results/scorecard.json and prints the derivation table. + +The score of every dimension is COMPUTED here from the evidence via rubric.py — +no score is ever written by hand. To change a score you change the measured +metric or a checklist answer in evidence.json, or the standard in rubric.py. +""" +import copy +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import rubric # noqa: E402 + +# Evidence keys that are provenance/prose, not scored inputs. +_NON_INPUT_KEYS = ("citations", "source", "lecture", "branch") + + +def _perturbations(evidence): + """Yield (dotted-path, old, new, mutated-evidence) — one input changed per + yield: booleans flipped; integer counts ±1; measured floats ±10%.""" + def walk(node, path): + for k, v in node.items(): + if k.startswith("_") or k in _NON_INPUT_KEYS: + continue + p = path + [k] + if isinstance(v, dict): + yield from walk(v, p) + elif isinstance(v, bool): + yield p, v, [not v] + elif isinstance(v, int): + yield p, v, [v - 1, v + 1] + elif isinstance(v, float): + yield p, v, [v * 0.9, v * 1.1] + + for path, old, alts in walk(evidence, []): + for alt in alts: + mutated = copy.deepcopy(evidence) + node = mutated + for k in path[:-1]: + node = node[k] + node[path[-1]] = alt + yield ".".join(path), old, alt, mutated + + +def sensitivity(evidence, base): + """One-flip sensitivity stamp: is the *final* verdict (band after gating, + plus the no-conversion flag) stable under single-input perturbations?""" + outcome0 = (base["no_conversion"], base["band_verdict"]) + tested, skipped, flips = 0, [], [] + for label, old, new, mutated in _perturbations(evidence): + try: + r = rubric.score_all(mutated) + except Exception as exc: + # A perturbation that breaks scoring cannot be deciding — but it is + # also not evidence of stability, so it is reported rather than + # silently counted in the denominator the stamp is judged on. + skipped.append({"input": label, "error": f"{type(exc).__name__}: {exc}"}) + continue + tested += 1 + if (r["no_conversion"], r["band_verdict"]) != outcome0: + flips.append({ + "input": label, "from": old, "to": new, "total": r["total"], + "outcome": (("no-conversion; " if r["no_conversion"] else "") + + r["band_verdict"]), + }) + # A verdict already in the bottom band cannot be perturbed downward: no + # single input can make "net regression" worse. So "no deciding flips" + # there is partly the band's geometry rather than the evidence's strength — + # only upward moves were ever available to the search. Reporting that as + # plain "robust" asserts a support the run did not demonstrate, and + # SKILL.md carries the stamp verbatim into the report. + floored = base.get("band_index") == 0 + if flips: + stamp, note = "fragile", "" + elif floored: + stamp = "robust-at-floor" + note = ("no perturbation changed the outcome, but the verdict is " + "already in the bottom band, where no single input can make it " + "worse — only upward moves were available to this search, so " + "the stability is partly structural, not purely evidential") + else: + stamp, note = "robust", "" + + return {"stamp": stamp, "stamp_note": note, + "perturbations_tested": tested, "deciding_flips": flips, + "perturbations_skipped": skipped} + + +def main(lecture_dir): + lec_dir = os.path.abspath(lecture_dir) + lecture = os.path.basename(lec_dir) + ev_path = os.path.join(lec_dir, "evidence.json") + if not os.path.exists(ev_path): + sys.exit(f"no evidence file at {ev_path}") + with open(ev_path, encoding="utf-8") as f: + evidence = json.load(f) + + # The authoring contracts are enforced, not just documented: every scored + # input the verdict gates read must be present, and a structural criterion + # that moves a score up must say what it rests on. + problems = rubric.validate_evidence(evidence) + if problems: + sys.exit(f"evidence.json is not valid ({len(problems)} problem(s)):" + "\n - " + "\n - ".join(problems)) + + result = rubric.score_all(evidence) + sens = sensitivity(evidence, result) + + # ---- print an auditable table ---- + print(f"\nSCORECARD — {lecture} (branch {evidence.get('branch', '?')})") + print("=" * 78) + print(f"{'dimension':34s} {'kind':11s} {'wt':>4s} {'sc':>3s} {'wtd':>5s}") + print("-" * 78) + for r in result["rows"]: + print(f"{r['title']:34s} {r['kind']:11s} {r['weight']:>4.2f} " + f"{r['score']:>3d} {r['weighted']:>5.2f}") + print(f" └ {r['reason']}") + print("-" * 78) + print(f"{'WEIGHTED TOTAL':34s} {'':11s} {'':>4s} {'':>3s} {result['total']:>5.2f}") + print(f"VERDICT: {result['verdict']}") + skipped = sens["perturbations_skipped"] + print(f"SENSITIVITY: {sens['stamp']} " + f"({sens['perturbations_tested']} single-input perturbations scored" + + (f"; {len(skipped)} skipped — see scorecard.json" if skipped else "") + + ")") + if sens["stamp_note"]: + print(f" └ {sens['stamp_note']}") + for fl in sens["deciding_flips"]: + print(f" └ {fl['input']}: {fl['from']} → {fl['to']} " + f"⇒ total {fl['total']:.2f}, {fl['outcome']}") + + out = { + "lecture": lecture, + "branch": evidence.get("branch"), + "weighted_total_out_of_5": result["total"], + "verdict": result["verdict"], + "band_verdict": result["band_verdict"], + "verdict_gate": result["verdict_gate"], + "no_conversion": result["no_conversion"], + "sensitivity": sens, + "dimensions": result["rows"], + "_note": "Scores are computed by scripts/scoring/rubric.py from " + f"{lecture}/evidence.json; do not edit by hand.", + } + res_dir = os.path.join(lec_dir, "results") + os.makedirs(res_dir, exist_ok=True) + dst = os.path.join(res_dir, "scorecard.json") + with open(dst, "w", encoding="utf-8") as f: + json.dump(out, f, indent=2, ensure_ascii=False) + print(f"\nwrote {os.path.relpath(dst)}") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit("usage: python scripts/scoring/score.py ") + main(sys.argv[1]) diff --git a/benchmark/skills/review-acceleration/SKILL.md b/benchmark/skills/review-acceleration/SKILL.md index ac003cf..90049f4 100644 --- a/benchmark/skills/review-acceleration/SKILL.md +++ b/benchmark/skills/review-acceleration/SKILL.md @@ -5,25 +5,57 @@ description: Review whether an accelerated implementation (JAX, Numba) of QuantE # review-acceleration -> **Status: under construction.** This skill is being built collaboratively with @xuanguang-li from the evaluation system he published on [QuantEcon/lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717). The supporting scripts referenced below are being collected into this plugin's `scripts/` directory. Tracking: [QuantEcon/meta#335](https://github.com/QuantEcon/meta/issues/335) (workstream B). +> **Status: evaluation system landed (rubric v2); skill wired for workspace runs.** The system was developed and validated by @xuanguang-li on [QuantEcon/lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654) and now lives in this plugin: the rubric in [`references/EVALUATION_FRAMEWORK.md`](../../references/EVALUATION_FRAMEWORK.md), the deterministic scoring engine in `scripts/scoring/`, and two complete worked evaluations in `references/examples/`. Rubric v2 (verdict gates, no-conversion, sensitivity stamp, K-repeat as-used) implements the surviving critiques of the 2026-07-21 design review. Tracking: [QuantEcon/skills#4](https://github.com/QuantEcon/skills/issues/4), [QuantEcon/meta#335](https://github.com/QuantEcon/meta/issues/335) (workstream B). + +## Where things live at run time + +When this skill runs from the installed plugin, the plugin's files are **read-only** at `${CLAUDE_PLUGIN_ROOT}` (the engine in `${CLAUDE_PLUGIN_ROOT}/scripts/scoring/`, templates and worked examples in `${CLAUDE_PLUGIN_ROOT}/references/examples/`). The evaluation itself is built in the **user's workspace** — normally the lecture repo checkout under review: + +``` +/benchmark-eval// + scripts/ # adapted per-lecture from a worked example's scripts/ + evidence.json # started from ${CLAUDE_PLUGIN_ROOT}/scripts/scoring/EVIDENCE_TEMPLATE.json + results/ # written by the pipeline + scorer + _REPORT.md +``` + +Never write into the plugin directory. Scoring works on any directory: `python ${CLAUDE_PLUGIN_ROOT}/scripts/scoring/score.py benchmark-eval/`. The scaffolded `run_all.py` reads `CLAUDE_PLUGIN_ROOT` from the environment to find the shared engine — export it (or keep the adapted script's path pointing at the plugin) before running the pipeline. + +**Preconditions to verify before starting** (fail loudly, don't improvise silently): a checkout of the lecture repo with both refs fetchable (baseline, usually `main`, and the candidate branch); a Python environment with `jax`, `numpy`, and the lecture's imports (the reference runs used the `quantecon` conda env); CPU-only is the calibrated regime. Record the environment via the provenance stamp — `run_all.py` does this automatically, including failed-step titles, so a partial run cannot claim full provenance. ## Guiding principle QuantEcon lectures are teaching materials first and programs second. A rewrite that is faster or more modern but harder for a learner to read, or that silently changes published numbers, is not an improvement. "Uses JAX" is never a goal in itself — the accelerated implementation must earn its place on each lecture. -## Procedure (v0 outline) +## Procedure + +Given a baseline implementation (usually `main`) and a candidate (usually a PR branch) for one lecture, follow the measure → record-evidence → score contract in [`scripts/README.md`](../../scripts/README.md) — **scores are never typed by hand**: + +1. **Scaffold** — create `/benchmark-eval//` from a worked example under `${CLAUDE_PLUGIN_ROOT}/references/examples/`: extract `model_old.py` (baseline) and `model_new.py` (candidate) **verbatim** from the lecture's code cells (disclose any deviation in the report), and adapt the measurement templates (`check_equivalence.py`, `static_metrics.py`, `benchmark.py`, `as_used_total.py`, plus lecture-specific ones) to the lecture's actual examples and call sequence. Adapting templates per lecture is this skill's job — there is deliberately no rigid harness. Before measuring, diff the extracted code and the replayed call sequence against the lecture's cells and fix mismatches — construction-pattern drift here invalidates everything downstream. +2. **Measure** — `run_all.py`: equivalence under the default dtype AND `jax_enable_x64` (report `max|Δ|` per regime); static metrics (prerequisite concepts, docstring coverage); the **as-used benchmark** — replay the lecture's *actual* solver call sequence at its *actual* sizes in a fresh interpreter so trace/compile time counts, repeated ≥3 times per side with the **median** as the headline (`as_used_speedup = baseline median / candidate median`), with warm timings alongside (never alone), a crossover-n scaling curve, and a recompile audit. A provenance stamp (`results/env.json`, generated per-run) records the environment and any failed steps. +3. **Record evidence** — fill `evidence.json` from the results: measured numbers into the quantitative slots with sources (including `baseline_as_used_seconds` and the per-run `as_used_runs`); each structural checklist item answered true/false **with a citation to the diff**. This file is the only place judgement is recorded. +4. **Score** — `python ${CLAUDE_PLUGIN_ROOT}/scripts/scoring/score.py benchmark-eval/` computes all seven dimensions and the weighted total deterministically. The weights, threshold anchors, and verdict bands are defined in [`references/EVALUATION_FRAMEWORK.md`](../../references/EVALUATION_FRAMEWORK.md) §1–2 and machine-encoded in `scripts/scoring/rubric.py` — never restate or re-derive them here. v2 outputs you must carry into the report verbatim: the **verdict gate** (correctness 1/2 caps the band), the **no-conversion** verdict (don't-convert profile beats polish), and the **sensitivity stamp** (robust / robust-at-floor / fragile, with the deciding flips). Carry the stamp as printed — *robust-at-floor* means the outcome held only because it is already in the bottom band and could not get worse, so never report it as *robust*. Because one stamp currently covers both measurement and judgement perturbations ([framework §1](../../references/EVALUATION_FRAMEWORK.md)), quote the deciding-flip list rather than resting the report's confidence on the word alone. +5. **Report** — write `_REPORT.md` from the scorecard + evidence, following the worked examples' format: TL;DR with the weighted score and the *full* verdict (including gate/no-conversion/sensitivity), the dimension table with drivers, evidence per dimension, and a must-fix list mapping each recommendation to the dimension it lifts. + +Never present warm-only speedups as the headline — the ge_arrow case measured 1.4–4.8× faster warm and ~45× slower as-used. + +## Triage mode (no candidate yet) + +When the question is "should this lecture be converted at all," run the prospective subset — only the existing lecture is needed: + +1. **Baseline as-used total**: adapt just the baseline half of an `as_used_total.py` template and replay the lecture's real call sequence — this bounds the maximum possible win (a 30 ms lecture has nothing to give). +2. **Pattern-match** against the calibrated poles: aiyagari-shaped (large fixed shapes, many re-solves, stable static args → ~24× win) vs ge_arrow-shaped (tiny models, fresh static args per call → ~45× loss). +3. **Crossover check**: the lecture's problem sizes vs warm crossover-n. +4. **Readability-cost forecast**: which prerequisite concepts the conversion would force. -Given a baseline implementation (usually `main`) and a candidate (usually a PR branch) for one lecture: +Decision rule from the weights: efficiency (0.15) gains at most +0.30 weighted; readability (0.25) losing two bands costs −0.50 — a conversion that costs meaningful readability cannot break even on speed alone, and structural wins are usually achievable in the baseline library. Report a predicted verdict band with the binding constraint named, not a scorecard. Validated 2026-07-21: blind triage on ge_arrow, markov_asset (both sub-second baselines → don't convert) and the aiyagari pattern (~54 s → convert) reproduced all three known verdicts, from the triage-time baseline measurements recorded in the plugin README; triage cannot predict conversion-quality defects (markov_asset's build bug), and must say so. Rubric v2 closes the loop from the review side: when a full evaluation's efficiency evidence shows the don't-convert profile (baseline under the 1 s floor, candidate slower as-used), the scorecard itself emits the **no-conversion** verdict — review and triage can no longer disagree on that question. -1. **Equivalence check** (`scripts/check_equivalence.py`, pending): run both implementations over every example in the lecture; diff all published objects under the default dtype AND with `jax_enable_x64` enabled; report `max|Δ|` for each regime. -2. **Static metrics** (`scripts/static_metrics.py`, pending): prerequisite-concept count, docstring coverage, code lines, number of definitions, closure-nesting depth — for both implementations. -3. **As-used benchmark** (`scripts/as_used_total.py`, pending): replay the lecture's *actual* solver call sequence, at its *actual* problem sizes, in a fresh interpreter so JIT trace/compile time counts. Compute `as_used_speedup = baseline total wall time / candidate total wall time`. Record warm timings alongside (never alone), a crossover-n scaling curve, and a recompile audit (one recompile per distinct static-argument value or shape). -4. **Score seven dimensions** (1–5 against the rubric anchors) and combine with weights: correctness & numerical fidelity 0.20, readability & pedagogical clarity 0.25, computational efficiency as-used 0.15, logic & design 0.15, coding style & idiom 0.10, API ergonomics 0.10, maintainability 0.05. Readability deliberately outranks efficiency. -5. **Report** with per-dimension evidence and the weighted total: ≥ 4.0 merge; 3.0–3.9 merge after addressing fixable regressions; 2.5–2.9 revisit before merging; < 2.5 do not merge as-is. Include a concrete fix list mapping each recommendation to the dimension it lifts. +## Calibration baseline (regression anchors) -The full rubric with numeric scoring anchors and worked HIGH/LOW examples lives in the [#717 thread](https://github.com/QuantEcon/lecture-python.myst/pull/717) and will move into this plugin's `references/` and the QuantEcon manual ([QuantEcon.manual#104](https://github.com/QuantEcon/QuantEcon.manual/issues/104)). +The two worked evaluations in `references/examples/` are the validation baseline — re-running their pipelines must reproduce these verdicts. Confirmed end-to-end 2026-07-22: a fresh-checkout workspace run of ge_arrow (#717, base `8cfba4c`) on a different machine and jax **0.10.1** (reference: 0.4.35) reproduced 2.85 / no-conversion / fragile with the same deciding flips — every measured quantity moved only within its band. Evidence files record `source_pr` + base/head SHAs: -## Calibration cases +- **`ge_arrow`** ([#717](https://github.com/QuantEcon/lecture-python.myst/pull/717)): **2.85/5 — no-conversion** (candidate band mixed/wash; sensitivity: fragile). Tiny 2×2/3×3 economies, fresh static args per call → ~45× slower as-used despite warm wins, on a 0.035 s baseline. +- **`markov_asset`** ([#654](https://github.com/QuantEcon/lecture-python.myst/pull/654)): **2.25/5 — no-conversion + gated net regression** (sensitivity: robust-at-floor). A stray `err.throw()` that crashes in any clean namespace and, in notebook order, silently disables the checkify stability validation (a masked failure — see the REPORT erratum); float32 drift near a critical stability margin. +- **HIGH anchor:** the aiyagari Bellman pattern (`scripts/calibration/bellman_bench.py`) — large fixed-shape arrays, many re-solves; ~25× faster as-used → the "score 5" calibration. -- **HIGH:** the `aiyagari.md` Bellman pattern — large fixed-shape arrays, many re-solves; measured ~25× faster as-used under JAX. -- **LOW:** the `ge_arrow.md` conversion (lecture-python.myst#717) — 2×2/3×3 economies, fresh static args per call; measured ~45× slower as-used despite warm speedups. +The rubric will also be distilled into the QuantEcon manual as the companion to the JAX style page ([QuantEcon.manual#104](https://github.com/QuantEcon/QuantEcon.manual/issues/104)). diff --git a/docs/developing-skills.md b/docs/developing-skills.md new file mode 100644 index 0000000..320ca24 --- /dev/null +++ b/docs/developing-skills.md @@ -0,0 +1,88 @@ +# Developing skills + +For contributors adding or modifying plugins in this repo. (Using them: [using-skills.md](using-skills.md).) + +## Repo layout + +``` +.claude-plugin/marketplace.json # the catalogue — every plugin registers here +scripts/validate.py # manifest + frontmatter validation (CI runs this) +docs/ # these guides +/ # one directory per plugin + .claude-plugin/plugin.json # name, description, version + README.md # the plugin's user guide + skills//SKILL.md # one directory per skill + scripts/ # deterministic scripts the skills drive + references/ # rule/rubric content the skills read +``` + +**Only `SKILL.md` is required.** A skill that is purely a procedure — nothing deterministic to run, no long reference material to point at — is one file in one directory, and should stay that way. `scripts/` appears when there is something mechanical worth doing in code; `references/` when the skill needs more context than belongs in its body. Adding either before you need it just makes the skill harder to read. + +The three live plugins show some of the range: `qe` (umbrella skill plus thin per-category sub-skills, sharing plugin-level rules and scripts), `benchmark` (one skill driving a deterministic engine, with worked examples as its regression baseline), and `audit` (sibling procedures sharing a method document). None of these is the house style — they are what three problems happened to need. + +## Conventions + +Guidance rather than gates. The repo is early, and most of what follows generalises from one or two worked examples; where something is genuinely load-bearing it says so and gives the reason. Departing from the rest is fine when you have a reason — and worth mentioning in the PR, since a second example is how any of this eventually becomes a real convention ([CATALOG.md § Principles](../CATALOG.md#principles)). + +- **Invocations read as commands** — the whole `/plugin:skill` string, not the skill name alone. `/qe:check-style` puts the verb in the skill because `qe` names a domain; `/audit:issues` puts it in the plugin and leaves the skill as the object. Both read as imperatives, which is the only part that matters. There is no rule yet about which to prefer — three plugins is too few to know, so pick what reads best and let a convention emerge from use. +- **Description quality matters**: the SKILL.md frontmatter `description` is what natural-language invocation matches against. State what the skill does, what it measures, and when to use it. `validate.py` rejects descriptions too short to trigger reliably. +- **Report first, fix on request** (load-bearing) — skills never silently edit; anything `build_risk` or output-changing (RNG streams) is presented, never auto-applied. +- **Deterministic before LLM** (load-bearing, and the reason is that a reader has to be able to check a skill's output without re-running it): put what is mechanical in `scripts/` (checkable, testable, zero-false-positive bar); reserve the skill's judgement for what genuinely needs it. The discipline scales with what the skill outputs: + 1. *Every skill*: claims carry citations — rule ID + `file:line`, or a number + its source. A findings list needs nothing more; don't add ceremony to simple skills. + 2. *Skills that judge*: record judgement as discrete answers (true/false per criterion, each cited), not free prose — so it's checkable. + 3. *Skills that score*: when multiple judgements aggregate into a verdict with stakes, use the benchmark plugin's evidence-file pattern — judgement lives only in an evidence file, a deterministic engine computes every score, and no score is ever typed by hand. Aggregation is where hand-waving hides; the engine eliminates it. +- **Don't duplicate content across docs** — one canonical location, pointers elsewhere. Rule text, weights, and thresholds especially: restated copies drift. +- **Self-contained plugins** (load-bearing — this one is a hard constraint of how plugins install, not a preference): an installed plugin ships only its own directory. No relative links or paths that escape the plugin root; use absolute GitHub URLs for repo-level files, and anchor runtime paths for the installed context (issue #4 tracks the `${CLAUDE_PLUGIN_ROOT}` pattern). + +## Development loop + +```bash +# validate everything the marketplace serves +python scripts/validate.py +``` + +`validate.py` checks: every catalogue entry resolves to a real directory; `plugin.json` agrees with `marketplace.json` on name/version/description; every SKILL.md has frontmatter whose `name` matches its directory. Negative-test your changes (break something on purpose; the validator must fail loudly) — a malformed manifest breaks installation silently in every consuming repo. + +## Testing locally + +Two tiers, fastest first. Either way, **test from a real consuming project** (a lecture repo checkout), not from inside this repo — the whole class of path-resolution bugs (`${CLAUDE_PLUGIN_ROOT}`, workspace-vs-plugin working directories) only surfaces when the plugin runs read-only from an install location while the working directory is somewhere else. + +**Tier 1 — skill iteration, no install.** Load one plugin directly into a session: + +```bash +claude --plugin-dir /path/to/skills/ # e.g. .../skills/benchmark +``` + +Nothing is installed and no marketplace state is touched. Best while editing SKILL.md or scripts; restart the session to pick up changes. + +**Tier 2 — full install simulation, before merging to `main`.** Add your checkout as a local-path marketplace so you exercise exactly what users get (marketplace metadata, install, versioning, plugin-root resolution). In a Claude Code session in the consuming project: + +``` +/plugin marketplace add /path/to/your/skills-checkout +/plugin install benchmark@quantecon +``` + +Two things to know: + +- **The marketplace serves whatever your checkout has checked out.** To test a PR branch, leave the working tree on that branch for the duration of the test. +- **Local and GitHub sources share the marketplace name** (`quantecon`, from `marketplace.json`) and cannot coexist — if the production marketplace is already added, `/plugin marketplace remove quantecon` first. + +**Switching back to production** once the PR has merged: + +``` +/plugin marketplace remove quantecon +/plugin marketplace add QuantEcon/skills +/plugin install benchmark@quantecon +``` + +Confirm with `/plugin marketplace list` (the source should read `QuantEcon/skills`, not your local path) and `/plugin list` (the version should match the merged `plugin.json`). Routine setup and updating for end users is covered in [using-skills.md](using-skills.md). + +## Versioning and releases + +Bump the version in **both** `plugin.json` and the plugin's `marketplace.json` entry — the validator enforces they match. Scaffolding → first usable content is a minor bump (the benchmark plugin's evaluation-system landing was 0.1.0 → 0.2.0). + +## PR flow + +- Branch, PR, CI must be green. This repo **squash-merges** — stacked branches need `git rebase --onto origin/main ` after the base PR merges (already-upstream commits drop automatically). +- External contributions land with the contributor as git author (`--author`, GitHub noreply address unless they prefer otherwise) and integration fixes as separate commits — see PR #5 for the pattern. +- [CATALOG.md](../CATALOG.md) lists what has merged and nothing else, so a PR that adds a skill updates it and a PR that plans one does not. Work in flight belongs in the plugin's tracking issue ([#3](https://github.com/QuantEcon/skills/issues/3) `qe`, [#4](https://github.com/QuantEcon/skills/issues/4) `benchmark`, [#12](https://github.com/QuantEcon/skills/issues/12) `audit`); ideas nobody has committed to belong in [FUTURE-IDEAS.md](../FUTURE-IDEAS.md). The style-guide rule content is authored in `QuantEcon/style-guide`, never here — this repo's `qe` plugin consumes a rendered snapshot ([project-style-guide#6](https://github.com/QuantEcon/project-style-guide/issues/6)). diff --git a/docs/tutorial-run-an-evaluation.md b/docs/tutorial-run-an-evaluation.md new file mode 100644 index 0000000..e3e93d7 --- /dev/null +++ b/docs/tutorial-run-an-evaluation.md @@ -0,0 +1,111 @@ +# Tutorial: run a full evaluation by hand + +This walks the `/benchmark:review-acceleration` procedure end-to-end **by hand**, using the recorded [ge_arrow validation run](../reviews/validation-run-ge_arrow-2026-07-22.md) as the worked example — so every number you produce can be checked against a committed reference. When you invoke the skill, Claude drives these same steps for you; doing it manually once is the fastest way to understand what the skill measures, what the scorecard means, and how to debug a run that goes wrong. + +Canonical references (this tutorial points, never restates): the procedure in [SKILL.md](../benchmark/skills/review-acceleration/SKILL.md), the rubric in [EVALUATION_FRAMEWORK.md](../benchmark/references/EVALUATION_FRAMEWORK.md), the engine contract in [scripts/README.md](../benchmark/scripts/README.md). + +## What you need + +- The `benchmark` plugin installed (or this repo checked out — its `benchmark/` directory serves as the plugin root). +- A Python environment with `jax`, `numpy`, and the lecture's imports. The reference used jax 0.4.35; the validation run used jax **0.10.1** — the verdict reproduced anyway, which is the point of band-based scoring. +- A checkout of the lecture repo. Evaluations always compare two refs: a **baseline** (the lecture before the conversion) and a **candidate** (the conversion PR's head). + +## Step 0 — check out the exact states under evaluation + +Every committed evidence file records its provenance: `source_pr` plus base/head SHAs. For ge_arrow ([lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717)): + +```bash +git clone --filter=blob:none https://github.com/QuantEcon/lecture-python.myst +cd lecture-python.myst +git fetch origin update_ge_arrow:update_ge_arrow +git merge-base origin/main update_ge_arrow # → 8cfba4c = the state prior to the PR +``` + +The merge-base is the "world before the PR" — that's the baseline. The PR branch head (`8c2d0d7`) is the candidate. + +## Step 1 — scaffold the workspace + +Evaluations live in **your workspace**, never inside the plugin (which is read-only when installed): + +```bash +export CLAUDE_PLUGIN_ROOT=/path/to/plugin/benchmark # or the installed plugin root +mkdir -p benchmark-eval/ge_arrow/scripts +cp $CLAUDE_PLUGIN_ROOT/references/examples/ge_arrow/scripts/*.py benchmark-eval/ge_arrow/scripts/ +cp $CLAUDE_PLUGIN_ROOT/scripts/scoring/EVIDENCE_TEMPLATE.json benchmark-eval/ge_arrow/evidence.json +``` + +Because we are *reproducing* the ge_arrow evaluation, we copy its already-adapted scripts. For a **new** lecture you adapt them — extract `model_old.py` from the lecture at the baseline ref and `model_new.py` at the candidate ref **verbatim** (disclose any deviation), and rewrite the measurement scripts around the lecture's actual examples and call sequence. That adaptation is the skill's real work; there is deliberately no rigid harness. Either way, before measuring, diff your extractions against the lecture's cells — the validation run did exactly this and caught an undisclosed whitespace normalisation in the committed baseline extraction. + +## Step 2 — measure + +```bash +conda run -n quantecon python benchmark-eval/ge_arrow/scripts/run_all.py +``` + +`run_all.py` runs every measurement and aggregates results into `benchmark-eval/ge_arrow/results/`. The headline is the **as-used benchmark**: the lecture's real call sequence, at its real sizes, in a fresh interpreter so JIT compile time counts — repeated 3× per side, median taken. From the validation run: + +``` +== As-used total (numpy) == +{"mode": "numpy", "total_s": 0.0272} {"total_s": 0.0440} {"total_s": 0.0292} +== As-used total (jax) == +{"mode": "jax", "total_s": 1.1947} {"total_s": 1.2470} {"total_s": 1.1603} +``` + +→ `results/as_used.json` records both `runs` lists, the medians, per-run speedups, and `baseline_as_used_seconds` (0.0292 s here). A provenance stamp (`results/env.json`) records the environment and any failed steps. + +### The two precision regimes + +Correctness is measured **twice**, and the two runs answer different questions: + +- **As shipped (float32).** JAX computes in float32 by default, NumPy in float64 — so this run measures what a *reader actually experiences*: how far the published numbers drift. It drives the correctness Δ-bands (validation run: worst max|Δ| = 1.01e-4 → correctness 3). +- **Under x64 (`JAX_ENABLE_X64=1`).** This flips JAX to float64, putting both implementations at the *same* precision. Any remaining divergence cannot be rounding — it means the two implementations compute **different economics**, which forces correctness 1 and the logic-design bug cap. Agreement here (validation run: 1.42e-13) proves the drift in the first run is purely precision, not logic. + +```bash +cd benchmark-eval/ge_arrow/scripts +JAX_ENABLE_X64=1 python check_equivalence.py # writes results/equivalence_x64.json +``` + +The script writes one file **per regime** (`equivalence.json` / `equivalence_x64.json`) so the second run can't clobber the first — a fix that came out of the validation run, which found the x64 rerun silently overwriting the as-shipped results. + +## Step 3 — record evidence + +Fill `benchmark-eval/ge_arrow/evidence.json` from `results/`: measured numbers into the quantitative slots (each with its source file named), and each structural checklist item answered true/false **with a citation to the diff**. This file is the only place judgement lives — for a reproduction run the structural answers carry over unchanged (same diff), and only the measured quantities update. + +## Step 4 — score + +```bash +python $CLAUDE_PLUGIN_ROOT/scripts/scoring/score.py benchmark-eval/ge_arrow +``` + +No score is ever typed by hand — the engine computes all seven dimensions and prints the derivation of each. The validation run's tail: + +``` +WEIGHTED TOTAL 2.85 +VERDICT: no-conversion — the baseline as-used total 0.0292 s is under the 1 s +materiality floor and the candidate is slower as-used (0.0251×): this lecture +should not be converted, whatever the candidate's polish. Candidate quality +for the record: 2.85/5, mixed/wash +SENSITIVITY: fragile (29 single-input perturbations scored) + └ quantitative.correctness.builds: True → False ⇒ total 2.30, ... + └ quantitative.correctness.matches_under_x64: True → False ⇒ total 2.30, ... + └ structural.logic_design.criteria.good_algorithmic_choices: False → True ⇒ total 3.00, ... +``` + +Three things to read off a v2 scorecard beyond the total: the **verdict gate** (broken correctness caps the band regardless of polish), the **no-conversion** verdict (a lecture with nothing to gain shouldn't be converted, however good the candidate), and the **sensitivity stamp** (would any single contestable input flip the outcome? here: yes, three would — the scorecard says so instead of hiding it). + +## Step 5 — cross-compare + +Raw numbers are machine- and version-dependent; **bands are the reproducibility contract**. What must match the reference, and what may drift: + +| Quantity | Reference | Validation run | Contract | +|---|---|---|---| +| float32 worst max\|Δ\| | 1.7e-4 | 1.01e-4 | same Δ-band (→ correctness 3) | +| as-used speedup | 0.022× | 0.0251× (median of 3) | same efficiency band (→ 2) | +| baseline total | 0.035 s | 0.0292 s | same side of the 1 s floor | +| **Total / verdict / stamp** | 2.85, no-conversion, fragile | 2.85, no-conversion, fragile (same 3 flips) | **exact** | + +If your bands move, something real changed — check `results/env.json` first, then the extraction diff from Step 1. + +## Step 6 — report + +Write `_REPORT.md` from the scorecard + evidence following the worked examples' format ([ge_arrow](../benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md), [markov_asset](../benchmark/references/examples/markov_asset/markov_asset_REPORT.md)): TL;DR with the full verdict, dimension table, evidence per dimension, and a must-fix list. For the validation run the "report" is the [cross-comparison record](../reviews/validation-run-ge_arrow-2026-07-22.md) itself. diff --git a/docs/using-skills.md b/docs/using-skills.md new file mode 100644 index 0000000..dd571f1 --- /dev/null +++ b/docs/using-skills.md @@ -0,0 +1,46 @@ +# Using QuantEcon skills + +For lecture authors, reviewers, and maintainers who want to *use* the skills. (Building new ones: [developing-skills.md](developing-skills.md).) + +## Setup + +**In a lecture repo that has opted in** — nothing to do. Repos that check the marketplace into `.claude/settings.json` (see the [repo README](../README.md)) install everything automatically when you open the repo in Claude Code and trust it. + +**Anywhere else** — three commands in a Claude Code session (the marketplace first, then the plugins you want): + +``` +/plugin marketplace add QuantEcon/skills +/plugin install qe@quantecon # author-facing base skills +/plugin install benchmark@quantecon # lecture-evaluation tooling +``` + +**In CI** — `anthropics/claude-code-action@v1` accepts `plugin_marketplaces` and `plugins` inputs directly; see the repo README for the workflow snippet. + +## Invoking a skill + +Three ways, all equivalent: + +1. **Slash command** — type `/` and pick from the menu, e.g. `/benchmark:review-acceleration 717`. Trailing words are passed to the skill as arguments. +2. **Natural language** — describe what you want ("check this lecture's figures against the style guide"; "is this JAX conversion actually an improvement?") and the matching skill triggers on its description. +3. **Category entry points** — some plugins expose thin sub-skills (`/qe:check-figures`, `/qe:check-math`, …) so a narrow check is one keystroke and shows up in autocomplete. + +## What to expect + +- **Report first, fix on request.** Skills produce a structured report and *offer* changes; they never silently edit your files. Risky fixes (anything that breaks builds or changes published figures, e.g. RNG-stream changes) are presented but never auto-applied. +- **Evidence, not vibes.** Reports cite rule IDs, `file:line` locations, and measured numbers. The benchmark plugin goes further: its scores are computed by a deterministic engine from recorded evidence — the session shows the full derivation. +- **The same skill works pre-PR and in review.** Run it on your working copy before opening a PR (catch issues early), or point it at an open PR (consistent review). + +## The plugins + +| Plugin | Skills | What they do | Status | +|---|---|---|---| +| `qe` | `/qe:check-style` + `check-{writing,math,code,figures,jax,refs}` | Style-guide compliance for lecture source, by rule ID | scaffolding — [skills#3](https://github.com/QuantEcon/skills/issues/3) | +| `benchmark` | `/benchmark:review-acceleration` | Score a NumPy→JAX/Numba conversion (review mode) or assess whether a lecture is worth converting (triage mode) | system landed — [guide](../benchmark/README.md), [skills#4](https://github.com/QuantEcon/skills/issues/4) | + +## Updating and troubleshooting + +- **Update**: `/plugin` → marketplace → update, or reinstall; repos with the settings.json opt-in track the marketplace automatically. +- **Skill not in the menu?** Check the plugin is installed and enabled (`/plugin`), and that you trusted the repo. In settings-managed repos, `enabledPlugins` must list it. +- **`Unknown command: /benchmark:review-acceleration`?** The plugin-prefixed slash form needs a recent Claude Code (v2.1.216+; check with `claude --version`). On older versions the skill still registers under the bare `/review-acceleration`, and **natural-language invocation works on any version** — just describe the task ("is this JAX conversion worth merging?"). If it resolves under none of these, the install didn't complete — re-run `/plugin install benchmark@quantecon`. +- **A skill reports "not yet operational"** — it's scaffolding; its issue link says what's pending. +- **Version pinning**: plugin versions live in the marketplace catalogue; CI validates that every manifest is consistent, so a broken install is a bug — please open an issue. diff --git a/qe/skills/check-code/SKILL.md b/qe/skills/check-code/SKILL.md index 00d3344..08aa5ba 100644 --- a/qe/skills/check-code/SKILL.md +++ b/qe/skills/check-code/SKILL.md @@ -5,7 +5,7 @@ description: Check a QuantEcon lecture's code cells against the style guide — # check-code -> **Status: scaffolding.** Rule content lands in follow-up PRs; see [CATALOG.md](https://github.com/QuantEcon/skills/blob/main/CATALOG.md). Until then this skill reports that it is not yet operational. +> **Status: scaffolding.** Rule content lands in follow-up PRs; see [issue #3](https://github.com/QuantEcon/skills/issues/3). Until then this skill reports that it is not yet operational. Category entry point for the `code` rules — code style and library idiom. diff --git a/qe/skills/check-figures/SKILL.md b/qe/skills/check-figures/SKILL.md index 2ac974c..deb4ef8 100644 --- a/qe/skills/check-figures/SKILL.md +++ b/qe/skills/check-figures/SKILL.md @@ -5,7 +5,7 @@ description: Check a QuantEcon lecture's figures against the style guide — mys # check-figures -> **Status: scaffolding.** Rule content lands in follow-up PRs; see [CATALOG.md](https://github.com/QuantEcon/skills/blob/main/CATALOG.md). Until then this skill reports that it is not yet operational. +> **Status: scaffolding.** Rule content lands in follow-up PRs; see [issue #3](https://github.com/QuantEcon/skills/issues/3). Until then this skill reports that it is not yet operational. Category entry point for the `figures` rules — figure and plotting conventions. diff --git a/qe/skills/check-jax/SKILL.md b/qe/skills/check-jax/SKILL.md index 9d2c2fd..4cd4a62 100644 --- a/qe/skills/check-jax/SKILL.md +++ b/qe/skills/check-jax/SKILL.md @@ -5,7 +5,7 @@ description: Check a QuantEcon lecture's JAX usage against the style guide — G # check-jax -> **Status: scaffolding.** Rule content lands in follow-up PRs; see [CATALOG.md](https://github.com/QuantEcon/skills/blob/main/CATALOG.md). Until then this skill reports that it is not yet operational. +> **Status: scaffolding.** Rule content lands in follow-up PRs; see [issue #3](https://github.com/QuantEcon/skills/issues/3). Until then this skill reports that it is not yet operational. Category entry point for the `jax` rules — JAX conventions and anti-patterns. diff --git a/qe/skills/check-math/SKILL.md b/qe/skills/check-math/SKILL.md index 6d6ce6a..2666b11 100644 --- a/qe/skills/check-math/SKILL.md +++ b/qe/skills/check-math/SKILL.md @@ -5,7 +5,7 @@ description: Check a QuantEcon lecture's mathematical notation against the style # check-math -> **Status: scaffolding.** Rule content lands in follow-up PRs; see [CATALOG.md](https://github.com/QuantEcon/skills/blob/main/CATALOG.md). Until then this skill reports that it is not yet operational. +> **Status: scaffolding.** Rule content lands in follow-up PRs; see [issue #3](https://github.com/QuantEcon/skills/issues/3). Until then this skill reports that it is not yet operational. Category entry point for the `math` rules — mathematical notation conventions. diff --git a/qe/skills/check-refs/SKILL.md b/qe/skills/check-refs/SKILL.md index de530e0..2f9eba6 100644 --- a/qe/skills/check-refs/SKILL.md +++ b/qe/skills/check-refs/SKILL.md @@ -5,7 +5,7 @@ description: Check a QuantEcon lecture's citations and cross-document links agai # check-refs -> **Status: scaffolding.** Rule content lands in follow-up PRs; see [CATALOG.md](https://github.com/QuantEcon/skills/blob/main/CATALOG.md). Until then this skill reports that it is not yet operational. +> **Status: scaffolding.** Rule content lands in follow-up PRs; see [issue #3](https://github.com/QuantEcon/skills/issues/3). Until then this skill reports that it is not yet operational. Category entry point for the `refs` rules — citations and cross-document links. diff --git a/qe/skills/check-style/SKILL.md b/qe/skills/check-style/SKILL.md index d64b9d0..cf9cb1f 100644 --- a/qe/skills/check-style/SKILL.md +++ b/qe/skills/check-style/SKILL.md @@ -5,7 +5,7 @@ description: Check a QuantEcon lecture against the QuantEcon style guide and rep # check-style -> **Status: scaffolding.** The rule content (`references/rules/`) and deterministic preflight scripts (`scripts/`) land in follow-up PRs, tracked in [CATALOG.md](https://github.com/QuantEcon/skills/blob/main/CATALOG.md) and the work plan in `QuantEcon/project-style-guide`. Until they land this skill reports that it is not yet operational. +> **Status: scaffolding.** The rule content (`references/rules/`) and deterministic preflight scripts (`scripts/`) land in follow-up PRs, tracked in [issue #3](https://github.com/QuantEcon/skills/issues/3) and the work plan in `QuantEcon/project-style-guide`. Until they land this skill reports that it is not yet operational. Umbrella style check for one lecture. Runs every category, or only the categories named in the arguments. diff --git a/qe/skills/check-writing/SKILL.md b/qe/skills/check-writing/SKILL.md index c4a990f..6c4390b 100644 --- a/qe/skills/check-writing/SKILL.md +++ b/qe/skills/check-writing/SKILL.md @@ -5,7 +5,7 @@ description: Check a QuantEcon lecture's prose against the style guide's writing # check-writing -> **Status: scaffolding.** Rule content lands in follow-up PRs; see [CATALOG.md](https://github.com/QuantEcon/skills/blob/main/CATALOG.md). Until then this skill reports that it is not yet operational. +> **Status: scaffolding.** Rule content lands in follow-up PRs; see [issue #3](https://github.com/QuantEcon/skills/issues/3). Until then this skill reports that it is not yet operational. Category entry point for the `writing` rules — prose style and structure. diff --git a/reviews/benchmark-design-2026-07-21-independent.md b/reviews/benchmark-design-2026-07-21-independent.md new file mode 100644 index 0000000..954df3f --- /dev/null +++ b/reviews/benchmark-design-2026-07-21-independent.md @@ -0,0 +1,159 @@ +# First-principles design review — `benchmark/` lecture-evaluation system + +**Date:** 2026-07-21 +**Branch reviewed:** `docs-skills-usage` +**Scope:** the full evaluation system in `benchmark/` — rubric dimensions and weights, metrics, aggregation and verdict method, thresholds and calibration, measurement architecture. Everything treated as open to revision; the goal is the best possible evaluation system for QuantEcon, where lectures are teaching materials first. +**Method:** read every design document ([README.md](benchmark/README.md), [EVALUATION_FRAMEWORK.md](benchmark/references/EVALUATION_FRAMEWORK.md), [scripts/README.md](benchmark/scripts/README.md), [SKILL.md](benchmark/skills/review-acceleration/SKILL.md), [examples README](benchmark/references/examples/README.md)), the scoring engine ([rubric.py](benchmark/scripts/scoring/rubric.py), [score.py](benchmark/scripts/scoring/score.py)), both worked evaluations (evidence, results, reports), and the measurement scripts; then ran the rubric against synthetic edge cases to test the aggregation empirically. The numbered edge-case results below (A–D) were produced by executing `rubric.score_all` directly on constructed evidence. + +## Bottom line + +The system's core measurement doctrine — **as-used, fresh-process, compile-time-counted** — is genuinely right and well-executed, but the scoring layer on top of it has structural flaws that can produce wrong verdicts. Three were confirmed empirically: + +- A lecture that **does not build** can score 3.9–4.2 → "merge" territory (compensatory aggregation, no gates). +- A **no-op conversion** (candidate byte-identical to baseline) scores 3.40 → "net positive" (the scale's zero is displaced). +- Moving a **single hand-counted concept** flips markov_asset's verdict band (2.25 "net regression" → 2.50 "wash"). + +The critique is ranked by decision impact; each item carries evidence and a recommended replacement design. + +--- + +## What the system gets right (keep these) + +- **The as-used doctrine.** Fresh process, real call sequence, real sizes, compile time counted, warm numbers never the headline. The ge_arrow case (1.4–4.8× faster warm, 45× slower as-used) proves this rule earns its keep. This is the system's central insight and should survive any redesign. +- **Evidence → score determinism.** Scores computed, never typed; every score carries its derivation; scorecards reproduce byte-identically; the 5⁷ brute-force band-edge audit found and fixed real FP bugs. This auditability discipline is rare and valuable. +- **Verbatim extraction with disclosed deviations**, including keeping markov_asset's build-breaking bug in `model_new.py`. +- **The caveat register** (M1/m3/n6 in the examples README) is unusually honest — M1 already anticipates part of finding 4. +- **Triage's bounding logic** — "the baseline as-used total bounds the entire possible win" — is elegant, cheap, and correct. + +--- + +## 1. The verdict is compensatory with no gates — a lecture that doesn't build can be told "merge" + +The weighted mean lets any dimension buy off any other. Running the rubric on a candidate with `builds: false` and strong scores elsewhere: + +| Synthetic case | Total | Verdict (`rubric.py:75-82`) | +|---|--:|---| +| Does not build, all else max, bug-override set | **3.90** | "net positive with fixable regressions — merge after addressing them" | +| Same, evaluator forgets the override flag | **4.20** | "clear improvement — merge" | +| Every dimension at its "wash" anchor (all 3s) | **3.00** | "net positive with fixable regressions" | + +markov_asset landed at 2.25 only because its *other* dimensions were also weak — the rubric has no mechanism guaranteeing that outcome. Note also that the build-breaking bug must be recorded twice (`builds: false` in correctness *and* `introduces_correctness_bug: true` in logic_design) with nothing enforcing consistency; forgetting the duplicate is worth +0.30 and a band. + +The all-3s case exposes a band mislabeling: every dimension's 3 is defined as "wash" (efficiency 0.8–1.3× is literally labeled "wash" in the anchors), yet a total of 3.0 maps to "net positive — merge after addressing." The wash band (2.5–2.9) sits *below* the scale's center. The ge_arrow report feels this: it calls 2.85 "net mixed, *slightly negative*" while the band says "mixed/wash." + +**Recommended design.** Non-compensatory gates checked *before* any weighted total: + +- **G1 — executes end-to-end** on the shipped configuration. Fail → "do not merge," full stop, whatever the total. +- **G2 — published numbers preserved** (within tolerance under the shipped dtype config) *or* the change is explicitly flagged and justified in the PR. Silent changes to printed numbers fail the gate — this is the framework's own stated principle ("silently changes the numbers"), which the current scoring only *discounts* rather than blocks. + +Then re-center the bands so 3.0 is a wash, and make band labels *descriptive* ("net improvement" / "wash" / "net regression") rather than imperative ("merge after addressing them" asserts fixability the score cannot know — the must-fix list is where fixability belongs). + +## 2. The scale's zero is displaced — a no-op conversion scores 3.40 "net positive" + +The verdict claims to answer "did this change improve the lecture?" but roughly 40% of the weight scores the *candidate in absolute terms*: the four structural checklists (0.45 combined weight) ask whether the new code is pure, global-free, idiomatic, testable; the docstring column of readability is `docstring_cov_new`, an absolute; ergonomics counts the candidate's statements. + +The identity test — candidate byte-identical to ge_arrow's baseline, evidence filled per the committed conventions — yields **3.40, "net positive with fixable regressions — merge after addressing them"** for a PR that changes nothing. + +The converse also holds: converting an already-pristine baseline (lake_model-style) inherits structural 4s–5s from virtues the baseline already had, and `fixes_prior_bugs` means a conversion of a *clean* lecture caps logic at 4 — the candidate is punished for the baseline's quality. The checklists even mix framings internally: `fixes_prior_bugs` is a delta, `pure_no_order_dependence` is an absolute. + +**Recommended design.** Score **both implementations on the same absolute anchors** and derive the verdict from the delta profile. The machinery barely changes: `static_metrics.py` already measures both sides; the checklists just get answered twice. Output becomes two absolute scorecards plus a per-dimension delta table; the verdict comes from the weighted delta (identity = 0.0 by construction) with the gates from finding 1. The dual scorecards are independently useful for QuantEcon: "the baseline itself scores 2.4 — the fix is a rewrite, not necessarily a JAX rewrite" is exactly the recommendation the ge_arrow report reached by hand. + +## 3. Measurement architecture: the system measures a hand-built reconstruction, not the lecture + +Each evaluation hand-adapts `as_used_total.py` to re-enact the lecture's call sequence. This puts the headline metric at the mercy of adaptation choices: the ge_arrow script implements the λ-sweep as a 100-iteration Python loop on the NumPy side and one jitted `fori_loop` on the JAX side (`as_used_total.py:50-54` vs `:89-99`) — defensible (caveat n6 documents it), but it's an evaluator's judgement sitting inside the decisive number, and nothing verifies the reconstruction against the lecture. markov_asset needed a hand-patched copy just to produce a timing. And the whole thing is timed **once** — a single pass per side feeding threshold cliffs at 0.8×/1.3×/3×. + +Beyond that: the deliverable is the *lecture* (prose + code + outputs), but only extracted code is evaluated — markov_asset's prose edits went unassessed, and prose is where pedagogy actually lives. + +**Recommended design.** Measure the lecture itself. The lectures are executable MyST documents: for each branch, convert with jupytext and execute with nbclient in a fresh kernel, recording **per-cell wall time** (repeat K≥5, take medians). As-used time is then the real reader/CI wait, by definition — no reconstruction, no asymmetry class, no per-lecture timing scripts. Equivalence becomes an **executed-output diff**: compare the printed numbers between branches numerically — which is *directly* "were the published numbers preserved?" (gate G2), and it catches build breaks natively (markov_asset's `NameError` fails execution — no `smoke_test.py` needed). The current micro-benchmarks (crossover-n, recompile audit, cold-start) remain valuable as *diagnosis* feeding the must-fix list, not as verdict inputs. This is also the only architecture that scales across ~200 QuantEcon lectures, and it makes triage nearly free (baseline-side timing = execute the current lecture). + +## 4. Readability — the heaviest weight rides on the weakest instrument + +`n_prerequisite_concepts` drives the 0.25-weight dimension, and it is a hand-curated list embedded in each lecture's script (`static_metrics.py:41-52`) — caveat M1 admits this. The counting granularity is arbitrary ("jax.jit & tracing" and "static_argnames & recompilation" are two concepts; "NumPy arrays & slicing" is one), and the verdict is exquisitely sensitive to it: **recounting markov_asset's +5 as +4 — one concept — moves the total from 2.25 ("net regression — do not merge") to 2.50 ("mixed/wash")**, verified against the committed evidence. + +Further problems: the delta treats concepts as fungible (7 OOP concepts out, 7 JAX concepts in = +0 = band 5, though the audience already knows the former from the series and none of the latter); `docstring_cov_new` is absolute, not a delta, and is a weak proxy for lecture code whose real documentation is the surrounding prose; and the "math-to-code distance" tie-breaker promised in EVALUATION_FRAMEWORK.md §2 never appears in `score_readability` (`rubric.py:104-111`). + +**Recommended design.** Curate the concept inventory **once at series level, not per lecture**: a versioned `concepts.yml` mapping detection patterns → canonical concepts → the lecture where the series first teaches each. The existing `CONCEPTS` regex dict shows most JAX concepts have syntactic signatures (`jax.jit`, `static_argnames`, `.at[].set`, `lax.fori_loop`) — the fragile step was per-lecture deduplication into "ideas," which a shared map eliminates. The metric becomes mechanical: concepts used by the candidate that the series has *not taught at or before this lecture*, minus the same for baseline. Pair it with a prose check — for each new concept, does the PR's added prose explain it near first use? (A concept explained in prose is pedagogy; one that appears bare is burden.) Judgement that remains moves into `evidence.json` as cited slots — the M1 fix already proposed on skills PR #5, endorsed and extended here. + +## 5. Efficiency: ratio-only scoring with a saturating floor and no materiality test + +Three problems in `score_efficiency` (`rubric.py:114-124`): + +1. Everything below 0.8× collapses to one band: 0.75× slower and 45× slower both score 2. +2. `correct_or_fixable` is near-vacuous (all code is "fixable"; both committed cases say so), so score 1 is practically unreachable — it's a 4-point scale in disguise. +3. Most important, **the ratio ignores absolute materiality**: ge_arrow's "45× slower" is 0.035 s → 1.56 s — 1.5 wall-clock seconds a reader would never notice. Triage mode already knows this ("a lecture whose compute totals 30 ms has nothing to give") but review mode doesn't: it hands ge_arrow a −2-band penalty for an imperceptible cost, while the actual crimes (readability, precision) are elsewhere. Ratios of tiny denominators are also noise-dominated — a single unrepeated pass deciding a banded score. + +**Recommended design.** Two-dimensional scoring: a **materiality zone** first (|Δtotal| below a threshold — say 2 s of reader/CI wait — is automatically a 3/wash, whatever the ratio), then **log-symmetric ratio bands** outside it, so 3× faster and 3× slower are equidistant from wash. Under this, ge_arrow's efficiency is a 3 (correct: the case against it is pedagogy, not seconds), aiyagari's −52 s is a decisive 5, and the headline stops sounding catastrophic ("45×!") for immaterial stakes. Require K fresh-process repeats with the band assignment stable across the spread before it's recorded. + +## 6. Calibration is two in-sample points, and triage's validation is circular + +The thresholds claim calibration "against two measured end points" — 25× and 0.022×. Two points pin the two extreme bands; every interior boundary (0.8, 1.3, 3× for efficiency; +1–2/+3–4 for concepts; the docstring cuts) has **zero observed cases** — both real evaluations landed in band 2 of readability and band 2 of efficiency. + +The triage validation table in the plugin README "reproduces every known verdict" on exactly the three cases the thresholds were built from — in-sample prediction presented as validation. + +The HIGH anchor's provenance is inconsistent: EVALUATION_FRAMEWORK.md (§2, dimension 3) cites 1664 ms / 29.3 s while the committed `bellman_bench.json` says 2955 ms / 54.3 s — same ~24× ratio, visibly different run, in a system whose brand is citation fidelity. + +**Recommended design.** Label all interior thresholds *provisional* in the framework; pre-register a recalibration protocol (after every N evaluations, re-fit band edges against the accumulated evidence files, version the rubric, re-run the regression anchors); treat the next several real evaluations as out-of-sample tests of triage and report hits/misses. Reconcile the anchor numbers now — either re-run and update the doc, or cite the JSON. + +## 7. Cross-dimension double counting quietly rewrites the weights + +Single root causes score in multiple dimensions: + +- **float32** hits correctness (the entire band-3 rung), maintainability (`dtype_precision_safe`), and readability (the "float32/x64 flag" concept). +- **Purity** appears in logic (`pure_no_order_dependence`), maintainability (`pure_unit_testable`), and ergonomics (`fragile_protocol`). +- **Vectorisation** appears in logic (`good_algorithmic_choices`), style (`vectorised_where_natural` *and* `correct_control_flow_primitive`), efficiency, and readability (the `fori_loop`/carry concepts). + +So the effective weight of these facts exceeds any nominal number, and the headline claim "readability (0.25) outranks efficiency (0.15)" isn't reliably true of the system's actual behavior. Seven dimensions also carry overlap costs: maintainability at 0.05 can swing the total by at most 0.20 — ceremony without leverage. + +**Recommended design.** Either assign each observable to exactly one home dimension and publish the assignment, or — better — consolidate to four orthogonal dimensions re-derived from the teaching-first principle: + +| Dimension | Weight (proposed) | Contents | +|---|:--:|---| +| Fidelity | ~0.25 | gates + precision policy | +| Pedagogy | ~0.40 | concept burden, math-to-code, prose explanation, exercise integrity | +| Cost | ~0.15 | as-used time with materiality, dependency/install burden | +| Code quality | ~0.20 | purity / idiom / API / maintainability merged | + +Fewer, cleaner dimensions make the weights mean what they say. + +## 8. No reliability engineering around the human/AI judgement slots + +The system's motto — "no score is ever typed by hand" — is true but subtly overstated: the *scores* are computed, but the checklist booleans, `fragile_protocol`, `correct_or_fixable`, concept lists, and `statements_for_one_result` are hand-typed judgements that map deterministically to scores. Determinism relocated the subjectivity; it didn't remove it. There is no inter-rater data — no evidence that two independent evaluators (or two AI-skill runs) fill `evidence.json` the same way, on an instrument where finding 4 shows one boolean or one concept can flip a verdict. + +**Recommended design.** Since the skill automates the fill, reliability testing is nearly free: run the evidence-fill twice in independent sessions, diff the judgement slots, and surface disagreements for human adjudication rather than silently keeping one. Track agreement rates across evaluations — that number, not the determinism claim, is what makes the structural dimensions trustworthy. + +## 9. Smaller defects (fix opportunistically) + +- `score_correctness` (`rubric.py:92`): the x64-divergence guard fires only when `d > 1e-8` — divergent logic with small shipped drift slips into bands 4–5. Also the band-3 reason string hardcodes "ships float32" as the explanation regardless of actual cause. +- Correctness thresholds are **absolute** `max|Δ|`; for lectures whose published objects are large-magnitude, 1e-3 absolute may be ~1e-6 relative and still scores 2. Use relative error (or per-object normalization). +- `score_ergonomics` (`rubric.py:129`) gives base 2 for any n≥4, but the prose anchor reserves 2 for "ordered, side-effecting" protocols — code and prose disagree. +- Dependency cost is unmeasured: adding `jax`/`jaxlib` to a lecture is a real installability burden for students (platform wheels, Windows) and belongs in the Cost dimension. +- Hardware policy should be pinned in the framework: "as-used" = the target series' actual build environment (CPU runners for lecture-python.myst, GPU for lecture-jax), recorded by `env_stamp.py`. A conversion verdict is environment-relative and should say so. +- Anchors were measured on jax 0.4.35 (old by now); compile costs and defaults drift across JAX releases. The recalibration trigger lives only in a script docstring — promote it to the framework. + +--- + +## Summary + +The measurement doctrine (as-used, fresh-process) and the auditability discipline are the right foundation — keep both. The redesign priorities: + +1. **Gates before any weighted total** (builds; published numbers preserved), with re-centered, descriptive bands. +2. **Symmetric dual scoring** so the verdict measures the *change*, with identity = 0. +3. **Execute the actual lecture on both branches** (per-cell timing + output diff) in place of hand-adapted replay scripts — simultaneously fixes gate G2, kills the reconstruction-fidelity risk, and scales to the whole series. +4. **Series-level concept inventory** with mechanical detection in place of per-lecture hand lists. +5. **Materiality zone plus log-symmetric bands** for efficiency. + +Items 1, 2, and 5 are pure `rubric.py` changes that could land quickly and be validated against the two committed evidence files; item 3 is the one real piece of new engineering; item 4 is a natural companion to the QuantEcon.manual style-page work already tracked in QuantEcon.manual#104. + +--- + +## Appendix — empirical verification runs + +All produced by calling `rubric.score_all` directly (engine at `benchmark/scripts/scoring/rubric.py`, evidence conventions as in the two committed `evidence.json` files): + +| Case | Construction | Total | Verdict | +|---|---|--:|---| +| A | No-op conversion: candidate ≡ ge_arrow baseline; quantitative slots take the baseline's own measured values (Δ=0, speedup 1.0×, docstrings 0.90, 4 ordered statements); checklists describe the baseline code (order-dependent methods, module globals, typo) | **3.40** | net positive with fixable regressions — merge after addressing them | +| B | `builds: false`, every other slot maximal, `introduces_correctness_bug: true` (logic capped at 3) | **3.90** | net positive with fixable regressions — merge after addressing them | +| B2 | Same as B but the duplicate override flag left false | **4.20** | clear improvement — merge | +| C | Every dimension at its "wash" anchor (score 3 on all seven) | **3.00** | net positive with fixable regressions — merge after addressing them | +| D | markov_asset committed evidence with `delta_prereq_concepts` 5 → 4 (one fewer hand-counted concept; docstrings 0.75 → band 4, so the concept column pins the readability score) | 2.25 → **2.50** | net regression → mixed / wash | diff --git a/reviews/benchmark-design-2026-07-21-merged.md b/reviews/benchmark-design-2026-07-21-merged.md new file mode 100644 index 0000000..b283cf1 --- /dev/null +++ b/reviews/benchmark-design-2026-07-21-merged.md @@ -0,0 +1,73 @@ +# Benchmark evaluation system — merged design review (2026-07-21) + +Synthesis of two independent design critiques of the evaluation system, run deliberately in isolation from each other as a bias control: + +- **Review A** ([benchmark-design-2026-07-21-independent.md](benchmark-design-2026-07-21-independent.md)) — a fresh session with an unframed prompt; 9 findings, empirically verified against the engine. +- **Review B** — a 36-agent adversarial workflow: six critics attacking from independent angles, every non-minor critique then passed to a **steelman defender** instructed to save the original design; only critiques surviving the strongest defense are reported. Two critiques survived outright (CRITIQUE_STANDS); most were PARTIALLY_DEFENDED — the residuals below are what remains after the best defense. + +Findings present in **both** independent runs are the most robust. All demonstrations referenced here were produced by executing `rubric.py`/the notebooks, not by inspection alone. The scope was the *design*; the system's architecture (evidence → deterministic engine, verbatim extraction, the as-used doctrine) was validated previously and both reviews independently concluded it should survive any redesign. + +--- + +## 1. Corrections of record (already applied) + +The review process falsified three claims our own documents made — corrected via erratum and rewording on this branch. (Evaluation findings briefly posted to [lecture-python.myst#654](https://github.com/QuantEcon/lecture-python.myst/pull/654) were withdrawn; that PR will receive one authoritative evaluation after the v2 revision, rather than a comment-and-correction trail.) + +1. **"markov_asset's lecture does not build as shipped" — false as worded.** Executing all cells of the PR branch's notebook in order completes cleanly (preview CI passes): earlier cells bind a global `err` that the stray `err.throw()` inside `call_option` silently resolves to. The true finding is *subtler and worse*: the stale-global masking means **the checkify stability validation never actually runs in the shipped lecture**, on the model whose spectral radius sits 0.002 below the stability bound; a reader copying the function into a clean namespace hits the `NameError`. `builds: false` remains a true measurement under the system's declared fresh-process regime; the sentence about the lecture was wrong. +2. **"The replayed sequence mirrors the lecture exactly" — false for both reference cases.** ge_arrow's replay constructs 12 model objects where the lecture constructs 6 and reuses them across initial states; markov_asset's replay builds a fresh model per γ where the lecture mutates one object, and omits two calls. The reconstruction-fidelity risk the review process flagged abstractly had already occurred, undetected. +3. **"Medians over repeats" — false for the headline metric.** The as-used totals are single passes per side; only the warm/scaling benchmarks use medians. The one metric that solely decides a 0.15-weight dimension is the least-replicated measurement in the system. + +## 2. Findings that survive the steelman defense + +### 2.1 The verdict's safety couplings exist only by convention (both reviews; residual after defense) + +The defense established something the critiques missed: **with the documented evidence convention followed** (build failure recorded in both `builds` and the logic-design bug flag), a non-building lecture's ceiling is exactly **3.90 < 4.0** — the two overrides jointly form a designed soft gate on the unconditional-merge band. But that coupling is enforced by reviewer discipline, not code: one forgotten boolean yields **4.20 "clear improvement — merge" for a lecture that crashes** — in a system whose stated contract is that scores are deterministic functions of evidence. Worse, Review B closed the "requires dishonest evidence" escape: `builds: true`, `matches_under_x64: true`, `max|Δ| > 1e-1` (a float32 catastrophe with no logic bug) reaches **4.2 with fully honest evidence**. + +### 2.2 The readability instrument inverts its own ground truth (CRITIQUE_STANDS) + +`score_readability` = worse-of(Δprereq-concepts, docstring coverage). Executed against the framework's own labeled exemplars: **odu.py — the framework's LOW-readability example — measures 0.86 docstring coverage**, and code written in the flagship aiyagari style (inline shape comments, undocumented closures/NamedTuples) measures ~0.41–0.55 and is **mechanically capped at readability 2** regardless of concept count. The metric anti-correlates with the framework's own judgements at both ends, and the construct the prose says defines the dimension ("math-to-code distance") has no encoding in the scorer at all. + +### 2.3 Review mode and triage mode contradict each other (residual after defense) + +A fully-polished ge_arrow — every fix from its own report applied, the intrinsic 0.022× as-used unchanged — scores **4.0 "clear improvement — merge"** while the (blind-validated) triage rule says *don't convert* at a 0.028 s baseline. The band vocabulary cannot express "no-conversion" — a verdict the system's own authors needed twice and both times delivered in prose outside the score. + +### 2.4 Band labels use delta language an absolute-hybrid total cannot license (both reviews; narrowed by defense) + +The defense partly refuted Review A's "3 is neutral" (only efficiency anchors 3 as wash; correctness/readability anchor *no-change at 5*). What survives: ~40% of the weight scores the candidate absolutely, so a no-op rewrite of a ge_arrow-quality baseline scores ~3.35–3.55 — rewrites landing in [3.0, 3.55) are *worse than doing nothing* yet labeled "net positive." The homogeneous-population defense (baselines share a known-bad house style, so absolute criteria are deltas in disguise) is honest but population-bound — a clean-baseline conversion scores 4.30 "clear improvement" for virtues the baseline already had. + +### 2.5 Judgement noise exceeds band resolution (both reviews; narrowed by defense) + +One hand-counted concept flips markov_asset 2.25→2.50 across a band; one contestable checklist boolean flips ge_arrow 2.85→3.00; ~27% of ordering-preserving weight vectors flip ge_arrow's band. The defense's strongest point: every demonstrated flip crosses a *deliberation* boundary (the operational next step is identical), never the merge/reject gates, which sit ≥0.35 away. What survives: the 2-decimal total communicates precision the instrument lacks; the concept-count grain rule exists only by example; a one-flip sensitivity stamp (~20 lines in score.py) would make fragility visible. + +### 2.6 Fact-level fan-out is undocumented (both reviews; substantially defended) + +The per-consequence billing defense is strong — a decision harming readers *and* callers *and* editors *should* cost multiply, and most of the demonstrated 1.35-point checkify swing decomposes into legitimately distinct harms. What survives: two near-verbatim duplicate criteria across logic/style (`good_algorithmic_choices` glossed as "vectorised where natural" vs style's `vectorised_where_natural`); and no document states that a root cause's total influence is the sum over its manifestations — the weight table invites misreading. + +### 2.7 "Calibrated" overclaims (both reviews) + +The two measured anchors pin the efficiency scale's *extremes*; every interior edge (0.8/1.3/3×; the Δ bands; the concept and coverage cuts; the 4.0/3.0/2.5 verdict cutoffs) has zero observed cases and no recorded derivation. The defense showed the wash band is deliberately wider than the noise floor (a real design rationale) — but nothing in the repo records it. Same class: the triage "3/3 validation" is in-sample (now noted in the docs). + +## 3. Defended — no change recommended + +- **The efficiency ratio-only form** (Review A's materiality critique): the ratio is the right construct for "did the conversion meet its stated goal"; log-rescoring changes no committed verdict; absolute reader-seconds belong in *triage* (where they already are) and in the report prose (where they already are). +- **The weighted total per se**: it provides a real total order for programme-level triage and a distance-to-merge trajectory; the four-gate alternative was shown to be a lossy projection of the rubric fitted on its own calibration set — its "3/3 agreement" validates the rubric, not the gates. +- **min() aggregation in readability** (direction): the gameable input (docstrings) has no upward power under min() — Goodhart-resistant by shape. The problem is the input (2.2), not the aggregation. +- **Per-consequence multi-counting as a principle** (2.6's core). + +## 4. Recommended v2 changes + +Ordered by (impact ÷ effort); items 1–5 are engine/doc changes validatable against the committed evidence files; item 6 needs @xuanguang-li's design input. + +| # | Change | Effort | +|---|---|---| +| 1 | **Enforce the couplings in code**: derive the logic-design cap from `builds`/x64-divergence in `score_all`; gate the verdict — correctness 1 (any cause) caps the verdict at "net regression," correctness 2 caps at "mixed/wash" | ~5 lines | +| 2 | **Add a "no-conversion" verdict**: when the efficiency evidence shows the triage don't-convert profile (immaterial baseline total + slower as-used), the verdict says so instead of scoring the polish | ~10 lines | +| 3 | **Sensitivity stamp**: score.py perturbs each boolean and band-adjacent value, marks the scorecard `robust` or `fragile (deciding flips listed)`; report totals at the precision the instrument supports | ~20 lines | +| 4 | **K-repeat as-used** (median of ≥3 fresh-process runs; contested-band annotation when the spread crosses an edge) | script change | +| 5 | **Documentation honesty pass**: thresholds labeled policy choices with derivation notes; fan-out paragraph; concept-grain rule stated (one item per reader-facing API surface, symmetric old/new); "calibrated" → "anchored" | prose | +| 6 | **Readability instrument v2**: replace docstring coverage with equation-traceability (per numbered equation: can a reviewer cite the single implementing expression? fraction traceable, old vs new — same citation discipline as the checklists); move concept lists into evidence.json as cited slots (extends the M1 proposal) | design + rubric | +| 7 | **Extraction/replay verification**: a mechanical step diffing extracted code against the lecture's cells, and the replay's call sequence against the lecture's — closing the fidelity gap that produced §1.2. Longer-term: evaluate executing the lecture itself at both refs (nbclient per-cell timings; meta#335 telemetry) as the as-used source, with the current scripts as the diagnostic layer | design | + +## 5. What both reviews agree must survive + +The as-used, fresh-process, compile-counted doctrine; evidence → deterministic scoring with printed derivations; verbatim extraction with disclosed deviations; the calibration-anchor discipline; the caveat register; triage's bounding logic. The redesign is of the scoring superstructure, not the measurement foundation. diff --git a/reviews/pr5-review-2026-07-25.md b/reviews/pr5-review-2026-07-25.md new file mode 100644 index 0000000..b858175 --- /dev/null +++ b/reviews/pr5-review-2026-07-25.md @@ -0,0 +1,195 @@ +# Review suggestions — QuantEcon/skills#5 + +**PR:** [Land the lecture evaluation system (benchmark plugin 0.3.0: rubric v2, skill wired)](https://github.com/QuantEcon/skills/pull/5) +**Branch:** `land-evaluation-system` (14 commits, 56 files, +5,306 / −40) +**Reviewed:** 2026-07-25 + +> **Disposition — added when this record was committed, 2026-07-25.** Everything actionable before merge was applied on the branch in commits `4fffbc9..8388e86`; the item-by-item mapping is in [the PR response comment](https://github.com/QuantEcon/skills/pull/5#issuecomment-5077239291). Of the discussion items, C3 and C1's floor half were also applied; the remaining methodology items (C2, C4, C5, D, C1's measured/adjudicated split) are filed on [#7](https://github.com/QuantEcon/skills/issues/7), the housekeeping remainder (E1, E6, E7) on [#4](https://github.com/QuantEcon/skills/issues/4). Item numbers cited in those issues refer to this document. The text below is otherwise verbatim as received. + +## How this was verified + +Everything below was checked against a local clone of the branch, not read off the diff: + +- `python scripts/validate.py` → green (2 plugins valid) +- `python scripts/scoring/score.py references/examples/{ge_arrow,markov_asset}` → both scorecards regenerate **byte-identically**. The headline reproducibility claim holds. +- Additional probes: dropping `introduces_correctness_bug`, stripping all `citations` blocks, flipping `builds`, and deleting `benchmark/skills/` to exercise the validator's error paths. + +## Overall + +The architecture is right and the discipline is unusual for this kind of system. Separating cited judgement (`evidence.json`) from arithmetic (`rubric.py`) is the correct call, the as-used metric is the honest one, and deriving the bug cap from correctness evidence rather than trusting a hand-set boolean is exactly the right instinct. The findings below are about making the system enforce the standard it already describes. + +Items are numbered so feedback can reference them. Severity: **A** = fix before merge, **B** = should fix, **C** = discussion / judgement call, **D** = housekeeping. + +--- + +## A. Blocking + +### A1 — `scripts/validate.py` raises `NameError` on three error paths + +The `resolve_source` refactor removed the `path` local from `check_plugin`, but three error strings still reference it: lines **146**, **159**, **163**. CI is green only because those branches never fire on the current tree. + +Reproduced by deleting `benchmark/skills/`: + +``` +File "scripts/validate.py", line 159, in check_plugin + error(f"{path}: no skills/ directory") +NameError: name 'path' is not defined. Did you mean: 'Path'? +``` + +The validator crashes instead of reporting the problem it exists to report. Exit code is still 1 by accident, so CI would fail — but with a traceback rather than the diagnostic. + +**Fix:** reuse the `rel` computation already present a few lines above, or assign `path = plugin_dir.relative_to(ROOT)` once after `resolve_source` returns. + +**Effort:** one line. + +--- + +## B. Coverage and enforcement + +### B1 — Neither rubric v2 headline feature is exercised by the regression baselines + +The PR description says the v2 changes were "each validated against the committed evidence files." In the committed tree: + +| Feature | Status in baselines | +|---|---| +| K-repeat as-used (`as_used_runs`) | **Absent from both** `evidence.json` files. Both take the single-run fallback and print the *"the v2 standard is a median of ≥3 fresh-process runs"* caveat. Contested-band annotation never fires. | +| Derived bug cap | markov_asset hand-sets `introduces_correctness_bug: true`, so the derived path never runs. Its reason string reads *"introduces a correctness bug"*, not *"derived from correctness evidence"*. | + +The derived cap does work — dropping the manual flag produces the derived message and the same 2.25 — but nothing in the repo will notice if it stops. + +**Suggested fix:** either backfill `as_used_runs` into at least one worked example, or add a third small fixture (a synthetic `evidence.json`) whose only job is to exercise the v2 paths. A fixture is probably cleaner than re-running the pipelines. + +### B2 — No CI check on the regression baseline + +`.github/workflows/validate.yml` runs `validate.py` only. The PR calls the two worked examples "the regression baseline" and states the byte-identical regeneration was verified — but that verification is manual and will rot on the first refactor. + +**Suggested fix:** + +```yaml +- name: Scorecards reproduce from evidence + run: | + cd benchmark + python scripts/scoring/score.py references/examples/ge_arrow + python scripts/scoring/score.py references/examples/markov_asset + git diff --exit-code -- references/examples/*/results/scorecard.json +``` + +This is the single highest-leverage item in the document — it turns the claim in the PR body into an invariant. + +### B3 — New evidence fields fail open + +v2's stated motivation was that one forgotten boolean shouldn't silently weaken a verdict. It then introduces two fields that do exactly that, both shipped with permissive defaults in `EVIDENCE_TEMPLATE.json` (lines 20–21): + +- `baseline_as_used_seconds: null` → omit it and the no-conversion gate silently never fires (`rubric.py:289` requires non-`None`). +- `as_used_runs: []` → omit it and you silently get a single-run score. + +Same failure class the derived cap was written to close. + +**Suggested fix:** a `validate_evidence()` pass in `score.py` that errors on a missing scored input, run before `score_all`. Keeps the template as documentation while making omission loud. + +### B4 — Citations are documented as mandatory but unenforced + +`rubric.py`'s module docstring: *"Every checklist answer in evidence.json must carry a citation."* `EVALUATION_FRAMEWORK.md` §1 repeats it. Neither is enforced — emptying every `citations` block in markov_asset's evidence produces an identical 2.25. + +**Suggested fix:** reject a structural criterion whose key is absent from `citations`. A few lines in `score_structural`, and it makes the stated contract real. + +--- + +## C. Methodology — worth arguing about + +These are judgement calls, not defects. Flagging them because the system's authority rests on the rubric being defensible. + +### C1 — The sensitivity stamp conflates measurement noise with adjudicated facts + +`score.py:_perturbations` flips every boolean, including ones that record observations rather than estimates. ge_arrow stamps **fragile** partly because flipping `builds: True → False` changes the verdict — but `builds` is something you watched happen. Under this definition nearly every evaluation stamps fragile, which drains the signal from the stamp. + +The reverse case is worse. markov_asset stamps **robust**, but partly because it is already at the floor: flipping `builds` to `True` only raises the total to 2.45, still net regression. It takes two simultaneous changes to move it. "Robust" here partly means "cannot get any worse." + +**Suggested fix:** partition inputs into *measured* (perturb; report as **measurement fragility**) and *adjudicated* (flip; report separately as **judgement sensitivity**). Two stamps, each meaning one thing. The current single stamp is reported in the scorecard and carried into the report verbatim per `SKILL.md` step 5, so the ambiguity propagates. + +### C2 — The no-conversion rule is asymmetric + +`rubric.py:289` requires `sp < 1.0` as well as a sub-floor baseline. So a candidate that is 1.2× faster on a 0.035 s baseline buys 7 ms and gets a normal band, while one that is 0.9× gets no-conversion. If the principle is materiality — *there is nothing here worth buying* — the floor should key on the baseline alone, with the speedup only shaping the wording. + +Worth deciding explicitly, since `NO_CONVERSION_BASELINE_S` is documented as a policy choice and this is part of that policy. + +### C3 — `score_correctness` has a hole in the case it exists to catch + +`rubric.py:126` — x64 divergence is only fatal when `max_delta_shipped > 1e-8`. A candidate whose logic diverges under x64 but happens to agree at float32 to within 1e-8 scores 4 or 5. `score_all`'s derived cap (line 246) uses the same condition, so it does not cap either. + +That is precisely "wrong economics masked by low precision," which §1 of the framework names as the thing correctness is guarding. + +**Suggested fix:** make `matches_under_x64: false` independently capping, regardless of the shipped delta — the shipped agreement is luck, not correctness. + +### C4 — `run_all.py` prints a scorecard for stale evidence + +The pipeline computes `as_used_speedup_runs` and `baseline_as_used_seconds` into `results/as_used.json` (which is gitignored), then ends by invoking `score.py` on a hand-filled `evidence.json` from a *previous* run. The printed scorecard may not reflect the measurements just taken. The field names differ too — `as_used_speedup_runs` in results vs `as_used_runs` in evidence — forcing manual transcription at the one point in the system where a typo is least detectable. + +**Suggested fix:** a `fill_evidence.py` that populates the quantitative slots mechanically from `results/*.json`, leaving only the structural judgements by hand. This removes the whole error class and makes the "measure → record → score" contract literal rather than aspirational. + +### C5 — Paired-run ratios in `as_used_speedup_runs` + +`run_all.py:95–96` (ge_arrow) / `85–86` (markov_asset) zips numpy run *i* with jax run *i*. These are independent fresh processes, so the pairing is arbitrary; `zip` truncates silently if one side broke early; and median-of-ratios is not ratio-of-medians. Since this list drives the contested-band annotation, the definition is worth pinning down — either all-pairs ratios, or state that the pairing is by run index and why that is acceptable. + +--- + +## D. Scope question: comparative vs absolute + +Raising this separately because it is about what the rubric *means*, not how it is coded. + +The scorecard is not a symmetric comparison — you cannot run it backwards to grade the baseline. Only three inputs are genuinely relative (`max_delta_shipped`, `as_used_speedup`, `fixes_prior_bugs`). The rest are absolute properties of the candidate. + +The clearest instance is inside a single dimension. `score_readability` (`rubric.py:139`) takes `delta_prereq_concepts` — relative — and `docstring_cov_new` — absolute. ge_arrow's `static_metrics.json` records coverage going **0.90 → 0.55**; the rubric reads only the 0.55 and scores it 2. A candidate that *improved* coverage 0.10 → 0.55 scores identically to one that *degraded* it 0.90 → 0.55. The distinguishing measurement is already being collected and then discarded. + +The same shape appears in `style_idiom` and `maintainability`, graded entirely on the candidate's own merits — a rewrite loses points for a flaw the original had just as badly. + +This is defensible if the question is *"is this good enough to ship in a teaching resource."* It is wrong if the question is *"is this better than what we have."* The verdict vocabulary — **net regression**, **clear improvement** — is squarely the second question. + +**Two options:** + +1. Keep the mix, but state it explicitly in §1 of the framework: which dimensions are comparative and which are absolute, and why. +2. Make `docstring_cov` a delta like its sibling input, since the data is already in `static_metrics.json`. This changes ge_arrow's readability score, so it is a v3 conversation, not a merge blocker. + +Related, and cheaper: **triage mode currently reads as a subsection of the skill**, but it may be the more valuable half. Both worked examples concluded *no-conversion* — the lecture should not have been converted at all — and triage would have reached the same answer beforehand, from the existing lecture only, in minutes. Worth considering whether it deserves equal billing in `SKILL.md` and the plugin description. + +--- + +## E. Housekeeping + +| # | Item | Where | +|---|---|---| +| E1 | `results/env.json` is gitignored for the reference examples, but those results *are* the baseline — there is no committed record of what produced ge_arrow's `benchmark.json`. The `.gitignore` comment says "the committed provenance is evidence.json," but evidence records source PRs and SHAs, not library versions. Suggest committing `env.reference.json` for the two worked cases. | `.gitignore:7–9` | +| E2 | Number drift: the framework, `SKILL.md`, `benchmark/README.md` and `rubric.py`'s floor comment all cite baselines of 0.028 s / 0.087 s, but the gate actually reads 0.035 / 0.18 from the evidence files. Two different measurements of "baseline as-used" in a system whose selling point is that numbers come from evidence. | `EVALUATION_FRAMEWORK.md:41`, `SKILL.md:51`, `benchmark/README.md:50–51`, `rubric.py:109` | +| E3 | CRLF line endings in two files; everything else is LF. A `.gitattributes` would stop this recurring. | `EVALUATION_FRAMEWORK.md`, `ge_arrow_REPORT.md` | +| E4 | markov_asset still described as "2.25 net regression"; the scorecard now leads with no-conversion. | `benchmark/README.md:51` | +| E5 | `tested += 1` fires before the `try`, so perturbations that raise inflate the "29 single-input perturbations" denominator. | `score.py:62` | +| E6 | `docstring_cov_new` bottoms out at 2 while the prereq sub-score can reach 1 — minor asymmetry in the worse-of-two. | `rubric.py:139–147` | +| E7 | `sys.path.insert(0, HERE); import rubric` works, but there is no `__init__.py`, so `python -m scripts.scoring.score` does not. Low priority unless the engine gets imported elsewhere. | `score.py:22–24` | + +--- + +## F. On the PR shape + +Landing four things at once — the original system, #6's docs, rubric v2, and the skill wiring — is defensible given the goal of testing the whole thing from one branch, and the commit split is clean and well-labelled. + +The cost is that rubric v2's semantics changes, which **move published verdicts**, arrive in the same review as ~3,000 lines of worked-example measurement scripts. Both ge_arrow and markov_asset changed verdict wording in this PR. If there is a next change of that kind, the engine diff deserves to stand alone where it can be read closely. + +--- + +## Reproduction + +```bash +git clone https://github.com/QuantEcon/skills.git && cd skills +git fetch origin land-evaluation-system:land-evaluation-system +git checkout land-evaluation-system + +python scripts/validate.py # green + +cd benchmark +python scripts/scoring/score.py references/examples/ge_arrow # 2.85, no-conversion, fragile +python scripts/scoring/score.py references/examples/markov_asset # 2.25, no-conversion, gated +git diff --stat # byte-identical + +# A1 repro +cd .. && rm -rf benchmark/skills && python scripts/validate.py # NameError +``` diff --git a/reviews/validation-run-ge_arrow-2026-07-22.md b/reviews/validation-run-ge_arrow-2026-07-22.md new file mode 100644 index 0000000..39d47e7 --- /dev/null +++ b/reviews/validation-run-ge_arrow-2026-07-22.md @@ -0,0 +1,39 @@ +# Validation run — ge_arrow re-evaluated from a fresh checkout (2026-07-22) + +The dry run required by [skills#8](https://github.com/QuantEcon/skills/issues/8) §3: check out the lecture repo at the state prior to the motivating PR, drive the wired skill procedure end-to-end in a user workspace, and cross-compare against the committed reference evaluation. Target: **ge_arrow / [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717)** — the PR the system was first developed on, still open, and independent of the #654 acceptance test that issue #8 reserves for the adjudicated run. Nothing was posted to either upstream PR. + +## Setup + +| | | +|---|---| +| Refs | base `8cfba4c` (merge-base of `update_ge_arrow` with `main` — also #654's merge-base), head `8c2d0d7` (PR head; last pushed 2026-07-09, so identical to the state the reference evaluation measured) | +| Workspace | fresh partial clone; evaluation under `benchmark-eval/ge_arrow/` per the wired SKILL.md; plugin read-only via `CLAUDE_PLUGIN_ROOT` | +| Environment | python 3.13.9, **jax 0.10.1, numpy 2.3.5**, macOS arm64 — deliberately *not* the reference environment (jax 0.4.35, numpy 2.1.3), so this also tests robustness to library drift | + +## Cross-comparison + +| Quantity | Reference (committed) | Fresh re-run | Band agreement | +|---|---|---|---| +| Candidate extraction | `model_new.py` | byte-identical to lecture cell at head | verbatim confirmed | +| Baseline extraction | `model_old.py` (globals fix disclosed) | matches, two deviation classes (below) | confirmed with findings | +| float32 worst max\|Δ\| | 1.7e-4 (ex2) | 1.01e-4 (ex3_s0/J) | same band → correctness 3 | +| x64 worst max\|Δ\| | 1.4e-14 | 1.42e-13, all match | same conclusion (≪1e-8) | +| as-used speedup | 0.022× (single pass) | 0.0251× (median of 3; spread 0.0227–0.0353× within one band) | same band → efficiency 2 | +| baseline as-used total | 0.035 s | 0.0292 s (median of 3) | both under the 1 s floor | +| Δprereq / docstrings / statements | +6 / 0.90→0.55 / 1 | identical | identical | +| **Weighted total** | **2.85** | **2.85** | exact | +| **Verdict** | no-conversion (candidate band mixed/wash) | no-conversion (candidate band mixed/wash) | exact | +| **Sensitivity** | fragile, 3 deciding flips | fragile, same 3 flips (`builds`, `matches_under_x64`, `good_algorithmic_choices`) | exact | + +The v2 additions all exercised on real data: K-repeat medians with per-run speedups, the no-conversion verdict, the sensitivity stamp, `CLAUDE_PLUGIN_ROOT` resolution from a workspace, and the per-run provenance stamp (which recorded the environment difference). + +## Findings (the point of a dry run) + +1. **ge_arrow's `check_equivalence.py` had no x64 handling** — it always wrote `results/equivalence.json`, so the x64 run clobbered the as-shipped run, and the evidence file's dual-regime citation was not reproducible from `run_all.py` alone (markov_asset's template already wrote per-regime files). **Fixed**: the script now writes `equivalence_x64.json` when `JAX_ENABLE_X64=1` and records the regime in its summary. +2. **Undisclosed cosmetic deviation in the baseline extraction**: `model_old.py` normalises arithmetic spacing (`T+1` → `T + 1`, `t-1` → `t - 1`) beyond its one disclosed deviation class (the globals fix). Semantics identical; now disclosed in the file's fidelity note per the v2 verbatim rule. +3. **Median-of-ratios vs ratio-of-medians**: with `as_used_runs` present the engine scores the median of per-run speedups (0.0251×) rather than the ratio of median totals (0.0244×). Same band here; the evidence template documents `as_used_runs` as per-run speedups, which is the authoritative form. +4. Upstream provenance was recorded by PR number and branch but **not by SHA**; both committed evidence files now carry `source_pr` + base/head SHAs (both PRs share merge-base `8cfba4c`). + +## Conclusion + +The skill procedure runs end-to-end from a clean checkout in a user workspace and **reproduces the reference evaluation exactly at the level the rubric claims to be reproducible** (bands, total, verdict, sensitivity stamp) across a major JAX version change, with measured quantities moving only within their bands. The #654 acceptance run can proceed on this procedure. diff --git a/scripts/validate.py b/scripts/validate.py index 8613782..c1075fd 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -81,6 +81,45 @@ def check_skill(skill_dir, plugin_name): print(f" !! /{plugin_name}:{skill_dir.name} — see problems below") +def resolve_source(entry, name): + """Locate a plugin's directory from its `source`, and reject sources that + would break install. + + A plugin co-located with this marketplace must use a relative-path source + (`"./qe"`) so install uses the already-present marketplace copy. A remote + source pointing back at this repo forces an install-time re-clone — the SSH + failure reported in QuantEcon/skills#10 — so we flag it here rather than let + it reach users. Returns the resolved directory Path, or None if the source + is malformed. + """ + source = entry.get("source") + if source is None: + error(f"marketplace.json: plugin `{name}` missing `source`") + return None + if isinstance(source, str): + if not source.startswith("./"): + error( + f"marketplace.json: plugin `{name}` source `{source}` must start " + f"with `./` (a path relative to the marketplace root)" + ) + return None + return ROOT / source[2:] + if isinstance(source, dict): + target = str(source.get("repo") or source.get("url") or "") + if "QuantEcon/skills" in target: + error( + f"marketplace.json: plugin `{name}` uses a remote source pointing " + f"back at this repo, which forces an install-time re-clone (SSH " + f"failure in #10). Use a relative path: \"source\": \"./{name}\"." + ) + return ROOT / source.get("path", name) + error( + f"marketplace.json: plugin `{name}` source must be a relative path string " + f"or an object, got {type(source).__name__}" + ) + return None + + def check_plugin(entry): if not isinstance(entry, dict): error(f"marketplace.json: plugin entry must be an object, got {type(entry).__name__}") @@ -89,8 +128,13 @@ def check_plugin(entry): if not name: error("marketplace.json: plugin entry missing `name`") return - path = entry.get("source", {}).get("path", name) - plugin_dir = ROOT / path + + plugin_dir = resolve_source(entry, name) + if plugin_dir is None: + return + # Repo-relative label for diagnostics; every error below reports the plugin + # location, so it is computed once here rather than per-branch. + path = plugin_dir.relative_to(ROOT) if ROOT in plugin_dir.parents else plugin_dir if not plugin_dir.is_dir(): error(f"marketplace.json: plugin `{name}` points at missing directory `{path}`") return @@ -129,6 +173,12 @@ def main(): print("\n".join(errors), file=sys.stderr) return 1 + # `owner` is a required top-level object in the marketplace schema; a manifest + # missing it is rejected by `/plugin marketplace add` (QuantEcon/skills#10). + owner = marketplace.get("owner") + if not isinstance(owner, dict) or not owner.get("name"): + error("marketplace.json: missing required `owner` object with a `name` field") + plugins = marketplace.get("plugins", []) if not isinstance(plugins, list): error(f"marketplace.json: `plugins` must be a list, got {type(plugins).__name__}")