Repository navigation
Expand file tree
/
Copy pathMakefile
More file actions
300 lines (276 loc) · 17.7 KB
/
Copy pathMakefile
File metadata and controls
300 lines (276 loc) · 17.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
# openflipbook — local demo stack shortcuts.
#
# cp .env.example .env # add FAL_KEY (+ OPENROUTER_API_KEY)
# make demo # → http://localhost:3000/play
#
.PHONY: demo demo-world demo-mock demo-local demo-down demo-clean demo-logs release selfhost-check
# Local stores (Mongo + a local S3 store, RustFS) + backend + web, with cloud AI.
# Needs FAL_KEY + OPENROUTER_API_KEY in .env (see .env.example).
demo:
docker compose up --build
# Full first-release authoring shell, private assets and independent workers.
# Set SHARED_TOKEN in .env. Provider generation still requires priced opt-ins.
release:
docker compose -f docker-compose.yml -f docker-compose.release.yml --profile world-build up --build
# Fresh isolated fixture stack; never mounts existing data or loads user keys.
selfhost-check:
node scripts/selfhost/check.mjs
# Same stack with World Mode + scale-ladder nav on (tap to ENTER places, go
# deeper, ascend back out, pan to neighbours). Same two keys; rebuilds the web
# image because the NEXT_PUBLIC_* flags are baked at build time.
demo-world:
docker compose -f docker-compose.yml -f docker-compose.demo.yml up --build
# Zero-key on-ramp: stubbed providers (MOCK_PROVIDERS=1) — no FAL/OpenRouter
# keys needed. The whole stack boots and serves mocked generations so a
# first-time deployer sees /play work instead of a confusing key error.
demo-mock:
MOCK_PROVIDERS=1 docker compose up --build
# Adds Ollama so the planner + click VLM run locally too — only FAL_KEY needed
# (images stay on fal). First run pulls multi-GB models; CPU-slow.
demo-local:
docker compose -f docker-compose.yml -f docker-compose.local.yml up --build
# Stop + remove containers (keeps the mongo / s3 / ollama data volumes).
demo-down:
docker compose -f docker-compose.yml -f docker-compose.local.yml down
# Stop + wipe volumes too — fresh slate (empty DB + blob store + models).
demo-clean:
docker compose -f docker-compose.yml -f docker-compose.local.yml down -v
# Tail logs from the running stack.
demo-logs:
docker compose logs -f --tail=100
# ── Eval gates (geometric world model) ───────────────────────────────────────
# Free gates run always (deterministic, no spend). Paid gates spend fal/openrouter
# and only run when their *_BENCH_RUN flag is set — `make eval` excludes them.
PY := apps/modal-backend/.venv/bin/python
.PHONY: eval eval-geometry eval-silhouette eval-layout eval-grounding eval-repair eval-edit eval-paid
# The always-on gate: every free phase gate + lints + typechecks. Run between
# phases — must be green before the next phase's commit.
eval:
cd apps/modal-backend && .venv/bin/python -m pytest -m "not paid" -q
cd apps/modal-backend && .venv/bin/ruff check . && .venv/bin/mypy providers obs.py generate.py
cd apps/web && pnpm exec vitest run && pnpm exec tsc --noEmit && pnpm run check:circular
# P1 — pure 2.5D projection parity (TS golden + Py golden + cross-lang fuzz).
eval-geometry:
cd apps/modal-backend && .venv/bin/python -m pytest -m geometry -q
cd apps/web && pnpm exec vitest run lib/world-geometry.test.ts lib/world-geometry.fuzz.test.ts
# P3 layout-fidelity A/B: generate each scene with vs without the geometry layout
# clause + VLM-judge both → the lift. PAID (~4 fal gens + judge calls). Needs
# FAL_KEY + OPENROUTER_API_KEY (auto-loaded from apps/modal-backend/.env).
# Silhouette fidelity: the block render knows which pixels belong to which
# place (the footprint is stored data) -- segment the painted building in the
# same spot and compare. PAID (~one SAM-3 call per camera+building).
# Needs masks from the TS side first; see the runner's docstring.
eval-silhouette:
cd apps/modal-backend && SILHOUETTE_BENCH_RUN=1 .venv/bin/python -m tests.world_bench.silhouette_runner $(MASKS) $(FRAMES)
eval-layout:
cd apps/modal-backend && .venv/bin/python -m tests.world_bench.layout_runner
# P4 grounding-verify: generate from the layout clause, detect the expected
# entities, diff vs intent → the grounded confirmation signal. PAID (fal + VLM).
eval-grounding:
cd apps/modal-backend && GROUNDING_BENCH_RUN=1 .venv/bin/python -m tests.world_bench.grounding_runner
# B2 segmenter smoke: VLM polygon borders + anchored absolute heights over a
# few existing report JPGs — eyeball the output. PAID (~$0.05, Gemini only,
# zero fal). Override images: SEGMENT_SMOKE_IMAGES=/path/a.jpg,...
eval-segment-smoke:
cd apps/modal-backend && SEGMENT_BENCH_RUN=1 .venv/bin/python -m tests.world_bench.segment_smoke
# Detector canary: the real detector on a committed synthetic image with a
# 10-label ask (the size class that used to truncate + silently return []).
# Fails on any parse failure/salvage or zero detections — catches live-model
# drift the free fixture tests can't. PAID (~$0.01, one Gemini call).
eval-detector-smoke:
cd apps/modal-backend && DETECTOR_SMOKE_RUN=1 .venv/bin/python -m tests.world_bench.detector_smoke
# Ground-truth map corpus: fetch the public-domain scans (free, a few MB;
# --pin on first run records sha256s into the manifest) and VLM-draft a
# description for human verification (PAID ~$0.015/map, Gemini only):
# make corpus-fetch
# make corpus-draft id=fantasy-treasure-island (or id=all)
corpus-fetch:
cd apps/modal-backend && .venv/bin/python scripts/fetch_corpus.py --pin
corpus-draft:
cd apps/modal-backend && CORPUS_DRAFT_RUN=1 .venv/bin/python -m tests.map_corpus.draft $(or $(id),all)
# Ensemble annotate (the evolvable upgrade to corpus-draft): fan the describe
# call over N VLMs, reconcile deterministically, judge, refine once on a low
# score, and AUTO-PROMOTE to review.status=verified when the judge AND ensemble
# agreement both clear their gates (else needs_human). Non-destructive: a
# verified description is preserved; output lands in descriptions/candidates/
# unless CORPUS_ANNOTATE_FORCE=1. PAID (~$0.03-0.10/map).
# Preview the ensemble + gates with no flag; CORPUS_ANNOTATE_RUN=1 to spend.
# make corpus-annotate-preview
# make corpus-annotate id=fantasy-treasure-island (or id=all)
# Knobs (env): CORPUS_ANNOTATE_MODELS=google/gemini-3-pro-preview,anthropic/claude-sonnet-4-6,qwen/qwen3-vl-235b-a22b-instruct
# (middle-ground ensemble — avoid Opus, the describe cost driver; unset = single cheap VLM)
# CORPUS_JUDGE_THRESHOLD (7.0) CORPUS_AGREEMENT_THRESHOLD (0.6)
# CORPUS_ANNOTATE_MAX_ITERS (2) CORPUS_MIN_VOTES (majority; =1 union for scenes)
# CORPUS_ARBITER_MODEL=google/gemini-3-flash-preview (text arbiter that semantically
# merges synonym fixtures; unset = deterministic merge. NOTE deepseek-v4-pro flakily
# returns {} -> silent fallback; use gemini-3-flash / gpt-4o-mini / deepseek-v4-flash.)
corpus-annotate-preview:
cd apps/modal-backend && .venv/bin/python -m tests.map_corpus.annotate $(or $(id),all)
corpus-annotate:
cd apps/modal-backend && CORPUS_ANNOTATE_RUN=1 .venv/bin/python -m tests.map_corpus.annotate $(or $(id),all)
# Visual in->out: draw a description's entities (detector boxes + segmenter
# borders + labels) back onto its source map. FREE (no API). Verified by
# default; add cand=1 to render the ensemble candidate instead. Writes
# tests/map_corpus/overlays/<id>.<source>.png.
# make corpus-overlay id=fantasy-treasure-island
# make corpus-overlay id=fantasy-treasure-island cand=1
corpus-overlay:
cd apps/modal-backend && .venv/bin/python -m tests.map_corpus.overlay $(or $(id),fantasy-treasure-island) $(if $(cand),--candidate,)
# Reconstruction bench: regenerate each VERIFIED corpus map from its authored
# description (graph = product planning path, direct = ground-truth layout),
# extract + score vs ground truth + VLM judges. Rides the matrix chassis:
# cached, budget-capped, dry-run without the flag. PAID (~$0.40 first run,
# cached after): make eval-recon (preview: make eval-recon-dry)
eval-recon-dry:
cd apps/modal-backend && .venv/bin/python -m tests.recon_bench.runner
eval-recon:
cd apps/modal-backend && RECON_BENCH_RUN=1 .venv/bin/python -m tests.recon_bench.runner
# Per-tier recon for the CLOSEUP tier (verified Met objects): same chassis, an
# object-worded render template + object-parts judge (feature_articulation
# instead of map_plausibility). RECON_SWEEP picks the sweep + report + baseline.
eval-recon-closeup-dry:
cd apps/modal-backend && RECON_SWEEP=recon_closeup .venv/bin/python -m tests.recon_bench.runner
eval-recon-closeup:
cd apps/modal-backend && RECON_SWEEP=recon_closeup RECON_BENCH_RUN=1 .venv/bin/python -m tests.recon_bench.runner
# Per-tier recon for the INTERIOR tier (verified scene descriptions): scene-worded
# template + image judges (style/alignment/feature_articulation); geo weighted low
# since interior fixture positions are approximate (prose-decoupled, not grounded).
eval-recon-interior-dry:
cd apps/modal-backend && RECON_SWEEP=recon_interior .venv/bin/python -m tests.recon_bench.runner
eval-recon-interior:
cd apps/modal-backend && RECON_SWEEP=recon_interior RECON_BENCH_RUN=1 .venv/bin/python -m tests.recon_bench.runner
# Descent reconstruction (M4): ENTER a parent map's linked place (a child
# interior/closeup tied via parent_id+parent_ref to a parent entity), score the
# region-conditioned descent against the REAL child photo (style + continuity)
# vs a fresh baseline. Dry (resolve chains + cost, $0): make eval-descent-dry
# Live (DESCENT_BENCH_RUN=1, ~$0.30/chain): make eval-descent
eval-descent-dry:
cd apps/modal-backend && .venv/bin/python -m tests.descent_bench.runner
eval-descent:
cd apps/modal-backend && DESCENT_BENCH_RUN=1 .venv/bin/python -m tests.descent_bench.runner
eval-repair:
cd apps/modal-backend && .venv/bin/python -m pytest -m repair -q
eval-edit:
cd apps/modal-backend && .venv/bin/python -m pytest -m edit -q
# B3 sub-part coherence A/B: ENTER each place WITH vs WITHOUT region-conditioning
# (+ the B2 faithful preamble) → judge faithfulness vs the parent map crop → the
# lift. PAID (fal gens + judge). Needs a live session + the web app running:
# COHERENCE_BENCH_SESSION=session_xxx make eval-coherence
eval-coherence:
cd apps/modal-backend && COHERENCE_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.coherence_runner
# Style medium-consistency A/B: edit a styled source WITH vs WITHOUT the medium
# lock → judge how faithfully each edit keeps the source's medium → the lift + a
# pass threshold. Guards the edit-path style fix. PAID (fal edits + Gemini judge);
# self-contained, no session needed: make eval-style
eval-style:
cd apps/modal-backend && STYLE_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.style_runner
# B2 OUTWARD-drift A/B: synthesize each container TWICE — the default zero-drift
# BRIA outpaint vs the SCALE_OUTWARD_RERENDER fresh path — and judge how faithfully
# each keeps the source's medium → the drift number + a trust threshold. Run before
# enabling SCALE_OUTWARD_RERENDER. PAID (fal gens + Gemini judge); no session:
# make eval-outward-drift
eval-outward-drift:
cd apps/modal-backend && OUTWARD_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.outward_runner
# MULTI-HOP DRIFT (Risk #1): walk a styled source OUTWARD k hops (each container
# conditioned on the PREVIOUS hop — the compounding path) and score style
# faithfulness to the ORIGINAL after every hop → a half-life (the hop drift crosses
# the floor). Product rule falls out: cap auto-OUTWARD at half_life - 1.
# Dry preview ($0): make eval-chain-drift-dry ; PAID: make eval-chain-drift
eval-chain-drift-dry:
cd apps/modal-backend && .venv/bin/python -m tests.continuity_bench.chain_runner
eval-chain-drift:
cd apps/modal-backend && CHAIN_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.chain_runner
# ENTER-consistency A/B: tap a landmark on a styled map and render the entered
# scene the OLD way (fresh text-to-image, refs ignored) vs the NEW way (edit
# endpoint on the region crop) → "same place?" judge vs the tapped region → the
# LIFT. THE metric for cross-hop visual consistency. Optional extra arms:
# ENTER_BENCH_MODELS=fal-ai/flux-pro/kontext,openai/gpt-image-2/edit \
# make eval-enter-drift
# PAID (fal gens + Gemini judge); self-contained, no session needed.
eval-enter-drift:
cd apps/modal-backend && ENTER_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.enter_runner
# VIEW-conformance bench: render the enter five ways (legacy + the four
# deliberate projections) → judge "is it ACTUALLY that projection?" + the
# same-place floor; plus a positioning probe (layout clause + top_down camera
# clause → detector → grounding.diff on correct-register bins). The view
# grammar's gate. PAID (~$2.5; fal gens + Gemini judge); no session needed.
eval-view:
cd apps/modal-backend && VIEW_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.view_runner
# Same bench with the PRODUCTION render loop on the steep arms (judged retries
# with critic feedback, max 3 attempts) — measures what users actually get.
eval-view-loop:
cd apps/modal-backend && VIEW_BENCH_RUN=1 VIEW_BENCH_LOOP=1 .venv/bin/python -m tests.continuity_bench.view_runner
# E1's gating probe: does gpt-image-2/edit honor mask_url, and with which mask
# convention? 4 gpt arms (3 conventions + no-mask churn control) + the dormant
# flux-pro/v1/fill slot, scored by pixel-diff only. PAID (~$0.5, no VLM).
eval-edit-mask-smoke:
cd apps/modal-backend && EDIT_REGION_BENCH_RUN=1 .venv/bin/python -m tests.edit_bench.mask_smoke
# E5: the mask-scoped edit bench — the asked change LANDS (alignment judge on
# the inside crop), the edit CONFINES (outside pixel-diff, free, per-case),
# the MEDIUM holds (style judge). EDIT_REGION_BENCH_MODELS adds A/B arms;
# EDIT_REGION_BENCH_LOOP=1 measures the production edit loop;
# EDIT_REGION_BENCH_WHOLE=1 adds the EDIT_JUDGE whole-image arm. PAID (~$1).
eval-edit-region:
cd apps/modal-backend && EDIT_REGION_BENCH_RUN=1 .venv/bin/python -m tests.edit_bench.runner
# The before/after regression sweep: every PAID eval that has a committed
# baseline band (tests/eval_baselines.json), run back to back. Each prints
# PASS / REGRESSION / IMPROVED vs its band — run it before AND after a risky
# generation-path change and diff the verdicts. ~$7/run; keeps going past a
# single bench failure so one flaky run doesn't hide the rest.
# Free coverage twin: make coverage (no spend).
eval-baselines:
-cd apps/modal-backend && .venv/bin/python -m tests.world_bench.layout_runner
-cd apps/modal-backend && STYLE_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.style_runner
-cd apps/modal-backend && GROUNDING_BENCH_RUN=1 .venv/bin/python -m tests.world_bench.grounding_runner
-cd apps/modal-backend && ENTER_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.enter_runner
-cd apps/modal-backend && VIEW_BENCH_RUN=1 .venv/bin/python -m tests.continuity_bench.view_runner
-cd apps/modal-backend && EDIT_REGION_BENCH_RUN=1 .venv/bin/python -m tests.edit_bench.runner
# Matrix bench (the evolvable eval): scenarios × arms × models × prompt-
# variants, disk-cached so prompt evolution re-bills only changed cells.
# DRY-RUN IS THE DEFAULT — eval-matrix-dry prints the per-cell cost table
# (the mandatory preview, $0). eval-matrix runs uncached cells under the
# hard budget cap (MATRIX_BUDGET_USD, default $3; MATRIX_ALLOW_PARTIAL=1
# runs to the cap instead of refusing). Scenarios come from the map corpus
# once the recon bench (tests/recon_bench) lands.
eval-matrix-dry:
cd apps/modal-backend && .venv/bin/python -m tests.matrix_bench.runner
eval-matrix:
cd apps/modal-backend && MATRIX_BENCH_RUN=1 .venv/bin/python -m tests.matrix_bench.runner
# ── Scenario Lab (unified test bench) ───────────────────────────────────────
# bench-dry is the mandatory $0 cost preview. bench-run spends under the cap.
# SWEEP= picks a sweep file under tests/scenario_lab/sweeps/ (default: layout).
SWEEP ?= layout
.PHONY: bench-dry bench-run bench-compare bench-baselines scenario-new ux-bench-dry ux-bench chain-bench chain-bench-dry
bench-dry: eval-matrix-dry
cd apps/modal-backend && MATRIX_SWEEP=tests/scenario_lab/sweeps/$(SWEEP).json .venv/bin/python -m tests.scenario_lab.runner
bench-run:
cd apps/modal-backend && MATRIX_BENCH_RUN=1 MATRIX_SWEEP=tests/scenario_lab/sweeps/$(SWEEP).json .venv/bin/python -m tests.scenario_lab.runner
bench-compare:
cd apps/modal-backend && .venv/bin/python -m tests.scenario_lab.bench_compare
bench-baselines: eval-baselines
scenario-new:
cd apps/modal-backend && .venv/bin/python -m tests.scenario_lab.scenario_new $(id)
ux-bench-dry:
pnpm tsx scripts/ux-bench/run.ts
ux-bench:
UX_BENCH_RUN=1 pnpm tsx scripts/ux-bench/run.ts
chain-bench-dry:
cd apps/modal-backend && .venv/bin/python -m tests.scenario_lab.chain_runner
chain-bench:
cd apps/modal-backend && CHAIN_BENCH_RUN=1 .venv/bin/python -m tests.scenario_lab.chain_runner
# Free coverage report (backend lines + the web view-path files).
coverage:
cd apps/modal-backend && .venv/bin/python -m pytest -q -m "not paid" --cov=providers --cov=generate --cov-report=term | tail -30
cd apps/web && pnpm exec vitest run --coverage --coverage.reporter=text 2>/dev/null | grep -E "geo-tap|click-route|world-geometry|image-condition|ClickDetail|All files" || true
# Alias for eval-baselines — the full paid regression sweep.
eval-paid: eval-baselines
# The ladder proof: Playwright drives the REAL app (localhost:3000) through
# map → closeup tap → enter tap across 5 place types, saves the image
# gallery + wire manifests to ladder-proof-runs/<name>, then numeric judges
# score both hops. The eyes-on pass (adversarial subagent verdicts on the
# images) stays manual — no ladder change ships without it. PAID: ~$0.55
# per scenario + ~$0.03 judging. LADDER_ONLY=city,castle narrows scenarios.
ladder-proof:
cd apps/web && node scripts/ladder-proof.mjs ../../ladder-proof-runs/$${LADDER_RUN:-$$(date +%Y%m%d-%H%M%S)}
ladder-judge:
cd apps/modal-backend && .venv/bin/python -m tests.ladder_judge ../../ladder-proof-runs/$(LADDER_RUN)