feat(cache): incremental persistence + full-cache-hit short-circuit (Plan A)

tests/test_aggregator.py

@@ -159,3 +159,71 @@ def test_aggregate_records_cache_entry_after_synthesis(tmp_path, mock_provider_f
     )
     aggregate_all(layout=layout, provider=provider, cache=cache)
     assert section.id in cache.aggregation
+
+
+def test_aggregate_persist_cache_called_after_each_section(tmp_path, mock_provider_factory):
+    """The persist callback fires once per cache update.
+
+    This is the contract that turns a Ctrl-C mid-stage-3 into a
+    survivable event — without per-section persistence, every
+    aggregation entry computed in this stage would be lost if anything
+    raised before the final walk-end save.
+    """
+    layout = _setup(tmp_path)
+    # Populate two primary sections so we get two persist invocations.
+    s1, s2 = PRIMARY_SECTIONS[0], PRIMARY_SECTIONS[1]
+    append_note(layout, s1, {"file": "a.py", "summary": "x", "finding": "Entity A."})
+    append_note(layout, s2, {"file": "b.py", "summary": "x", "finding": "Capability B."})
+
+    cache = WalkCache()
+    persist_calls = {"n": 0}
+
+    def persist():
+        persist_calls["n"] += 1
+
+    provider = mock_provider_factory(
+        json_factory=lambda schema, system, user: SectionBody(body="body"),
+    )
+    aggregate_all(layout=layout, provider=provider, cache=cache, persist_cache=persist)
+    assert persist_calls["n"] == 2  # one per section that got a cache update
+
+
+def test_aggregate_persist_cache_survives_mid_stage_failure(tmp_path, mock_provider_factory):
+    """Sections that *did* aggregate before a crash must still be on disk.
+
+    Simulate the deriver-stage Ctrl-C scenario: after section 1 succeeds
+    and persists, section 2's LLM call raises. The cache file on disk
+    should contain section 1's entry — that's the resumability gain.
+    """
+    from wikifi.cache import load as load_cache
+    from wikifi.cache import save_aggregation
+
+    layout = _setup(tmp_path)
+    s1, s2 = PRIMARY_SECTIONS[0], PRIMARY_SECTIONS[1]
+    append_note(layout, s1, {"file": "a.py", "summary": "x", "finding": "Entity A."})
+    append_note(layout, s2, {"file": "b.py", "summary": "x", "finding": "Capability B."})
+
+    cache = WalkCache()
+    call_count = {"n": 0}
+
+    def factory(schema, system, user):
+        call_count["n"] += 1
+        if call_count["n"] >= 2:
+            raise RuntimeError("simulated mid-stage-3 crash")
+        return SectionBody(body="Synthesized.")
+
+    provider = mock_provider_factory(json_factory=factory)
+    aggregate_all(
+        layout=layout,
+        provider=provider,
+        cache=cache,
+        persist_cache=lambda: save_aggregation(layout, cache),
+    )
+
+    # Reload from disk to confirm section 1's body persisted before the
+    # second section's failure rolled the in-memory cache back.
+    on_disk = load_cache(layout)
+    assert s1.id in on_disk.aggregation
+    # Section 2's per-section catch wrote a fallback body, so its cache
+    # entry was never recorded.
+    assert s2.id not in on_disk.aggregation

tests/test_cache.py

@@ -8,11 +8,19 @@
     CACHE_VERSION,
     WalkCache,
     aggregation_cache_path,
+    derivation_cache_path,
     extraction_cache_path,
+    hash_introspection_scope,
     hash_section_notes,
+    hash_upstream_bodies,
+    introspection_cache_path,
     load,
     reset,
     save,
+    save_aggregation,
+    save_derivation,
+    save_extraction,
+    save_introspection,
 )
 from wikifi.wiki import WikiLayout, initialize
 
@@ -141,6 +149,173 @@ def test_cache_version_is_pinned():
     assert CACHE_VERSION >= 1
 
 
+def test_derivation_cache_round_trip(tmp_path: Path):
+    """Cached derivative bodies survive a full save/load cycle."""
+    layout = _layout(tmp_path)
+    cache = WalkCache()
+    cache.record_derivation("personas", upstream_hash="uh1", body="P body", revised=False)
+    cache.record_derivation("user_stories", upstream_hash="uh2", body="US body", revised=True)
+    save(layout, cache)
+    assert derivation_cache_path(layout).exists()
+
+    loaded = load(layout)
+    hit = loaded.lookup_derivation("personas", "uh1", expect_revised=False)
+    assert hit is not None and hit.body == "P body"
+    revised = loaded.lookup_derivation("user_stories", "uh2", expect_revised=True)
+    assert revised is not None and revised.revised is True
+
+
+def test_derivation_cache_misses_on_upstream_hash_change(tmp_path: Path):
+    """A different `upstream_hash` must miss; that's how upstream edits flow through."""
+    _layout(tmp_path)
+    cache = WalkCache()
+    cache.record_derivation("personas", upstream_hash="uh1", body="P", revised=False)
+    assert cache.lookup_derivation("personas", "uh2", expect_revised=False) is None
+    assert cache.derivation_misses == 1
+
+
+def test_derivation_cache_rejects_unrevised_body_when_review_requested(tmp_path: Path):
+    """`--review` walks must not silently reuse a body produced without the critic."""
+    _layout(tmp_path)
+    cache = WalkCache()
+    cache.record_derivation("personas", upstream_hash="uh1", body="P", revised=False)
+    assert cache.lookup_derivation("personas", "uh1", expect_revised=True) is None
+    # The inverse — review off, cached body was revised — is still a hit:
+    cache.record_derivation("user_stories", upstream_hash="uh2", body="US", revised=True)
+    assert cache.lookup_derivation("user_stories", "uh2", expect_revised=False) is not None
+
+
+def test_introspection_cache_round_trip(tmp_path: Path):
+    """The prior walk's scope hash + payload survive save/load."""
+    layout = _layout(tmp_path)
+    cache = WalkCache()
+    cache.record_introspection(
+        scope_hash="sh1",
+        payload={"include": ["src/"], "exclude": ["build/"], "primary_languages": ["python"]},
+    )
+    save(layout, cache)
+    assert introspection_cache_path(layout).exists()
+
+    loaded = load(layout)
+    assert loaded.lookup_introspection("sh1") is not None
+    assert loaded.lookup_introspection("different") is None
+    assert loaded.introspection.payload["include"] == ["src/"]
+
+
+def test_hash_upstream_bodies_is_stable_across_dict_order():
+    """Insertion order must not affect the upstream-body hash."""
+    a = {"capabilities": "cap body", "intent": "intent body"}
+    b = {"intent": "intent body", "capabilities": "cap body"}
+    assert hash_upstream_bodies(a) == hash_upstream_bodies(b)
+    different = {"capabilities": "cap body", "intent": "DIFFERENT"}
+    assert hash_upstream_bodies(a) != hash_upstream_bodies(different)
+
+
+def test_hash_introspection_scope_ignores_non_scope_fields():
+    """Only ``include`` + ``exclude`` should drive the scope hash."""
+    a = hash_introspection_scope(include=["src/"], exclude=["build/"])
+    b = hash_introspection_scope(include=["src/"], exclude=["build/"])
+    assert a == b
+    # Order-insensitive within each list.
+    a2 = hash_introspection_scope(include=["b/", "a/"], exclude=["x/", "y/"])
+    a3 = hash_introspection_scope(include=["a/", "b/"], exclude=["y/", "x/"])
+    assert a2 == a3
+    # Adding an exclude shifts the hash.
+    different = hash_introspection_scope(include=["src/"], exclude=["build/", "tmp/"])
+    assert a != different
+
+
+def test_save_aggregation_only_writes_aggregation_file(tmp_path: Path):
+    """Per-stage save helpers must touch only their own file.
+
+    Without this the per-section persist callback would re-write every
+    cache file on every section, defeating the point of splitting them.
+    """
+    layout = _layout(tmp_path)
+    cache = WalkCache()
+    cache.record_aggregation("intent", notes_hash="h", body="b", claims=[], contradictions=[])
+    # Pre-create extraction file so we can verify it's untouched.
+    cache.record_extraction("a.py", fingerprint="x", findings=[], summary="", chunks_processed=0)
+    save_extraction(layout, cache)
+    extraction_mtime = extraction_cache_path(layout).stat().st_mtime_ns
+
+    # Wait long enough that any rewrite would change the mtime.
+    import time
+
+    time.sleep(0.01)
+    save_aggregation(layout, cache)
+    assert aggregation_cache_path(layout).exists()
+    # extraction file untouched by save_aggregation
+    assert extraction_cache_path(layout).stat().st_mtime_ns == extraction_mtime
+
+
+def test_save_introspection_no_op_when_unset(tmp_path: Path):
+    """Calling ``save_introspection`` on a cache without an introspection record is a no-op."""
+    layout = _layout(tmp_path)
+    cache = WalkCache()
+    save_introspection(layout, cache)
+    assert not introspection_cache_path(layout).exists()
+
+
+def test_reset_drops_every_scope(tmp_path: Path):
+    """``reset`` must clear the new derivation + introspection files too."""
+    layout = _layout(tmp_path)
+    cache = WalkCache()
+    cache.record_extraction("a.py", fingerprint="x", findings=[], summary="", chunks_processed=0)
+    cache.record_aggregation("intent", notes_hash="h", body="b", claims=[], contradictions=[])
+    cache.record_derivation("personas", upstream_hash="u", body="p", revised=False)
+    cache.record_introspection(scope_hash="s", payload={"include": [], "exclude": []})
+    save(layout, cache)
+
+    reset(layout)
+    assert not extraction_cache_path(layout).exists()
+    assert not aggregation_cache_path(layout).exists()
+    assert not derivation_cache_path(layout).exists()
+    assert not introspection_cache_path(layout).exists()
+
+
+def test_save_derivation_only_writes_derivation_file(tmp_path: Path):
+    """Stage 4's per-section persist must not rewrite stages 2/3."""
+    layout = _layout(tmp_path)
+    cache = WalkCache()
+    cache.record_extraction("a.py", fingerprint="x", findings=[], summary="", chunks_processed=0)
+    cache.record_aggregation("intent", notes_hash="h", body="b", claims=[], contradictions=[])
+    save_extraction(layout, cache)
+    save_aggregation(layout, cache)
+    extraction_mtime = extraction_cache_path(layout).stat().st_mtime_ns
+    aggregation_mtime = aggregation_cache_path(layout).stat().st_mtime_ns
+
+    import time
+
+    time.sleep(0.01)
+    cache.record_derivation("personas", upstream_hash="u", body="p", revised=False)
+    save_derivation(layout, cache)
+    assert derivation_cache_path(layout).exists()
+    assert extraction_cache_path(layout).stat().st_mtime_ns == extraction_mtime
+    assert aggregation_cache_path(layout).stat().st_mtime_ns == aggregation_mtime
+
+
+def test_v1_caches_are_invalidated_by_version_bump(tmp_path: Path):
+    """A v1 cache file on disk loads to empty under CACHE_VERSION=2.
+
+    This is the upgrade path: existing wikis re-extract on the first
+    walk after the upgrade, then enjoy the new short-circuit on every
+    walk after.
+    """
+    layout = _layout(tmp_path)
+    extraction_cache_path(layout).parent.mkdir(parents=True, exist_ok=True)
+    extraction_cache_path(layout).write_text(
+        '{"version": 1, "entries": {"a.py": {"fingerprint": "abc", "findings": [],'
+        ' "summary": "", "chunks_processed": 0}}}'
+    )
+    aggregation_cache_path(layout).write_text(
+        '{"version": 1, "entries": {"intent": {"notes_hash": "h", "body": "b", "claims": [], "contradictions": []}}}'
+    )
+    cache = load(layout)
+    assert cache.extraction == {}
+    assert cache.aggregation == {}
+
+
 def test_hash_section_notes_changes_when_sources_change():
     """The aggregation cache key must reflect each note's `sources`.
 

tests/test_deriver.py

@@ -2,6 +2,7 @@
 
 import re
 
+from wikifi.cache import WalkCache, hash_upstream_bodies
 from wikifi.deriver import DerivedSection, derive_all
 from wikifi.sections import DERIVATIVE_SECTIONS, SECTIONS_BY_ID
 from wikifi.wiki import WikiLayout, initialize, write_section
@@ -138,3 +139,117 @@ def factory(schema, system, user):
     # The user_stories prompt must include the *derived* personas content,
     # confirming Stage 4 cascades correctly.
     assert "Derived body for personas." in seen["user_stories"]
+
+
+def test_derive_uses_cache_when_upstream_bodies_unchanged(tmp_path, mock_provider_factory):
+    """A second derive run with identical upstreams replays the cached body, no LLM call."""
+    layout = _setup(tmp_path)
+    for section in DERIVATIVE_SECTIONS:
+        _populate_upstreams(layout, section)
+
+    cache = WalkCache()
+    call_count = {"n": 0}
+
+    def factory(schema, system, user):
+        call_count["n"] += 1
+        return DerivedSection(body="First-pass body.")
+
+    provider = mock_provider_factory(json_factory=factory)
+    derive_all(layout=layout, provider=provider, cache=cache)
+    first_calls = call_count["n"]
+    assert first_calls == len(DERIVATIVE_SECTIONS)
+
+    # Second pass — same upstreams, same cache → zero LLM calls.
+    derive_all(layout=layout, provider=provider, cache=cache)
+    assert call_count["n"] == first_calls
+    # Stats should reflect that this run cached every section.
+    stats = derive_all(layout=layout, provider=provider, cache=cache)
+    assert stats.sections_cached == len(DERIVATIVE_SECTIONS)
+    assert call_count["n"] == first_calls  # still no new calls
+
+
+def test_derive_cache_misses_when_upstream_changes(tmp_path, mock_provider_factory):
+    """Editing an upstream body invalidates the derivative cache for that section."""
+    layout = _setup(tmp_path)
+    section = DERIVATIVE_SECTIONS[0]
+    _populate_upstreams(layout, section)
+
+    cache = WalkCache()
+    upstream_bodies = {
+        upstream_id: layout.section_path(upstream_id).read_text() for upstream_id in section.derived_from
+    }
+    cache.record_derivation(
+        section.id,
+        upstream_hash=hash_upstream_bodies(upstream_bodies),
+        body="Cached body.",
+        revised=False,
+    )
+
+    # Mutate one upstream — cache should miss and the LLM should be called.
+    first_upstream = SECTIONS_BY_ID[section.derived_from[0]]
+    write_section(layout, first_upstream, "Edited upstream content.")
+
+    call_count = {"n": 0}
+
+    def factory(schema, system, user):
+        call_count["n"] += 1
+        return DerivedSection(body="Re-synthesized body.")
+
+    provider = mock_provider_factory(json_factory=factory)
+    derive_all(layout=layout, provider=provider, cache=cache)
+    assert call_count["n"] >= 1
+    body = layout.section_path(section).read_text()
+    assert "Re-synthesized body." in body
+
+
+def test_derive_review_walk_does_not_reuse_unrevised_cached_body(tmp_path, mock_provider_factory):
+    """``--review`` must not silently inherit a cache entry produced without the critic."""
+    from wikifi.critic import Critique
+
+    layout = _setup(tmp_path)
+    section = DERIVATIVE_SECTIONS[0]
+    _populate_upstreams(layout, section)
+    upstream_bodies = {
+        upstream_id: layout.section_path(upstream_id).read_text() for upstream_id in section.derived_from
+    }
+
+    cache = WalkCache()
+    cache.record_derivation(
+        section.id,
+        upstream_hash=hash_upstream_bodies(upstream_bodies),
+        body="Unrevised cached body.",
+        revised=False,
+    )
+
+    call_count = {"n": 0}
+
+    def factory(schema, system, user):
+        call_count["n"] += 1
+        if schema is Critique:
+            return Critique(score=9, summary="ok")
+        return DerivedSection(body="New body under review.")
+
+    provider = mock_provider_factory(json_factory=factory)
+    derive_all(layout=layout, provider=provider, cache=cache, review=True)
+    # Review-mode walk forces re-synthesis even though the upstream hash
+    # matches the cache.
+    assert call_count["n"] >= 1
+
+
+def test_derive_persist_cache_called_after_each_section(tmp_path, mock_provider_factory):
+    """Stage 4's persist callback fires once per derivative cache update."""
+    layout = _setup(tmp_path)
+    for section in DERIVATIVE_SECTIONS:
+        _populate_upstreams(layout, section)
+
+    cache = WalkCache()
+    persist_calls = {"n": 0}
+
+    def persist():
+        persist_calls["n"] += 1
+
+    provider = mock_provider_factory(
+        json_factory=lambda schema, system, user: DerivedSection(body="body"),
+    )
+    derive_all(layout=layout, provider=provider, cache=cache, persist_cache=persist)
+    assert persist_calls["n"] == len(DERIVATIVE_SECTIONS)