fix(memory-ingest): stop dropping capture_origin at ingest (issue #368)

benchmarks/lib/bench_db.py

@@ -15,6 +15,7 @@
 import os
 from typing import Any, Callable
 
+from benchmarks.lib.capture_origin_mix import assign_capture_origins
 from mcp_server.core.memory_ingest import ingest_memories_batch
 from mcp_server.core.pg_recall import (
     assemble_context as pg_assemble_context,
@@ -25,6 +26,23 @@
 from mcp_server.infrastructure.pg_store import PgMemoryStore
 
 
+def _apply_capture_origin_mix(memories: list[dict[str, Any]]) -> None:
+    """Stamp every memory lacking `capture_origin` with a value drawn from
+    the measured production mixture, in place.
+
+    Extracted as a free function (rather than inlined in `load_memories`) so
+    it is testable without a live PostgreSQL connection — see
+    `tests_py/benchmarks/test_bench_db_capture_origin.py`.
+
+    A memory that already sets `capture_origin` (e.g. the adversarial-corpus
+    montage, which needs specific per-pair values, not the aggregate mix) is
+    left untouched — `setdefault` only fills what's missing.
+    """
+    origins = assign_capture_origins(len(memories))
+    for mem, origin in zip(memories, origins, strict=True):
+        mem.setdefault("capture_origin", origin)
+
+
 class BenchmarkDB:
     """Thin passthrough to the production PG pipeline.
 
@@ -165,8 +183,19 @@ def load_memories(
         """Delegate to mcp_server.core.memory_ingest.ingest_memories_batch().
 
         Returns (ids, source_map) where source_map maps memory_id → source string.
+
+        Assigns a `capture_origin` to every memory that does not already carry
+        one, drawn from the measured production mixture
+        (benchmarks/lib/capture_origin_mix.py) instead of falling through to
+        insert_memory's "unknown" default. Without this every LME/LoCoMo/BEAM
+        row landed in `capture_origin='unknown'`, which
+        core.capture_origin.trust_factor demotes uniformly — a uniform
+        multiplier cannot change WRRF order, so the trust-factor W sweep
+        (docs/provenance/trust-factor-calibration.md) could not discriminate
+        between values of W (issue #368 follow-up).
         """
         assert self._store is not None, "Call open() first"
+        _apply_capture_origin_mix(memories)
         ids, source_map = ingest_memories_batch(
             memories,
             self._store,

benchmarks/lib/capture_origin_mix.py

@@ -0,0 +1,101 @@
+"""Realistic capture_origin mixture for benchmark corpora (issue #368 fix).
+
+Why this exists: the trust-factor gated arm
+(docs/provenance/trust-factor-calibration.md) reported LME identical to four
+decimals across W in {1.0, 0.7, 0.6, 0.5} because LME/LoCoMo/BEAM never set
+capture_origin on the memories they insert -- every row fell through to the
+column default 'unknown' (pg_schema.py, pg_store.py:567), which
+mcp_server.core.capture_origin.trust_factor demotes uniformly. A uniform
+multiplier cannot change WRRF order, so the gated arm was structurally
+incapable of measuring W's effect on ranking; it could only prove the
+non-regression of everything else in the pipeline.
+
+This module assigns each benchmark memory a capture_origin drawn from the
+distribution that ACTUALLY occurs in production Cortex usage, so the gated
+arm mixes trusted and untrusted content the way a live store does and can
+therefore discriminate W.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+# source: measured 2026-08-10 on this machine's own Claude Code history --
+#   886 session transcripts under ~/.claude/projects/**/*.jsonl (the only
+#   representative "existing store" of real Cortex-adjacent usage available
+#   in this environment; the local memory.db predates the capture_origin
+#   migration and carries no such column to sample). Two greps:
+#
+#   1) tool_use frequency for every tool name that
+#      mcp_server.core.capture_origin.classify_capture_origin maps to a
+#      non-UNKNOWN origin (issue #365 _LOCAL_ACTION_TOOLS / _NETWORK_TOOLS),
+#      restricted to the tools hooks/post_tool_capture.py actually
+#      auto-captures (_HIGH_VALUE_TOOLS + _LIGHT_VALUE_TOOLS +
+#      _CONDITIONAL_TOOLS):
+#
+#        grep -rhoE '"name": ?"(Edit|Write|MultiEdit|NotebookEdit|
+#          NotebookRead|Bash|Read|Glob|Grep|WebFetch|WebSearch)"'
+#          ~/.claude/projects/ | sort | uniq -c
+#
+#      -> Bash 29088, Read 5656, Edit 3992, Write 1162, WebFetch 583,
+#         WebSearch 466, Glob 5, MultiEdit/NotebookEdit/NotebookRead/Grep 0.
+#      local_action (Bash+Read+Edit+Write+Glob) = 39903
+#      network (WebFetch+WebSearch)             = 1049
+#
+#   2) explicit `remember` MCP tool_use calls (ORIGIN_DELIBERATE -- a direct
+#      remember with no origin_tool resolves DELIBERATE per
+#      handlers/remember.py) across the same transcripts:
+#
+#        grep -rhoE '"name": ?"[a-zA-Z0-9_.-]*remember[a-zA-Z0-9_.-]*"'
+#          ~/.claude/projects/ | sort | uniq -c
+#
+#      -> mcp__plugin_cortex_cortex__remember 278,
+#         mcp__plugin_hypermnesia-mcp_cortex__remember 83 => deliberate = 361
+#
+#   Pool = 39903 + 1049 + 361 = 41313.
+#     local_action = 39903 / 41313 = 0.9659
+#     network      =  1049 / 41313 = 0.0254
+#     deliberate   =   361 / 41313 = 0.0087
+#   Rounded to 3dp so the three sum to 1.000 exactly.
+#
+#   Limitation, stated rather than smoothed over: this is one user's
+#   tool-call frequency, not a filtered count of rows that actually pass
+#   _should_capture's length/content gates, and not a multi-user production
+#   sample. It is nonetheless a measurement of real usage, not an invented
+#   split, and it is the only "existing store" available to measure from in
+#   this environment. unknown/legacy are omitted at 0.0: every current
+#   writer (remember handler, post_tool_capture hook) resolves one of the
+#   three origins below; legacy is written only once, by the migration,
+#   never by a live write path, and no live writer produces unknown for a
+#   tool name it recognises -- the measured 0.0 IS the production rate for a
+#   fully-migrated store, not a gap in the count.
+CAPTURE_ORIGIN_MIX: tuple[tuple[str, float], ...] = (
+    ("local_action", 0.966),
+    ("network", 0.025),
+    ("deliberate", 0.009),
+)
+
+_ORIGINS = tuple(o for o, _ in CAPTURE_ORIGIN_MIX)
+_WEIGHTS = tuple(w for _, w in CAPTURE_ORIGIN_MIX)
+
+_SUM_TOLERANCE = 1e-9
+assert abs(sum(_WEIGHTS) - 1.0) < _SUM_TOLERANCE, "CAPTURE_ORIGIN_MIX must sum to 1.0"
+
+# Deterministic across runs (benchmarks/reproduce.sh's whole premise is "hit
+# play, get the same numbers" -- see its module docstring). Not a calibrated
+# quantity, just a fixed draw seed; any constant works, this one is arbitrary.
+DEFAULT_SEED = 368
+
+
+def assign_capture_origins(n: int, seed: int = DEFAULT_SEED) -> list[str]:
+    """Return `n` capture_origin values drawn i.i.d. from CAPTURE_ORIGIN_MIX.
+
+    Pre: n >= 0.
+    Post: len(result) == n; each element is one of _ORIGINS; deterministic
+    for a given (n, seed): same call, same output, every process.
+    """
+    rng = np.random.default_rng(seed)
+    if n == 0:
+        return []
+    idx = rng.choice(len(_ORIGINS), size=n, p=_WEIGHTS)
+    return [_ORIGINS[i] for i in idx]

benchmarks/lib/write_manifest.py

@@ -8,13 +8,138 @@
 Usage (from reproduce.sh):
     python benchmarks/lib/write_manifest.py \\
         RESULTS_DIR GIT_SHA DATASET_SHA256 PG_IMAGE CONTAINER PG_PORT RUNNER_PID
+
+    # Cell-start snapshot (call BEFORE start_db, so it also predates the
+    # benchmark's own container/DB overhead):
+    python benchmarks/lib/write_manifest.py --snapshot RESULTS_DIR
 """
 
 import json
+import os
 import platform
+import subprocess
 import sys
+from datetime import datetime, timezone
 from pathlib import Path
 
+_START_SNAPSHOT_NAME = "START_SNAPSHOT.json"
+
+
+def machine_load_snapshot() -> dict:
+    """Load average + concurrent pytest/container counts, as this run saw them.
+
+    2026-08-10 incident: a 5-cell trust-factor sweep ran while three other
+    agents' full pytest suites were active on the same machine (load average
+    ~11-14 on a 10-core box); one cell crashed on a native fatal error, and
+    the crash was the ONLY visible signal — cells that merely finished under
+    the same contention could have returned degraded numbers (saturated
+    connection pool, cold cache, GC pressure) with nothing in the artifact to
+    show it. The whole grid was discarded and re-run rather than salvaged,
+    per this project's own rule: a measurement from a harness with a known
+    defect is invalid and is redone, not patched after the fact — and
+    contention is exactly such a defect. This snapshot is recorded so that
+    rule can be applied by inspection later, instead of by asking whoever
+    happened to be watching at the time.
+
+    Taken TWICE per cell (2026-08-10 follow-up, same incident): once at cell
+    START (`write_start_snapshot`, called before `start_db` so it predates
+    the container/DB overhead too) and once at cell END (inside
+    `build_manifest`, the pre-existing call). A crash is the visible failure
+    mode; a cell that merely FINISHES under contention (saturated pool, cold
+    cache, GC pressure) is the invisible one, and a single end-of-run
+    snapshot cannot distinguish "this cell ran under load throughout" from
+    "load spiked right at the end". Two points at least bound the window.
+
+    Best-effort: any probe that fails records `None`/`"unresolved"` rather
+    than aborting manifest generation, matching this module's other fields.
+    """
+    try:
+        load1, load5, load15 = os.getloadavg()
+    except OSError:  # not available on this platform (e.g. Windows)
+        load1 = load5 = load15 = None
+
+    def _run(cmd: list[str], *, env: dict[str, str] | None = None) -> str | None:
+        try:
+            return subprocess.run(
+                cmd, capture_output=True, text=True, timeout=10, check=False, env=env
+            ).stdout
+        except (OSError, subprocess.SubprocessError):
+            return None
+
+    # Filtered in Python, not via a shell `grep -c "[p]ytest"` idiom: a
+    # subprocess.run argv has no shell to bracket-escape a self-match, so the
+    # filter runs here instead, over the same process list that idiom reads.
+    #
+    # The `COLUMNS` override below fixes a real undercount, not just a test
+    # flake (caught by
+    # tests_py/benchmarks/test_write_manifest_machine_load.py's own
+    # self-referential assertion failing on GitHub's Linux CI runner,
+    # 2026-08-10): both BSD ps (macOS) and GNU procps (Linux) truncate the
+    # COMMAND column to `$COLUMNS` when stdout is not a terminal and COLUMNS
+    # is unset, and `ps aux`'s fixed-width USER/PID/... columns alone can
+    # exceed a default 80-column budget before COMMAND even starts — cutting
+    # off the "pytest" substring entirely on a long interpreter path. A wide
+    # COLUMNS override is the standard fix for both implementations.
+    ps_out = _run(["ps", "aux"], env={**os.environ, "COLUMNS": "1000"})
+    pytest_procs = (
+        None
+        if ps_out is None
+        else sum(
+            1 for line in ps_out.splitlines() if "pytest" in line and "grep" not in line
+        )
+    )
+
+    docker_out = _run(["docker", "ps", "-q"])
+    docker_containers = (
+        None
+        if docker_out is None
+        else len([line for line in docker_out.splitlines() if line.strip()])
+    )
+
+    return {
+        "load_average_1m": load1,
+        "load_average_5m": load5,
+        "load_average_15m": load15,
+        "cpu_count": os.cpu_count(),
+        "concurrent_pytest_processes": pytest_procs,
+        "concurrent_docker_containers": docker_containers,
+    }
+
+
+def write_start_snapshot(results_dir: str) -> Path:
+    """Capture + persist the cell-start machine-load snapshot.
+
+    Called from reproduce.sh before `start_db`, so `RESULTS_DIR` already
+    exists (created by `main()`'s `mkdir -p`) but nothing benchmark-specific
+    has run yet. `build_manifest` reads this file back at cell end and folds
+    it into the final MANIFEST.json as `machine_load_at_start`, alongside the
+    end-of-run `machine_load_at_end` — see `machine_load_snapshot`'s
+    docstring for why both points are recorded.
+    """
+    out = Path(results_dir) / _START_SNAPSHOT_NAME
+    payload = {
+        "captured_at_utc": datetime.now(timezone.utc).isoformat(),
+        "machine_load": machine_load_snapshot(),
+    }
+    out.write_text(json.dumps(payload, indent=2))
+    return out
+
+
+def _read_start_snapshot(results_dir: str) -> dict | None:
+    """Read back the cell-start snapshot written by `write_start_snapshot`.
+
+    Returns None (never raises) when absent — e.g. a `reproduce.sh` call
+    that predates this fix, or a caller that skipped the `--snapshot` step.
+    A missing start snapshot must not block the end-of-run manifest from
+    being written; `machine_load_at_start` is simply absent in that case,
+    which is itself an observable fact rather than a silent guess.
+    """
+    path = Path(results_dir) / _START_SNAPSHOT_NAME
+    try:
+        return json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return None
+
 
 def ver(pkg: str) -> str:
     try:
@@ -89,8 +214,17 @@ def build_manifest(
     pg_port: str,
     pid: str,
 ) -> dict:
+    start_snapshot = _read_start_snapshot(results_dir)
     return {
         "git_sha": git_sha,
+        # Alongside git_sha, not buried: see machine_load_snapshot's
+        # docstring for why (2026-08-10 sweep-contention incident). Two
+        # points, not one — `_at_start` is None when reproduce.sh's
+        # `--snapshot` step was never called for this results_dir.
+        "machine_load_at_start": (
+            start_snapshot["machine_load"] if start_snapshot else None
+        ),
+        "machine_load_at_end": machine_load_snapshot(),
         "longmemeval_dataset_sha256": ds_sha,
         "pg_image": pg_image,
         # Per-run container isolation fix (2026-07-11, incident: two concurrent
@@ -115,11 +249,19 @@ def build_manifest(
         },
         "embedding_model_revision": embedding_revision(),
         **reranker_fields(),
-        "results_files": sorted(p.name for p in Path(results_dir).glob("*.json")),
+        "results_files": sorted(
+            p.name
+            for p in Path(results_dir).glob("*.json")
+            if p.name != _START_SNAPSHOT_NAME
+        ),
     }
 
 
 def main(argv: list[str]) -> None:
+    if len(argv) > 1 and argv[1] == "--snapshot":
+        out = write_start_snapshot(argv[2])
+        print(f"==> Wrote {out}")
+        return
     results_dir = argv[1]
     manifest = build_manifest(*argv[1:8])
     out = Path(results_dir) / "MANIFEST.json"

benchmarks/reproduce.sh

@@ -395,6 +395,17 @@ main() {
     fi
     mkdir -p "$RESULTS_DIR"
 
+    # Machine-load snapshot at cell START, before start_db so it predates
+    # even the container/DB overhead — the counterpart to write_manifest's
+    # end-of-run snapshot (2026-08-10 sweep-contention incident: a crashed
+    # cell was the only visible symptom of contention that also silently
+    # affected cells which merely finished; see write_manifest.py's
+    # machine_load_snapshot docstring). Uses the base interpreter, not
+    # `--extra benchmarks`, deliberately: this must succeed even if the
+    # benchmarks extra install itself is what's under contention.
+    uv run python "$REPO_ROOT/benchmarks/lib/write_manifest.py" \
+        --snapshot "$RESULTS_DIR"
+
     # Only fetch datasets for benchmarks that will actually run.
     if [ "$RUN_BENCHMARKS" = "1" ] && want_bench longmemeval; then fetch_longmemeval; fi
     if [ "$RUN_ABLATION" = "1" ] && [ "$ABLATE_ON" = "longmemeval-s" ]; then fetch_longmemeval; fi

mcp_server/core/memory_ingest.py

@@ -141,6 +141,18 @@ def ingest_memory(
                 "is_protected": auto_protect,
                 "agent_context": agent_ctx,
                 "is_global": is_global,
+                # issue #368 gated-arm fix: pass the caller's capture_origin
+                # through like every other metadata field above (source,
+                # tags, heat, ...) instead of dropping it silently. Without
+                # this, every ingest_memory caller — including the LME/
+                # LoCoMo/BEAM benchmark harnesses — fell through to
+                # insert_memory's "unknown" default regardless of what the
+                # caller set, which is why the trust-factor calibration's
+                # gated arm could not discriminate W (docs/provenance/
+                # trust-factor-calibration.md §"What this measurement does
+                # and does not establish"). Default unchanged: a caller that
+                # omits the field still gets "unknown", identical to before.
+                "capture_origin": memory.get("capture_origin", "unknown"),
             }
         )
         ids.append(mid)