Draft regulated-memory evaluation preprint

paper/regulated-memory-eval/README.md

@@ -0,0 +1,42 @@
+# Regulated-memory evaluation preprint
+
+This directory contains the source for a preprint about evaluating agent-memory
+systems as auditable records rather than only as conversational recall systems.
+
+## Build
+
+```bash
+pdflatex main.tex
+bibtex main
+pdflatex main.tex
+pdflatex main.tex
+```
+
+The repository does not vendor a TeX distribution. The source is intentionally
+limited to common arXiv-compatible packages.
+
+The source was also built successfully with Tectonic 0.16.9 on July 17, 2026.
+The verified output is four letter-sized pages. Text extraction reported no
+unresolved references, replacement glyphs, or Unicode em dashes. All four rendered
+pages were visually inspected after hiding PDF link borders.
+
+## Evidence status
+
+- Lians, mem0 OSS, and Graphiti OSS were executed live.
+- A fresh Lians 0.4.1 live run is archived under `evidence/` and validates against
+  the versioned JSON Schema.
+- Letta, Hindsight, and Supermemory are capability-assessed from public APIs.
+- Every vendor has a public right of reply.
+- Capability-assessed columns must not be described as live results until their
+  adapters are executed against a supported vendor configuration.
+
+## Before submission
+
+- Incorporate material vendor corrections.
+- Rerun mem0 OSS and Graphiti OSS, then archive exact dependency versions, run
+  identifiers, and machine-readable per-cell evidence. The original July 4 raw
+  outputs were not retained and are not reconstructed from prose.
+- Complete author and affiliation metadata.
+- Build and visually inspect the PDF. Completed July 17, 2026.
+- Run an external methodology review.
+

paper/regulated-memory-eval/evidence/README.md

@@ -0,0 +1,17 @@
+# Machine-readable evaluation evidence
+
+This directory archives exact outputs from the regulated-memory harness. Every live-run file records the harness commit, package version, adapter mode, execution configuration, timestamp, and per-invariant details.
+
+Validate an evidence file against `schema.json` before citing it. The JSON Schema separates executed results from capability assessments so an unexecuted vendor column cannot be mistaken for live evidence.
+
+## Archived runs
+
+| File | System | Status | Result |
+|---|---|---|---|
+| `lians-local-2026-07-17.json` | Lians 0.4.1 with `LocalLiansClient` | Executed live | 5 of 5 invariants passed |
+
+## Evidence still required
+
+The July 4 mem0 OSS and Graphiti OSS runs were summarized in `docs/regulated-eval-results.md`, but their original machine-readable outputs were not archived. They must be rerun and saved here before the preprint leaves draft status. No JSON files are reconstructed from narrative descriptions.
+
+Vendor-hosted columns and capability-assessed columns must use `execution_status: capability_assessed` unless a complete live harness report is available.

paper/regulated-memory-eval/evidence/lians-local-2026-07-17.json

@@ -0,0 +1,80 @@
+{
+  "evaluation": "lians-regulated-memory-five-invariant",
+  "executed_at": "2026-07-17T20:29:02.204939+00:00",
+  "report": {
+    "checks": [
+      {
+        "capability_absent": false,
+        "check": "stale_revision_suppression",
+        "detail": {
+          "current_retrieved": true,
+          "stale_excluded": true
+        },
+        "passed": true,
+        "status": "pass"
+      },
+      {
+        "capability_absent": false,
+        "check": "point_in_time_reconstruction",
+        "detail": {
+          "as_of_value_retrieved": true
+        },
+        "passed": true,
+        "status": "pass"
+      },
+      {
+        "capability_absent": false,
+        "check": "erasure_proof",
+        "detail": {
+          "content_unrecoverable": true,
+          "proof_artifact": true
+        },
+        "passed": true,
+        "status": "pass"
+      },
+      {
+        "capability_absent": false,
+        "check": "lookahead_contamination_detection",
+        "detail": {
+          "flags": 1,
+          "is_clean": false
+        },
+        "passed": true,
+        "status": "pass"
+      },
+      {
+        "capability_absent": false,
+        "check": "audit_state_reconstruction",
+        "detail": {
+          "total": 1
+        },
+        "passed": true,
+        "status": "pass"
+      }
+    ],
+    "note": "barrier_leakage verified separately on Postgres RLS (non-superuser role)",
+    "passed": 5,
+    "score": 1.0,
+    "total": 5
+  },
+  "run_id": "lians-local-2026-07-17",
+  "schema_version": "1.0",
+  "source": {
+    "git_commit": "b3fbbd29b798727e5dae37257c340047fcc80eb3",
+    "harness": "agentmem/benchmarks/regulated_eval.py",
+    "repository": "https://github.com/Lians-ai/Lians"
+  },
+  "system": {
+    "adapter": "LocalLiansClient",
+    "configuration": {
+      "embedding_provider": "local",
+      "ephemeral_test_encryption_key": true,
+      "external_api_key": false,
+      "storage": "temporary SQLite"
+    },
+    "execution_status": "executed_live",
+    "name": "Lians",
+    "package": "lians-sdk",
+    "package_version": "0.4.1"
+  }
+}

paper/regulated-memory-eval/evidence/schema.json

@@ -0,0 +1,81 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/Lians-ai/Lians/blob/master/paper/regulated-memory-eval/evidence/schema.json",
+  "title": "Lians regulated-memory evaluation evidence",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "evaluation",
+    "run_id",
+    "executed_at",
+    "system",
+    "source",
+    "report"
+  ],
+  "properties": {
+    "schema_version": { "const": "1.0" },
+    "evaluation": { "const": "lians-regulated-memory-five-invariant" },
+    "run_id": { "type": "string", "minLength": 1 },
+    "executed_at": { "type": "string", "format": "date-time" },
+    "system": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "execution_status", "adapter", "package", "package_version", "configuration"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "execution_status": { "enum": ["executed_live", "capability_assessed"] },
+        "adapter": { "type": "string", "minLength": 1 },
+        "package": { "type": "string", "minLength": 1 },
+        "package_version": { "type": "string", "minLength": 1 },
+        "configuration": { "type": "object" }
+      }
+    },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["repository", "git_commit", "harness"],
+      "properties": {
+        "repository": { "type": "string", "format": "uri" },
+        "git_commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
+        "harness": { "type": "string", "minLength": 1 }
+      }
+    },
+    "report": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["checks", "passed", "total", "score", "note"],
+      "properties": {
+        "checks": {
+          "type": "array",
+          "minItems": 5,
+          "maxItems": 5,
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["check", "passed", "status", "capability_absent", "detail"],
+            "properties": {
+              "check": {
+                "enum": [
+                  "stale_revision_suppression",
+                  "point_in_time_reconstruction",
+                  "erasure_proof",
+                  "lookahead_contamination_detection",
+                  "audit_state_reconstruction"
+                ]
+              },
+              "passed": { "type": "boolean" },
+              "status": { "enum": ["pass", "partial", "fail"] },
+              "capability_absent": { "type": "boolean" },
+              "detail": { "type": "object" }
+            }
+          }
+        },
+        "passed": { "type": "integer", "minimum": 0, "maximum": 5 },
+        "total": { "const": 5 },
+        "score": { "type": "number", "minimum": 0, "maximum": 1 },
+        "note": { "type": "string" }
+      }
+    }
+  }
+}