diff --git a/paper/regulated-memory-eval/README.md b/paper/regulated-memory-eval/README.md new file mode 100644 index 0000000..1d7a143 --- /dev/null +++ b/paper/regulated-memory-eval/README.md @@ -0,0 +1,42 @@ +# Regulated-memory evaluation preprint + +This directory contains the source for a preprint about evaluating agent-memory +systems as auditable records rather than only as conversational recall systems. + +## Build + +```bash +pdflatex main.tex +bibtex main +pdflatex main.tex +pdflatex main.tex +``` + +The repository does not vendor a TeX distribution. The source is intentionally +limited to common arXiv-compatible packages. + +The source was also built successfully with Tectonic 0.16.9 on July 17, 2026. +The verified output is four letter-sized pages. Text extraction reported no +unresolved references, replacement glyphs, or Unicode em dashes. All four rendered +pages were visually inspected after hiding PDF link borders. + +## Evidence status + +- Lians, mem0 OSS, and Graphiti OSS were executed live. +- A fresh Lians 0.4.1 live run is archived under `evidence/` and validates against + the versioned JSON Schema. +- Letta, Hindsight, and Supermemory are capability-assessed from public APIs. +- Every vendor has a public right of reply. +- Capability-assessed columns must not be described as live results until their + adapters are executed against a supported vendor configuration. + +## Before submission + +- Incorporate material vendor corrections. +- Rerun mem0 OSS and Graphiti OSS, then archive exact dependency versions, run + identifiers, and machine-readable per-cell evidence. The original July 4 raw + outputs were not retained and are not reconstructed from prose. +- Complete author and affiliation metadata. +- Build and visually inspect the PDF. Completed July 17, 2026. +- Run an external methodology review. + diff --git a/paper/regulated-memory-eval/evidence/README.md b/paper/regulated-memory-eval/evidence/README.md new file mode 100644 index 0000000..b136053 --- /dev/null +++ b/paper/regulated-memory-eval/evidence/README.md @@ -0,0 +1,17 @@ +# Machine-readable evaluation evidence + +This directory archives exact outputs from the regulated-memory harness. Every live-run file records the harness commit, package version, adapter mode, execution configuration, timestamp, and per-invariant details. + +Validate an evidence file against `schema.json` before citing it. The JSON Schema separates executed results from capability assessments so an unexecuted vendor column cannot be mistaken for live evidence. + +## Archived runs + +| File | System | Status | Result | +|---|---|---|---| +| `lians-local-2026-07-17.json` | Lians 0.4.1 with `LocalLiansClient` | Executed live | 5 of 5 invariants passed | + +## Evidence still required + +The July 4 mem0 OSS and Graphiti OSS runs were summarized in `docs/regulated-eval-results.md`, but their original machine-readable outputs were not archived. They must be rerun and saved here before the preprint leaves draft status. No JSON files are reconstructed from narrative descriptions. + +Vendor-hosted columns and capability-assessed columns must use `execution_status: capability_assessed` unless a complete live harness report is available. diff --git a/paper/regulated-memory-eval/evidence/lians-local-2026-07-17.json b/paper/regulated-memory-eval/evidence/lians-local-2026-07-17.json new file mode 100644 index 0000000..832fdf5 --- /dev/null +++ b/paper/regulated-memory-eval/evidence/lians-local-2026-07-17.json @@ -0,0 +1,80 @@ +{ + "evaluation": "lians-regulated-memory-five-invariant", + "executed_at": "2026-07-17T20:29:02.204939+00:00", + "report": { + "checks": [ + { + "capability_absent": false, + "check": "stale_revision_suppression", + "detail": { + "current_retrieved": true, + "stale_excluded": true + }, + "passed": true, + "status": "pass" + }, + { + "capability_absent": false, + "check": "point_in_time_reconstruction", + "detail": { + "as_of_value_retrieved": true + }, + "passed": true, + "status": "pass" + }, + { + "capability_absent": false, + "check": "erasure_proof", + "detail": { + "content_unrecoverable": true, + "proof_artifact": true + }, + "passed": true, + "status": "pass" + }, + { + "capability_absent": false, + "check": "lookahead_contamination_detection", + "detail": { + "flags": 1, + "is_clean": false + }, + "passed": true, + "status": "pass" + }, + { + "capability_absent": false, + "check": "audit_state_reconstruction", + "detail": { + "total": 1 + }, + "passed": true, + "status": "pass" + } + ], + "note": "barrier_leakage verified separately on Postgres RLS (non-superuser role)", + "passed": 5, + "score": 1.0, + "total": 5 + }, + "run_id": "lians-local-2026-07-17", + "schema_version": "1.0", + "source": { + "git_commit": "b3fbbd29b798727e5dae37257c340047fcc80eb3", + "harness": "agentmem/benchmarks/regulated_eval.py", + "repository": "https://github.com/Lians-ai/Lians" + }, + "system": { + "adapter": "LocalLiansClient", + "configuration": { + "embedding_provider": "local", + "ephemeral_test_encryption_key": true, + "external_api_key": false, + "storage": "temporary SQLite" + }, + "execution_status": "executed_live", + "name": "Lians", + "package": "lians-sdk", + "package_version": "0.4.1" + } +} diff --git a/paper/regulated-memory-eval/evidence/schema.json b/paper/regulated-memory-eval/evidence/schema.json new file mode 100644 index 0000000..f82fe1e --- /dev/null +++ b/paper/regulated-memory-eval/evidence/schema.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Lians-ai/Lians/blob/master/paper/regulated-memory-eval/evidence/schema.json", + "title": "Lians regulated-memory evaluation evidence", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "evaluation", + "run_id", + "executed_at", + "system", + "source", + "report" + ], + "properties": { + "schema_version": { "const": "1.0" }, + "evaluation": { "const": "lians-regulated-memory-five-invariant" }, + "run_id": { "type": "string", "minLength": 1 }, + "executed_at": { "type": "string", "format": "date-time" }, + "system": { + "type": "object", + "additionalProperties": false, + "required": ["name", "execution_status", "adapter", "package", "package_version", "configuration"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "execution_status": { "enum": ["executed_live", "capability_assessed"] }, + "adapter": { "type": "string", "minLength": 1 }, + "package": { "type": "string", "minLength": 1 }, + "package_version": { "type": "string", "minLength": 1 }, + "configuration": { "type": "object" } + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "git_commit", "harness"], + "properties": { + "repository": { "type": "string", "format": "uri" }, + "git_commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "harness": { "type": "string", "minLength": 1 } + } + }, + "report": { + "type": "object", + "additionalProperties": false, + "required": ["checks", "passed", "total", "score", "note"], + "properties": { + "checks": { + "type": "array", + "minItems": 5, + "maxItems": 5, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["check", "passed", "status", "capability_absent", "detail"], + "properties": { + "check": { + "enum": [ + "stale_revision_suppression", + "point_in_time_reconstruction", + "erasure_proof", + "lookahead_contamination_detection", + "audit_state_reconstruction" + ] + }, + "passed": { "type": "boolean" }, + "status": { "enum": ["pass", "partial", "fail"] }, + "capability_absent": { "type": "boolean" }, + "detail": { "type": "object" } + } + } + }, + "passed": { "type": "integer", "minimum": 0, "maximum": 5 }, + "total": { "const": 5 }, + "score": { "type": "number", "minimum": 0, "maximum": 1 }, + "note": { "type": "string" } + } + } + } +} diff --git a/paper/regulated-memory-eval/main.tex b/paper/regulated-memory-eval/main.tex new file mode 100644 index 0000000..e0d3e9b --- /dev/null +++ b/paper/regulated-memory-eval/main.tex @@ -0,0 +1,228 @@ +\documentclass[10pt]{article} + +\usepackage[margin=1in]{geometry} +\usepackage{booktabs} +\usepackage[hidelinks]{hyperref} +\usepackage{microtype} +\usepackage{tabularx} + +\title{Can an Agent Memory Layer Prove What the Agent Knew?\\ +An Evaluation of Regulated-Memory Invariants} +\author{Ethan Beirne\\Lians\\\texttt{https://www.lians.ai}} +\date{July 2026} + +\begin{document} +\maketitle + +\begin{abstract} +Long-term agent-memory benchmarks primarily measure whether a system can retrieve +useful evidence from conversational or interaction histories. Regulated uses add a +different requirement: an operator may need to reconstruct what an agent knew at a +past decision time, demonstrate that later facts could not influence that decision, +and prove that erased material is no longer recoverable. We introduce an open +evaluation of five regulated-memory invariants: stale-revision suppression, +point-in-time recall, provable erasure, lookahead protection, and historical +audit-state reconstruction. The harness evaluates product-level primitives rather +than general architecture quality. Lians, mem0 OSS, and Graphiti OSS are executed +live; Letta, Hindsight, and Supermemory are capability-assessed from their public API +surfaces pending vendor-supplied live configurations. Under the stated scoring rule, +Lians scores 5.0 of 5, Graphiti 2.0, Letta 1.0, Hindsight 1.0, Supermemory 1.0, and +mem0 OSS 0.5. These results do not establish general memory superiority. They expose +a distinct evaluation axis and provide runnable adapters, per-cell evidence, and a +public correction process for testing it. +\end{abstract} + +\section{Introduction} + +Persistent memory is becoming a standard component of long-running agents. +LoCoMo and LongMemEval test long-horizon conversational recall +\cite{locomo2024,longmemeval2024}, while LongMemEval-V2 evaluates whether agents can +internalize environment-specific experience \cite{longmemevalv2_2026}. These tasks +are important, but they do not directly test whether a deployed memory layer can +serve as evidence about a past agent decision. + +That distinction matters when facts change. A system can answer the current question +correctly while being unable to reconstruct the knowledge state used yesterday. It +can delete a row while being unable to produce an erasure artifact. It can retrieve a +highly relevant fact that was not knowable at the simulated decision time. Recent +work has made state coordination, contradiction resolution, and bitemporal operators +first-class memory problems \cite{toki2026,worlddb2026,atma2026}. We complement that +work with a product-facing evaluation of auditable operational primitives. + +This paper contributes: + +\begin{enumerate} + \item Five falsifiable regulated-memory invariants with explicit pass, partial, + and absent criteria. + \item A runnable adapter interface that separates live behavioral evidence from + capability assessment. + \item An initial six-system comparison with per-cell evidence and public vendor + right of reply. + \item A methodology for evaluating what an agent could have known at a past time, + rather than only whether it can answer correctly now. +\end{enumerate} + +\section{Evaluation Contract} + +The evaluation treats each invariant as a turnkey product primitive. A pass requires +a supported API path that satisfies the full invariant. Partial credit indicates a +related capability that leaves required filtering, proof construction, or state +reconstruction to the application. Absent means that no supported primitive was +identified. Scores are one point for pass, one half point for partial, and zero for +absent. + +\subsection{Invariants} + +\paragraph{Stale-revision suppression.} +After a keyed fact is revised, default retrieval must not return the superseded value +as current context. Preserving old values for history is compatible with a pass if +the retrieval contract distinguishes them deterministically. + +\paragraph{Point-in-time recall.} +The caller can query the knowledge state as known at a prior transaction time. This +is distinct from filtering only on when a fact was valid in the external world. + +\paragraph{Provable erasure.} +Erased content becomes cryptographically unrecoverable and the system emits a proof +artifact or stable request reference. Behavioral non-retrieval without a proof +artifact receives partial credit. + +\paragraph{Lookahead protection.} +The system prevents or flags retrieval of facts that were not knowable at a declared +simulation or decision time. + +\paragraph{Historical audit-state reconstruction.} +The caller can reproduce the complete knowledge state that was available at a past +time, including the effect of revisions and erasures under the documented policy. + +\section{Systems and Execution Status} + +Table~\ref{tab:status} distinguishes behavioral runs from capability assessment. This +distinction is central. A capability-assessed cell is a documented hypothesis about +the public product surface, not a claim that the system was executed. + +\begin{table}[h] +\centering +\caption{Evidence status by system.} +\label{tab:status} +\begin{tabular}{lll} +\toprule +System & Evidence mode & Configuration \\ +\midrule +Lians & Live & LocalLiansClient, version 0.4.0 \\ +mem0 & Live & OSS 2.0.11, documented OpenAI configuration \\ +Graphiti & Live & OSS 0.29.2, OpenAI clients, embedded Kuzu \\ +Letta & Capability-assessed & Public API and adapter \\ +Hindsight & Capability-assessed & Public API and adapter \\ +Supermemory & Capability-assessed & Public API and adapter \\ +\bottomrule +\end{tabular} +\end{table} + +The live OSS configurations follow vendor quickstarts. Any accommodation required to +complete a run is recorded in the public results appendix. Hosted adapters switch to +live execution when the corresponding credentials are supplied. The harness does not +convert an unexecuted capability map into behavioral evidence. + +\section{Initial Results} + +\begin{table*}[t] +\centering +\caption{Initial regulated-memory results. P indicates pass, H indicates half credit, +and A indicates absent. Capability-assessed columns remain subject to live vendor +configuration.} +\label{tab:results} +\begin{tabularx}{\textwidth}{Xcccccc} +\toprule +Invariant & Lians & Graphiti & Letta & Hindsight & Supermemory & mem0 \\ +\midrule +Stale revision suppressed & P & H & H & H & H & A \\ +Point-in-time recall & P & H & A & H & A & A \\ +Provable erasure & P & H & H & A & H & H \\ +Lookahead protection & P & A & A & A & A & A \\ +Audit-state snapshot & P & H & A & A & A & A \\ +\midrule +Score & 5.0 & 2.0 & 1.0 & 1.0 & 1.0 & 0.5 \\ +\bottomrule +\end{tabularx} +\end{table*} + +The results identify a product gap rather than a universal ranking. Graphiti's +contradiction invalidation correctly backdated stale edges in the live run, but +default retrieval returned invalidated edges unless the caller filtered them. mem0 +made erased content behaviorally unretrievable but emitted no proof artifact. These +capabilities receive partial credit under the same rule applied to Lians and all +other systems. + +\section{Reproducibility and Correction Process} + +The implementation, adapters, generated table, and per-cell notes are available in +the Lians repository \cite{lians2026}. A basic run is: + +\begin{verbatim} +cd agentmem +python -m benchmarks.compare_regulated +\end{verbatim} + +Optional vendor credentials activate hosted adapters. The public results document +records exact OSS versions and accommodations. Each evaluated vendor received a +public request to identify missed APIs, correct characterizations, or provide a +supported configuration. Accepted corrections change the adapter and the published +result, whether they improve or reduce a vendor's score. + +\section{Threats to Validity} + +\paragraph{Evaluator conflict of interest.} +The evaluation is authored by the developer of Lians. Open adapters, public evidence, +and vendor correction requests reduce but do not eliminate this conflict. Independent +replication is required. + +\paragraph{Mixed evidence modes.} +Three columns are capability-assessed rather than live. They must not be interpreted +as behavioral measurements. The primary near-term objective is to replace these +columns with supported live configurations. + +\paragraph{Configuration sensitivity.} +Memory behavior can change with model, prompt, version, storage backend, and hosted +product tier. Results apply only to the recorded configuration. + +\paragraph{Binary product criteria.} +Turnkey primitives favor systems that expose explicit APIs over architectures from +which an application could construct equivalent behavior. This is intentional for +procurement analysis, but it is not a measure of theoretical expressiveness. + +\paragraph{No utility or latency claim.} +The evaluation does not measure personalization quality, answer accuracy, cost, or +latency. A system that scores lower here may be superior for many agent workloads. + +\section{Related Work} + +LoCoMo and LongMemEval established long-horizon conversational evaluation +\cite{locomo2024,longmemeval2024}. LongMemEval-V2 shifts attention toward memory of +environment experience \cite{longmemevalv2_2026}. Look-Ahead-Bench evaluates +model-level temporal leakage in financial workflows by comparing performance and +alpha decay across market regimes \cite{lookaheadbench2026}. Our lookahead invariant +is complementary: it tests whether a memory product prevents facts that were not +knowable at the declared decision time from entering retrieval context, even when +the underlying model is temporally clean. WorldDB evaluates immutable, +content-addressed worlds and write-time reconciliation \cite{worlddb2026}. TOKI +formalizes contradiction resolution as bitemporal concurrency control with provenance +contracts \cite{toki2026}. A-TMA separates bank maintenance, retrieval, and answer-time +state resolution, exposing ghost-memory failures hidden by final answer accuracy +\cite{atma2026}. Our evaluation operates at a complementary layer: whether a product +surface exposes auditable primitives that a regulated operator can execute directly. + +\section{Conclusion} + +Agent memory has two different evaluation questions. One asks whether the agent can +recall useful information. The other asks whether an operator can prove which +information was available, prevent future information from leaking backward, and +produce evidence about revision and erasure. The initial results suggest that the +second question is not captured by current recall benchmarks or most default product +surfaces. The proposed invariants, open harness, and correction process provide a +starting point for independent evaluation rather than a final ranking. + +\bibliographystyle{plain} +\bibliography{references} + +\end{document} diff --git a/paper/regulated-memory-eval/references.bib b/paper/regulated-memory-eval/references.bib new file mode 100644 index 0000000..51aa8f2 --- /dev/null +++ b/paper/regulated-memory-eval/references.bib @@ -0,0 +1,64 @@ +@article{locomo2024, + title = {Evaluating Very Long-Term Conversational Memory of LLM Agents}, + author = {Maharana, Adyasha and others}, + journal = {arXiv preprint arXiv:2402.17753}, + year = {2024}, + url = {https://arxiv.org/abs/2402.17753} +} + +@article{longmemeval2024, + title = {LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory}, + author = {Wu, Di and others}, + journal = {arXiv preprint arXiv:2410.10813}, + year = {2024}, + url = {https://arxiv.org/abs/2410.10813} +} + +@article{longmemevalv2_2026, + title = {LongMemEval-V2: Evaluating Long-Term Agent Memory Toward Experienced Colleagues}, + author = {Wu, Di and Ji, Zixiang and Kawatkar, Asmi and Kwan, Bryan and Gu, Jia-Chen and Peng, Nanyun and Chang, Kai-Wei}, + journal = {arXiv preprint arXiv:2605.12493}, + year = {2026}, + url = {https://arxiv.org/abs/2605.12493} +} + +@article{lookaheadbench2026, + title = {Look-Ahead-Bench: a Standardized Benchmark of Look-ahead Bias in Point-in-Time LLMs for Finance}, + author = {Benhenda, Mostapha}, + journal = {arXiv preprint arXiv:2601.13770}, + year = {2026}, + url = {https://arxiv.org/abs/2601.13770} +} + +@article{toki2026, + title = {TOKI: A Bitemporal Operator Algebra for Contradiction Resolution in LLM-Agent Persistent Memory}, + author = {Wang, Ziming}, + journal = {arXiv preprint arXiv:2606.06240}, + year = {2026}, + url = {https://arxiv.org/abs/2606.06240} +} + +@article{worlddb2026, + title = {WorldDB: A Vector Graph-of-Worlds Memory Engine with Ontology-Aware Write-Time Reconciliation}, + author = {Ganesan, Harish Santhanalakshmi}, + journal = {arXiv preprint arXiv:2604.18478}, + year = {2026}, + url = {https://arxiv.org/abs/2604.18478} +} + +@article{atma2026, + title = {A-TMA: Decoupling State-Aware Memory Failures in Long-Term Agent Memory}, + author = {Shi, Zitong and Tang, Yixuan and Tung, Anthony Kum Hoe}, + journal = {arXiv preprint arXiv:2607.01935}, + year = {2026}, + url = {https://arxiv.org/abs/2607.01935} +} + +@misc{lians2026, + title = {Lians: Bitemporal Long-Term Memory for AI Agents}, + author = {Beirne, Ethan}, + year = {2026}, + howpublished = {Software, version 0.4.0}, + url = {https://github.com/Lians-ai/Lians} +} +