feat: memory retrieval quality sprint — metadata strip, trivial augmentation, extraction window, preamble

.gitignore

@@ -2,5 +2,3 @@ node_modules/
 *.tsbuildinfo
 .env
 .DS_Store
-src/
-tsconfig.json

src/index.ts

@@ -28,6 +28,9 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
 import { mkdirSync, existsSync } from "node:fs";
 import { dirname, join } from "node:path";
 
+// Module-level flag: emit hardcoded-key warning only once per process lifetime
+let _apiKeyWarningEmitted = false;
+
 // Dynamic require for node:sqlite (available in Node 22+, avoids TS import issues)
 let NodeDatabaseSync: any;
 try {
@@ -87,6 +90,10 @@ function resolveEnv(value: string): string {
   return value.replace(/\$\{([^}]+)\}/g, (_, key) => process.env[key] ?? "");
 }
 
+function isEnvInterpolation(value: unknown): value is string {
+  return typeof value === "string" && /^\$\{[^}]+\}$/.test(value.trim());
+}
+
 function parseConfig(raw: unknown): EvaMemoryConfig {
   const defaults: EvaMemoryConfig = {
     cortexUrl: "http://localhost:8000",
@@ -98,7 +105,7 @@ function parseConfig(raw: unknown): EvaMemoryConfig {
     retrievalBudget: 2000,
     maxInjectionChars: 8000,
     maxInjectedMemories: 8,
-    minRelevanceScore: 0.25,
+    minRelevanceScore: 0.30,
     retrievalMode: "fast",
     recencyFilterMinutes: 15,
     injectCornerstones: false,
@@ -780,10 +787,29 @@ async function syncMemoryCache(
   }
 }
 
+// --- Inbound metadata stripper ---
+
+function stripInboundMetadata(prompt: string): string {
+  let text = prompt;
+  // Strip Sender/Conversation info metadata blocks
+  text = text.replace(/(?:Sender|Conversation info) \(untrusted metadata\):\n```json\n[\s\S]*?```\n?/g, "");
+  // Strip only OpenClaw-decorated system lines (not bare "System:" which could be user content)
+  text = text.replace(/^System \(untrusted\):.*$/gm, "");
+  // Strip [Thread history ...] block: header line + indented/prefixed lines until blank line
+  text = text.replace(/^\[Thread history[^\n]*\n(?:(?:  |\w+:)[^\n]*\n)*\n?/gm, "");
+  // Strip timestamp prefixes at start of text like [Mon 2026-04-02 14:51 GMT+7]
+  text = text.replace(/^\[[A-Za-z]{3}\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}[^\]]*\]\s*/gm, "");
+  // Strip <relevant-memories>...</relevant-memories> blocks
+  text = text.replace(/<relevant-memories>[\s\S]*?<\/relevant-memories>/g, "");
+  // Collapse multiple newlines to single
+  text = text.replace(/\n{2,}/g, "\n");
+  return text.trim();
+}
+
 // --- Memory-relevance heuristic ---
 
 const MEMORY_KEYWORDS = /\b(remember|forgot|recall|last time|previously|before|earlier|you said|you told|we discussed|we decided|my preference|my name|who am i|what do i|do you know|history|past|memory|memorize|don'?t forget)\b/i;
-const TRIVIAL_PATTERNS = /^(hi|hello|hey|thanks|ok|yes|no|sure|bye|good morning|good night|👍|😊)[\s!?.]*$/i;
+const TRIVIAL_PATTERNS = /^(hi|hello|hey|thanks|thank you|ok|okay|yes|no|sure|bye|good morning|good night|go ahead|sounds good|do it|got it|lets do it|let's do it|lets go|let's go|cool|nice|great|agreed|done|will do|on it|yep|nope|nah|alright|perfect|roger|absolutely|definitely|for sure|ship it|merge it|deploy it|👍|😊|👌|🙏|✅)[\s!?.]*$/i;
 
 // Sub-agent completion events and runtime system events should not trigger memory retrieval
 const SYSTEM_EVENT_PATTERNS = /\[Internal task completion event\]|<<<BEGIN_UNTRUSTED_CHILD_RESULT>>>|source: subagent|type: subagent task|^\[.*\] Exec (?:completed|failed)|^\[.*\] OpenClaw runtime context/m;
@@ -897,7 +923,24 @@ function filterEchoMemories(
   });
 }
 
-function formatMemoryContext(items: RetrievedItem[], maxChars: number, maxCount = 8, minScore = 0.25): string {
+const MEMORY_PREAMBLE = `These are long-term memories retrieved from your memory system based on this conversation.
+Each has an [item_id], [date], and [salience/category] label.
+
+How to use these memories:
+- identity/preferences/decisions: Trust as stable context. Reference when relevant.
+- episodic: Specific past events. Check the date — may be outdated.
+- goals: Strategic direction. Verify if context has changed since the date.
+- behavioral/relational: Patterns and dynamics. Descriptive, not prescriptive.
+- high salience: Identity-defining or emotionally significant. Handle with care.
+
+Guidelines:
+- Surface relevant preferences or prior decisions naturally in your response.
+- If a memory might change your approach, ask a clarifying question before proceeding.
+- If memories contradict the current request, flag the contradiction.
+- Never expose raw item_ids, scores, or metadata to the user.
+- To look up more context on any memory, use cortex_search with the item_id.`;
+
+function formatMemoryContext(items: RetrievedItem[], maxChars: number, maxCount = 8, minScore = 0.30): string {
   if (!items.length) return "";
 
   // Filter by relevance score and cap count
@@ -907,7 +950,8 @@ function formatMemoryContext(items: RetrievedItem[], maxChars: number, maxCount
   if (!relevant.length) return "";
 
   const lines: string[] = ["<relevant-memories>"];
-  let charCount = 0;
+  lines.push(MEMORY_PREAMBLE);
+  let charCount = MEMORY_PREAMBLE.length;
 
   for (const item of relevant) {
     const tag = item.source === "cornerstone" ? " [cornerstone]" : "";
@@ -930,7 +974,7 @@ function formatMemoryContext(items: RetrievedItem[], maxChars: number, maxCount
     charCount += line.length;
   }
 
-  if (lines.length === 1) return ""; // Only header, no items fit
+  if (lines.length <= 2) return ""; // Only header + preamble, no actual items fit
   lines.push("</relevant-memories>");
   return lines.join("\n");
 }
@@ -940,10 +984,16 @@ function formatMemoryContext(items: RetrievedItem[], maxChars: number, maxCount
 function extractMessages(rawMessages: unknown[]): Array<{ role: string; content: string }> {
   const result: Array<{ role: string; content: string }> = [];
 
-  for (const msg of rawMessages.slice(-10)) {
+  // Walk backwards from end to collect last 5 user/assistant messages
+  // Avoids filtering the entire session history (can be 30K+ messages / 70MB+)
+  const tail: Record<string, unknown>[] = [];
+  for (let i = rawMessages.length - 1; i >= 0 && tail.length < 5; i--) {
+    const msg = rawMessages[i];
     if (!msg || typeof msg !== "object") continue;
     const m = msg as Record<string, unknown>;
-    if (m.role !== "user" && m.role !== "assistant") continue;
+    if (m.role === "user" || m.role === "assistant") tail.unshift(m);
+  }
+  for (const m of tail) {
 
     let text = "";
     if (typeof m.content === "string") {
@@ -1003,7 +1053,7 @@ const cortexPlugin = {
         retrievalBudget: { type: "number" },
         maxInjectionChars: { type: "number" },
         maxInjectedMemories: { type: "number", description: "Max memories to inject per turn (default: 8)" },
-        minRelevanceScore: { type: "number", description: "Min score to inject a memory (default: 0.25)" },
+        minRelevanceScore: { type: "number", description: "Min score to inject a memory (default: 0.30)" },
         retrievalMode: { type: "string", enum: ["auto", "fast", "thorough"], description: "Retrieval mode for memory search (default: auto)" },
         recencyFilterMinutes: { type: "number", description: "Filter out memories created within this many minutes to suppress echo (default: 15, 0 to disable)" },
       },
@@ -1012,9 +1062,13 @@ const cortexPlugin = {
   },
 
   register(api: OpenClawPluginApi) {
+    const rawConfig = api.pluginConfig && typeof api.pluginConfig === "object" && !Array.isArray(api.pluginConfig)
+      ? api.pluginConfig as Record<string, unknown>
+      : null;
     const cfg = parseConfig(api.pluginConfig);
-
-    if (cfg.apiKey && !cfg.apiKey.startsWith("${")) {
+
+    if (cfg.apiKey && !isEnvInterpolation(rawConfig?.apiKey) && !_apiKeyWarningEmitted) {
+      _apiKeyWarningEmitted = true;
       api.logger.warn("cortex: API key appears to be hardcoded in config. Consider using environment variable: apiKey: '${CORTEX_API_KEY}'");
     }
 
@@ -1040,11 +1094,6 @@ const cortexPlugin = {
       api.logger.info("cortex: node:sqlite not available — local cache disabled");
     }
 
-    // Security: warn if API key is hardcoded in config instead of env var
-    if (cfg.apiKey && !cfg.apiKey.startsWith("${")) {
-      api.logger.warn("cortex: API key appears to be hardcoded in config. Consider using environment variable: apiKey: '${CORTEX_API_KEY}'");
-    }
-
     api.logger.info(
       `cortex: registered (cortex=${cfg.cortexUrl}, owner=${cfg.ownerId}, recall=${cfg.autoRecall}, capture=${cfg.autoCapture}, shadow=${cfg.shadowMode})`,
     );
@@ -1406,6 +1455,9 @@ const cortexPlugin = {
     // Hooks
     // -------------------------------------------------------------------------
 
+    // Cache last assistant turn per session for trivial-query augmentation
+    const lastAssistantTurnCache = new Map<string, string>();
+
     // Auto-recall: inject cornerstones (always) + relevant memories (when relevant)
     // Lane guards prevent injection on heartbeat, boot, subagent, cron, isolated lanes.
     // Server-first: always call Cortex API (semantic embeddings). Local cache is fallback only.
@@ -1421,7 +1473,40 @@ const cortexPlugin = {
 
       // --- Fetch contextual memories (+ optional cornerstones) ---
       if (cfg.autoRecall) {
-        const doRetrieve = event.prompt && isMemoryRelevant(event.prompt);
+        // Strip metadata from prompt for retrieval decisions
+        const cleanedPrompt = event.prompt ? stripInboundMetadata(event.prompt) : "";
+
+        // Determine retrieval strategy
+        const hasQuestion = cleanedPrompt.includes("?");
+        const isTrivial = TRIVIAL_PATTERNS.test(cleanedPrompt.trim());
+        const isRelevant = cleanedPrompt && isMemoryRelevant(cleanedPrompt);
+
+        let retrievalQuery = cleanedPrompt;
+        let maxMemories = cfg.maxInjectedMemories; // default (8)
+        let doRetrieve = false;
+
+        if (hasQuestion) {
+          // Questions always retrieve
+          doRetrieve = true;
+        } else if (isTrivial) {
+          // Trivial patterns: augment with last assistant turn if available
+          const sessionKey = ctx.sessionKey ?? "";
+          const lastAssistant = sessionKey ? lastAssistantTurnCache.get(sessionKey) : undefined;
+          if (lastAssistant) {
+            // Take last ~200 chars, trim to nearest word boundary on left edge
+            let tail = lastAssistant.slice(-200).trim();
+            const spaceIdx = tail.indexOf(" ");
+            if (spaceIdx > 0 && spaceIdx < 40) tail = tail.slice(spaceIdx + 1);
+            const lastTwo = tail;
+            retrievalQuery = lastTwo + " " + cleanedPrompt;
+            doRetrieve = true;
+            maxMemories = 3; // Cap injection for trivial queries
+          }
+          // If no cached assistant turn, skip retrieval (no signal)
+        } else if (isRelevant) {
+          doRetrieve = true;
+        }
+
         const doCornerstones = cfg.injectCornerstones; // default false — cornerstones in SOUL.md
 
         // --- Memory retrieval (server-first, local fallback) ---
@@ -1439,7 +1524,7 @@ const cortexPlugin = {
           // Fire cornerstones + API retrieve in parallel
           const [cornerstonesResult, retrieveResult] = await Promise.allSettled([
             doCornerstones ? client.getCornerstones() : Promise.resolve(null),
-            client.retrieve(event.prompt!, cfg.retrievalBudget, cfg.retrievalMode),
+            client.retrieve(retrievalQuery, cfg.retrievalBudget, cfg.retrievalMode),
           ]);
 
           // Process cornerstones
@@ -1467,7 +1552,7 @@ const cortexPlugin = {
             // Server down — fall back to local cache if available
             if (memoryCache) {
               try {
-                const fallbackResults = memoryCache.search(event.prompt!, 20);
+                const fallbackResults = memoryCache.search(retrievalQuery, 20);
                 if (fallbackResults.length) {
                   memoryItems = fallbackResults;
                   api.logger.info(`cortex: using local cache fallback (${fallbackResults.length} results)`);
@@ -1495,7 +1580,7 @@ const cortexPlugin = {
               `cortex: echo filter removed ${memoryItems.length - filtered.length} same-session/recent memories`,
             );
           }
-          const context = formatMemoryContext(filtered, cfg.maxInjectionChars, cfg.maxInjectedMemories, cfg.minRelevanceScore);
+          const context = formatMemoryContext(filtered, cfg.maxInjectionChars, maxMemories, cfg.minRelevanceScore);
           if (context) {
             const elapsed = Date.now() - startMs;
             if (elapsed <= 3000) {
@@ -1530,6 +1615,18 @@ const cortexPlugin = {
 
         const messages = extractMessages(event.messages);
 
+        // Cache last assistant message for next-turn augmentation
+        const lastAssistantMsg = messages.filter(m => m.role === "assistant").pop();
+        const cacheKey = ctx.sessionKey ?? "";
+        if (lastAssistantMsg?.content && cacheKey) {
+          lastAssistantTurnCache.set(cacheKey, lastAssistantMsg.content.slice(0, 2000));
+          // Limit cache entries to prevent memory leak
+          if (lastAssistantTurnCache.size > 100) {
+            const firstKey = lastAssistantTurnCache.keys().next().value;
+            if (firstKey) lastAssistantTurnCache.delete(firstKey);
+          }
+        }
+
         // Require at least 2 real messages — single-message captures are noise
         if (messages.length < 2) return;