feat: temporal scoring, staleness tracking, and drift detection

plugin/skills/mem-search/SKILL.md

@@ -93,6 +93,36 @@ get_observations(ids=[11131, 10942])
 
 **Returns:** Complete observation objects with title, subtitle, narrative, facts, concepts, files (~500-1000 tokens each)
 
+## Saving Memories
+
+Use the `save_memory` MCP tool to store manual observations:
+
+```
+save_memory(text="Important discovery about the auth system", title="Auth Architecture", project="my-project")
+```
+
+**Parameters:**
+
+- `text` (string, required) - Content to remember
+- `title` (string, optional) - Short title, auto-generated if omitted
+- `project` (string, optional) - Project name, defaults to "claude-mem"
+
+## Correcting Outdated Memories
+
+Use the `contradict` MCP tool when a stored memory is no longer accurate:
+
+```
+contradict(stale_id=11131, correction="Auth now uses session cookies, not JWT tokens", title="Auth Architecture Update")
+```
+
+This marks the old observation (`stale_id`) as stale and saves the correction as a new observation linked to it.
+
+**Parameters:**
+
+- `stale_id` (number, required) - ID of the memory to mark as stale
+- `correction` (string, required) - The corrected or updated information
+- `title` (string, optional) - Short title for the correction, auto-generated if omitted
+
 ## Examples
 
 **Find recent bug fixes:**

src/servers/mcp-server.ts

@@ -349,6 +349,50 @@ NEVER fetch full details without filtering first. 10x token savings.`,
         }]
       };
     }
+  },
+  {
+    name: 'contradict',
+    description: 'Mark an existing memory as stale/outdated and record a correction. Use when you discover that a previously stored memory is no longer accurate.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        stale_id: { type: 'number', description: 'ID of the memory to mark as stale' },
+        correction: { type: 'string', description: 'The corrected or updated information to store' },
+        title: { type: 'string', description: 'Optional title for the correction memory' }
+      },
+      required: ['stale_id', 'correction']
+    },
+    handler: async (args: any) => {
+      return await callWorkerAPIPost('/api/memory/contradict', args);
+    }
+  },
+  {
+    name: 'set_importance',
+    description: 'Set the importance score (1-10) for a memory. Higher importance = slower decay. Use 8-10 for critical architectural decisions, 1-3 for ephemeral notes. Default is 5 (90-day half-life).',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        id: { type: 'number', description: 'ID of the memory to update' },
+        importance: { type: 'number', description: 'Importance score 1-10 (5=default/90d half-life, 10=180d, 1=18d)' }
+      },
+      required: ['id', 'importance']
+    },
+    handler: async (args: any) => {
+      return await callWorkerAPIPost('/api/memory/importance', args);
+    }
+  },
+  {
+    name: 'drift_check',
+    description: 'Analyze memory clusters for semantic drift — detects concept areas where old memories have gone stale or been superseded by recent work. Run when architecture or tech decisions change.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        project: { type: 'string', description: 'Limit check to a specific project (optional, checks all projects if omitted)' }
+      }
+    },
+    handler: async (args: any) => {
+      return await callWorkerAPIPost('/api/memory/drift-check', args);
+    }
   }
 ];
 

src/services/sqlite/SessionSearch.ts

@@ -272,12 +272,11 @@ export class SessionSearch {
    * Vector search is handled by ChromaDB - this only supports filtering without query text.
    */
   searchObservations(query: string | undefined, options: SearchOptions = {}): ObservationSearchResult[] {
-    const params: any[] = [];
-    const { limit = 50, offset = 0, orderBy = 'relevance', ...filters } = options;
-
     // FILTER-ONLY PATH: When no query text, query table directly
     // This enables date filtering which Chroma cannot do (requires direct SQLite access)
     if (!query) {
+      const params: any[] = [];
+      const { limit = 50, offset = 0, orderBy = 'relevance', ...filters } = options;
       const filterClause = this.buildFilterClause(filters, params, 'o');
       if (!filterClause) {
         throw new Error('Either query or filters required for search');
@@ -297,10 +296,47 @@ export class SessionSearch {
       return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
     }
 
-    // Vector search with query text should be handled by ChromaDB
-    // This method only supports filter-only queries (query=undefined)
-    logger.warn('DB', 'Text search not supported - use ChromaDB for vector search');
-    return [];
+    // FTS5 search path - used when ChromaDB is not available
+    const { limit = 50, offset = 0, orderBy = 'relevance', ...filters } = options;
+    const ftsQuery = query.replace(/['"*]/g, ' ').trim();
+    if (!ftsQuery) return [];
+
+    const params: any[] = [ftsQuery];
+    let sql = `
+      SELECT o.*
+      FROM observations o
+      JOIN observations_fts fts ON o.id = fts.rowid
+      WHERE fts.observations_fts MATCH ?
+    `;
+
+    const filterClauses: string[] = [];
+    if (filters.project) {
+      filterClauses.push('o.project = ?');
+      params.push(filters.project);
+    }
+    if (filters.type) {
+      if (Array.isArray(filters.type)) {
+        const placeholders = filters.type.map(() => '?').join(',');
+        filterClauses.push(`o.type IN (${placeholders})`);
+        params.push(...filters.type);
+      } else {
+        filterClauses.push('o.type = ?');
+        params.push(filters.type);
+      }
+    }
+    if (filterClauses.length > 0) {
+      sql += ` AND ${filterClauses.join(' AND ')}`;
+    }
+
+    sql += ` ORDER BY rank LIMIT ? OFFSET ?`;
+    params.push(limit, offset);
+
+    try {
+      return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
+    } catch (err) {
+      logger.error('DB', 'FTS5 search failed', { query: ftsQuery }, err as Error);
+      return [];
+    }
   }
 
   /**
@@ -598,6 +634,142 @@ export class SessionSearch {
     return stmt.all(contentSessionId) as UserPromptRow[];
   }
 
+  /**
+   * Rank observations by temporal decay score.
+   * Half-life scales with importance (1-10): 18d at imp=1, 90d at imp=5, 180d at imp=10.
+   * Access count and staleness also factor in.
+   */
+  rankByTemporalScore(observations: ObservationSearchResult[]): ObservationSearchResult[] {
+    if (observations.length === 0) return observations;
+
+    const now = Date.now() / 1000;
+
+    return observations
+      .map(obs => {
+        const importance = Math.min(10, Math.max(1, obs.importance ?? 5));
+        const halfLifeDays = 90 * (importance / 5); // 18d at imp=1, 90d at imp=5, 180d at imp=10
+        const lambda = Math.log(2) / halfLifeDays;
+        const createdEpoch = obs.created_at_epoch / 1000; // created_at_epoch is ms
+        const daysSince = Math.max(0, (now - createdEpoch) / 86400);
+        const decayFactor = Math.exp(-lambda * daysSince);
+        const accessBoost = Math.log1p(obs.access_count ?? 0) * 0.1;
+        const stalenessPenalty = obs.is_stale ? 0.1 : 1.0;
+        const score = decayFactor * (1 + accessBoost) * stalenessPenalty;
+
+        return { obs, score };
+      })
+      .sort((a, b) => b.score - a.score)
+      .map(({ obs }) => obs);
+  }
+
+  /**
+   * Update last_accessed_at and increment access_count for a set of observation IDs.
+   */
+  updateAccessTracking(ids: number[]): void {
+    if (ids.length === 0) return;
+    const now = Math.floor(Date.now() / 1000);
+    const placeholders = ids.map(() => '?').join(',');
+    this.db.prepare(
+      `UPDATE observations SET last_accessed_at = ?, access_count = COALESCE(access_count, 0) + 1 WHERE id IN (${placeholders})`
+    ).run(now, ...ids);
+  }
+
+  /**
+   * Detect semantic drift in concept clusters.
+   * Flags clusters where old memories are unaccessed or heavily stale,
+   * suggesting the project has moved on and those memories may be outdated.
+   */
+  detectDrift(project?: string): {
+    driftedConcepts: Array<{
+      project: string;
+      concept: string;
+      totalCount: number;
+      recentCount: number;
+      oldCount: number;
+      unaccessedOld: number;
+      staleCount: number;
+      stalePct: number;
+      signal: 'high-stale' | 'likely-outdated' | 'monitor';
+    }>;
+    summary: string;
+  } {
+    const thirtyDaysAgoMs = Date.now() - 30 * 86400 * 1000;
+
+    let sql = `
+      WITH concept_stats AS (
+        SELECT
+          o.project,
+          je.value as concept,
+          COUNT(*) as total_count,
+          SUM(CASE WHEN o.created_at_epoch > ? THEN 1 ELSE 0 END) as recent_count,
+          SUM(CASE WHEN o.created_at_epoch <= ? THEN 1 ELSE 0 END) as old_count,
+          SUM(CASE WHEN o.created_at_epoch <= ? AND COALESCE(o.access_count, 0) = 0 THEN 1 ELSE 0 END) as unaccessed_old,
+          SUM(CASE WHEN o.is_stale = 1 THEN 1 ELSE 0 END) as stale_count
+        FROM observations o, json_each(o.concepts) je
+        WHERE o.concepts IS NOT NULL
+          AND o.concepts != '[]'
+          AND o.concepts != 'null'
+          AND json_valid(o.concepts)
+    `;
+
+    const params: (string | number)[] = [thirtyDaysAgoMs, thirtyDaysAgoMs, thirtyDaysAgoMs];
+
+    if (project) {
+      sql += ` AND o.project = ?`;
+      params.push(project);
+    }
+
+    sql += `
+        GROUP BY o.project, je.value
+        HAVING total_count >= 2
+      )
+      SELECT
+        project, concept, total_count, recent_count, old_count, unaccessed_old, stale_count,
+        ROUND(CAST(stale_count AS REAL) / total_count * 100) as stale_pct,
+        CASE
+          WHEN CAST(stale_count AS REAL) / total_count > 0.3 THEN 'high-stale'
+          WHEN recent_count > 0 AND old_count > 0
+            AND CAST(unaccessed_old AS REAL) / old_count > 0.5 THEN 'likely-outdated'
+          ELSE 'monitor'
+        END as signal
+      FROM concept_stats
+      WHERE
+        (CAST(stale_count AS REAL) / total_count > 0.3)
+        OR (recent_count > 0 AND old_count > 0 AND CAST(unaccessed_old AS REAL) / NULLIF(old_count, 0) > 0.5)
+      ORDER BY stale_pct DESC, unaccessed_old DESC
+      LIMIT 20
+    `;
+
+    let rows: any[] = [];
+    try {
+      rows = this.db.prepare(sql).all(...params) as any[];
+    } catch (err) {
+      logger.error('DB', 'Drift check query failed', { project: project ?? 'all' }, err as Error);
+      return { driftedConcepts: [], summary: 'Drift check unavailable (query failed — see worker logs).' };
+    }
+
+    const driftedConcepts = rows.map(r => ({
+      project: r.project as string,
+      concept: r.concept as string,
+      totalCount: r.total_count as number,
+      recentCount: r.recent_count as number,
+      oldCount: r.old_count as number,
+      unaccessedOld: r.unaccessed_old as number,
+      staleCount: r.stale_count as number,
+      stalePct: r.stale_pct as number,
+      signal: r.signal as 'high-stale' | 'likely-outdated' | 'monitor',
+    }));
+
+    const highStale = driftedConcepts.filter(c => c.signal === 'high-stale');
+    const likelyOutdated = driftedConcepts.filter(c => c.signal === 'likely-outdated');
+
+    const summary = driftedConcepts.length === 0
+      ? 'No drift detected. All concept clusters appear current.'
+      : `Drift detected in ${driftedConcepts.length} concept cluster(s): ${highStale.length} high-stale, ${likelyOutdated.length} likely-outdated. Consider using contradict() on stale memories in flagged clusters.`;
+
+    return { driftedConcepts, summary };
+  }
+
   /**
    * Close the database connection
    */

src/services/sqlite/SessionStore.ts

@@ -1181,6 +1181,25 @@ export class SessionStore {
     return stmt.get(id) as ObservationRecord | undefined || null;
   }
 
+  /**
+   * Mark an observation as stale and record the ID of its correction
+   */
+  markObservationStale(staleId: number, correctedById: number): void {
+    this.db.prepare(
+      'UPDATE observations SET is_stale = 1, corrected_by_id = ? WHERE id = ?'
+    ).run(correctedById, staleId);
+  }
+
+  /**
+   * Set the importance score (1-10) for an observation
+   */
+  setObservationImportance(id: number, importance: number): void {
+    const clamped = Math.min(10, Math.max(1, Math.round(importance)));
+    this.db.prepare(
+      'UPDATE observations SET importance = ? WHERE id = ?'
+    ).run(clamped, id);
+  }
+
   /**
    * Get observations by array of IDs with ordering and limit
    */

src/services/sqlite/migrations/runner.ts

@@ -34,6 +34,7 @@ export class MigrationRunner {
     this.addOnUpdateCascadeToForeignKeys();
     this.addObservationContentHashColumn();
     this.addSessionCustomTitleColumn();
+    this.addTemporalScoringColumns();
   }
 
   /**
@@ -614,6 +615,36 @@ export class MigrationRunner {
     this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(19, new Date().toISOString());
   }
 
+  /**
+   * Add temporal scoring and staleness tracking columns to observations (migration 22)
+   */
+  private addTemporalScoringColumns(): void {
+    const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(24) as SchemaVersion | undefined;
+    if (applied) return;
+
+    const tableInfo = this.db.query('PRAGMA table_info(observations)').all() as TableColumnInfo[];
+    const existingColumns = new Set(tableInfo.map(col => col.name));
+
+    if (!existingColumns.has('last_accessed_at')) {
+      this.db.run('ALTER TABLE observations ADD COLUMN last_accessed_at INTEGER');
+    }
+    if (!existingColumns.has('access_count')) {
+      this.db.run('ALTER TABLE observations ADD COLUMN access_count INTEGER NOT NULL DEFAULT 0');
+    }
+    if (!existingColumns.has('importance')) {
+      this.db.run('ALTER TABLE observations ADD COLUMN importance INTEGER NOT NULL DEFAULT 5');
+    }
+    if (!existingColumns.has('is_stale')) {
+      this.db.run('ALTER TABLE observations ADD COLUMN is_stale INTEGER NOT NULL DEFAULT 0');
+    }
+    if (!existingColumns.has('corrected_by_id')) {
+      this.db.run('ALTER TABLE observations ADD COLUMN corrected_by_id INTEGER');
+    }
+    logger.debug('DB', 'Ensured temporal scoring columns on observations table');
+
+    this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(24, new Date().toISOString());
+  }
+
   /**
    * Add failed_at_epoch column to pending_messages (migration 20)
    * Used by markSessionMessagesFailed() for error recovery tracking

src/services/sqlite/types.ts

@@ -218,6 +218,12 @@ export interface ObservationRow {
   discovery_tokens: number; // ROI metrics: tokens spent discovering this observation
   created_at: string;
   created_at_epoch: number;
+  // Temporal scoring fields (added by migration v24, may be absent on pre-migration rows)
+  last_accessed_at?: number | null;
+  access_count?: number;
+  importance?: number;
+  is_stale?: number;
+  corrected_by_id?: number | null;
 }
 
 export interface SessionSummaryRow {

src/services/worker/http/routes/DataRoutes.ts

@@ -140,6 +140,15 @@ export class DataRoutes extends BaseRouteHandler {
     const store = this.dbManager.getSessionStore();
     const observations = store.getObservationsByIds(ids, { orderBy, limit, project });
 
+    // fire-and-forget — don't block response
+    setTimeout(() => {
+      try {
+        this.dbManager.getSessionSearch().updateAccessTracking(ids.map(Number));
+      } catch {
+        // ignore tracking failures
+      }
+    }, 0);
+
     res.json(observations);
   });