feat(learning): add lesson effectiveness telemetry

packages/franken-critique/README.md

@@ -172,6 +172,18 @@ Every `LessonRecorder.record()` result exposes `learningBacklogPrioritizationRep
 
 Report items identify their source as `recorded-lesson`, `blocker-pattern`, or `cooldown-suppression`, include task/evaluator context when available, and carry a concise rationale plus recommended next action. High-priority recorded lessons should go through promotion review with their traceability verifier, blocker patterns should route to a durable mitigation owner, and low-priority cooldown suppressions should reuse the existing in-cooldown lesson instead of creating duplicate backlog churn.
 
+## Lesson effectiveness telemetry
+
+`LessonEffectivenessTelemetry` correlates an injected lesson with the task result that follows it. Call `record()` after the task outcome is known with the stable lesson id, reviewed lesson scope, injection context, injection/outcome timestamps, task success, blocker counts before/after injection, review-finding count, and whether the user corrected the result. Scope expiry and audit approval are validated at the injection timestamp so a valid lesson remains attributable when a long-running task finishes after expiry, while later scope approvals cannot legitimize earlier injections. Outcomes timestamped before injection are rejected. The emitted `lesson-effectiveness-event-v1` contains only those bounded signals; its schema deliberately has no raw prompt, reviewer finding text, or correction text fields.
+
+Outcome attribution is deterministic:
+
+- `positive`: the task succeeded, blockers decreased, no review findings remained, and no user correction was recorded.
+- `negative`: the task failed, blockers increased, or the user corrected the result.
+- `neutral`: the remaining mixed or unchanged outcomes, including successful work with unchanged blockers or residual review findings.
+
+`report()` aggregates events by stable lesson id and scope. Its score ranges from `-1` (all negative) to `1` (all positive), alongside correction, blocker-reduction, and blocker-regression counts. A positive majority recommends `promote`, a negative majority recommends `retire`, and ties/no demonstrated improvement recommend `monitor`. These are evidence inputs to the existing reviewed lifecycle—not automatic mutations: promotion still requires traceability and scope review, while retirement should use the quarantine/rollback workflow so operators can audit the decision.
+
 ## Package scripts
 
 Run these from the package directory with `npm run <script>`, or from the repository root with `npm run <script> --workspace @franken/critique`.

packages/franken-critique/src/index.ts

@@ -65,6 +65,12 @@ export type {
   LessonCritiqueAgentFinding,
   LessonMultiAgentCritique,
   LessonInjectionContext,
+  LessonEffectivenessOutcome,
+  LessonEffectivenessSignals,
+  LessonEffectivenessEvent,
+  LessonLifecycleRecommendation,
+  LessonEffectivenessTrend,
+  LessonEffectivenessReport,
   LessonScopeAuditEntry,
   LessonScopeKind,
   LessonScopeMetadata,
@@ -143,6 +149,8 @@ export type {
   LessonHumanFeedbackRequest,
   LessonScopeReviewRequest,
 } from './memory/lesson-recorder.js';
+export { LessonEffectivenessTelemetry } from './memory/lesson-effectiveness.js';
+export type { LessonEffectivenessRecordInput } from './memory/lesson-effectiveness.js';
 
 // Evaluators
 export { SafetyEvaluator } from './evaluators/safety.js';

packages/franken-critique/src/memory/lesson-effectiveness.ts

@@ -0,0 +1,320 @@
+import type {
+  CritiqueLesson,
+  LessonEffectivenessEvent,
+  LessonEffectivenessOutcome,
+  LessonEffectivenessReport,
+  LessonEffectivenessTrend,
+  LessonInjectionContext,
+  LessonLifecycleRecommendation,
+  LessonScopeKind,
+  LessonScopeMetadata,
+} from '../types/contracts.js';
+import type { TaskId } from '../types/common.js';
+import { isLessonApplicable } from './lesson-recorder.js';
+
+const LESSON_SCOPE_KINDS: readonly LessonScopeKind[] = [
+  'global',
+  'repo',
+  'role',
+  'profile',
+  'task',
+];
+
+export interface LessonEffectivenessRecordInput {
+  readonly lessonId: string;
+  readonly lessonScope: LessonScopeMetadata;
+  readonly injectionContext: Omit<LessonInjectionContext, 'now'>;
+  readonly injectedAt: string;
+  readonly observedAt: string;
+  readonly taskSucceeded: boolean;
+  readonly blockersBefore: number;
+  readonly blockersAfter: number;
+  readonly reviewFindingCount: number;
+  readonly userCorrection: boolean;
+}
+
+interface MutableLessonEffectivenessTrend {
+  lessonId: string;
+  lessonScope: LessonScopeKind;
+  observations: number;
+  positive: number;
+  neutral: number;
+  negative: number;
+  correctionSignals: number;
+  blockerReductions: number;
+  blockerRegressions: number;
+}
+
+/**
+ * Records transcript-free outcome signals for injected lessons and aggregates
+ * them into deterministic lifecycle trends. Raw prompts, finding text, and
+ * correction text are deliberately absent from the input and event schemas.
+ */
+export class LessonEffectivenessTelemetry {
+  private readonly events: LessonEffectivenessEvent[] = [];
+  private readonly now: () => string;
+
+  constructor(now: () => Date | string = (): Date => new Date()) {
+    this.now = (): string => normalizeTimestamp(now(), 'report timestamp');
+  }
+
+  record(input: LessonEffectivenessRecordInput): LessonEffectivenessEvent {
+    const lessonId = requireNonEmptyString(input.lessonId, 'lessonId');
+    const lessonScope = normalizeLessonScope(input.lessonScope?.scope);
+    const injectedAt = normalizeTimestamp(input.injectedAt, 'injectedAt');
+    const observedAt = normalizeTimestamp(input.observedAt, 'observedAt');
+    if (Date.parse(observedAt) < Date.parse(injectedAt)) {
+      throw new RangeError('observedAt must not precede injectedAt.');
+    }
+    const injectionContext = normalizeInjectionContext(input.injectionContext);
+    requireApplicableScope(
+      lessonId,
+      input.lessonScope,
+      injectionContext,
+      injectedAt,
+    );
+    const blockersBefore = requireCount(input.blockersBefore, 'blockersBefore');
+    const blockersAfter = requireCount(input.blockersAfter, 'blockersAfter');
+    const reviewFindingCount = requireCount(
+      input.reviewFindingCount,
+      'reviewFindingCount',
+    );
+    if (typeof input.taskSucceeded !== 'boolean') {
+      throw new TypeError('taskSucceeded must be a boolean.');
+    }
+    if (typeof input.userCorrection !== 'boolean') {
+      throw new TypeError('userCorrection must be a boolean.');
+    }
+
+    const blockerDelta = blockersAfter - blockersBefore;
+    const outcome = attributeOutcome({
+      taskSucceeded: input.taskSucceeded,
+      blockerDelta,
+      reviewFindingCount,
+      userCorrection: input.userCorrection,
+    });
+    const event: LessonEffectivenessEvent = {
+      schemaVersion: 'lesson-effectiveness-event-v1',
+      lessonId,
+      lessonScope,
+      injectionContext,
+      injectedAt,
+      observedAt,
+      outcome,
+      signals: {
+        taskSucceeded: input.taskSucceeded,
+        blockerDelta,
+        blockerReduced: blockerDelta < 0,
+        reviewFindingCount,
+        userCorrection: input.userCorrection,
+      },
+    };
+    this.events.push({
+      ...event,
+      injectionContext: { ...event.injectionContext },
+      signals: { ...event.signals },
+    });
+    return event;
+  }
+
+  report(): LessonEffectivenessReport {
+    const trends = new Map<string, MutableLessonEffectivenessTrend>();
+    for (const event of this.events) {
+      const key = `${event.lessonScope}\u0000${event.lessonId}`;
+      const trend = trends.get(key) ?? {
+        lessonId: event.lessonId,
+        lessonScope: event.lessonScope,
+        observations: 0,
+        positive: 0,
+        neutral: 0,
+        negative: 0,
+        correctionSignals: 0,
+        blockerReductions: 0,
+        blockerRegressions: 0,
+      };
+      trend.observations += 1;
+      trend[event.outcome] += 1;
+      if (event.signals.userCorrection) trend.correctionSignals += 1;
+      if (event.signals.blockerDelta < 0) trend.blockerReductions += 1;
+      if (event.signals.blockerDelta > 0) trend.blockerRegressions += 1;
+      trends.set(key, trend);
+    }
+
+    const lessons = [...trends.values()]
+      .sort(
+        (left, right) =>
+          left.lessonId.localeCompare(right.lessonId) ||
+          left.lessonScope.localeCompare(right.lessonScope),
+      )
+      .map(toPublicTrend);
+
+    return {
+      schemaVersion: 'lesson-effectiveness-report-v1',
+      generatedAt: this.now(),
+      totalEvents: this.events.length,
+      lessons,
+    };
+  }
+}
+
+function attributeOutcome(signals: {
+  readonly taskSucceeded: boolean;
+  readonly blockerDelta: number;
+  readonly reviewFindingCount: number;
+  readonly userCorrection: boolean;
+}): LessonEffectivenessOutcome {
+  if (
+    signals.userCorrection ||
+    !signals.taskSucceeded ||
+    signals.blockerDelta > 0
+  ) {
+    return 'negative';
+  }
+  if (
+    signals.taskSucceeded &&
+    signals.blockerDelta < 0 &&
+    signals.reviewFindingCount === 0
+  ) {
+    return 'positive';
+  }
+  return 'neutral';
+}
+
+function toPublicTrend(
+  trend: MutableLessonEffectivenessTrend,
+): LessonEffectivenessTrend {
+  const effectivenessScore = roundScore(
+    (trend.positive - trend.negative) / trend.observations,
+  );
+  return {
+    lessonId: trend.lessonId,
+    lessonScope: trend.lessonScope,
+    observations: trend.observations,
+    positive: trend.positive,
+    neutral: trend.neutral,
+    negative: trend.negative,
+    effectivenessScore,
+    correctionSignals: trend.correctionSignals,
+    blockerReductions: trend.blockerReductions,
+    blockerRegressions: trend.blockerRegressions,
+    lifecycleRecommendation: recommendLifecycle(trend),
+  };
+}
+
+function recommendLifecycle(
+  trend: MutableLessonEffectivenessTrend,
+): LessonLifecycleRecommendation {
+  if (trend.negative * 2 > trend.observations) return 'retire';
+  if (trend.positive * 2 > trend.observations) return 'promote';
+  return 'monitor';
+}
+
+function requireApplicableScope(
+  lessonId: string,
+  lessonScope: LessonScopeMetadata,
+  injectionContext: Omit<LessonInjectionContext, 'now'>,
+  injectedAt: string,
+): void {
+  const injectionTime = Date.parse(injectedAt);
+  const effectiveAuditTrail = lessonScope.auditTrail.filter((entry) => {
+    const changedAt = normalizeTimestamp(
+      entry.changedAt,
+      'lessonScope.auditTrail.changedAt',
+    );
+    return Date.parse(changedAt) <= injectionTime;
+  });
+  const attributionLesson: CritiqueLesson = {
+    evaluatorName: 'lesson-effectiveness-attribution',
+    failureDescription: lessonId,
+    correctionApplied: 'Effectiveness observation',
+    taskId: (injectionContext.taskId ??
+      lessonScope.provenance.taskId ??
+      'lesson-effectiveness-attribution') as TaskId,
+    timestamp: injectedAt,
+    lifecycleStatus: 'active',
+    lessonScope: {
+      ...lessonScope,
+      auditTrail: effectiveAuditTrail,
+    },
+  };
+  if (
+    !isLessonApplicable(attributionLesson, {
+      ...injectionContext,
+      now: injectedAt,
+    })
+  ) {
+    throw new RangeError(
+      'Lesson effectiveness injection context is outside the reviewed lesson scope.',
+    );
+  }
+}
+
+function normalizeInjectionContext(
+  context: Omit<LessonInjectionContext, 'now'>,
+): Omit<LessonInjectionContext, 'now'> {
+  if (!context || typeof context !== 'object') {
+    throw new TypeError('injectionContext must be an object.');
+  }
+  return {
+    ...(context.repo !== undefined
+      ? { repo: requireNonEmptyString(context.repo, 'injectionContext.repo') }
+      : {}),
+    ...(context.role !== undefined
+      ? { role: requireNonEmptyString(context.role, 'injectionContext.role') }
+      : {}),
+    ...(context.profile !== undefined
+      ? {
+          profile: requireNonEmptyString(
+            context.profile,
+            'injectionContext.profile',
+          ),
+        }
+      : {}),
+    ...(context.taskId !== undefined
+      ? {
+          taskId: requireTaskId(context.taskId),
+        }
+      : {}),
+  };
+}
+
+function normalizeLessonScope(scope: LessonScopeKind): LessonScopeKind {
+  if (!LESSON_SCOPE_KINDS.includes(scope)) {
+    throw new RangeError(`Unsupported lesson scope: ${String(scope)}.`);
+  }
+  return scope;
+}
+
+function requireNonEmptyString(value: string, label: string): string {
+  if (typeof value !== 'string' || !value.trim()) {
+    throw new TypeError(`${label} must be a non-empty string.`);
+  }
+  return value.trim();
+}
+
+function requireTaskId(
+  value: NonNullable<LessonInjectionContext['taskId']>,
+): NonNullable<LessonInjectionContext['taskId']> {
+  return requireNonEmptyString(value, 'injectionContext.taskId') as NonNullable<
+    LessonInjectionContext['taskId']
+  >;
+}
+
+function requireCount(value: number, label: string): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new RangeError(`${label} must be a non-negative safe integer.`);
+  }
+  return value;
+}
+
+function normalizeTimestamp(value: Date | string, label: string): string {
+  const date = value instanceof Date ? value : new Date(value);
+  if (!Number.isFinite(date.getTime())) {
+    throw new RangeError(`${label} must be a valid timestamp.`);
+  }
+  return date.toISOString();
+}
+
+function roundScore(value: number): number {
+  return Math.round(value * 1000) / 1000;
+}