diff --git a/.env.example b/.env.example index 719a2147..866cac37 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,12 @@ BCG_GRAPH_START_TIMEOUT="120" BCG_GRAPH_TIMEOUT_MS="300000" BCG_RECENT_TURNS="2" +# Optional local RAG context tuning. The default database path is session-local +# under ~/.bcg/agent/rag; set BCG_RAG_DB_PATH only to override it. +BCG_RAG_DB_PATH="" +BCG_RAG_TOP_K="6" +BCG_RAG_MAX_CHARS="12000" + # Optional local GPU service profiles. These names are deliberately separate # from MODEL, which is the remote/API model used by Agent rollouts. VLLM_MODEL="" diff --git a/agent-cli/README.md b/agent-cli/README.md index 09417235..68fef047 100644 --- a/agent-cli/README.md +++ b/agent-cli/README.md @@ -6,12 +6,17 @@ Interactive terminal Agent for The public command is `bcg`, provided by the Python package. This Node package installs the internal `bcg-agent` runtime used by that launcher. -The runtime supports three session-level context modes. Default retains normal -full context with compaction. BCG and Summary both pin the initial user input, -retain a configurable number of recent completed turns, and evict older turns -in the same batches. BCG converts those batches into a belief graph; Summary -updates one rolling LLM summary. Either memory block is injected into the -system prompt. +The runtime supports five session-level context modes. Every bounded mode +permanently pins the initial user input and retains a configurable number of +recent completed turns: + +- **Default** keeps the full conversation with automatic compaction. +- **Recent-Only** drops older turns and leaves the system prompt unchanged. +- **RAG** stores dropped turns in a session-local SQLite FTS5 database, queries + it with the recent raw turns, and injects retrieved history into the system + prompt. +- **Summary** compresses dropped turns into one rolling LLM summary. +- **BCG** converts dropped turns into a confidence-aware belief graph. Configuration and sessions live under `~/.bcg/agent/`. API credentials can be entered with `/login`; custom OpenAI-compatible endpoints use diff --git a/agent-cli/src/core/agent-session.ts b/agent-cli/src/core/agent-session.ts index d8baa4c2..a572401c 100644 --- a/agent-cli/src/core/agent-session.ts +++ b/agent-cli/src/core/agent-session.ts @@ -62,7 +62,7 @@ import { prepareCompaction, shouldCompact, } from "./compaction/index.ts"; -import { getSessionContextMode } from "./context/context-mode.ts"; +import { getSessionContextMode, usesBoundedContext } from "./context/context-mode.ts"; import { DEFAULT_THINKING_LEVEL } from "./defaults.ts"; import { exportSessionToHtml, type ToolHtmlRenderer } from "./export-html/index.ts"; import { createToolHtmlRenderer } from "./export-html/tool-renderer.ts"; @@ -1776,8 +1776,8 @@ export class AgentSession { */ async compact(customInstructions?: string): Promise { const contextMode = getSessionContextMode(this.sessionManager); - if (contextMode === "bcg" || contextMode === "summary") { - throw new Error("Traditional compaction is disabled in BCG and Summary modes."); + if (usesBoundedContext(contextMode)) { + throw new Error("Traditional compaction is disabled in bounded-context modes."); } this._disconnectFromAgent(); await this.abort(); @@ -1950,7 +1950,7 @@ export class AgentSession { */ private async _checkCompaction(assistantMessage: AssistantMessage, skipAbortedCheck = true): Promise { const contextMode = getSessionContextMode(this.sessionManager); - if (contextMode === "bcg" || contextMode === "summary") { + if (usesBoundedContext(contextMode)) { return false; } const settings = this.settingsManager.getCompactionSettings(); @@ -2226,7 +2226,7 @@ export class AgentSession { /** Whether auto-compaction is enabled */ get autoCompactionEnabled(): boolean { const contextMode = getSessionContextMode(this.sessionManager); - return contextMode !== "bcg" && contextMode !== "summary" && this.settingsManager.getCompactionEnabled(); + return !usesBoundedContext(contextMode) && this.settingsManager.getCompactionEnabled(); } async bindExtensions(bindings: ExtensionBindings): Promise { diff --git a/agent-cli/src/core/context/context-mode.ts b/agent-cli/src/core/context/context-mode.ts index 094d94a5..f6d479ef 100644 --- a/agent-cli/src/core/context/context-mode.ts +++ b/agent-cli/src/core/context/context-mode.ts @@ -3,8 +3,18 @@ import type { ContextManagementProvider } from "../settings-manager.ts"; export const CONTEXT_MODE_ENTRY_TYPE = "bcg.context_mode"; +export function usesBoundedContext(provider: ContextManagementProvider | undefined): boolean { + return provider === "bcg" || provider === "summary" || provider === "recent-only" || provider === "rag"; +} + function isContextManagementProvider(value: unknown): value is ContextManagementProvider { - return value === "default" || value === "bcg" || value === "summary"; + return ( + value === "default" || + value === "bcg" || + value === "summary" || + value === "recent-only" || + value === "rag" + ); } export function getSessionContextMode(sessionManager: SessionManager): ContextManagementProvider | undefined { diff --git a/agent-cli/src/core/context/recent-context.ts b/agent-cli/src/core/context/recent-context.ts new file mode 100644 index 00000000..29ae0c8e --- /dev/null +++ b/agent-cli/src/core/context/recent-context.ts @@ -0,0 +1,404 @@ +import { createHash } from "node:crypto"; +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import type { AgentMessage } from "@bigai-nlco/bcg-agent-core"; +import { + contextMessageKey, + contextMessageText, + partitionContextTurns, + splitBcgTurns, +} from "./bcg-context.ts"; + +const RAG_CONTEXT_GUIDE = ` +Earlier completed turns have been omitted from the raw conversation. The excerpts below were retrieved from the session's local history database using the recent raw turns as the query. They are prior working context, not verified evidence. Use relevant excerpts to continue prior work and avoid repeating completed actions; ignore irrelevant excerpts and prefer the current raw conversation when they conflict. +`; + +const DEFAULT_TOP_K = 6; +const DEFAULT_MAX_CHARS = 12_000; +const MAX_QUERY_TERMS = 48; + +const QUERY_STOP_WORDS = new Set([ + "about", + "after", + "again", + "also", + "assistant", + "before", + "could", + "from", + "have", + "into", + "just", + "more", + "result", + "search", + "should", + "that", + "their", + "then", + "there", + "these", + "they", + "this", + "tool", + "using", + "what", + "when", + "where", + "which", + "with", + "would", +]); + +export interface RecentContextManagerOptions { + recentTurns: number; + getInitialUserMessage?: () => AgentMessage | undefined; +} + +export interface RagContextManagerOptions extends RecentContextManagerOptions { + databasePath: string; + topK?: number; + maxChars?: number; + onWarning?: (message: string) => void; + onRagContext?: (trace: RagContextTrace) => void; +} + +export interface RagContextTrace { + query: string; + storedTurns: number; + retrievedTurns: number; + retrievedTurnIndices: number[]; + chars: number; + text: string; +} + +interface PartitionedContext { + initialUser: AgentMessage; + evicted: AgentMessage[][]; + retained: AgentMessage[][]; +} + +interface RetrievedTurn { + id: number; + turnIndex: number; + content: string; + rank: number; +} + +interface SqliteModule { + DatabaseSync: typeof import("node:sqlite").DatabaseSync; +} + +let sqliteModulePromise: Promise | undefined; + +function loadSqlite(): Promise { + if (!sqliteModulePromise) { + // Node 22.19 satisfies this package's engine floor and ships node:sqlite, + // but still labels it experimental. Hide only that one runtime warning so + // entering RAG mode does not pollute the terminal or benchmark stderr. + const originalEmitWarning = process.emitWarning; + process.emitWarning = ((warning: string | Error, ...args: unknown[]) => { + const warningType = typeof args[0] === "string" ? args[0] : undefined; + if (warningType === "ExperimentalWarning" && String(warning).includes("SQLite")) { + return; + } + (originalEmitWarning as (...values: unknown[]) => void)(warning, ...args); + }) as typeof process.emitWarning; + sqliteModulePromise = (import("node:sqlite") as Promise).finally(() => { + process.emitWarning = originalEmitWarning; + }); + } + return sqliteModulePromise; +} + +function resolveInitialUser( + messages: AgentMessage[], + configured?: () => AgentMessage | undefined, +): AgentMessage | undefined { + const explicit = configured?.(); + if (explicit?.role === "user") return explicit; + return messages.find((message) => message.role === "user"); +} + +function partitionMessages( + messages: AgentMessage[], + recentTurns: number, + configuredInitial?: () => AgentMessage | undefined, +): PartitionedContext { + const initialUser = resolveInitialUser(messages, configuredInitial); + if (!initialUser) { + throw new Error("the session has no initial user input"); + } + + const initialKey = contextMessageKey(initialUser); + let removedInitial = false; + const rest = messages.filter((message) => { + if (!removedInitial && contextMessageKey(message) === initialKey) { + removedInitial = true; + return false; + } + return true; + }); + const { evicted, retained } = partitionContextTurns(splitBcgTurns(rest), recentTurns); + return { initialUser, evicted, retained }; +} + +function messageRole(message: AgentMessage): string { + switch (message.role) { + case "toolResult": + case "bashExecution": + return "tool"; + case "branchSummary": + case "compactionSummary": + case "custom": + return "user"; + default: + return message.role; + } +} + +function renderTurn(messages: AgentMessage[]): string { + return messages + .map((message) => { + const content = contextMessageText(message).trim(); + return `[${messageRole(message)}]\n${content || "(empty)"}`; + }) + .join("\n\n"); +} + +function turnKey(messages: AgentMessage[]): string { + return createHash("sha256") + .update(messages.map((message) => contextMessageKey(message)).join("\u0001")) + .digest("hex"); +} + +function queryTerms(text: string): string[] { + const terms = text + .normalize("NFKC") + .toLocaleLowerCase() + .match(/[\p{L}\p{N}_-]{2,}/gu); + if (!terms) return []; + + const unique: string[] = []; + const seen = new Set(); + for (let index = terms.length - 1; index >= 0 && unique.length < MAX_QUERY_TERMS; index -= 1) { + const term = terms[index]; + if (QUERY_STOP_WORDS.has(term) || seen.has(term)) continue; + seen.add(term); + unique.push(term); + } + return unique.reverse(); +} + +function ftsQuery(text: string): string { + return queryTerms(text) + .map((term) => `"${term.replaceAll('"', '""')}"`) + .join(" OR "); +} + +function renderRetrievedHistory(turns: RetrievedTurn[], maxChars: number): string { + const sections: string[] = []; + let used = 0; + for (const turn of turns) { + const section = `### Earlier turn ${turn.turnIndex}\n${turn.content}`; + const separator = sections.length > 0 ? 2 : 0; + if (used + separator + section.length <= maxChars) { + sections.push(section); + used += separator + section.length; + continue; + } + const remaining = maxChars - used - separator; + if (remaining > 80) { + sections.push(`${section.slice(0, remaining - 1).trimEnd()}…`); + } + break; + } + return sections.join("\n\n"); +} + +/** Keep the initial request plus the configured number of recent completed turns. */ +export class RecentOnlyContextManager { + private readonly recentTurns: number; + private readonly getInitialUserMessage?: () => AgentMessage | undefined; + + constructor(options: RecentContextManagerOptions) { + this.recentTurns = Math.max(-1, Math.trunc(options.recentTurns)); + this.getInitialUserMessage = options.getInitialUserMessage; + } + + async transform(messages: AgentMessage[]): Promise { + const { initialUser, retained } = partitionMessages( + messages, + this.recentTurns, + this.getInitialUserMessage, + ); + return [initialUser, ...retained.flat()]; + } + + augmentSystemPrompt(systemPrompt: string | undefined): string | undefined { + return systemPrompt; + } +} + +/** SQLite FTS-backed retrieval over turns evicted by the recent-context window. */ +export class RagContextManager { + private readonly recentTurns: number; + private readonly databasePath: string; + private readonly topK: number; + private readonly maxChars: number; + private readonly getInitialUserMessage?: () => AgentMessage | undefined; + private readonly onWarning: (message: string) => void; + private readonly onRagContext?: (trace: RagContextTrace) => void; + private database: DatabaseSync | undefined; + private databasePromise: Promise | undefined; + private retrievedText = ""; + private requestReady = false; + private warned = false; + private closed = false; + + constructor(options: RagContextManagerOptions) { + this.recentTurns = Math.max(-1, Math.trunc(options.recentTurns)); + this.databasePath = options.databasePath; + this.topK = Math.max(1, Math.trunc(options.topK ?? DEFAULT_TOP_K)); + this.maxChars = Math.max(256, Math.trunc(options.maxChars ?? DEFAULT_MAX_CHARS)); + this.getInitialUserMessage = options.getInitialUserMessage; + this.onWarning = options.onWarning ?? ((message) => console.warn(message)); + this.onRagContext = options.onRagContext; + } + + async transform(messages: AgentMessage[]): Promise { + this.requestReady = false; + const partitioned = partitionMessages(messages, this.recentTurns, this.getInitialUserMessage); + const bounded = [partitioned.initialUser, ...partitioned.retained.flat()]; + if (this.recentTurns < 0) { + this.retrievedText = ""; + this.requestReady = true; + return bounded; + } + try { + const database = await this.getDatabase(); + this.storeTurns(database, partitioned.evicted); + const query = partitioned.retained.flat().map(contextMessageText).filter(Boolean).join("\n\n"); + const retrieved = this.retrieve(database, query); + this.retrievedText = renderRetrievedHistory(retrieved, this.maxChars); + this.requestReady = true; + this.warned = false; + this.onRagContext?.({ + query, + storedTurns: this.storedTurnCount(database), + retrievedTurns: retrieved.length, + retrievedTurnIndices: retrieved.map((turn) => turn.turnIndex), + chars: this.retrievedText.length, + text: this.retrievedText, + }); + } catch (error) { + this.retrievedText = ""; + if (!this.warned) { + const detail = error instanceof Error ? error.message : String(error); + this.onWarning(`[RAG context] ${detail}; using recent-only context for this request.`); + this.warned = true; + } + } + return bounded; + } + + augmentSystemPrompt(systemPrompt: string | undefined): string | undefined { + if (!this.requestReady || !this.retrievedText) return systemPrompt; + const block = `\n${this.retrievedText}\n`; + return [systemPrompt, RAG_CONTEXT_GUIDE, block].filter(Boolean).join("\n\n"); + } + + release(): void { + if (this.closed) return; + this.closed = true; + this.database?.close(); + this.database = undefined; + } + + private async getDatabase(): Promise { + if (this.closed) throw new Error("the RAG history database is already closed"); + if (this.database) return this.database; + this.databasePromise ??= this.openDatabase(); + return this.databasePromise; + } + + private async openDatabase(): Promise { + mkdirSync(dirname(this.databasePath), { recursive: true }); + const { DatabaseSync } = await loadSqlite(); + const database = new DatabaseSync(this.databasePath); + database.exec(` + PRAGMA journal_mode = WAL; + CREATE TABLE IF NOT EXISTS memory ( + id INTEGER PRIMARY KEY, + turn_key TEXT NOT NULL UNIQUE, + turn_index INTEGER NOT NULL, + content TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5( + content, + content='memory', + content_rowid='id', + tokenize='unicode61 remove_diacritics 2' + ); + CREATE TRIGGER IF NOT EXISTS memory_insert AFTER INSERT ON memory BEGIN + INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content); + END; + CREATE TRIGGER IF NOT EXISTS memory_delete AFTER DELETE ON memory BEGIN + INSERT INTO memory_fts(memory_fts, rowid, content) VALUES ('delete', old.id, old.content); + END; + CREATE TRIGGER IF NOT EXISTS memory_update AFTER UPDATE ON memory BEGIN + INSERT INTO memory_fts(memory_fts, rowid, content) VALUES ('delete', old.id, old.content); + INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content); + END; + `); + this.database = database; + return database; + } + + private storeTurns(database: DatabaseSync, turns: AgentMessage[][]): void { + if (turns.length === 0) return; + const nextIndexRow = database.prepare("SELECT COALESCE(MAX(turn_index), 0) + 1 AS next_index FROM memory").get() as + | { next_index?: number } + | undefined; + let nextIndex = Number(nextIndexRow?.next_index ?? 1); + const insert = database.prepare( + "INSERT OR IGNORE INTO memory(turn_key, turn_index, content, created_at) VALUES (?, ?, ?, ?)", + ); + for (const turn of turns) { + if (turn.length === 0) continue; + const result = insert.run(turnKey(turn), nextIndex, renderTurn(turn), new Date().toISOString()); + if (Number(result.changes) > 0) nextIndex += 1; + } + } + + private retrieve(database: DatabaseSync, query: string): RetrievedTurn[] { + const match = ftsQuery(query); + if (!match) return []; + const rows = database + .prepare( + `SELECT memory.id AS id, memory.turn_index AS turn_index, memory.content AS content, + bm25(memory_fts) AS rank + FROM memory_fts + JOIN memory ON memory.id = memory_fts.rowid + WHERE memory_fts MATCH ? + ORDER BY rank ASC, memory.turn_index DESC + LIMIT ?`, + ) + .all(match, this.topK) as Array>; + return rows + .map((row) => ({ + id: Number(row.id), + turnIndex: Number(row.turn_index), + content: String(row.content ?? ""), + rank: Number(row.rank ?? 0), + })) + .sort((left, right) => left.turnIndex - right.turnIndex); + } + + private storedTurnCount(database: DatabaseSync): number { + const row = database.prepare("SELECT COUNT(*) AS count FROM memory").get() as { count?: number } | undefined; + return Number(row?.count ?? 0); + } +} diff --git a/agent-cli/src/core/sdk.ts b/agent-cli/src/core/sdk.ts index 9a409519..dec10a57 100644 --- a/agent-cli/src/core/sdk.ts +++ b/agent-cli/src/core/sdk.ts @@ -9,6 +9,7 @@ import { AgentSession } from "./agent-session.ts"; import { formatNoModelsAvailableMessage } from "./auth-guidance.ts"; import { BcgContextManager } from "./context/bcg-context.ts"; import { ensureSessionContextMode, getSessionContextMode } from "./context/context-mode.ts"; +import { RagContextManager, RecentOnlyContextManager } from "./context/recent-context.ts"; import { SummaryContextManager } from "./context/summary-context.ts"; import { DEFAULT_THINKING_LEVEL } from "./defaults.ts"; import type { ExtensionRunner, LoadExtensionsResult, SessionStartEvent, ToolDefinition } from "./extensions/index.ts"; @@ -308,10 +309,13 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} const bcgProblemId = `${sessionManager.getSessionId()}:${randomUUID()}`; const graphTracePath = process.env.BCG_GRAPH_TRACE_PATH?.trim(); const summaryTracePath = process.env.BCG_SUMMARY_TRACE_PATH?.trim(); + const ragTracePath = process.env.BCG_RAG_TRACE_PATH?.trim(); const modelIoTracePath = process.env.BCG_MODEL_IO_TRACE_PATH?.trim(); const modelIoTrace = modelIoTracePath ? new ModelIoTraceRecorder(modelIoTracePath) : undefined; let bcgContextManager: BcgContextManager | undefined; let summaryContextManager: SummaryContextManager | undefined; + let recentOnlyContextManager: RecentOnlyContextManager | undefined; + let ragContextManager: RagContextManager | undefined; const getInitialUserMessage = (): AgentMessage | undefined => { for (const entry of sessionManager.getBranch()) { if (entry.type === "message" && entry.message.role === "user") { @@ -407,7 +411,50 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} }); return summaryContextManager; }; - const getActiveContextManager = () => getBcgContextManager() ?? getSummaryContextManager(); + const getRecentOnlyContextManager = (): RecentOnlyContextManager | undefined => { + if (getSessionContextMode(sessionManager) !== "recent-only") { + return undefined; + } + const contextManagementSettings = settingsManager.getContextManagementSettings(); + recentOnlyContextManager ??= new RecentOnlyContextManager({ + recentTurns: contextManagementSettings.recentOnly.recentTurns, + getInitialUserMessage, + }); + return recentOnlyContextManager; + }; + const getRagContextManager = (): RagContextManager | undefined => { + if (getSessionContextMode(sessionManager) !== "rag") { + return undefined; + } + const contextManagementSettings = settingsManager.getContextManagementSettings(); + const ragSettings = contextManagementSettings.rag; + const databasePath = ragSettings.databasePath + ? resolvePath(ragSettings.databasePath) + : join(agentDir, "rag", `${sessionManager.getSessionId()}.sqlite`); + ragContextManager ??= new RagContextManager({ + recentTurns: ragSettings.recentTurns, + databasePath, + topK: ragSettings.topK, + maxChars: ragSettings.maxChars, + getInitialUserMessage, + onRagContext: ragTracePath + ? (trace) => { + mkdirSync(dirname(ragTracePath), { recursive: true }); + appendFileSync( + ragTracePath, + `${JSON.stringify({ timestamp: new Date().toISOString(), ...trace })}\n`, + "utf8", + ); + } + : undefined, + }); + return ragContextManager; + }; + const getActiveContextManager = () => + getBcgContextManager() ?? + getSummaryContextManager() ?? + getRecentOnlyContextManager() ?? + getRagContextManager(); agent = new Agent({ initialState: { @@ -552,6 +599,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} if (summaryUsage) { session.emitSummaryUsage({ ...summaryUsage }); } + ragContextManager?.release(); }, }; } diff --git a/agent-cli/src/core/settings-manager.ts b/agent-cli/src/core/settings-manager.ts index 101e4ce9..f4b76e45 100644 --- a/agent-cli/src/core/settings-manager.ts +++ b/agent-cli/src/core/settings-manager.ts @@ -14,7 +14,7 @@ export interface CompactionSettings { keepRecentTokens?: number; // default: 20000 } -export type ContextManagementProvider = "default" | "bcg" | "summary"; +export type ContextManagementProvider = "default" | "bcg" | "summary" | "recent-only" | "rag"; export type BcgGraphView = "full" | "compact"; export interface BcgContextSettings { @@ -36,10 +36,23 @@ export interface SummaryContextSettings { thinkingLevel?: ThinkingLevel; // default: "off" } +export interface RecentOnlyContextSettings { + recentTurns?: number; // default: 2; -1 keeps all raw turns +} + +export interface RagContextSettings { + recentTurns?: number; // default: 2; -1 keeps all raw turns + databasePath?: string; // default: ~/.bcg/agent/rag/.sqlite + topK?: number; // default: 6 retrieved historical turns + maxChars?: number; // default: 12000 injected history characters +} + export interface ContextManagementSettings { provider?: ContextManagementProvider; // default: "default" bcg?: BcgContextSettings; summary?: SummaryContextSettings; + recentOnly?: RecentOnlyContextSettings; + rag?: RagContextSettings; } export interface ResolvedContextManagementSettings { @@ -61,6 +74,15 @@ export interface ResolvedContextManagementSettings { maxTokens: number; thinkingLevel: ThinkingLevel; }; + recentOnly: { + recentTurns: number; + }; + rag: { + recentTurns: number; + databasePath: string; + topK: number; + maxChars: number; + }; } export interface BranchSummarySettings { @@ -881,10 +903,33 @@ export class SettingsManager { ? Math.max(1, Math.trunc(configuredSummaryMaxTokens)) : 2048; const configuredProvider = settings?.provider; + const configuredRecentOnlyTurns = settings?.recentOnly?.recentTurns; + const recentOnlyTurns = + typeof configuredRecentOnlyTurns === "number" && Number.isFinite(configuredRecentOnlyTurns) + ? Math.max(-1, Math.trunc(configuredRecentOnlyTurns)) + : recentTurns; + const configuredRagRecentTurns = settings?.rag?.recentTurns; + const ragRecentTurns = + typeof configuredRagRecentTurns === "number" && Number.isFinite(configuredRagRecentTurns) + ? Math.max(-1, Math.trunc(configuredRagRecentTurns)) + : recentTurns; + const configuredRagTopK = settings?.rag?.topK; + const ragTopK = + typeof configuredRagTopK === "number" && Number.isFinite(configuredRagTopK) + ? Math.max(1, Math.trunc(configuredRagTopK)) + : 6; + const configuredRagMaxChars = settings?.rag?.maxChars; + const ragMaxChars = + typeof configuredRagMaxChars === "number" && Number.isFinite(configuredRagMaxChars) + ? Math.max(256, Math.trunc(configuredRagMaxChars)) + : 12000; return { provider: - configuredProvider === "bcg" || configuredProvider === "summary" + configuredProvider === "bcg" || + configuredProvider === "summary" || + configuredProvider === "recent-only" || + configuredProvider === "rag" ? configuredProvider : "default", bcg: { @@ -908,6 +953,15 @@ export class SettingsManager { maxTokens: summaryMaxTokens, thinkingLevel: settings?.summary?.thinkingLevel ?? "off", }, + recentOnly: { + recentTurns: recentOnlyTurns, + }, + rag: { + recentTurns: ragRecentTurns, + databasePath: settings?.rag?.databasePath?.trim() || process.env.BCG_RAG_DB_PATH?.trim() || "", + topK: ragTopK, + maxChars: ragMaxChars, + }, }; } diff --git a/agent-cli/src/core/slash-commands.ts b/agent-cli/src/core/slash-commands.ts index 8906cebc..77e5c7a2 100644 --- a/agent-cli/src/core/slash-commands.ts +++ b/agent-cli/src/core/slash-commands.ts @@ -21,8 +21,8 @@ export const BUILTIN_SLASH_COMMANDS: ReadonlyArray = [ { name: "model", description: "Select the inference model", argumentHint: "" }, { name: "mode", - description: "Choose Default, BCG, or Summary context", - argumentHint: "", + description: "Choose the session context strategy", + argumentHint: "", }, { name: "login", description: "Configure the model API key", argumentHint: "" }, { name: "logout", description: "Remove a saved model API key" }, diff --git a/agent-cli/src/index.ts b/agent-cli/src/index.ts index 6e57d7ae..3b678315 100644 --- a/agent-cli/src/index.ts +++ b/agent-cli/src/index.ts @@ -253,6 +253,8 @@ export { type ImageSettings, type PackageSource, type ResolvedContextManagementSettings, + type RagContextSettings, + type RecentOnlyContextSettings, type RetrySettings, SettingsManager, type SettingsManagerCreateOptions, diff --git a/agent-cli/src/modes/interactive/components/context-mode-selector.ts b/agent-cli/src/modes/interactive/components/context-mode-selector.ts index 082bf216..d511383b 100644 --- a/agent-cli/src/modes/interactive/components/context-mode-selector.ts +++ b/agent-cli/src/modes/interactive/components/context-mode-selector.ts @@ -29,6 +29,16 @@ export class ContextModeSelectorComponent extends Container { label: "Summary", description: "Rolling LLM summary · initial request + recent turns", }, + { + value: "rag", + label: "RAG", + description: "Retrieved local history · initial request + recent turns", + }, + { + value: "recent-only", + label: "Recent-Only", + description: "Initial request + recent turns · no long-term memory", + }, { value: "default", label: "Default", @@ -37,8 +47,9 @@ export class ContextModeSelectorComponent extends Container { ]; this.addChild(new DynamicBorder()); - this.selectList = new SelectList(items, 5, getSelectListTheme(), CONTEXT_MODE_SELECT_LIST_LAYOUT); - this.selectList.setSelectedIndex(currentMode === "bcg" ? 0 : currentMode === "summary" ? 1 : 2); + this.selectList = new SelectList(items, 7, getSelectListTheme(), CONTEXT_MODE_SELECT_LIST_LAYOUT); + const selectedIndex = items.findIndex((item) => item.value === currentMode); + this.selectList.setSelectedIndex(selectedIndex >= 0 ? selectedIndex : items.length - 1); this.selectList.onSelect = (item) => { onSelect(item.value as ContextManagementProvider); }; diff --git a/agent-cli/src/modes/interactive/interactive-mode.ts b/agent-cli/src/modes/interactive/interactive-mode.ts index 21d7ace6..5805bcee 100644 --- a/agent-cli/src/modes/interactive/interactive-mode.ts +++ b/agent-cli/src/modes/interactive/interactive-mode.ts @@ -605,6 +605,9 @@ export class InteractiveMode { const normalizedPrefix = prefix.trim().toLowerCase(); return [ { value: "bcg", label: "bcg", description: "Graph memory context" }, + { value: "rag", label: "rag", description: "Retrieved local history context" }, + { value: "recent-only", label: "recent-only", description: "Recent raw turns only" }, + { value: "summary", label: "summary", description: "Rolling summary context" }, { value: "default", label: "default", description: "Full context with automatic compaction" }, ].filter((item) => item.value.startsWith(normalizedPrefix)); }; @@ -2151,11 +2154,13 @@ ${block.body}`, 0, 0), } else if ( requestedMode === "default" || requestedMode === "bcg" || - requestedMode === "summary" + requestedMode === "summary" || + requestedMode === "recent-only" || + requestedMode === "rag" ) { this.handleContextModeChange(requestedMode); } else { - this.showWarning("Usage: /mode "); + this.showWarning("Usage: /mode "); } return; } @@ -4509,7 +4514,11 @@ ${block.body}`, 0, 0), ? theme.fg("success", "● GRAPH ACTIVE") : mode === "summary" ? theme.fg("success", "● SUMMARY ACTIVE") - : theme.fg("warning", "○ MEMORY DISABLED"); + : mode === "rag" + ? theme.fg("success", "● RAG ACTIVE") + : mode === "recent-only" + ? theme.fg("warning", "○ RECENT WINDOW") + : theme.fg("warning", "○ MEMORY DISABLED"); const modelProvider = model?.provider === "bcg-openai" ? "bcg" : model?.provider; const modelName = model ? `${modelProvider}/${model.id}` : "not configured"; const contextLine = @@ -4519,9 +4528,17 @@ ${block.body}`, 0, 0), ? context.summary.recentTurns < 0 ? "Context initial request pinned · raw history retained" : `Context initial request pinned · ${context.summary.recentTurns} recent turns · older turns → summary` - : context.bcg.recentTurns < 0 - ? "Context initial request pinned · raw history retained" - : `Context initial request pinned · ${context.bcg.recentTurns} recent turns · older turns → graph`; + : mode === "rag" + ? context.rag.recentTurns < 0 + ? "Context initial request pinned · raw history retained" + : `Context initial request pinned · ${context.rag.recentTurns} recent turns · older turns → local RAG` + : mode === "recent-only" + ? context.recentOnly.recentTurns < 0 + ? "Context initial request pinned · raw history retained" + : `Context initial request pinned · ${context.recentOnly.recentTurns} recent turns · no memory` + : context.bcg.recentTurns < 0 + ? "Context initial request pinned · raw history retained" + : `Context initial request pinned · ${context.bcg.recentTurns} recent turns · older turns → graph`; const graphEndpoint = mode === "bcg" ? ` ${context.bcg.url}` : ""; const commandLine = `${theme.fg("accent", "/help")} commands ${theme.fg("accent", "/model")} model ${theme.fg("accent", "/mode")} context mode`; return `${title}\n${graphState}${graphEndpoint}\nMode ${mode}\nModel ${modelName}\n${contextLine}\n${commandLine}`; @@ -4553,7 +4570,11 @@ ${block.body}`, 0, 0), ? "Mode: BCG · graph memory with recent raw turns" : mode === "summary" ? "Mode: Summary · rolling LLM summary with recent raw turns" - : "Mode: Default · full context with automatic compaction", + : mode === "rag" + ? "Mode: RAG · retrieved local history with recent raw turns" + : mode === "recent-only" + ? "Mode: Recent-Only · initial request with recent raw turns" + : "Mode: Default · full context with automatic compaction", ); } @@ -4607,6 +4628,12 @@ ${block.body}`, 0, 0), const rawWindow = context.bcg.recentTurns < 0 ? "all turns" : `${context.bcg.recentTurns} completed turns`; const summaryRawWindow = context.summary.recentTurns < 0 ? "all turns" : `${context.summary.recentTurns} completed turns`; + const recentOnlyRawWindow = + context.recentOnly.recentTurns < 0 + ? "all turns" + : `${context.recentOnly.recentTurns} completed turns`; + const ragRawWindow = + context.rag.recentTurns < 0 ? "all turns" : `${context.rag.recentTurns} completed turns`; const info = mode === "default" ? [ @@ -4626,16 +4653,34 @@ ${block.body}`, 0, 0), `${theme.fg("dim", "Raw context:")} initial user input + ${summaryRawWindow}`, `${theme.fg("dim", "Summary injection:")} system prompt · Markdown`, ] - : [ - theme.bold(theme.fg("accent", "BCG Graph")), - "", - `${theme.fg("dim", "Mode:")} bcg`, - `${theme.fg("dim", "Status:")} ${status}`, - `${theme.fg("dim", "Endpoint:")} ${context.bcg.url}`, - `${theme.fg("dim", "Raw context:")} initial user input + ${rawWindow}`, - `${theme.fg("dim", "Graph injection:")} system prompt · Markdown`, - `${theme.fg("dim", "Relations:")} ${context.bcg.includeRelations ? "enabled" : "disabled"}`, - ]; + : mode === "recent-only" + ? [ + theme.bold(theme.fg("accent", "Recent-Only Context")), + "", + `${theme.fg("dim", "Mode:")} recent-only`, + `${theme.fg("dim", "Raw context:")} initial user input + ${recentOnlyRawWindow}`, + `${theme.fg("dim", "Memory injection:")} disabled`, + ] + : mode === "rag" + ? [ + theme.bold(theme.fg("accent", "RAG Context")), + "", + `${theme.fg("dim", "Mode:")} rag`, + `${theme.fg("dim", "Raw context:")} initial user input + ${ragRawWindow}`, + `${theme.fg("dim", "History store:")} SQLite FTS5`, + `${theme.fg("dim", "Retrieved turns:")} top ${context.rag.topK}`, + `${theme.fg("dim", "RAG injection:")} system prompt · Markdown`, + ] + : [ + theme.bold(theme.fg("accent", "BCG Graph")), + "", + `${theme.fg("dim", "Mode:")} bcg`, + `${theme.fg("dim", "Status:")} ${status}`, + `${theme.fg("dim", "Endpoint:")} ${context.bcg.url}`, + `${theme.fg("dim", "Raw context:")} initial user input + ${rawWindow}`, + `${theme.fg("dim", "Graph injection:")} system prompt · Markdown`, + `${theme.fg("dim", "Relations:")} ${context.bcg.includeRelations ? "enabled" : "disabled"}`, + ]; this.chatContainer.addChild(new Spacer(1)); this.chatContainer.addChild(new Text(info.join("\n"), 1, 0)); this.ui.requestRender(); diff --git a/agent-cli/test/bcg-context.test.ts b/agent-cli/test/bcg-context.test.ts index 42567de4..04541644 100644 --- a/agent-cli/test/bcg-context.test.ts +++ b/agent-cli/test/bcg-context.test.ts @@ -721,6 +721,15 @@ describe("BCG context management", () => { maxTokens: 2048, thinkingLevel: "off", }, + recentOnly: { + recentTurns: -1, + }, + rag: { + recentTurns: -1, + databasePath: "", + topK: 6, + maxChars: 12000, + }, }); }); @@ -735,6 +744,10 @@ describe("BCG context management", () => { expect(getSessionContextMode(session)).toBe("bcg"); setSessionContextMode(session, "summary"); expect(getSessionContextMode(session)).toBe("summary"); + setSessionContextMode(session, "recent-only"); + expect(getSessionContextMode(session)).toBe("recent-only"); + setSessionContextMode(session, "rag"); + expect(getSessionContextMode(session)).toBe("rag"); session.appendMessage(user("first message", 1)); expect(hasSessionConversationStarted(session)).toBe(true); diff --git a/agent-cli/test/recent-context.test.ts b/agent-cli/test/recent-context.test.ts new file mode 100644 index 00000000..f30c404b --- /dev/null +++ b/agent-cli/test/recent-context.test.ts @@ -0,0 +1,158 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AgentMessage } from "@bigai-nlco/bcg-agent-core"; +import type { AssistantMessage, ToolResultMessage, Usage } from "@bigai-nlco/bcg-ai/compat"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { RagContextManager, RecentOnlyContextManager } from "../src/core/context/recent-context.ts"; + +const USAGE: Usage = { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function temporaryDatabase(): string { + const directory = mkdtempSync(join(tmpdir(), "bcg-rag-context-")); + temporaryDirectories.push(directory); + return join(directory, "history.sqlite"); +} + +function user(text: string, timestamp: number): AgentMessage { + return { role: "user", content: text, timestamp }; +} + +function assistant(text: string, timestamp: number): AssistantMessage { + return { + role: "assistant", + content: [{ type: "text", text }], + api: "test-api", + provider: "test-provider", + model: "test-model", + usage: USAGE, + stopReason: "toolUse", + timestamp, + }; +} + +function tool(text: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId: `call-${timestamp}`, + toolName: "web_search", + content: [{ type: "text", text }], + isError: false, + timestamp, + }; +} + +describe("recent-only context management", () => { + it("permanently pins the initial user input and keeps two recent completed turns", async () => { + const initial = user("original task that must remain visible", 1); + const first = [assistant("first action", 2), tool("first evidence", 3)]; + const second = [assistant("second action", 4), tool("second evidence", 5)]; + const third = [assistant("third action", 6), tool("third evidence", 7)]; + const manager = new RecentOnlyContextManager({ + recentTurns: 2, + getInitialUserMessage: () => initial, + }); + + const transformed = await manager.transform([initial, ...first, ...second, ...third]); + + expect(transformed).toEqual([initial, ...second, ...third]); + expect(transformed).not.toContain(first[0]); + expect(manager.augmentSystemPrompt("unchanged system prompt")).toBe("unchanged system prompt"); + }); +}); + +describe("RAG context management", () => { + it("stores evicted turns, retrieves them from SQLite, and injects them into the system prompt", async () => { + const traces: Array<{ storedTurns: number; retrievedTurns: number }> = []; + const databasePath = temporaryDatabase(); + const initial = user("original task that must remain visible", 1); + const earlier = [ + assistant("Investigate Saturn ring composition", 2), + tool("Saturn's rings are predominantly water ice", 3), + ]; + const recent = [ + assistant("Use the Saturn evidence to prepare the answer", 4), + tool("The question asks specifically about Saturn", 5), + ]; + const manager = new RagContextManager({ + recentTurns: 1, + databasePath, + topK: 3, + getInitialUserMessage: () => initial, + onRagContext: (trace) => traces.push(trace), + }); + + const transformed = await manager.transform([initial, ...earlier, ...recent]); + const augmented = manager.augmentSystemPrompt("base system prompt"); + + expect(transformed).toEqual([initial, ...recent]); + expect(augmented).toContain("base system prompt"); + expect(augmented).toContain(""); + expect(augmented).toContain("Saturn's rings are predominantly water ice"); + expect(augmented).not.toContain("original task that must remain visible"); + expect(augmented?.match(/base system prompt/g)).toHaveLength(1); + expect(traces.at(-1)).toMatchObject({ storedTurns: 1, retrievedTurns: 1 }); + + await manager.transform([initial, ...earlier, ...recent]); + expect(traces.at(-1)).toMatchObject({ storedTurns: 1, retrievedTurns: 1 }); + manager.release(); + }); + + it("persists history across manager instances without duplicating the pinned user input", async () => { + const databasePath = temporaryDatabase(); + const initial = user("persistent original task", 1); + const earlier = [assistant("Find the Zephyr codename", 2), tool("Zephyr maps to project Aurora", 3)]; + const recent = [assistant("Recall the Zephyr mapping", 4), tool("Need the Aurora answer", 5)]; + const firstManager = new RagContextManager({ recentTurns: 1, databasePath }); + await firstManager.transform([initial, ...earlier, ...recent]); + firstManager.release(); + + const traces: Array<{ storedTurns: number }> = []; + const resumedManager = new RagContextManager({ + recentTurns: 1, + databasePath, + onRagContext: (trace) => traces.push(trace), + }); + await resumedManager.transform([initial, ...earlier, ...recent]); + + expect(resumedManager.augmentSystemPrompt("system")).toContain("Zephyr maps to project Aurora"); + expect(traces.at(-1)?.storedTurns).toBe(1); + resumedManager.release(); + }); + + it("falls back to the pinned recent-only window when the database cannot be opened", async () => { + const warning = vi.fn(); + const directoryPath = temporaryDatabase().replace(/\/history\.sqlite$/, ""); + const initial = user("task", 1); + const first = [assistant("old work", 2), tool("old result", 3)]; + const recent = [assistant("new work", 4), tool("new result", 5)]; + const manager = new RagContextManager({ + recentTurns: 1, + databasePath: directoryPath, + onWarning: warning, + }); + + const transformed = await manager.transform([initial, ...first, ...recent]); + + expect(transformed).toEqual([initial, ...recent]); + expect(manager.augmentSystemPrompt("system")).toBe("system"); + expect(warning).toHaveBeenCalledOnce(); + manager.release(); + }); +}); diff --git a/bcg/apps/agent_runtime.py b/bcg/apps/agent_runtime.py index 1c364882..8101a1c9 100644 --- a/bcg/apps/agent_runtime.py +++ b/bcg/apps/agent_runtime.py @@ -87,6 +87,12 @@ def ensure_agent_configuration(graph_url: str) -> Path: summary_settings = context.get("summary") if not isinstance(summary_settings, dict): summary_settings = {} + recent_only_settings = context.get("recentOnly") + if not isinstance(recent_only_settings, dict): + recent_only_settings = {} + rag_settings = context.get("rag") + if not isinstance(rag_settings, dict): + rag_settings = {} try: recent_turns = int( @@ -102,6 +108,14 @@ def ensure_agent_configuration(graph_url: str) -> Path: ) ) summary_max_tokens = int(os.environ.get("BCG_SUMMARY_MAX_TOKENS", "2048")) + rag_top_k = int( + os.environ.get("BCG_RAG_TOP_K", str(rag_settings.get("topK", 6))) + ) + rag_max_chars = int( + os.environ.get( + "BCG_RAG_MAX_CHARS", str(rag_settings.get("maxChars", 12000)) + ) + ) timeout_ms = int( os.environ.get( "BCG_GRAPH_TIMEOUT_MS", @@ -122,7 +136,8 @@ def ensure_agent_configuration(graph_url: str) -> Path: "BCG_RECENT_TURNS, BCG_GRAPH_MAX_TURNS, BCG_GRAPH_TIMEOUT_MS, " "BCG_GRAPH_FINALIZATION_TIMEOUT_MS, " "BCG_SUMMARY_RECENT_TURNS, BCG_SUMMARY_TIMEOUT_MS, and " - "BCG_SUMMARY_MAX_TOKENS must be integers." + "BCG_SUMMARY_MAX_TOKENS, BCG_RAG_TOP_K, and BCG_RAG_MAX_CHARS " + "must be integers." ) from exc bcg_settings.update( @@ -158,13 +173,33 @@ def ensure_agent_configuration(graph_url: str) -> Path: .lower(), } ) + recent_only_settings.update({"recentTurns": max(-1, recent_turns)}) + rag_settings.update( + { + "recentTurns": max(-1, recent_turns), + "databasePath": os.environ.get( + "BCG_RAG_DB_PATH", str(rag_settings.get("databasePath", "")) + ).strip(), + "topK": max(1, rag_top_k), + "maxChars": max(256, rag_max_chars), + } + ) configured_context_provider = os.environ.get("BCG_CONTEXT_MODE", "").strip() - if configured_context_provider in {"default", "bcg", "summary"}: + valid_context_providers = { + "default", + "bcg", + "summary", + "recent-only", + "rag", + } + if configured_context_provider in valid_context_providers: context["provider"] = configured_context_provider - elif context.get("provider") not in {"default", "bcg", "summary"}: + elif context.get("provider") not in valid_context_providers: context["provider"] = "bcg" context["bcg"] = bcg_settings context["summary"] = summary_settings + context["recentOnly"] = recent_only_settings + context["rag"] = rag_settings settings["contextManagement"] = context settings.setdefault("defaultThinkingLevel", "off") settings["enableInstallTelemetry"] = False diff --git a/bcg/apps/benchmark/README.md b/bcg/apps/benchmark/README.md index 73bb59aa..9a86e013 100644 --- a/bcg/apps/benchmark/README.md +++ b/bcg/apps/benchmark/README.md @@ -17,16 +17,21 @@ Data policy: ## Benchmark Adapter -The reference Agent can be evaluated head-to-head in **Default**, **BCG**, and **Summary** modes against **BrowseComp** and **BrowseComp (ZH)**. All modes use the same Agent model, prompt, and scorer; only context management changes. +The reference Agent can be evaluated head-to-head in **Default**, **Recent-Only**, **RAG**, **Summary**, and **BCG** modes against **BrowseComp** and **BrowseComp (ZH)**. All modes use the same Agent model, prompt, and scorer; only context management changes. Every bounded mode permanently retains the initial user input. ```bash -bcg benchmark run browsecomp browsecomp_zh --modes default,bcg,summary \ +bcg benchmark run browsecomp browsecomp_zh --modes default,recent-only,rag,summary,bcg \ --thinking off \ --summary-model gpt-4.1-mini \ --summary-thinking off \ + --recent-turns 2 \ + --rag-top-k 6 \ + --rag-max-chars 12000 \ --max-problems 100 \ --workers 8 \ --output-dir results/browsecomp-comparison ``` +RAG stores one SQLite database per task under `BENCHMARK/rag/rag-memory/` and writes retrieved-context snapshots under `BENCHMARK/rag/rag-contexts/`. + See [Evaluate with benchmarks](https://belief-context-graph.docs.buildwithfern.com/operate/benchmarking) for dataset setup, scoring, output artifacts, and every `bcg benchmark run` option. diff --git a/bcg/apps/benchmark/cli.py b/bcg/apps/benchmark/cli.py index acc27209..a29a46d9 100644 --- a/bcg/apps/benchmark/cli.py +++ b/bcg/apps/benchmark/cli.py @@ -33,7 +33,7 @@ def _bootstrap_env() -> None: app = typer.Typer( name="bcg benchmark", - help="Evaluate the reference Agent in Default, BCG, and Summary modes.", + help="Evaluate the reference Agent across supported context modes.", add_completion=False, context_settings={"help_option_names": ["-h", "--help"]}, rich_markup_mode="rich", @@ -43,7 +43,7 @@ def _bootstrap_env() -> None: @app.callback() def _root() -> None: - """Evaluate the reference Agent in Default, BCG, and Summary modes.""" + """Evaluate the reference Agent across supported context modes.""" @app.command("run") @@ -65,7 +65,9 @@ def run( ] = None, modes: Annotated[ str, - typer.Option(help="Comma-separated context modes: default,bcg,summary."), + typer.Option( + help=("Comma-separated context modes: default,bcg,summary,recent-only,rag.") + ), ] = "default,bcg", output_dir: Annotated[ Path | None, @@ -142,9 +144,19 @@ def run( int, typer.Option( min=-1, - help="Completed turns retained verbatim in BCG and Summary modes.", + help="Completed turns retained verbatim in bounded-context modes.", ), ] = 2, + rag_top_k: Annotated[ + int, + typer.Option(min=1, help="Maximum historical turns retrieved in RAG mode."), + ] = 6, + rag_max_chars: Annotated[ + int, + typer.Option( + min=256, help="Maximum retrieved-history characters injected in RAG mode." + ), + ] = 12_000, summary_model: Annotated[ str | None, typer.Option(help="Rolling-summary model; defaults to the Agent model."), @@ -190,6 +202,12 @@ def run( help="Score Summary tasks even when summarization falls back to raw context." ), ] = False, + allow_rag_fallback: Annotated[ + bool, + typer.Option( + help="Score RAG tasks even when retrieval falls back to Recent-Only." + ), + ] = False, allow_no_search: Annotated[ bool, typer.Option(help="Allow BrowseComp/HotpotQA without SERPER_API_KEY."), @@ -314,10 +332,12 @@ def run( raise typer.BadParameter("--graph-view must be `full` or `compact`.") resolved_modes = tuple(value.strip() for value in modes.split(",") if value.strip()) - invalid_modes = set(resolved_modes) - {"default", "bcg", "summary"} + valid_modes = {"default", "bcg", "summary", "recent-only", "rag"} + invalid_modes = set(resolved_modes) - valid_modes if not resolved_modes or invalid_modes: raise typer.BadParameter( - "--modes must contain only `default`, `bcg`, and/or `summary`." + "--modes must contain only `default`, `bcg`, `summary`, " + "`recent-only`, and/or `rag`." ) resolved_summary_model = (summary_model or resolved_model).strip() resolved_summary_base_url = (summary_base_url or resolved_base_url).strip() @@ -352,6 +372,8 @@ def run( graph_finalization_timeout_ms=graph_finalization_timeout_ms, graph_max_turns=graph_max_turns, recent_turns=recent_turns, + rag_top_k=rag_top_k, + rag_max_chars=rag_max_chars, graph_view=graph_view, summary_model=resolved_summary_model, summary_base_url=resolved_summary_base_url, @@ -361,6 +383,7 @@ def run( summary_max_tokens=summary_max_tokens, allow_graph_fallback=allow_graph_fallback, allow_summary_fallback=allow_summary_fallback, + allow_rag_fallback=allow_rag_fallback, allow_no_search=allow_no_search, overwrite=overwrite, agent_command=tuple(shlex.split(agent_command)) if agent_command else None, diff --git a/bcg/apps/benchmark/runner.py b/bcg/apps/benchmark/runner.py index 60ec9532..efe836ba 100644 --- a/bcg/apps/benchmark/runner.py +++ b/bcg/apps/benchmark/runner.py @@ -39,6 +39,7 @@ "cancelled_after_quota", "graph_fallback", "summary_fallback", + "rag_fallback", } @@ -63,6 +64,8 @@ class RunConfig: graph_finalization_timeout_ms: int = 900_000 graph_max_turns: int = 160 recent_turns: int = 2 + rag_top_k: int = 6 + rag_max_chars: int = 12_000 graph_view: str = "full" summary_model: str = "" summary_base_url: str = "" @@ -72,6 +75,7 @@ class RunConfig: summary_max_tokens: int = 2048 allow_graph_fallback: bool = False allow_summary_fallback: bool = False + allow_rag_fallback: bool = False allow_no_search: bool = False overwrite: bool = False agent_command: tuple[str, ...] | None = None @@ -104,6 +108,8 @@ def run_benchmarks( "graph_finalization_timeout_ms": config.graph_finalization_timeout_ms, "graph_max_turns": config.graph_max_turns, "recent_turns": config.recent_turns, + "rag_top_k": config.rag_top_k, + "rag_max_chars": config.rag_max_chars, "graph_view": config.graph_view, "summary_model": config.summary_model, "summary_base_url": config.summary_base_url, @@ -112,6 +118,7 @@ def run_benchmarks( "summary_max_tokens": config.summary_max_tokens, "allow_graph_fallback": config.allow_graph_fallback, "allow_summary_fallback": config.allow_summary_fallback, + "allow_rag_fallback": config.allow_rag_fallback, "benchmarks": { benchmark: len(tasks) for benchmark, tasks in tasks_by_benchmark.items() }, @@ -278,7 +285,13 @@ def _validate_run( ) -> None: if config.graph_view not in {"full", "compact"}: raise ValueError("graph_view must be 'full' or 'compact'.") - invalid_modes = set(config.modes) - {"default", "bcg", "summary"} + invalid_modes = set(config.modes) - { + "default", + "bcg", + "summary", + "recent-only", + "rag", + } if invalid_modes: raise ValueError(f"Invalid context modes: {', '.join(sorted(invalid_modes))}.") if not config.model.strip() or not config.base_url.strip(): @@ -342,6 +355,15 @@ def _write_agent_configuration( "maxTokens": config.summary_max_tokens, "thinkingLevel": config.summary_thinking, }, + "recentOnly": { + "recentTurns": config.recent_turns, + }, + "rag": { + "recentTurns": config.recent_turns, + "databasePath": "", + "topK": config.rag_top_k, + "maxChars": config.rag_max_chars, + }, }, } is_gpt_56 = "gpt-5.6" in config.model.casefold() @@ -420,6 +442,20 @@ def _run_one( / "summary-contexts" / f"{safe_key}.jsonl" ) + rag_context_trace_path = ( + config.output_dir.expanduser().resolve() + / task.benchmark + / mode + / "rag-contexts" + / f"{safe_key}.jsonl" + ) + rag_database_path = ( + config.output_dir.expanduser().resolve() + / task.benchmark + / mode + / "rag-memory" + / f"{safe_key}.sqlite" + ) model_io_trace_path = ( config.output_dir.expanduser().resolve() / task.benchmark @@ -429,6 +465,11 @@ def _run_one( ) model_io_trace_path.parent.mkdir(parents=True, exist_ok=True) model_io_trace_path.unlink(missing_ok=True) + if mode == "rag": + rag_database_path.parent.mkdir(parents=True, exist_ok=True) + rag_database_path.unlink(missing_ok=True) + rag_database_path.with_suffix(".sqlite-shm").unlink(missing_ok=True) + rag_database_path.with_suffix(".sqlite-wal").unlink(missing_ok=True) with tempfile.TemporaryDirectory(prefix="bcg-benchmark-") as temporary: workspace = Path(temporary) @@ -468,6 +509,8 @@ def _run_one( "BCG_SKIP_VERSION_CHECK": "1", "BCG_GRAPH_TRACE_PATH": str(graph_context_trace_path), "BCG_SUMMARY_TRACE_PATH": str(summary_context_trace_path), + "BCG_RAG_TRACE_PATH": str(rag_context_trace_path), + "BCG_RAG_DB_PATH": str(rag_database_path), "BCG_MODEL_IO_TRACE_PATH": str(model_io_trace_path), "SUMMARY_API_KEY": config.summary_api_key or config.api_key or "EMPTY", } @@ -489,6 +532,7 @@ def _run_one( ) graph_finalization_warning = mode == "bcg" and "[BCG finalization]" in stderr summary_fallback = mode == "summary" and "[Summary context]" in stderr + rag_fallback = mode == "rag" and "[RAG context]" in stderr status = "completed" error: str | None = None score = None @@ -536,6 +580,12 @@ def _run_one( "Summary context failed and the Agent fell back to full raw context; " "this sample is excluded from accuracy." ) + elif rag_fallback and not config.allow_rag_fallback: + status = "rag_fallback" + error = ( + "RAG retrieval failed and the Agent fell back to recent-only context; " + "this sample is excluded from accuracy." + ) elif not task.answers: status = "unscored" error = "This dataset split has no public reference answers." @@ -587,6 +637,7 @@ def _run_one( "graph_fallback": graph_fallback, "graph_finalization_warning": graph_finalization_warning, "summary_fallback": summary_fallback, + "rag_fallback": rag_fallback, "agent_exit_code": return_code, "agent_stop_reason": parsed["stop_reason"], "stderr": stderr, @@ -601,6 +652,12 @@ def _run_one( if summary_context_trace_path.is_file() else None ), + "rag_context_trace": ( + str(rag_context_trace_path) if rag_context_trace_path.is_file() else None + ), + "rag_database": ( + str(rag_database_path) if rag_database_path.is_file() else None + ), "model_io_trace": ( str(model_io_trace_path) if model_io_trace_path.is_file() else None ), @@ -861,6 +918,7 @@ def summarize_results(results: Iterable[dict[str, Any]]) -> dict[str, Any]: "summary_fallbacks": sum( bool(value.get("summary_fallback")) for value in values ), + "rag_fallbacks": sum(bool(value.get("rag_fallback")) for value in values), "wall_time_seconds_total": sum( float(value.get("wall_time_seconds", 0)) for value in values ), @@ -1099,6 +1157,7 @@ def _unexpected_failure( "graph_fallback": False, "graph_finalization_warning": False, "summary_fallback": False, + "rag_fallback": False, "tool_calls": {}, "search_calls": 0, "metrics": {}, diff --git a/bcg/apps/cli.py b/bcg/apps/cli.py index 813460d7..505385ec 100644 --- a/bcg/apps/cli.py +++ b/bcg/apps/cli.py @@ -97,7 +97,7 @@ def _construct(ctx: typer.Context) -> None: @app.command( "benchmark", - help="Evaluate the Agent in Default, BCG, and Summary context modes.", + help="Evaluate the Agent across full, bounded, retrieved, summary, and BCG context modes.", context_settings=_FORWARD_CONTEXT, add_help_option=False, ) diff --git a/bcg/apps/setup.py b/bcg/apps/setup.py index 1c11f8f7..9e359f01 100644 --- a/bcg/apps/setup.py +++ b/bcg/apps/setup.py @@ -556,6 +556,8 @@ def apply_user_configuration( "BCG_GRAPH_EMBEDDING_KEY": str(graph.get("embeddingKey") or EMBEDDING_KEY), "BCG_RECENT_TURNS": str(context.get("recentTurns", 2)), "BCG_CONTEXT_MODE": str(context.get("mode") or "bcg"), + "BCG_RAG_TOP_K": str(context.get("ragTopK", 6)), + "BCG_RAG_MAX_CHARS": str(context.get("ragMaxChars", 12000)), "BCG_SUMMARY_MODEL": str(summary.get("model") or agent.get("model") or ""), "BCG_SUMMARY_BASE_URL": str( summary.get("baseUrl") or agent.get("baseUrl") or "" @@ -721,12 +723,14 @@ def run_setup( ("bcg", "BCG graph-backed context"), ("default", "Default full-context agent with compaction"), ("summary", "Rolling LLM summary with recent raw turns"), + ("recent-only", "Initial user input plus recent raw turns only"), + ("rag", "Retrieved local history plus recent raw turns"), ], default=_current_default(current, "context", "mode", "bcg"), input_fn=input_fn, ) recent_turns = 2 - if context_mode in {"bcg", "summary"}: + if context_mode in {"bcg", "summary", "recent-only", "rag"}: recent_turns_text = _ask( f"Recent completed turns kept verbatim in {context_mode} mode", default=str( diff --git a/tests/test_agent_runtime.py b/tests/test_agent_runtime.py index 602ff927..6c4748ae 100644 --- a/tests/test_agent_runtime.py +++ b/tests/test_agent_runtime.py @@ -18,6 +18,9 @@ def _clean_generated_runtime_environment(monkeypatch) -> None: "BCG_SUMMARY_MODEL", "BCG_SUMMARY_BASE_URL", "BCG_SUMMARY_API_KEY", + "BCG_RAG_DB_PATH", + "BCG_RAG_TOP_K", + "BCG_RAG_MAX_CHARS", ): monkeypatch.delenv(name, raising=False) @@ -60,6 +63,13 @@ def test_agent_configuration_enables_bcg_and_references_env_key( "maxTokens": 2048, "thinkingLevel": "off", }, + "recentOnly": {"recentTurns": 2}, + "rag": { + "recentTurns": 2, + "databasePath": "", + "topK": 6, + "maxChars": 12000, + }, } assert settings["defaultProvider"] == "bcg" assert settings["defaultModel"] == "test-model" @@ -103,6 +113,34 @@ def test_summary_context_uses_independent_model_configuration( assert models["providers"]["bcg-summary"]["models"][0]["id"] == ("summary-model") +@pytest.mark.parametrize("mode", ["recent-only", "rag"]) +def test_bounded_context_modes_are_written_to_agent_settings( + monkeypatch, + tmp_path: Path, + mode: str, +) -> None: + monkeypatch.setenv("BCG_HOME", str(tmp_path)) + monkeypatch.setenv("OPENAI_BASE_URL", "https://agent.test/v1") + monkeypatch.setenv("OPENAI_MODEL", "agent-model") + monkeypatch.setenv("BCG_CONTEXT_MODE", mode) + monkeypatch.setenv("BCG_RECENT_TURNS", "2") + monkeypatch.setenv("BCG_RAG_DB_PATH", str(tmp_path / "rag.sqlite")) + monkeypatch.setenv("BCG_RAG_TOP_K", "4") + monkeypatch.setenv("BCG_RAG_MAX_CHARS", "4096") + + agent_dir = agent_runtime.ensure_agent_configuration("http://127.0.0.1:8848") + settings = json.loads((agent_dir / "settings.json").read_text()) + + assert settings["contextManagement"]["provider"] == mode + assert settings["contextManagement"]["recentOnly"] == {"recentTurns": 2} + assert settings["contextManagement"]["rag"] == { + "recentTurns": 2, + "databasePath": str(tmp_path / "rag.sqlite"), + "topK": 4, + "maxChars": 4096, + } + + def test_summary_provider_is_generated_when_agent_uses_login( monkeypatch, tmp_path: Path, diff --git a/tests/test_benchmark_adapters.py b/tests/test_benchmark_adapters.py index 332f782c..f2596811 100644 --- a/tests/test_benchmark_adapters.py +++ b/tests/test_benchmark_adapters.py @@ -111,6 +111,31 @@ def test_benchmark_summary_uses_an_independent_model_configuration( assert models["providers"]["summary"]["models"][0]["id"] == "summary-model" +@pytest.mark.parametrize("mode", ["recent-only", "rag"]) +def test_benchmark_writes_bounded_context_mode_configuration( + tmp_path: Path, + mode: str, +) -> None: + config = RunConfig( + output_dir=tmp_path, + model="agent-model", + base_url="https://agent.test/v1", + recent_turns=2, + ) + + agent_dir = _write_agent_configuration(tmp_path, config, mode) + settings = json.loads((agent_dir / "settings.json").read_text(encoding="utf-8")) + + assert settings["contextManagement"]["provider"] == mode + assert settings["contextManagement"]["recentOnly"] == {"recentTurns": 2} + assert settings["contextManagement"]["rag"] == { + "recentTurns": 2, + "databasePath": "", + "topK": 6, + "maxChars": 12000, + } + + def test_loads_all_supported_benchmark_schemas(tmp_path: Path) -> None: _write_json( tmp_path / "browse_comp" / "data.json", @@ -609,6 +634,54 @@ def test_final_graph_warning_is_not_runtime_graph_fallback(tmp_path: Path) -> No assert result["graph_finalization_warning"] is True +def test_runner_excludes_rag_recent_only_fallback_from_accuracy(tmp_path: Path) -> None: + fake_agent = tmp_path / "rag_fallback_agent.py" + fake_agent.write_text( + """ +import json +import sys + +print("[RAG context] database unavailable; using recent-only context for this request.", file=sys.stderr) +print(json.dumps({ + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "FINAL ANSWER: A"}], + "usage": {}, + "stopReason": "stop", + }, +})) +""".strip() + + "\n", + encoding="utf-8", + ) + task = BenchmarkTask( + benchmark="mmlu_pro", + task_id="rag-fallback", + question="Question\n\nA. yes\nB. no", + answers=("A",), + ) + output = tmp_path / "results" + config = RunConfig( + output_dir=output, + model="fake", + base_url="https://unused.test/v1", + modes=("rag",), + workers=1, + agent_command=(sys.executable, str(fake_agent)), + ) + + summary = run_benchmarks({"mmlu_pro": [task]}, config, judge=None) + result = json.loads( + (output / "mmlu_pro" / "rag" / "tasks" / "rag-fallback.json").read_text() + ) + + assert result["status"] == "rag_fallback" + assert result["rag_fallback"] is True + assert summary["benchmarks"]["mmlu_pro"]["rag"]["evaluated"] == 0 + assert summary["benchmarks"]["mmlu_pro"]["rag"]["rag_fallbacks"] == 1 + + def test_runner_persists_per_task_model_io_trace_reference(tmp_path: Path) -> None: fake_agent = tmp_path / "trace_agent.py" fake_agent.write_text( diff --git a/tests/test_setup.py b/tests/test_setup.py index cb294792..3b9b5573 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -5,6 +5,8 @@ import stat from pathlib import Path +import pytest + from bcg.apps import setup @@ -241,6 +243,41 @@ def test_setup_configures_summary_mode_with_agent_endpoint( assert "BCG_SUMMARY_API_KEY=agent-secret\n" in credentials +@pytest.mark.parametrize( + ("selection", "mode"), + [("4", "recent-only"), ("5", "rag")], +) +def test_setup_configures_bounded_context_modes( + monkeypatch, + tmp_path: Path, + selection: str, + mode: str, +) -> None: + monkeypatch.setenv("BCG_HOME", str(tmp_path)) + answers = iter( + [ + "1", # API key authentication + "https://agent.test/v1", + "agent-model", + "2", # disable Serper + selection, + "", # two recent completed turns + "2", # existing Graph server (not used by these modes) + "https://graph.test", + ] + ) + + config = setup.run_setup( + input_fn=lambda _prompt: next(answers), + secret_fn=lambda _prompt: "agent-secret", + ) + + assert config["context"] == {"mode": mode, "recentTurns": 2} + setup.apply_user_configuration(config, override=True) + assert os.environ["BCG_CONTEXT_MODE"] == mode + assert os.environ["BCG_RECENT_TURNS"] == "2" + + def test_apply_user_configuration_uses_global_values( monkeypatch, tmp_path: Path,