diff --git a/README.md b/README.md index d1d9b3fd..5076a33d 100644 --- a/README.md +++ b/README.md @@ -827,6 +827,8 @@ ollama pull nomic-embed-text The built-in `ollama` provider uses Ollama's native `/api/embeddings` endpoint and is the simplest setup when you want to use `nomic-embed-text`. +For the built-in Ollama path, the plugin budgets `nomic-embed-text` against an observed effective input limit of about **2048 tokens**, not the model's higher advertised theoretical context. This keeps batching and chunk text generation aligned with real Ollama embedding runtime behavior. + If you want to use a different Ollama embedding model through its OpenAI-compatible API, use the `custom` provider instead and set `customProvider.baseUrl` to `http://127.0.0.1:11434/v1` so the plugin calls `.../v1/embeddings`. ## 📈 Performance diff --git a/native/src/db.rs b/native/src/db.rs index 085b8929..e45c641c 100644 --- a/native/src/db.rs +++ b/native/src/db.rs @@ -768,7 +768,13 @@ pub fn chunk_exists_on_branch(conn: &Connection, branch: &str, chunk_id: &str) - /// Get all branches pub fn get_all_branches(conn: &Connection) -> DbResult> { - let mut stmt = conn.prepare("SELECT DISTINCT branch FROM branch_chunks")?; + let mut stmt = conn.prepare( + r#" + SELECT branch FROM branch_chunks + UNION + SELECT branch FROM branch_symbols + "#, + )?; let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; let mut results = Vec::new(); @@ -1494,7 +1500,14 @@ pub fn get_stats(conn: &Connection) -> DbResult { let branch_chunk_count: i64 = conn.query_row("SELECT COUNT(*) FROM branch_chunks", [], |row| row.get(0))?; let branch_count: i64 = conn.query_row( - "SELECT COUNT(DISTINCT branch) FROM branch_chunks", + r#" + SELECT COUNT(*) + FROM ( + SELECT branch FROM branch_chunks + UNION + SELECT branch FROM branch_symbols + ) + "#, [], |row| row.get(0), )?; diff --git a/src/config/constants.ts b/src/config/constants.ts index de5545a9..c06f6daa 100644 --- a/src/config/constants.ts +++ b/src/config/constants.ts @@ -77,7 +77,7 @@ export const EMBEDDING_MODELS = { provider: "ollama", model: "nomic-embed-text", dimensions: 768, - maxTokens: 8192, + maxTokens: 2048, costPer1MTokens: 0.00, }, "mxbai-embed-large": { diff --git a/src/embeddings/provider.ts b/src/embeddings/provider.ts index 52592001..bb0480c7 100644 --- a/src/embeddings/provider.ts +++ b/src/embeddings/provider.ts @@ -291,35 +291,69 @@ class OllamaEmbeddingProvider implements EmbeddingProviderInterface { }; } + private estimateTokens(text: string): number { + return Math.ceil(text.length / 4); + } + + private truncateToTokenLimit(text: string, maxTokens: number): string { + const maxChars = Math.max(1, maxTokens * 4); + if (text.length <= maxChars) { + return text; + } + + return `${text.slice(0, Math.max(0, maxChars - 17))}\n... [truncated]`; + } + + private async embedSingle(text: string): Promise<{ embedding: number[]; tokensUsed: number }> { + const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: this.modelInfo.model, + prompt: text, + truncate: false, + }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`Ollama embedding API error: ${response.status} - ${error}`); + } + + const data = (await response.json()) as { + embedding: number[]; + }; + + return { + embedding: data.embedding, + tokensUsed: this.estimateTokens(text), + }; + } + async embedBatch(texts: string[]): Promise { - const results = await Promise.all( - texts.map(async (text) => { - const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model: this.modelInfo.model, - prompt: text, - }), - }); + const results: Array<{ embedding: number[]; tokensUsed: number }> = []; - if (!response.ok) { - const error = await response.text(); - throw new Error(`Ollama embedding API error: ${response.status} - ${error}`); + for (const text of texts) { + try { + results.push(await this.embedSingle(text)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const shouldRetryWithTruncation = message.includes("input length exceeds the context length"); + + if (!shouldRetryWithTruncation) { + throw error; } - const data = (await response.json()) as { - embedding: number[]; - }; + const truncated = this.truncateToTokenLimit(text, this.modelInfo.maxTokens); + if (truncated === text) { + throw error; + } - return { - embedding: data.embedding, - tokensUsed: Math.ceil(text.length / 4), - }; - }) - ); + results.push(await this.embedSingle(truncated)); + } + } return { embeddings: results.map((r) => r.embedding), diff --git a/src/indexer/index.ts b/src/indexer/index.ts index b12c6ed6..7086759e 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -20,7 +20,7 @@ import { InvertedIndex, Database, parseFiles, - createEmbeddingText, + createEmbeddingTexts, generateChunkId, generateChunkHash, ChunkMetadata, @@ -30,6 +30,7 @@ import { hashContent, extractCalls, parseFileAsText, + estimateTokens, } from "../native/index.js"; import type { SymbolData, CallEdgeData } from "../native/index.js"; import { getBranchOrDefault, getBaseBranch, isGitRepo } from "../git/index.js"; @@ -87,6 +88,23 @@ function isRateLimitError(error: unknown): boolean { return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests"); } +function getSafeEmbeddingChunkTokenLimit(provider: ConfiguredProviderInfo): number { + const providerMaxTokens = provider.modelInfo.maxTokens; + const maxChunkTokens = Math.max(256, Math.floor(providerMaxTokens * 0.75)); + return Math.min(2000, maxChunkTokens); +} + +function getDynamicBatchOptions(provider: ConfiguredProviderInfo): { maxBatchTokens?: number; maxBatchItems?: number } { + if (provider.provider === "ollama") { + return { + maxBatchTokens: provider.modelInfo.maxTokens, + maxBatchItems: 1, + }; + } + + return {}; +} + function isSqliteCorruptionError(error: unknown): boolean { const message = getErrorMessage(error).toLowerCase(); return message.includes("database disk image is malformed") @@ -166,12 +184,23 @@ export type ProgressCallback = (progress: IndexProgress) => void; interface PendingChunk { id: string; - text: string; + texts: Array<{ + text: string; + tokenCount: number; + }>; + storageText: string; content: string; contentHash: string; metadata: ChunkMetadata; } +interface PendingEmbeddingRequest { + chunk: PendingChunk; + partIndex: number; + text: string; + tokenCount: number; +} + interface FailedBatch { chunks: PendingChunk[]; error: string; @@ -179,6 +208,18 @@ interface FailedBatch { lastAttempt: string; } +interface RetryableFailedChunk { + chunk: PendingChunk; + attemptCount: number; +} + +interface SerializedFailedBatch { + chunks: unknown[]; + error: string; + attemptCount: number; + lastAttempt: string; +} + type RankedCandidate = { id: string; score: number; metadata: ChunkMetadata }; interface RerankDocumentPayload { @@ -207,6 +248,7 @@ interface IndexMetadata { embeddingProvider: string; embeddingModel: string; embeddingDimensions: number; + embeddingStrategyVersion: string; createdAt: string; updatedAt: string; } @@ -214,6 +256,7 @@ interface IndexMetadata { enum IncompatibilityCode { DIMENSION_MISMATCH = "DIMENSION_MISMATCH", MODEL_MISMATCH = "MODEL_MISMATCH", + EMBEDDING_STRATEGY_MISMATCH = "EMBEDDING_STRATEGY_MISMATCH", } interface IndexCompatibility { @@ -224,8 +267,214 @@ interface IndexCompatibility { } const INDEX_METADATA_VERSION = "1"; +const EMBEDDING_STRATEGY_VERSION = "2"; const RANKING_TOKEN_CACHE_LIMIT = 4096; +function createPendingChunkStorageText(texts: PendingChunk["texts"]): string { + const primaryText = texts[0]?.text ?? ""; + if (texts.length <= 1) { + return primaryText; + } + + return `${primaryText}\n\n... [split into ${texts.length} parts for embedding]`; +} + +function normalizePendingChunk(rawChunk: unknown, maxChunkTokens?: number): PendingChunk | null { + if (!rawChunk || typeof rawChunk !== "object") { + return null; + } + + const chunk = rawChunk as { + id?: unknown; + text?: unknown; + texts?: Array<{ text?: unknown; tokenCount?: unknown }>; + storageText?: unknown; + content?: unknown; + contentHash?: unknown; + metadata?: unknown; + }; + + if (typeof chunk.id !== "string" || typeof chunk.contentHash !== "string" || !chunk.metadata || typeof chunk.metadata !== "object") { + return null; + } + + const texts = Array.isArray(chunk.texts) + ? chunk.texts + .map((entry) => { + if (!entry || typeof entry.text !== "string") { + return null; + } + + return { + text: entry.text, + tokenCount: typeof entry.tokenCount === "number" && Number.isFinite(entry.tokenCount) + ? entry.tokenCount + : estimateTokens(entry.text), + }; + }) + .filter((entry): entry is PendingChunk["texts"][number] => entry !== null) + : []; + + if (texts.length === 0 && typeof chunk.text === "string") { + if (typeof chunk.content === "string" && chunk.content.length > 0 && chunk.metadata && typeof chunk.metadata === "object") { + const metadata = chunk.metadata as Partial; + const rebuiltChunk = { + content: chunk.content, + startLine: typeof metadata.startLine === "number" ? metadata.startLine : 1, + endLine: typeof metadata.endLine === "number" ? metadata.endLine : 1, + chunkType: typeof metadata.chunkType === "string" ? metadata.chunkType : "other", + name: typeof metadata.name === "string" ? metadata.name : undefined, + language: typeof metadata.language === "string" ? metadata.language : "text", + }; + const filePath = typeof metadata.filePath === "string" ? metadata.filePath : "unknown"; + texts.push( + ...createEmbeddingTexts(rebuiltChunk, filePath, maxChunkTokens).map((text) => ({ + text, + tokenCount: estimateTokens(text), + })) + ); + } else { + texts.push({ + text: chunk.text, + tokenCount: estimateTokens(chunk.text), + }); + } + } + + if (texts.length === 0) { + return null; + } + + return { + id: chunk.id, + texts, + storageText: typeof chunk.storageText === "string" ? chunk.storageText : createPendingChunkStorageText(texts), + content: typeof chunk.content === "string" ? chunk.content : "", + contentHash: chunk.contentHash, + metadata: chunk.metadata as ChunkMetadata, + }; +} + +function getPendingChunkFilePath(rawChunk: unknown): string | null { + if (!rawChunk || typeof rawChunk !== "object") { + return null; + } + + const chunk = rawChunk as { metadata?: unknown }; + if (!chunk.metadata || typeof chunk.metadata !== "object") { + return null; + } + + const metadata = chunk.metadata as { filePath?: unknown }; + return typeof metadata.filePath === "string" ? metadata.filePath : null; +} + +function normalizeFailedBatch(batch: SerializedFailedBatch, maxChunkTokens?: number): FailedBatch | null { + const chunks = batch.chunks + .map((chunk) => normalizePendingChunk(chunk, maxChunkTokens)) + .filter((chunk): chunk is PendingChunk => chunk !== null); + + if (chunks.length === 0) { + return null; + } + + return { + chunks, + error: batch.error, + attemptCount: batch.attemptCount, + lastAttempt: batch.lastAttempt, + } satisfies FailedBatch; +} + +function createPendingEmbeddingRequests(chunks: PendingChunk[]): PendingEmbeddingRequest[] { + return chunks.flatMap((chunk) => + chunk.texts.map((textPart, partIndex) => ({ + chunk, + partIndex, + text: textPart.text, + tokenCount: textPart.tokenCount, + })) + ); +} + +function createPendingEmbeddingRequestBatches( + chunks: PendingChunk[], + options: { maxBatchTokens?: number; maxBatchItems?: number } = {} +): PendingEmbeddingRequest[][] { + return createDynamicBatches(createPendingEmbeddingRequests(chunks), options); +} + +function getUniquePendingChunksFromRequests(requests: PendingEmbeddingRequest[]): PendingChunk[] { + const uniqueChunks = new Map(); + for (const request of requests) { + uniqueChunks.set(request.chunk.id, request.chunk); + } + return Array.from(uniqueChunks.values()); +} + +function coalesceFailedBatches(batches: FailedBatch[]): FailedBatch[] { + const grouped = new Map(); + + for (const batch of batches) { + const key = `${batch.attemptCount}:${batch.lastAttempt}:${batch.error}`; + const existing = grouped.get(key); + if (!existing) { + grouped.set(key, { + ...batch, + chunks: [...batch.chunks], + }); + continue; + } + + existing.chunks.push(...batch.chunks); + } + + return Array.from(grouped.values()); +} + +function poolEmbeddingVectors(vectors: number[][], weights: number[]): number[] { + const firstVector = vectors[0]; + if (!firstVector) { + return []; + } + + const pooled = new Array(firstVector.length).fill(0); + let totalWeight = 0; + + for (let index = 0; index < vectors.length; index++) { + const vector = vectors[index]; + const weight = Math.max(1, weights[index] ?? 1); + totalWeight += weight; + + for (let dimension = 0; dimension < vector.length; dimension++) { + pooled[dimension] += vector[dimension] * weight; + } + } + + if (totalWeight === 0) { + return firstVector; + } + + return pooled.map((value) => value / totalWeight); +} + +function hasAllEmbeddingParts( + parts: Array<{ vector: number[]; tokenCount: number } | undefined>, + expectedPartCount: number +): boolean { + if (parts.length !== expectedPartCount) { + return false; + } + + for (let index = 0; index < expectedPartCount; index++) { + if (parts[index] === undefined) { + return false; + } + } + + return true; +} + function isPathWithinRoot(filePath: string, rootPath: string): boolean { const normalizedFilePath = path.resolve(filePath); const normalizedRoot = path.resolve(rootPath); @@ -1535,6 +1784,100 @@ export class Indexer { return `index.globalBranchMigration.${projectHash}`; } + private getProjectEmbeddingStrategyMetadataKey(): string { + const projectHash = hashContent(path.resolve(this.projectRoot)).slice(0, 16); + return `index.embeddingStrategyVersion.${projectHash}`; + } + + private getProjectForceReembedMetadataKey(): string { + const projectHash = hashContent(path.resolve(this.projectRoot)).slice(0, 16); + return `index.forceReembed.${projectHash}`; + } + + private hasProjectForceReembedPending(): boolean { + return this.config.scope === "global" && this.database?.getMetadata(this.getProjectForceReembedMetadataKey()) === "true"; + } + + private hasScopedIndexedData(): boolean { + if (!this.store || this.config.scope !== "global") { + return false; + } + + if (this.hasProjectForceReembedPending()) { + return false; + } + + const roots = this.getScopedRoots(); + + if (Array.from(this.fileHashCache.keys()).some((filePath) => this.isFileInCurrentScope(filePath, roots))) { + return true; + } + + if (this.loadSerializedFailedBatches().some((batch) => + batch.chunks.some((chunk) => { + const filePath = getPendingChunkFilePath(chunk); + return filePath !== null && this.isFileInCurrentScope(filePath, roots); + }) + )) { + return true; + } + + if (!this.database) { + return false; + } + + if (this.getBranchCatalogKeys().some((branchKey) => { + const branchChunkIds = this.database!.getBranchChunkIds(branchKey); + if (branchChunkIds.length > 0) { + return true; + } + + return this.database!.getBranchSymbolIds(branchKey).length > 0; + })) { + return true; + } + + const hasAnyBranchRows = this.database.getAllBranches().some((branchKey) => { + const branchChunkIds = this.database!.getBranchChunkIds(branchKey); + if (branchChunkIds.length > 0) { + return true; + } + + return this.database!.getBranchSymbolIds(branchKey).length > 0; + }); + if (hasAnyBranchRows) { + return false; + } + + return this.store.getAllMetadata().some(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots)); + } + + private loadStoredEmbeddingStrategyVersion(): string | null { + if (!this.database) { + return null; + } + + if (this.hasProjectForceReembedPending()) { + return null; + } + + if (this.config.scope !== "global") { + return this.database.getMetadata("index.embeddingStrategyVersion") ?? "1"; + } + + const projectVersion = this.database.getMetadata(this.getProjectEmbeddingStrategyMetadataKey()); + if (projectVersion) { + return projectVersion; + } + + const legacySharedVersion = this.database.getMetadata("index.embeddingStrategyVersion"); + if (legacySharedVersion && this.hasScopedIndexedData()) { + return legacySharedVersion; + } + + return null; + } + private getBranchCatalogKeys(): string[] { const primary = this.getBranchCatalogKey(); if (this.config.scope !== "global") { @@ -1549,6 +1892,85 @@ export class Indexer { return primary === legacy ? [primary] : [primary, legacy]; } + private getBranchCatalogCleanupKeys(): string[] { + const primary = this.getBranchCatalogKey(); + if (this.config.scope !== "global") { + return [primary]; + } + + const legacy = this.getLegacyBranchCatalogKey(); + return primary === legacy ? [primary] : [primary, legacy]; + } + + private getProjectLocalScopedOwnershipIds(roots: string[]): { + chunkIds: Set; + symbolIds: Set; + } { + const chunkIds = new Set(); + const symbolIds = new Set(); + if (!this.database) { + return { chunkIds, symbolIds }; + } + + const projectRootPath = path.resolve(this.projectRoot); + const projectLocalFilePaths = new Set([ + ...Array.from(this.fileHashCache.keys()).filter( + (filePath) => this.isFileInCurrentScope(filePath, roots) && isPathWithinRoot(filePath, projectRootPath) + ), + ...(this.store?.getAllMetadata() ?? []) + .map(({ metadata }) => metadata.filePath) + .filter( + (filePath) => this.isFileInCurrentScope(filePath, roots) && isPathWithinRoot(filePath, projectRootPath) + ), + ]); + + for (const filePath of projectLocalFilePaths) { + for (const chunk of this.database.getChunksByFile(filePath)) { + chunkIds.add(chunk.chunkId); + } + + for (const symbol of this.database.getSymbolsByFile(filePath)) { + symbolIds.add(symbol.id); + } + } + + return { chunkIds, symbolIds }; + } + + private getProjectScopedBranchCatalogCleanupKeys(projectChunkIds: string[], projectSymbolIds: string[]): string[] { + if (this.config.scope !== "global") { + return this.getBranchCatalogCleanupKeys(); + } + + const projectHash = hashContent(path.resolve(this.projectRoot)).slice(0, 16); + const keys = new Set(); + const projectChunkIdSet = new Set(projectChunkIds); + const projectSymbolIdSet = new Set(projectSymbolIds); + + for (const branchKey of this.database?.getAllBranches() ?? []) { + if (branchKey.startsWith(`${projectHash}:`)) { + keys.add(branchKey); + continue; + } + + if (branchKey.includes(":")) { + continue; + } + + const referencesProjectChunks = this.database?.getBranchChunkIds(branchKey).some((chunkId) => projectChunkIdSet.has(chunkId)) ?? false; + const referencesProjectSymbols = this.database?.getBranchSymbolIds(branchKey).some((symbolId) => projectSymbolIdSet.has(symbolId)) ?? false; + if (referencesProjectChunks || referencesProjectSymbols) { + keys.add(branchKey); + } + } + + for (const branchKey of this.getBranchCatalogCleanupKeys()) { + keys.add(branchKey); + } + + return Array.from(keys); + } + private isFileInCurrentScope(filePath: string, roots: string[]): boolean { return roots.some((root) => isPathWithinRoot(filePath, root)); } @@ -1576,16 +1998,25 @@ export class Indexer { this.saveFileHashCache(); } - private partitionFailedBatches(roots: string[]): { scoped: FailedBatch[]; retained: FailedBatch[] } { + private partitionFailedBatches(roots: string[], maxChunkTokens?: number): { scoped: FailedBatch[]; retained: SerializedFailedBatch[] } { const scoped: FailedBatch[] = []; - const retained: FailedBatch[] = []; + const retained: SerializedFailedBatch[] = []; - for (const batch of this.loadFailedBatches()) { - const scopedChunks = batch.chunks.filter((chunk) => this.isFileInCurrentScope(chunk.metadata.filePath, roots)); - const retainedChunks = batch.chunks.filter((chunk) => !this.isFileInCurrentScope(chunk.metadata.filePath, roots)); + for (const batch of this.loadSerializedFailedBatches()) { + const scopedChunks = batch.chunks.filter((chunk) => { + const filePath = getPendingChunkFilePath(chunk); + return filePath !== null && this.isFileInCurrentScope(filePath, roots); + }); + const retainedChunks = batch.chunks.filter((chunk) => { + const filePath = getPendingChunkFilePath(chunk); + return filePath === null || !this.isFileInCurrentScope(filePath, roots); + }); if (scopedChunks.length > 0) { - scoped.push({ ...batch, chunks: scopedChunks }); + const normalizedBatch = normalizeFailedBatch({ ...batch, chunks: scopedChunks }, maxChunkTokens); + if (normalizedBatch) { + scoped.push(normalizedBatch); + } } if (retainedChunks.length > 0) { @@ -1610,6 +2041,39 @@ export class Indexer { return retained.length > 0; } + private hasForeignScopedBranchData(): boolean { + if (!this.database || this.config.scope !== "global") { + return false; + } + + const projectHash = hashContent(path.resolve(this.projectRoot)).slice(0, 16); + const roots = this.getScopedRoots(); + const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots); + + return this.database.getAllBranches().some( + (branchKey) => { + const branchChunkIds = this.database!.getBranchChunkIds(branchKey); + const branchSymbolIds = this.database!.getBranchSymbolIds(branchKey); + const hasBranchData = branchChunkIds.length > 0 || branchSymbolIds.length > 0; + if (!hasBranchData) { + return false; + } + + if (branchKey.startsWith(`${projectHash}:`)) { + return false; + } + + if (!branchKey.includes(":")) { + const referencesCurrentProjectChunks = branchChunkIds.some((chunkId) => projectLocalChunkIds.has(chunkId)); + const referencesCurrentProjectSymbols = branchSymbolIds.some((symbolId) => projectLocalSymbolIds.has(symbolId)); + return !(referencesCurrentProjectChunks || referencesCurrentProjectSymbols); + } + + return true; + } + ); + } + private saveScopedFailedBatches(batches: FailedBatch[], roots: string[]): void { const { retained } = this.partitionFailedBatches(roots); this.saveFailedBatches([...retained, ...batches]); @@ -1628,6 +2092,11 @@ export class Indexer { ...scopedEntries.map(({ metadata }) => metadata.filePath), ]); + const projectRootPath = path.resolve(this.projectRoot); + const projectLocalFilePaths = new Set( + Array.from(filePaths).filter((filePath) => isPathWithinRoot(filePath, projectRootPath)) + ); + const removedChunkIds = new Set(scopedEntries.map(({ key }) => key)); for (const filePath of filePaths) { for (const chunk of database.getChunksByFile(filePath)) { @@ -1636,7 +2105,29 @@ export class Indexer { } const removedChunkIdList = Array.from(removedChunkIds); - for (const branchKey of this.getBranchCatalogKeys()) { + const projectLocalChunkIds = new Set( + scopedEntries + .filter(({ metadata }) => isPathWithinRoot(metadata.filePath, projectRootPath)) + .map(({ key }) => key) + ); + for (const filePath of projectLocalFilePaths) { + for (const chunk of database.getChunksByFile(filePath)) { + projectLocalChunkIds.add(chunk.chunkId); + } + } + + const symbolIds: string[] = []; + const projectLocalSymbolIds = new Set(); + for (const filePath of filePaths) { + for (const symbol of database.getSymbolsByFile(filePath)) { + symbolIds.push(symbol.id); + if (projectLocalFilePaths.has(filePath)) { + projectLocalSymbolIds.add(symbol.id); + } + } + } + + for (const branchKey of this.getProjectScopedBranchCatalogCleanupKeys(Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds))) { database.deleteBranchChunksForBranch(branchKey, removedChunkIdList); } const sharedChunkIds = new Set(database.getReferencedChunkIds(removedChunkIdList)); @@ -1647,14 +2138,7 @@ export class Indexer { invertedIndex.removeChunk(chunkId); } - const symbolIds: string[] = []; - for (const filePath of filePaths) { - for (const symbol of database.getSymbolsByFile(filePath)) { - symbolIds.push(symbol.id); - } - } - - for (const branchKey of this.getBranchCatalogKeys()) { + for (const branchKey of this.getProjectScopedBranchCatalogCleanupKeys(Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds))) { database.deleteBranchSymbolsForBranch(branchKey, symbolIds); } const sharedSymbolIds = new Set(database.getReferencedSymbolIds(symbolIds)); @@ -1721,19 +2205,47 @@ export class Indexer { this.logger.info("Recovery complete, next index will re-process all files"); } - private loadFailedBatches(): FailedBatch[] { + private loadFailedBatches(maxChunkTokens?: number): FailedBatch[] { try { - if (existsSync(this.failedBatchesPath)) { - const data = readFileSync(this.failedBatchesPath, "utf-8"); - return JSON.parse(data) as FailedBatch[]; - } + return this.loadSerializedFailedBatches() + .map((batch) => normalizeFailedBatch(batch, maxChunkTokens)) + .filter((batch): batch is FailedBatch => batch !== null); } catch { return []; } - return []; } - private saveFailedBatches(batches: FailedBatch[]): void { + private loadSerializedFailedBatches(): SerializedFailedBatch[] { + if (!existsSync(this.failedBatchesPath)) { + return []; + } + + const data = readFileSync(this.failedBatchesPath, "utf-8"); + const parsed = JSON.parse(data) as Array<{ + chunks?: unknown[]; + error?: unknown; + attemptCount?: unknown; + lastAttempt?: unknown; + }>; + + return parsed + .map((batch) => { + const chunks = Array.isArray(batch.chunks) ? batch.chunks : []; + if (chunks.length === 0) { + return null; + } + + return { + chunks, + error: typeof batch.error === "string" ? batch.error : "Unknown embedding error", + attemptCount: typeof batch.attemptCount === "number" ? batch.attemptCount : 1, + lastAttempt: typeof batch.lastAttempt === "string" ? batch.lastAttempt : new Date().toISOString(), + } satisfies SerializedFailedBatch; + }) + .filter((batch): batch is SerializedFailedBatch => batch !== null); + } + + private saveFailedBatches(batches: SerializedFailedBatch[]): void { if (batches.length === 0) { if (existsSync(this.failedBatchesPath)) { try { @@ -1749,11 +2261,12 @@ export class Indexer { private collectRetryableFailedChunks( currentFileHashes: Map, - unchangedFilePaths: Set - ): PendingChunk[] { - const retryableById = new Map(); + unchangedFilePaths: Set, + maxChunkTokens?: number + ): RetryableFailedChunk[] { + const retryableById = new Map(); - for (const batch of this.loadFailedBatches()) { + for (const batch of this.loadFailedBatches(maxChunkTokens)) { for (const chunk of batch.chunks) { const filePath = chunk.metadata.filePath; if (!currentFileHashes.has(filePath)) { @@ -1762,7 +2275,14 @@ export class Indexer { if (!unchangedFilePaths.has(filePath)) { continue; } - retryableById.set(chunk.id, chunk); + + const existing = retryableById.get(chunk.id); + if (!existing || batch.attemptCount > existing.attemptCount) { + retryableById.set(chunk.id, { + chunk, + attemptCount: batch.attemptCount, + }); + } } } @@ -2068,6 +2588,19 @@ export class Indexer { dbIsNew = true; } + if (isGitRepo(this.projectRoot)) { + this.currentBranch = getBranchOrDefault(this.projectRoot); + this.baseBranch = getBaseBranch(this.projectRoot); + this.logger.branch("info", "Detected git repository", { + currentBranch: this.currentBranch, + baseBranch: this.baseBranch, + }); + } else { + this.currentBranch = "default"; + this.baseBranch = "default"; + this.logger.branch("debug", "Not a git repository, using default branch"); + } + // Recover from interrupted indexing AFTER store, invertedIndex, and database // are all initialized. healthCheck() calls ensureInitialized() which checks // these fields — if they're not set, it re-enters initialize() causing infinite @@ -2080,6 +2613,8 @@ export class Indexer { this.migrateFromLegacyIndex(); } + this.loadFileHashCache(); + this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo); if (!this.indexCompatibility.compatible) { this.logger.warn("Index compatibility issue detected", { @@ -2089,19 +2624,6 @@ export class Indexer { }); } - if (isGitRepo(this.projectRoot)) { - this.currentBranch = getBranchOrDefault(this.projectRoot); - this.baseBranch = getBaseBranch(this.projectRoot); - this.logger.branch("info", "Detected git repository", { - currentBranch: this.currentBranch, - baseBranch: this.baseBranch, - }); - } else { - this.currentBranch = "default"; - this.baseBranch = "default"; - this.logger.branch("debug", "Not a git repository, using default branch"); - } - // Auto-GC: Run garbage collection if enabled and interval has elapsed if (this.config.indexing.autoGc) { await this.maybeRunAutoGc(); @@ -2259,14 +2781,15 @@ export class Indexer { const version = this.database.getMetadata("index.version"); if (!version) return null; - return { - indexVersion: version, - embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "", - embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "", - embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10), - createdAt: this.database.getMetadata("index.createdAt") ?? "", - updatedAt: this.database.getMetadata("index.updatedAt") ?? "", - }; + return { + indexVersion: version, + embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "", + embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "", + embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10), + embeddingStrategyVersion: this.loadStoredEmbeddingStrategyVersion() ?? EMBEDDING_STRATEGY_VERSION, + createdAt: this.database.getMetadata("index.createdAt") ?? "", + updatedAt: this.database.getMetadata("index.updatedAt") ?? "", + }; } private saveIndexMetadata(provider: ConfiguredProviderInfo): void { @@ -2274,13 +2797,22 @@ export class Indexer { const now = new Date().toISOString(); const existingCreatedAt = this.database.getMetadata("index.createdAt"); + const completeProjectEmbeddingStrategyReset = !this.hasProjectForceReembedPending(); this.database.setMetadata("index.version", INDEX_METADATA_VERSION); this.database.setMetadata("index.embeddingProvider", provider.provider); this.database.setMetadata("index.embeddingModel", provider.modelInfo.model); this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString()); if (this.config.scope === "global") { + if (completeProjectEmbeddingStrategyReset) { + this.database.setMetadata(this.getProjectEmbeddingStrategyMetadataKey(), EMBEDDING_STRATEGY_VERSION); + } this.database.setMetadata(this.getLegacyMigrationMetadataKey(), "done"); + if (completeProjectEmbeddingStrategyReset) { + this.database.deleteMetadata(this.getProjectForceReembedMetadataKey()); + } + } else { + this.database.setMetadata("index.embeddingStrategyVersion", EMBEDDING_STRATEGY_VERSION); } this.database.setMetadata("index.updatedAt", now); @@ -2318,6 +2850,15 @@ export class Indexer { }; } + if (storedMetadata.embeddingStrategyVersion !== EMBEDDING_STRATEGY_VERSION) { + return { + compatible: false, + code: IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH, + reason: `Embedding strategy mismatch: index was built with embedding strategy v${storedMetadata.embeddingStrategyVersion}, but the current code requires v${EMBEDDING_STRATEGY_VERSION}. Run index_codebase with force=true to rebuild cached embeddings.`, + storedMetadata, + }; + } + if (storedMetadata.embeddingProvider !== currentProvider) { this.logger.warn("Provider changed", { storedProvider: storedMetadata.embeddingProvider, @@ -2381,6 +2922,8 @@ export class Indexer { const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized(); const scopedRoots = this.config.scope === "global" ? this.getScopedRoots() : null; const branchCatalogKey = this.getBranchCatalogKey(); + const forceScopedReembed = scopedRoots !== null && database.getMetadata(this.getProjectForceReembedMetadataKey()) === "true"; + const failedForcedChunkIds = new Set(); if (!this.indexCompatibility?.compatible) { throw new Error( @@ -2482,6 +3025,9 @@ export class Indexer { if (scopedRoots && !this.isFileInCurrentScope(metadata.filePath, scopedRoots)) { continue; } + if (forceScopedReembed && scopedRoots && this.isFileInCurrentScope(metadata.filePath, scopedRoots)) { + continue; + } existingChunks.set(key, metadata.hash); const fileChunks = existingChunksByFile.get(metadata.filePath) || new Set(); fileChunks.add(key); @@ -2552,7 +3098,10 @@ export class Indexer { continue; } - const text = createEmbeddingText(chunk, parsed.path); + const texts = createEmbeddingTexts(chunk, parsed.path, getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)).map((text) => ({ + text, + tokenCount: estimateTokens(text), + })); const metadata: ChunkMetadata = { filePath: parsed.path, startLine: chunk.startLine, @@ -2563,15 +3112,32 @@ export class Indexer { hash: contentHash, }; - pendingChunks.push({ id, text, content: chunk.content, contentHash, metadata }); + pendingChunks.push({ + id, + texts, + storageText: createPendingChunkStorageText(texts), + content: chunk.content, + contentHash, + metadata, + }); fileChunkCount++; } } - const retryableFailedChunks = this.collectRetryableFailedChunks(currentFileHashes, unchangedFilePaths); + const retryableFailedChunks = this.collectRetryableFailedChunks( + currentFileHashes, + unchangedFilePaths, + getSafeEmbeddingChunkTokenLimit(configuredProviderInfo) + ); + const retryableFailedAttemptCounts = new Map(); + const retryableChunksWithExistingData = new Set(); if (retryableFailedChunks.length > 0) { const pendingChunkIds = new Set(pendingChunks.map((chunk) => chunk.id)); - for (const chunk of retryableFailedChunks) { + for (const { chunk, attemptCount } of retryableFailedChunks) { + retryableFailedAttemptCounts.set(chunk.id, attemptCount); + if (existingChunks.has(chunk.id)) { + retryableChunksWithExistingData.add(chunk.id); + } if (!pendingChunkIds.has(chunk.id)) { pendingChunks.push(chunk); pendingChunkIds.add(chunk.id); @@ -2770,9 +3336,12 @@ export class Indexer { const allContentHashes = pendingChunks.map((c) => c.contentHash); const missingHashes = new Set(database.getMissingEmbeddings(allContentHashes)); + const forcedReembedChunkIds = forceScopedReembed + ? new Set(pendingChunks.map((chunk) => chunk.id)) + : new Set(); - const chunksNeedingEmbedding = pendingChunks.filter((c) => missingHashes.has(c.contentHash)); - const chunksWithExistingEmbedding = pendingChunks.filter((c) => !missingHashes.has(c.contentHash)); + const chunksNeedingEmbedding = pendingChunks.filter((c) => forcedReembedChunkIds.has(c.id) || missingHashes.has(c.contentHash)); + const chunksWithExistingEmbedding = pendingChunks.filter((c) => !forcedReembedChunkIds.has(c.id) && !missingHashes.has(c.contentHash)); this.logger.cache("info", "Embedding cache lookup", { needsEmbedding: chunksNeedingEmbedding.length, @@ -2797,10 +3366,17 @@ export class Indexer { interval: providerRateLimits.intervalMs, intervalCap: providerRateLimits.concurrency }); - const dynamicBatches = createDynamicBatches(chunksNeedingEmbedding); + const pendingChunksById = new Map(chunksNeedingEmbedding.map((chunk) => [chunk.id, chunk])); + const embeddingPartsByChunk = new Map>(); + const completedChunkIds = new Set(); + const failedChunkIds = new Set(); + const requestBatches = createPendingEmbeddingRequestBatches( + chunksNeedingEmbedding, + getDynamicBatchOptions(configuredProviderInfo) + ); let rateLimitBackoffMs = 0; - for (const batch of dynamicBatches) { + for (const requestBatch of requestBatches) { queue.add(async () => { if (rateLimitBackoffMs > 0) { await new Promise(resolve => setTimeout(resolve, rateLimitBackoffMs)); @@ -2809,7 +3385,7 @@ export class Indexer { try { const result = await pRetry( async () => { - const texts = batch.map((c) => c.text); + const texts = requestBatch.map((request) => request.text); return provider.embedBatch(texts); }, { @@ -2841,34 +3417,96 @@ export class Indexer { rateLimitBackoffMs = Math.max(0, rateLimitBackoffMs - 2000); } - const items = batch.map((chunk, idx) => ({ - id: chunk.id, - vector: result.embeddings[idx], - metadata: chunk.metadata, - })); + const touchedChunkIds = new Set(); - store.addBatch(items); + requestBatch.forEach((request, idx) => { + if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) { + return; + } + + const vector = result.embeddings[idx]; + if (!vector) { + throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`); + } + + const parts = embeddingPartsByChunk.get(request.chunk.id) ?? []; + parts[request.partIndex] = { + vector, + tokenCount: request.tokenCount, + }; + embeddingPartsByChunk.set(request.chunk.id, parts); + touchedChunkIds.add(request.chunk.id); + }); + + const pooledResults: Array<{ chunk: PendingChunk; vector: number[] }> = []; + for (const chunkId of touchedChunkIds) { + if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) { + continue; + } + + const chunk = pendingChunksById.get(chunkId); + if (!chunk) { + continue; + } - const embeddingBatchItems = batch.map((chunk, i) => ({ - contentHash: chunk.contentHash, - embedding: float32ArrayToBuffer(result.embeddings[i]), - chunkText: chunk.text, - model: configuredProviderInfo.modelInfo.model, - })); - database.upsertEmbeddingsBatch(embeddingBatchItems); - - for (const chunk of batch) { - invertedIndex.removeChunk(chunk.id); - invertedIndex.addChunk(chunk.id, chunk.content); + const parts = embeddingPartsByChunk.get(chunk.id) ?? []; + if (!hasAllEmbeddingParts(parts, chunk.texts.length)) { + continue; + } + + const orderedParts = parts as Array<{ vector: number[]; tokenCount: number }>; + pooledResults.push({ + chunk, + vector: poolEmbeddingVectors( + orderedParts.map((part) => part.vector), + orderedParts.map((part) => part.tokenCount) + ), + }); + } + + if (pooledResults.length > 0) { + const items = pooledResults.map(({ chunk, vector }) => ({ + id: chunk.id, + vector, + metadata: chunk.metadata, + })); + + store.addBatch(items); + + const embeddingBatchItems = pooledResults.map(({ chunk, vector }) => ({ + contentHash: chunk.contentHash, + embedding: float32ArrayToBuffer(vector), + chunkText: chunk.storageText, + model: configuredProviderInfo.modelInfo.model, + })); + + try { + database.upsertEmbeddingsBatch(embeddingBatchItems); + } catch (dbError) { + // Rollback vectors added to store if DB write fails + for (const { chunk } of pooledResults) { + store.remove(chunk.id); + } + throw dbError; + } + + for (const { chunk } of pooledResults) { + invertedIndex.removeChunk(chunk.id); + invertedIndex.addChunk(chunk.id, chunk.content); + completedChunkIds.add(chunk.id); + embeddingPartsByChunk.delete(chunk.id); + } + + stats.indexedChunks += pooledResults.length; + this.logger.recordChunksEmbedded(pooledResults.length); } - stats.indexedChunks += batch.length; stats.tokensUsed += result.totalTokensUsed; - this.logger.recordChunksEmbedded(batch.length); this.logger.recordEmbeddingApiCall(result.totalTokensUsed); this.logger.embedding("debug", `Embedded batch`, { - batchSize: batch.length, + batchSize: pooledResults.length, + requestCount: requestBatch.length, tokens: result.totalTokensUsed, }); @@ -2880,17 +3518,48 @@ export class Indexer { totalChunks: pendingChunks.length, }); } catch (error) { - stats.failedChunks += batch.length; - failedBatchesForCurrentRun.push({ - chunks: batch, - error: getErrorMessage(error), - attemptCount: 1, - lastAttempt: new Date().toISOString(), - }); + const failedChunks = getUniquePendingChunksFromRequests(requestBatch) + .filter((chunk) => !completedChunkIds.has(chunk.id)); + const failureMessage = getErrorMessage(error); + const failureTimestamp = new Date().toISOString(); + + for (const chunk of failedChunks) { + if (!failedChunkIds.has(chunk.id)) { + failedChunkIds.add(chunk.id); + stats.failedChunks += 1; + } + + if (forceScopedReembed) { + failedForcedChunkIds.add(chunk.id); + } + + embeddingPartsByChunk.delete(chunk.id); + + const existingFailedBatchIndex = failedBatchesForCurrentRun.findIndex( + (failedBatch) => failedBatch.chunks[0]?.id === chunk.id + ); + const existingFailedBatch = existingFailedBatchIndex === -1 + ? undefined + : failedBatchesForCurrentRun[existingFailedBatchIndex]; + const failedBatch = { + chunks: [chunk], + error: failureMessage, + attemptCount: (existingFailedBatch?.attemptCount ?? retryableFailedAttemptCounts.get(chunk.id) ?? 0) + 1, + lastAttempt: failureTimestamp, + } satisfies FailedBatch; + + if (existingFailedBatchIndex === -1) { + failedBatchesForCurrentRun.push(failedBatch); + } else { + failedBatchesForCurrentRun[existingFailedBatchIndex] = failedBatch; + } + } + this.logger.recordEmbeddingError(); this.logger.embedding("error", `Failed to embed batch after retries`, { - batchSize: batch.length, - error: getErrorMessage(error), + batchSize: failedChunks.length, + requestCount: requestBatch.length, + error: failureMessage, }); } }); @@ -2898,9 +3567,9 @@ export class Indexer { await queue.onIdle(); if (scopedRoots) { - this.saveScopedFailedBatches(failedBatchesForCurrentRun, scopedRoots); + this.saveScopedFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun), scopedRoots); } else { - this.saveFailedBatches(failedBatchesForCurrentRun); + this.saveFailedBatches(coalesceFailedBatches(failedBatchesForCurrentRun)); } onProgress?.({ @@ -2911,8 +3580,15 @@ export class Indexer { totalChunks: pendingChunks.length, }); + const branchChunkIds = Array.from(currentChunkIds).filter( + (chunkId) => { + const isNewlyFailed = failedChunkIds.has(chunkId) && !retryableChunksWithExistingData.has(chunkId); + const isForcedFailed = forceScopedReembed && failedForcedChunkIds.has(chunkId); + return !isNewlyFailed && !isForcedFailed; + } + ); database.clearBranch(branchCatalogKey); - database.addChunksToBranchBatch(branchCatalogKey, Array.from(currentChunkIds)); + database.addChunksToBranchBatch(branchCatalogKey, branchChunkIds); database.clearBranchSymbols(branchCatalogKey); database.addSymbolsToBranchBatch(branchCatalogKey, Array.from(allSymbolIds)); @@ -2950,6 +3626,9 @@ export class Indexer { stats.durationMs = Date.now() - startTime; + if (forceScopedReembed && failedForcedChunkIds.size === 0) { + database.deleteMetadata(this.getProjectForceReembedMetadataKey()); + } this.saveIndexMetadata(configuredProviderInfo); this.indexCompatibility = { compatible: true }; @@ -3377,10 +4056,21 @@ export class Indexer { const allMetadata = store.getAllMetadata(); const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || + this.hasForeignScopedBranchData() || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots); if (!compatibility.compatible && hasForeignData) { + if (compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH) { + this.clearSharedIndexProjectData(store, invertedIndex, database, roots); + this.clearScopedFileHashCache(roots); + this.clearScopedFailedBatches(roots); + database.setMetadata(this.getProjectForceReembedMetadataKey(), "true"); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); + this.indexCompatibility = { compatible: true }; + return; + } + throw new Error( `Global index compatibility reset is unsafe because the shared index contains files from other projects. ` + `The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. ` + @@ -3404,6 +4094,9 @@ export class Indexer { database.deleteMetadata("index.embeddingProvider"); database.deleteMetadata("index.embeddingModel"); database.deleteMetadata("index.embeddingDimensions"); + database.deleteMetadata("index.embeddingStrategyVersion"); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); + database.deleteMetadata(this.getProjectForceReembedMetadataKey()); database.deleteMetadata(this.getLegacyMigrationMetadataKey()); database.deleteMetadata("index.createdAt"); database.deleteMetadata("index.updatedAt"); @@ -3446,6 +4139,9 @@ export class Indexer { database.deleteMetadata("index.embeddingProvider"); database.deleteMetadata("index.embeddingModel"); database.deleteMetadata("index.embeddingDimensions"); + database.deleteMetadata("index.embeddingStrategyVersion"); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); + database.deleteMetadata(this.getProjectForceReembedMetadataKey()); database.deleteMetadata(this.getLegacyMigrationMetadataKey()); database.deleteMetadata("index.createdAt"); database.deleteMetadata("index.updatedAt"); @@ -3531,12 +4227,14 @@ export class Indexer { } async retryFailedBatches(): Promise<{ succeeded: number; failed: number; remaining: number }> { - const { store, provider, invertedIndex } = await this.ensureInitialized(); + const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized(); + const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo); + const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider); const roots = this.config.scope === "global" ? this.getScopedRoots() : null; const { scoped: scopedFailedBatches, retained: retainedFailedBatches } = roots - ? this.partitionFailedBatches(roots) - : { scoped: this.loadFailedBatches(), retained: [] as FailedBatch[] }; + ? this.partitionFailedBatches(roots, maxChunkTokens) + : { scoped: this.loadFailedBatches(maxChunkTokens), retained: [] as FailedBatch[] }; const failedBatches = scopedFailedBatches; if (failedBatches.length === 0) { return { succeeded: 0, failed: 0, remaining: 0 }; @@ -3547,51 +4245,177 @@ export class Indexer { const stillFailing: FailedBatch[] = []; for (const batch of failedBatches) { + const batchChunksById = new Map(batch.chunks.map((chunk) => [chunk.id, chunk])); + const embeddingPartsByChunk = new Map>(); + const completedChunkIds = new Set(); + const failedChunkIds = new Set(); + const failedChunksForBatch = new Map(); + const pooledResults: Array<{ chunk: PendingChunk; vector: number[] }> = []; try { - const result = await pRetry( - async () => { - const texts = batch.chunks.map((c) => c.text); - return provider.embedBatch(texts); - }, - { - retries: this.config.indexing.retries, - minTimeout: this.config.indexing.retryDelayMs, - } + const requestBatches = createPendingEmbeddingRequestBatches( + batch.chunks, + getDynamicBatchOptions(configuredProviderInfo) ); - const items = batch.chunks.map((chunk, idx) => ({ + for (const requestBatch of requestBatches) { + try { + const result = await pRetry( + async () => { + const texts = requestBatch.map((request) => request.text); + return provider.embedBatch(texts); + }, + { + retries: this.config.indexing.retries, + minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs), + maxTimeout: providerRateLimits.maxRetryMs, + factor: 2, + shouldRetry: (error) => !((error as { error?: Error }).error instanceof CustomProviderNonRetryableError), + } + ); + + const touchedChunkIds = new Set(); + requestBatch.forEach((request, idx) => { + if (failedChunkIds.has(request.chunk.id) || completedChunkIds.has(request.chunk.id)) { + return; + } + + const vector = result.embeddings[idx]; + if (!vector) { + throw new Error(`Embedding API returned too few vectors for chunk ${request.chunk.id}`); + } + + const parts = embeddingPartsByChunk.get(request.chunk.id) ?? []; + parts[request.partIndex] = { + vector, + tokenCount: request.tokenCount, + }; + embeddingPartsByChunk.set(request.chunk.id, parts); + touchedChunkIds.add(request.chunk.id); + }); + + for (const chunkId of touchedChunkIds) { + if (failedChunkIds.has(chunkId) || completedChunkIds.has(chunkId)) { + continue; + } + + const chunk = batchChunksById.get(chunkId); + if (!chunk) { + continue; + } + + const parts = embeddingPartsByChunk.get(chunk.id) ?? []; + if (!hasAllEmbeddingParts(parts, chunk.texts.length)) { + continue; + } + + const orderedParts = parts as Array<{ vector: number[]; tokenCount: number }>; + pooledResults.push({ + chunk, + vector: poolEmbeddingVectors( + orderedParts.map((part) => part.vector), + orderedParts.map((part) => part.tokenCount) + ), + }); + } + + this.logger.recordEmbeddingApiCall(result.totalTokensUsed); + } catch (error) { + const failureMessage = String(error); + const failureTimestamp = new Date().toISOString(); + const failedChunks = getUniquePendingChunksFromRequests(requestBatch) + .filter((chunk) => !completedChunkIds.has(chunk.id) && !failedChunkIds.has(chunk.id)); + + for (const chunk of failedChunks) { + failedChunkIds.add(chunk.id); + embeddingPartsByChunk.delete(chunk.id); + failedChunksForBatch.set(chunk.id, { + chunks: [chunk], + attemptCount: batch.attemptCount + 1, + lastAttempt: failureTimestamp, + error: failureMessage, + }); + } + + failed += failedChunks.length; + this.logger.recordEmbeddingError(); + } + } + + const successfulResults = pooledResults.filter(({ chunk }) => !failedChunkIds.has(chunk.id)); + + const items = successfulResults.map(({ chunk, vector }) => ({ id: chunk.id, - vector: result.embeddings[idx], + vector, metadata: chunk.metadata, })); - store.addBatch(items); + if (items.length > 0) { + store.addBatch(items); + } - for (const chunk of batch.chunks) { + if (successfulResults.length > 0) { + try { + database.upsertEmbeddingsBatch( + successfulResults.map(({ chunk, vector }) => ({ + contentHash: chunk.contentHash, + embedding: float32ArrayToBuffer(vector), + chunkText: chunk.storageText, + model: configuredProviderInfo.modelInfo.model, + })) + ); + } catch (dbError) { + // Rollback vectors added to store if DB write fails + for (const { chunk } of successfulResults) { + store.remove(chunk.id); + } + throw dbError; + } + } + + for (const { chunk } of successfulResults) { invertedIndex.removeChunk(chunk.id); invertedIndex.addChunk(chunk.id, chunk.content); + completedChunkIds.add(chunk.id); + embeddingPartsByChunk.delete(chunk.id); } - this.logger.recordChunksEmbedded(batch.chunks.length); - this.logger.recordEmbeddingApiCall(result.totalTokensUsed); + database.addChunksToBranchBatch( + this.getBranchCatalogKey(), + successfulResults.map(({ chunk }) => chunk.id) + ); + + this.logger.recordChunksEmbedded(successfulResults.length); - succeeded += batch.chunks.length; + succeeded += successfulResults.length; + stillFailing.push(...failedChunksForBatch.values()); } catch (error) { - failed += batch.chunks.length; + const failureMessage = getErrorMessage(error); + const failureTimestamp = new Date().toISOString(); + const unaccountedChunks = batch.chunks.filter( + (chunk) => !failedChunksForBatch.has(chunk.id) && !completedChunkIds.has(chunk.id) + ); + + for (const chunk of unaccountedChunks) { + failedChunksForBatch.set(chunk.id, { + chunks: [chunk], + attemptCount: batch.attemptCount + 1, + lastAttempt: failureTimestamp, + error: failureMessage, + }); + } + + failed += unaccountedChunks.length; this.logger.recordEmbeddingError(); - stillFailing.push({ - ...batch, - attemptCount: batch.attemptCount + 1, - lastAttempt: new Date().toISOString(), - error: String(error), - }); + stillFailing.push(...coalesceFailedBatches(Array.from(failedChunksForBatch.values()))); } } + const persistedStillFailing = coalesceFailedBatches(stillFailing); + if (roots) { - this.saveFailedBatches([...retainedFailedBatches, ...stillFailing]); + this.saveFailedBatches([...retainedFailedBatches, ...persistedStillFailing]); } else { - this.saveFailedBatches(stillFailing); + this.saveFailedBatches(persistedStillFailing); } if (succeeded > 0) { @@ -3599,7 +4423,13 @@ export class Indexer { invertedIndex.save(); } - return { succeeded, failed, remaining: stillFailing.length }; + if (roots && succeeded > 0 && persistedStillFailing.length === 0 && this.hasProjectForceReembedPending()) { + database.deleteMetadata(this.getProjectForceReembedMetadataKey()); + this.saveIndexMetadata(configuredProviderInfo); + this.indexCompatibility = { compatible: true }; + } + + return { succeeded, failed, remaining: persistedStillFailing.length }; } getFailedBatchesCount(): number { diff --git a/src/native/index.ts b/src/native/index.ts index 71daa41a..494b2e39 100644 --- a/src/native/index.ts +++ b/src/native/index.ts @@ -321,27 +321,27 @@ export class VectorStore { // Token estimation: ~4 chars per token for code (conservative) const CHARS_PER_TOKEN = 4; const MAX_BATCH_TOKENS = 7500; // Leave buffer under 8192 API limit -const MAX_SINGLE_CHUNK_TOKENS = 2000; // Truncate individual chunks beyond this +const MAX_SINGLE_CHUNK_TOKENS = 2000; // Default truncation cap for individual chunks export function estimateTokens(text: string): number { return Math.ceil(text.length / CHARS_PER_TOKEN); } -export function createEmbeddingText(chunk: CodeChunk, filePath: string): string { +function getEmbeddingHeaderParts(chunk: CodeChunk, filePath: string): string[] { const parts: string[] = []; - + const fileName = filePath.split("/").pop() || filePath; const dirPath = filePath.split("/").slice(-3, -1).join("/"); - + const langDescriptors: Record = { typescript: "TypeScript", - javascript: "JavaScript", + javascript: "JavaScript", python: "Python", rust: "Rust", go: "Go", java: "Java", }; - + const typeDescriptors: Record = { function_declaration: "function", function: "function", @@ -364,48 +364,103 @@ export function createEmbeddingText(chunk: CodeChunk, filePath: string): string const lang = langDescriptors[chunk.language] || chunk.language; const typeDesc = typeDescriptors[chunk.chunkType] || chunk.chunkType; - + if (chunk.name) { parts.push(`${lang} ${typeDesc} "${chunk.name}"`); } else { parts.push(`${lang} ${typeDesc}`); } - + if (dirPath) { parts.push(`in ${dirPath}/${fileName}`); } else { parts.push(`in ${fileName}`); } - + const semanticHints = extractSemanticHints(chunk.name || "", chunk.content); if (semanticHints.length > 0) { parts.push(`Purpose: ${semanticHints.join(", ")}`); } - - parts.push(""); - - let content = chunk.content; - const headerLength = parts.join("\n").length; - const maxContentChars = (MAX_SINGLE_CHUNK_TOKENS * CHARS_PER_TOKEN) - headerLength; - - if (content.length > maxContentChars) { - content = content.slice(0, maxContentChars) + "\n... [truncated]"; + + return parts; +} + +function buildEmbeddingText(headerParts: string[], content: string, partIndex?: number, partCount?: number): string { + const parts = [...headerParts]; + if (partCount && partCount > 1 && partIndex) { + parts.push(`Part ${partIndex}/${partCount}`); } - + parts.push(""); parts.push(content); - return parts.join("\n"); } -export function createDynamicBatches(chunks: T[]): T[][] { +function splitOversizedContent(content: string, maxContentChars: number): string[] { + if (content.length <= maxContentChars) { + return [content]; + } + + const overlapChars = Math.max(CHARS_PER_TOKEN * 32, Math.min(Math.floor(maxContentChars * 0.15), CHARS_PER_TOKEN * 128)); + const stepChars = Math.max(1, maxContentChars - overlapChars); + const segments: string[] = []; + + for (let start = 0; start < content.length; start += stepChars) { + const end = Math.min(content.length, start + maxContentChars); + segments.push(content.slice(start, end)); + if (end >= content.length) { + break; + } + } + + return segments; +} + +export function createEmbeddingTexts(chunk: CodeChunk, filePath: string, maxChunkTokens = MAX_SINGLE_CHUNK_TOKENS): string[] { + const headerParts = getEmbeddingHeaderParts(chunk, filePath); + const headerLength = buildEmbeddingText(headerParts, "", 1, 9).length; + const maxContentChars = Math.max(1, (maxChunkTokens * CHARS_PER_TOKEN) - headerLength); + const segments = splitOversizedContent(chunk.content, maxContentChars); + + if (segments.length === 1) { + return [buildEmbeddingText(headerParts, segments[0])]; + } + + return segments.map((segment, index) => buildEmbeddingText(headerParts, segment, index + 1, segments.length)); +} + +export function createEmbeddingText(chunk: CodeChunk, filePath: string, maxChunkTokens = MAX_SINGLE_CHUNK_TOKENS): string { + const text = createEmbeddingTexts(chunk, filePath, maxChunkTokens)[0]; + if (!text) { + return ""; + } + + const maxChars = maxChunkTokens * CHARS_PER_TOKEN; + if (text.length <= maxChars) { + return text; + } + + return text.slice(0, Math.max(0, maxChars - 17)) + "\n... [truncated]"; +} + +export interface DynamicBatchOptions { + maxBatchTokens?: number; + maxBatchItems?: number; +} + +export function createDynamicBatches(chunks: T[], options: DynamicBatchOptions = {}): T[][] { const batches: T[][] = []; let currentBatch: T[] = []; let currentTokens = 0; + const maxBatchTokens = Math.max(1, options.maxBatchTokens ?? MAX_BATCH_TOKENS); + const maxBatchItems = Math.max(1, options.maxBatchItems ?? Number.MAX_SAFE_INTEGER); for (const chunk of chunks) { - const chunkTokens = estimateTokens(chunk.text); - - if (currentBatch.length > 0 && currentTokens + chunkTokens > MAX_BATCH_TOKENS) { + const chunkTokens = chunk.tokenCount ?? estimateTokens(chunk.text); + + if ( + currentBatch.length > 0 + && (currentTokens + chunkTokens > maxBatchTokens || currentBatch.length >= maxBatchItems) + ) { batches.push(currentBatch); currentBatch = []; currentTokens = 0; diff --git a/tests/config.test.ts b/tests/config.test.ts index 7614bec3..06a6eca9 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -902,6 +902,11 @@ describe("config schema", () => { expect(EMBEDDING_MODELS["ollama"]["mxbai-embed-large"].costPer1MTokens).toBe(0); }); + it("should use the observed effective token budget for built-in ollama models", () => { + expect(EMBEDDING_MODELS["ollama"]["nomic-embed-text"].maxTokens).toBe(2048); + expect(EMBEDDING_MODELS["ollama"]["mxbai-embed-large"].maxTokens).toBe(512); + }); + it("should have non-zero cost for paid providers", () => { expect(EMBEDDING_MODELS["openai"]["text-embedding-3-small"].costPer1MTokens).toBeGreaterThan(0); expect(EMBEDDING_MODELS["openai"]["text-embedding-3-large"].costPer1MTokens).toBeGreaterThan(0); diff --git a/tests/custom-provider.test.ts b/tests/custom-provider.test.ts index 50a67468..bd04566e 100644 --- a/tests/custom-provider.test.ts +++ b/tests/custom-provider.test.ts @@ -3,6 +3,7 @@ import { createEmbeddingProvider, CustomProviderNonRetryableError } from "../src import { createCustomProviderInfo, type ConfiguredProviderInfo } from "../src/embeddings/detector.js"; import { Indexer } from "../src/indexer/index.js"; import { parseConfig } from "../src/config/schema.js"; +import { EMBEDDING_MODELS } from "../src/config/constants.js"; import pRetry from "p-retry"; import * as fs from "fs"; import * as os from "os"; @@ -435,6 +436,79 @@ describe("CustomEmbeddingProvider", () => { }); }); +describe("OllamaEmbeddingProvider", () => { + let fetchSpy: ReturnType; + + beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch"); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + function createOllamaProvider(model: keyof typeof EMBEDDING_MODELS.ollama = "nomic-embed-text") { + return createEmbeddingProvider({ + provider: "ollama", + credentials: { + provider: "ollama", + baseUrl: "http://localhost:11434", + }, + modelInfo: EMBEDDING_MODELS.ollama[model], + }); + } + + it("retries oversize prompts with truncation for ollama", async () => { + fetchSpy + .mockResolvedValueOnce(new Response(JSON.stringify({ error: "the input length exceeds the context length" }), { status: 500 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ embedding: new Array(768).fill(0.1) }), { status: 200 })); + + const provider = createOllamaProvider(); + const oversized = "x".repeat(9000); + const result = await provider.embedBatch([oversized]); + + expect(fetchSpy).toHaveBeenCalledTimes(2); + const firstBody = JSON.parse((fetchSpy.mock.calls[0] as [string, RequestInit])[1].body as string) as { prompt: string; truncate: boolean }; + const secondBody = JSON.parse((fetchSpy.mock.calls[1] as [string, RequestInit])[1].body as string) as { prompt: string; truncate: boolean }; + expect(firstBody.truncate).toBe(false); + expect(secondBody.truncate).toBe(false); + expect(secondBody.prompt.length).toBeLessThan(firstBody.prompt.length); + expect(result.embeddings).toHaveLength(1); + }); + + it("processes ollama embedBatch requests sequentially", async () => { + let activeRequests = 0; + let maxActiveRequests = 0; + + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { truncate?: boolean }; + expect(body.truncate).toBe(false); + + activeRequests += 1; + maxActiveRequests = Math.max(maxActiveRequests, activeRequests); + + await new Promise((resolve) => setTimeout(resolve, 5)); + + activeRequests -= 1; + return new Response(JSON.stringify({ embedding: new Array(768).fill(0.1) }), { status: 200 }); + }); + + const provider = createOllamaProvider(); + const result = await provider.embedBatch(["first", "second", "third"]); + + expect(result.embeddings).toHaveLength(3); + expect(maxActiveRequests).toBe(1); + expect(fetchSpy).toHaveBeenCalledTimes(3); + }); + + it("rethrows non-context ollama errors", async () => { + fetchSpy.mockResolvedValueOnce(new Response(JSON.stringify({ error: "boom" }), { status: 500 })); + + const provider = createOllamaProvider(); + await expect(provider.embedBatch(["hello"])).rejects.toThrow("Ollama embedding API error: 500"); + }); +}); + describe("Indexer custom provider initialization", () => { let tempDir: string; diff --git a/tests/database.test.ts b/tests/database.test.ts index 5e255b2a..e682c9f7 100644 --- a/tests/database.test.ts +++ b/tests/database.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import * as fs from "fs"; import * as path from "path"; import * as os from "os"; -import { Database, ChunkData } from "../src/native/index.js"; +import { Database, ChunkData, SymbolData } from "../src/native/index.js"; describe("Database", () => { let tempDir: string; @@ -163,6 +163,30 @@ describe("Database", () => { expect(branches).toContain("feature"); }); + it("should include branches that only have symbols", () => { + const testSymbol: SymbolData = { + id: "sym_abc123", + filePath: "/path/to/file.ts", + name: "testFunction", + kind: "function", + startLine: 10, + startCol: 0, + endLine: 20, + endCol: 0, + language: "typescript", + }; + + db.upsertSymbol(testSymbol); + db.addSymbolsToBranchBatch("symbols-only", [testSymbol.id]); + db.upsertChunk(testChunk); + db.addChunksToBranch("chunks-only", [testChunk.chunkId]); + + const branches = db.getAllBranches(); + + expect(branches).toContain("symbols-only"); + expect(branches).toContain("chunks-only"); + }); + it("should compute branch delta", () => { db.upsertChunk(testChunk); db.upsertChunk({ ...testChunk, chunkId: "chunk_main_only" }); @@ -332,6 +356,38 @@ describe("Database", () => { expect(stats.branchChunkCount).toBe(1); expect(stats.branchCount).toBe(1); }); + + it("should count branches that only have symbols", () => { + const testSymbol: SymbolData = { + id: "sym_stats", + filePath: "/file.ts", + name: "statsFunction", + kind: "function", + startLine: 1, + startCol: 0, + endLine: 5, + endCol: 0, + language: "typescript", + }; + + db.upsertChunk({ + chunkId: "chunk_stats", + contentHash: "hash_stats", + filePath: "/chunk.ts", + startLine: 1, + endLine: 5, + language: "typescript", + }); + db.addChunksToBranch("chunk-branch", ["chunk_stats"]); + db.upsertSymbol(testSymbol); + db.addSymbolsToBranchBatch("symbol-branch", [testSymbol.id]); + + const stats = db.getStats(); + + expect(stats.branchChunkCount).toBe(1); + expect(stats.branchCount).toBe(2); + expect(stats.symbolCount).toBe(1); + }); }); describe("batch operations", () => { diff --git a/tests/indexer-clear-index.test.ts b/tests/indexer-clear-index.test.ts index c44dfe2b..55b004d8 100644 --- a/tests/indexer-clear-index.test.ts +++ b/tests/indexer-clear-index.test.ts @@ -127,6 +127,24 @@ describe("indexer clearIndex force rebuild", () => { expect(floatCount).toBe(4); }); + it("marks older embedding strategy metadata as incompatible until force rebuild", async () => { + embeddingDimensions = 8; + const indexer = createIndexer(tempDir, 8); + const stats = await indexer.index(); + expect(stats.failedChunks).toBe(0); + + const dbPath = path.join(tempDir, ".opencode", "index", "codebase.db"); + const db = new Database(dbPath); + db.setMetadata("index.embeddingStrategyVersion", "1"); + + const restartedIndexer = createIndexer(tempDir, 8); + const status = await restartedIndexer.getStatus(); + + expect(status.compatibility?.compatible).toBe(false); + expect(status.compatibility?.reason).toContain("Embedding strategy mismatch"); + await expect(restartedIndexer.index()).rejects.toThrow("Run index_codebase with force=true to rebuild the index"); + }); + it("rejects force clearing an inherited project index from a fresh worktree", async () => { const mainRepoDir = path.join(tempDir, "main-repo"); const worktreeDir = path.join(tempDir, "worktree-feature"); @@ -267,6 +285,469 @@ describe("indexer clearIndex force rebuild", () => { expect(rebuiltStats.indexedChunks).toBeGreaterThan(0); }); + it("allows a global embedding strategy rebuild without deleting other projects", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return 'b'; }\n", "utf-8"); + + const indexerA = createIndexer(projectA, 8, "global"); + const indexerB = createIndexer(projectB, 8, "global"); + + await indexerA.index(); + await indexerB.index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const projectBHash = hashContent(path.resolve(projectB)).slice(0, 16); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const restartedIndexerA = createIndexer(projectA, 8, "global"); + const statusBefore = await restartedIndexerA.getStatus(); + expect(statusBefore.compatibility?.compatible).toBe(false); + expect(statusBefore.compatibility?.reason).toContain("Embedding strategy mismatch"); + + await restartedIndexerA.clearIndex(); + + expect(db.getChunksByFile(projectAFile)).toHaveLength(0); + expect(db.getChunksByFile(projectBFile).length).toBeGreaterThan(0); + expect(db.getMetadata(`index.embeddingStrategyVersion.${projectAHash}`)).toBeNull(); + expect(db.getMetadata(`index.embeddingStrategyVersion.${projectBHash}`)).toBe("2"); + + const rebuiltStats = await restartedIndexerA.index(); + expect(rebuiltStats.failedChunks).toBe(0); + expect(rebuiltStats.indexedChunks).toBeGreaterThan(0); + + expect(db.getChunksByFile(projectAFile).length).toBeGreaterThan(0); + expect(db.getChunksByFile(projectBFile).length).toBeGreaterThan(0); + }); + + it("detects global embedding strategy mismatch from DB-only scoped state", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + const indexer = createIndexer(projectA, 8, "global"); + await indexer.index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const chunk = db.getChunksByFile(projectAFile)[0]; + const projectHash = hashContent(path.resolve(projectA)).slice(0, 16); + const branchKey = `${projectHash}:default`; + + db.setMetadata("index.embeddingStrategyVersion", "1"); + db.deleteMetadata(`index.embeddingStrategyVersion.${projectHash}`); + db.deleteBranchChunksForBranch(branchKey, [chunk.chunkId]); + db.addChunksToBranchBatch(branchKey, [chunk.chunkId]); + + const storeFile = path.join(tempHome, ".opencode", "global-index", "vectors.usearch"); + fs.rmSync(storeFile, { force: true }); + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors"), { recursive: true, force: true }); + + const restartedIndexer = createIndexer(projectA, 8, "global"); + const status = await restartedIndexer.getStatus(); + + expect(status.compatibility?.compatible).toBe(false); + expect(status.compatibility?.reason).toContain("Embedding strategy mismatch"); + }); + + it("detects DB-only scoped mismatch on a non-default branch during startup", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + fs.mkdirSync(path.join(projectA, ".git", "refs", "heads", "feature"), { recursive: true }); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(path.join(projectA, ".git", "HEAD"), "ref: refs/heads/feature/test\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "feature", "test"), "1111111111111111111111111111111111111111\n"); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + const indexer = createIndexer(projectA, 8, "global"); + await indexer.index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const chunk = db.getChunksByFile(projectAFile)[0]; + const projectHash = hashContent(path.resolve(projectA)).slice(0, 16); + const branchKey = `${projectHash}:feature/test`; + + db.setMetadata("index.embeddingStrategyVersion", "1"); + db.deleteMetadata(`index.embeddingStrategyVersion.${projectHash}`); + db.deleteBranchChunksForBranch(branchKey, [chunk.chunkId]); + db.addChunksToBranchBatch(branchKey, [chunk.chunkId]); + + const storeFile = path.join(tempHome, ".opencode", "global-index", "vectors.usearch"); + fs.rmSync(storeFile, { force: true }); + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors"), { recursive: true, force: true }); + + const restartedIndexer = createIndexer(projectA, 8, "global"); + const status = await restartedIndexer.getStatus(); + + expect(status.currentBranch).toBe("feature/test"); + expect(status.compatibility?.compatible).toBe(false); + expect(status.compatibility?.reason).toContain("Embedding strategy mismatch"); + }); + + it("detects file-hash-only scoped mismatch during startup status checks", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + await createIndexer(projectA, 8, "global").index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectHash = hashContent(path.resolve(projectA)).slice(0, 16); + const branchKey = `${projectHash}:default`; + + db.setMetadata("index.embeddingStrategyVersion", "1"); + db.deleteMetadata(`index.embeddingStrategyVersion.${projectHash}`); + db.clearBranch(branchKey); + db.deleteChunksByFile(projectAFile); + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors.usearch"), { force: true }); + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors"), { recursive: true, force: true }); + + const status = await createIndexer(projectA, 8, "global").getStatus(); + + expect(status.compatibility?.compatible).toBe(false); + expect(status.compatibility?.reason).toContain("Embedding strategy mismatch"); + }); + + it("detects symbol-only scoped mismatch during startup status checks", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + await createIndexer(projectA, 8, "global").index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectHash = hashContent(path.resolve(projectA)).slice(0, 16); + const branchKey = `${projectHash}:default`; + const projectSymbol = `sym_${hashContent(`${projectAFile}:alpha:function:1`).slice(0, 16)}`; + + db.setMetadata("index.embeddingStrategyVersion", "1"); + db.deleteMetadata(`index.embeddingStrategyVersion.${projectHash}`); + db.clearBranch(branchKey); + db.deleteBranchSymbolsForBranch(branchKey, [projectSymbol]); + db.deleteChunksByFile(projectAFile); + db.addSymbolsToBranchBatch(branchKey, [projectSymbol]); + + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors.usearch"), { force: true }); + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors"), { recursive: true, force: true }); + fs.writeFileSync( + path.join(tempHome, ".opencode", "global-index", "file-hashes.json"), + JSON.stringify({}, null, 2), + "utf-8" + ); + + expect(db.getBranchChunkIds(branchKey)).toHaveLength(0); + expect(db.getBranchSymbolIds(branchKey)).toContain(projectSymbol); + + const status = await createIndexer(projectA, 8, "global").getStatus(); + + expect(status.compatibility?.compatible).toBe(false); + expect(status.compatibility?.reason).toContain("Embedding strategy mismatch"); + }); + + it("re-embeds shared knowledge-base chunks after a global embedding strategy reset", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const kbDir = path.join(tempDir, "shared-kb"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + const kbFile = path.join(kbDir, "docs", "shared.ts"); + + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.mkdirSync(path.dirname(kbFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(kbFile, "export function sharedDoc() { return 'shared'; }\n", "utf-8"); + + const embedInputs: string[][] = []; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + embedInputs.push(texts); + + const data = texts.map((text) => { + let seed = 0; + for (const ch of text) { + seed = (seed * 31 + ch.charCodeAt(0)) % 1000; + } + const embedding = Array.from( + { length: embeddingDimensions }, + (_, idx) => ((seed + idx * 17) % 997) / 997 + ); + return { embedding }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * embeddingDimensions) }, + }), + { status: 200 } + ); + }); + + const createKbIndexer = (projectRoot: string) => new Indexer(projectRoot, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "mock-8d", + dimensions: 8, + }, + scope: "global", + knowledgeBases: [kbDir], + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + }, + })); + + await createKbIndexer(projectA).index(); + await createKbIndexer(projectB).index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const beforeResetCalls = embedInputs.length; + const restartedIndexer = createKbIndexer(projectA); + await restartedIndexer.clearIndex(); + + const resetStatus = await restartedIndexer.getStatus(); + expect(resetStatus.compatibility?.compatible).toBe(true); + + const rebuiltStats = await restartedIndexer.index(); + expect(rebuiltStats.failedChunks).toBe(0); + + const afterResetInputs = embedInputs.slice(beforeResetCalls).flat(); + expect(afterResetInputs.some((text) => text.includes("sharedDoc"))).toBe(true); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + }); + + it("keeps forced re-embed pending across restart until a failed shared chunk is re-embedded", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const kbDir = path.join(tempDir, "shared-kb"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + const kbFile = path.join(kbDir, "docs", "shared.ts"); + + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.mkdirSync(path.dirname(kbFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(kbFile, "export function sharedDoc() { return 'shared'; }\n", "utf-8"); + + const kbPrompt = "export function sharedDoc() { return 'shared'; }"; + let failSharedKbEmbedding = false; + const embedInputs: string[][] = []; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + embedInputs.push(texts); + + if (failSharedKbEmbedding && texts.some((text) => text.includes(kbPrompt))) { + return new Response(JSON.stringify({ error: "simulated shared kb failure" }), { status: 500 }); + } + + const data = texts.map((text) => { + let seed = 0; + for (const ch of text) { + seed = (seed * 31 + ch.charCodeAt(0)) % 1000; + } + const embedding = Array.from( + { length: embeddingDimensions }, + (_, idx) => ((seed + idx * 17) % 997) / 997 + ); + return { embedding }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * embeddingDimensions) }, + }), + { status: 200 } + ); + }); + + const createKbIndexer = (projectRoot: string) => new Indexer(projectRoot, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "mock-8d", + dimensions: 8, + }, + scope: "global", + knowledgeBases: [kbDir], + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + }, + })); + + await createKbIndexer(projectA).index(); + await createKbIndexer(projectB).index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const projectABranch = `${projectAHash}:default`; + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const resettingIndexer = createKbIndexer(projectA); + await resettingIndexer.clearIndex(); + + failSharedKbEmbedding = true; + const failedStats = await resettingIndexer.index(); + expect(failedStats.failedChunks).toBeGreaterThan(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + const sharedChunkId = db.getChunksByFile(kbFile)[0]?.chunkId; + expect(sharedChunkId).toBeTruthy(); + expect(db.chunkExistsOnBranch(projectABranch, sharedChunkId!)).toBe(false); + + failSharedKbEmbedding = false; + const restartedIndexer = createKbIndexer(projectA); + const restartStatus = await restartedIndexer.getStatus(); + expect(restartStatus.compatibility?.compatible).toBe(true); + + const beforeRecoveryCalls = embedInputs.length; + const recoveredStats = await restartedIndexer.index(); + expect(recoveredStats.failedChunks).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + expect(db.chunkExistsOnBranch(projectABranch, sharedChunkId!)).toBe(true); + + const recoveryInputs = embedInputs.slice(beforeRecoveryCalls).flat(); + expect(recoveryInputs.some((text) => text.includes(kbPrompt))).toBe(true); + }); + + it("rejects a full global reset when another tenant survives only in DB branch rows", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return 'b'; }\n", "utf-8"); + + await createIndexer(projectA, 8, "global").index(); + await createIndexer(projectB, 8, "global").index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const projectBHash = hashContent(path.resolve(projectB)).slice(0, 16); + const projectAChunk = db.getChunksByFile(projectAFile)[0]; + const projectBChunk = db.getChunksByFile(projectBFile)[0]; + + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors.usearch"), { force: true }); + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors"), { recursive: true, force: true }); + fs.writeFileSync( + path.join(tempHome, ".opencode", "global-index", "file-hashes.json"), + JSON.stringify({}, null, 2), + "utf-8" + ); + db.clearBranch(`${projectAHash}:default`); + db.clearBranch(`${projectBHash}:default`); + db.addChunksToBranchBatch(`${projectAHash}:default`, [projectAChunk.chunkId]); + db.addChunksToBranchBatch(`${projectBHash}:default`, [projectBChunk.chunkId]); + + embeddingDimensions = 4; + await expect(createIndexer(projectA, 4, "global").clearIndex()).rejects.toThrow( + "Global index compatibility reset is unsafe" + ); + }); + + it("rejects a full global reset when another tenant survives only in DB branch symbol rows", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return 'b'; }\n", "utf-8"); + + await createIndexer(projectA, 8, "global").index(); + await createIndexer(projectB, 8, "global").index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const projectBHash = hashContent(path.resolve(projectB)).slice(0, 16); + const projectAChunk = db.getChunksByFile(projectAFile)[0]; + const projectBChunk = db.getChunksByFile(projectBFile)[0]; + const projectASymbol = `sym_${hashContent(`${projectAFile}:alpha:function:1`).slice(0, 16)}`; + const projectBSymbol = `sym_${hashContent(`${projectBFile}:beta:function:1`).slice(0, 16)}`; + + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors.usearch"), { force: true }); + fs.rmSync(path.join(tempHome, ".opencode", "global-index", "vectors"), { recursive: true, force: true }); + fs.writeFileSync( + path.join(tempHome, ".opencode", "global-index", "file-hashes.json"), + JSON.stringify({}, null, 2), + "utf-8" + ); + + db.clearBranch(`${projectAHash}:default`); + db.clearBranch(`${projectBHash}:default`); + db.deleteBranchSymbolsForBranch(`${projectAHash}:default`, [projectASymbol]); + db.deleteBranchSymbolsForBranch(`${projectBHash}:default`, [projectBSymbol]); + db.deleteChunksByFile(projectAFile); + db.deleteChunksByFile(projectBFile); + db.addSymbolsToBranchBatch(`${projectAHash}:default`, [projectASymbol]); + db.addSymbolsToBranchBatch(`${projectBHash}:default`, [projectBSymbol]); + + expect(db.getBranchChunkIds(`${projectBHash}:default`)).toHaveLength(0); + expect(db.getBranchSymbolIds(`${projectBHash}:default`)).toContain(projectBSymbol); + expect(projectAChunk).toBeTruthy(); + expect(projectBChunk).toBeTruthy(); + + embeddingDimensions = 4; + await expect(createIndexer(projectA, 4, "global").clearIndex()).rejects.toThrow( + "Global index compatibility reset is unsafe" + ); + }); + it("resets a corrupted local sqlite index during health check and reports rebuild guidance", async () => { embeddingDimensions = 8; const indexer = createIndexer(tempDir, 8); @@ -513,6 +994,225 @@ describe("indexer clearIndex force rebuild", () => { expect(searchResults.filter((result) => result.filePath === projectAFile)).toHaveLength(1); }); + it("clears namespaced and legacy branch rows for the current repo's other branches during strategy reset", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + fs.mkdirSync(path.join(projectA, ".git", "refs", "heads", "feature"), { recursive: true }); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(path.join(projectA, ".git", "HEAD"), "ref: refs/heads/default\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "default"), "1111111111111111111111111111111111111111\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "feature", "test"), "2222222222222222222222222222222222222222\n"); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + embeddingDimensions = 8; + const indexerA = createIndexer(projectA, 8, "global"); + await indexerA.index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const defaultBranch = `${projectAHash}:default`; + const featureBranch = `${projectAHash}:feature/test`; + const legacyFeatureBranch = "feature/test"; + const projectAChunk = db.getChunksByFile(projectAFile)[0]; + + db.addChunksToBranchBatch(featureBranch, [projectAChunk.chunkId]); + db.addChunksToBranchBatch(legacyFeatureBranch, [projectAChunk.chunkId]); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const globalIndexDir = path.join(tempHome, ".opencode", "global-index"); + fs.writeFileSync( + path.join(globalIndexDir, "file-hashes.json"), + JSON.stringify({ [projectAFile]: "project-a-hash" }, null, 2), + "utf-8" + ); + fs.writeFileSync(path.join(globalIndexDir, "failed-batches.json"), "[]", "utf-8"); + + embeddingDimensions = 8; + const resettingIndexer = createIndexer(projectA, 8, "global"); + await resettingIndexer.clearIndex(); + + expect(db.chunkExistsOnBranch(defaultBranch, projectAChunk.chunkId)).toBe(false); + expect(db.chunkExistsOnBranch(featureBranch, projectAChunk.chunkId)).toBe(false); + expect(db.chunkExistsOnBranch(legacyFeatureBranch, projectAChunk.chunkId)).toBe(false); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + }); + + it("allows incompatible global reset when only same-project non-current branches have data", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + fs.mkdirSync(path.join(projectA, ".git", "refs", "heads", "feature"), { recursive: true }); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(path.join(projectA, ".git", "HEAD"), "ref: refs/heads/default\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "default"), "1111111111111111111111111111111111111111\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "feature", "test"), "2222222222222222222222222222222222222222\n"); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + embeddingDimensions = 8; + await createIndexer(projectA, 8, "global").index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const defaultBranch = `${projectAHash}:default`; + const featureBranch = `${projectAHash}:feature/test`; + const projectAChunk = db.getChunksByFile(projectAFile)[0]; + + db.addChunksToBranchBatch(featureBranch, [projectAChunk.chunkId]); + expect(db.chunkExistsOnBranch(defaultBranch, projectAChunk.chunkId)).toBe(true); + expect(db.chunkExistsOnBranch(featureBranch, projectAChunk.chunkId)).toBe(true); + + embeddingDimensions = 4; + const incompatibleIndexer = createIndexer(projectA, 4, "global"); + await expect(incompatibleIndexer.clearIndex()).resolves.toBeUndefined(); + + expect(db.chunkExistsOnBranch(defaultBranch, projectAChunk.chunkId)).toBe(false); + expect(db.chunkExistsOnBranch(featureBranch, projectAChunk.chunkId)).toBe(false); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + }); + + it("clears DB-only legacy branch rows for deleted same-project branches during strategy reset", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + fs.mkdirSync(path.join(projectA, ".git", "refs", "heads"), { recursive: true }); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(path.join(projectA, ".git", "HEAD"), "ref: refs/heads/default\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "default"), "1111111111111111111111111111111111111111\n"); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + await createIndexer(projectA, 8, "global").index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const defaultBranch = `${projectAHash}:default`; + const deletedLegacyBranch = "feature/old"; + const projectAChunk = db.getChunksByFile(projectAFile)[0]; + + db.addChunksToBranchBatch(deletedLegacyBranch, [projectAChunk.chunkId]); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const globalIndexDir = path.join(tempHome, ".opencode", "global-index"); + fs.writeFileSync( + path.join(globalIndexDir, "file-hashes.json"), + JSON.stringify({ [projectAFile]: "project-a-hash" }, null, 2), + "utf-8" + ); + fs.writeFileSync(path.join(globalIndexDir, "failed-batches.json"), "[]", "utf-8"); + + await createIndexer(projectA, 8, "global").clearIndex(); + + expect(db.chunkExistsOnBranch(defaultBranch, projectAChunk.chunkId)).toBe(false); + expect(db.chunkExistsOnBranch(deletedLegacyBranch, projectAChunk.chunkId)).toBe(false); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + }); + + it("allows incompatible global reset when only same-project legacy bare branch rows remain", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectAFile = path.join(projectA, "src", "a.ts"); + fs.mkdirSync(path.join(projectA, ".git", "refs", "heads", "feature"), { recursive: true }); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(path.join(projectA, ".git", "HEAD"), "ref: refs/heads/default\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "default"), "1111111111111111111111111111111111111111\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "feature", "test"), "2222222222222222222222222222222222222222\n"); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + + await createIndexer(projectA, 8, "global").index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const namespacedDefaultBranch = `${projectAHash}:default`; + const legacyFeatureBranch = "feature/test"; + const projectAChunk = db.getChunksByFile(projectAFile)[0]; + + db.addChunksToBranchBatch(legacyFeatureBranch, [projectAChunk.chunkId]); + db.clearBranch(namespacedDefaultBranch); + + expect(db.getBranchChunkIds(namespacedDefaultBranch)).toHaveLength(0); + expect(db.chunkExistsOnBranch(legacyFeatureBranch, projectAChunk.chunkId)).toBe(true); + + embeddingDimensions = 4; + const incompatibleIndexer = createIndexer(projectA, 4, "global"); + await expect(incompatibleIndexer.clearIndex()).resolves.toBeUndefined(); + + expect(db.chunkExistsOnBranch(legacyFeatureBranch, projectAChunk.chunkId)).toBe(false); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + }); + + it("preserves foreign legacy shared-kb branch rows during strategy reset", async () => { + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const sharedDir = path.join(tempDir, "shared-kb"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + const sharedFile = path.join(sharedDir, "shared.ts"); + + fs.mkdirSync(path.join(projectA, ".git", "refs", "heads", "feature"), { recursive: true }); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.writeFileSync(path.join(projectA, ".git", "HEAD"), "ref: refs/heads/default\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "default"), "1111111111111111111111111111111111111111\n"); + fs.writeFileSync(path.join(projectA, ".git", "refs", "heads", "feature", "test"), "2222222222222222222222222222222222222222\n"); + + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.mkdirSync(path.dirname(sharedFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(sharedFile, "export function sharedDoc() { return 'shared'; }\n", "utf-8"); + + const createKbIndexer = (projectRoot: string) => new Indexer(projectRoot, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "mock-8d", + dimensions: 8, + }, + scope: "global", + knowledgeBases: [sharedDir], + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + }, + })); + + await createKbIndexer(projectA).index(); + await createKbIndexer(projectB).index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectAHash = hashContent(path.resolve(projectA)).slice(0, 16); + const projectAProjectChunk = db.getChunksByFile(projectAFile)[0]; + const sharedChunk = db.getChunksByFile(sharedFile)[0]; + const foreignLegacyBranch = "feature/test"; + + db.addChunksToBranchBatch(foreignLegacyBranch, [sharedChunk.chunkId]); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const globalIndexDir = path.join(tempHome, ".opencode", "global-index"); + fs.writeFileSync( + path.join(globalIndexDir, "file-hashes.json"), + JSON.stringify({ [projectAFile]: "project-a-hash", [projectBFile]: "project-b-hash", [sharedFile]: "shared-hash" }, null, 2), + "utf-8" + ); + fs.writeFileSync(path.join(globalIndexDir, "failed-batches.json"), "[]", "utf-8"); + + await createKbIndexer(projectA).clearIndex(); + + expect(db.chunkExistsOnBranch(foreignLegacyBranch, sharedChunk.chunkId)).toBe(true); + expect(db.chunkExistsOnBranch(foreignLegacyBranch, projectAProjectChunk.chunkId)).toBe(false); + }); + it("preserves foreign failed-batch and file-hash state during global clear when no vectors exist yet", async () => { vi.stubEnv("HOME", tempHome); diff --git a/tests/indexer-failed-batches.test.ts b/tests/indexer-failed-batches.test.ts index 013f49bd..66264dbd 100644 --- a/tests/indexer-failed-batches.test.ts +++ b/tests/indexer-failed-batches.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { parseConfig } from "../src/config/schema.js"; import { Indexer } from "../src/indexer/index.js"; +import { Database, VectorStore, hashContent } from "../src/native/index.js"; import { formatStatus } from "../src/tools/utils.js"; describe("indexer failed batch recovery", () => { @@ -17,8 +18,25 @@ describe("indexer failed batch recovery", () => { beforeEach(() => { failEmbeddings = false; fetchSpy = vi.spyOn(globalThis, "fetch"); - fetchSpy.mockImplementation(async (_url, init) => { - const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + fetchSpy.mockImplementation(async (url, init) => { + if (String(url).endsWith("/api/tags")) { + return new Response(JSON.stringify({ + models: [{ name: "nomic-embed-text" }], + }), { status: 200 }); + } + + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[]; prompt?: string }; + + if (body.prompt) { + if (body.prompt.includes("triggerFailure") && !body.prompt.includes("Part ")) { + return new Response(JSON.stringify({ error: "the input length exceeds the context length" }), { status: 500 }); + } + + return new Response(JSON.stringify({ + embedding: Array.from({ length: 768 }, () => 0.1), + }), { status: 200 }); + } + const texts = Array.isArray(body.input) ? body.input : []; if (failEmbeddings) { @@ -63,6 +81,7 @@ describe("indexer failed batch recovery", () => { afterEach(() => { fetchSpy.mockRestore(); + vi.unstubAllEnvs(); fs.rmSync(tempDir, { recursive: true, force: true }); }); @@ -84,6 +103,39 @@ describe("indexer failed batch recovery", () => { return new Indexer(tempDir, config); } + function createLimitedBatchIndexer(maxBatchSize: number): Indexer { + const config = parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "mock-embedding-model", + dimensions: 8, + maxBatchSize, + }, + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + }, + }); + + return new Indexer(tempDir, config); + } + + function createOllamaIndexer(): Indexer { + const config = parseConfig({ + embeddingProvider: "ollama", + embeddingModel: "nomic-embed-text", + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + }, + }); + + return new Indexer(tempDir, config); + } + it("retries saved failed batches on a later successful rerun without force", async () => { const indexer = createIndexer(); @@ -145,4 +197,1302 @@ describe("indexer failed batch recovery", () => { expect(message).toContain("retry the saved failed batches"); expect(message).toContain("Use force=true only for a full rebuild or compatibility reset"); }); + + it("isolates ollama embedding failures to the offending chunk", async () => { + const safeFile = path.join(tempDir, "src", "safe.ts"); + fs.writeFileSync(safeFile, "export function safeChunk() { return 'ok'; }\n", "utf-8"); + + fs.writeFileSync( + sourceFile, + [ + "export const alpha = 'alpha';", + "export const beta = 'beta';", + "export const gamma = 'gamma';", + "export const delta = 'delta';", + "export const epsilon = 'epsilon';", + "export const zeta = 'zeta';", + "export const eta = 'eta';", + "export const theta = 'theta';", + "export const triggerFailure = 'triggerFailure';", + "export const iota = 'iota';", + "export const kappa = 'kappa';", + "export const lambda = 'lambda';", + "export const mu = 'mu';", + "export const nu = 'nu';", + "export const xi = 'xi';", + "export const omicron = 'omicron';", + "export const pi = 'pi';", + "export const rho = 'rho';", + "export const sigma = 'sigma';", + "export const tau = 'tau';", + "export const upsilon = 'upsilon';", + "export const phi = 'phi';", + "export const chi = 'chi';", + "export const psi = 'psi';", + "export const omega = 'omega';", + "export const stillWorks = 'stillWorks';", + ].join("\n"), + "utf-8" + ); + + const indexer = createOllamaIndexer(); + const stats = await indexer.index(); + + expect(stats.indexedChunks).toBeGreaterThan(0); + expect(stats.failedChunks).toBeGreaterThan(0); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBeGreaterThan(0); + }); + + it("splits oversized ollama chunks into pooled sub-requests before embedding", async () => { + const embedPrompts: string[] = []; + fetchSpy.mockImplementation(async (url, init) => { + if (String(url).endsWith("/api/tags")) { + return new Response(JSON.stringify({ + models: [{ name: "nomic-embed-text" }], + }), { status: 200 }); + } + + const body = JSON.parse(String(init?.body ?? "{}")) as { prompt?: string }; + const prompt = body.prompt ?? ""; + embedPrompts.push(prompt); + + if (prompt.length > 8200) { + return new Response(JSON.stringify({ error: "the input length exceeds the context length" }), { status: 500 }); + } + + const seed = prompt.length % 17; + return new Response(JSON.stringify({ + embedding: Array.from({ length: 768 }, (_, idx) => seed + idx / 1000), + }), { status: 200 }); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function oversizedChunk() {", + ` const blob = ${JSON.stringify("triggerFailure ".repeat(900))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createOllamaIndexer(); + const stats = await indexer.index(); + + expect(stats.failedChunks).toBe(0); + expect(stats.indexedChunks).toBeGreaterThan(0); + expect(embedPrompts.length).toBeGreaterThan(1); + expect(embedPrompts.some((prompt) => prompt.includes("Part 1/"))).toBe(true); + expect(embedPrompts.some((prompt) => prompt.includes("Part 2/"))).toBe(true); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(0); + }); + + it("rebuilds legacy failed-batch prompts with the current split strategy", async () => { + const indexer = createOllamaIndexer(); + await indexer.initialize(); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + fs.mkdirSync(path.dirname(failedBatchesPath), { recursive: true }); + fs.writeFileSync( + failedBatchesPath, + JSON.stringify([ + { + chunks: [ + { + id: "legacy-oversized", + text: "triggerFailure legacy prompt", + content: "triggerFailure ".repeat(900), + contentHash: "legacy-hash", + metadata: { + filePath: sourceFile, + startLine: 1, + endLine: 1, + language: "typescript", + chunkType: "function", + hash: "legacy-hash", + name: "legacyOversized", + }, + }, + ], + error: "legacy oversize failure", + attemptCount: 1, + lastAttempt: new Date().toISOString(), + }, + ], null, 2), + "utf-8" + ); + + const embedPrompts: string[] = []; + fetchSpy.mockImplementation(async (url, init) => { + if (String(url).endsWith("/api/tags")) { + return new Response(JSON.stringify({ + models: [{ name: "nomic-embed-text" }], + }), { status: 200 }); + } + + const body = JSON.parse(String(init?.body ?? "{}")) as { prompt?: string }; + const prompt = body.prompt ?? ""; + embedPrompts.push(prompt); + + if (prompt.includes("triggerFailure") && !prompt.includes("Part ")) { + return new Response(JSON.stringify({ error: "the input length exceeds the context length" }), { status: 500 }); + } + + return new Response(JSON.stringify({ + embedding: Array.from({ length: 768 }, () => 0.1), + }), { status: 200 }); + }); + + const retry = await indexer.retryFailedBatches(); + + expect(retry.failed).toBe(0); + expect(retry.remaining).toBe(0); + expect(retry.succeeded).toBe(1); + expect(embedPrompts.length).toBeGreaterThan(1); + expect(embedPrompts.some((prompt) => prompt.includes("Part 1/"))).toBe(true); + expect(embedPrompts.some((prompt) => prompt.includes("Part 2/"))).toBe(true); + }); + + it("rebuilds retryable legacy failed-batch prompts with the provider-aware split budget", async () => { + const indexer = createOllamaIndexer(); + await indexer.initialize(); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + fs.mkdirSync(path.dirname(failedBatchesPath), { recursive: true }); + fs.writeFileSync( + failedBatchesPath, + JSON.stringify([ + { + chunks: [ + { + id: "legacy-retryable", + text: "triggerFailure legacy retryable prompt", + content: "triggerFailure ".repeat(900), + contentHash: "legacy-retryable-hash", + metadata: { + filePath: sourceFile, + startLine: 1, + endLine: 1, + language: "typescript", + chunkType: "function", + hash: "legacy-retryable-hash", + name: "legacyRetryable", + }, + }, + ], + error: "legacy retryable oversize failure", + attemptCount: 1, + lastAttempt: new Date().toISOString(), + }, + ], null, 2), + "utf-8" + ); + + fs.writeFileSync( + sourceFile, + [ + "export function legacyRetryable() {", + ` return ${JSON.stringify("triggerFailure ".repeat(900))};`, + "}", + ].join("\n"), + "utf-8" + ); + + const embedPrompts: string[] = []; + fetchSpy.mockImplementation(async (url, init) => { + if (String(url).endsWith("/api/tags")) { + return new Response(JSON.stringify({ + models: [{ name: "nomic-embed-text" }], + }), { status: 200 }); + } + + const body = JSON.parse(String(init?.body ?? "{}")) as { prompt?: string }; + const prompt = body.prompt ?? ""; + embedPrompts.push(prompt); + + if (prompt.includes("triggerFailure") && !prompt.includes("Part ")) { + return new Response(JSON.stringify({ error: "the input length exceeds the context length" }), { status: 500 }); + } + + return new Response(JSON.stringify({ + embedding: Array.from({ length: 768 }, () => 0.1), + }), { status: 200 }); + }); + + const stats = await indexer.index(); + + expect(stats.failedChunks).toBe(0); + expect(stats.indexedChunks).toBeGreaterThan(0); + expect(embedPrompts.length).toBeGreaterThan(1); + expect(embedPrompts.some((prompt) => prompt.includes("Part 1/"))).toBe(true); + expect(embedPrompts.some((prompt) => prompt.includes("Part 2/"))).toBe(true); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(0); + }); + + it("pools split custom-provider chunks across multiple embedBatch calls", async () => { + const requestSizes: number[] = []; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + requestSizes.push(texts.length); + + const data = texts.map((text, textIndex) => { + const seed = (text.length + textIndex) % 23; + return { + embedding: Array.from({ length: 8 }, (_, idx) => seed + idx / 100), + }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * 8) }, + }), + { status: 200 } + ); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function oversizedCustomChunk() {", + ` const blob = ${JSON.stringify("segment ".repeat(1500))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createLimitedBatchIndexer(1); + const stats = await indexer.index(); + + expect(stats.failedChunks).toBe(0); + expect(stats.indexedChunks).toBe(1); + expect(requestSizes.length).toBeGreaterThan(1); + expect(requestSizes.every((size) => size <= 1)).toBe(true); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(0); + }); + + it("waits for all split parts before pooling when custom-provider calls complete out of order", async () => { + let callCount = 0; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + const currentCall = callCount++; + + if (currentCall === 0) { + await new Promise((resolve) => setTimeout(resolve, 40)); + } + + const data = texts.map((text, textIndex) => { + const seed = (text.length + textIndex + currentCall) % 31; + return { + embedding: Array.from({ length: 8 }, (_, idx) => seed + idx / 100), + }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * 8) }, + }), + { status: 200 } + ); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function outOfOrderSplitChunk() {", + ` const blob = ${JSON.stringify("out-of-order ".repeat(1500))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createLimitedBatchIndexer(1); + const stats = await indexer.index(); + + expect(stats.failedChunks).toBe(0); + expect(stats.indexedChunks).toBe(1); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(0); + }); + + it("retries split custom-provider failed batches across multiple embedBatch calls", async () => { + let firstChunkAttempt = true; + const requestSizes: number[] = []; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + requestSizes.push(texts.length); + + if (firstChunkAttempt && texts.some((text) => text.includes("Part 1/"))) { + firstChunkAttempt = false; + return new Response(JSON.stringify({ error: "transient batch failure" }), { status: 500 }); + } + + const data = texts.map((text, textIndex) => { + const seed = (text.length + textIndex) % 29; + return { + embedding: Array.from({ length: 8 }, (_, idx) => seed + idx / 100), + }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * 8) }, + }), + { status: 200 } + ); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function retryableSplitChunk() {", + ` const blob = ${JSON.stringify("retryable ".repeat(1400))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createLimitedBatchIndexer(1); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(1); + expect(failedStats.indexedChunks).toBe(0); + + const retry = await indexer.retryFailedBatches(); + + expect(retry.failed).toBe(0); + expect(retry.remaining).toBe(0); + expect(retry.succeeded).toBe(1); + expect(requestSizes.length).toBeGreaterThan(1); + expect(requestSizes.every((size) => size <= 1)).toBe(true); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(0); + }); + + it("waits for all split retry parts before pooling when retry calls complete out of order", async () => { + let firstChunkAttempt = true; + let callCount = 0; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + const currentCall = callCount++; + + if (firstChunkAttempt && texts.some((text) => text.includes("Part 1/"))) { + firstChunkAttempt = false; + return new Response(JSON.stringify({ error: "transient batch failure" }), { status: 500 }); + } + + if (currentCall % 2 === 1) { + await new Promise((resolve) => setTimeout(resolve, 40)); + } + + const data = texts.map((text, textIndex) => { + const seed = (text.length + textIndex + currentCall) % 41; + return { + embedding: Array.from({ length: 8 }, (_, idx) => seed + idx / 100), + }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * 8) }, + }), + { status: 200 } + ); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function outOfOrderRetrySplitChunk() {", + ` const blob = ${JSON.stringify("retry-out-of-order ".repeat(1400))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createLimitedBatchIndexer(1); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(1); + expect(failedStats.indexedChunks).toBe(0); + + const retry = await indexer.retryFailedBatches(); + + expect(retry.failed).toBe(0); + expect(retry.remaining).toBe(0); + expect(retry.succeeded).toBe(1); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(0); + }); + + it("deduplicates repeated retry failures for the same split chunk", async () => { + const requestSizes: number[] = []; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + requestSizes.push(texts.length); + + return new Response(JSON.stringify({ error: "persistent split failure" }), { status: 500 }); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function persistentlyFailingSplitChunk() {", + ` const blob = ${JSON.stringify("persistent ".repeat(1400))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createLimitedBatchIndexer(1); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(1); + expect(failedStats.indexedChunks).toBe(0); + + const retry = await indexer.retryFailedBatches(); + + expect(retry.succeeded).toBe(0); + expect(retry.failed).toBe(1); + expect(retry.remaining).toBe(1); + expect(requestSizes.length).toBeGreaterThan(1); + expect(requestSizes.every((size) => size <= 1)).toBe(true); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(1); + }); + + it("increments attemptCount when the same split chunk fails multiple times in one index run", async () => { + const embedPrompts: string[] = []; + fetchSpy.mockImplementation(async (url, init) => { + if (String(url).endsWith("/api/tags")) { + return new Response(JSON.stringify({ + models: [{ name: "nomic-embed-text" }], + }), { status: 200 }); + } + + const body = JSON.parse(String(init?.body ?? "{}")) as { prompt?: string }; + const prompt = body.prompt ?? ""; + embedPrompts.push(prompt); + + if (prompt.includes("same-run-failure")) { + return new Response(JSON.stringify({ error: "persistent split failure" }), { status: 500 }); + } + + return new Response(JSON.stringify({ + embedding: Array.from({ length: 768 }, () => 0.1), + }), { status: 200 }); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function persistentlyFailingSameRunSplitChunk() {", + ` const blob = ${JSON.stringify("same-run-failure ".repeat(900))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createOllamaIndexer(); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(1); + expect(failedStats.indexedChunks).toBe(0); + expect(embedPrompts.length).toBeGreaterThan(1); + expect(embedPrompts.every((prompt) => prompt.includes("Part "))).toBe(true); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + const persistedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ id: string }>; + attemptCount: number; + error: string; + }>; + + expect(persistedBatches).toHaveLength(1); + expect(persistedBatches[0]?.chunks[0]?.id).toBeDefined(); + expect(persistedBatches[0]?.attemptCount).toBeGreaterThan(1); + expect(persistedBatches[0]?.error).toContain("persistent split failure"); + }); + + it("persists failed batches when storage fails after pooling embeddings", async () => { + const addBatchSpy = vi.spyOn(VectorStore.prototype, "addBatch").mockImplementation(() => { + throw new Error("vector store write failed"); + }); + + try { + const indexer = createIndexer(); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(1); + expect(failedStats.indexedChunks).toBe(0); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + const persistedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ id: string }>; + error: string; + attemptCount: number; + }>; + + expect(persistedBatches).toHaveLength(1); + expect(persistedBatches.every((batch) => batch.error.includes("vector store write failed"))).toBe(true); + expect(persistedBatches.every((batch) => batch.attemptCount === 1)).toBe(true); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(1); + } finally { + addBatchSpy.mockRestore(); + } + }); + + it("does not double-count mixed request failures and storage failures during retry", async () => { + let firstRetryRun = true; + const addBatchSpy = vi.spyOn(VectorStore.prototype, "addBatch").mockImplementation(() => { + if (firstRetryRun) { + throw new Error("vector store write failed"); + } + }); + + try { + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + + if (firstRetryRun && texts.some((text) => text.includes("Part 1/"))) { + return new Response(JSON.stringify({ error: "transient batch failure" }), { status: 500 }); + } + + const data = texts.map((text, textIndex) => { + const seed = (text.length + textIndex) % 37; + return { + embedding: Array.from({ length: 8 }, (_, idx) => seed + idx / 100), + }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * 8) }, + }), + { status: 200 } + ); + }); + + fs.writeFileSync( + sourceFile, + [ + "export function mixedRetryFailureChunk() {", + ` const blob = ${JSON.stringify("retry-mixed ".repeat(1400))};`, + " return blob.length;", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createLimitedBatchIndexer(1); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(1); + expect(failedStats.indexedChunks).toBe(0); + + const retry = await indexer.retryFailedBatches(); + firstRetryRun = false; + + expect(retry.succeeded).toBe(0); + expect(retry.failed).toBe(1); + expect(retry.remaining).toBe(1); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + const persistedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ id: string }>; + error: string; + attemptCount: number; + }>; + + expect(persistedBatches).toHaveLength(1); + expect(persistedBatches[0]?.attemptCount).toBe(2); + expect(persistedBatches[0]?.error).toContain("transient batch failure"); + } finally { + addBatchSpy.mockRestore(); + } + }); + + it("keeps a retried chunk in failed batches when storage fails after pooling", async () => { + let failStorageOnRetry = false; + const addBatchSpy = vi.spyOn(VectorStore.prototype, "addBatch").mockImplementation(() => { + if (failStorageOnRetry) { + throw new Error("vector store write failed"); + } + }); + + try { + const indexer = createIndexer(); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(0); + expect(failedStats.indexedChunks).toBeGreaterThan(0); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + fs.writeFileSync( + failedBatchesPath, + JSON.stringify([ + { + chunks: [ + { + id: "chunk_abc123", + text: "export function alpha() { return 'alpha'; }", + content: "export function alpha() { return 'alpha'; }", + contentHash: "retry-hash", + metadata: { + filePath: sourceFile, + startLine: 1, + endLine: 3, + language: "typescript", + chunkType: "function", + hash: "retry-hash", + name: "alpha", + }, + }, + ], + error: "previous failure", + attemptCount: 1, + lastAttempt: new Date().toISOString(), + }, + ], null, 2), + "utf-8" + ); + + failStorageOnRetry = true; + const retry = await indexer.retryFailedBatches(); + + expect(retry.succeeded).toBe(0); + expect(retry.failed).toBe(1); + expect(retry.remaining).toBe(1); + + const persistedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ id: string }>; + error: string; + attemptCount: number; + }>; + + expect(persistedBatches).toHaveLength(1); + expect(persistedBatches[0]?.chunks[0]?.id).toBe("chunk_abc123"); + expect(persistedBatches[0]?.error).toContain("vector store write failed"); + expect(persistedBatches[0]?.attemptCount).toBe(2); + } finally { + addBatchSpy.mockRestore(); + } + }); + + it("coalesces same-run failed chunks back into one persisted failed batch", async () => { + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + + if (texts.length > 0) { + return new Response(JSON.stringify({ error: "shared batch failure" }), { status: 500 }); + } + + return new Response( + JSON.stringify({ + data: [], + usage: { total_tokens: 0 }, + }), + { status: 200 } + ); + }); + + const secondFile = path.join(tempDir, "src", "second.ts"); + fs.writeFileSync( + sourceFile, + [ + "export function alpha() {", + " return 'alpha';", + "}", + "", + "export function beta() {", + " return alpha();", + "}", + ].join("\n"), + "utf-8" + ); + fs.writeFileSync( + secondFile, + [ + "export function gamma() {", + " return 'gamma';", + "}", + ].join("\n"), + "utf-8" + ); + + const indexer = createIndexer(); + const failedStats = await indexer.index(); + + expect(failedStats.failedChunks).toBe(2); + expect(failedStats.indexedChunks).toBe(0); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + const persistedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ id: string }>; + error: string; + attemptCount: number; + }>; + + expect(persistedBatches).toHaveLength(1); + expect(persistedBatches[0]?.chunks).toHaveLength(2); + expect(persistedBatches[0]?.error).toContain("shared batch failure"); + expect(persistedBatches[0]?.attemptCount).toBe(1); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBe(1); + }); + + it("reports remaining failed batches using the coalesced persisted count", async () => { + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + + if (texts.length > 0) { + return new Response(JSON.stringify({ error: "shared retry failure" }), { status: 500 }); + } + + return new Response( + JSON.stringify({ + data: [], + usage: { total_tokens: 0 }, + }), + { status: 200 } + ); + }); + + const secondFile = path.join(tempDir, "src", "retry-second.ts"); + fs.writeFileSync(sourceFile, "export function alpha() { return 'alpha'; }\n", "utf-8"); + fs.writeFileSync(secondFile, "export function beta() { return 'beta'; }\n", "utf-8"); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + fs.mkdirSync(path.dirname(failedBatchesPath), { recursive: true }); + fs.writeFileSync( + failedBatchesPath, + JSON.stringify([ + { + chunks: [ + { + id: "chunk_alpha", + text: "export function alpha() { return 'alpha'; }", + content: "export function alpha() { return 'alpha'; }", + contentHash: "retry-alpha-hash", + metadata: { + filePath: sourceFile, + startLine: 1, + endLine: 1, + language: "typescript", + chunkType: "function", + hash: "retry-alpha-hash", + name: "alpha", + }, + }, + { + id: "chunk_beta", + text: "export function beta() { return 'beta'; }", + content: "export function beta() { return 'beta'; }", + contentHash: "retry-beta-hash", + metadata: { + filePath: secondFile, + startLine: 1, + endLine: 1, + language: "typescript", + chunkType: "function", + hash: "retry-beta-hash", + name: "beta", + }, + }, + ], + error: "previous grouped failure", + attemptCount: 1, + lastAttempt: new Date().toISOString(), + }, + ], null, 2), + "utf-8" + ); + + const indexer = createIndexer(); + await indexer.initialize(); + + const retry = await indexer.retryFailedBatches(); + const status = await indexer.getStatus(); + const persistedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ id: string }>; + error: string; + attemptCount: number; + }>; + + expect(retry.succeeded).toBe(0); + expect(retry.failed).toBe(2); + expect(retry.remaining).toBe(1); + expect(status.failedBatchesCount).toBe(1); + expect(persistedBatches).toHaveLength(1); + expect(persistedBatches[0]?.chunks).toHaveLength(2); + expect(persistedBatches[0]?.error).toContain("shared retry failure"); + }); + + it("preserves foreign legacy failed batches without rewriting them during global scoped saves", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "failed-batches-global-home-")); + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return 'a'; }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return 'b'; }\n", "utf-8"); + + const globalFailedBatchesPath = path.join(tempHome, ".opencode", "global-index", "failed-batches.json"); + fs.mkdirSync(path.dirname(globalFailedBatchesPath), { recursive: true }); + fs.writeFileSync( + globalFailedBatchesPath, + JSON.stringify([ + { + chunks: [ + { + id: "foreign-legacy", + text: "triggerFailure foreign legacy prompt", + content: "triggerFailure ".repeat(900), + contentHash: "foreign-legacy-hash", + metadata: { + filePath: projectBFile, + startLine: 1, + endLine: 1, + language: "typescript", + chunkType: "function", + hash: "foreign-legacy-hash", + name: "foreignLegacy", + }, + }, + ], + error: "foreign legacy oversize failure", + attemptCount: 1, + lastAttempt: new Date().toISOString(), + }, + ], null, 2), + "utf-8" + ); + + const indexer = new Indexer(projectA, parseConfig({ + embeddingProvider: "ollama", + embeddingModel: "nomic-embed-text", + scope: "global", + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + }, + })); + + const stats = await indexer.index(); + expect(stats.failedChunks).toBe(0); + + const persistedBatches = JSON.parse(fs.readFileSync(globalFailedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ metadata: { filePath: string }; text?: string; texts?: unknown[] }>; + }>; + const foreignChunk = persistedBatches + .flatMap((batch) => batch.chunks) + .find((chunk) => chunk.metadata.filePath === projectBFile); + + expect(foreignChunk).toBeDefined(); + expect(typeof foreignChunk?.text).toBe("string"); + expect(foreignChunk?.texts).toBeUndefined(); + + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("does not retry custom-provider non-retryable errors during retryFailedBatches()", async () => { + let embedCallCount = 0; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + if (Array.isArray(body.input)) { + embedCallCount += 1; + } + + return new Response(JSON.stringify({ error: "invalid api key" }), { status: 401 }); + }); + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + fs.mkdirSync(path.dirname(failedBatchesPath), { recursive: true }); + fs.writeFileSync( + failedBatchesPath, + JSON.stringify([ + { + chunks: [ + { + id: "non-retryable-custom-provider-chunk", + text: "export function alpha() { return 'alpha'; }", + content: "export function alpha() { return 'alpha'; }", + contentHash: "non-retryable-custom-provider-hash", + metadata: { + filePath: sourceFile, + startLine: 1, + endLine: 1, + language: "typescript", + chunkType: "function", + hash: "non-retryable-custom-provider-hash", + name: "alpha", + }, + }, + ], + error: "previous custom provider failure", + attemptCount: 1, + lastAttempt: new Date().toISOString(), + }, + ], null, 2), + "utf-8" + ); + + const retryingIndexer = new Indexer(tempDir, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "mock-embedding-model", + dimensions: 8, + }, + indexing: { + watchFiles: false, + retries: 3, + retryDelayMs: 1, + }, + })); + + await retryingIndexer.initialize(); + const retry = await retryingIndexer.retryFailedBatches(); + + expect(embedCallCount).toBe(1); + expect(retry.succeeded).toBe(0); + expect(retry.failed).toBe(1); + expect(retry.remaining).toBe(1); + + const persistedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + attemptCount: number; + error: string; + }>; + expect(persistedBatches[0]?.attemptCount).toBe(2); + expect(persistedBatches[0]?.error).toContain("invalid api key"); + }); + + it("clears pending global force re-embed metadata after retryFailedBatches() recovers all remaining chunks", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "failed-batches-force-reembed-home-")); + vi.stubEnv("HOME", tempHome); + + const projectA = path.join(tempDir, "project-a"); + const projectB = path.join(tempDir, "project-b"); + const kbDir = path.join(tempDir, "shared-kb"); + const projectAFile = path.join(projectA, "src", "a.ts"); + const projectBFile = path.join(projectB, "src", "b.ts"); + const kbFile = path.join(kbDir, "docs", "shared.ts"); + + fs.mkdirSync(path.dirname(projectAFile), { recursive: true }); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.mkdirSync(path.dirname(kbFile), { recursive: true }); + fs.writeFileSync(projectAFile, "export function alpha() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(projectBFile, "export function beta() { return sharedDoc(); }\n", "utf-8"); + fs.writeFileSync(kbFile, "export function sharedDoc() { return 'shared'; }\n", "utf-8"); + + const kbPrompt = "export function sharedDoc() { return 'shared'; }"; + let failSharedKbEmbedding = false; + fetchSpy.mockImplementation(async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = Array.isArray(body.input) ? body.input : []; + + if (failSharedKbEmbedding && texts.some((text) => text.includes(kbPrompt))) { + return new Response(JSON.stringify({ error: "simulated shared kb failure" }), { status: 500 }); + } + + const data = texts.map((text) => { + let seed = 0; + for (const ch of text) { + seed = (seed * 31 + ch.charCodeAt(0)) % 1000; + } + const embedding = Array.from({ length: 8 }, (_, idx) => ((seed + idx * 17) % 997) / 997); + return { embedding }; + }); + + return new Response( + JSON.stringify({ + data, + usage: { total_tokens: Math.max(1, texts.length * 8) }, + }), + { status: 200 } + ); + }); + + const createGlobalKbIndexer = (projectRoot: string) => new Indexer(projectRoot, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "mock-8d", + dimensions: 8, + }, + scope: "global", + knowledgeBases: [kbDir], + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + }, + })); + + await createGlobalKbIndexer(projectA).index(); + await createGlobalKbIndexer(projectB).index(); + + const dbPath = path.join(tempHome, ".opencode", "global-index", "codebase.db"); + const db = new Database(dbPath); + const projectHash = hashContent(path.resolve(projectA)).slice(0, 16); + const projectABranch = `${projectHash}:default`; + db.setMetadata(`index.embeddingStrategyVersion.${projectHash}`, "1"); + + const resettingIndexer = createGlobalKbIndexer(projectA); + await resettingIndexer.clearIndex(); + + failSharedKbEmbedding = true; + const failedStats = await resettingIndexer.index(); + expect(failedStats.failedChunks).toBeGreaterThan(0); + expect(db.getMetadata(`index.forceReembed.${projectHash}`)).toBe("true"); + + const sharedChunkId = db.getChunksByFile(kbFile)[0]?.chunkId; + expect(sharedChunkId).toBeTruthy(); + expect(db.chunkExistsOnBranch(projectABranch, sharedChunkId!)).toBe(false); + + failSharedKbEmbedding = false; + const retryingIndexer = createGlobalKbIndexer(projectA); + const retry = await retryingIndexer.retryFailedBatches(); + + expect(retry.failed).toBe(0); + expect(retry.remaining).toBe(0); + expect(retry.succeeded).toBeGreaterThan(0); + expect(db.getMetadata(`index.forceReembed.${projectHash}`)).toBeNull(); + expect(db.chunkExistsOnBranch(projectABranch, sharedChunkId!)).toBe(true); + + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("preserves previously indexed chunk in branch catalog when stale failed-batch retry fails", async () => { + const indexer = createIndexer(); + + // Step 1: Initial successful index + const initialStats = await indexer.index(); + expect(initialStats.failedChunks).toBe(0); + expect(initialStats.indexedChunks).toBeGreaterThan(0); + + // Step 2: Capture the actual indexed chunk id from the database + const dbPath = path.join(tempDir, ".opencode", "index", "codebase.db"); + const dbBefore = new Database(dbPath); + const branches = dbBefore.getAllBranches(); + const branchKey = branches.find((b) => b.includes("default")) || branches[0]; + expect(branchKey).toBeDefined(); + + const branchChunksBefore = dbBefore.getBranchChunkIds(branchKey!); + expect(branchChunksBefore.length).toBeGreaterThan(0); + const existingChunkId = branchChunksBefore[0]; + expect(existingChunkId).toBeDefined(); + + // Step 3: Manually write a stale failed-batch entry for that same unchanged chunk + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + fs.writeFileSync( + failedBatchesPath, + JSON.stringify([ + { + chunks: [ + { + id: existingChunkId, + text: "export function alpha() { return 'alpha'; }", + content: "export function alpha() { return 'alpha'; }", + contentHash: "stale-hash", + metadata: { + filePath: sourceFile, + startLine: 1, + endLine: 3, + language: "typescript", + chunkType: "function", + hash: "stale-hash", + name: "alpha", + }, + }, + ], + error: "stale previous failure", + attemptCount: 1, + lastAttempt: new Date().toISOString(), + }, + ], null, 2), + "utf-8" + ); + + // Step 4: Make the later index() rerun fail embedding for that retryable chunk + failEmbeddings = true; + const retryStats = await indexer.index(); + + // Step 5: Assert the failed batch persists AND the branch catalog still contains that chunk id + expect(retryStats.failedChunks).toBeGreaterThan(0); + + const failedBatchesAfter = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + chunks: Array<{ id: string }>; + error: string; + }>; + expect(failedBatchesAfter.length).toBeGreaterThan(0); + expect(failedBatchesAfter.some((batch) => batch.chunks.some((c) => c.id === existingChunkId))).toBe(true); + + const dbAfter = new Database(dbPath); + const branchChunksAfter = dbAfter.getBranchChunkIds(branchKey!); + expect(branchChunksAfter).toContain(existingChunkId); + }); + + it("restores recovered chunks to the branch catalog during retryFailedBatches()", async () => { + failEmbeddings = true; + const initialIndexer = createIndexer(); + const failedStats = await initialIndexer.index(); + expect(failedStats.failedChunks).toBeGreaterThan(0); + + const dbPath = path.join(tempDir, ".opencode", "index", "codebase.db"); + const dbBefore = new Database(dbPath); + const branchKey = "default"; + expect(dbBefore.getBranchChunkIds(branchKey)).toHaveLength(0); + + failEmbeddings = false; + const retryIndexer = createIndexer(); + const retry = await retryIndexer.retryFailedBatches(); + + expect(retry.failed).toBe(0); + expect(retry.remaining).toBe(0); + expect(retry.succeeded).toBeGreaterThan(0); + + const dbAfter = new Database(dbPath); + const branchChunksAfter = dbAfter.getBranchChunkIds(branchKey); + expect(branchChunksAfter.length).toBe(retry.succeeded); + for (const chunkId of branchChunksAfter) { + expect(dbAfter.getChunk(chunkId)).not.toBeNull(); + } + }); + + it("rolls back vectors and excludes failed chunks from branch when database upsert fails during index()", async () => { + const originalUpsert = Database.prototype.upsertEmbeddingsBatch; + let callCount = 0; + + vi.spyOn(Database.prototype, "upsertEmbeddingsBatch").mockImplementation( + function ( + this: Database, + items: Array<{ contentHash: string; embedding: Buffer; chunkText: string; model: string }> + ) { + callCount += 1; + if (callCount === 1) { + throw new Error("database write failed"); + } + return originalUpsert.call(this, items); + } + ); + + const indexer = createIndexer(); + const stats = await indexer.index(); + + expect(stats.failedChunks).toBeGreaterThan(0); + expect(stats.indexedChunks).toBe(0); + + const status = await indexer.getStatus(); + expect(status.indexed).toBe(false); + expect(status.failedBatchesCount).toBeGreaterThan(0); + + const dbPath = path.join(tempDir, ".opencode", "index", "codebase.db"); + const dbAfter = new Database(dbPath); + const branches = dbAfter.getAllBranches(); + const branchKey = branches.find((b) => b.includes("default")) || branches[0]; + if (branchKey) { + const branchChunks = dbAfter.getBranchChunkIds(branchKey); + expect(branchChunks.length).toBe(0); + } + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + if (fs.existsSync(failedBatchesPath)) { + const failedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + error: string; + }>; + expect(failedBatches.length).toBeGreaterThan(0); + expect(failedBatches[0]?.error).toContain("database write failed"); + } + }); + + it("rolls back vectors and excludes failed chunks from branch when database upsert fails during retryFailedBatches()", async () => { + const indexer = createIndexer(); + + failEmbeddings = true; + await indexer.index(); + + failEmbeddings = false; + + const originalUpsert = Database.prototype.upsertEmbeddingsBatch; + let callCount = 0; + + vi.spyOn(Database.prototype, "upsertEmbeddingsBatch").mockImplementation( + function ( + this: Database, + items: Array<{ contentHash: string; embedding: Buffer; chunkText: string; model: string }> + ) { + callCount += 1; + if (callCount === 1) { + throw new Error("database write failed during retry"); + } + return originalUpsert.call(this, items); + } + ); + + const retry = await indexer.retryFailedBatches(); + + expect(retry.succeeded).toBe(0); + expect(retry.failed).toBeGreaterThan(0); + + const status = await indexer.getStatus(); + expect(status.failedBatchesCount).toBeGreaterThan(0); + + const dbPath = path.join(tempDir, ".opencode", "index", "codebase.db"); + const dbAfter = new Database(dbPath); + const branches = dbAfter.getAllBranches(); + const branchKey = branches.find((b) => b.includes("default")) || branches[0]; + if (branchKey) { + const branchChunks = dbAfter.getBranchChunkIds(branchKey); + expect(branchChunks.length).toBe(0); + } + + const failedBatchesPath = path.join(tempDir, ".opencode", "index", "failed-batches.json"); + if (fs.existsSync(failedBatchesPath)) { + const failedBatches = JSON.parse(fs.readFileSync(failedBatchesPath, "utf-8")) as Array<{ + error: string; + }>; + expect(failedBatches.length).toBeGreaterThan(0); + expect(failedBatches[0]?.error).toContain("database write failed during retry"); + } + }); }); diff --git a/tests/native.test.ts b/tests/native.test.ts index 78e74a7b..a7f78c4a 100644 --- a/tests/native.test.ts +++ b/tests/native.test.ts @@ -8,6 +8,7 @@ import { hashContent, hashFile, VectorStore, + createEmbeddingTexts, createEmbeddingText, createDynamicBatches, generateChunkId, @@ -589,6 +590,84 @@ trait Timestampable { expect(batches.length).toBe(2); }); + + it("should respect maxBatchItems option", () => { + const chunks = [ + { text: "a".repeat(100), id: "1" }, + { text: "b".repeat(100), id: "2" }, + { text: "c".repeat(100), id: "3" }, + ]; + + const batches = createDynamicBatches(chunks, { maxBatchItems: 1 }); + + expect(batches).toHaveLength(3); + expect(batches.every((batch) => batch.length === 1)).toBe(true); + }); + + it("should respect maxBatchTokens override", () => { + const chunks = [ + { text: "a".repeat(1000), id: "1" }, + { text: "b".repeat(1000), id: "2" }, + ]; + + const batches = createDynamicBatches(chunks, { maxBatchTokens: 300 }); + + expect(batches).toHaveLength(2); + }); + }); + + describe("createEmbeddingText", () => { + it("should respect a lower max token override", () => { + const chunk: CodeChunk = { + content: "x".repeat(10000), + startLine: 1, + endLine: 50, + chunkType: "function", + name: "hugeChunk", + language: "typescript", + }; + + const text = createEmbeddingText(chunk, "/src/huge.ts", 256); + + expect(text.length).toBeLessThan(256 * 4 + 64); + expect(text).toContain("... [truncated]"); + }); + }); + + describe("createEmbeddingTexts", () => { + it("splits oversized chunks into multiple embedding texts with part markers", () => { + const chunk: CodeChunk = { + content: "x".repeat(8000), + startLine: 1, + endLine: 200, + chunkType: "function", + name: "hugeChunk", + language: "typescript", + }; + + const texts = createEmbeddingTexts(chunk, "/src/huge.ts", 256); + + expect(texts.length).toBeGreaterThan(1); + expect(texts[0]).toContain("Part 1/"); + expect(texts[1]).toContain("Part 2/"); + expect(texts.every((text) => text.length <= 256 * 4 + 128)).toBe(true); + }); + + it("returns a single text when the chunk fits the token budget", () => { + const chunk: CodeChunk = { + content: "function small() { return 1; }", + startLine: 1, + endLine: 3, + chunkType: "function", + name: "small", + language: "typescript", + }; + + const texts = createEmbeddingTexts(chunk, "/src/small.ts", 512); + + expect(texts).toHaveLength(1); + expect(texts[0]).not.toContain("Part 1/"); + }); }); describe("generateChunkId", () => {