diff --git a/src/lib/agents/search/api.ts b/src/lib/agents/search/api.ts index 924bc68f4..9f98d405d 100644 --- a/src/lib/agents/search/api.ts +++ b/src/lib/agents/search/api.ts @@ -49,13 +49,26 @@ class APISearchAgent { type: 'researchComplete', }); - const finalContext = - searchResults?.searchFindings - .map( - (f, index) => - `${f.content}`, - ) - .join('\n') || ''; + // Cap each result and total context to stay within reasonable token budgets + const maxCharsPerResult = 24000; + const maxTotalChars = 80000; + + let totalChars = 0; + const contextParts: string[] = []; + + if (searchResults?.searchFindings) { + for (let i = 0; i < searchResults.searchFindings.length; i++) { + const f = searchResults.searchFindings[i]; + const truncated = f.content.slice(0, maxCharsPerResult); + const part = `${truncated}`; + + if (totalChars + part.length > maxTotalChars) break; + totalChars += part.length; + contextParts.push(part); + } + } + + const finalContext = contextParts.join('\n'); const widgetContext = widgetOutputs .map((o) => { diff --git a/src/lib/agents/search/index.ts b/src/lib/agents/search/index.ts index 859183293..3711b2d27 100644 --- a/src/lib/agents/search/index.ts +++ b/src/lib/agents/search/index.ts @@ -98,13 +98,26 @@ class SearchAgent { type: 'researchComplete', }); - const finalContext = - searchResults?.searchFindings - .map( - (f, index) => - `${f.content}`, - ) - .join('\n') || ''; + // Cap each result and total context to stay within reasonable token budgets + const maxCharsPerResult = 24000; + const maxTotalChars = 80000; + + let totalChars = 0; + const contextParts: string[] = []; + + if (searchResults?.searchFindings) { + for (let i = 0; i < searchResults.searchFindings.length; i++) { + const f = searchResults.searchFindings[i]; + const truncated = f.content.slice(0, maxCharsPerResult); + const part = `${truncated}`; + + if (totalChars + part.length > maxTotalChars) break; + totalChars += part.length; + contextParts.push(part); + } + } + + const finalContext = contextParts.join('\n'); const widgetContext = widgetOutputs .map((o) => { diff --git a/src/lib/agents/search/researcher/actions/scrapeURL.ts b/src/lib/agents/search/researcher/actions/scrapeURL.ts index c702a7014..c7f297d43 100644 --- a/src/lib/agents/search/researcher/actions/scrapeURL.ts +++ b/src/lib/agents/search/researcher/actions/scrapeURL.ts @@ -3,6 +3,7 @@ import { ResearchAction } from '../../types'; import { Chunk, ReadingResearchBlock } from '@/lib/types'; import TurnDown from 'turndown'; import path from 'path'; +import { splitText } from '@/lib/utils/splitText'; const turndownService = new TurnDown(); @@ -40,11 +41,18 @@ const scrapeURLAction: ResearchAction = { params.urls.map(async (url) => { try { const res = await fetch(url); - const text = await res.text(); + let text = await res.text(); const title = text.match(/(.*?)<\/title>/i)?.[1] || `Content from ${url}`; + // Cap raw HTML before Turndown so we don't spend CPU converting + // megabytes of markup we'll mostly throw away after tokenization. + const maxHtmlChars = 200_000; + if (text.length > maxHtmlChars) { + text = text.slice(0, maxHtmlChars); + } + if ( !readingEmitted && researchBlock && @@ -110,8 +118,14 @@ const scrapeURLAction: ResearchAction<typeof schema> = { const markdown = turndownService.turndown(text); + // Limit scraped content to avoid blowing up the context window. + // splitText chunks by token count — we only keep the first chunk. + const maxTokensPerPage = 6000; + const chunks = splitText(markdown, maxTokensPerPage, 0); + const content = chunks.length > 0 ? chunks[0] : markdown; + results.push({ - content: markdown, + content, metadata: { url, title: title,