From 346c4c910a5438fa324cebf3c9cbf216eb77bed3 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 2 Oct 2025 09:53:58 +0200 Subject: [PATCH 1/7] fix: compact question->routing tests, prompt improvements Signed-off-by: anilb --- frontend/lib/chat/prompts/router.ts | 9 ++ frontend/lib/chat/tests/router.test.ts | 190 +++++++++---------------- 2 files changed, 76 insertions(+), 123 deletions(-) diff --git a/frontend/lib/chat/prompts/router.ts b/frontend/lib/chat/prompts/router.ts index 0c339535b..6f1e6147e 100644 --- a/frontend/lib/chat/prompts/router.ts +++ b/frontend/lib/chat/prompts/router.ts @@ -95,6 +95,15 @@ ${pipeToolQuestion} It must help the following agent to answer the question. You must capture the intent and leave no room for ambiguity. You can include things you learned/know, like country codes, timeframes, granularity, etc. +**CRITICAL: Disambiguate vague terms to prevent over-complicated queries:** +- "activity" or "commit activity" → reformulate as "commit count" (single metric, not lines changed/added/deleted) +- "fork activity" → reformulate as "fork count" +- "star activity" → reformulate as "star count" +- "PR activity" or "pull request activity" → reformulate as "pull request count" +- "issue activity" → reformulate as "issue count" +- **Default to COUNT as the metric unless user explicitly asks for detailed metrics like "lines changed", "lines added", "lines deleted", "files modified", etc.** +- Example: "Show me commit activity by company" → reformulate as "Show me commit count grouped by company" + ## Reasoning: It must be something user-friendly. - If the action is "stop", the reasoning must be something like "I'm unable to answer this question with the available data sources, I am missing access to {DATA, explained in non-technical natural language}. If this looks like a mistake, please contact us." diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index 68645b4b8..e0731265c 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -105,7 +105,7 @@ describe('Router Agent', () => { function createTestInput(userQuery: string): RouterAgentInput { const messages: ChatMessage[] = [{ role: 'user', content: userQuery }] - console.warn("📝 Creating test input for query:", userQuery) + console.warn('📝 Creating test input for query:', userQuery) return { model, @@ -141,7 +141,6 @@ describe('Router Agent', () => { expect(router.temperature).toBe(0) }) - test('should validate output schema for both implementations', () => { const originalRouter = new RouterAgent() @@ -179,133 +178,78 @@ describe('Router Agent', () => { describe('Real AI routing decisions', () => { describe('PIPES routing', () => { - test('should route activity queries correctly to PIPE', async () => { - if (skipIfNoCredentials()) return - - const router = new RouterAgent() - const input = createTestInput('Show me commits this week') - - console.warn('🤖 Sending query to router agent: Show me commits this week') - const result = await router.execute(input) - console.warn('🤖 Router agent response:', result) - - expect(result.next_action).toBeDefined() - expect(Object.values(RouterDecisionAction)).toContain(result.next_action) - expect(RouterDecisionAction.PIPES).toBe(result.next_action) - expect(result.reasoning).toBeTruthy() - expect(result.reformulated_question).toBeTruthy() - expect(Array.isArray(result.tools)).toBe(true) - expect(result.usage.totalTokens).toBeGreaterThan(0) - - console.warn(`🔍 Activity query routed to: ${result.next_action}`) - console.warn(`🔍 Reasoning: ${result.reasoning}`) - }, 15000) - - test('should route stars query for previous week to PIPE', async () => { - if (skipIfNoCredentials()) return - - const router = new RouterAgent() - const input = createTestInput('Show me stars for the previous week') - - const result = await router.execute(input) - - expect(result.next_action).toBe(RouterDecisionAction.PIPES) - expect(result.reasoning).toBeTruthy() - expect(result.reformulated_question).toBeTruthy() - expect(Array.isArray(result.tools)).toBe(true) - - console.warn(`🔍 Stars query routed to: ${result.next_action}`) - console.warn(`🔍 Reasoning: ${result.reasoning}`) - }, 15000) - - test('should route forks query for last year to PIPE', async () => { - if (skipIfNoCredentials()) return - - const router = new RouterAgent() - const input = createTestInput('Show me forks for the last year') - - const result = await router.execute(input) - - expect(result.next_action).toBe(RouterDecisionAction.PIPES) - expect(result.reasoning).toBeTruthy() - expect(result.reformulated_question).toBeTruthy() - expect(Array.isArray(result.tools)).toBe(true) - - console.warn(`🔍 Forks query routed to: ${result.next_action}`) - console.warn(`🔍 Reasoning: ${result.reasoning}`) - }, 15000) - - test('should route companies contributing query to PIPE', async () => { - if (skipIfNoCredentials()) return - - const router = new RouterAgent() - const input = createTestInput('List of companies contributing in project') - - const result = await router.execute(input) - - expect(result.next_action).toBe(RouterDecisionAction.PIPES) - expect(result.reasoning).toBeTruthy() - expect(result.reformulated_question).toBeTruthy() - expect(Array.isArray(result.tools)).toBe(true) - - console.warn(`🔍 Companies query routed to: ${result.next_action}`) - console.warn(`🔍 Reasoning: ${result.reasoning}`) - }, 15000) - - test('should route geographical queries correctly', async () => { - if (skipIfNoCredentials()) return - - const router = new RouterAgent() - const input = createTestInput('Show me contributors from Brazil') - - const result = await router.execute(input) - - expect(result.next_action).toBeDefined() - expect(Object.values(RouterDecisionAction)).toContain(result.next_action) - expect(result.reasoning).toBeTruthy() - expect(result.reformulated_question).toContain('Brazil') - - console.warn(`🔍 Geographic query routed to: ${result.next_action}`) - console.warn(`🔍 Reasoning: ${result.reasoning}`) - }, 15000) + test.each([ + 'Show me commits this week', + 'Show me stars for the previous week', + 'Show me forks for the last year', + 'List of companies contributing in project', + ])( + 'should route "%s" to PIPES', + async (query) => { + if (skipIfNoCredentials()) return + + console.warn(`🤖 Testing query: "${query}"`) + const router = new RouterAgent() + const input = createTestInput(query) + const result = await router.execute(input) + + expect(result.next_action).toBe(RouterDecisionAction.PIPES) + expect(result.reasoning).toBeTruthy() + expect(result.reformulated_question).toBeTruthy() + expect(Array.isArray(result.tools)).toBe(true) + + console.warn(`✅ "${query}" → ${result.next_action}`) + console.warn(`🔍 Reasoning: ${result.reasoning}`) + }, + 15000, + ) }) describe('CREATE_QUERY (TEXT_TO_SQL) routing', () => { - test('should route commit activity by company query to CREATE_QUERY', async () => { - if (skipIfNoCredentials()) return - - const router = new RouterAgent() - const input = createTestInput('Show me commit activity by company over all time period') - - const result = await router.execute(input) - - expect(result.next_action).toBe(RouterDecisionAction.CREATE_QUERY) - expect(result.reasoning).toBeTruthy() - expect(result.reformulated_question).toBeTruthy() - - console.warn(`🔍 Commit activity by company query routed to: ${result.next_action}`) - console.warn(`🔍 Reasoning: ${result.reasoning}`) - }, 15000) + test.each(['Show me commit activity by company over all time period'])( + 'should route "%s" to CREATE_QUERY', + async (query) => { + if (skipIfNoCredentials()) return + + console.warn(`🤖 Testing query: "${query}"`) + const router = new RouterAgent() + const input = createTestInput(query) + const result = await router.execute(input) + + expect(result.next_action).toBe(RouterDecisionAction.CREATE_QUERY) + expect(result.reasoning).toBeTruthy() + expect(result.reformulated_question).toBeTruthy() + + console.warn(`✅ "${query}" → ${result.next_action}`) + console.warn(`🔍 Reasoning: ${result.reasoning}`) + }, + 15000, + ) }) describe('STOP routing', () => { - test('should route impossible queries to STOP', async () => { - if (skipIfNoCredentials()) return - - const router = new RouterAgent() - const input = createTestInput("What's the weather forecast for contributors?") - - const result = await router.execute(input) - - expect(result.next_action).toBe(RouterDecisionAction.STOP) - expect(result.reasoning).toContain('weather') - expect(result.tools).toEqual([]) - - console.warn(`🔍 Weather query routed to: ${result.next_action}`) - console.warn(`🔍 Reasoning: ${result.reasoning}`) - }, 15000) + test.each([ + "What's the weather forecast for contributors?", + "Show me contributors from Brazil", + ])( + 'should route "%s" to STOP', + async (query) => { + if (skipIfNoCredentials()) return + + console.warn(`🤖 Testing query: "${query}"`) + const router = new RouterAgent() + const input = createTestInput(query) + const result = await router.execute(input) + + expect(result.next_action).toBe(RouterDecisionAction.STOP) + expect(result.reasoning).toBeTruthy() + expect(result.tools).toEqual([]) + + console.warn(`✅ "${query}" → ${result.next_action}`) + console.warn(`🔍 Reasoning: ${result.reasoning}`) + }, + 15000, + ) }) - }) - }) From 6992159d715c379afc664590ef59560e2c9d27a1 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 2 Oct 2025 16:25:35 +0200 Subject: [PATCH 2/7] feat: clarification questions by router agent Signed-off-by: anilb --- ..._addAskClarificationRouterResponseType.sql | 9 ++ .../copilot/store/copilot.api.service.ts | 7 +- .../modules/copilot/types/copilot.types.ts | 9 +- frontend/lib/chat/agents/router.ts | 1 + frontend/lib/chat/data-copilot.ts | 139 +++++++++++++++++- frontend/lib/chat/enums.ts | 2 + frontend/lib/chat/prompts/router.ts | 32 +++- frontend/lib/chat/tests/router.test.ts | 27 ++++ frontend/lib/chat/types.ts | 9 +- frontend/server/api/chat/stream.ts | 13 +- frontend/server/repo/chat.repo.ts | 42 ++++-- 11 files changed, 260 insertions(+), 30 deletions(-) create mode 100644 database/migrations/V1759392166__addAskClarificationRouterResponseType.sql diff --git a/database/migrations/V1759392166__addAskClarificationRouterResponseType.sql b/database/migrations/V1759392166__addAskClarificationRouterResponseType.sql new file mode 100644 index 000000000..84fb0e825 --- /dev/null +++ b/database/migrations/V1759392166__addAskClarificationRouterResponseType.sql @@ -0,0 +1,9 @@ +-- Drop the existing check constraint +ALTER TABLE chat_responses DROP CONSTRAINT chat_responses_router_response_check; + +-- Add the new check constraint with 'ask_clarification' +ALTER TABLE chat_responses ADD CONSTRAINT chat_responses_router_response_check + CHECK (router_response IN ('pipes', 'create_query', 'stop', 'ask_clarification')); + +-- Add clarification_question column to store the clarification question +ALTER TABLE chat_responses ADD COLUMN IF NOT EXISTS clarification_question TEXT; diff --git a/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts b/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts index a841631ad..4ffb1ae5c 100644 --- a/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts +++ b/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts @@ -229,7 +229,7 @@ class CopilotApiService { if ( data.type === 'router-status' && - (data.status === 'complete' || data.status === 'error') + (data.status === 'complete' || data.status === 'error' || data.status === 'ask_clarification') ) { if (!assistantMessageId) { assistantMessageId = this.generateId() @@ -240,9 +240,10 @@ class CopilotApiService { role: 'assistant', type: 'router-status', status: data.status, - content: data.reasoning || '', + content: data.status === 'ask_clarification' ? (data.question || '') : (data.reasoning || ''), explanation: data.status === 'error' ? data.error : undefined, routerReasoning: data.reasoning, + question: data.question, // Include the clarification question timestamp: Date.now(), }, -1, @@ -353,6 +354,8 @@ class CopilotApiService { return 'Analyzing your question...' case 'complete': return reasoning ? `Analysis: ${reasoning}` : 'Analysis complete' + case 'ask_clarification': + return reasoning || 'I need more information to answer your question.' default: return `Error: ${error || 'An error occurred'}` } diff --git a/frontend/app/components/shared/modules/copilot/types/copilot.types.ts b/frontend/app/components/shared/modules/copilot/types/copilot.types.ts index b6a575e2a..2fcbbf481 100644 --- a/frontend/app/components/shared/modules/copilot/types/copilot.types.ts +++ b/frontend/app/components/shared/modules/copilot/types/copilot.types.ts @@ -21,7 +21,7 @@ export type MessagePartType = 'router-status' | 'pipe-result' | 'chat-response-id' | 'conversation-id'; -export type MessageStatus = 'analyzing' | 'complete' | 'error'; +export type MessageStatus = 'analyzing' | 'complete' | 'error' | 'ask_clarification'; export type MessageData = Record; export type MessageRole = 'user' | 'assistant'; @@ -38,17 +38,18 @@ export interface ResultsHistory { } export interface AIMessage { - id: string; - role: MessageRole; + id: string; + role: MessageRole; type: MessagePartType; status: MessageStatus; - content: string; + content: string; timestamp: number; conversationId?: string; sql?: string; data?: Array; explanation?: string; routerReasoning?: string; + question?: string; // Clarification question when status is 'ask_clarification' instructions?: { pipes: Array<{ id: string diff --git a/frontend/lib/chat/agents/router.ts b/frontend/lib/chat/agents/router.ts index d7a7a01fc..1dbefae1d 100644 --- a/frontend/lib/chat/agents/router.ts +++ b/frontend/lib/chat/agents/router.ts @@ -24,6 +24,7 @@ export class RouterAgent extends BaseAgent { input.parametersString, input.segmentId, input.toolsOverview, + input.previousWasClarification, ) } diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 5965ac465..9ecd8f763 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -15,6 +15,7 @@ import { TextToSqlAgent, PipeAgent, RouterAgent } from './agents' import { executePipeInstructions, executeTextToSqlInstructions } from './instructions' import type { AgentResponseCompleteParams, + ChatMessage, DataCopilotQueryInput, PipeAgentInput, PipeAgentStreamInput, @@ -147,6 +148,7 @@ export class DataCopilot { pipe, parametersString, segmentId, + previousWasClarification, }: Omit) { const agent = new RouterAgent() return agent.execute({ @@ -159,6 +161,7 @@ export class DataCopilot { pipe, parametersString, segmentId, + previousWasClarification, }) } @@ -272,11 +275,79 @@ export class DataCopilot { }) } + /** + * Build messages array from conversation history + * Handles clarification merging if the previous response was ASK_CLARIFICATION + */ + private async buildMessagesFromConversation( + currentQuestion: string, + conversationId: string | undefined, + insightsDbPool: Pool, + ): Promise<{ messages: ChatMessage[]; previousWasClarification: boolean }> { + const chatRepo = new ChatRepository(insightsDbPool) + + if (!conversationId) { + // No conversation history, just return the current question + return { + messages: [{ role: 'user', content: currentQuestion }], + previousWasClarification: false, + } + } + + const previousChatResponses = await chatRepo.getChatResponsesByConversation(conversationId) + + if (previousChatResponses.length === 0) { + // No previous responses in this conversation + return { + messages: [{ role: 'user', content: currentQuestion }], + previousWasClarification: false, + } + } + + // Check if the latest response was ASK_CLARIFICATION + const latestResponse = previousChatResponses[previousChatResponses.length - 1] as ChatResponse + const previousWasClarification = latestResponse.routerResponse === RouterDecisionAction.ASK_CLARIFICATION + + if (previousWasClarification) { + // Merge the clarification: combine the ambiguous question with the clarification answer + const ambiguousQuestion = latestResponse.userPrompt + const mergedQuestion = `Original question: ${ambiguousQuestion}\n\nClarification provided: ${currentQuestion}` + + // Build messages: [older history before clarification] + [merged question] + const messages = previousChatResponses.slice(0, -1).map((response) => ({ + role: 'user' as const, + content: response.userPrompt, + })) + + // Add the merged question as the current message + messages.push({ + role: 'user', + content: mergedQuestion, + }) + + return { messages, previousWasClarification: true } + } + + // Normal case: build messages from all previous responses + current question + const messages = previousChatResponses.map((response) => ({ + role: 'user' as const, + content: response.userPrompt, + })) + + // Add the current question + messages.push({ + role: 'user', + content: currentQuestion, + }) + + return { messages, previousWasClarification: false } + } + /** * Main streaming handler that orchestrates the entire AI agent workflow */ async streamingAgentRequestHandler({ - messages, + currentQuestion, segmentId, projectName, pipe, @@ -289,8 +360,15 @@ export class DataCopilot { const parametersString = JSON.stringify(parameters || {}) const date = new Date().toISOString().slice(0, 10) + // Build messages from conversation history + const { messages, previousWasClarification } = await this.buildMessagesFromConversation( + currentQuestion, + conversationId, + insightsDbPool, + ) + const responseData: ChatResponse = { - userPrompt: messages[messages.length - 1]?.content || '', + userPrompt: currentQuestion, inputTokens: 0, outputTokens: 0, model: this.BEDROCK_MODEL_ID, @@ -315,6 +393,7 @@ export class DataCopilot { pipe, parametersString, segmentId: segmentId as string, + previousWasClarification, }) // Accumulate token usage from router @@ -336,6 +415,19 @@ export class DataCopilot { return } + if (routerOutput.next_action === RouterDecisionAction.ASK_CLARIFICATION) { + await this.handleAskClarificationAction( + messages[messages.length - 1]?.content || '', + routerOutput, + responseData, + dataStream, + insightsDbPool, + userEmail, + conversationId, + ) + return + } + dataStream.writeData({ type: StreamDataType.ROUTER_STATUS, status: StreamDataStatus.COMPLETE, @@ -437,6 +529,49 @@ export class DataCopilot { }) } + /** + * Handle router 'ask_clarification' action - ask user for clarification + */ + private async handleAskClarificationAction( + userPrompt: string, + routerOutput: RouterOutput, + responseData: ChatResponse, + dataStream: any, + insightsDbPool: Pool, + userEmail: string, + conversationId?: string, + ): Promise { + dataStream.writeData({ + type: StreamDataType.ROUTER_STATUS, + status: StreamDataStatus.ASK_CLARIFICATION, + question: routerOutput.clarification_question, + reasoning: routerOutput.reasoning, + }) + + const chatResponseId = await this.saveChatResponse( + { + userPrompt, + inputTokens: responseData.inputTokens, + outputTokens: responseData.outputTokens, + routerResponse: RouterDecisionAction.ASK_CLARIFICATION, + routerReason: routerOutput.reasoning, + clarificationQuestion: routerOutput.clarification_question || undefined, + pipeInstructions: undefined, + sqlQuery: undefined, + model: this.BEDROCK_MODEL_ID, + conversationId: conversationId, + }, + insightsDbPool, + userEmail, + ) + + dataStream.writeData({ + type: StreamDataType.CHAT_RESPONSE_ID, + id: chatResponseId, + conversationId: conversationId || '', + }) + } + /** * Handle router 'create_query' action - generate and execute SQL query */ diff --git a/frontend/lib/chat/enums.ts b/frontend/lib/chat/enums.ts index 32192e973..82b515c2f 100644 --- a/frontend/lib/chat/enums.ts +++ b/frontend/lib/chat/enums.ts @@ -10,10 +10,12 @@ export enum StreamDataType { export enum StreamDataStatus { ANALYZING = 'analyzing', COMPLETE = 'complete', + ASK_CLARIFICATION = 'ask_clarification', } export enum RouterDecisionAction { STOP = 'stop', CREATE_QUERY = 'create_query', PIPES = 'pipes', + ASK_CLARIFICATION = 'ask_clarification', } diff --git a/frontend/lib/chat/prompts/router.ts b/frontend/lib/chat/prompts/router.ts index 6f1e6147e..0b1c15e38 100644 --- a/frontend/lib/chat/prompts/router.ts +++ b/frontend/lib/chat/prompts/router.ts @@ -9,6 +9,7 @@ export const routerPrompt = ( parametersString: string, segmentId: string | null, toolsOverview: string, + previousWasClarification?: boolean, ) => { const dashboardDescription = pipe ? `Project "${projectName}" using ${pipe} tool with parameters: ${parametersString}` @@ -16,7 +17,21 @@ export const routerPrompt = ( const pipeToolQuestion = pipe ? `- Can ${pipe} tool answer this with different parameters?` : '' - return `You are a routing agent that analyzes user questions and determines the appropriate next action. Your job is to evaluate questions and decide whether they can be answered with existing tools, need custom queries, or cannot be answered. + const clarificationWarning = previousWasClarification + ? `\n\n## ⚠️ IMPORTANT: CLARIFICATION LOOP PREVENTION +**The previous response was ASK_CLARIFICATION. You have already asked the user for clarification once.** + +CRITICAL RULES: +- **STRONGLY PREFER** answering the question with available data or tools over asking for another clarification +- **ONLY** ask for clarification again if the question is still completely impossible to interpret +- Make reasonable assumptions based on the context and user's clarification +- If you can answer the question with ANY interpretation, choose "pipes" or "create_query" instead of "ask_clarification" +- **NEVER** create a clarification loop by repeatedly asking for clarification + +The user has already provided clarification. Work with what you have.` + : '' + + return `You are a routing agent that analyzes user questions and determines the appropriate next action. Your job is to evaluate questions and decide whether they can be answered with existing tools, need custom queries, or cannot be answered.${clarificationWarning} # DATE AND CONTEXT Today's date: ${date} @@ -72,7 +87,8 @@ ${pipeToolQuestion} **Step 2: Check Data Sources (only if Step 1 is NO)** - Use list_datasources to examine available tables and fields - Check if the required fields exist in any data source -- Pay special attention to the activityRelations_deduplicated_cleaned_ds table +- Pay special attention to the pull_requests_analyzed, issues_analyzed, activityRelations_deduplicated_cleaned_ds tables +- If something can be answered by avoiding using activityRelations_deduplicated_cleaned_ds, prefer that (e.g., use pull_requests_analyzed for PR counts, issues_analyzed for issue counts, etc.) - If the needed fields exist → Question is VALID, route to "create_query" action - If fields don't exist → Question is INVALID, route to "stop" action - If the question is referencing a field about contributors/people that we have only for organizations, the question is INVALID @@ -81,6 +97,10 @@ ${pipeToolQuestion} - "stop": The question cannot be answered with available data - "create_query": Custom SQL query needed using available data sources (tools can be empty) - "pipes": Existing tools can answer the question (specify which tools in the tools array) +- "ask_clarification": The question is ambiguous or missing critical information. Ask user for clarification before proceeding. + * Use this when: timeframe is unclear, metrics are ambiguous, grouping/dimension is not specified, or multiple interpretations exist + * The clarification_question must be specific, user-friendly, and guide the user toward providing the missing information + * Example: "I need to know the time period for this analysis. Should I show data for the last week, month, or year?" # IMPORTANT - Always check data availability before routing @@ -105,9 +125,15 @@ You can include things you learned/know, like country codes, timeframes, granula - Example: "Show me commit activity by company" → reformulate as "Show me commit count grouped by company" ## Reasoning: -It must be something user-friendly. +It must be something user-friendly. - If the action is "stop", the reasoning must be something like "I'm unable to answer this question with the available data sources, I am missing access to {DATA, explained in non-technical natural language}. If this looks like a mistake, please contact us." - If the action is "create_query", the reasoning must be something like "I'll create a query to answer the question." - If the action is "pipes", the reasoning must be something like "I'll use the widgets and to answer the question." +- If the action is "ask_clarification", the reasoning must explain what information is missing or ambiguous. + +## Clarification Question (only for ask_clarification action): +- Must be a clear, specific question that helps the user provide the missing information +- Should offer options or examples when appropriate +- Must be conversational and friendly ` } diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index e0731265c..9267642c5 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -251,5 +251,32 @@ describe('Router Agent', () => { 15000, ) }) + + describe('ASK_CLARIFICATION routing', () => { + test.each([ + "Show me the activity", + "Give me stats for last period", + "Show me metrics", + ])( + 'should route "%s" to ASK_CLARIFICATION', + async (query) => { + if (skipIfNoCredentials()) return + + console.warn(`🤖 Testing query: "${query}"`) + const router = new RouterAgent() + const input = createTestInput(query) + const result = await router.execute(input) + + expect(result.next_action).toBe(RouterDecisionAction.ASK_CLARIFICATION) + expect(result.reasoning).toBeTruthy() + expect(result.clarification_question).toBeTruthy() + + console.warn(`✅ "${query}" → ${result.next_action}`) + console.warn(`🔍 Reasoning: ${result.reasoning}`) + console.warn(`❓ Clarification: ${result.clarification_question}`) + }, + 15000, + ) + }) }) }) diff --git a/frontend/lib/chat/types.ts b/frontend/lib/chat/types.ts index 24f41cea5..cbc434217 100644 --- a/frontend/lib/chat/types.ts +++ b/frontend/lib/chat/types.ts @@ -94,10 +94,16 @@ export const routerOutputSchema = z.object({ RouterDecisionAction.STOP, RouterDecisionAction.CREATE_QUERY, RouterDecisionAction.PIPES, + RouterDecisionAction.ASK_CLARIFICATION, ]), reasoning: z.string().describe('Maximum 2 sentences explaining the decision'), reformulated_question: z.string().describe('Enhanced query with all parameters'), tools: z.array(z.string()).describe('Tools needed for next agent'), + clarification_question: z + .string() + .optional() + .nullable() + .describe('Question to ask user when next_action is ASK_CLARIFICATION'), }) // Pipe agent output schema @@ -131,6 +137,7 @@ export interface RouterAgentInput { pipe: string parametersString: string segmentId: string | null + previousWasClarification?: boolean } export interface PipeAgentStreamInput extends Omit { @@ -154,7 +161,7 @@ export interface PipeAgentInput { } export interface DataCopilotQueryInput { - messages: ChatMessage[] + currentQuestion: string // The current user question segmentId?: string projectName?: string pipe: string diff --git a/frontend/server/api/chat/stream.ts b/frontend/server/api/chat/stream.ts index 048891d51..9464e4f1b 100644 --- a/frontend/server/api/chat/stream.ts +++ b/frontend/server/api/chat/stream.ts @@ -4,12 +4,11 @@ import type { Pool } from 'pg' import { createDataStreamResponse } from 'ai' import { DataCopilot } from '~~/lib/chat/data-copilot' import { InsightsProjectsRepository } from '~~/server/repo/insightsProjects.repo' -import { ChatMessage } from '~~/lib/chat/types' export const maxDuration = 30 interface IStreamRequestBody { - messages: ChatMessage[] + messages: Array<{ role: 'user' | 'assistant'; content: string }> projectSlug?: string projectName?: string pipe: string @@ -34,6 +33,12 @@ export default defineEventHandler(async (event): Promise => { return createError({ statusCode: 400, statusMessage: 'Project slug is required' }) } + const question = messages?.filter((m) => m.role === 'user').pop()?.content + + if (!question) { + return createError({ statusCode: 400, statusMessage: 'Question is required' }) + } + // Generate conversationId if not provided const finalConversationId = conversationId || crypto.randomUUID() @@ -55,7 +60,7 @@ export default defineEventHandler(async (event): Promise => { return createDataStreamResponse({ execute: async (dataStream) => { await dataCopilot.streamingAgentRequestHandler({ - messages, + currentQuestion: question, segmentId: insightsProjects.segmentId, projectName, pipe, @@ -63,7 +68,7 @@ export default defineEventHandler(async (event): Promise => { conversationId: finalConversationId, insightsDbPool, userEmail: event.context.user.email, - dataStream, // Pass the dataStream to the class + dataStream, }) }, }) diff --git a/frontend/server/repo/chat.repo.ts b/frontend/server/repo/chat.repo.ts index da56fe989..d4b3eeeee 100644 --- a/frontend/server/repo/chat.repo.ts +++ b/frontend/server/repo/chat.repo.ts @@ -50,6 +50,7 @@ export interface ChatResponse { routerReason: string pipeInstructions?: PipeInstructions sqlQuery?: string + clarificationQuestion?: string model: string inputTokens?: number outputTokens?: number @@ -61,21 +62,22 @@ export class ChatRepository { async saveChatResponse(response: ChatResponse, userEmail: string): Promise { try { const query = ` - INSERT INTO chat_responses - ( - created_by, + INSERT INTO chat_responses + ( + created_by, user_prompt, - router_response, - router_reason, - pipe_instructions, - sql_query, - model, - input_tokens, - output_tokens, + router_response, + router_reason, + pipe_instructions, + sql_query, + clarification_question, + model, + input_tokens, + output_tokens, feedback, conversation_id ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id ` @@ -86,6 +88,7 @@ export class ChatRepository { response.routerReason, response.pipeInstructions ? JSON.stringify(response.pipeInstructions) : null, response.sqlQuery, + response.clarificationQuestion, response.model, response.inputTokens, response.outputTokens, @@ -122,13 +125,24 @@ export class ChatRepository { async getLatestChatResponseByConversation(conversationId: string): Promise { const query = ` - SELECT * FROM chat_responses - WHERE conversation_id = $1 - ORDER BY created_at DESC + SELECT * FROM chat_responses + WHERE conversation_id = $1 + ORDER BY created_at DESC LIMIT 1 ` const result = await this.pool.query(query, [conversationId]) return result.rows.length > 0 ? result.rows[0] : null } + + async getChatResponsesByConversation(conversationId: string): Promise { + const query = ` + SELECT * FROM chat_responses + WHERE conversation_id = $1 + ORDER BY created_at ASC + ` + + const result = await this.pool.query(query, [conversationId]) + return result.rows + } } From c98ea36af56c700d6d9daf67c1259f16596a0d7c Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 6 Oct 2025 16:04:47 +0200 Subject: [PATCH 3/7] feat: auditor agent and router agent ask_clarification mode Signed-off-by: anilb --- .../copilot/store/copilot.api.service.ts | 45 +- .../shared/modules/copilot/store/test.json | 60 --- .../shared/modules/copilot/store/test2.json | 378 -------------- .../shared/modules/copilot/store/test3.json | 264 ---------- frontend/lib/chat/agents/auditor.ts | 43 ++ frontend/lib/chat/agents/base-agent.ts | 4 +- frontend/lib/chat/agents/index.ts | 3 +- frontend/lib/chat/agents/text-to-sql.ts | 1 + frontend/lib/chat/data-copilot.ts | 466 ++++++++++++------ frontend/lib/chat/enums.ts | 5 + frontend/lib/chat/prompts/auditor.ts | 146 ++++++ frontend/lib/chat/prompts/router.ts | 13 +- frontend/lib/chat/tests/auditor.test.ts | 187 +++++++ frontend/lib/chat/types.ts | 24 +- frontend/lib/chat/utils/data-summary.ts | 135 +++++ frontend/server/repo/chat.repo.ts | 20 +- 16 files changed, 937 insertions(+), 857 deletions(-) delete mode 100644 frontend/app/components/shared/modules/copilot/store/test.json delete mode 100644 frontend/app/components/shared/modules/copilot/store/test2.json delete mode 100644 frontend/app/components/shared/modules/copilot/store/test3.json create mode 100644 frontend/lib/chat/agents/auditor.ts create mode 100644 frontend/lib/chat/prompts/auditor.ts create mode 100644 frontend/lib/chat/tests/auditor.test.ts create mode 100644 frontend/lib/chat/utils/data-summary.ts diff --git a/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts b/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts index 4ffb1ae5c..65435375c 100644 --- a/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts +++ b/frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts @@ -10,11 +10,8 @@ import type { MessageStatus, } from '../types/copilot.types' import type { CopilotParams } from '../types/copilot.types' -// import testData from './test.json' -import testData3 from './test3.json' import type { Project } from '~~/types/project' -export const tempData = testData3 as AIMessage[] class CopilotApiService { // Generate unique ID for messages generateId = () => Date.now().toString(36) + Math.random().toString(36).substring(2); @@ -228,19 +225,38 @@ class CopilotApiService { statusCallBack(statusText) if ( - data.type === 'router-status' && - (data.status === 'complete' || data.status === 'error' || data.status === 'ask_clarification') + (data.type === 'router-status' || data.type === 'auditor-status') && + ( + data.status === 'complete' || + data.status === 'error' || + data.status === 'ask_clarification' || + data.status === 'validated' + ) ) { if (!assistantMessageId) { assistantMessageId = this.generateId() + let content: string + if (data.status === 'ask_clarification') { + content = data.question || 'I need more information to answer your question.' + } + else if (data.status === 'error') { + content = data.error || 'An error occurred.' + } + else if (data.status === 'validated') { + content = data.summary || 'Data validated successfully.' + } + else { + content = data.reasoning || 'Analysis complete.' + } + messageCallBack( { id: assistantMessageId, role: 'assistant', type: 'router-status', status: data.status, - content: data.status === 'ask_clarification' ? (data.question || '') : (data.reasoning || ''), + content, explanation: data.status === 'error' ? data.error : undefined, routerReasoning: data.reasoning, question: data.question, // Include the clarification question @@ -339,6 +355,8 @@ class CopilotApiService { switch (type) { case 'router-status': return this.getStatusTextRouterStatus(status, reasoning, error) + case 'auditor-status': + return this.getStatusTextAuditorStatus(status, reasoning) case 'sql-result': return 'SQL query executed successfully' case 'pipe-result': @@ -360,6 +378,21 @@ class CopilotApiService { return `Error: ${error || 'An error occurred'}` } } + + getStatusTextAuditorStatus(status: string, reasoning: string): string { + switch (status) { + case 'validating': + return 'Validating data quality...' + case 'validated': + return reasoning ? `Validation passed: ${reasoning}` : 'Data validated successfully' + case 'retrying': + return 'Retrying with improved query...' + case 'max_retries': + return reasoning ? `Validation feedback: ${reasoning}` : 'Maximum validation attempts reached' + default: + return 'Validating...' + } + } } export const copilotApiService = new CopilotApiService() diff --git a/frontend/app/components/shared/modules/copilot/store/test.json b/frontend/app/components/shared/modules/copilot/store/test.json deleted file mode 100644 index 6ef869653..000000000 --- a/frontend/app/components/shared/modules/copilot/store/test.json +++ /dev/null @@ -1,60 +0,0 @@ -[ - { - "id": "me2kfiwi0izfbw1epv7h", - "role": "user", - "type": "text", - "status": "complete", - "content": "Who are the top 3 contributors?", - "timestamp": 1754641811826 - }, - { - "id": "me2kfxz22tj6e3jpky9", - "role": "assistant", - "type": "router-status", - "status": "complete", - "content": "I'll create a query to identify the top 3 contributors by activity count for the Kubernetes project. The activityRelations_deduplicated_cleaned_ds table contains the necessary fields (memberId, username, segmentId) to rank contributors by their contribution activity.", - "timestamp": 1754641831358 - }, - { - "id": "me2kg525frdy8woetmu", - "role": "assistant", - "type": "sql-result", - "content": "This query finds the top 3 contributors by counting all activities per username in the activityRelations_deduplicated_cleaned_ds table, filtered by the segment ID for Kubernetes project and the specified date range, then ordering by activity count in descending order.", - "timestamp": 1754641840541, - "sql": "SELECT \n username,\n COUNT(*) as activity_count\nFROM activityRelations_deduplicated_cleaned_ds\nWHERE \n segmentId = 'd3e1742e-253b-4a16-af63-b50e15a5e505'\n AND timestamp >= '2024-07-23 00:00:00'\n AND timestamp <= '2025-07-23 23:59:59'\n AND username IS NOT NULL\n AND username != ''\nGROUP BY username\nORDER BY activity_count DESC\nLIMIT 3", - "data": [ - { "username": "k8s-ci-robot", "activity_count": 96909 }, - { "username": "dims", "activity_count": 4081 }, - { "username": "liggitt", "activity_count": 3976 } - ] - }, - { - "id": "me2kgkyrpqpvry4bfy", - "role": "user", - "type": "text", - "status": "complete", - "content": "Can you remove the username k8s-ci-robot", - "timestamp": 1754641861155 - }, - { - "id": "me2kgu1jihu6j0vivys", - "role": "assistant", - "type": "router-status", - "status": "complete", - "content": "I'll create a query to find the top 3 contributors excluding the k8s-ci-robot username from the results.", - "timestamp": 1754641872919 - }, - { - "id": "me2kgzzcd3pr7aa9m9p", - "role": "assistant", - "type": "sql-result", - "content": "This query finds the top 3 contributors by counting all activities per username in the Kubernetes project, excluding 'k8s-ci-robot' using a NOT EQUAL filter, and ordering by activity count in descending order within the specified date range.", - "timestamp": 1754641880616, - "sql": "SELECT username, COUNT(*) as activity_count FROM activityRelations_deduplicated_cleaned_ds WHERE segmentId = 'd3e1742e-253b-4a16-af63-b50e15a5e505' AND timestamp >= '2024-07-23 23:59:59' AND timestamp <= '2025-07-23 23:59:59' AND username != 'k8s-ci-robot' GROUP BY username ORDER BY activity_count DESC LIMIT 3", - "data": [ - { "username": "dependabot[bot]", "activity_count": 25033 }, - { "username": "k8s-infra-cherrypick-robot", "activity_count": 20076 }, - { "username": "dims", "activity_count": 6418 } - ] - } -] diff --git a/frontend/app/components/shared/modules/copilot/store/test2.json b/frontend/app/components/shared/modules/copilot/store/test2.json deleted file mode 100644 index b3e6fed63..000000000 --- a/frontend/app/components/shared/modules/copilot/store/test2.json +++ /dev/null @@ -1,378 +0,0 @@ -[ - { - "id": "me8389ggz916xpsvo6", - "role": "user", - "type": "text", - "status": "complete", - "content": "Give me the top 5 contributors per month", - "timestamp": 1754975716576 - }, - { - "id": "me838imbz4vxu7ph2pk", - "role": "assistant", - "type": "router-status", - "status": "complete", - "content": "I need to create a custom SQL query to get the top 5 contributors per month for the Kubernetes project. The activityRelations_deduplicated_cleaned_ds table contains the necessary fields (memberId, username, timestamp, segmentId) to identify contributors and their monthly activity counts.", - "timestamp": 1754975728451 - }, - { - "id": "me83956x7nn0juea848", - "role": "assistant", - "type": "sql-result", - "content": "This query uses a CTE approach to first aggregate activity counts by month and username, then ranks contributors within each month by activity count (descending) and selects the top 5 for each month. The results are ordered chronologically by month and then by activity count within each month.", - "timestamp": 1754975757706, - "sql": "WITH monthly_activity AS (\n SELECT \n toStartOfMonth(timestamp) as month,\n username,\n count(*) as activity_count\n FROM activityRelations_deduplicated_cleaned_ds\n WHERE segmentId = 'd3e1742e-253b-4a16-af63-b50e15a5e505'\n AND timestamp >= '2024-08-01'\n AND timestamp <= '2025-08-31'\n AND username IS NOT NULL\n AND username != ''\n GROUP BY month, username\n),\nranked_contributors AS (\n SELECT \n month,\n username,\n activity_count,\n row_number() OVER (PARTITION BY month ORDER BY activity_count DESC, username) as rank\n FROM monthly_activity\n)\nSELECT \n month,\n username,\n activity_count\nFROM ranked_contributors\nWHERE rank <= 5\nORDER BY month ASC, activity_count DESC\nLIMIT 1000", - "data": [ - { - "month": "2024-08-01", - "username": "k8s-ci-robot", - "activity_count": 68772 - }, - { - "month": "2024-08-01", - "username": "cncf-infra", - "activity_count": 2236 - }, - { - "month": "2024-08-01", - "username": "dims", - "activity_count": 2041 - }, - { - "month": "2024-08-01", - "username": "pacoxu", - "activity_count": 1938 - }, - { - "month": "2024-08-01", - "username": "sttts", - "activity_count": 1845 - }, - { - "month": "2024-09-01", - "username": "k8s-ci-robot", - "activity_count": 70063 - }, - { - "month": "2024-09-01", - "username": "dims", - "activity_count": 2644 - }, - { - "month": "2024-09-01", - "username": "liggitt", - "activity_count": 2281 - }, - { - "month": "2024-09-01", - "username": "cncf-infra", - "activity_count": 2142 - }, - { - "month": "2024-09-01", - "username": "pacoxu", - "activity_count": 1839 - }, - { - "month": "2024-10-01", - "username": "k8s-ci-robot", - "activity_count": 69554 - }, - { - "month": "2024-10-01", - "username": "dims", - "activity_count": 2755 - }, - { - "month": "2024-10-01", - "username": "cncf-infra", - "activity_count": 2135 - }, - { - "month": "2024-10-01", - "username": "liggitt", - "activity_count": 1905 - }, - { - "month": "2024-10-01", - "username": "neolit123", - "activity_count": 1859 - }, - { - "month": "2024-11-01", - "username": "k8s-ci-robot", - "activity_count": 68651 - }, - { - "month": "2024-11-01", - "username": "dims", - "activity_count": 2728 - }, - { - "month": "2024-11-01", - "username": "cncf-infra", - "activity_count": 2100 - }, - { - "month": "2024-11-01", - "username": "liggitt", - "activity_count": 1949 - }, - { - "month": "2024-11-01", - "username": "SataQiu", - "activity_count": 1754 - }, - { - "month": "2024-12-01", - "username": "k8s-ci-robot", - "activity_count": 64549 - }, - { - "month": "2024-12-01", - "username": "dims", - "activity_count": 2464 - }, - { - "month": "2024-12-01", - "username": "cncf-infra", - "activity_count": 1932 - }, - { - "month": "2024-12-01", - "username": "neolit123", - "activity_count": 1756 - }, - { - "month": "2024-12-01", - "username": "liggitt", - "activity_count": 1682 - }, - { - "month": "2025-01-01", - "username": "k8s-ci-robot", - "activity_count": 68080 - }, - { - "month": "2025-01-01", - "username": "dims", - "activity_count": 2498 - }, - { - "month": "2025-01-01", - "username": "cncf-infra", - "activity_count": 2086 - }, - { - "month": "2025-01-01", - "username": "liggitt", - "activity_count": 2023 - }, - { - "month": "2025-01-01", - "username": "neolit123", - "activity_count": 1877 - }, - { - "month": "2025-02-01", - "username": "k8s-ci-robot", - "activity_count": 62336 - }, - { - "month": "2025-02-01", - "username": "dims", - "activity_count": 2295 - }, - { - "month": "2025-02-01", - "username": "cncf-infra", - "activity_count": 1827 - }, - { - "month": "2025-02-01", - "username": "liggitt", - "activity_count": 1797 - }, - { - "month": "2025-02-01", - "username": "neolit123", - "activity_count": 1731 - }, - { - "month": "2025-03-01", - "username": "k8s-ci-robot", - "activity_count": 71209 - }, - { - "month": "2025-03-01", - "username": "dims", - "activity_count": 2544 - }, - { - "month": "2025-03-01", - "username": "cncf-infra", - "activity_count": 2185 - }, - { - "month": "2025-03-01", - "username": "liggitt", - "activity_count": 2104 - }, - { - "month": "2025-03-01", - "username": "neolit123", - "activity_count": 1936 - }, - { - "month": "2025-04-01", - "username": "k8s-ci-robot", - "activity_count": 68869 - }, - { - "month": "2025-04-01", - "username": "dims", - "activity_count": 2498 - }, - { - "month": "2025-04-01", - "username": "cncf-infra", - "activity_count": 2144 - }, - { - "month": "2025-04-01", - "username": "liggitt", - "activity_count": 2021 - }, - { - "month": "2025-04-01", - "username": "neolit123", - "activity_count": 1869 - }, - { - "month": "2025-05-01", - "username": "k8s-ci-robot", - "activity_count": 71207 - }, - { - "month": "2025-05-01", - "username": "dims", - "activity_count": 2559 - }, - { - "month": "2025-05-01", - "username": "cncf-infra", - "activity_count": 2137 - }, - { - "month": "2025-05-01", - "username": "liggitt", - "activity_count": 2083 - }, - { - "month": "2025-05-01", - "username": "neolit123", - "activity_count": 1909 - }, - { - "month": "2025-06-01", - "username": "k8s-ci-robot", - "activity_count": 68669 - }, - { - "month": "2025-06-01", - "username": "dims", - "activity_count": 2442 - }, - { - "month": "2025-06-01", - "username": "cncf-infra", - "activity_count": 2086 - }, - { - "month": "2025-06-01", - "username": "liggitt", - "activity_count": 2013 - }, - { - "month": "2025-06-01", - "username": "neolit123", - "activity_count": 1847 - }, - { - "month": "2025-07-01", - "username": "k8s-ci-robot", - "activity_count": 71036 - }, - { - "month": "2025-07-01", - "username": "dims", - "activity_count": 2563 - }, - { - "month": "2025-07-01", - "username": "cncf-infra", - "activity_count": 2126 - }, - { - "month": "2025-07-01", - "username": "liggitt", - "activity_count": 2036 - }, - { - "month": "2025-07-01", - "username": "neolit123", - "activity_count": 1943 - }, - { - "month": "2025-08-01", - "username": "k8s-ci-robot", - "activity_count": 28103 - }, - { - "month": "2025-08-01", - "username": "dims", - "activity_count": 1012 - }, - { - "month": "2025-08-01", - "username": "cncf-infra", - "activity_count": 842 - }, - { - "month": "2025-08-01", - "username": "liggitt", - "activity_count": 806 - }, - { - "month": "2025-08-01", - "username": "neolit123", - "activity_count": 770 - } - ] - }, - { - "id": "me839okxadzt6lzxc6s", - "role": "user", - "type": "text", - "status": "complete", - "content": "Can you remove the username k8s-ci-robot", - "timestamp": 1754975782833 - }, - { - "id": "me839wxzwyeobmbjs7i", - "role": "assistant", - "type": "router-status", - "status": "complete", - "content": "I'll create a query to get the top 5 contributors per month for the Kubernetes project, excluding the k8s-ci-robot username from the results.", - "timestamp": 1754975793671 - }, - { - "id": "me83ag5u0o263m4vjlqk", - "role": "assistant", - "type": "sql-result", - "content": "This query excludes the 'k8s-ci-robot' username by adding a WHERE clause filter (username != 'k8s-ci-robot'). It aggregates activity counts by month and username, ranks contributors within each month by activity volume, and returns the top 5 contributors per month from August 2024 to August 2025 for the Kubernetes project.", - "timestamp": 1754975818579, - "sql": "WITH monthly_contributors AS (\n SELECT \n toStartOfMonth(timestamp) as month,\n username,\n count() as activity_count\n FROM activityRelations_deduplicated_cleaned_ds\n WHERE segmentId = 'd3e1742e-253b-4a16-af63-b50e15a5e505'\n AND timestamp >= '2024-08-01 00:00:00'\n AND timestamp < '2025-09-01 00:00:00'\n AND username != 'k8s-ci-robot'\n GROUP BY month, username\n),\nranked_contributors AS (\n SELECT \n month,\n username,\n activity_count,\n row_number() OVER (PARTITION BY month ORDER BY activity_count DESC) as rank\n FROM monthly_contributors\n)\nSELECT \n month,\n username,\n activity_count,\n rank\nFROM ranked_contributors\nWHERE rank <= 5\nORDER BY month ASC, rank ASC\nLIMIT 100", - "data": "The query returns 13 months of data (Aug 2024 - Aug 2025) with the top 5 contributors per month. Key contributors include dims (consistently #1 with 800-1000+ activities), thockin (#2 with 600-800 activities), liggitt (#3 with 600-700 activities), justaugustus (#4 with 500-600 activities), and BenTheElder (#5 with 400-500 activities). August 2025 shows partial month data with lower counts as expected." - } -] \ No newline at end of file diff --git a/frontend/app/components/shared/modules/copilot/store/test3.json b/frontend/app/components/shared/modules/copilot/store/test3.json deleted file mode 100644 index 74346164f..000000000 --- a/frontend/app/components/shared/modules/copilot/store/test3.json +++ /dev/null @@ -1,264 +0,0 @@ -[ - { - "id": "mehys54cf7kqj8riihp", - "role": "user", - "type": "text", - "status": "complete", - "content": "Show me the monthly active contributors over the past 6 months", - "timestamp": 1755572907756 - }, - { - "id": "mehysfemctwiu2zfxnc", - "role": "assistant", - "type": "router-status", - "status": "complete", - "content": "I'll use the active_contributors tool to show monthly active contributors for the Kubernetes project over the past 6 months with monthly granularity.", - "timestamp": 1755572921086 - }, - { - "id": "mehysmf4hnmwsaqyq2h", - "role": "assistant", - "type": "pipe-result", - "data": [ - { - "Month Start": "2025-02-01", - "Month End": "2025-02-28", - "Active Contributors": 1544 - }, - { - "Month Start": "2025-03-01", - "Month End": "2025-03-31", - "Active Contributors": 3010 - }, - { - "Month Start": "2025-04-01", - "Month End": "2025-04-30", - "Active Contributors": 2797 - }, - { - "Month Start": "2025-05-01", - "Month End": "2025-05-31", - "Active Contributors": 2940 - }, - { - "Month Start": "2025-06-01", - "Month End": "2025-06-30", - "Active Contributors": 2785 - }, - { - "Month Start": "2025-07-01", - "Month End": "2025-07-31", - "Active Contributors": 2524 - }, - { - "Month Start": "2025-08-01", - "Month End": "2025-08-31", - "Active Contributors": 1628 - } - ], - "content": "Using the active_contributors pipe with monthly granularity for the past 6 months (February 2025 to August 2025) to show the trend of unique contributors over time for the Kubernetes project.", - "explanation": "Using the active_contributors pipe with monthly granularity for the past 6 months (February 2025 to August 2025) to show the trend of unique contributors over time for the Kubernetes project.", - "instructions": { - "pipes": [ - { - "id": "monthly_contributors", - "name": "active_contributors", - "inputs": { - "project": "k8s", - "startDate": "2025-02-19 00:00:00", - "endDate": "2025-08-19 23:59:59", - "granularity": "monthly" - } - } - ], - "output": [ - { - "type": "direct", - "name": "Month Start", - "pipeId": "monthly_contributors", - "sourceColumn": "startDate" - }, - { - "type": "direct", - "name": "Month End", - "pipeId": "monthly_contributors", - "sourceColumn": "endDate" - }, - { - "type": "direct", - "name": "Active Contributors", - "pipeId": "monthly_contributors", - "sourceColumn": "contributorCount" - } - ] - }, - "timestamp": 1755572930176 - }, - { - "id": "mehysmh8tukhbfqiqeh", - "role": "assistant", - "type": "chat-response-id", - "content": "a8c72343-6695-47d2-b804-fb5e8a802baf", - "timestamp": 1755572930253 - }, - { - "id": "mehytgctwsyk87tr08n", - "role": "user", - "type": "text", - "status": "complete", - "content": "Show the comparison with the previous 6 months", - "timestamp": 1755572968973 - }, - { - "id": "mehytpojvflfyw3tkk", - "role": "assistant", - "type": "router-status", - "status": "complete", - "content": "I'll use the active_contributors widget twice with different time ranges to compare the current 6 months (February-August 2025) with the previous 6 months (August 2024-January 2025).", - "timestamp": 1755572981059 - }, - { - "id": "mehyu8c9uzellpsp9b", - "role": "assistant", - "type": "pipe-result", - "data": [ - { - "Month": "2025-02-01", - "Current Period Contributors": 2804, - "Previous Period Contributors": 2982, - "Change": -178, - "Growth %": -5.969148222669349 - }, - { - "Month": "2025-03-01", - "Current Period Contributors": 3010, - "Previous Period Contributors": 3835, - "Change": -825, - "Growth %": -21.51238591916558 - }, - { - "Month": "2025-04-01", - "Current Period Contributors": 2797, - "Previous Period Contributors": 4742, - "Change": -1945, - "Growth %": -41.01644875579924 - }, - { - "Month": "2025-05-01", - "Current Period Contributors": 2940, - "Previous Period Contributors": 5226, - "Change": -2286, - "Growth %": -43.74282433983927 - }, - { - "Month": "2025-06-01", - "Current Period Contributors": 2785, - "Previous Period Contributors": 3975, - "Change": -1190, - "Growth %": -29.937106918238992 - }, - { - "Month": "2025-07-01", - "Current Period Contributors": 2524, - "Previous Period Contributors": 3007, - "Change": -483, - "Growth %": -16.062520784835385 - }, - { - "Month": "2025-08-01", - "Current Period Contributors": 1628, - "Previous Period Contributors": null, - "Change": 1628, - "Growth %": null - } - ], - "content": "Using two active_contributors pipes to get monthly data for both the current period (Feb-Aug 2025) and previous period (Aug 2024-Jan 2025), then combining them with calculated growth percentages for comparison", - "explanation": "Using two active_contributors pipes to get monthly data for both the current period (Feb-Aug 2025) and previous period (Aug 2024-Jan 2025), then combining them with calculated growth percentages for comparison", - "instructions": { - "pipes": [ - { - "id": "current_period", - "name": "active_contributors", - "inputs": { - "project": "k8s", - "startDate": "2025-02-01 00:00:00", - "endDate": "2025-08-19 23:59:59", - "granularity": "monthly" - } - }, - { - "id": "previous_period", - "name": "active_contributors", - "inputs": { - "project": "k8s", - "startDate": "2024-08-01 00:00:00", - "endDate": "2025-01-31 23:59:59", - "granularity": "monthly" - } - } - ], - "output": [ - { - "type": "direct", - "name": "Month", - "pipeId": "current_period", - "sourceColumn": "startDate" - }, - { - "type": "direct", - "name": "Current Period Contributors", - "pipeId": "current_period", - "sourceColumn": "contributorCount" - }, - { - "type": "direct", - "name": "Previous Period Contributors", - "pipeId": "previous_period", - "sourceColumn": "contributorCount" - }, - { - "type": "formula", - "name": "Change", - "formula": "current - previous", - "dependencies": [ - { - "variable": "current", - "pipeId": "current_period", - "sourceColumn": "contributorCount" - }, - { - "variable": "previous", - "pipeId": "previous_period", - "sourceColumn": "contributorCount" - } - ] - }, - { - "type": "formula", - "name": "Growth %", - "formula": "((current - previous) / previous) * 100", - "dependencies": [ - { - "variable": "current", - "pipeId": "current_period", - "sourceColumn": "contributorCount" - }, - { - "variable": "previous", - "pipeId": "previous_period", - "sourceColumn": "contributorCount" - } - ] - } - ] - }, - "timestamp": 1755573005241 - }, - { - "id": "mehyu8dsa781n19zyi9", - "role": "assistant", - "type": "chat-response-id", - "content": "7128ab69-d5a4-4689-bfe0-b8c820a183b6", - "timestamp": 1755573005296 - } -] \ No newline at end of file diff --git a/frontend/lib/chat/agents/auditor.ts b/frontend/lib/chat/agents/auditor.ts new file mode 100644 index 000000000..7d9ec35e9 --- /dev/null +++ b/frontend/lib/chat/agents/auditor.ts @@ -0,0 +1,43 @@ +// Copyright (c) 2025 The Linux Foundation and each contributor. +// SPDX-License-Identifier: MIT +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { AuditorOutput, AuditorAgentInput } from '../types' +import { auditorOutputSchema } from '../types' +import { auditorPrompt } from '../prompts/auditor' +import { BaseAgent } from './base-agent' + +export class AuditorAgent extends BaseAgent { + readonly name = 'Auditor' + readonly outputSchema = auditorOutputSchema + readonly temperature = 0.2 // Slightly creative for summaries + readonly maxSteps = 1 + + protected getModel(input: AuditorAgentInput): any { + return input.model + } + + protected getSystemPrompt(input: AuditorAgentInput): string { + return auditorPrompt( + input.originalQuestion, + input.reformulatedQuestion, + input.dataSummary, + input.attemptNumber, + input.previousFeedback, + ) + } + + protected getUserPrompt(_input: AuditorAgentInput): string { + return '' + } + + protected getTools(_input: AuditorAgentInput): Record { + return {} + } + + protected createError(error: unknown): Error { + if (error instanceof Error) { + return new Error(`Auditor agent error: ${error.message}`) + } + return new Error(`Auditor agent error: ${String(error)}`) + } +} diff --git a/frontend/lib/chat/agents/base-agent.ts b/frontend/lib/chat/agents/base-agent.ts index 1f362d573..3a83bdd0a 100644 --- a/frontend/lib/chat/agents/base-agent.ts +++ b/frontend/lib/chat/agents/base-agent.ts @@ -99,7 +99,7 @@ export abstract class BaseAgent { const jsonInstructions = this.generateJSONInstructions() const conversationHistoryReceipt = this.generateConversationHistoryReceipt(input.messages) - const fullSystemPrompt = conversationHistoryReceipt + systemPrompt + jsonInstructions + const fullSystemPrompt = systemPrompt + conversationHistoryReceipt + jsonInstructions // Check if we have messages in the input const hasMessages = @@ -223,7 +223,7 @@ export abstract class BaseAgent { ${conversationHistory} - ## END OF CONVERSATION HISTORY` + ## END OF CONVERSATION HISTORY\n` } catch (error) { console.error('Error generating conversation history context', error) return '' diff --git a/frontend/lib/chat/agents/index.ts b/frontend/lib/chat/agents/index.ts index 4091c8ac4..1b27c4c6c 100644 --- a/frontend/lib/chat/agents/index.ts +++ b/frontend/lib/chat/agents/index.ts @@ -1,7 +1,8 @@ // Copyright (c) 2025 The Linux Foundation and each contributor. // SPDX-License-Identifier: MIT +import { AuditorAgent } from './auditor' import { PipeAgent } from './pipe' import { RouterAgent } from './router' import { TextToSqlAgent } from './text-to-sql' -export { PipeAgent, RouterAgent, TextToSqlAgent } +export { AuditorAgent, PipeAgent, RouterAgent, TextToSqlAgent } diff --git a/frontend/lib/chat/agents/text-to-sql.ts b/frontend/lib/chat/agents/text-to-sql.ts index bcb6de756..7ab9c6a99 100644 --- a/frontend/lib/chat/agents/text-to-sql.ts +++ b/frontend/lib/chat/agents/text-to-sql.ts @@ -48,6 +48,7 @@ export class TextToSqlAgent extends BaseAgent { const fullSystemPrompt = conversationHistoryReceipt + systemPrompt + ` ## CRITICAL INSTRUCTIONS +0. NEVER use the functions that are NOT provided under AVAILABLE FUNCTIONS 1. Use tools BRIEFLY to understand schema (max 2 tool calls) 2. Then STOP calling tools and write the SQL query 3. Put your SQL in a markdown code block: \`\`\`sql ... \`\`\` diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 9ecd8f763..49bc86a0a 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -2,19 +2,17 @@ // SPDX-License-Identifier: MIT /* eslint-disable @typescript-eslint/no-explicit-any */ import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock' -import { - experimental_createMCPClient as createMCPClient, - type LanguageModelV1, -} from 'ai' +import { experimental_createMCPClient as createMCPClient, type LanguageModelV1 } from 'ai' import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' import type { Pool } from 'pg' -import type { ChatResponse } from '../../server/repo/chat.repo' +import type { ChatResponse, IChatResponseDb } from '../../server/repo/chat.repo' import { ChatRepository } from '../../server/repo/chat.repo' -import { TextToSqlAgent, PipeAgent, RouterAgent } from './agents' +import { TextToSqlAgent, PipeAgent, RouterAgent, AuditorAgent } from './agents' import { executePipeInstructions, executeTextToSqlInstructions } from './instructions' import type { AgentResponseCompleteParams, + AuditorAgentInput, ChatMessage, DataCopilotQueryInput, PipeAgentInput, @@ -26,6 +24,7 @@ import type { TextToSqlAgentStreamInput, } from './types' import { RouterDecisionAction, StreamDataStatus, StreamDataType } from './enums' +import { generateDataSummary } from './utils/data-summary' const bedrock = createAmazonBedrock({ accessKeyId: process.env.NUXT_AWS_BEDROCK_ACCESS_KEY_ID, @@ -66,6 +65,9 @@ export class DataCopilot { /** Bedrock model identifier */ private readonly BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0' + /** Maximum number of auditor retry attempts */ + private readonly MAX_AUDITOR_RETRIES = 1 + constructor() { this.model = bedrock(this.BEDROCK_MODEL_ID) this.tbMcpUrl = `https://mcp.tinybird.co?token=${process.env.NUXT_INSIGHTS_DATA_COPILOT_TINYBIRD_TOKEN}&host=${process.env.NUXT_TINYBIRD_BASE_URL}` @@ -191,7 +193,6 @@ export class DataCopilot { const followUpTools = this.tbTools delete followUpTools['execute_query'] - const agent = new TextToSqlAgent() return agent.execute({ model: this.model, @@ -252,6 +253,226 @@ export class DataCopilot { }) } + /** + * Executes the auditor agent to validate whether retrieved data actually answers the user's question. + * Uses statistical analysis of data structure and content without requiring full dataset transmission. + * + * @param messages - Conversation history for context + * @param originalQuestion - The user's original question + * @param reformulatedQuestion - Router's enhanced interpretation of the question + * @param data - Retrieved data to validate + * @param attemptNumber - Current retry attempt (0 for first attempt) + * @param previousFeedback - Feedback from previous auditor run if this is a retry + * @returns Validation result with summary or feedback for router + */ + private async runAuditorAgent({ + messages, + originalQuestion, + reformulatedQuestion, + data, + attemptNumber, + previousFeedback, + }: Omit & { data: any[] }) { + const dataSummary = generateDataSummary(data) + const agent = new AuditorAgent() + return agent.execute({ + model: this.model, + messages, + originalQuestion, + reformulatedQuestion, + dataSummary, + attemptNumber, + previousFeedback, + }) + } + + /** + * Run execution and validation loop with auditor feedback + * Handles router execution, query/pipes execution, validation, and retries + * + * @returns Router action and results after validation + */ + private async runExecutionWithAuditorLoop({ + messages, + currentQuestion, + date, + projectName, + pipe, + parametersString, + segmentId, + previousWasClarification, + dataStream, + responseData, + }: { + messages: ChatMessage[] + currentQuestion: string + date: string + projectName: string + pipe: string + parametersString: string + segmentId: string + previousWasClarification: boolean + dataStream: any + responseData: ChatResponse + }): Promise<{ + action: RouterDecisionAction + routerOutput: RouterOutput + sqlQuery?: string + pipeInstructions?: PipeInstructions + }> { + let attemptNumber = 0 + let previousFeedback: string | undefined = undefined + let currentMessages = messages + let routerOutput: RouterOutput + let sqlQuery: string | undefined = undefined + let pipeInstructions: PipeInstructions | undefined = undefined + + while (attemptNumber <= this.MAX_AUDITOR_RETRIES) { + // Run router agent + dataStream.writeData({ + type: StreamDataType.ROUTER_STATUS, + status: StreamDataStatus.ANALYZING, + }) + + routerOutput = await this.runRouterAgent({ + messages: currentMessages, + date, + projectName, + pipe, + parametersString, + segmentId, + previousWasClarification: attemptNumber === 0 ? previousWasClarification : false, + }) + + // Accumulate router token usage + if (routerOutput.usage) { + responseData.inputTokens += routerOutput.usage.promptTokens || 0 + responseData.outputTokens += routerOutput.usage.completionTokens || 0 + } + + // Handle STOP and ASK_CLARIFICATION - no auditor needed + if ( + routerOutput.next_action === RouterDecisionAction.STOP || + routerOutput.next_action === RouterDecisionAction.ASK_CLARIFICATION + ) { + return { action: routerOutput.next_action, routerOutput } + } + + // Router decided on CREATE_QUERY or PIPES - continue with execution + dataStream.writeData({ + type: StreamDataType.ROUTER_STATUS, + status: StreamDataStatus.COMPLETE, + reasoning: routerOutput.reasoning, + reformulatedQuestion: routerOutput.reformulated_question, + }) + + let data: any[] = [] + + // Execute based on router decision + if (routerOutput.next_action === RouterDecisionAction.CREATE_QUERY) { + const result = await this.handleCreateQueryAction({ + messages: currentMessages, + date, + projectName, + pipe, + parametersString, + segmentId, + reformulatedQuestion: routerOutput.reformulated_question, + dataStream, + }) + sqlQuery = result.sqlQuery + data = result.data + } else if (routerOutput.next_action === RouterDecisionAction.PIPES) { + const result = await this.handlePipesAction({ + messages: currentMessages, + date, + projectName, + pipe, + parametersString, + segmentId, + reformulatedQuestion: routerOutput.reformulated_question, + toolNames: routerOutput.tools, + dataStream, + responseData, + routerOutput, + }) + pipeInstructions = result.pipeInstructions + data = result.data + } + + // Stream auditor status + dataStream.writeData({ + type: StreamDataType.AUDITOR_STATUS, + status: attemptNumber === 0 ? StreamDataStatus.VALIDATING : StreamDataStatus.RETRYING, + attempt: attemptNumber + 1, + maxAttempts: this.MAX_AUDITOR_RETRIES + 1, + }) + + // Run auditor validation + const auditorResult = await this.runAuditorAgent({ + messages: currentMessages, + originalQuestion: currentQuestion, + reformulatedQuestion: routerOutput.reformulated_question, + data, + attemptNumber, + previousFeedback, + }) + + // Accumulate auditor token usage + if (auditorResult.usage) { + responseData.inputTokens += auditorResult.usage.promptTokens || 0 + responseData.outputTokens += auditorResult.usage.completionTokens || 0 + } + + if (auditorResult.is_valid) { + // Data is valid, stream summary and exit loop + dataStream.writeData({ + type: StreamDataType.AUDITOR_STATUS, + status: StreamDataStatus.VALIDATED, + summary: auditorResult.summary, + reasoning: auditorResult.reasoning, + }) + return { action: routerOutput.next_action, routerOutput, sqlQuery, pipeInstructions } + } + + // Data is invalid + if (attemptNumber >= this.MAX_AUDITOR_RETRIES) { + // Max retries reached, send final status + dataStream.writeData({ + type: StreamDataType.AUDITOR_STATUS, + status: StreamDataStatus.MAX_RETRIES, + feedback: auditorResult.feedback_to_router, + reasoning: auditorResult.reasoning, + }) + return { action: routerOutput.next_action, routerOutput, sqlQuery, pipeInstructions } + } + + // Prepare for retry - add feedback to messages and loop + previousFeedback = auditorResult.feedback_to_router + attemptNumber++ + + dataStream.writeData({ + type: StreamDataType.AUDITOR_STATUS, + status: StreamDataStatus.RETRYING, + feedback: previousFeedback, + attempt: attemptNumber + 1, + }) + + // Add feedback to conversation context for next iteration + currentMessages = [ + ...currentMessages, + { + role: 'system', + content: `Previous attempt did not produce valid results. Auditor feedback: ${previousFeedback}. \n + Please adjust your approach based on this feedback.`, + }, + ] + } + + // This should never be reached, but TypeScript needs it + throw new Error('Auditor loop completed without returning a result') + } + /** * Send keepalive message to prevent Cloudflare timeout */ @@ -305,18 +526,19 @@ export class DataCopilot { } // Check if the latest response was ASK_CLARIFICATION - const latestResponse = previousChatResponses[previousChatResponses.length - 1] as ChatResponse - const previousWasClarification = latestResponse.routerResponse === RouterDecisionAction.ASK_CLARIFICATION + const latestResponse = previousChatResponses[previousChatResponses.length - 1] as IChatResponseDb + const previousWasClarification = + latestResponse.router_response === RouterDecisionAction.ASK_CLARIFICATION if (previousWasClarification) { // Merge the clarification: combine the ambiguous question with the clarification answer - const ambiguousQuestion = latestResponse.userPrompt + const ambiguousQuestion = latestResponse.user_prompt const mergedQuestion = `Original question: ${ambiguousQuestion}\n\nClarification provided: ${currentQuestion}` // Build messages: [older history before clarification] + [merged question] const messages = previousChatResponses.slice(0, -1).map((response) => ({ role: 'user' as const, - content: response.userPrompt, + content: response.user_prompt, })) // Add the merged question as the current message @@ -331,7 +553,7 @@ export class DataCopilot { // Normal case: build messages from all previous responses + current question const messages = previousChatResponses.map((response) => ({ role: 'user' as const, - content: response.userPrompt, + content: response.user_prompt, })) // Add the current question @@ -368,124 +590,80 @@ export class DataCopilot { ) const responseData: ChatResponse = { - userPrompt: currentQuestion, - inputTokens: 0, - outputTokens: 0, - model: this.BEDROCK_MODEL_ID, - conversationId: conversationId || '', - routerResponse: RouterDecisionAction.STOP, - routerReason: '', - pipeInstructions: undefined as PipeInstructions | undefined, - sqlQuery: undefined as string | undefined, - } + userPrompt: currentQuestion, + inputTokens: 0, + outputTokens: 0, + model: this.BEDROCK_MODEL_ID, + conversationId: conversationId || '', + routerResponse: RouterDecisionAction.STOP, + routerReason: '', + pipeInstructions: undefined as PipeInstructions | undefined, + sqlQuery: undefined as string | undefined, + } - try { - dataStream.writeData({ - type: StreamDataType.ROUTER_STATUS, - status: StreamDataStatus.ANALYZING, - }) - // Add padding for Cloudflare streaming threshold - - const routerOutput = await this.runRouterAgent({ - messages, - date, - projectName: projectName as string, - pipe, - parametersString, - segmentId: segmentId as string, - previousWasClarification, - }) - - // Accumulate token usage from router - if (routerOutput.usage) { - responseData.inputTokens += routerOutput.usage.promptTokens || 0 - responseData.outputTokens += routerOutput.usage.completionTokens || 0 - } - - if (routerOutput.next_action === RouterDecisionAction.STOP) { - await this.handleStopAction( - messages[messages.length - 1]?.content || '', - routerOutput, - responseData, - dataStream, - insightsDbPool, - userEmail, - conversationId, - ) - return - } - - if (routerOutput.next_action === RouterDecisionAction.ASK_CLARIFICATION) { - await this.handleAskClarificationAction( - messages[messages.length - 1]?.content || '', - routerOutput, - responseData, - dataStream, - insightsDbPool, - userEmail, - conversationId, - ) - return - } - - dataStream.writeData({ - type: StreamDataType.ROUTER_STATUS, - status: StreamDataStatus.COMPLETE, - reasoning: routerOutput.reasoning, - reformulatedQuestion: routerOutput.reformulated_question, - }) - - let sqlQuery: string | undefined = undefined - let pipeInstructions: PipeInstructions | undefined = undefined - - if (routerOutput.next_action === RouterDecisionAction.CREATE_QUERY) { - const result = await this.handleCreateQueryAction({ - messages, - date, - projectName: projectName as string, - pipe, - parametersString, - segmentId: segmentId as string, - reformulatedQuestion: routerOutput.reformulated_question, - dataStream, - }) - sqlQuery = result.sqlQuery - } else if (routerOutput.next_action === RouterDecisionAction.PIPES) { - const result = await this.handlePipesAction({ - messages, - date, - projectName: projectName as string, - pipe, - parametersString, - segmentId: segmentId as string, - reformulatedQuestion: routerOutput.reformulated_question, - toolNames: routerOutput.tools, - dataStream, - responseData, - routerOutput, - }) - pipeInstructions = result.pipeInstructions - } - - await this.handleResponseComplete({ - userPrompt: messages[messages.length - 1]?.content || '', - responseData, - routerOutput, - pipeInstructions, - sqlQuery, - conversationId, - insightsDbPool, - userEmail, - dataStream, - }) - } catch (error) { - dataStream.writeData({ - type: 'router-status', - status: 'error', - error: error instanceof Error ? error.message : 'An error occurred', - }) - throw error - } + try { + // Run execution with auditor loop (handles router, execution, validation, retries) + const { action, routerOutput, sqlQuery, pipeInstructions } = + await this.runExecutionWithAuditorLoop({ + messages, + currentQuestion, + date, + projectName: projectName as string, + pipe, + parametersString, + segmentId: segmentId as string, + previousWasClarification, + dataStream, + responseData, + }) + + // Handle STOP and ASK_CLARIFICATION actions + if (action === RouterDecisionAction.STOP) { + await this.handleStopAction( + messages[messages.length - 1]?.content || '', + routerOutput, + responseData, + dataStream, + insightsDbPool, + userEmail, + conversationId, + ) + return + } + + if (action === RouterDecisionAction.ASK_CLARIFICATION) { + await this.handleAskClarificationAction( + messages[messages.length - 1]?.content || '', + routerOutput, + responseData, + dataStream, + insightsDbPool, + userEmail, + conversationId, + ) + return + } + + // Handle completed execution (CREATE_QUERY or PIPES) + await this.handleResponseComplete({ + userPrompt: messages[messages.length - 1]?.content || '', + responseData, + routerOutput, + pipeInstructions, + sqlQuery, + conversationId, + insightsDbPool, + userEmail, + dataStream, + }) + } catch (error) { + dataStream.writeData({ + type: 'router-status', + status: 'error', + error: error instanceof Error ? error.message : 'An error occurred', + }) + throw error + } } /** @@ -584,7 +762,7 @@ export class DataCopilot { segmentId, reformulatedQuestion, dataStream, - }: TextToSqlAgentStreamInput): Promise<{ sqlQuery: string }> { + }: TextToSqlAgentStreamInput): Promise<{ sqlQuery: string; data: any[] }> { // Send progress update before starting TextToSql agent this.sendProgress(dataStream, 'progress', 'Analyzing database schema...') @@ -595,17 +773,21 @@ export class DataCopilot { try { const textToSqlOutput = await this.runTextToSqlAgent({ - messages, - date, - projectName, - pipe, - parametersString, - segmentId, - reformulatedQuestion, - }) + messages, + date, + projectName, + pipe, + parametersString, + segmentId, + reformulatedQuestion, + }) clearInterval(keepaliveInterval) - this.sendProgress(dataStream, 'progress', `SQL query generated! Executing [${textToSqlOutput.instructions}]...`) + this.sendProgress( + dataStream, + 'progress', + `SQL query generated! Executing [${textToSqlOutput.instructions}]...`, + ) const queryData = await executeTextToSqlInstructions(textToSqlOutput.instructions) @@ -616,7 +798,7 @@ export class DataCopilot { data: queryData, }) - return { sqlQuery: textToSqlOutput.instructions } + return { sqlQuery: textToSqlOutput.instructions, data: queryData } } catch (error) { clearInterval(keepaliveInterval) throw error @@ -637,7 +819,7 @@ export class DataCopilot { toolNames, dataStream, responseData, - }: PipeAgentStreamInput): Promise<{ pipeInstructions: PipeInstructions }> { + }: PipeAgentStreamInput): Promise<{ pipeInstructions: PipeInstructions; data: any[] }> { const pipeOutput = await this.runPipeAgent({ messages, date, @@ -665,7 +847,7 @@ export class DataCopilot { data: combinedData, }) - return { pipeInstructions: pipeOutput.instructions } + return { pipeInstructions: pipeOutput.instructions, data: combinedData } } /** diff --git a/frontend/lib/chat/enums.ts b/frontend/lib/chat/enums.ts index 82b515c2f..028203d67 100644 --- a/frontend/lib/chat/enums.ts +++ b/frontend/lib/chat/enums.ts @@ -5,12 +5,17 @@ export enum StreamDataType { CHAT_RESPONSE_ID = 'chat-response-id', SQL_RESULT = 'sql-result', PIPE_RESULT = 'pipe-result', + AUDITOR_STATUS = 'auditor-status', } export enum StreamDataStatus { ANALYZING = 'analyzing', COMPLETE = 'complete', ASK_CLARIFICATION = 'ask_clarification', + VALIDATING = 'validating', + VALIDATED = 'validated', + RETRYING = 'retrying', + MAX_RETRIES = 'max_retries', } export enum RouterDecisionAction { diff --git a/frontend/lib/chat/prompts/auditor.ts b/frontend/lib/chat/prompts/auditor.ts new file mode 100644 index 000000000..4fdad86a6 --- /dev/null +++ b/frontend/lib/chat/prompts/auditor.ts @@ -0,0 +1,146 @@ +// Copyright (c) 2025 The Linux Foundation and each contributor. +// SPDX-License-Identifier: MIT +import type { DataSummary } from '../utils/data-summary' + +export const auditorPrompt = ( + originalQuestion: string, + reformulatedQuestion: string, + dataSummary: DataSummary, + attemptNumber: number, + previousFeedback?: string, +) => { + const statsFormatted = Object.entries(dataSummary.columnStats) + .map(([col, stats]) => { + const lines = [`- ${col} (${stats.type}):`] + + if (stats.nullPercentage > 0) { + lines.push(` • ${stats.nullPercentage}% null values`) + } + + if (stats.type === 'numeric') { + lines.push(` • Range: ${stats.min} to ${stats.max}`) + lines.push(` • Average: ${stats.avg}`) + if (stats.hasAllZeros) lines.push(` • ⚠️ All values are zero`) + } + + if (stats.type === 'date') { + lines.push(` • Date range: ${stats.dateRange}`) + lines.push(` • ${stats.distinctCount} distinct dates`) + } + + if (stats.type === 'string') { + lines.push(` • ${stats.distinctCount} distinct values`) + } + + return lines.join('\n') + }) + .join('\n') + + return `You are an Auditor agent that validates whether retrieved data can answer the user's question. + +## USER'S QUESTION +${originalQuestion} + +## ENHANCED QUERY (Router's Interpretation) +${reformulatedQuestion} + +## DATA SUMMARY +**Total Rows:** ${dataSummary.rowCount} +**Columns:** ${dataSummary.columns.join(', ')} + +**Column Statistics:** +${statsFormatted} + +${ + attemptNumber > 0 + ? ` +## RETRY ATTEMPT #${attemptNumber + 1} +Previous feedback: ${previousFeedback} +⚠️ The router already tried once. Check if the issue was addressed. +` + : '' +} + +--- + +## YOUR TASK + +Make a **BINARY decision**: Can this data answer the user's question? + +### Validation Checklist + +**1. Column Coverage** +- Are all required columns present? +- Do column names semantically match the question? + +**2. Data Quality** +- Row count > 0? +- Key columns not 100% null? +- Numeric metrics not all zeros? + +**3. Time Dimension (if applicable)** +- If question mentions time period, does date range match? +- If "daily", are there enough distinct dates? +- If "monthly", is data aggregated correctly? + +**4. Granularity** +- If question asks "by company", is there a company/organization column? +- If question asks for breakdown, are grouping columns present? + +**5. Metric Presence** +- If question asks for "count", "activity", "growth", etc., is there a numeric metric? +- Is the metric column name relevant to the question? + +### Decision Criteria + +✅ **is_valid = true** IF: +- All required columns exist (even if imperfect names) +- Data has > 0 rows with non-null values +- Time range matches question (if time-based) +- Granularity is appropriate +- At least one relevant metric present + +❌ **is_valid = false** IF: +- Missing critical columns (e.g., no metric for "show activity") +- 0 rows or all nulls/zeros in key columns +- Wrong time period (e.g., 2023 data for "2024" question) +- Wrong aggregation (e.g., monthly when daily requested) +- Columns completely irrelevant to question + +### Output Requirements + +**IF is_valid = true:** +- Set \`is_valid: true\` +- Write a brief \`summary\` (2-3 sentences) for the user: + - What the data shows + - Key findings based on statistics + - Direct answer to their question + - Example: "Commit activity in 2024 ranged from 0 to 453 per day across 12 companies, + with an average of 87 commits daily." + +**IF is_valid = false:** +- Set \`is_valid: false\` +- Write \`feedback_to_router\` with SPECIFIC fixes: + - What column is missing? (e.g., "Need commit_count or activity metric") + - What's wrong with data? (e.g., "Date range is 2023, but question asks for 2024") + - What should router try instead? (e.g., "Use active_contributors_by_date pipe instead") + - Be direct and actionable + +### Important Notes +- **If there's no date column, don't check period validity:** +If user asks for a single metric and you have a relevant numeric column, it's likely valid +- **Statistics are your friend:** Use min/max/avg/range to validate without seeing raw data +- **Date ranges are critical:** If question has timeframe, validate earliestDate/latestDate (for timeseries) +- **Distinct counts matter:** Low distinctCount on grouping columns = problem +- **Don't be overly strict:** If data can partially answer, mark valid +- ${attemptNumber >= 1 ? '**This is a RETRY:** Be slightly more lenient unless clearly broken' : ''} + +--- + +## REASONING FORMAT +Explain your decision in 2-3 sentences: +1. What you validated in the statistics +2. Whether it matches the question requirements +3. Your final decision +` +} diff --git a/frontend/lib/chat/prompts/router.ts b/frontend/lib/chat/prompts/router.ts index 0b1c15e38..00c4a9156 100644 --- a/frontend/lib/chat/prompts/router.ts +++ b/frontend/lib/chat/prompts/router.ts @@ -58,6 +58,11 @@ ${toolsOverview} **Step 1: Check Existing Tools (HIGHEST PRIORITY)** - **FIRST: For activity-count-related queries (stars count, forks count, commits count, etc.) → Consider activities_count or activities_cumulative_count pipes** ${pipeToolQuestion} +- ** Each activity type and its description can be found in activityTypes datasource. + * Refer to this datasource when user asks for specific activity types and use this field for filtering types. + * Keep in mind that for some activities, there can be multiple types to filter. (For instance to get commits => "commited-commit" and "authored-commit") + * When activities are reffered as "contributions" find all types in "activityTypes" with "isCodeContribution" = true. + * When activities are reffered as "collaboration" find all types in "activityTypes" with "isCollaboration" = true. - **MANDATORY VALIDATION: Before routing to pipes, verify the pipe can FULLY answer the question:** - **Check dimensions/groupings:** Does the query ask for breakdowns the pipe doesn't support? * Example: "commits by company" → activities_count cannot group by company → USE create_query @@ -87,13 +92,16 @@ ${pipeToolQuestion} **Step 2: Check Data Sources (only if Step 1 is NO)** - Use list_datasources to examine available tables and fields - Check if the required fields exist in any data source -- Pay special attention to the pull_requests_analyzed, issues_analyzed, activityRelations_deduplicated_cleaned_ds tables -- If something can be answered by avoiding using activityRelations_deduplicated_cleaned_ds, prefer that (e.g., use pull_requests_analyzed for PR counts, issues_analyzed for issue counts, etc.) +- Pay special attention to the pull_requests_analyzed, issues_analyzed, activityRelations_data_copilot tables +- If something can be answered by avoiding using activityRelations_data_copilot, prefer that (e.g., use pull_requests_analyzed for PR counts, issues_analyzed for issue counts, etc.) - If the needed fields exist → Question is VALID, route to "create_query" action - If fields don't exist → Question is INVALID, route to "stop" action - If the question is referencing a field about contributors/people that we have only for organizations, the question is INVALID # ROUTING DECISIONS +** CRITICAL: Always consider the current dashboard information to disambiguate. + * Example: Split the count by repository to find the repo with the strongest growth. + * In the example above, if the user is coming from active-contributors dashboard, he's probably referring to "active contributor count" when asking for "count". - "stop": The question cannot be answered with available data - "create_query": Custom SQL query needed using available data sources (tools can be empty) - "pipes": Existing tools can answer the question (specify which tools in the tools array) @@ -135,5 +143,6 @@ It must be something user-friendly. - Must be a clear, specific question that helps the user provide the missing information - Should offer options or examples when appropriate - Must be conversational and friendly +- Always check for historical context: if with historical context the question is clear, do NOT ask for clarification ` } diff --git a/frontend/lib/chat/tests/auditor.test.ts b/frontend/lib/chat/tests/auditor.test.ts new file mode 100644 index 000000000..9c6d3cfa6 --- /dev/null +++ b/frontend/lib/chat/tests/auditor.test.ts @@ -0,0 +1,187 @@ +// Copyright (c) 2025 The Linux Foundation and each contributor. +// SPDX-License-Identifier: MIT +/* eslint-disable @typescript-eslint/no-explicit-any */ +/** + * Auditor Agent Tests + * + * Tests auditor agent validation logic with statistical data summaries + */ + +import { describe, test, expect, beforeAll } from 'vitest' +import { createAmazonBedrock, type AmazonBedrockProvider } from '@ai-sdk/amazon-bedrock' +import type { LanguageModelV1 } from 'ai' + +import { AuditorAgent } from '../agents/auditor' +import { generateDataSummary } from '../utils/data-summary' +import type { ChatMessage } from '../types' + +let bedrock: AmazonBedrockProvider | null = null + +describe('Auditor Agent', () => { + let model: LanguageModelV1 + + beforeAll(async () => { + // Check if we have the required environment variables + const hasAwsCredentials = process.env.NUXT_AWS_BEDROCK_ACCESS_KEY_ID + + if (!hasAwsCredentials) { + console.warn('⚠️ Skipping auditor tests - missing AWS credentials') + return + } + + // Initialize AWS Bedrock model + bedrock = createAmazonBedrock({ + accessKeyId: process.env.NUXT_AWS_BEDROCK_ACCESS_KEY_ID, + secretAccessKey: process.env.NUXT_AWS_BEDROCK_SECRET_ACCESS_KEY, + region: process.env.NUXT_AWS_BEDROCK_REGION, + }) + + const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0' + model = bedrock(BEDROCK_MODEL_ID) + }, 30000) + + test('should validate data that answers the question correctly', async () => { + if (!bedrock) { + console.warn('⚠️ Skipping test - no AWS credentials') + return + } + + const messages: ChatMessage[] = [ + { role: 'user', content: 'How many commits were made in January 2025?' }, + ] + + // Mock data that clearly answers the question + const mockData = [ + { month: '2025-01', commit_count: 150 }, + { month: '2025-02', commit_count: 200 }, + ] + + const dataSummary = generateDataSummary(mockData) + const agent = new AuditorAgent() + + const result = await agent.execute({ + model, + messages, + originalQuestion: 'How many commits were made in January 2025?', + reformulatedQuestion: 'Get monthly commit counts for Q1 2025', + dataSummary, + attemptNumber: 0, + }) + + expect(result.is_valid).toBe(true) + expect(result.summary).toBeDefined() + expect(result.reasoning).toBeDefined() + }, 30000) + + test('should invalidate data that does not answer the question', async () => { + if (!bedrock) { + console.warn('⚠️ Skipping test - no AWS credentials') + return + } + + const messages: ChatMessage[] = [ + { role: 'user', content: 'What is the average response time for API endpoints?' }, + ] + + // Mock data that doesn't answer the question (commit data instead of API data) + const mockData = [ + { author: 'john', commit_count: 50 }, + { author: 'jane', commit_count: 75 }, + ] + + const dataSummary = generateDataSummary(mockData) + const agent = new AuditorAgent() + + const result = await agent.execute({ + model, + messages, + originalQuestion: 'What is the average response time for API endpoints?', + reformulatedQuestion: 'Calculate average API response time across all endpoints', + dataSummary, + attemptNumber: 0, + }) + + expect(result.is_valid).toBe(false) + expect(result.feedback_to_router).toBeDefined() + expect(result.reasoning).toBeDefined() + }, 30000) + + test('should generate efficient token-optimized data summary', () => { + const mockData = [ + { date: '2025-01-01', commits: 10, lines_added: 500, lines_removed: 200 }, + { date: '2025-01-02', commits: 15, lines_added: 600, lines_removed: 150 }, + { date: '2025-01-03', commits: 8, lines_added: 300, lines_removed: 100 }, + ] + + const summary = generateDataSummary(mockData) + + expect(summary.rowCount).toBe(3) + expect(summary.columns).toContain('commits') + expect(summary.columns).toContain('lines_added') + expect(summary.columnStats.commits).toBeDefined() + expect(summary.columnStats.commits?.type).toBe('numeric') + expect(summary.columnStats.commits?.min).toBe(8) + expect(summary.columnStats.commits?.max).toBe(15) + expect(summary.columnStats.commits?.avg).toBeDefined() + }) + + test('should handle empty data gracefully', async () => { + if (!bedrock) { + console.warn('⚠️ Skipping test - no AWS credentials') + return + } + + const messages: ChatMessage[] = [ + { role: 'user', content: 'How many commits were made in January 2025?' }, + ] + + const emptyData: any[] = [] + const dataSummary = generateDataSummary(emptyData) + const agent = new AuditorAgent() + + const result = await agent.execute({ + model, + messages, + originalQuestion: 'How many commits were made in January 2025?', + reformulatedQuestion: 'Get monthly commit counts for Q1 2025', + dataSummary, + attemptNumber: 0, + }) + + expect(result.is_valid).toBe(false) + expect(result.feedback_to_router).toBeDefined() + }, 30000) + + test('should provide feedback for retry on second attempt', async () => { + if (!bedrock) { + console.warn('⚠️ Skipping test - no AWS credentials') + return + } + + const messages: ChatMessage[] = [ + { role: 'user', content: 'What is the commit trend over time?' }, + ] + + // Mock data missing time dimension + const mockData = [{ total_commits: 500 }] + + const dataSummary = generateDataSummary(mockData) + const agent = new AuditorAgent() + + const previousFeedback = + 'The data lacks time dimension. Please group by date or month to show trends.' + + const result = await agent.execute({ + model, + messages, + originalQuestion: 'What is the commit trend over time?', + reformulatedQuestion: 'Show commit activity trends grouped by month', + dataSummary, + attemptNumber: 1, + previousFeedback, + }) + + expect(result.is_valid).toBe(false) + expect(result.reasoning).toContain('time') + }, 30000) +}) diff --git a/frontend/lib/chat/types.ts b/frontend/lib/chat/types.ts index cbc434217..33ebe13cc 100644 --- a/frontend/lib/chat/types.ts +++ b/frontend/lib/chat/types.ts @@ -114,9 +114,21 @@ export const pipeOutputSchema = z.object({ ), }) +// Auditor agent output schema +export const auditorOutputSchema = z.object({ + is_valid: z.boolean().describe('true = data answers question, false = needs retry'), + reasoning: z.string().describe('2-3 sentences explaining the validation decision'), + feedback_to_router: z + .string() + .optional() + .describe('If invalid, specific guidance for router to fix the issue'), + summary: z.string().optional().describe('If valid, user-friendly summary of findings'), +}) + // TypeScript types for agent outputs export type RouterOutput = z.infer & { usage?: any } export type PipeOutput = z.infer & { usage?: any } +export type AuditorOutput = z.infer & { usage?: any } // ============================================ // Agent Input Types @@ -166,7 +178,7 @@ export interface DataCopilotQueryInput { projectName?: string pipe: string parameters?: Record - conversationId?: string + conversationId: string insightsDbPool: Pool userEmail: string dataStream: DataStreamWriter // DataStreamWriter from AI SDK @@ -193,6 +205,16 @@ export interface TextToSqlAgentStreamInput { dataStream: any } +export interface AuditorAgentInput { + model: any + messages: ChatMessage[] + originalQuestion: string + reformulatedQuestion: string + dataSummary: import('./utils/data-summary').DataSummary + attemptNumber: number + previousFeedback?: string +} + export interface AgentResponseCompleteParams { userPrompt: string responseData: ChatResponse diff --git a/frontend/lib/chat/utils/data-summary.ts b/frontend/lib/chat/utils/data-summary.ts new file mode 100644 index 000000000..77d0b6801 --- /dev/null +++ b/frontend/lib/chat/utils/data-summary.ts @@ -0,0 +1,135 @@ +// Copyright (c) 2025 The Linux Foundation and each contributor. +// SPDX-License-Identifier: MIT + +export interface DataSummary { + rowCount: number + columns: string[] + columnStats: Record +} + +export interface ColumnStats { + type: 'numeric' | 'string' | 'date' | 'boolean' + nullCount: number + nullPercentage: number + + // Numeric stats + min?: number + max?: number + sum?: number + avg?: number + hasAllZeros?: boolean + + // String/categorical stats + distinctCount?: number + minLength?: number + maxLength?: number + + // Date stats + earliestDate?: string + latestDate?: string + dateRange?: string +} + +/** + * Generate statistical summary of dataset + * Token-efficient: ~400-500 tokens for typical dataset + * No raw data samples sent to LLM - only statistics + * + * @param data - Array of data rows + * @returns Statistical summary optimized for auditor validation + */ +export function generateDataSummary>(data: T[]): DataSummary { + if (!data || data.length === 0) { + return { + rowCount: 0, + columns: [], + columnStats: {}, + } + } + + const columns = Object.keys(data[0] || {}) + const columnStats: Record = {} + + for (const col of columns) { + const values = data.map((row) => row[col]) + const nonNullValues = values.filter((v) => v !== null && v !== undefined && v !== '') + const nullCount = data.length - nonNullValues.length + const nullPercentage = Math.round((nullCount / data.length) * 100) + + if (nonNullValues.length === 0) { + // All nulls - mark as string type with full null percentage + columnStats[col] = { + type: 'string', + nullCount, + nullPercentage, + } + continue + } + + const firstValue = nonNullValues[0] + let stats: ColumnStats + + // Numeric columns + if (typeof firstValue === 'number') { + const numericValues = nonNullValues as number[] + const sum = numericValues.reduce((a, b) => a + b, 0) + const hasAllZeros = numericValues.every((v) => v === 0) + + stats = { + type: 'numeric', + nullCount, + nullPercentage, + min: Math.min(...numericValues), + max: Math.max(...numericValues), + sum, + avg: Math.round((sum / numericValues.length) * 100) / 100, // Round to 2 decimals + hasAllZeros, + } + } + // Boolean columns + else if (typeof firstValue === 'boolean') { + stats = { + type: 'boolean', + nullCount, + nullPercentage, + distinctCount: new Set(nonNullValues).size, + } + } + // Date columns (detect date strings) + else if (typeof firstValue === 'string' && !isNaN(Date.parse(firstValue))) { + const dates = nonNullValues.map((v) => new Date(v as string)) + const earliest = new Date(Math.min(...dates.map((d) => d.getTime()))) + const latest = new Date(Math.max(...dates.map((d) => d.getTime()))) + + stats = { + type: 'date', + nullCount, + nullPercentage, + distinctCount: new Set(nonNullValues).size, + earliestDate: earliest.toISOString().split('T')[0], + latestDate: latest.toISOString().split('T')[0], + dateRange: `${earliest.toISOString().split('T')[0]} to ${latest.toISOString().split('T')[0]}`, + } + } + // String columns + else { + const stringValues = nonNullValues.map((v) => String(v)) + stats = { + type: 'string', + nullCount, + nullPercentage, + distinctCount: new Set(stringValues).size, + minLength: Math.min(...stringValues.map((s) => s.length)), + maxLength: Math.max(...stringValues.map((s) => s.length)), + } + } + + columnStats[col] = stats + } + + return { + rowCount: data.length, + columns, + columnStats, + } +} diff --git a/frontend/server/repo/chat.repo.ts b/frontend/server/repo/chat.repo.ts index d4b3eeeee..8ee032a22 100644 --- a/frontend/server/repo/chat.repo.ts +++ b/frontend/server/repo/chat.repo.ts @@ -56,6 +56,24 @@ export interface ChatResponse { outputTokens?: number } +export interface IChatResponseDb { + id: string + created_at: Date + created_by: string + user_prompt: string + router_response: RouterDecisionAction + router_reason: string + pipe_instructions: PipeInstructions | null + sql_query: string | null + clarification_question: string | null + model: string + input_tokens: number | null + output_tokens: number | null + feedback: number | null + conversation_id: string | null + +} + export class ChatRepository { constructor(private pool: Pool) {} @@ -135,7 +153,7 @@ export class ChatRepository { return result.rows.length > 0 ? result.rows[0] : null } - async getChatResponsesByConversation(conversationId: string): Promise { + async getChatResponsesByConversation(conversationId: string): Promise { const query = ` SELECT * FROM chat_responses WHERE conversation_id = $1 From 2a830a7a08481f505a40c64d139c43b701d71a01 Mon Sep 17 00:00:00 2001 From: anilb Date: Wed, 8 Oct 2025 14:42:23 +0200 Subject: [PATCH 4/7] feat(chat): quality improvements - self healing text-to-sql agent - text-to-sql agent sql api restriction improvements - improved ui - auditor and pipe agent improvements Signed-off-by: anilb --- .../components/chat-history/chat-result.vue | 33 --- .../components/results/results-section.vue | 13 +- frontend/lib/chat/agents/router.ts | 7 +- frontend/lib/chat/agents/text-to-sql.ts | 33 ++- frontend/lib/chat/data-copilot.ts | 229 ++++++++++++---- frontend/lib/chat/enums.ts | 3 + frontend/lib/chat/prompts/auditor.ts | 37 ++- frontend/lib/chat/prompts/pipe.ts | 13 +- frontend/lib/chat/prompts/router.ts | 109 +++++++- frontend/lib/chat/prompts/text-to-sql.ts | 258 ++++++++++-------- .../lib/chat/prompts/tinybird-functions.md | 78 ++++++ .../lib/chat/prompts/tinybird-patterns.md | 135 +++++++++ frontend/lib/chat/types.ts | 8 + frontend/nuxt.config.ts | 1 + frontend/server/middleware/database.ts | 13 +- 15 files changed, 725 insertions(+), 245 deletions(-) create mode 100644 frontend/lib/chat/prompts/tinybird-functions.md create mode 100644 frontend/lib/chat/prompts/tinybird-patterns.md diff --git a/frontend/app/components/shared/modules/copilot/components/chat-history/chat-result.vue b/frontend/app/components/shared/modules/copilot/components/chat-history/chat-result.vue index 495888bd6..b6c03b183 100644 --- a/frontend/app/components/shared/modules/copilot/components/chat-history/chat-result.vue +++ b/frontend/app/components/shared/modules/copilot/components/chat-history/chat-result.vue @@ -24,33 +24,11 @@ SPDX-License-Identifier: MIT > {{ reasoning }} -
{{ message.content }}
- - - - - -