feat: data copilot improvements

database/migrations/V1759392166__addAskClarificationRouterResponseType.sql

@@ -0,0 +1,9 @@
+-- Drop the existing check constraint
+ALTER TABLE chat_responses DROP CONSTRAINT chat_responses_router_response_check;
+
+-- Add the new check constraint with 'ask_clarification'
+ALTER TABLE chat_responses ADD CONSTRAINT chat_responses_router_response_check
+    CHECK (router_response IN ('pipes', 'create_query', 'stop', 'ask_clarification'));
+
+-- Add clarification_question column to store the clarification question
+ALTER TABLE chat_responses ADD COLUMN IF NOT EXISTS clarification_question TEXT;

database/migrations/V1759927411__createChatResponseAgentStepsTable.sql

@@ -0,0 +1,27 @@
+-- Create enum for agent types
+CREATE TYPE agent_type AS ENUM ('ROUTER', 'PIPE', 'TEXT_TO_SQL', 'AUDITOR', 'CHART', 'EXECUTE_INSTRUCTIONS');
+
+-- Create table to track individual agent execution steps
+CREATE TABLE IF NOT EXISTS chat_response_agent_steps (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    chat_response_id UUID NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    model TEXT,
+    agent agent_type NOT NULL,
+    response JSONB,
+    input_tokens INTEGER DEFAULT 0,
+    output_tokens INTEGER DEFAULT 0,
+    response_time_seconds NUMERIC NOT NULL DEFAULT 0,
+    instructions TEXT,
+    error_message TEXT,
+
+    CONSTRAINT fk_chat_response
+        FOREIGN KEY (chat_response_id)
+        REFERENCES chat_responses(id)
+        ON DELETE CASCADE
+);
+
+-- Create indexes for efficient querying
+CREATE INDEX idx_agent_steps_chat_response_id ON chat_response_agent_steps(chat_response_id);
+CREATE INDEX idx_agent_steps_created_at ON chat_response_agent_steps(created_at DESC);
+CREATE INDEX idx_agent_steps_agent_type ON chat_response_agent_steps(agent);

database/migrations/V1759927412__makeChatResponsesNullable.sql

@@ -0,0 +1,18 @@
+-- Make router fields nullable to allow early creation of chat_responses
+ALTER TABLE chat_responses ALTER COLUMN router_response DROP NOT NULL;
+ALTER TABLE chat_responses ALTER COLUMN router_reason DROP NOT NULL;
+
+-- Drop existing constraints
+ALTER TABLE chat_responses DROP CONSTRAINT IF EXISTS chat_responses_router_response_check;
+ALTER TABLE chat_responses DROP CONSTRAINT IF EXISTS check_pipe_instructions;
+
+-- Add new constraint that allows NULL or valid enum values
+ALTER TABLE chat_responses ADD CONSTRAINT chat_responses_router_response_check
+    CHECK (router_response IS NULL OR router_response IN ('pipes', 'create_query', 'stop', 'ask_clarification'));
+
+-- Recreate pipe_instructions check with NULL handling
+ALTER TABLE chat_responses ADD CONSTRAINT check_pipe_instructions CHECK (
+    router_response IS NULL OR
+    (router_response = 'pipes' AND pipe_instructions IS NOT NULL) OR
+    (router_response != 'pipes' AND pipe_instructions IS NULL)
+);

frontend/app/components/shared/modules/copilot/components/chat-history/chat-result.vue

@@ -24,33 +24,11 @@ SPDX-License-Identifier: MIT
     >
       {{ reasoning }}
     </div>
-    <div class="my-4">{{ message.content }}</div>
-
-    <span 
-      class="flex items-center p-3 border border-solid border-neutral-200 
-      rounded-xl bg-white justify-between cursor-pointer hover:bg-neutral-50"
-      @click="emit('select')"
-    >
-      <lfx-chat-result-label
-        :version="version"
-        :label="getTitle(message.id)"
-      />
-      <lfx-icon
-        v-if="!isSelected"
-        name="arrow-rotate-left"
-        :size="16"
-        class="text-neutral-400"
-      />
-    </span>
-
   </div>
 </template>
 <script setup lang="ts">
 import { computed, ref } from 'vue';
-import { storeToRefs } from 'pinia';
 import type { AIMessage } from '../../types/copilot.types'
-import LfxChatResultLabel from '../shared/result-label.vue'
-import { useCopilotStore } from '../../store/copilot.store';
 import LfxChatLabel from './chat-label.vue'
 import LfxIcon from '~/components/uikit/icon/icon.vue'
 
@@ -60,23 +38,12 @@ const props = defineProps<{
   isSelected: boolean | undefined
 }>()
 
-const { resultData } = storeToRefs(useCopilotStore());
-
 const isReasonExpanded = ref(false);
-// TODO: Implement feedback backend
-
-const emit = defineEmits<{
-  (e: 'select'): void
-}>()
 
 const reasoning = computed(() => {
   return props.message.explanation || props.message.sql;
 })
 
-const getTitle = (id: string) => {
-  const result = resultData.value.find(r => String(r.id) === String(id));
-  return result?.title || 'Loading...';
-}
 </script>
 
 <script lang="ts">

frontend/app/components/shared/modules/copilot/components/results/results-section.vue

@@ -75,7 +75,7 @@ SPDX-License-Identifier: MIT
 </template>
 
 <script setup lang="ts">
-import { computed, ref, watch } from 'vue';
+import { computed, ref } from 'vue';
 import { storeToRefs } from 'pinia';
 import LfxCopilotLoadingState from '../shared/loading-state.vue';
 import LfxCopilotEmptyResult from '../info/empty-result.vue';
@@ -98,9 +98,9 @@ const props = defineProps<{
 const { resultData, selectedResultId } = storeToRefs(useCopilotStore());
 
 const isChartError = ref(false);
-const selectedTab = ref('chart');
+const selectedTab = ref('data');
 const isSnapshotModalOpen = ref(false);
-const isChartLoading = ref(true);
+const isChartLoading = ref(false);
 const selectedResultConfig = computed<Config | null>(() => {
   return resultData.value.find(result => result.id === selectedResultId.value)?.chartConfig || null;
 });
@@ -147,11 +147,8 @@ const handleChartLoading = (value: boolean) => {
   emit('update:isChartLoading', value);
 }
 
-watch(isChartLoading, (value) => {
-  if (value) {
-    selectedTab.value = 'chart';
-  } 
-})
+// Removed watcher that forced chart tab selection during loading
+// Now users stay on data tab by default and can manually switch to chart if desired
 </script>
 
 <script lang="ts">

frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts

@@ -10,11 +10,8 @@ import type {
   MessageStatus,
 } from '../types/copilot.types'
 import type { CopilotParams } from '../types/copilot.types'
-// import testData from './test.json'
-import testData3 from './test3.json'
 import type { Project } from '~~/types/project'
 
-export const tempData = testData3 as AIMessage[]
 class CopilotApiService {
   // Generate unique ID for messages
   generateId = () => Date.now().toString(36) + Math.random().toString(36).substring(2);
@@ -228,21 +225,41 @@ class CopilotApiService {
           statusCallBack(statusText)
 
           if (
-            data.type === 'router-status' &&
-            (data.status === 'complete' || data.status === 'error')
+            (data.type === 'router-status' || data.type === 'auditor-status') &&
+            (
+             data.status === 'complete' || 
+             data.status === 'error' || 
+             data.status === 'ask_clarification' || 
+             data.status === 'validated' 
+            )
           ) {
             if (!assistantMessageId) {
               assistantMessageId = this.generateId()
 
+              let content: string
+              if (data.status === 'ask_clarification') {
+                content = data.question || 'I need more information to answer your question.'
+              }
+              else if (data.status === 'error') {
+                content = data.error || 'An error occurred.'
+              }
+              else if (data.status === 'validated') {
+                content = data.summary || 'Data validated successfully.'
+              }
+              else {
+                content = data.reasoning || 'Analysis complete.'
+              }
+
               messageCallBack(
                 {
                   id: assistantMessageId,
                   role: 'assistant',
                   type: 'router-status',
                   status: data.status,
-                  content: data.reasoning || '',
+                  content,
                   explanation: data.status === 'error' ? data.error : undefined,
                   routerReasoning: data.reasoning,
+                  question: data.question, // Include the clarification question
                   timestamp: Date.now(),
                 },
                 -1,
@@ -338,6 +355,8 @@ class CopilotApiService {
     switch (type) {
       case 'router-status':
         return this.getStatusTextRouterStatus(status, reasoning, error)
+      case 'auditor-status':
+        return this.getStatusTextAuditorStatus(status, reasoning)
       case 'sql-result':
         return 'SQL query executed successfully'
       case 'pipe-result':
@@ -353,10 +372,27 @@ class CopilotApiService {
         return 'Analyzing your question...'
       case 'complete':
         return reasoning ? `Analysis: ${reasoning}` : 'Analysis complete'
+      case 'ask_clarification':
+        return reasoning || 'I need more information to answer your question.'
       default:
         return `Error: ${error || 'An error occurred'}`
     }
   }
+
+  getStatusTextAuditorStatus(status: string, reasoning: string): string {
+    switch (status) {
+      case 'validating':
+        return 'Validating data quality...'
+      case 'validated':
+        return reasoning ? `Validation passed: ${reasoning}` : 'Data validated successfully'
+      case 'retrying':
+        return 'Retrying with improved query...'
+      case 'max_retries':
+        return reasoning ? `Validation feedback: ${reasoning}` : 'Maximum validation attempts reached'
+      default:
+        return 'Validating...'
+    }
+  }
 }
 
 export const copilotApiService = new CopilotApiService()