feat: new chat protocol extension points (#170)

Tested working.

packages/chat-component/src/components/chat-controller.ts

@@ -138,7 +138,7 @@ export class ChatController implements ReactiveController {
       followingSteps.push(...(processedText.arrays[1] as string[]));
       followupQuestions.push(...(processedText.arrays[2] as string[]));
       thoughts = generatedResponse.context?.thoughts ?? '';
-      dataPoints = generatedResponse.context?.data_points ?? [];
+      dataPoints = generatedResponse.context?.data_points?.text ?? [];
 
       await updateChatWithMessageOrChunk(messageToUpdate, false);
     }

packages/chat-component/src/core/parser/index.ts

@@ -49,7 +49,7 @@ export async function parseStreamedMessages({
 
     const { content, context } = chunk.choices[0].delta;
     if (context?.data_points) {
-      updatedEntry.dataPoints = context.data_points ?? [];
+      updatedEntry.dataPoints = context.data_points?.text ?? [];
       updatedEntry.thoughts = context.thoughts ?? '';
 
       continue;

packages/chat-component/src/types.d.ts

@@ -82,7 +82,10 @@ declare interface BotResponseChunk {
 
 declare type BotResponseMessage = Message & {
   context?: Record<string, any> & {
-    data_points?: string[];
+    data_points?: {
+      text?: string[];
+      images?: string[];
+    };
     thoughts?: string;
   };
   session_state?: Record<string, any>;

packages/search/src/lib/approaches/approach.ts

@@ -19,7 +19,10 @@ export interface ApproachResponseChunk {
 
 export type ApproachResponseMessage = Message & {
   context?: Record<string, any> & {
-    data_points?: string[];
+    data_points?: {
+      text?: string[];
+      images?: string[];
+    };
     thoughts?: string;
   };
   session_state?: Record<string, any>;

packages/search/src/lib/approaches/ask-read-retrieve-read.ts

@@ -110,7 +110,9 @@ export class AskReadRetrieveRead extends ApproachBase implements AskApproach {
             content: answer,
             role: 'assistant' as const,
             context: {
-              data_points: searchResults,
+              data_points: {
+                text: searchResults,
+              },
               thoughts: htmlTracer.getAndResetLog(),
             },
           },

packages/search/src/lib/approaches/ask-retrieve-then-read.ts

@@ -79,7 +79,9 @@ export class AskRetrieveThenRead extends ApproachBase implements AskApproach {
             role: 'assistant' as const,
             content: chatCompletion.choices[0].message.content ?? '',
             context: {
-              data_points: results,
+              data_points: {
+                text: results,
+              },
               thoughts: `Question:<br>${query}<br><br>Prompt:<br>${messageToDisplay.replace('\n', '<br>')}`,
             },
           },

packages/search/src/lib/approaches/chat-read-retrieve-read.ts

@@ -78,7 +78,9 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
             content: chatContent,
             role: 'assistant',
             context: {
-              data_points: dataPoints,
+              data_points: {
+                text: dataPoints,
+              },
               thoughts: thoughts,
             },
           },
@@ -108,7 +110,7 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
               content: chunk.choices[0].delta.content ?? '',
               role: 'assistant' as const,
               context: {
-                data_points: id === 0 ? dataPoints : undefined,
+                data_points: id === 0 ? { text: dataPoints } : undefined,
                 thoughts: id === 0 ? thoughts : undefined,
               },
             },

packages/search/src/plugins/schemas.ts

@@ -59,8 +59,17 @@ export const messageSchema = {
       type: 'object',
       properties: {
         data_points: {
-          type: 'array',
-          items: { type: 'string' },
+          type: 'object',
+          properties: {
+            text: {
+              type: 'array',
+              items: { type: 'string' },
+            },
+            images: {
+              type: 'array',
+              items: { type: 'string' },
+            },
+          },
         },
         thoughts: { type: 'string' },
       },

tests/e2e/hars/error-chat-response-stream.har

@@ -61,7 +61,7 @@
           "content": {
             "size": -1,
             "mimeType": "application/x-ndjson",
-            "text": "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{\"data_points\":[],\"thoughts\":\"Here's how to do it: 1.\"}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"To\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" search\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" book\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" rentals\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"content_filter\"}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" on\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"stop\"}],\"object\":\"chat.completion.chunk\"}\n"
+            "text": "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{\"data_points\":{},\"thoughts\":\"Here's how to do it: 1.\"}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"To\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" search\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" book\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" rentals\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"content_filter\"}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" on\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"stop\"}],\"object\":\"chat.completion.chunk\"}\n"
           },
           "headersSize": -1,
           "bodySize": -1,