From 542cad492db96e5d41e861c6622fbb56733b19b2 Mon Sep 17 00:00:00 2001 From: Subin C Date: Tue, 24 Sep 2024 18:34:44 +0530 Subject: [PATCH 1/4] Image url validation #2838 --- .../ai/core/prompt/convert-to-language-model-prompt.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/ai/core/prompt/convert-to-language-model-prompt.ts b/packages/ai/core/prompt/convert-to-language-model-prompt.ts index 7e5fb2bd3adb..ab0dfa708f2d 100644 --- a/packages/ai/core/prompt/convert-to-language-model-prompt.ts +++ b/packages/ai/core/prompt/convert-to-language-model-prompt.ts @@ -131,6 +131,10 @@ export function convertToLanguageModelMessage( // try to convert string image parts to urls if (typeof part.image === 'string') { + const invalidUrlPattern = /\s/; + if (invalidUrlPattern.test(part.image)) { + throw new Error(`Invalid Image URL: ${part.image}`); + } try { const url = new URL(part.image); @@ -146,7 +150,8 @@ export function convertToLanguageModelMessage( part.experimental_providerMetadata, }; } else { - const downloadedImage = downloadedImages[part.image]; + const downloadedImage = + downloadedImages[url.toString()]; return { type: 'image', image: downloadedImage.data, From 7d425c40d96a33230b1def52fca8ce2cadf98abb Mon Sep 17 00:00:00 2001 From: Subin C Date: Mon, 30 Sep 2024 19:46:23 +0530 Subject: [PATCH 2/4] Include the createdAt field in AssistantResponse and useAssistant #3129 --- packages/ai/streams/assistant-response.ts | 1 + packages/react/src/use-assistant.ts | 2 + packages/react/src/use-assistant.ui.test.tsx | 86 ++++++++++++++++++++ packages/ui-utils/src/stream-parts.ts | 2 +- packages/ui-utils/src/types.ts | 1 + 5 files changed, 91 insertions(+), 1 deletion(-) diff --git a/packages/ai/streams/assistant-response.ts b/packages/ai/streams/assistant-response.ts index c4d872483c8f..40e4f83a880a 100644 --- a/packages/ai/streams/assistant-response.ts +++ b/packages/ai/streams/assistant-response.ts @@ -92,6 +92,7 @@ export function AssistantResponse( textEncoder.encode( formatStreamPart('assistant_message', { id: value.data.id, + createdAt: value.data.created_at ? new Date(value.data.created_at * 1000) : undefined, role: 'assistant', content: [{ type: 'text', text: { value: '' } }], }), diff --git a/packages/react/src/use-assistant.ts b/packages/react/src/use-assistant.ts index 7a366f451fe3..b3ead342bda7 100644 --- a/packages/react/src/use-assistant.ts +++ b/packages/react/src/use-assistant.ts @@ -186,6 +186,7 @@ export function useAssistant({ { id: value.id, role: value.role, + createdAt: value.createdAt, content: value.content[0].text.value, }, ]); @@ -201,6 +202,7 @@ export function useAssistant({ { id: lastMessage.id, role: lastMessage.role, + createdAt: lastMessage.createdAt, content: lastMessage.content + value, }, ]; diff --git a/packages/react/src/use-assistant.ui.test.tsx b/packages/react/src/use-assistant.ui.test.tsx index 37dd58405b49..80f6164946f1 100644 --- a/packages/react/src/use-assistant.ui.test.tsx +++ b/packages/react/src/use-assistant.ui.test.tsx @@ -23,6 +23,7 @@ describe('stream data stream', () => {
{m.role === 'user' ? 'User: ' : 'AI: '} {m.content} + {m.createdAt && ` (Created at: ${m.createdAt})`}
))} @@ -85,6 +86,91 @@ describe('stream data stream', () => { ); }); + it('should show streamed response with createdAt', async () => { + const { requestBody } = mockFetchDataStream({ + url: 'https://example.com/api/assistant', + chunks: [ + formatStreamPart('assistant_control_data', { + threadId: 't0', + messageId: 'm0', + }), + formatStreamPart('assistant_message', { + id: 'm0', + role: 'assistant', + createdAt: new Date(1727703377 * 1000), + content: [{ type: 'text', text: { value: '' } }], + }), + // text parts: + '0:"Hello"\n', + '0:","\n', + '0:" world"\n', + '0:"."\n', + ], + }); + + await userEvent.click(screen.getByTestId('do-append')); + + await screen.findByTestId('message-0'); + expect(screen.getByTestId('message-0')).toHaveTextContent('User: hi'); + + await screen.findByTestId('message-1'); + const messageContent = screen.getByTestId('message-1').textContent; + console.log(messageContent); + + expect(messageContent).toContain('AI: Hello, world.'); + expect(messageContent).toContain(`Created at: ${new Date(1727703377 * 1000).toISOString()}`); + + // check that correct information was sent to the server: + expect(await requestBody).toStrictEqual( + JSON.stringify({ + threadId: null, + message: 'hi', + }), + ); + }); + + it('should show streamed response without createdAt', async () => { + const { requestBody } = mockFetchDataStream({ + url: 'https://example.com/api/assistant', + chunks: [ + formatStreamPart('assistant_control_data', { + threadId: 't0', + messageId: 'm0', + }), + formatStreamPart('assistant_message', { + id: 'm0', + role: 'assistant', + content: [{ type: 'text', text: { value: '' } }], + }), + // text parts: + '0:"Hello"\n', + '0:","\n', + '0:" world"\n', + '0:"."\n', + ], + }); + + await userEvent.click(screen.getByTestId('do-append')); + + await screen.findByTestId('message-0'); + expect(screen.getByTestId('message-0')).toHaveTextContent('User: hi'); + + await screen.findByTestId('message-1'); + const messageContent = screen.getByTestId('message-1').textContent; + console.log(messageContent); + + expect(messageContent).toContain('AI: Hello, world.'); + expect(messageContent).not.toContain('Created at:'); + + // check that correct information was sent to the server: + expect(await requestBody).toStrictEqual( + JSON.stringify({ + threadId: null, + message: 'hi', + }), + ); + }); + it('should show error response', async () => { mockFetchError({ statusCode: 500, errorMessage: 'Internal Error' }); diff --git a/packages/ui-utils/src/stream-parts.ts b/packages/ui-utils/src/stream-parts.ts index 4b5470943ad8..b169c66c7fdb 100644 --- a/packages/ui-utils/src/stream-parts.ts +++ b/packages/ui-utils/src/stream-parts.ts @@ -121,7 +121,7 @@ const assistantMessageStreamPart: StreamPart< return { type: 'assistant_message', - value: value as AssistantMessage, + value: value as unknown as AssistantMessage, }; }, }; diff --git a/packages/ui-utils/src/types.ts b/packages/ui-utils/src/types.ts index df59a5d42e34..53a30dd95a8e 100644 --- a/packages/ui-utils/src/types.ts +++ b/packages/ui-utils/src/types.ts @@ -604,6 +604,7 @@ export type JSONValue = export type AssistantMessage = { id: string; role: 'assistant'; + createdAt?: Date; content: Array<{ type: 'text'; text: { From 3c47bfac4178dbf951a515bff8b922bdb087bb94 Mon Sep 17 00:00:00 2001 From: Subin C Date: Mon, 30 Sep 2024 19:57:14 +0530 Subject: [PATCH 3/4] removed wrong commit --- packages/ai/core/prompt/convert-to-language-model-prompt.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/ai/core/prompt/convert-to-language-model-prompt.ts b/packages/ai/core/prompt/convert-to-language-model-prompt.ts index ab0dfa708f2d..f3f47adf9150 100644 --- a/packages/ai/core/prompt/convert-to-language-model-prompt.ts +++ b/packages/ai/core/prompt/convert-to-language-model-prompt.ts @@ -131,10 +131,6 @@ export function convertToLanguageModelMessage( // try to convert string image parts to urls if (typeof part.image === 'string') { - const invalidUrlPattern = /\s/; - if (invalidUrlPattern.test(part.image)) { - throw new Error(`Invalid Image URL: ${part.image}`); - } try { const url = new URL(part.image); @@ -150,8 +146,6 @@ export function convertToLanguageModelMessage( part.experimental_providerMetadata, }; } else { - const downloadedImage = - downloadedImages[url.toString()]; return { type: 'image', image: downloadedImage.data, From 3141104b8e7f10f48519bcca86f0e3661f51f0b2 Mon Sep 17 00:00:00 2001 From: Subin C Date: Mon, 30 Sep 2024 19:57:57 +0530 Subject: [PATCH 4/4] removed wrong commit --- packages/ai/core/prompt/convert-to-language-model-prompt.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ai/core/prompt/convert-to-language-model-prompt.ts b/packages/ai/core/prompt/convert-to-language-model-prompt.ts index f3f47adf9150..7e5fb2bd3adb 100644 --- a/packages/ai/core/prompt/convert-to-language-model-prompt.ts +++ b/packages/ai/core/prompt/convert-to-language-model-prompt.ts @@ -146,6 +146,7 @@ export function convertToLanguageModelMessage( part.experimental_providerMetadata, }; } else { + const downloadedImage = downloadedImages[part.image]; return { type: 'image', image: downloadedImage.data,