From c157adde9e15e1422f0649e1c4e3a4d51f1dbe13 Mon Sep 17 00:00:00 2001 From: jeremyphilemon Date: Fri, 27 Sep 2024 20:37:12 +0530 Subject: [PATCH 1/4] feat (core): support converting attachments to file parts --- .../ai/core/prompt/attachments-to-parts.ts | 28 +++++++++++++-- .../prompt/convert-to-core-messages.test.ts | 18 ++++++++++ pnpm-lock.yaml | 36 +++++++++---------- 3 files changed, 62 insertions(+), 20 deletions(-) diff --git a/packages/ai/core/prompt/attachments-to-parts.ts b/packages/ai/core/prompt/attachments-to-parts.ts index f510807845f6..9594691fcb4d 100644 --- a/packages/ai/core/prompt/attachments-to-parts.ts +++ b/packages/ai/core/prompt/attachments-to-parts.ts @@ -1,11 +1,11 @@ import { Attachment } from '@ai-sdk/ui-utils'; -import { ImagePart, TextPart } from './content-part'; +import { FilePart, ImagePart, TextPart } from './content-part'; import { convertDataContentToUint8Array, convertUint8ArrayToText, } from './data-content'; -type ContentPart = TextPart | ImagePart; +type ContentPart = TextPart | ImagePart | FilePart; /** * Converts a list of attachments to a list of content parts @@ -29,6 +29,18 @@ export function attachmentsToParts(attachments: Attachment[]): ContentPart[] { case 'https:': { if (attachment.contentType?.startsWith('image/')) { parts.push({ type: 'image', image: url }); + } else { + if (!attachment.contentType) { + throw new Error( + 'If the attachment is not an image or text, it must specify a content type', + ); + } + + parts.push({ + type: 'file', + data: url, + mimeType: attachment.contentType, + }); } break; } @@ -61,6 +73,18 @@ export function attachmentsToParts(attachments: Attachment[]): ContentPart[] { convertDataContentToUint8Array(base64Content), ), }); + } else { + if (!attachment.contentType) { + throw new Error( + 'If the attachment is not an image or text, it must specify a content type', + ); + } + + parts.push({ + type: 'file', + data: convertDataContentToUint8Array(base64Content), + mimeType: attachment.contentType, + }); } break; diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index d839a15cd766..55aad2675597 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -87,6 +87,24 @@ describe('user message', () => { }).toThrow('Invalid URL: invalid-url'); }); + it('should throw an error for file attachments without contentType', () => { + const attachment: Attachment = { + url: 'data:application/pdf;base64,dGVzdA==', + }; + + expect(() => { + convertToCoreMessages([ + { + role: 'user', + content: 'Check this file', + experimental_attachments: [attachment], + }, + ]); + }).toThrow( + 'If the attachment is not an image or text, it must specify a content type', + ); + }); + it('should throw an error for invalid data URL format', () => { const attachment: Attachment = { contentType: 'image/jpeg', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4478ae312abb..eba766d510d2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1243,13 +1243,13 @@ importers: version: link:../../.. next: specifier: canary - version: 15.0.0-canary.171(@playwright/test@1.46.0)(react-dom@19.0.0-rc-67fee58b-20240926)(react@19.0.0-rc-67fee58b-20240926) + version: 15.0.0-canary.171(@playwright/test@1.46.0)(react-dom@19.0.0-rc-04bd67a4-20240924)(react@19.0.0-rc-04bd67a4-20240924) react: specifier: rc - version: 19.0.0-rc-67fee58b-20240926 + version: 19.0.0-rc-04bd67a4-20240924 react-dom: specifier: rc - version: 19.0.0-rc-67fee58b-20240926(react@19.0.0-rc-67fee58b-20240926) + version: 19.0.0-rc-04bd67a4-20240924(react@19.0.0-rc-04bd67a4-20240924) packages/amazon-bedrock: dependencies: @@ -20166,7 +20166,7 @@ packages: - babel-plugin-macros dev: false - /next@15.0.0-canary.171(@playwright/test@1.46.0)(react-dom@19.0.0-rc-67fee58b-20240926)(react@19.0.0-rc-67fee58b-20240926): + /next@15.0.0-canary.171(@playwright/test@1.46.0)(react-dom@19.0.0-rc-04bd67a4-20240924)(react@19.0.0-rc-04bd67a4-20240924): resolution: {integrity: sha512-Yic4FzN0CiJrCRI++FAkONZsen0vY5XYe9UkliEFGAmbO9oVx24opzC/JA0N36dTjYtsLFVxcRafoT1dTlJmwg==} engines: {node: '>=18.18.0'} hasBin: true @@ -20194,9 +20194,9 @@ packages: busboy: 1.6.0 caniuse-lite: 1.0.30001649 postcss: 8.4.31 - react: 19.0.0-rc-67fee58b-20240926 - react-dom: 19.0.0-rc-67fee58b-20240926(react@19.0.0-rc-67fee58b-20240926) - styled-jsx: 5.1.6(react@19.0.0-rc-67fee58b-20240926) + react: 19.0.0-rc-04bd67a4-20240924 + react-dom: 19.0.0-rc-04bd67a4-20240924(react@19.0.0-rc-04bd67a4-20240924) + styled-jsx: 5.1.6(react@19.0.0-rc-04bd67a4-20240924) optionalDependencies: '@next/swc-darwin-arm64': 15.0.0-canary.171 '@next/swc-darwin-x64': 15.0.0-canary.171 @@ -22210,13 +22210,13 @@ packages: react: 18.3.1 scheduler: 0.23.2 - /react-dom@19.0.0-rc-67fee58b-20240926(react@19.0.0-rc-67fee58b-20240926): - resolution: {integrity: sha512-fyLfI8S7pXwVguPdS9z8gY8SWOLP3ZiraOAYM0mAeMrjeU2PbN2lXx4huN5WlbBkCMCKNxTkLJpOpWfJ0Gi2jw==} + /react-dom@19.0.0-rc-04bd67a4-20240924(react@19.0.0-rc-04bd67a4-20240924): + resolution: {integrity: sha512-9MbUcK/HGBwzdDsbgHO6TtSMBO4/lUPyxVhTuKz2shznd4t7T17ObtM7m39JhPaz0uCdlOeErWkzzCKNxt57/g==} peerDependencies: - react: 19.0.0-rc-67fee58b-20240926 + react: 19.0.0-rc-04bd67a4-20240924 dependencies: - react: 19.0.0-rc-67fee58b-20240926 - scheduler: 0.25.0-rc-67fee58b-20240926 + react: 19.0.0-rc-04bd67a4-20240924 + scheduler: 0.25.0-rc-04bd67a4-20240924 dev: false /react-dom@19.0.0-rc-cc1ec60d0d-20240607(react@19.0.0-rc-cc1ec60d0d-20240607): @@ -22271,8 +22271,8 @@ packages: dependencies: loose-envify: 1.4.0 - /react@19.0.0-rc-67fee58b-20240926: - resolution: {integrity: sha512-VCp63TASDf1XDh+oGCV0kV/jTFOn2sxg5HrbuMyjbZXHdTzcNvUw/sEbrtC1QoU+vledZiR4AV7wQW7Ka9qruw==} + /react@19.0.0-rc-04bd67a4-20240924: + resolution: {integrity: sha512-oldOS+RU4b0cXzOxK0TuJXVhcMMtsjEFew0GQFsRTcYH1vwlWnwcAEBwRRWczZfhykpA1VFcCzS6+sZ3qdnBzw==} engines: {node: '>=0.10.0'} dev: false @@ -22747,8 +22747,8 @@ packages: dependencies: loose-envify: 1.4.0 - /scheduler@0.25.0-rc-67fee58b-20240926: - resolution: {integrity: sha512-saTMMKVkRbIoAyrqnFgRa864RnAxiU85yXZaBXBAXQDaXPz+EdtjZy8BQCnSRZwq8CHvXvzDI8Ox9icBftZ/nw==} + /scheduler@0.25.0-rc-04bd67a4-20240924: + resolution: {integrity: sha512-gAD5Ob6qimTeOixl9Icmzrgs6572HsxD3AoK/b6VAMKgoDhIa/NXSD8Po1f4SFNysAXVBDCpT1CSvFwy8MtpRQ==} dev: false /scheduler@0.25.0-rc-cc1ec60d0d-20240607: @@ -23536,7 +23536,7 @@ packages: react: 18.3.1 dev: false - /styled-jsx@5.1.6(react@19.0.0-rc-67fee58b-20240926): + /styled-jsx@5.1.6(react@19.0.0-rc-04bd67a4-20240924): resolution: {integrity: sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==} engines: {node: '>= 12.0.0'} peerDependencies: @@ -23550,7 +23550,7 @@ packages: optional: true dependencies: client-only: 0.0.1 - react: 19.0.0-rc-67fee58b-20240926 + react: 19.0.0-rc-04bd67a4-20240924 dev: false /styled-jsx@5.1.6(react@19.0.0-rc-cc1ec60d0d-20240607): From 7140eb7282ee359918d687d6c4077046536c723f Mon Sep 17 00:00:00 2001 From: jeremyphilemon Date: Fri, 27 Sep 2024 20:42:30 +0530 Subject: [PATCH 2/4] Update error message --- packages/ai/core/prompt/attachments-to-parts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ai/core/prompt/attachments-to-parts.ts b/packages/ai/core/prompt/attachments-to-parts.ts index 9594691fcb4d..c019b9a6c099 100644 --- a/packages/ai/core/prompt/attachments-to-parts.ts +++ b/packages/ai/core/prompt/attachments-to-parts.ts @@ -32,7 +32,7 @@ export function attachmentsToParts(attachments: Attachment[]): ContentPart[] { } else { if (!attachment.contentType) { throw new Error( - 'If the attachment is not an image or text, it must specify a content type', + 'If the attachment is not an image, it must specify a content type', ); } From 81ec27a32721cb954795677ea324119804058adf Mon Sep 17 00:00:00 2001 From: jeremyphilemon Date: Fri, 27 Sep 2024 20:46:15 +0530 Subject: [PATCH 3/4] Add changeset --- .changeset/silly-pigs-judge.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/silly-pigs-judge.md diff --git a/.changeset/silly-pigs-judge.md b/.changeset/silly-pigs-judge.md new file mode 100644 index 000000000000..589c7ae8a716 --- /dev/null +++ b/.changeset/silly-pigs-judge.md @@ -0,0 +1,5 @@ +--- +'ai': patch +--- + +feat (core): support converting attachments to file parts From 42c9e277900b21318228fa9623ff9b0de730da4b Mon Sep 17 00:00:00 2001 From: jeremyphilemon Date: Fri, 27 Sep 2024 21:22:55 +0530 Subject: [PATCH 4/4] Preserve base64 and update tests --- .../ai/core/prompt/attachments-to-parts.ts | 2 +- .../prompt/convert-to-core-messages.test.ts | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/packages/ai/core/prompt/attachments-to-parts.ts b/packages/ai/core/prompt/attachments-to-parts.ts index c019b9a6c099..91adca52f2e2 100644 --- a/packages/ai/core/prompt/attachments-to-parts.ts +++ b/packages/ai/core/prompt/attachments-to-parts.ts @@ -82,7 +82,7 @@ export function attachmentsToParts(attachments: Attachment[]): ContentPart[] { parts.push({ type: 'file', - data: convertDataContentToUint8Array(base64Content), + data: base64Content, mimeType: attachment.contentType, }); } diff --git a/packages/ai/core/prompt/convert-to-core-messages.test.ts b/packages/ai/core/prompt/convert-to-core-messages.test.ts index 55aad2675597..b7e80bb43b3f 100644 --- a/packages/ai/core/prompt/convert-to-core-messages.test.ts +++ b/packages/ai/core/prompt/convert-to-core-messages.test.ts @@ -45,6 +45,35 @@ describe('user message', () => { ]); }); + it('should handle user message with attachments (file)', () => { + const attachment: Attachment = { + contentType: 'application/pdf', + url: 'https://example.com/document.pdf', + }; + + const result = convertToCoreMessages([ + { + role: 'user', + content: 'Check this document', + experimental_attachments: [attachment], + }, + ]); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'Check this document' }, + { + type: 'file', + data: new URL('https://example.com/document.pdf'), + mimeType: 'application/pdf', + }, + ], + }, + ]); + }); + it('should handle user message with attachment URLs', () => { const attachment: Attachment = { contentType: 'image/jpeg', @@ -70,6 +99,35 @@ describe('user message', () => { ]); }); + it('should handle user message with attachment URLs (file)', () => { + const attachment: Attachment = { + contentType: 'application/pdf', + url: 'data:application/pdf;base64,dGVzdA==', + }; + + const result = convertToCoreMessages([ + { + role: 'user', + content: 'Check this document', + experimental_attachments: [attachment], + }, + ]); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'Check this document' }, + { + type: 'file', + data: 'dGVzdA==', + mimeType: 'application/pdf', + }, + ], + }, + ]); + }); + it('should throw an error for invalid attachment URLs', () => { const attachment: Attachment = { contentType: 'image/jpeg',