Skip to content

Commit 1d4d9b5

Browse files
authored
fix(ai): preserve OpenAI Chat image URLs (#48862)
1 parent 752d980 commit 1d4d9b5

2 files changed

Lines changed: 53 additions & 1 deletion

File tree

‎packages/ai/src/protocols/openai-chat.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,11 @@ const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart
323323
const media = ProviderShared.normalizeMedia(part)
324324
if (!media.mime.startsWith("image/"))
325325
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.mediaType}`)
326-
return { type: "image_url" as const, image_url: { url: media.dataUrl } }
326+
const url =
327+
typeof part.data === "string" && (part.data.startsWith("https://") || part.data.startsWith("http://"))
328+
? part.data
329+
: media.dataUrl
330+
return { type: "image_url" as const, image_url: { url } }
327331
})
328332

329333
const openAICompatibleReasoningContent = (native: unknown) =>

‎packages/ai/test/provider/openai-chat.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -698,6 +698,54 @@ describe("OpenAI Chat route", () => {
698698
}),
699699
)
700700

701+
it.effect("preserves HTTP and HTTPS image URLs in user content", () =>
702+
Effect.gen(function* () {
703+
const urls = ["https://example.com/image.png?size=64#preview", "http://example.com/image.jpg"]
704+
const prepared = yield* compileRequest(
705+
LLM.request({
706+
model,
707+
prompt: urls.map((data) => ({ type: "media" as const, mediaType: "image/png", data })),
708+
}),
709+
)
710+
expect(prepared.body.messages).toEqual([
711+
{ role: "user", content: urls.map((url) => ({ type: "image_url", image_url: { url } })) },
712+
])
713+
}),
714+
)
715+
716+
it.effect("preserves remote image URLs from tool results", () =>
717+
Effect.gen(function* () {
718+
const url = "https://example.com/tool-image.png?version=2"
719+
const prepared = yield* compileRequest(
720+
LLM.request({
721+
model,
722+
messages: [
723+
Message.user("Describe the image."),
724+
Message.assistant([ToolCallPart.make({ id: "call_image", name: "read_image", input: {} })]),
725+
Message.tool({
726+
id: "call_image",
727+
name: "read_image",
728+
resultType: "content",
729+
result: [
730+
{ type: "text", text: "Image attached." },
731+
{ type: "file", mime: "image/png", uri: url },
732+
],
733+
}),
734+
],
735+
}),
736+
)
737+
expect(prepared.body.messages).toContainEqual({
738+
role: "tool",
739+
tool_call_id: "call_image",
740+
content: "Image attached.",
741+
})
742+
expect(prepared.body.messages.at(-1)).toEqual({
743+
role: "user",
744+
content: [{ type: "image_url", image_url: { url } }],
745+
})
746+
}),
747+
)
748+
701749
it.effect("rejects non-image media that cannot be lowered", () =>
702750
Effect.gen(function* () {
703751
const error = yield* compileRequest(

0 commit comments

Comments
 (0)