diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cfe886..1af41b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Notable additions, fixes, or breaking changes to the Freeplay SDK. +## [0.6.2] - 2026-03-13 + +### Fixed + +- Media content in recorded messages now uses provider-formatted objects instead of internal content_part_type objects, fixing validation errors when recording prompts with media inputs. + ## [0.6.1] - 2026-03-10 ### Fixed diff --git a/package-lock.json b/package-lock.json index a966130..a440d14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "freeplay", - "version": "0.6.1", + "version": "0.6.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "freeplay", - "version": "0.6.1", + "version": "0.6.2", "license": "Apache-2.0", "dependencies": { "@anthropic-ai/sdk": "^0.39.0", diff --git a/package.json b/package.json index 0864ecf..5451d32 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "freeplay", - "version": "0.6.1", + "version": "0.6.2", "description": "Node.js/Typescript SDK for Freeplay AI", "type": "module", "typesVersions": { diff --git a/src/resources/prompts.ts b/src/resources/prompts.ts index c9f9f52..c246b41 100644 --- a/src/resources/prompts.ts +++ b/src/resources/prompts.ts @@ -477,12 +477,37 @@ export class BoundPrompt { provider: llmAdapter.provider(), } : this.promptInfo; - const prepared = prepareMessages( + let prepared = prepareMessages( this.messages, llmAdapter.roleSupport, finalFlavor, ); const llmFormat = llmAdapter.toLLMSyntax(prepared); + + // Kind of a hack: bound messages may contain internal content objects + // (content_part_type: "text", "media_base64", etc.) that aren't in any + // provider format. Ideally bind() wouldn't produce these internal objects. + // For now, re-run each media message through the adapter individually to + // get provider-formatted content while preserving messages the adapter + // would otherwise drop (e.g. system for Responses API). + prepared = prepared.map((msg) => { + if ( + Array.isArray(msg.content) && + msg.content.some( + (part: unknown) => + typeof part === "object" && + part !== null && + "content_part_type" in part, + ) + ) { + const formatted = llmAdapter.toLLMSyntax([msg]); + if (Array.isArray(formatted) && formatted.length > 0) { + return formatted[0]; + } + return msg; + } + return msg; + }); const llmFormatText = typeof llmFormat === "string" ? llmFormat : undefined; const formattedToolSchema = this.toolSchema ? this.formatToolSchema(this.toolSchema, finalFlavor) diff --git a/test/resources/prompts.test.ts b/test/resources/prompts.test.ts index 932aeb4..fe60f17 100644 --- a/test/resources/prompts.test.ts +++ b/test/resources/prompts.test.ts @@ -123,6 +123,77 @@ describe("prompts", () => { expect(formattedPrompt.outputSchema).toEqual(outputSchema); }); + test("media messages in allMessages have provider format, not internal content_part_type", () => { + const promptInfo: PromptInfo = { + promptTemplateId: "test-id", + promptTemplateVersionId: "test-version-id", + templateName: "test-template", + modelParameters: {}, + provider: "openai", + model: "gpt-4o", + flavorName: "openai_responses", + }; + + const messages: TemplateMessage[] = [ + { role: "system", content: "You are a helpful assistant." }, + { + role: "user", + content: "Describe this image: {{description}}", + media_slots: [{ type: "image", placeholder_name: "photo" }], + }, + ]; + + const media: MediaInputMap = { + photo: { + type: "base64", + content_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUg==", + }, + }; + + const templatePrompt = new TemplatePrompt(promptInfo, messages); + const boundPrompt = templatePrompt.bind( + { description: "a sunset" }, + undefined, + media, + ); + const formattedPrompt = boundPrompt.format("openai_responses"); + + const completionOutput = { + role: "assistant", + content: "It shows a beautiful sunset.", + }; + const allMsgs = formattedPrompt.allMessages(completionOutput); + + // allMessages should be JSON-serializable (JS objects always are, but verify no cycles etc.) + expect(() => JSON.stringify(allMsgs)).not.toThrow(); + + // No content block should have the internal `content_part_type` field + for (const msg of allMsgs) { + if (Array.isArray(msg.content)) { + for (const block of msg.content) { + expect(block).not.toHaveProperty("content_part_type"); + expect(block).not.toHaveProperty("slot_name"); + expect(block).not.toHaveProperty("slot_type"); + } + } + } + + // System message should be preserved in allMessages + const systemMsg = allMsgs.find((m) => m.role === "system"); + expect(systemMsg).toBeDefined(); + expect(systemMsg!.content).toBe("You are a helpful assistant."); + + // The user message should have provider-formatted content blocks with `type`, not `content_part_type` + const userMsg = allMsgs.find( + (m) => m.role === "user" && Array.isArray(m.content), + ); + expect(userMsg).toBeDefined(); + for (const block of userMsg!.content as Array>) { + expect(block).toHaveProperty("type"); + } + }); + test("output schema with unsupported provider throws error", () => { const outputSchema = { type: "object", diff --git a/test/sdk.test.ts b/test/sdk.test.ts index 65bd58a..4fc77bb 100644 --- a/test/sdk.test.ts +++ b/test/sdk.test.ts @@ -1634,21 +1634,18 @@ describe("Chat Completions", function () { role: "user", content: [ { - content_part_type: "text", + type: "text", text: "Answer this question: How do these images look?", }, { - content_part_type: "media_url", - url: "http://localhost/bird.png", - slot_name: "image-1", - slot_type: "image", + type: "image_url", + image_url: { url: "http://localhost/bird.png" }, }, { - content_part_type: "media_base64", - content_type: "image/png", - data: onePixelPng, - slot_name: "image-2", - slot_type: "image", + type: "image_url", + image_url: { + url: `data:image/png;base64,${onePixelPng}`, + }, }, ], },