Skip to content

fix(responses): keep images in tool results instead of dropping them #2117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
looplj merged 2 commits into from
Aug 1, 2026
+268 −7
Merged

fix(responses): keep images in tool results instead of dropping them #2117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a7399a3
fix(responses): keep images in tool results instead of dropping them
GhostFlying Jul 30, 2026
94ba2a5
fix(responses): default tool image detail
GhostFlying Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions llm/transformer/openai/responses/outbound_convert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,11 +310,31 @@ func convertToolMessageWithType(msg llm.Message, itemType string) Item {
output.Text = msg.Content.Content
} else if len(msg.Content.MultipleContent) > 0 {
for _, p := range msg.Content.MultipleContent {
if p.Type == "text" && p.Text != nil {
output.Items = append(output.Items, Item{
Type: "input_text",
Text: p.Text,
})
switch p.Type {
case "text":
if p.Text != nil {
output.Items = append(output.Items, Item{
Type: "input_text",
Text: p.Text,
})
}
case "image_url":
// Tool results can carry images (Codex's view_image, MCP screenshot
// tools, ...); the Responses schema allows text/image/file content in
// function_call_output and custom_tool_call_output. Skipping them left
// output empty, which the fallback below turned into "" — a blank but
// successful tool result the model cannot distinguish from a real one.
if p.ImageURL != nil {
// `detail` is required by InputImageContent, which is what a
// custom_tool_call_output's content array resolves to; the
// function_call_output param schema makes it optional. Both
// document "auto" as the default, so always send one.
output.Items = append(output.Items, Item{
Type: "input_image",
ImageURL: &p.ImageURL.URL,
Detail: lo.ToPtr(lo.FromPtrOr(p.ImageURL.Detail, "auto")),
})
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
}
}
}
Expand Down
245 changes: 243 additions & 2 deletions llm/transformer/openai/responses/outbound_convert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ func TestConvertToolMessage(t *testing.T) {
},
},
{
name: "tool message with multiple content - mixed types (only text extracted)",
name: "tool message with multiple content - text and image preserved in order",
msg: llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_789"),
Expand Down Expand Up @@ -139,6 +139,11 @@ func TestConvertToolMessage(t *testing.T) {
Type: "input_text",
Text: lo.ToPtr("Text result"),
},
{
Type: "input_image",
ImageURL: lo.ToPtr("https://example.com/image.jpg"),
Detail: lo.ToPtr("auto"),
},
{
Type: "input_text",
Text: lo.ToPtr("More text"),
Expand Down Expand Up @@ -176,7 +181,7 @@ func TestConvertToolMessage(t *testing.T) {
},
},
{
name: "tool message with multiple content but no text parts",
name: "tool message with image and unsupported parts keeps the image",
msg: llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_no_text"),
Expand All @@ -201,6 +206,162 @@ func TestConvertToolMessage(t *testing.T) {
expected: Item{
Type: "function_call_output",
CallID: "call_no_text",
Output: &Input{
Items: []Item{
{
Type: "input_image",
ImageURL: lo.ToPtr("https://example.com/image.jpg"),
Detail: lo.ToPtr("auto"),
},
},
},
},
},
{
// Shape produced by Codex's view_image tool: the result is a single
// base64 image with no text at all. Dropping it made the model see an
// empty-but-successful tool result and silently hallucinate instead.
name: "image-only tool result is preserved as input_image",
msg: llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_view_image"),
Content: llm.MessageContent{
MultipleContent: []llm.MessageContentPart{
{
Type: "image_url",
ImageURL: &llm.ImageURL{
URL: "data:image/png;base64,iVBORw0KGgo=",
Detail: lo.ToPtr("high"),
},
},
},
},
},
expected: Item{
Type: "function_call_output",
CallID: "call_view_image",
Output: &Input{
Items: []Item{
{
Type: "input_image",
ImageURL: lo.ToPtr("data:image/png;base64,iVBORw0KGgo="),
Detail: lo.ToPtr("high"),
},
},
},
},
},
{
name: "tool result with text and image keeps both in order",
msg: llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_mixed"),
Content: llm.MessageContent{
MultipleContent: []llm.MessageContentPart{
{
Type: "text",
Text: lo.ToPtr("screenshot attached"),
},
{
Type: "image_url",
ImageURL: &llm.ImageURL{
URL: "https://example.com/shot.png",
},
},
},
},
},
expected: Item{
Type: "function_call_output",
CallID: "call_mixed",
Output: &Input{
Items: []Item{
{
Type: "input_text",
Text: lo.ToPtr("screenshot attached"),
},
{
Type: "input_image",
ImageURL: lo.ToPtr("https://example.com/shot.png"),
Detail: lo.ToPtr("auto"),
},
},
},
},
},
{
// custom_tool_call_output shares the FunctionAndCustomToolCallOutput
// content union with function_call_output, so images are allowed here too.
name: "custom tool output keeps images",
msg: llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_custom_img"),
Content: llm.MessageContent{
MultipleContent: []llm.MessageContentPart{
{
Type: "image_url",
ImageURL: &llm.ImageURL{
URL: "data:image/png;base64,iVBORw0KGgo=",
},
},
},
},
},
expected: Item{
Type: "custom_tool_call_output",
CallID: "call_custom_img",
Output: &Input{
Items: []Item{
{
Type: "input_image",
ImageURL: lo.ToPtr("data:image/png;base64,iVBORw0KGgo="),
Detail: lo.ToPtr("auto"),
},
},
},
},
},
{
// Known gap, unchanged by this fix: part kinds the current AxonHub
// Responses Item model cannot express in a tool result are still dropped
// silently, and an all-dropped result is still indistinguishable from a
// genuinely empty one. Surfacing that needs structured logging (no logger
// reaches this pure converter), so it is left for a separate change rather
// than papered over with synthetic output text.
name: "tool message with only unsupported parts still degrades to empty string",
msg: llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_audio_only"),
Content: llm.MessageContent{
MultipleContent: []llm.MessageContentPart{
{
Type: "input_audio",
InputAudio: &llm.InputAudio{
Data: "audio-data",
Format: "wav",
},
},
},
},
},
expected: Item{
Type: "function_call_output",
CallID: "call_audio_only",
Output: &Input{
Text: lo.ToPtr(""),
},
},
},
{
name: "tool message with no content at all still uses the empty string",
msg: llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_truly_empty"),
Content: llm.MessageContent{MultipleContent: []llm.MessageContentPart{}},
},
expected: Item{
Type: "function_call_output",
CallID: "call_truly_empty",
Output: &Input{
Text: lo.ToPtr(""),
},
Expand All @@ -221,6 +382,86 @@ func TestConvertToolMessage(t *testing.T) {
}
}

// The wire shape matters as much as the struct: a tool result carrying an image
// has to serialise as an output array of content parts, not as a string.
func TestConvertToolMessageImageSerializesAsOutputArray(t *testing.T) {
item := convertToolMessageWithType(llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_wire"),
Content: llm.MessageContent{
MultipleContent: []llm.MessageContentPart{
{Type: "text", Text: lo.ToPtr("look")},
{
Type: "image_url",
ImageURL: &llm.ImageURL{
URL: "data:image/png;base64,iVBORw0KGgo=",
Detail: lo.ToPtr("high"),
},
},
},
},
}, "function_call_output")

raw, err := json.Marshal(item)
require.NoError(t, err)

var decoded struct {
Type string `json:"type"`
CallID string `json:"call_id"`
Output []struct {
Type string `json:"type"`
Text *string `json:"text"`
ImageURL *string `json:"image_url"`
Detail *string `json:"detail"`
} `json:"output"`
}

require.NoError(t, json.Unmarshal(raw, &decoded))
require.Equal(t, "function_call_output", decoded.Type)
require.Equal(t, "call_wire", decoded.CallID)
require.Len(t, decoded.Output, 2)
require.Equal(t, "input_text", decoded.Output[0].Type)
require.Equal(t, "look", lo.FromPtr(decoded.Output[0].Text))
require.Equal(t, "input_image", decoded.Output[1].Type)
require.Equal(t, "data:image/png;base64,iVBORw0KGgo=", lo.FromPtr(decoded.Output[1].ImageURL))
require.Equal(t, "high", lo.FromPtr(decoded.Output[1].Detail))
}

// custom_tool_call_output resolves to InputImageContent, which requires
// `detail`; assert it reaches the wire even when the source part omits it.
func TestConvertToolMessageCustomOutputImageCarriesDetail(t *testing.T) {
item := convertToolMessageWithType(llm.Message{
Role: "tool",
ToolCallID: lo.ToPtr("call_custom_wire"),
Content: llm.MessageContent{
MultipleContent: []llm.MessageContentPart{
{Type: "image_url", ImageURL: &llm.ImageURL{URL: "https://example.com/shot.png"}},
},
},
}, "custom_tool_call_output")

raw, err := json.Marshal(item)
require.NoError(t, err)

var decoded struct {
Type string `json:"type"`
CallID string `json:"call_id"`
Output []struct {
Type string `json:"type"`
ImageURL *string `json:"image_url"`
Detail *string `json:"detail"`
} `json:"output"`
}

require.NoError(t, json.Unmarshal(raw, &decoded))
require.Equal(t, "custom_tool_call_output", decoded.Type)
require.Equal(t, "call_custom_wire", decoded.CallID)
require.Len(t, decoded.Output, 1)
require.Equal(t, "input_image", decoded.Output[0].Type)
require.Equal(t, "https://example.com/shot.png", lo.FromPtr(decoded.Output[0].ImageURL))
require.Equal(t, "auto", lo.FromPtr(decoded.Output[0].Detail))
}

func TestConvertWebSearchToTool(t *testing.T) {
tests := []struct {
name string
Expand Down
Loading