Recover from provider rejecting images instead of poisoning the chat

Providers like xAI reject requests carrying images below a minimum size
(e.g. tiny images from MCP tool results), which failed every subsequent
request in the chat. Classify these 400s as :invalid-image, retry once
and keep omitting images at send time while the same model is selected.

Author: ericdallo

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Recover when the provider rejects an image in the request (e.g. xAI's 512px minimum on MCP tool images): retry without images instead of failing every subsequent prompt.
+
 ## 0.148.1
 
 - Fix OpenRouter models failing every prompt with a context-window error: ignore catalog output limits >= context window and use limits from the provider's `/models` endpoint.

integration-test/entrypoint.clj

@@ -20,6 +20,7 @@
     integration.chat.hooks-test
     integration.chat.commands-test
     integration.chat.mcp-remote-test
+    integration.chat.invalid-image-test
     integration.chat.background-jobs-test
     integration.chat.subagent-test
     integration.chat.list-test

integration-test/integration/chat/invalid_image_test.clj

@@ -0,0 +1,85 @@
+(ns integration.chat.invalid-image-test
+  "Covers recovery from providers rejecting image content: an MCP tool returns
+   a tiny image, the (mocked) provider rejects any request carrying it with a
+   400 (mimicking xAI's 512 total-pixels minimum), and ECA must retry without
+   images instead of poisoning the chat forever."
+  (:require
+   [clojure.string :as string]
+   [clojure.test :refer [deftest is testing]]
+   [integration.eca :as eca]
+   [integration.fixture :as fixture]
+   [llm-mock.mocks :as llm.mocks]
+   [matcher-combinators.core :as mc]
+   [matcher-combinators.test :refer [match?]]
+   [mcp-mock.server :as mcp-mock]))
+
+(eca/clean-after-test)
+
+(def ^:private init-options
+  (merge fixture/default-init-options
+         {:mcpServers {"test-mcp" {:url (str "http://localhost:" mcp-mock/port "/mcp")}}}))
+
+(defn ^:private await-content
+  "Consumes chat/contentReceived notifications until one matches role+content,
+   failing the test after 40 non-matching notifications."
+  [chat-id role content]
+  (loop [remaining 40]
+    (if (zero? remaining)
+      (is false (str "No contentReceived matched role=" role " content=" content))
+      (let [actual (eca/client-awaits-server-notification :chat/contentReceived)]
+        (when-not (mc/indicates-match?
+                   (mc/match {:chatId chat-id :role role :content content} actual))
+          (recur (dec remaining)))))))
+
+(deftest invalid-image-recovery
+  (eca/start-process!)
+  (mcp-mock/reset-requests!)
+  (eca/request! (fixture/initialize-request {:initializationOptions init-options}))
+  (eca/notify! (fixture/initialized-notification))
+
+  ;; Wait for MCP server to be ready: native, mcp starting, mcp running
+  (eca/client-awaits-server-notification :tool/serverUpdated)
+  (eca/client-awaits-server-notification :tool/serverUpdated)
+  (eca/client-awaits-server-notification :tool/serverUpdated)
+
+  (testing "provider rejecting a tool-result image triggers a retry without images"
+    (llm.mocks/set-case! :invalid-image-0)
+    (let [resp (eca/request! (fixture/chat-prompt-request
+                              {:model "anthropic/claude-sonnet-4-6"
+                               :message "Call the tiny-image tool"}))
+          chat-id (:chatId resp)]
+
+      (is (match? {:chatId string?
+                   :model "anthropic/claude-sonnet-4-6"
+                   :status "prompting"}
+                  resp))
+
+      ;; MCP tool runs and returns text + image contents (the image is split
+      ;; into its own content notification).
+      (await-content chat-id "assistant" {:type "toolCalled"
+                                          :origin "mcp"
+                                          :name "tiny-image"
+                                          :error nil
+                                          :outputs [{:type "text" :text "Evaluation result rendered as image:"}]})
+      (await-content chat-id "assistant" {:type "image"
+                                          :mediaType "image/png"
+                                          :base64 mcp-mock/tiny-png-base64})
+
+      ;; The continuation request carries the image -> mocked 400 -> recovery.
+      (await-content chat-id "system" {:type "text"
+                                       :text #(string/includes? % "rejected an image")})
+
+      ;; Retry (without images) succeeds and the turn completes.
+      (await-content chat-id "assistant" {:type "text" :text "Recovered without the image"})
+      (await-content chat-id "system" {:type "progress" :state "finished"})
+
+      (testing "MCP mock received exactly one tools/call"
+        (is (= 1 (count (mcp-mock/get-requests-by-method "tools/call")))))
+
+      (testing "retry request replaced the image with a placeholder"
+        (let [retry-body (llm.mocks/get-req-body :invalid-image-0)
+              as-str (pr-str retry-body)]
+          (is (not (string/includes? as-str "\"image\""))
+              "no image content may be replayed after the provider rejected it")
+          (is (string/includes? as-str "[image removed: rejected by the LLM provider]")
+              "the tool result keeps a placeholder where the image was"))))))

integration-test/llm_mock/anthropic.clj

@@ -339,6 +339,64 @@
         (sse-send! ch "message_stop" {:type "message_stop"})
         (hk/close ch)))))
 
+(def invalid-image-error-body
+  "Anthropic-shaped 400 mimicking a provider rejecting an image (e.g. xAI's
+   512 total-pixels minimum relayed through OpenRouter)."
+  (json/generate-string
+   {:type "error"
+    :error {:type "invalid_request_error"
+            :message "messages.2.content.1.image: Image has 100 total pixels (10x10), which is below the minimum of 512 pixels."}}))
+
+(defn ^:private message-with-image?
+  "True when any message content block (including nested tool_result content)
+   is an image block."
+  [messages]
+  (some #(and (map? %) (= "image" (:type %)))
+        (tree-seq coll? seq messages)))
+
+(defn ^:private invalid-image-0 [ch body]
+  (let [second-stage? (some (fn [{:keys [content]}]
+                              (some #(= "tool_result" (:type %)) content))
+                            (:messages body))]
+    (if-not second-stage?
+      (do
+        (sse-send! ch "content_block_delta"
+                   {:type "content_block_delta"
+                    :index 0
+                    :delta {:type "text_delta" :text "Calling the image tool"}})
+        (sse-send! ch "content_block_start"
+                   {:type "content_block_start"
+                    :index 1
+                    :content_block {:type "tool_use"
+                                    :id "img-tool-1"
+                                    :name "testMcp__tiny-image"}})
+        (sse-send! ch "content_block_delta"
+                   {:type "content_block_delta"
+                    :index 1
+                    :delta {:type "input_json_delta"
+                            :partial_json "{}"}})
+        (sse-send! ch "message_delta"
+                   {:type "message_delta"
+                    :delta {:stop_reason "tool_use"}
+                    :usage {:input_tokens 10
+                            :output_tokens 20}})
+        (sse-send! ch "message_stop" {:type "message_stop"})
+        (hk/close ch))
+      ;; Second stage only reached when the retry no longer carries the image
+      ;; (requests with images get a 400 before the SSE channel opens).
+      (do
+        (sse-send! ch "content_block_delta"
+                   {:type "content_block_delta"
+                    :index 0
+                    :delta {:type "text_delta" :text "Recovered without the image"}})
+        (sse-send! ch "message_delta"
+                   {:type "message_delta"
+                    :delta {:stop_reason "end_turn"}
+                    :usage {:input_tokens 15
+                            :output_tokens 10}})
+        (sse-send! ch "message_stop" {:type "message_stop"})
+        (hk/close ch)))))
+
 (defn ^:private compact-0 [ch]
   ;; LLM calls eca__compact_chat with a summary — no text or reasoning
   (sse-send! ch "content_block_start"
@@ -376,9 +434,10 @@
   (let [body (some-> (slurp (:body req))
                      (json/parse-string true))
         title-req? (string/includes? (:text (last (:system body))) llm.mocks/chat-title-generator-str)]
-    (if (and (= :rate-limited-0 llm.mocks/*case*)
-             (not title-req?)
-             (zero? @rate-limited-count*))
+    (cond
+      (and (= :rate-limited-0 llm.mocks/*case*)
+           (not title-req?)
+           (zero? @rate-limited-count*))
       (do
         (swap! rate-limited-count* inc)
         (llm.mocks/set-req-body! llm.mocks/*case* body)
@@ -388,6 +447,18 @@
          :body (json/generate-string {:type "error"
                                       :error {:type "rate_limit_error"
                                               :message "Rate limit exceeded"}})})
+
+      ;; Mimic a provider rejecting any request carrying an image.
+      (and (= :invalid-image-0 llm.mocks/*case*)
+           (not title-req?)
+           (message-with-image? (:messages body)))
+      (do
+        (llm.mocks/set-req-body! llm.mocks/*case* body)
+        {:status 400
+         :headers {"Content-Type" "application/json"}
+         :body invalid-image-error-body})
+
+      :else
       (hk/as-channel
        req
        {:on-open (fn [ch]
@@ -411,4 +482,5 @@
                          :mcp-add-tool-0 (mcp-add-tool-0 ch body)
                          :bg-shell-0 (bg-shell-0 ch body)
                          :compact-0 (compact-0 ch)
+                         :invalid-image-0 (invalid-image-0 ch body)
                          :rate-limited-0 (simple-text-0 ch)))))}))))

integration-test/mcp_mock/server.clj

@@ -39,7 +39,15 @@
     :description "Dynamically registers a new tool on this server"
     :inputSchema {:type "object"
                   :properties {:name {:type "string" :description "Name of the tool to add"}}
-                  :required ["name"]}}])
+                  :required ["name"]}}
+   {:name "tiny-image"
+    :description "Returns the evaluation result rendered as a tiny PNG image"
+    :inputSchema {:type "object"
+                  :properties {}}}])
+
+(def tiny-png-base64
+  "10x10 8-bit RGBA PNG (100 total pixels), below xAI's minimum of 512."
+  "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC")
 
 (def ^:private default-instructions
   "This is a test MCP server for integration testing.")
@@ -134,13 +142,18 @@
       {:content [{:type "text" :text (str "Unknown tool template: " tool-name)}]
        :isError true})))
 
+(defn ^:private call-tiny-image [_arguments]
+  {:content [{:type "text" :text "Evaluation result rendered as image:"}
+             {:type "image" :data tiny-png-base64 :mimeType "image/png"}]})
+
 (defn ^:private handle-tool-call [body]
   (let [tool-name (get-in body [:params :name])
         arguments (get-in body [:params :arguments])
         result (case tool-name
                  "echo" (call-echo arguments)
                  "add" (call-add arguments)
                  "add-tool" (call-add-tool arguments)
+                 "tiny-image" (call-tiny-image arguments)
                  {:content [{:type "text" :text (str "Unknown tool: " tool-name)}]
                   :isError true})]
     {:status 200

src/eca/features/chat.clj

@@ -187,6 +187,49 @@
       (swap! db* assoc-in [:chats chat-id :messages] pruned-messages))
     freed-tokens))
 
+(def ^:private removed-image-placeholder
+  {:type :text :text "[image removed: rejected by the LLM provider]"})
+
+(defn ^:private strip-messages-images
+  "Replaces :image content entries (user attachments and MCP tool result
+   images) with a text placeholder in the given messages. Returns
+   {:messages messages :stripped n} where n is the number of images replaced."
+  [messages]
+  (let [stripped* (volatile! 0)
+        strip-contents (fn [contents]
+                         (if (sequential? contents)
+                           (mapv (fn [content]
+                                   (if (and (map? content) (= :image (:type content)))
+                                     (do (vswap! stripped* inc)
+                                         removed-image-placeholder)
+                                     content))
+                                 contents)
+                           contents))
+        messages (mapv (fn [{:keys [role] :as msg}]
+                         (case role
+                           "user" (update msg :content strip-contents)
+                           "tool_call_output" (if (sequential? (get-in msg [:content :output :contents]))
+                                                (update-in msg [:content :output :contents] strip-contents)
+                                                msg)
+                           msg))
+                       messages)]
+    {:messages messages
+     :stripped @stripped*}))
+
+(defn ^:private messages-to-send
+  "History slice sent to the LLM. When this chat's current model previously
+   rejected an image (`:images-rejected-by-model`, e.g. below xAI's minimum
+   pixels), image contents are stripped at send time so images (including new
+   tool results) cannot poison every subsequent request. History itself is
+   left untouched: clients keep displaying images and switching to an
+   image-capable model replays them again."
+  [db chat-id full-model]
+  (let [messages (shared/messages-after-last-compact-marker
+                  (get-in db [:chats chat-id :messages] []))]
+    (if (= full-model (get-in db [:chats chat-id :images-rejected-by-model]))
+      (:messages (strip-messages-images messages))
+      messages)))
+
 (defn ^:private message-content->chat-content [role message-content content-id]
   (case role
     ("user"
@@ -985,8 +1028,7 @@
                 :model-capabilities model-capabilities
                 :user-messages user-messages
                 :instructions  instructions
-                :past-messages (shared/messages-after-last-compact-marker
-                                (get-in @db* [:chats chat-id :messages] []))
+                :past-messages (messages-to-send @db* chat-id full-model)
                 :config  config
                 :tools all-tools
                 :provider-auth provider-auth
@@ -1147,8 +1189,7 @@
                                                  (consume-steer-message! chat-id db* chat-ctx add-to-history!)
                                                  (consume-pending-job-notifications! chat-id db* add-to-history!)
                                                  {:tools tc-all-tools
-                                                  :new-messages (shared/messages-after-last-compact-marker
-                                                                 (get-in @db* [:chats chat-id :messages]))})))))
+                                                  :new-messages (messages-to-send @db* chat-id full-model)})))))
                                   received-msgs* add-to-history! user-messages)
                 :on-reason (fn [{:keys [status id text external-id delta-reasoning? redacted? data]}]
                              (lifecycle/assert-chat-not-stopped! chat-ctx)
@@ -1328,6 +1369,50 @@
                                   (prune-tool-results! db* chat-id {})
                                   (trigger-auto-compact! chat-ctx all-tools user-messages))
 
+                                ;; Provider rejected an image in the request (e.g. an MCP tool
+                                ;; returned an image below xAI's 512 total-pixels minimum).
+                                ;; Such requests fail deterministically on every replay, which
+                                ;; would otherwise poison the chat forever. Replace images with
+                                ;; text placeholders and retry once.
+                                (and (= :invalid-image error-type)
+                                     (not compacting?)
+                                     (not (:image-retried? chat-ctx)))
+                                (let [;; When the failure happened before the first response,
+                                      ;; user-messages were never added to history and must be
+                                      ;; re-sent (image-stripped) on retry; otherwise just nudge
+                                      ;; the model to continue from the sanitized history.
+                                      user-msgs-in-history? (boolean (when-let [user-content-id (:user-content-id chat-ctx)]
+                                                                       (some #(= user-content-id (:content-id %))
+                                                                             (get-in @db* [:chats chat-id :messages]))))
+                                      {stripped-user-messages :messages user-stripped :stripped} (strip-messages-images user-messages)
+                                      retry-messages (if user-msgs-in-history?
+                                                       [{:role "user"
+                                                         :content [{:type :text
+                                                                    :text "An image in the conversation was rejected by the LLM provider and was replaced with a placeholder. Continue the task with the remaining content, do not redo completed steps."}]}]
+                                                       stripped-user-messages)]
+                                  (logger/warn logger-tag "Provider rejected an image, omitting images for this model and retrying"
+                                               {:chat-id chat-id
+                                                :full-model full-model
+                                                :user-messages-stripped user-stripped})
+                                  ;; Makes messages-to-send strip images from every
+                                  ;; subsequent request while this model is selected, so
+                                  ;; new tool-result images cannot re-poison the chat.
+                                  (swap! db* assoc-in [:chats chat-id :images-rejected-by-model] full-model)
+                                  (lifecycle/send-content! chat-ctx :system
+                                                           {:type :text
+                                                            :text "The LLM provider rejected an image in the conversation. Retrying without images (omitted while this model is selected)..."})
+                                  (swap! db* assoc-in [:chats chat-id :auto-compacting?] true)
+                                  (lifecycle/finish-chat-prompt! :idle
+                                                                 (assoc chat-ctx
+                                                                        :on-finished-side-effect
+                                                                        (fn []
+                                                                          (swap! db* update-in [:chats chat-id] dissoc :auto-compacting?))
+                                                                        :on-after-finish!
+                                                                        (fn []
+                                                                          (prompt-messages! retry-messages
+                                                                                            :invalid-image-retry
+                                                                                            (assoc chat-ctx :image-retried? true))))))
+
                                 :else
                                 (let [partial-text @received-msgs*
                                       transient-error? (contains? #{:overloaded :premature-stop} error-type)