From 82ba66d9073f4f8666cdd3f4e29bb2c92aa8e86e Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Tue, 23 Jun 2026 08:07:32 -0700 Subject: [PATCH] test(e2e): don't assert exact image count for duplicate images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test_multi_images_mixed` sends the same pug twice (mixed base64 + URL) and asserted the model counts "3". Engines legitimately differ on byte-identical multimodal inputs: vLLM deduplicates them (encodes the duplicate once), sglang keeps both — so the literal count is engine-dependent. This surfaced after #1604 made image decode bit-deterministic, so identical inputs now produce identical pixel tensors that vLLM dedups (sglang is unaffected). Drop the exact-count assertion and keep the duplicate-detection assertion (which both engines satisfy) plus the dog/pug content checks, so the test still validates multi-image + mixed base64/URL + duplicate handling. Signed-off-by: Simo Lin --- e2e_test/chat_completions/test_multimodal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e_test/chat_completions/test_multimodal.py b/e2e_test/chat_completions/test_multimodal.py index 087621367..93f28beba 100644 --- a/e2e_test/chat_completions/test_multimodal.py +++ b/e2e_test/chat_completions/test_multimodal.py @@ -142,10 +142,10 @@ def test_multi_images_mixed(self, model, setup_backend): assert text is not None and len(text) > 0 text_lower = text.lower() - # Should acknowledge all 3 images - assert "3" in text or "three" in text_lower, ( - f"Expected model to count 3 images, got: {text}" - ) + # Don't assert an exact image count: the two pug inputs are byte-identical, + # and engines legitimately differ on whether identical multimodal inputs are + # deduplicated (vLLM encodes the duplicate once; sglang keeps both). The + # duplicate-detection assertion below covers the intent of this test. # Should identify both dog and pug assert any(k in text_lower for k in ["dog", "puppy", "labrador"]), ( f"Expected dog-related content, got: {text}"