From 8dc622b94ff9a02629d8c54a02ff0e3d9243dc1a Mon Sep 17 00:00:00 2001 From: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:16:29 +0800 Subject: [PATCH 1/3] Fix dead image branch in OpenAICompatible._conversation_to_list The image-modality branch tested `if "image" in turn.content.data_type`, but data_type is a (mimetype, encoding) tuple from mimetypes.guess_type, e.g. ("image/gif", None). Membership on a tuple checks element equality, so "image" in ("image/gif", None) is always False. The image branch was therefore never taken: every image turn fell through to the audio elif (which fails on the mimetype subtype) and then to the else, raising GarakException("Data type image/gif not supported."). This broke all image/multimodal probes (e.g. visual_jailbreak) against OpenAI-compatible vision generators. Test the mimetype string via data_type[0], mirroring the audio branch two lines below. Also add the missing comma in the base64 data URI ("...;base64," per RFC 2397), which the now-reachable branch would otherwise emit malformed. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> --- garak/generators/openai.py | 4 +-- tests/generators/test_openai_compatible.py | 29 +++++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/garak/generators/openai.py b/garak/generators/openai.py index fa97ce830..21e277a53 100644 --- a/garak/generators/openai.py +++ b/garak/generators/openai.py @@ -204,14 +204,14 @@ def _conversation_to_list(conversation: Conversation) -> list[dict]: data_b64 = base64.b64encode(turn.content.data).decode("utf-8") - if "image" in turn.content.data_type: + if "image" in turn.content.data_type[0]: transformed_turn = { "role": turn.role, "content": [ {"type": "input_text", "text": turn.content.text}, { "type": "input_image", - "image_url": f"data:{turn.content.data_type[0]};base64{data_b64}", + "image_url": f"data:{turn.content.data_type[0]};base64,{data_b64}", }, ], } diff --git a/tests/generators/test_openai_compatible.py b/tests/generators/test_openai_compatible.py index d69ad8f88..0f55582a0 100644 --- a/tests/generators/test_openai_compatible.py +++ b/tests/generators/test_openai_compatible.py @@ -11,7 +11,7 @@ from collections.abc import Iterable from garak.attempt import Message, Turn, Conversation -from garak.generators.openai import OpenAICompatible +from garak.generators.openai import OpenAICompatible, OpenAIGenerator from garak.generators.rest import RestGenerator # TODO: expand this when we have faster loading, currently to process all generator costs 30s for 3 tests @@ -141,3 +141,30 @@ def test_openai_multiple_generations(): assert ( oai_klass.supports_multiple_generations == True ), "OpenAI access expected to correctly support multiple generations by default" + + +def test_conversation_to_list_image_turn_is_reachable(): + """An image turn must produce an image content part, not raise GarakException. + + ``data_type`` is a ``(mimetype, encoding)`` tuple, so ``"image" in + turn.content.data_type`` was always False and the image branch was dead: + every image turn fell through to the ``else`` and raised + ``GarakException("Data type image/gif not supported.")``. + """ + generator = build_test_instance(OpenAIGenerator) + conv = Conversation( + [ + Turn( + "user", + Message(text="describe", data_path="tests/_assets/tinytrans.gif"), + ) + ] + ) + + result = generator._conversation_to_list(conv) + + content = result[0]["content"] + image_parts = [p for p in content if p.get("type") == "input_image"] + assert image_parts, "image turn did not produce an input_image content part" + # Well-formed base64 data URI: data:;base64, + assert image_parts[0]["image_url"].startswith("data:image/gif;base64,") From de601b048a48eb8e5900a021af42b600dc38ca2f Mon Sep 17 00:00:00 2001 From: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:48:25 +0800 Subject: [PATCH 2/3] Use chat/completions content part types for image turns Per review: input_text/input_image are the responses API spelling, which chat/completions rejects with 400 invalid_value ("Supported values are: 'text', 'image_url', ..."). Support for the responses API is #1806's scope. Switch the image branch to the chat/completions shape -- {"type": "text"} plus {"type": "image_url", "image_url": {"url": ...}}, where image_url is an object rather than a bare string. The neighbouring audio branch already emits {"type": "text"}, so this also makes the two branches consistent. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> --- garak/generators/openai.py | 8 +++++--- tests/generators/test_openai_compatible.py | 16 +++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/garak/generators/openai.py b/garak/generators/openai.py index 21e277a53..32fae59fb 100644 --- a/garak/generators/openai.py +++ b/garak/generators/openai.py @@ -208,10 +208,12 @@ def _conversation_to_list(conversation: Conversation) -> list[dict]: transformed_turn = { "role": turn.role, "content": [ - {"type": "input_text", "text": turn.content.text}, + {"type": "text", "text": turn.content.text}, { - "type": "input_image", - "image_url": f"data:{turn.content.data_type[0]};base64,{data_b64}", + "type": "image_url", + "image_url": { + "url": f"data:{turn.content.data_type[0]};base64,{data_b64}" + }, }, ], } diff --git a/tests/generators/test_openai_compatible.py b/tests/generators/test_openai_compatible.py index 0f55582a0..904219033 100644 --- a/tests/generators/test_openai_compatible.py +++ b/tests/generators/test_openai_compatible.py @@ -144,12 +144,16 @@ def test_openai_multiple_generations(): def test_conversation_to_list_image_turn_is_reachable(): - """An image turn must produce an image content part, not raise GarakException. + """An image turn must produce a chat/completions image part, not raise. ``data_type`` is a ``(mimetype, encoding)`` tuple, so ``"image" in turn.content.data_type`` was always False and the image branch was dead: every image turn fell through to the ``else`` and raised ``GarakException("Data type image/gif not supported.")``. + + The part types are the ones chat/completions accepts (``text`` / + ``image_url``); the responses API spelling (``input_text`` / + ``input_image``) is rejected with a 400 invalid_value. """ generator = build_test_instance(OpenAIGenerator) conv = Conversation( @@ -164,7 +168,9 @@ def test_conversation_to_list_image_turn_is_reachable(): result = generator._conversation_to_list(conv) content = result[0]["content"] - image_parts = [p for p in content if p.get("type") == "input_image"] - assert image_parts, "image turn did not produce an input_image content part" - # Well-formed base64 data URI: data:;base64, - assert image_parts[0]["image_url"].startswith("data:image/gif;base64,") + assert [p.get("type") for p in content] == ["text", "image_url"] + + text_part, image_part = content + assert text_part["text"] == "describe" + # image_url is an object with a url key, not a bare string + assert image_part["image_url"]["url"].startswith("data:image/gif;base64,") From a8f1fc7d83069fa4f9fc59b8da509e727b7a4a0c Mon Sep 17 00:00:00 2001 From: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> Date: Fri, 17 Jul 2026 08:32:15 +0800 Subject: [PATCH 3/3] Assert the image part round-trips to the original bytes Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> --- tests/generators/test_openai_compatible.py | 43 +++++++++++----------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/tests/generators/test_openai_compatible.py b/tests/generators/test_openai_compatible.py index 904219033..6fabb5e78 100644 --- a/tests/generators/test_openai_compatible.py +++ b/tests/generators/test_openai_compatible.py @@ -1,8 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import base64 import os import httpx +import pathlib import respx import pytest import importlib @@ -29,6 +31,8 @@ __file__ ) # use test path as hint encase env changes are missed +IMAGE_ASSET = pathlib.Path(__file__).parents[1] / "_assets" / "tinytrans.gif" + def compatible() -> Iterable[OpenAICompatible]: for classname in GENERATORS: @@ -143,34 +147,29 @@ def test_openai_multiple_generations(): ), "OpenAI access expected to correctly support multiple generations by default" -def test_conversation_to_list_image_turn_is_reachable(): - """An image turn must produce a chat/completions image part, not raise. - - ``data_type`` is a ``(mimetype, encoding)`` tuple, so ``"image" in - turn.content.data_type`` was always False and the image branch was dead: - every image turn fell through to the ``else`` and raised - ``GarakException("Data type image/gif not supported.")``. +def test_conversation_to_list_image_turn_uses_chat_completions_format(): + """Image turns render as chat/completions content parts. - The part types are the ones chat/completions accepts (``text`` / - ``image_url``); the responses API spelling (``input_text`` / - ``input_image``) is rejected with a 400 invalid_value. + `data_type` is a `(mimetype, encoding)` tuple, so `"image" in + turn.content.data_type` never matched and the image branch was unreachable. """ generator = build_test_instance(OpenAIGenerator) conv = Conversation( - [ - Turn( - "user", - Message(text="describe", data_path="tests/_assets/tinytrans.gif"), - ) - ] + [Turn("user", Message(text="describe", data_path=str(IMAGE_ASSET)))] ) result = generator._conversation_to_list(conv) - content = result[0]["content"] - assert [p.get("type") for p in content] == ["text", "image_url"] + assert len(result) == 1 + assert result[0]["role"] == "user" + + text_part, image_part = result[0]["content"] + assert text_part == {"type": "text", "text": "describe"} + assert image_part["type"] == "image_url" + # chat/completions expects an object with a `url` entry, not a bare string + assert isinstance(image_part["image_url"], dict) - text_part, image_part = content - assert text_part["text"] == "describe" - # image_url is an object with a url key, not a bare string - assert image_part["image_url"]["url"].startswith("data:image/gif;base64,") + header, separator, payload = image_part["image_url"]["url"].partition(",") + assert header == "data:image/gif;base64" + assert separator == "," + assert base64.b64decode(payload) == IMAGE_ASSET.read_bytes()