fix: handle image_url rejection by retrying without images

Replace the static provider-level supports_vision check with a
reactive fallback: when a model returns an image-unsupported error,
strip image_url blocks from messages and retry once. This avoids
maintaining an inaccurate vision capability table and correctly
handles gateway/unknown model scenarios.

Also extract _safe_chat() to deduplicate try/except boilerplate
in chat_with_retry().
This commit is contained in:
Xubin Ren
2026-03-15 14:18:08 +00:00
committed by Xubin Ren
parent de0b5b3d91
commit c4628038c6
4 changed files with 142 additions and 72 deletions

View File

@@ -123,3 +123,87 @@ async def test_chat_with_retry_explicit_override_beats_defaults() -> None:
assert provider.last_kwargs["temperature"] == 0.9
assert provider.last_kwargs["max_tokens"] == 9999
assert provider.last_kwargs["reasoning_effort"] == "low"
# ---------------------------------------------------------------------------
# Image-unsupported fallback tests
# ---------------------------------------------------------------------------
_IMAGE_MSG = [
{"role": "user", "content": [
{"type": "text", "text": "describe this"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},
]},
]
@pytest.mark.asyncio
async def test_image_unsupported_error_retries_without_images() -> None:
"""If the model rejects image_url, retry once with images stripped."""
provider = ScriptedProvider([
LLMResponse(
content="Invalid content type. image_url is only supported by certain models",
finish_reason="error",
),
LLMResponse(content="ok, no image"),
])
response = await provider.chat_with_retry(messages=_IMAGE_MSG)
assert response.content == "ok, no image"
assert provider.calls == 2
msgs_on_retry = provider.last_kwargs["messages"]
for msg in msgs_on_retry:
content = msg.get("content")
if isinstance(content, list):
assert all(b.get("type") != "image_url" for b in content)
assert any("[image omitted]" in (b.get("text") or "") for b in content)
@pytest.mark.asyncio
async def test_image_unsupported_error_no_retry_without_image_content() -> None:
"""If messages don't contain image_url blocks, don't retry on image error."""
provider = ScriptedProvider([
LLMResponse(
content="image_url is only supported by certain models",
finish_reason="error",
),
])
response = await provider.chat_with_retry(
messages=[{"role": "user", "content": "hello"}],
)
assert provider.calls == 1
assert response.finish_reason == "error"
@pytest.mark.asyncio
async def test_image_unsupported_fallback_returns_error_on_second_failure() -> None:
"""If the image-stripped retry also fails, return that error."""
provider = ScriptedProvider([
LLMResponse(
content="does not support image input",
finish_reason="error",
),
LLMResponse(content="some other error", finish_reason="error"),
])
response = await provider.chat_with_retry(messages=_IMAGE_MSG)
assert provider.calls == 2
assert response.content == "some other error"
assert response.finish_reason == "error"
@pytest.mark.asyncio
async def test_non_image_error_does_not_trigger_image_fallback() -> None:
"""Regular non-transient errors must not trigger image stripping."""
provider = ScriptedProvider([
LLMResponse(content="401 unauthorized", finish_reason="error"),
])
response = await provider.chat_with_retry(messages=_IMAGE_MSG)
assert provider.calls == 1
assert response.content == "401 unauthorized"