fix(deps): allow litellm releases past GHSA-7hp6-4w63-5g45

pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
     "httpx>=0.27.0",
     "huggingface_hub>=1.0",  # agent-env run hf:// resolves and downloads a Hub bundle; checked on 1.0, 1.33 and 2.4
     "jsonschema>=4.20",  # checks an app's settings against its Settings' schema before a deploy
-    "litellm>=1.96.0,<1.97",
+    "litellm>=1.104.0,<1.107",  # 1.104.0 is past every range GHSA-7hp6-4w63-5g45 lists (a proxy-server advisory; only the client is used); checked on 1.104.0 and 1.106.6
     "mcp>=1.25,<2",  # same floor as env/gateway/requirements.txt and agentenv-protocol; 2.0.0 removed mcp.server.fastmcp, imported at module scope by env/gateway
     "packaging>=22",  # parses a bundle.toml's requires; evaluates a plugin's requirement markers with no extra
     "a2a-sdk==0.3.26",

tst/unit/conftest.py

@@ -8,9 +8,17 @@
 
 A test that genuinely needs a socket can opt out with @pytest.mark.enable_socket.
 
+``import litellm`` fetches its model cost map from GitHub unless told to read the copy it ships, and in every release
+pyproject.toml allows a blocked socket fails the import instead of falling back, so the unit suite reads the shipped
+copy.
+
 The AWS backends' tests live in store/aws, whose conftest gives them placeholder credentials.
 """
 
+import os
+
+os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+
 import pytest
 from pytest_socket import disable_socket, enable_socket
 

tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py

@@ -0,0 +1,86 @@
+"""The rubrics judge's calls go through the installed litellm, not a stand-in.
+
+``_prompt_llm_judge`` is the only caller of litellm: ``acompletion`` with a strict ``json_schema`` response_format over
+text or text-and-image content, read through ``.choices[0].message.content``, sent again without the response_format
+when an endpoint refuses it with images. Each test hands the real ``litellm.acompletion`` exactly the kwargs the
+judge builds, adding only litellm's ``mock_response`` so nothing leaves the process, so a litellm release that stops
+accepting one of those shapes fails here instead of in a run.
+"""
+from __future__ import annotations
+
+import json
+
+import litellm
+import pytest
+
+from agent_env.config.model import ModelConfig
+from agent_env.store import LocalSecretStore
+from agent_env.task_step.context import TaskStepContext
+from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep
+
+_VERDICT = {"results": [{"criterion_id": "c1", "passed": True}]}
+_IMAGE = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}
+
+
+@pytest.fixture
+def judge(monkeypatch):
+    import agent_env.config as cfg_mod
+
+    config = cfg_mod.Config()
+    config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False))
+    monkeypatch.setattr(config, "_model_cfg", ModelConfig())
+    monkeypatch.setattr(cfg_mod, "get_config", lambda: config)
+    monkeypatch.delenv("LITELLM_BASE_URL", raising=False)
+    monkeypatch.delenv("LITELLM_API_KEY", raising=False)
+    return RubricsVerifierTaskStep(
+        id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=False,
+        default_model_api_base="https://judge.invalid/v1", verifier_id="vt",
+    )
+
+
+def _answer_with(monkeypatch, *answers):
+    """Send each call to the real acompletion with the next of ``answers`` as its mock_response; return the calls."""
+    real, calls, pending = litellm.acompletion, [], list(answers)
+
+    async def acompletion(**kwargs):
+        calls.append(kwargs)
+        return await real(**kwargs, mock_response=pending.pop(0))
+
+    monkeypatch.setattr(litellm, "acompletion", acompletion)
+    return calls
+
+
+@pytest.mark.asyncio
+async def test_a_text_judge_call_is_answered_through_a_strict_json_schema(judge, monkeypatch):
+    calls = _answer_with(monkeypatch, json.dumps(_VERDICT))
+
+    out = await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext())
+
+    assert json.loads(out["response"]) == _VERDICT
+    assert calls[0]["response_format"]["type"] == "json_schema"
+    assert calls[0]["response_format"]["json_schema"]["strict"] is True
+    assert calls[0]["api_base"] == "https://judge.invalid/v1"
+
+
+@pytest.mark.asyncio
+async def test_a_judge_call_with_screenshots_sends_text_and_image_content(judge, monkeypatch):
+    calls = _answer_with(monkeypatch, json.dumps(_VERDICT))
+
+    out = await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext(),
+                                        image_blocks=[_IMAGE])
+
+    assert json.loads(out["response"]) == _VERDICT
+    assert calls[0]["messages"][0]["content"] == [{"type": "text", "text": "grade it"}, _IMAGE]
+
+
+@pytest.mark.asyncio
+async def test_an_endpoint_refusing_the_schema_with_images_is_asked_again_without_it(judge, monkeypatch):
+    refusal = Exception("response_format json_schema is not supported with image inputs")
+    calls = _answer_with(monkeypatch, refusal, json.dumps(_VERDICT))
+
+    out = await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext(),
+                                        image_blocks=[_IMAGE])
+
+    assert json.loads(out["response"]) == _VERDICT
+    assert "response_format" in calls[0]
+    assert "response_format" not in calls[1]

tst/unit/test_optional_extras_stay_optional.py

@@ -114,13 +114,13 @@ def get_authorization_token(self):
 """
 
 
-def _closure(extras: frozenset[str]) -> set[str]:
-    """The distributions installing this package with ``extras`` pulls in."""
+def _closure(extras: frozenset[str], pruned: frozenset[str] = frozenset()) -> set[str]:
+    """The distributions installing this package with ``extras`` pulls in, not reaching any through ``pruned``."""
     seen: set[tuple[str, frozenset[str]]] = set()
     pending = [(SELF, extras)]
     while pending:
         name, wanted = pending.pop()
-        if (name, wanted) in seen:
+        if (name, wanted) in seen or name in pruned:
             continue
         seen.add((name, wanted))
         try:
@@ -135,14 +135,26 @@ def _closure(extras: frozenset[str]) -> set[str]:
     return {name for name, _ in seen} - {SELF}
 
 
+def _named_by(extra: str) -> frozenset[str]:
+    """The distributions ``extra`` requires by name."""
+    named = set()
+    for line in importlib.metadata.requires(SELF) or []:
+        requirement = Requirement(line)
+        marker = requirement.marker
+        if marker is not None and marker.evaluate({"extra": extra}) and not marker.evaluate({"extra": ""}):
+            named.add(canonicalize_name(requirement.name))
+    return frozenset(named)
+
+
 def _only_in(extra: str) -> list[str]:
     """Module paths the extra installs and neither a core install nor any other extra does, but for an extra that
     includes it (``trl`` includes ``openenv``): a distribution two extras share (grpcio, for gcp and tensorlake) stays
-    for the other's modules."""
+    for the other's modules. A distribution the extra names stays the extra's though a core dependency brings it too
+    (litellm requires boto3 from 1.104), so core is still shown to import without it."""
     mine = _closure(frozenset({extra}))
     others = [other for other in _optional_requirements()
               if other != extra and not mine <= _closure(frozenset({other}))]
-    core = _closure(frozenset()) | set().union(*(_closure(frozenset({other})) for other in others))
+    core = _closure(frozenset(), _named_by(extra)) | set().union(*(_closure(frozenset({other})) for other in others))
     provided_by_core = set().union(*(_module_paths(d) for d in core))
     extra_only = _closure(frozenset({extra})) - core
     return sorted(set().union(*(_module_paths(d) for d in extra_only)) - provided_by_core)