fix(benchmark): preserve explicit effort and staged model catalogs

benchmark/edgebench/run.py

@@ -21,6 +21,7 @@
 from sforge.harness.task_spec import make_task_spec
 
 from benchmark.runtime.codex import TASK_ENTRIES, turn_envelope_receipt
+from loopx.reasoning_effort import REASONING_EFFORTS
 from loopx.control_plane.work_items.stride_policy_codec import stride_strategy_options
 from benchmark.runtime.sforge import (
     DEFAULT_TIMEOUT_SECONDS, PROFILES, SForgeWorker,
@@ -111,7 +112,7 @@ def main(argv=None):
                         help="Opt-in: when a native Goal declares completion with budget left, start "
                              "continuation Goals until the budget edge (native-goal worker only)")
     parser.add_argument("--model", required=True)
-    parser.add_argument("--effort", choices=("low", "medium", "high", "xhigh"), required=True)
+    parser.add_argument("--effort", choices=REASONING_EFFORTS, required=True)
     parser.add_argument("--timeout", type=int,
                         help="Total trial seconds; task-defaults.json overrides the 18h fallback")
     parser.add_argument("--task-entry", choices=TASK_ENTRIES,

benchmark/runtime/codex.py

@@ -123,6 +123,7 @@ def prepare_codex_home(
     skills: Path | None,
     web_search: str = "disabled",
     model_catalog: Path | None = None,
+    model_catalog_destination: Path | None = None,
 ) -> None:
     """Materialize fixed inputs once; never delete sessions between iterations.
 
@@ -139,6 +140,9 @@ def prepare_codex_home(
     real client send ``tools: []``) are rejected before a trial; qualify a
     pinned binary/catalog pair at the first-request boundary with
     ``benchmark.runtime.toolface`` before relying on the rendered tools.
+    Host-side installers may set ``model_catalog_destination`` to the absolute
+    guest path where those validated catalog bytes are staged. It affects only
+    rendering; the host source must still pass the same validation.
     """
     if not model or not effort:
         raise ValueError("model and effort are required")
@@ -152,6 +156,10 @@ def prepare_codex_home(
         raise ValueError("unsupported web_search setting")
     if model_catalog is not None:
         validate_model_catalog(model_catalog, model)
+    if model_catalog_destination is not None and (
+        model_catalog is None or not model_catalog_destination.is_absolute()
+    ):
+        raise ValueError("catalog destination requires a validated catalog and an absolute guest path")
     home.mkdir(parents=True, exist_ok=True)
     settings = "\n".join(
         [
@@ -161,7 +169,7 @@ def prepare_codex_home(
         + (
             # Codex resolves this path from its own working directory, not the
             # caller's; a relative path validates here yet silently fails there.
-            [f"model_catalog_json = {json.dumps(str(model_catalog.resolve()))}"]
+            [f"model_catalog_json = {json.dumps(str(model_catalog_destination or model_catalog.resolve()))}"]
             if model_catalog is not None
             else []
         )

benchmark/runtime/codex_offline.py

@@ -41,6 +41,7 @@ def __init__(self, *args, goals="false", web_search="disabled", **kwargs):
         self._goals = str(goals)
         self._web_search = web_search
         self._model_catalog_staged = False
+        self._model_catalog_source: Path | None = None
         super().__init__(*args, **kwargs)
 
     @staticmethod
@@ -90,6 +91,7 @@ async def install(self, environment: BaseEnvironment) -> None:
         if catalog.is_file():
             await environment.upload_file(catalog, f"{_STAGE_DIR}/model-catalog.json")
             self._model_catalog_staged = True
+            self._model_catalog_source = catalog.resolve()
 
         await self.exec_as_root(
             environment,

benchmark/runtime/sforge.py

@@ -20,7 +20,7 @@
 from benchmark.edgebench.feedback_hook import FEEDBACK_PAYLOAD
 
 from .codex import DEFAULT_REPLAN_AFTER_TURNS, Execution, prepare_codex_home
-from .codex_offline import CodexOffline, stage_portable_python
+from .codex_offline import CodexOffline, MODEL_CATALOG_REMOTE, stage_portable_python
 from .harbor import (
     BenchmarkCodex, _CONTROL, _CODEX_HOME, _PYTHON, _SCHEDULER_STATE, _SRC,
 )
@@ -221,7 +221,7 @@ def install_stop_hook(self, backend, handle, log_dir, logger):
     def _install_worker(self, backend, handle, log_dir, logger):
         self.backend, self.handle = backend, handle
         self.log_dir = log_dir
-        effort = {"max": "xhigh"}.get(self._config.agent_effort, self._config.agent_effort)
+        effort = self._config.agent_effort
         common = dict(logs_dir=log_dir / "worker", model_name=self._config.agent_model,
                       reasoning_effort=effort)
         if self._config.agent_api_base_url:
@@ -231,7 +231,8 @@ def _install_worker(self, backend, handle, log_dir, logger):
                 "CODEX_AUTH_JSON_PATH": "",
             }
         if self.profile in {"official", "single"}:
-            asyncio.run(CodexOffline(**common).install(self.environment))
+            offline = CodexOffline(**common)
+            asyncio.run(offline.install(self.environment))
             result = backend.exec_run(handle, "mkdir -p /home/agent/.codex", user="agent")
             if result.exit_code:
                 raise RuntimeError("Could not create isolated Codex home")
@@ -245,6 +246,9 @@ def _install_worker(self, backend, handle, log_dir, logger):
                     model=self._config.agent_model, effort=effort,
                     base_url=self._config.agent_api_base_url or "",
                     api_key=self._config.agent_api_key or "", wire_api="responses", skills=None,
+                    model_catalog=offline._model_catalog_source,
+                    model_catalog_destination=(Path(MODEL_CATALOG_REMOTE)
+                                               if offline._model_catalog_staged else None),
                 )
                 for name in ("auth.json", "config.toml"):
                     if not (home / name).exists():
@@ -365,8 +369,15 @@ def format_run_cmd(self, prompt_path, *, model=None, cwd="", internet=True, resu
             (self.log_dir / "agent_prompt.md").write_text(self.feedback_prompt)
             self.prompt_installed = True
         if self.profile in {"official", "single"}:
-            return self._execution_command(super().format_run_cmd(prompt_path, model=model, cwd=cwd,
-                                          internet=internet, resume=resume))
+            command = super().format_run_cmd(prompt_path, model=model, cwd=cwd,
+                                            internet=internet, resume=resume)
+            # Native SForge predates clients/providers with a distinct max
+            # effort. Undo its legacy alias instead of silently downgrading
+            # the explicit trial input; all other native command behavior stays.
+            if self._config.agent_effort == "max":
+                command = command.replace('model_reasoning_effort="xhigh"',
+                                          'model_reasoning_effort="max"', 1)
+            return self._execution_command(command)
         if self.runtime is None:
             raise RuntimeError("Run the native SForge installation hook before execution")
         if not self.prepared:

benchmark/tests/test_sforge_runtime.py

@@ -109,6 +109,77 @@ def test_invalid_worker_inputs_fail_before_install(monkeypatch):
         SForgeWorker(config, profile="single", cwd="/task")
 
 
+@pytest.mark.parametrize("effort", ["max", "xhigh", "high"])
+@pytest.mark.parametrize("profile", ["official", "single", "heartbeat-explore"])
+def test_explicit_effort_survives_native_adapter_and_install(tmp_path, monkeypatch, effort, profile):
+    pytest.importorskip("sforge")
+    pytest.importorskip("harbor")
+    from sforge.harness.config import SForgeConfig
+    from benchmark.runtime.sforge import SForgeWorker, BenchmarkCodex, CodexOffline
+    monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/synthetic-credential")
+    async def installed(self, environment):
+        pass
+    monkeypatch.setattr(BenchmarkCodex, "install", installed)
+    monkeypatch.setattr(CodexOffline, "install", installed)
+    worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort=effort),
+                          profile=profile, cwd="/task")
+    if profile.startswith("heartbeat-"):
+        worker.install_stop_hook(None, None, tmp_path, None)
+        assert worker.runtime._worker_env(cwd="/task")["REASONING_EFFORT"] == effort
+        assert json.loads((tmp_path / "worker-profile.json").read_text())["reasoning_effort"] == effort
+    else:
+        for resume in (False, True):
+            command = worker.format_run_cmd("/task.md", internet=False, resume=resume)
+            assert f'model_reasoning_effort="{effort}"' in command
+
+
+@pytest.mark.parametrize("effort", ["max", "ultra", "high"])
+def test_edgebench_accepts_canonical_effort_before_provider_admission(tmp_path, effort, monkeypatch, capsys):
+    pytest.importorskip("sforge")
+    pytest.importorskip("harbor")
+    from benchmark.edgebench import run
+    monkeypatch.setenv("OPENAI_BASE_URL", "http://example.invalid/v1")
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    with pytest.raises(SystemExit):
+        run.main(["--task", "fixture", "--tasks-dir", str(tmp_path), "--log-dir", str(tmp_path),
+                  "--run-id", "fixture", "--worker", "official", "--model", "fixture",
+                  "--effort", effort, "--judge-url", "http://example.invalid:8080"])
+    assert "OPENAI_BASE_URL requires OPENAI_API_KEY" in capsys.readouterr().err
+    assert not (tmp_path / "runs").exists()
+
+
+@pytest.mark.parametrize("with_catalog", [False, True])
+def test_official_installer_binds_staged_catalog_without_leaking_host_path(tmp_path, monkeypatch, with_catalog):
+    from types import SimpleNamespace
+    import tomllib
+    pytest.importorskip("sforge")
+    pytest.importorskip("harbor")
+    from sforge.harness.agent.codex import CodexAgent
+    from sforge.harness.config import SForgeConfig
+    from benchmark.runtime.sforge import SForgeWorker, CodexOffline
+    from benchmark.runtime.codex_offline import MODEL_CATALOG_REMOTE
+    catalog = tmp_path / "host-catalog.json"
+    catalog.write_text(json.dumps({"models": [{"slug": "fixture"}]}))
+    async def installed(self, environment):
+        self._model_catalog_staged = with_catalog
+        self._model_catalog_source = catalog if with_catalog else None
+    monkeypatch.setattr(CodexOffline, "install", installed)
+    monkeypatch.setattr(CodexAgent, "install_stop_hook", lambda *args: None)
+    observed = {}
+    def copy(handle, source, target):
+        if source.name == "config.toml":
+            observed.update(tomllib.loads(source.read_text()))
+    backend = SimpleNamespace(copy_to_container=copy,
+        exec_run=lambda *args, **kwargs: SimpleNamespace(exit_code=0))
+    worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort="max",
+        agent_api_base_url="http://example.invalid/v1", agent_api_key="fixture"),
+        profile="official", cwd="/task")
+    worker.install_stop_hook(backend, None, tmp_path, None)
+    assert observed["model_reasoning_effort"] == "max"
+    assert observed.get("model_catalog_json") == (MODEL_CATALOG_REMOTE if with_catalog else None)
+    assert str(tmp_path) not in json.dumps(observed)
+
+
 @pytest.mark.parametrize("profile", ["official", "single", "native-goal", "heartbeat-resume", "heartbeat-explore"])
 @pytest.mark.parametrize("unreadable", [False, True])
 def test_best_only_install_requires_ordinary_worker_hook_readback(tmp_path, monkeypatch, profile, unreadable):

benchmark/tests/test_shared_codex_runtime.py

@@ -93,6 +93,24 @@ def test_baseline_isolated_from_loopx_skills(tmp_path):
         prepare_codex_home(**args)
 
 
+def test_catalog_staging_validates_host_bytes_but_renders_fixed_guest_path(tmp_path):
+    catalog = tmp_path / "catalog.json"
+    catalog.write_text(json.dumps({"models": [{"slug": "fixture-model"}]}))
+    args = settings(tmp_path, model_catalog=catalog,
+                    model_catalog_destination=Path("/usr/local/share/codex/model-catalog.json"))
+    prepare_codex_home(**args)
+    config = tomllib.loads((args["home"] / "config.toml").read_text())
+    assert config["model_catalog_json"] == "/usr/local/share/codex/model-catalog.json"
+    with pytest.raises(ValueError, match="settings changed"):
+        prepare_codex_home(**(args | {"model_catalog_destination": Path("/different/catalog.json")}))
+    for invalid in ({"model_catalog": None}, {"model_catalog_destination": Path("relative.json")}):
+        with pytest.raises(ValueError, match="catalog destination"):
+            prepare_codex_home(**(args | invalid))
+    catalog.write_text(json.dumps({"models": [{"slug": "other-model"}]}))
+    with pytest.raises(ValueError, match="no entry"):
+        prepare_codex_home(**args)
+
+
 def worker_env(tmp_path):
     task = tmp_path / "task.md"
     task.write_text("Synthetic task.\n")