From 647a360fc5762af28108f5fc422865ce1a430174 Mon Sep 17 00:00:00 2001 From: song Date: Sat, 17 Oct 2026 18:47:10 +0800 Subject: [PATCH] fix(benchmark): preserve explicit effort and staged model catalogs Signed-off-by: song --- benchmark/edgebench/run.py | 3 +- benchmark/runtime/codex.py | 10 ++- benchmark/runtime/codex_offline.py | 2 + benchmark/runtime/sforge.py | 21 ++++-- benchmark/tests/test_sforge_runtime.py | 71 ++++++++++++++++++++ benchmark/tests/test_shared_codex_runtime.py | 18 +++++ 6 files changed, 118 insertions(+), 7 deletions(-) diff --git a/benchmark/edgebench/run.py b/benchmark/edgebench/run.py index af5a0d00b9..e16e2a0e9e 100644 --- a/benchmark/edgebench/run.py +++ b/benchmark/edgebench/run.py @@ -21,6 +21,7 @@ from sforge.harness.task_spec import make_task_spec from benchmark.runtime.codex import TASK_ENTRIES, turn_envelope_receipt +from loopx.reasoning_effort import REASONING_EFFORTS from loopx.control_plane.work_items.stride_policy_codec import stride_strategy_options from benchmark.runtime.sforge import ( DEFAULT_TIMEOUT_SECONDS, PROFILES, SForgeWorker, @@ -111,7 +112,7 @@ def main(argv=None): help="Opt-in: when a native Goal declares completion with budget left, start " "continuation Goals until the budget edge (native-goal worker only)") parser.add_argument("--model", required=True) - parser.add_argument("--effort", choices=("low", "medium", "high", "xhigh"), required=True) + parser.add_argument("--effort", choices=REASONING_EFFORTS, required=True) parser.add_argument("--timeout", type=int, help="Total trial seconds; task-defaults.json overrides the 18h fallback") parser.add_argument("--task-entry", choices=TASK_ENTRIES, diff --git a/benchmark/runtime/codex.py b/benchmark/runtime/codex.py index 984ffcc20c..66d670d3f3 100644 --- a/benchmark/runtime/codex.py +++ b/benchmark/runtime/codex.py @@ -123,6 +123,7 @@ def prepare_codex_home( skills: Path | None, web_search: str = "disabled", model_catalog: Path | None = None, + model_catalog_destination: Path | None = None, ) -> None: """Materialize fixed inputs once; never delete sessions between iterations. @@ -139,6 +140,9 @@ def prepare_codex_home( real client send ``tools: []``) are rejected before a trial; qualify a pinned binary/catalog pair at the first-request boundary with ``benchmark.runtime.toolface`` before relying on the rendered tools. + Host-side installers may set ``model_catalog_destination`` to the absolute + guest path where those validated catalog bytes are staged. It affects only + rendering; the host source must still pass the same validation. """ if not model or not effort: raise ValueError("model and effort are required") @@ -152,6 +156,10 @@ def prepare_codex_home( raise ValueError("unsupported web_search setting") if model_catalog is not None: validate_model_catalog(model_catalog, model) + if model_catalog_destination is not None and ( + model_catalog is None or not model_catalog_destination.is_absolute() + ): + raise ValueError("catalog destination requires a validated catalog and an absolute guest path") home.mkdir(parents=True, exist_ok=True) settings = "\n".join( [ @@ -161,7 +169,7 @@ def prepare_codex_home( + ( # Codex resolves this path from its own working directory, not the # caller's; a relative path validates here yet silently fails there. - [f"model_catalog_json = {json.dumps(str(model_catalog.resolve()))}"] + [f"model_catalog_json = {json.dumps(str(model_catalog_destination or model_catalog.resolve()))}"] if model_catalog is not None else [] ) diff --git a/benchmark/runtime/codex_offline.py b/benchmark/runtime/codex_offline.py index e9355dfeec..20763d2aa7 100644 --- a/benchmark/runtime/codex_offline.py +++ b/benchmark/runtime/codex_offline.py @@ -41,6 +41,7 @@ def __init__(self, *args, goals="false", web_search="disabled", **kwargs): self._goals = str(goals) self._web_search = web_search self._model_catalog_staged = False + self._model_catalog_source: Path | None = None super().__init__(*args, **kwargs) @staticmethod @@ -90,6 +91,7 @@ async def install(self, environment: BaseEnvironment) -> None: if catalog.is_file(): await environment.upload_file(catalog, f"{_STAGE_DIR}/model-catalog.json") self._model_catalog_staged = True + self._model_catalog_source = catalog.resolve() await self.exec_as_root( environment, diff --git a/benchmark/runtime/sforge.py b/benchmark/runtime/sforge.py index 00f4bf40c9..124069b965 100644 --- a/benchmark/runtime/sforge.py +++ b/benchmark/runtime/sforge.py @@ -20,7 +20,7 @@ from benchmark.edgebench.feedback_hook import FEEDBACK_PAYLOAD from .codex import DEFAULT_REPLAN_AFTER_TURNS, Execution, prepare_codex_home -from .codex_offline import CodexOffline, stage_portable_python +from .codex_offline import CodexOffline, MODEL_CATALOG_REMOTE, stage_portable_python from .harbor import ( BenchmarkCodex, _CONTROL, _CODEX_HOME, _PYTHON, _SCHEDULER_STATE, _SRC, ) @@ -221,7 +221,7 @@ def install_stop_hook(self, backend, handle, log_dir, logger): def _install_worker(self, backend, handle, log_dir, logger): self.backend, self.handle = backend, handle self.log_dir = log_dir - effort = {"max": "xhigh"}.get(self._config.agent_effort, self._config.agent_effort) + effort = self._config.agent_effort common = dict(logs_dir=log_dir / "worker", model_name=self._config.agent_model, reasoning_effort=effort) if self._config.agent_api_base_url: @@ -231,7 +231,8 @@ def _install_worker(self, backend, handle, log_dir, logger): "CODEX_AUTH_JSON_PATH": "", } if self.profile in {"official", "single"}: - asyncio.run(CodexOffline(**common).install(self.environment)) + offline = CodexOffline(**common) + asyncio.run(offline.install(self.environment)) result = backend.exec_run(handle, "mkdir -p /home/agent/.codex", user="agent") if result.exit_code: raise RuntimeError("Could not create isolated Codex home") @@ -245,6 +246,9 @@ def _install_worker(self, backend, handle, log_dir, logger): model=self._config.agent_model, effort=effort, base_url=self._config.agent_api_base_url or "", api_key=self._config.agent_api_key or "", wire_api="responses", skills=None, + model_catalog=offline._model_catalog_source, + model_catalog_destination=(Path(MODEL_CATALOG_REMOTE) + if offline._model_catalog_staged else None), ) for name in ("auth.json", "config.toml"): if not (home / name).exists(): @@ -365,8 +369,15 @@ def format_run_cmd(self, prompt_path, *, model=None, cwd="", internet=True, resu (self.log_dir / "agent_prompt.md").write_text(self.feedback_prompt) self.prompt_installed = True if self.profile in {"official", "single"}: - return self._execution_command(super().format_run_cmd(prompt_path, model=model, cwd=cwd, - internet=internet, resume=resume)) + command = super().format_run_cmd(prompt_path, model=model, cwd=cwd, + internet=internet, resume=resume) + # Native SForge predates clients/providers with a distinct max + # effort. Undo its legacy alias instead of silently downgrading + # the explicit trial input; all other native command behavior stays. + if self._config.agent_effort == "max": + command = command.replace('model_reasoning_effort="xhigh"', + 'model_reasoning_effort="max"', 1) + return self._execution_command(command) if self.runtime is None: raise RuntimeError("Run the native SForge installation hook before execution") if not self.prepared: diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index 016a46628a..6e5effb6b3 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -109,6 +109,77 @@ def test_invalid_worker_inputs_fail_before_install(monkeypatch): SForgeWorker(config, profile="single", cwd="/task") +@pytest.mark.parametrize("effort", ["max", "xhigh", "high"]) +@pytest.mark.parametrize("profile", ["official", "single", "heartbeat-explore"]) +def test_explicit_effort_survives_native_adapter_and_install(tmp_path, monkeypatch, effort, profile): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from sforge.harness.config import SForgeConfig + from benchmark.runtime.sforge import SForgeWorker, BenchmarkCodex, CodexOffline + monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/synthetic-credential") + async def installed(self, environment): + pass + monkeypatch.setattr(BenchmarkCodex, "install", installed) + monkeypatch.setattr(CodexOffline, "install", installed) + worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort=effort), + profile=profile, cwd="/task") + if profile.startswith("heartbeat-"): + worker.install_stop_hook(None, None, tmp_path, None) + assert worker.runtime._worker_env(cwd="/task")["REASONING_EFFORT"] == effort + assert json.loads((tmp_path / "worker-profile.json").read_text())["reasoning_effort"] == effort + else: + for resume in (False, True): + command = worker.format_run_cmd("/task.md", internet=False, resume=resume) + assert f'model_reasoning_effort="{effort}"' in command + + +@pytest.mark.parametrize("effort", ["max", "ultra", "high"]) +def test_edgebench_accepts_canonical_effort_before_provider_admission(tmp_path, effort, monkeypatch, capsys): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from benchmark.edgebench import run + monkeypatch.setenv("OPENAI_BASE_URL", "http://example.invalid/v1") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + with pytest.raises(SystemExit): + run.main(["--task", "fixture", "--tasks-dir", str(tmp_path), "--log-dir", str(tmp_path), + "--run-id", "fixture", "--worker", "official", "--model", "fixture", + "--effort", effort, "--judge-url", "http://example.invalid:8080"]) + assert "OPENAI_BASE_URL requires OPENAI_API_KEY" in capsys.readouterr().err + assert not (tmp_path / "runs").exists() + + +@pytest.mark.parametrize("with_catalog", [False, True]) +def test_official_installer_binds_staged_catalog_without_leaking_host_path(tmp_path, monkeypatch, with_catalog): + from types import SimpleNamespace + import tomllib + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from sforge.harness.agent.codex import CodexAgent + from sforge.harness.config import SForgeConfig + from benchmark.runtime.sforge import SForgeWorker, CodexOffline + from benchmark.runtime.codex_offline import MODEL_CATALOG_REMOTE + catalog = tmp_path / "host-catalog.json" + catalog.write_text(json.dumps({"models": [{"slug": "fixture"}]})) + async def installed(self, environment): + self._model_catalog_staged = with_catalog + self._model_catalog_source = catalog if with_catalog else None + monkeypatch.setattr(CodexOffline, "install", installed) + monkeypatch.setattr(CodexAgent, "install_stop_hook", lambda *args: None) + observed = {} + def copy(handle, source, target): + if source.name == "config.toml": + observed.update(tomllib.loads(source.read_text())) + backend = SimpleNamespace(copy_to_container=copy, + exec_run=lambda *args, **kwargs: SimpleNamespace(exit_code=0)) + worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort="max", + agent_api_base_url="http://example.invalid/v1", agent_api_key="fixture"), + profile="official", cwd="/task") + worker.install_stop_hook(backend, None, tmp_path, None) + assert observed["model_reasoning_effort"] == "max" + assert observed.get("model_catalog_json") == (MODEL_CATALOG_REMOTE if with_catalog else None) + assert str(tmp_path) not in json.dumps(observed) + + @pytest.mark.parametrize("profile", ["official", "single", "native-goal", "heartbeat-resume", "heartbeat-explore"]) @pytest.mark.parametrize("unreadable", [False, True]) def test_best_only_install_requires_ordinary_worker_hook_readback(tmp_path, monkeypatch, profile, unreadable): diff --git a/benchmark/tests/test_shared_codex_runtime.py b/benchmark/tests/test_shared_codex_runtime.py index bf4b46da15..2bd1dc4fdf 100644 --- a/benchmark/tests/test_shared_codex_runtime.py +++ b/benchmark/tests/test_shared_codex_runtime.py @@ -93,6 +93,24 @@ def test_baseline_isolated_from_loopx_skills(tmp_path): prepare_codex_home(**args) +def test_catalog_staging_validates_host_bytes_but_renders_fixed_guest_path(tmp_path): + catalog = tmp_path / "catalog.json" + catalog.write_text(json.dumps({"models": [{"slug": "fixture-model"}]})) + args = settings(tmp_path, model_catalog=catalog, + model_catalog_destination=Path("/usr/local/share/codex/model-catalog.json")) + prepare_codex_home(**args) + config = tomllib.loads((args["home"] / "config.toml").read_text()) + assert config["model_catalog_json"] == "/usr/local/share/codex/model-catalog.json" + with pytest.raises(ValueError, match="settings changed"): + prepare_codex_home(**(args | {"model_catalog_destination": Path("/different/catalog.json")})) + for invalid in ({"model_catalog": None}, {"model_catalog_destination": Path("relative.json")}): + with pytest.raises(ValueError, match="catalog destination"): + prepare_codex_home(**(args | invalid)) + catalog.write_text(json.dumps({"models": [{"slug": "other-model"}]})) + with pytest.raises(ValueError, match="no entry"): + prepare_codex_home(**args) + + def worker_env(tmp_path): task = tmp_path / "task.md" task.write_text("Synthetic task.\n")