diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index a450a7f93a..00dbea3f34 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -185,6 +185,12 @@ GNU timeout after verified execution entry is a normal budget stop. Other nonzer or unknown exits, including an early exit with code 124, are `runner_failed` and do not publish `final_result.json`. Keep their captures and failure evidence; do not count an initial artifact's score as a solver outcome. +For native Goals with positive runtime, `partial_result.json` preserves the full +scoring result and its `runtime_status` / `native_goal_outcome` without making +SForge's visualizer count the attempt as done. Handled terminal stops retain +`final_result.json`; its presence does not independently qualify a score. Use +new source pins for new attempts and leave frozen histories unchanged; see the +[native outcome contract](../runtime/RUNTIME.md) for rollback and classification. New native archive-submission trials default to **no periodic auto-evaluation** (`--eval-interval 0`) and **3,600 seconds between accepted active submissions**. diff --git a/benchmark/edgebench/run.py b/benchmark/edgebench/run.py index 29e7b814cc..f568d7874a 100644 --- a/benchmark/edgebench/run.py +++ b/benchmark/edgebench/run.py @@ -87,7 +87,7 @@ def _result_status(*, interrupted, started, runtime_seconds, exit_code, timed_ou def _write_native_final_result(trial, result, *, status, agent, task, run_id, model, effort, feedback="native", native_goal_outcome=None): - """Preserve native scores separately from the solver's terminal outcome.""" + """Retain partial native scores without publishing a completion marker.""" if status != "terminal" and not (native_goal_outcome and result.runtime_seconds > 0): return final = dict(agent=agent, task=task, run_id=run_id, model=model, effort=effort, @@ -97,9 +97,11 @@ def _write_native_final_result(trial, result, *, status, agent, task, run_id, mo if feedback == "best-only": final["evaluation_coverage"] = "online_only" final["offline_scoring_complete"] = False - pending = trial / "final_result.json.tmp" + # SForge consumers treat final_result.json itself as a completed run. + destination = trial / ("final_result.json" if status == "terminal" else "partial_result.json") + pending = destination.with_suffix(".json.tmp") pending.write_text(json.dumps(final, indent=2, ensure_ascii=False)) - pending.replace(trial / "final_result.json") + pending.replace(destination) def _task_default(task_id: str, key: str, explicit: int | None, fallback: int) -> int: diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md index 978c9e4539..b95844ff8e 100644 --- a/benchmark/runtime/RUNTIME.md +++ b/benchmark/runtime/RUNTIME.md @@ -96,11 +96,16 @@ the terminal-error fields and may complete normally. The EdgeBench adapter carries this outcome through artifact collection into `runtime-receipt.json`. If a native Goal fails after execution starts, its -`final_result.json` still retains native scores, with explicit `runtime_status` -and `native_goal_outcome`; the presence of that file alone is not successful -completion or result qualification. Ordinary non-native profile handling is -unchanged. Adopt this behavior only for newly pinned attempts; frozen histories -retain their original receipts and require explicit post-run classification. +`partial_result.json` retains the complete native scoring result, with explicit +`runtime_status` and `native_goal_outcome`. Failed, cancelled, usage-limited and +unknown native outcomes do not publish `final_result.json`: existing SForge +consumers use that file as a completion marker without interpreting the outcome +labels. Handled terminal stops retain the existing final-result behavior; +neither artifact grants result qualification. Ordinary non-native profile +handling is unchanged. Adopt this behavior only for newly pinned attempts; +frozen histories retain their original receipts and require explicit post-run +classification. Roll back by pinning the previous source for another new attempt, +not by renaming artifacts in a frozen run. The optional `native_goal_continue: true` loop retains the original task's requirements and evaluation rules after completion. Its continuation prompt no diff --git a/benchmark/tests/test_native_goal_outcomes.py b/benchmark/tests/test_native_goal_outcomes.py index d10894de1f..fd59793a1d 100644 --- a/benchmark/tests/test_native_goal_outcomes.py +++ b/benchmark/tests/test_native_goal_outcomes.py @@ -3,6 +3,7 @@ import json import logging import os +import re import subprocess import sys import uuid @@ -154,13 +155,18 @@ def test_edgebench_retains_failed_native_score_without_claiming_completion(tmp_p _write_native_final_result(tmp_path, score, status=status, agent="fixture", task="fixture", run_id="fixture", model="fixture", effort="high", native_goal_outcome=receipt["native_goal_outcome"]) - final = json.loads((tmp_path / "final_result.json").read_text()) - assert final["best_score"] == 12.5 - assert final["runtime_status"] == "runner_failed" - assert final["native_goal_outcome"] == "failed" + partial = json.loads((tmp_path / "partial_result.json").read_text()) + assert all(partial[key] == value for key, value in score.to_dict().items()) + assert partial["runtime_status"] == "runner_failed" + assert partial["native_goal_outcome"] == "failed" + assert not (tmp_path / "final_result.json").exists() + assert not (tmp_path / "partial_result.json.tmp").exists() -@pytest.mark.parametrize("outcome,expected_code", [("failed", 1), ("blocked", 0), (None, 1)]) +@pytest.mark.parametrize("outcome,expected_code", [ + ("failed", 1), ("usage_limited", 1), ("cancelled", 130), (None, 1), + ("complete", 0), ("blocked", 0), ("paused", 0), ("budget_exhausted", 0), +]) def test_edgebench_cli_persists_native_outcome_even_when_transport_exits_zero(tmp_path, monkeypatch, outcome, expected_code): pytest.importorskip("sforge") pytest.importorskip("harbor") @@ -185,7 +191,7 @@ def test_edgebench_cli_persists_native_outcome_even_when_transport_exits_zero(tm trial = tmp_path / "runs" / "attempt" / "fixture" def completed_scoring(**kwargs): assert kwargs["disable_auto_resume"] is True - (trial / "started_at").touch() + (trial / "started_at").write_text("2024-01-01T00:00:00Z\n1704067200\n") return RunResult(runtime_seconds=2, best_score=4, total_rounds=1) monkeypatch.setattr(run, "run_agent", completed_scoring) (tmp_path / "fixture.json").write_text("{}") @@ -196,10 +202,35 @@ def completed_scoring(**kwargs): receipt = json.loads((trial / "runtime-receipt.json").read_text()) assert receipt["native_goal_outcome"] == (outcome or "unknown") assert receipt["score_countable"] is False - assert receipt["status"] == ("terminal" if expected_code == 0 else "runner_failed") - final = json.loads((trial / "final_result.json").read_text()) - assert final["runtime_status"] == receipt["status"] - assert final["best_score"] == 4 + assert receipt["status"] == ("terminal" if expected_code == 0 else + "cancelled" if expected_code == 130 else "runner_failed") + terminal = expected_code == 0 + artifact = trial / ("final_result.json" if terminal else "partial_result.json") + result = json.loads(artifact.read_text()) + assert result["runtime_status"] == receipt["status"] + assert result["best_score"] == 4 and result["total_rounds"] == 1 + assert (trial / "final_result.json").exists() is terminal + assert (trial / "partial_result.json").exists() is not terminal + + # Exercise the real consumer through normal directory discovery and HTTP + # rendering: outcome labels alone cannot override its completion marker. + pytest.importorskip("fastapi") + pytest.importorskip("httpx") + from fastapi.testclient import TestClient + from sforge.visualizer.scanner import RunsIndex + from sforge.visualizer.server import create_app + + discovered, = RunsIndex(tmp_path / "runs").list_runs(include_submissions=False) + assert discovered.has_final is terminal + assert discovered.aborted is not terminal + with TestClient(create_app(tmp_path / "runs")) as client: + response = client.get("/") + assert response.status_code == 200 + group = response.text.split('data-run-id="attempt"', 1)[1].split("", 1)[0] + done = re.search(r'font-medium">(\d+)]*>/1 done', group) + assert done and int(done[1]) == int(terminal) + assert (">done" in group) is terminal + assert (">aborted" in group) is not terminal @pytest.mark.skipif(os.environ.get("LOOPX_EDGEBENCH_DOCKER_SMOKE") != "1", @@ -261,8 +292,10 @@ def test_docker_collection_preserves_real_worker_outcome(tmp_path, goal, turn, e _write_native_final_result(tmp_path, RunResult(runtime_seconds=observed.elapsed_seconds, best_score=7), status=status, agent="fixture", task="fixture", run_id="fixture", model="fixture", effort="high", native_goal_outcome=native["native_goal_outcome"]) - final = json.loads((tmp_path / "final_result.json").read_text()) - assert final["best_score"] == 7 and final["runtime_status"] == expected_status + artifact = tmp_path / ("final_result.json" if expected_status == "terminal" else "partial_result.json") + result = json.loads(artifact.read_text()) + assert result["best_score"] == 7 and result["runtime_status"] == expected_status + assert (tmp_path / "final_result.json").exists() is (expected_status == "terminal") finally: if not cleaned: backend.cleanup_container(handle)