fix(benchmark): keep partial native scores out of completion markers

benchmark/edgebench/README.md

@@ -185,6 +185,12 @@ GNU timeout after verified execution entry is a normal budget stop. Other nonzer
 or unknown exits, including an early exit with code 124, are `runner_failed` and
 do not publish `final_result.json`. Keep their captures and failure evidence;
 do not count an initial artifact's score as a solver outcome.
+For native Goals with positive runtime, `partial_result.json` preserves the full
+scoring result and its `runtime_status` / `native_goal_outcome` without making
+SForge's visualizer count the attempt as done. Handled terminal stops retain
+`final_result.json`; its presence does not independently qualify a score. Use
+new source pins for new attempts and leave frozen histories unchanged; see the
+[native outcome contract](../runtime/RUNTIME.md) for rollback and classification.
 
 New native archive-submission trials default to **no periodic auto-evaluation**
 (`--eval-interval 0`) and **3,600 seconds between accepted active submissions**.

benchmark/edgebench/run.py

@@ -87,7 +87,7 @@ def _result_status(*, interrupted, started, runtime_seconds, exit_code, timed_ou
 
 def _write_native_final_result(trial, result, *, status, agent, task, run_id, model, effort,
                                feedback="native", native_goal_outcome=None):
-    """Preserve native scores separately from the solver's terminal outcome."""
+    """Retain partial native scores without publishing a completion marker."""
     if status != "terminal" and not (native_goal_outcome and result.runtime_seconds > 0):
         return
     final = dict(agent=agent, task=task, run_id=run_id, model=model, effort=effort,
@@ -97,9 +97,11 @@ def _write_native_final_result(trial, result, *, status, agent, task, run_id, mo
     if feedback == "best-only":
         final["evaluation_coverage"] = "online_only"
         final["offline_scoring_complete"] = False
-    pending = trial / "final_result.json.tmp"
+    # SForge consumers treat final_result.json itself as a completed run.
+    destination = trial / ("final_result.json" if status == "terminal" else "partial_result.json")
+    pending = destination.with_suffix(".json.tmp")
     pending.write_text(json.dumps(final, indent=2, ensure_ascii=False))
-    pending.replace(trial / "final_result.json")
+    pending.replace(destination)
 
 
 def _task_default(task_id: str, key: str, explicit: int | None, fallback: int) -> int:

benchmark/runtime/RUNTIME.md

@@ -96,11 +96,16 @@ the terminal-error fields and may complete normally.
 
 The EdgeBench adapter carries this outcome through artifact collection into
 `runtime-receipt.json`. If a native Goal fails after execution starts, its
-`final_result.json` still retains native scores, with explicit `runtime_status`
-and `native_goal_outcome`; the presence of that file alone is not successful
-completion or result qualification. Ordinary non-native profile handling is
-unchanged. Adopt this behavior only for newly pinned attempts; frozen histories
-retain their original receipts and require explicit post-run classification.
+`partial_result.json` retains the complete native scoring result, with explicit
+`runtime_status` and `native_goal_outcome`. Failed, cancelled, usage-limited and
+unknown native outcomes do not publish `final_result.json`: existing SForge
+consumers use that file as a completion marker without interpreting the outcome
+labels. Handled terminal stops retain the existing final-result behavior;
+neither artifact grants result qualification. Ordinary non-native profile
+handling is unchanged. Adopt this behavior only for newly pinned attempts;
+frozen histories retain their original receipts and require explicit post-run
+classification. Roll back by pinning the previous source for another new attempt,
+not by renaming artifacts in a frozen run.
 
 The optional `native_goal_continue: true` loop retains the original task's
 requirements and evaluation rules after completion. Its continuation prompt no

benchmark/tests/test_native_goal_outcomes.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import re
 import subprocess
 import sys
 import uuid
@@ -154,13 +155,18 @@ def test_edgebench_retains_failed_native_score_without_claiming_completion(tmp_p
     _write_native_final_result(tmp_path, score, status=status, agent="fixture", task="fixture",
                               run_id="fixture", model="fixture", effort="high",
                               native_goal_outcome=receipt["native_goal_outcome"])
-    final = json.loads((tmp_path / "final_result.json").read_text())
-    assert final["best_score"] == 12.5
-    assert final["runtime_status"] == "runner_failed"
-    assert final["native_goal_outcome"] == "failed"
+    partial = json.loads((tmp_path / "partial_result.json").read_text())
+    assert all(partial[key] == value for key, value in score.to_dict().items())
+    assert partial["runtime_status"] == "runner_failed"
+    assert partial["native_goal_outcome"] == "failed"
+    assert not (tmp_path / "final_result.json").exists()
+    assert not (tmp_path / "partial_result.json.tmp").exists()
 
 
-@pytest.mark.parametrize("outcome,expected_code", [("failed", 1), ("blocked", 0), (None, 1)])
+@pytest.mark.parametrize("outcome,expected_code", [
+    ("failed", 1), ("usage_limited", 1), ("cancelled", 130), (None, 1),
+    ("complete", 0), ("blocked", 0), ("paused", 0), ("budget_exhausted", 0),
+])
 def test_edgebench_cli_persists_native_outcome_even_when_transport_exits_zero(tmp_path, monkeypatch, outcome, expected_code):
     pytest.importorskip("sforge")
     pytest.importorskip("harbor")
@@ -185,7 +191,7 @@ def test_edgebench_cli_persists_native_outcome_even_when_transport_exits_zero(tm
     trial = tmp_path / "runs" / "attempt" / "fixture"
     def completed_scoring(**kwargs):
         assert kwargs["disable_auto_resume"] is True
-        (trial / "started_at").touch()
+        (trial / "started_at").write_text("2024-01-01T00:00:00Z\n1704067200\n")
         return RunResult(runtime_seconds=2, best_score=4, total_rounds=1)
     monkeypatch.setattr(run, "run_agent", completed_scoring)
     (tmp_path / "fixture.json").write_text("{}")
@@ -196,10 +202,35 @@ def completed_scoring(**kwargs):
     receipt = json.loads((trial / "runtime-receipt.json").read_text())
     assert receipt["native_goal_outcome"] == (outcome or "unknown")
     assert receipt["score_countable"] is False
-    assert receipt["status"] == ("terminal" if expected_code == 0 else "runner_failed")
-    final = json.loads((trial / "final_result.json").read_text())
-    assert final["runtime_status"] == receipt["status"]
-    assert final["best_score"] == 4
+    assert receipt["status"] == ("terminal" if expected_code == 0 else
+                                  "cancelled" if expected_code == 130 else "runner_failed")
+    terminal = expected_code == 0
+    artifact = trial / ("final_result.json" if terminal else "partial_result.json")
+    result = json.loads(artifact.read_text())
+    assert result["runtime_status"] == receipt["status"]
+    assert result["best_score"] == 4 and result["total_rounds"] == 1
+    assert (trial / "final_result.json").exists() is terminal
+    assert (trial / "partial_result.json").exists() is not terminal
+
+    # Exercise the real consumer through normal directory discovery and HTTP
+    # rendering: outcome labels alone cannot override its completion marker.
+    pytest.importorskip("fastapi")
+    pytest.importorskip("httpx")
+    from fastapi.testclient import TestClient
+    from sforge.visualizer.scanner import RunsIndex
+    from sforge.visualizer.server import create_app
+
+    discovered, = RunsIndex(tmp_path / "runs").list_runs(include_submissions=False)
+    assert discovered.has_final is terminal
+    assert discovered.aborted is not terminal
+    with TestClient(create_app(tmp_path / "runs")) as client:
+        response = client.get("/")
+    assert response.status_code == 200
+    group = response.text.split('data-run-id="attempt"', 1)[1].split("</details>", 1)[0]
+    done = re.search(r'font-medium">(\d+)<span[^>]*>/1</span> done', group)
+    assert done and int(done[1]) == int(terminal)
+    assert (">done</span>" in group) is terminal
+    assert (">aborted</span>" in group) is not terminal
 
 
 @pytest.mark.skipif(os.environ.get("LOOPX_EDGEBENCH_DOCKER_SMOKE") != "1",
@@ -261,8 +292,10 @@ def test_docker_collection_preserves_real_worker_outcome(tmp_path, goal, turn, e
         _write_native_final_result(tmp_path, RunResult(runtime_seconds=observed.elapsed_seconds, best_score=7),
                                   status=status, agent="fixture", task="fixture", run_id="fixture",
                                   model="fixture", effort="high", native_goal_outcome=native["native_goal_outcome"])
-        final = json.loads((tmp_path / "final_result.json").read_text())
-        assert final["best_score"] == 7 and final["runtime_status"] == expected_status
+        artifact = tmp_path / ("final_result.json" if expected_status == "terminal" else "partial_result.json")
+        result = json.loads(artifact.read_text())
+        assert result["best_score"] == 7 and result["runtime_status"] == expected_status
+        assert (tmp_path / "final_result.json").exists() is (expected_status == "terminal")
     finally:
         if not cleaned:
             backend.cleanup_container(handle)