From dd11aecaae105bf009f67fc04605e124827a64ff Mon Sep 17 00:00:00 2001
From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com>
Date: Sat, 17 Oct 2026 22:18:52 +0000
Subject: [PATCH 1/6] fix(backend): eval suite fails closed on unreadable
 inputs, missing scenes, and stale work
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review findings on PR #295's eval suite (burst 17):

- A configured site whose scene fixture is absent now stays in the run
  and fails every stage it reaches, instead of being filtered out — a
  wrong --inputs directory must not exit 0 having evaluated nothing.
- score_files and the produced/committed criteria-report reads are
  caught per row: one malformed labels or report file is a FAIL row
  with the error, not a crashed run that aborts the whole report.
- The work directory is cleared at the start of each run so a second
  run never stands on the previous run's verdicts or reports.

Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com>
---
 backend/server/eval/suite.py | 93 ++++++++++++++++++++++++++++++++----
 1 file changed, 84 insertions(+), 9 deletions(-)

diff --git a/backend/server/eval/suite.py b/backend/server/eval/suite.py
index a281279c8..703d17ec2 100644
--- a/backend/server/eval/suite.py
+++ b/backend/server/eval/suite.py
@@ -138,7 +138,24 @@ def _pipeline_rows(sites, inputs: Path, repo: Path, work: Path, engine_cfg: Crit
         site_dir = inputs / "scenes" / site / "gt-laser"
         target = f"{site}/gt-laser"
         out_dir = work / site / "criteria"
-        staged = _stage(site_dir, site, repo, work)
+        try:
+            staged = _stage(site_dir, site, repo, work)
+        except Exception as exc:
+            traceback.print_exc()
+            rows.append(
+                _row(
+                    f"pipeline/{site}",
+                    "pipeline",
+                    target,
+                    "criteria_engine",
+                    None,
+                    f"verdicts for {', '.join(engine_cfg.profiles)}",
+                    "FAIL",
+                    str(site_dir),
+                    f"scene fixture missing: {_err(exc)}",
+                )
+            )
+            continue
         hint = "" if _has_visibility_inputs(staged) else " (laser PLY absent; run make gt)"
         try:
             produced = evaluate_scene(staged, out_dir, config=engine_cfg)
@@ -208,7 +225,23 @@ def _criteria_rows(sites, inputs: Path, work: Path):
                 )
             )
             continue
-        produced = json.loads(report_path.read_text())
+        try:
+            produced = json.loads(report_path.read_text())
+        except ValueError as exc:
+            rows.append(
+                _row(
+                    f"criteria/{site}/report",
+                    "criteria",
+                    f"{site}/gt-laser",
+                    "criteria_report",
+                    None,
+                    "reproduces committed evidence",
+                    "FAIL",
+                    str(report_path),
+                    f"produced criteria report is unreadable: {_err(exc)}",
+                )
+            )
+            continue
         shares = {
             p: (produced.get("decidability") or {}).get(p, {}).get("share") for p in produced.get("profiles") or []
         }
@@ -228,7 +261,23 @@ def _criteria_rows(sites, inputs: Path, work: Path):
                 )
             )
             continue
-        committed = json.loads(committed_path.read_text())
+        try:
+            committed = json.loads(committed_path.read_text())
+        except ValueError as exc:
+            rows.append(
+                _row(
+                    f"criteria/{site}/report",
+                    "criteria",
+                    f"{site}/gt-laser",
+                    "criteria_report",
+                    shares,
+                    "reproduces committed evidence",
+                    "FAIL",
+                    str(committed_path),
+                    f"committed criteria report is unreadable: {_err(exc)}",
+                )
+            )
+            continue
         diffs = _report_diff(produced, committed)
         rows.append(
             _row(
@@ -257,8 +306,12 @@ def _criteria_rows(sites, inputs: Path, work: Path):
             if not produced_path.is_file() or not ref_path.is_file():
                 diffs.append(f"{profile}: missing produced or reference verdict")
                 continue
-            got = load_json(produced_path).get("site", {}).get("decision")
-            want = load_json(ref_path).get("site", {}).get("decision")
+            try:
+                got = load_json(produced_path).get("site", {}).get("decision")
+                want = load_json(ref_path).get("site", {}).get("decision")
+            except Exception as exc:
+                diffs.append(f"{profile}: unreadable produced or reference verdict: {_err(exc)}")
+                continue
             decisions[profile] = got
             if got != want:
                 diffs.append(f"{profile}: produced {got!r} vs reference {want!r}")
@@ -321,7 +374,24 @@ def _verdict_rows(sites, inputs: Path, work: Path, limits: dict):
                     )
                 )
                 continue
-            results = score_files(verdict_path, [labels_path], out_dir=work / "verdict-scores" / site)
+            try:
+                results = score_files(verdict_path, [labels_path], out_dir=work / "verdict-scores" / site)
+            except Exception as exc:
+                traceback.print_exc()
+                rows.append(
+                    _row(
+                        f"verdict/{target}",
+                        "verdicts",
+                        target,
+                        "verdictscore",
+                        None,
+                        "scored against committed labels",
+                        "FAIL",
+                        str(labels_path),
+                        f"scoring failed: {_err(exc)}",
+                    )
+                )
+                continue
             if not results:
                 rows.append(
                     _row(
@@ -590,10 +660,15 @@ def run_suite(config_path: Path, inputs: Path, out: Path, repo: Path | None = No
     repo = repo or Path(__file__).resolve().parents[2]
     config_path, inputs, out = Path(config_path), Path(inputs), Path(out)
     cfg = yaml.safe_load(config_path.read_text()) or {}
-    sites = [
-        site for site in (cfg.get("sites") or {}) if (inputs / "scenes" / site / "gt-laser" / "scene.json").is_file()
-    ]
+    # Every configured site stays in the run: a site whose scene fixture is missing
+    # is a FAIL row, never a silent omission — an empty or wrong --inputs directory
+    # must not exit 0 having evaluated nothing.
+    sites = list(cfg.get("sites") or {})
+    # Each run reads only what it produced: verdicts and reports left in the work
+    # directory by an earlier run would otherwise stand in for this run's results.
     work = out / "work"
+    if work.exists():
+        shutil.rmtree(work)
     engine_cfg = _engine_config(cfg)
     rows: list[dict] = []
     rows.extend(_fixture_rows(sites, inputs))

From 6ed7d330fc3ab8970753791377c23c0b0ab4a75d Mon Sep 17 00:00:00 2001
From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com>
Date: Sat, 17 Oct 2026 22:18:52 +0000
Subject: [PATCH 2/6] test(backend): eval receipt contract reads the saved
 suite.json
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- _run returns the receipt parsed from disk: suite.json is the seam a
  machine consumes, so the assertions must hold on the saved bytes, not
  the runner's return value.
- test_committed_fixtures_suite checks the produce-hint failures per
  recon row instead of an all-or-nothing asset flag, so a partially
  restored fixture tree (one cloud present, one absent) passes and any
  other failure still fails.
- The public-verdict assertion now requires every configured site's
  public row to exist before checking MEASURED — it could pass vacuously
  when no public rows were produced at all.
- New tests: a broken labels file fails its row and the run still
  reports; a broken committed criteria report fails its row; a
  configured site without its scene fails the run; the work directory
  is cleared between runs.
- _config_for replaces rather than merges overrides: a synthetic inputs
  directory carries only the sites the test wrote.

Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com>
---
 backend/server/eval/tests/test_suite.py | 169 +++++++++++++++++++++---
 1 file changed, 147 insertions(+), 22 deletions(-)

diff --git a/backend/server/eval/tests/test_suite.py b/backend/server/eval/tests/test_suite.py
index d3f3bf4d5..cec48bb7e 100644
--- a/backend/server/eval/tests/test_suite.py
+++ b/backend/server/eval/tests/test_suite.py
@@ -20,20 +20,24 @@
 
 
 def _run(config: Path, inputs: Path, out: Path) -> dict:
-    report = run_suite(config, inputs, out, repo=ROOT)
+    run_suite(config, inputs, out, repo=ROOT)
     assert (out / "suite.json").is_file()
     assert (out / "SUMMARY.md").is_file()
-    assert json.loads((out / "suite.json").read_text())["format"] == "hss-eval-suite/v0"
-    return report
+    # Assertions below inspect the saved receipt, not the runner's return value:
+    # suite.json is the seam a machine consumes, so a writer change that drops
+    # scores, reasons or rows must fail here, on the broken and green paths alike.
+    saved = json.loads((out / "suite.json").read_text())
+    assert saved["format"] == "hss-eval-suite/v0"
+    return saved
 
 
 def _config_for(tmp_path: Path, overrides: dict) -> Path:
     cfg = yaml.safe_load(CONFIG.read_text()) or {}
     for key, value in overrides.items():
-        if isinstance(value, dict) and isinstance(cfg.get(key), dict):
-            cfg[key].update(value)
-        else:
-            cfg[key] = value
+        # Replace rather than merge: a synthetic inputs directory carries only the
+        # sites the test wrote, and a merged site list would name scenes the run
+        # will (rightly) fail as missing.
+        cfg[key] = value
     dest = tmp_path / "suite.yaml"
     dest.write_text(yaml.safe_dump(cfg))
     return dest
@@ -183,19 +187,26 @@ def test_committed_fixtures_suite(tmp_path: Path):
     gt = ROOT / ".fixtures" / "gt" / "eth3d-facade" / "gt-laser"
     if not (gt / "laser.ply").is_file() or not (gt / "occluders.ply").is_file():
         pytest.skip("GT PLYs absent; run make gt for the full-fixture run")
-    has_recon_assets = bool(list((FIXTURES / "scenes").glob("*/colmap-*/points.ply")))
+    recon_dirs = sorted((FIXTURES / "scenes").glob("*/colmap-*"))
     report = _run(CONFIG, FIXTURES, tmp_path / "out")
     failed = [row for row in report["rows"] if row["status"] == "FAIL"]
-    if not has_recon_assets:
-        # The recon PLYs are gitignored Modal-pipeline outputs. On a checkout
-        # without them the geometry rows fail closed with the produce hint and
-        # every other stage must still be green.
-        assert failed and all(
-            row["id"].startswith("geometry/") and "scripts/modal_scene.py" in row["note"] for row in failed
-        ), json.dumps(failed, indent=1, default=str)
-        assert report["summary"]["pass"] > 0
-        return
-    assert not failed, json.dumps(failed, indent=1, default=str)
+    produce_hint = [
+        row for row in failed if row["id"].startswith("geometry/") and "scripts/modal_scene.py" in row["note"]
+    ]
+    other = [row for row in failed if row not in produce_hint]
+    # The recon PLYs are gitignored Modal-pipeline outputs. A recon cloud whose
+    # points.ply is absent must produce exactly its produce-hint failure — and
+    # only that: a restored cloud that fails, or any unrelated failure, is a bug.
+    # Checking per recon row keeps a partial restore (one cloud present, one
+    # absent) a pass instead of a broken test.
+    assert not other, json.dumps(other, indent=1, default=str)
+    expected_missing = sum(
+        1 for d in recon_dirs if (d / "scene.json").is_file() and not (d / "points.ply").is_file()
+    )
+    if expected_missing == 0:
+        assert not produce_hint, json.dumps(produce_hint, indent=1, default=str)
+    else:
+        assert len(produce_hint) == expected_missing, json.dumps(produce_hint, indent=1, default=str)
     assert report["summary"]["pass"] > 0
 
 
@@ -253,11 +264,12 @@ def broken(*args, **kwargs):
     assert all(row["status"] == "FAIL" and "criteria engine failed" in row["note"] for row in by_group["pipeline"])
     # skip-equivalent: the public profile has no committed labels, so it is
     # MEASURED (recorded, no bar) rather than FAIL or silently omitted
-    assert all(
-        row["id"].endswith("/public") and row["status"] == "MEASURED"
-        for row in by_group["verdicts"]
-        if row["id"].endswith("/public")
+    configured_sites = set(yaml.safe_load(CONFIG.read_text())["sites"])
+    public_ids = {f"verdict/{site}/public" for site in configured_sites}
+    assert {row["id"] for row in by_group["verdicts"] if row["id"].endswith("/public")} == public_ids, sorted(
+        row["id"] for row in by_group["verdicts"]
     )
+    assert all(row["status"] == "MEASURED" for row in by_group["verdicts"] if row["id"] in public_ids)
     assert any(row["status"] == "FAIL" for row in by_group["verdicts"] if not row["id"].endswith("/public"))
     # same contract on the green path: scored verdict rows keep the score keys
     monkeypatch.undo()
@@ -300,3 +312,116 @@ def broken(*args, **kwargs):
     assert green["summary"]["fail"] == 0
     verdict = next(row for row in green["rows"] if row["id"] == "verdict/flat-wall/C")
     assert verdict["status"] == "PASS" and verdict["value"]["unsafe_passes"] == 0
+
+
+def _synthetic_site(tmp_path: Path) -> tuple[Path, Path]:
+    """A flat-wall run the engine scores green: scene, empty C labels, and the
+    config naming exactly this site."""
+    inputs = tmp_path / "inputs"
+    scene = write_flat_wall(inputs / "scenes" / "flat-wall" / "gt-laser")
+    probe = evaluate_scene(
+        scene,
+        tmp_path / "probe",
+        config=CriteriaConfig(profiles=("C",), ground_rule="angle80", c2=False, c3=False),
+    )
+    labels_dir = inputs / "labels"
+    labels_dir.mkdir(parents=True)
+    (labels_dir / "flat-wall.C.json").write_text(
+        json.dumps(
+            {
+                "format": "base-labels/v0",
+                "site": "synthetic-flatwall",
+                "rules_profile": "C",
+                "site_decision": probe["C"]["site"]["decision"],
+                "labels": [],
+            }
+        )
+        + "\n"
+    )
+    config = _config_for(
+        tmp_path,
+        {
+            "sites": {"flat-wall": {"labels": ["C"]}},
+            "suite": {"criteria": {"profiles": ["C"], "ground_rule": "angle80", "c2": False}},
+            "verdicts": {
+                "unsafe_passes_max": 0,
+                "false_rejections_max": 0,
+                "coverage_contract_errors_max": 0,
+                "c5_margin_errors_max": 0,
+            },
+        },
+    )
+    return inputs, config
+
+
+def test_a_broken_labels_file_fails_its_row_and_the_run_still_reports(tmp_path: Path):
+    """One malformed labels file is a FAIL verdict row with the scoring error —
+    suite.json and SUMMARY.md still land, and the remaining rows still score."""
+    inputs, config = _synthetic_site(tmp_path)
+    (inputs / "labels" / "flat-wall.C.json").write_text("{not json\n")
+
+    report = _run(config, inputs, tmp_path / "out")
+
+    by_id = {row["id"]: row for row in report["rows"]}
+    verdict = by_id["verdict/flat-wall/C"]
+    assert verdict["status"] == "FAIL"
+    assert "scoring failed" in verdict["note"]
+    assert report["summary"]["fail"] >= 1
+    assert (tmp_path / "out" / "SUMMARY.md").is_file()
+
+
+def test_a_broken_committed_criteria_report_fails_its_row(tmp_path: Path):
+    """The committed criteria report is read per row: a malformed one is a FAIL
+    row naming the unreadable file, not a crashed run."""
+    inputs, config = _synthetic_site(tmp_path)
+    committed = inputs / "criteria" / "flat-wall" / "gt-laser"
+    committed.mkdir(parents=True)
+    (committed / "criteria_report.json").write_text("{not json\n")
+
+    report = _run(config, inputs, tmp_path / "out")
+
+    row = {r["id"]: r for r in report["rows"]}["criteria/flat-wall/report"]
+    assert row["status"] == "FAIL"
+    assert "committed criteria report is unreadable" in row["note"]
+
+
+def test_a_configured_site_without_its_scene_fails_the_run(tmp_path: Path):
+    """A configured site whose scene fixture is absent is a FAIL row on every
+    stage it reaches: an empty or wrong --inputs directory must not exit 0
+    having evaluated nothing."""
+    inputs = tmp_path / "inputs"
+    inputs.mkdir()
+    config = _config_for(tmp_path, {"sites": {"ghost": {"labels": ["C"]}}})
+
+    report = _run(config, inputs, tmp_path / "out")
+
+    by_id = {row["id"]: row for row in report["rows"]}
+    assert by_id["fixtures/ghost"]["status"] == "FAIL"
+    assert "scene.json" in by_id["fixtures/ghost"]["note"]
+    assert by_id["pipeline/ghost"]["status"] == "FAIL"
+    assert "scene fixture missing" in by_id["pipeline/ghost"]["note"]
+    assert report["summary"]["fail"] >= 1
+    assert report["summary"]["pass"] == 0
+
+
+def test_the_work_directory_is_cleared_between_runs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
+    """Each run reads only its own output: after a green run, a broken engine
+    must leave the criteria rows failing on missing reports, not passing on the
+    stale ones the previous run left in the work directory."""
+    inputs, config = _synthetic_site(tmp_path)
+    out = tmp_path / "out"
+    green = _run(config, inputs, out)
+    assert green["summary"]["fail"] == 0
+    stale = out / "work" / "flat-wall" / "criteria" / "criteria_report.json"
+    assert stale.is_file()
+
+    def broken(*args, **kwargs):
+        raise ImportError("not_merged:ws-09")
+
+    monkeypatch.setattr("server.eval.suite.evaluate_scene", broken)
+    second = _run(config, inputs, out)
+
+    by_id = {row["id"]: row for row in second["rows"]}
+    assert by_id["pipeline/flat-wall"]["status"] == "FAIL"
+    assert by_id["criteria/flat-wall/report"]["status"] == "FAIL"
+    assert "produced no criteria report" in by_id["criteria/flat-wall/report"]["note"]

From 06e84b926c73190fb04f368646026833baa3be9b Mon Sep 17 00:00:00 2001
From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com>
Date: Sat, 17 Oct 2026 22:18:52 +0000
Subject: [PATCH 3/6] fix(placement): refuse ground bands without depth and
 band counts past the wire limit

Review findings on PR #295's scene bridge (burst 17):

- A ground band without out_m has no wire out_ft (scene.schema.json
  requires out_ft on every ground band while the source contract leaves
  out_m optional): refuse with the pointer at the band's depth instead
  of converting a document the root contract rejects.
- The wire schema caps coverage.observed at 500 entries: a source with
  more bands now refuses at conversion instead of producing a wire
  scene the placement server cannot read.

Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com>
---
 backend/server/placement/scene_bridge.py      | 20 +++++++
 .../placement/tests/test_scene_bridge.py      | 52 ++++++++++++++++++-
 2 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/backend/server/placement/scene_bridge.py b/backend/server/placement/scene_bridge.py
index aada91a63..55feea7d9 100644
--- a/backend/server/placement/scene_bridge.py
+++ b/backend/server/placement/scene_bridge.py
@@ -107,6 +107,9 @@
 UNSUPPORTED = "placement_scene_unsupported"
 INVALID = "placement_scene_invalid"
 
+# scene.schema.json: coverage.observed allows at most 500 band entries.
+_WIRE_MAX_OBSERVED_BANDS = 500
+
 # 1 ft = 0.3048 m exactly, so lengths convert with one exact factor in one direction:
 # dividing by 0.3048, the correctly rounded conversion, never multiplying by a rounded
 # reciprocal.
@@ -588,6 +591,16 @@ def end_out(end, side: str) -> dict[str, Any]:
     def band_out(band, pointer: str) -> dict[str, Any]:
         if band.wall_id is not None and band.wall_id != wall_id:
             _invalid(f"the band names unknown wall {band.wall_id!r}", f"{pointer}/wall_id")
+        if band.band == "ground" and band.out_m is None:
+            # scene.schema.json requires out_ft on every ground band ("the ground in
+            # front of the wall out to out_ft"), while the source contract leaves
+            # out_m optional — this band cannot be represented, so refuse rather
+            # than guess a depth the capture never measured.
+            _unsupported(
+                "a ground band without a measured depth (out_m) has no wire out_ft; "
+                "the ground's depth was never captured",
+                f"{pointer}/out_m",
+            )
         record: dict[str, Any] = {
             "band": band.band,
             "span_ft": checks.span_ft(band.s_m, f"{pointer}/s_m", bounded=False),
@@ -606,6 +619,13 @@ def band_out(band, pointer: str) -> dict[str, Any]:
             # ends only — no observation is invented.
             band_out(band, f"/coverage/sensitivity/{sensitivity_index}/bands/{band_index}")
 
+    if len(observed) > _WIRE_MAX_OBSERVED_BANDS:
+        _unsupported(
+            f"{len(observed)} observed bands exceed the wire scene's limit of "
+            f"{_WIRE_MAX_OBSERVED_BANDS} (scene.schema.json coverage.observed maxItems); "
+            "thin or merge the capture's bands before conversion",
+            "/coverage/bands",
+        )
     if observed:
         out["observed"] = observed
     return out
diff --git a/backend/server/placement/tests/test_scene_bridge.py b/backend/server/placement/tests/test_scene_bridge.py
index 21904e1e2..28c5be4af 100644
--- a/backend/server/placement/tests/test_scene_bridge.py
+++ b/backend/server/placement/tests/test_scene_bridge.py
@@ -15,7 +15,7 @@
 from typing import Any
 
 import pytest
-from jsonschema import Draft202012Validator
+from jsonschema import Draft202012Validator, ValidationError
 
 from server.placement.adapter import load_solver
 from server.placement.scene_bridge import (
@@ -706,3 +706,53 @@ def test_the_committed_orchestrator_stub_scene_converts_and_parses() -> None:
         "left": {"kind": "limit"},
         "right": {"kind": "limit"},
     }
+
+
+def test_a_ground_band_without_a_measured_depth_refuses() -> None:
+    """The source contract leaves out_m optional, but the wire scene requires out_ft
+    on every ground band. The bridge must refuse the band at its depth instead of
+    converting a document the root contract rejects — and never guess a depth."""
+    document = base_scene()
+    document["coverage"]["bands"] = [{"band": "ground", "s_m": [-1.0, 1.0]}]
+
+    with pytest.raises(SceneBridgeError) as excinfo:
+        convert_scene(document)
+
+    assert excinfo.value.code == UNSUPPORTED
+    assert excinfo.value.pointer == "/coverage/bands/0/out_m"
+
+    # The premise, checked against the root contract: an observed ground band
+    # without out_ft is exactly what scene.schema.json refuses.
+    with pytest.raises(ValidationError):
+        _WIRE_SCHEMA.validate({"band": "ground", "span_ft": [-1.0 / 0.3048, 1.0 / 0.3048]})
+
+    # A measured ground band converts and validates as an observation.
+    document["coverage"]["bands"] = [{"band": "ground", "s_m": [-1.0, 1.0], "out_m": 2.5}]
+    converted = convert_scene(document)
+    _WIRE_SCHEMA.validate(converted)
+    assert converted["coverage"]["observed"] == [
+        {"band": "ground", "span_ft": [-1.0 / 0.3048, 1.0 / 0.3048], "out_ft": 2.5 / 0.3048}
+    ]
+
+
+def test_observed_band_counts_past_the_wire_limit_refuse() -> None:
+    """scene.schema.json caps coverage.observed at 500 entries: a valid source with
+    501 bands must refuse at conversion, not produce a wire scene the placement
+    server cannot read."""
+    document = base_scene()
+    document["coverage"]["bands"] = [
+        {"band": "wall", "s_m": [-1.0 + index * 0.01, 1.0 + index * 0.01]} for index in range(501)
+    ]
+
+    with pytest.raises(SceneBridgeError) as excinfo:
+        convert_scene(document)
+
+    assert excinfo.value.code == UNSUPPORTED
+    assert excinfo.value.pointer == "/coverage/bands"
+    assert "500" in str(excinfo.value)
+
+    # The limit itself still converts, and the output is a valid wire scene.
+    document["coverage"]["bands"] = document["coverage"]["bands"][:500]
+    converted = convert_scene(document)
+    assert len(converted["coverage"]["observed"]) == 500
+    _WIRE_SCHEMA.validate(converted)

From dc33f1e529e8b524d60077e922e7779f9c783197 Mon Sep 17 00:00:00 2001
From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com>
Date: Sat, 17 Oct 2026 22:18:52 +0000
Subject: [PATCH 4/6] fix(placement): turn the multiwall packet case into a
 real corner L

The fixture's notes claimed an L of two house walls, but rear ran
collinear with side and overlapped it. rear now turns 90 degrees from
side's right end, the chain carries the observed corner and updated
end limits, and the manifest's pinned sha256 is refreshed.

Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com>
---
 .../tests/data/capture_cases_v1/manifest.json |  2 +-
 .../multiwall-source-unsupported.source.json  | 27 ++++++++++++++-----
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/backend/server/placement/tests/data/capture_cases_v1/manifest.json b/backend/server/placement/tests/data/capture_cases_v1/manifest.json
index 2a5f278d5..e53e4ea62 100644
--- a/backend/server/placement/tests/data/capture_cases_v1/manifest.json
+++ b/backend/server/placement/tests/data/capture_cases_v1/manifest.json
@@ -188,7 +188,7 @@
       "kind": "negative_source",
       "source": {
         "path": "multiwall-source-unsupported.source.json",
-        "sha256": "3e13bace8381dd56bdaaa6e5e874cca488c8ea08617eb09c8bf4e19b8c75de05"
+        "sha256": "a4c52e79da950e0d22e3853cb9218f7e9252765a1b66739c8039f972a771d652"
       },
       "wire": null,
       "expected_error": {
diff --git a/backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json b/backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json
index 18f8ab413..9b8a28249 100644
--- a/backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json
+++ b/backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json
@@ -11,7 +11,7 @@
     "gravity_source": "ground_truth",
     "site": "burst8-multiwall-unsupported",
     "notes": [
-      "Authored for the burst8 supported-case packet: an L of two house walls, everything else inside the bridge's bounded scope (observed on-wall meter anchor, observed wall_rect hazard, level ground, band coverage). The chain has two walls, so the bridge must refuse it at /walls with placement_scene_unsupported."
+      "Authored for the burst8 supported-case packet: an L of two house walls — rear turns 90 degrees from side's right end — everything else inside the bridge's bounded scope (observed on-wall meter anchor, observed wall_rect hazard, level ground, band coverage). The chain has two walls, so the bridge must refuse it at /walls with placement_scene_unsupported."
     ]
   },
   "frame": {
@@ -45,12 +45,12 @@
       "role": "house",
       "baseline_m": [
         [
-          -0.4572,
+          1.524,
           0.0
         ],
         [
-          0.9144,
-          0.0
+          1.524,
+          -0.9144
         ]
       ],
       "height_m": 2.7432,
@@ -70,9 +70,24 @@
       },
       "right": {
         "kind": "limit",
-        "s_m": 1.524
+        "s_m": 2.4384
       }
-    }
+    },
+    "corners": [
+      {
+        "id": "corner-side-rear",
+        "left_wall": "side",
+        "right_wall": "rear",
+        "point_m": [
+          1.524,
+          0.0
+        ],
+        "s_m": 1.524,
+        "turn_deg": 90.0,
+        "plus_minus_m": 0.09,
+        "evidence": "observed"
+      }
+    ]
   },
   "ground": {
     "surface": {

From 1289c742ff1c2238e25213d01eae36478b8b5bb8 Mon Sep 17 00:00:00 2001
From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com>
Date: Sat, 17 Oct 2026 22:18:52 +0000
Subject: [PATCH 5/6] style: ruff format drift on alpha

make lint's ruff format --check fails on alpha at
test_greptile_followup.py (a long tuple line ruff now wraps); apply the
formatter so the branch's lint gate is green. No semantic change.

Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com>
---
 .../orchestrator/tests/test_greptile_followup.py    | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/backend/server/orchestrator/tests/test_greptile_followup.py b/backend/server/orchestrator/tests/test_greptile_followup.py
index 69a975242..f35446e42 100644
--- a/backend/server/orchestrator/tests/test_greptile_followup.py
+++ b/backend/server/orchestrator/tests/test_greptile_followup.py
@@ -160,7 +160,18 @@ def test_confirmation_updates_the_reads_record(tmp_path: Path) -> None:
     # The recompose preserves the upstream work as terminal records at attempt 1
     # — the re-run reads stage that once threatened the answer never executes.
     rows = {rec.stage: rec for rec in services.state.stages("run_reads")}
-    for upstream in ("validate", "poses", "objects", "reads", "scale", "dense", "coverage", "scene", "objects_3d", "placement"):
+    for upstream in (
+        "validate",
+        "poses",
+        "objects",
+        "reads",
+        "scale",
+        "dense",
+        "coverage",
+        "scene",
+        "objects_3d",
+        "placement",
+    ):
         assert rows[upstream].status == "done", upstream
         assert rows[upstream].attempt == 1, upstream
     assert rows["criteria"].status == "done"

From 00cdda9ba28a097619bed5af2f5c704b663c11e4 Mon Sep 17 00:00:00 2001
From: SamGu-NRX <sgu07966@gmail.com>
Date: Thu, 22 Oct 2026 09:09:14 -0500
Subject: [PATCH 6/6] fix(eval): refuse malformed reports without aborting

---
 backend/server/eval/suite.py                  | 51 +++++++++++++++++++
 backend/server/eval/tests/test_suite.py       | 36 +++++++++++--
 backend/server/placement/scene_bridge.py      |  8 +--
 .../placement/tests/test_scene_bridge.py      | 17 +++++++
 4 files changed, 104 insertions(+), 8 deletions(-)

diff --git a/backend/server/eval/suite.py b/backend/server/eval/suite.py
index 703d17ec2..dfd80bcb6 100644
--- a/backend/server/eval/suite.py
+++ b/backend/server/eval/suite.py
@@ -197,6 +197,25 @@ def _pipeline_rows(sites, inputs: Path, repo: Path, work: Path, engine_cfg: Crit
     return rows
 
 
+def _report_shape_error(report: object) -> str | None:
+    """Return a stable refusal for a parsed report with unusable containers."""
+    if not isinstance(report, dict):
+        return f"criteria report must be an object, got {type(report).__name__}"
+    profiles = report.get("profiles")
+    if profiles is not None and (
+        not isinstance(profiles, list) or not all(isinstance(profile, str) for profile in profiles)
+    ):
+        return "criteria report profiles must be a list of strings"
+    decidability = report.get("decidability")
+    if decidability is not None and not isinstance(decidability, dict):
+        return "criteria report decidability must be an object"
+    if isinstance(decidability, dict):
+        for profile, value in decidability.items():
+            if not isinstance(profile, str) or not isinstance(value, dict):
+                return "criteria report decidability entries must be objects keyed by profile"
+    return None
+
+
 def _report_diff(produced: dict, committed: dict) -> list[str]:
     diffs = []
     for field in REPORT_FIELDS:
@@ -242,6 +261,22 @@ def _criteria_rows(sites, inputs: Path, work: Path):
                 )
             )
             continue
+        shape_error = _report_shape_error(produced)
+        if shape_error:
+            rows.append(
+                _row(
+                    f"criteria/{site}/report",
+                    "criteria",
+                    f"{site}/gt-laser",
+                    "criteria_report",
+                    None,
+                    "reproduces committed evidence",
+                    "FAIL",
+                    str(report_path),
+                    shape_error,
+                )
+            )
+            continue
         shares = {
             p: (produced.get("decidability") or {}).get(p, {}).get("share") for p in produced.get("profiles") or []
         }
@@ -278,6 +313,22 @@ def _criteria_rows(sites, inputs: Path, work: Path):
                 )
             )
             continue
+        committed_shape_error = _report_shape_error(committed)
+        if committed_shape_error:
+            rows.append(
+                _row(
+                    f"criteria/{site}/report",
+                    "criteria",
+                    f"{site}/gt-laser",
+                    "criteria_report",
+                    shares,
+                    "reproduces committed evidence",
+                    "FAIL",
+                    str(committed_path),
+                    committed_shape_error,
+                )
+            )
+            continue
         diffs = _report_diff(produced, committed)
         rows.append(
             _row(
diff --git a/backend/server/eval/tests/test_suite.py b/backend/server/eval/tests/test_suite.py
index cec48bb7e..122ddca4f 100644
--- a/backend/server/eval/tests/test_suite.py
+++ b/backend/server/eval/tests/test_suite.py
@@ -11,7 +11,7 @@
 
 from server.eval.adapters.criteria import CriteriaConfig, evaluate_scene
 from server.eval.coverage import MissingOccluderError
-from server.eval.suite import _engine_config, _matches_reference, _reference, run_suite
+from server.eval.suite import _criteria_rows, _engine_config, _matches_reference, _reference, run_suite
 from server.eval.tests.flat_wall import write as write_flat_wall
 
 ROOT = Path(__file__).resolve().parents[3]
@@ -200,9 +200,7 @@ def test_committed_fixtures_suite(tmp_path: Path):
     # Checking per recon row keeps a partial restore (one cloud present, one
     # absent) a pass instead of a broken test.
     assert not other, json.dumps(other, indent=1, default=str)
-    expected_missing = sum(
-        1 for d in recon_dirs if (d / "scene.json").is_file() and not (d / "points.ply").is_file()
-    )
+    expected_missing = sum(1 for d in recon_dirs if (d / "scene.json").is_file() and not (d / "points.ply").is_file())
     if expected_missing == 0:
         assert not produce_hint, json.dumps(produce_hint, indent=1, default=str)
     else:
@@ -370,6 +368,36 @@ def test_a_broken_labels_file_fails_its_row_and_the_run_still_reports(tmp_path:
     assert (tmp_path / "out" / "SUMMARY.md").is_file()
 
 
+def test_malformed_produced_criteria_report_is_a_fail_row(tmp_path: Path):
+    """A parsed JSON scalar or list must fail its row instead of crashing the suite."""
+    inputs = tmp_path / "inputs"
+    work = tmp_path / "work"
+    report_path = work / "flat-wall" / "criteria" / "criteria_report.json"
+    report_path.parent.mkdir(parents=True)
+    report_path.write_text("null\n")
+
+    rows = _criteria_rows(["flat-wall"], inputs, work)
+
+    assert rows[0]["status"] == "FAIL"
+    assert "must be an object" in rows[0]["note"]
+
+
+def test_malformed_committed_criteria_report_is_a_fail_row(tmp_path: Path):
+    inputs = tmp_path / "inputs"
+    work = tmp_path / "work"
+    report_path = work / "flat-wall" / "criteria" / "criteria_report.json"
+    report_path.parent.mkdir(parents=True)
+    report_path.write_text(json.dumps({"profiles": ["C"], "decidability": {"C": {"share": 1.0}}}))
+    committed = inputs / "criteria" / "flat-wall" / "gt-laser"
+    committed.mkdir(parents=True)
+    (committed / "criteria_report.json").write_text("[]\n")
+
+    rows = _criteria_rows(["flat-wall"], inputs, work)
+
+    assert rows[0]["status"] == "FAIL"
+    assert "must be an object" in rows[0]["note"]
+
+
 def test_a_broken_committed_criteria_report_fails_its_row(tmp_path: Path):
     """The committed criteria report is read per row: a malformed one is a FAIL
     row naming the unreadable file, not a crashed run."""
diff --git a/backend/server/placement/scene_bridge.py b/backend/server/placement/scene_bridge.py
index 55feea7d9..70acebe3d 100644
--- a/backend/server/placement/scene_bridge.py
+++ b/backend/server/placement/scene_bridge.py
@@ -588,10 +588,10 @@ def end_out(end, side: str) -> dict[str, Any]:
 
     observed: list[dict[str, Any]] = []
 
-    def band_out(band, pointer: str) -> dict[str, Any]:
+    def band_out(band, pointer: str, *, require_ground_depth: bool) -> dict[str, Any]:
         if band.wall_id is not None and band.wall_id != wall_id:
             _invalid(f"the band names unknown wall {band.wall_id!r}", f"{pointer}/wall_id")
-        if band.band == "ground" and band.out_m is None:
+        if require_ground_depth and band.band == "ground" and band.out_m is None:
             # scene.schema.json requires out_ft on every ground band ("the ground in
             # front of the wall out to out_ft"), while the source contract leaves
             # out_m optional — this band cannot be represented, so refuse rather
@@ -610,14 +610,14 @@ def band_out(band, pointer: str) -> dict[str, Any]:
         return record
 
     for index, band in enumerate(scene.coverage.bands or []):
-        observed.append(band_out(band, f"/coverage/bands/{index}"))
+        observed.append(band_out(band, f"/coverage/bands/{index}", require_ground_depth=True))
     for sensitivity_index, sensitivity in enumerate(scene.coverage.sensitivity or []):
         for band_index, band in enumerate(sensitivity.bands):
             # Sensitivity variants are validated but are not observations: the wire's
             # observed list is the ground-truth coverage the device saw, and a what-if
             # sensitivity band is not that. A sensitivity-only scene still converts to
             # ends only — no observation is invented.
-            band_out(band, f"/coverage/sensitivity/{sensitivity_index}/bands/{band_index}")
+            band_out(band, f"/coverage/sensitivity/{sensitivity_index}/bands/{band_index}", require_ground_depth=False)
 
     if len(observed) > _WIRE_MAX_OBSERVED_BANDS:
         _unsupported(
diff --git a/backend/server/placement/tests/test_scene_bridge.py b/backend/server/placement/tests/test_scene_bridge.py
index 28c5be4af..2a517acde 100644
--- a/backend/server/placement/tests/test_scene_bridge.py
+++ b/backend/server/placement/tests/test_scene_bridge.py
@@ -339,6 +339,23 @@ def test_meter_wall_rect_geometry_places_the_meter_at_its_span_centre_mid_height
     assert parsed.point_at(0.0)[0] == pytest.approx(0.0, abs=1e-6)
 
 
+def test_ground_sensitivity_without_depth_is_discarded() -> None:
+    """Sensitivity bands are validated for span but never emitted to the wire."""
+    document = base_scene()
+    document["coverage"]["sensitivity"] = [
+        {
+            "surface": "ground",
+            "max_incidence_deg": 75,
+            "bands": [{"band": "ground", "s_m": [-2.0, 4.0]}],
+        }
+    ]
+
+    converted = convert_scene(document)
+
+    assert "observed" in converted["coverage"]
+    assert all(band["band"] != "ground" for band in converted["coverage"]["observed"])
+
+
 def test_sensitivity_only_coverage_converts_to_ends_only() -> None:
     """A sensitivity band is a what-if variant, not a device observation: a scene
     whose coverage carries only sensitivity bands must convert to ends only, with