From bad040ccfc057052ea752a932556d5ddfe20e770 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:47:11 +0000 Subject: [PATCH 01/19] fix(backend): make the case suite reject invalid manifests and forged input hashes One evaluator contract change, from the P1 (Codex r4287473089) and P2 (Greptile r4287478936/r4287478951) findings on #271: - load_cases requires a nonempty cases list, unique nonempty ids, a recognized kind, the fixture records each kind needs (positive: source plus wire; negative: source), meaningful positive expectations (a pinned decision plus at least one geometric expectation of the right shape) and a negative expected_error with nonempty code and pointer. Malformed types are rejected with the manifest path and field instead of a silent default; fixture hash/path errors are still validated before any record is returned. - _positive_case now requires stats.input_sha256 to equal the sha256 of the exact posted wire bytes; a discrepancy becomes a named mismatch, so it fails the case status, report.ok and the CLI exit. load_cases and the existing four-case manifest stay compatible: no fixture repinning, no new geometry, no solver changes. --- backend/server/placement/case_suite.py | 137 ++++++++++++++++-- backend/tests/test_placement_case_suite.py | 154 +++++++++++++++++++++ 2 files changed, 280 insertions(+), 11 deletions(-) diff --git a/backend/server/placement/case_suite.py b/backend/server/placement/case_suite.py index 16d97e464..f24dda085 100644 --- a/backend/server/placement/case_suite.py +++ b/backend/server/placement/case_suite.py @@ -65,6 +65,19 @@ _REPO_ROOT = Path(__file__).parents[3] RESULT_SCHEMA = _REPO_ROOT / "server" / "schemas" / "result.schema.json" SOURCE_SCHEMA = _REPO_ROOT / "backend" / "server" / "contracts" / "schemas" / "scene.v0.schema.json" +# Recognized expectation keys and the container shape each must have. +EXPECTATION_KEYS = ("decision", "spot", "checks", "missing_evidence", "sweep_failing_runs", "ends") +EXPECTATION_SHAPES = { + "decision": str, + "spot": dict, + "checks": dict, + "missing_evidence": list, + "sweep_failing_runs": list, + "ends": dict, +} +# A positive case's expectations are meaningful only when they pin the decision +# and at least one geometric expectation that can actually mismatch. +GEOMETRIC_EXPECTATION_KEYS = ("spot", "checks", "missing_evidence", "sweep_failing_runs", "ends") class CaseSuiteError(RuntimeError): @@ -108,6 +121,32 @@ def _check_fixture(name: str, fixture: dict[str, Any], base: Path, problems: lis return path +def _expectations_problem(expectations: Any) -> str | None: + """Return a named problem when a positive case's expectations could not + fail a solver regression, or ``None`` when they can.""" + if not isinstance(expectations, dict): + return "must be an object of pinned expectations" + for key in EXPECTATION_KEYS: + if key in expectations and not isinstance(expectations[key], EXPECTATION_SHAPES[key]): + shape = "a string" if EXPECTATION_SHAPES[key] is str else ( + "an object" if EXPECTATION_SHAPES[key] is dict else "an array" + ) + return f".{key} must be {shape} (got {type(expectations[key]).__name__})" + if not isinstance(expectations.get("decision"), str) or not expectations.get("decision"): + return "must pin a nonempty 'decision' string" + geometric = [ + key + for key in GEOMETRIC_EXPECTATION_KEYS + if isinstance(expectations.get(key), EXPECTATION_SHAPES[key]) and expectations[key] + ] + if not geometric: + return ( + "must pin at least one geometric expectation with content " + f"(one of {', '.join(GEOMETRIC_EXPECTATION_KEYS)}), so a solver change can fail the case" + ) + return None + + def _problems_message(problems: list[str]) -> str: return "case suite fixtures failed to load:\n" + "\n".join(f"- {p}" for p in problems) @@ -115,30 +154,97 @@ def _problems_message(problems: list[str]) -> str: def load_cases(path: Path) -> list[CaseRecord]: """Load the manifest at ``path`` and verify its fixtures. - Raises :class:`CaseSuiteError` naming every missing or drifted fixture path. + The loader is the gate against a silently weakened manifest: it requires a + nonempty ``cases`` list of unique nonempty ids with a recognized ``kind`` + (``positive`` or ``negative_source``), the fixture records each kind needs + (positive: ``source`` plus ``wire``; negative: ``source``), meaningful + positive expectations (a pinned ``decision`` plus at least one geometric + expectation of the right shape) and a negative ``expected_error`` with + nonempty ``code`` and ``pointer`` strings. Fixture existence and pinned + sha256s are checked before any record is returned. Every rejection names + the manifest path and the offending field (``cases[N].field``) instead of + guessing a default. + + Raises :class:`CaseSuiteError` with the manifest path and field on any + malformed entry, and naming every missing or drifted fixture path. """ problems: list[str] = [] if not path.is_file(): raise CaseSuiteError(_problems_message([f"missing case manifest: {path}"])) manifest = json.loads(path.read_text()) + if not isinstance(manifest, dict): + raise CaseSuiteError(f"case manifest {path}: top level must be a JSON object") if manifest.get("schema_version") != MANIFEST_SCHEMA_VERSION: problems.append(f"{path}: schema_version {manifest.get('schema_version')!r} != {MANIFEST_SCHEMA_VERSION!r}") raise CaseSuiteError(_problems_message(problems)) + raw_cases = manifest.get("cases") + if not isinstance(raw_cases, list) or not raw_cases: + raise CaseSuiteError( + f"case manifest {path}: field 'cases' must be a nonempty list of case objects " + f"(got {'nothing' if raw_cases is None else type(raw_cases).__name__})" + ) base = path.parent records: list[CaseRecord] = [] - for raw in manifest.get("cases", []): - name = raw.get("id", "") - source = _check_fixture(name, raw.get("source") or {}, base, problems) - wire = raw.get("wire") - wire_path = _check_fixture(name, wire, base, problems) if wire else None + seen_ids: set[str] = set() + for index, raw in enumerate(raw_cases): + field = f"cases[{index}]" + if not isinstance(raw, dict): + raise CaseSuiteError(f"case manifest {path}: field {field} must be an object (got {type(raw).__name__})") + name = raw.get("id") + if not isinstance(name, str) or not name: + raise CaseSuiteError(f"case manifest {path}: field {field}.id must be a nonempty string (got {name!r})") + if name in seen_ids: + raise CaseSuiteError(f"case manifest {path}: field {field}.id duplicates id {name!r} from an earlier case") + seen_ids.add(name) + kind = raw.get("kind") + if kind not in (KIND_POSITIVE, KIND_NEGATIVE): + raise CaseSuiteError( + f"case manifest {path}: field {field}.kind {kind!r} is not a recognized case kind " + f"(expected {KIND_POSITIVE!r} or {KIND_NEGATIVE!r})" + ) + source = raw.get("source") + if not isinstance(source, dict): + raise CaseSuiteError( + f"case manifest {path}: field {field}.source must be an object with a path and a pinned sha256" + ) + source_path = _check_fixture(name, source, base, problems) + expectations: dict[str, Any] = {} + expected_error: dict[str, str] | None = None + if kind == KIND_POSITIVE: + wire = raw.get("wire") + if not isinstance(wire, dict): + raise CaseSuiteError( + f"case manifest {path}: field {field}.wire must be an object with a path and a pinned " + "sha256 (a positive case is executed, so its wire fixture is mandatory)" + ) + wire_path = _check_fixture(name, wire, base, problems) + expectations = raw.get("expectations") + problem = _expectations_problem(expectations) + if problem is not None: + raise CaseSuiteError(f"case manifest {path}: field {field}.expectations {problem}") + expectations = dict(expectations) + else: + wire_path = None + expected_error = raw.get("expected_error") + if ( + not isinstance(expected_error, dict) + or not isinstance(expected_error.get("code"), str) + or not expected_error.get("code") + or not isinstance(expected_error.get("pointer"), str) + or not expected_error.get("pointer") + ): + raise CaseSuiteError( + f"case manifest {path}: field {field}.expected_error must be an object with a nonempty " + "'code' and 'pointer' string (the error the future converter must produce)" + ) records.append( CaseRecord( id=name, - kind=raw.get("kind", ""), - source_path=source, + kind=kind, + source_path=source_path, wire_path=wire_path, - expected_error=raw.get("expected_error"), - expectations=raw.get("expectations") or {}, + expected_error=expected_error, + expectations=expectations, note=raw.get("note"), ) ) @@ -266,7 +372,15 @@ def _positive_case( case["executions_agree"] = _agree_without_elapsed(first, second) stats = first.get("stats", {}) case["input_sha256"] = stats.get("input_sha256") - case["input_sha_matches_file"] = case["input_sha256"] == _file_sha256(record.wire_path) + posted_sha = _file_sha256(record.wire_path) + case["input_sha_matches_file"] = case["input_sha256"] == posted_sha + if not case["input_sha_matches_file"]: + # A response tied to different bytes must not pass, no matter how well + # its placement fields happen to satisfy the expectations. + case["mismatches"].append( + f"stats.input_sha256 {case['input_sha256']!r} does not match the sha256 of the " + f"posted wire bytes ({posted_sha})" + ) case["policy"] = first.get("policy") case["decision"] = first.get("decision") case["spot"] = first.get("spot") @@ -413,5 +527,6 @@ def main(argv: list[str] | None = None) -> int: return 0 if report["ok"] else 1 + if __name__ == "__main__": raise SystemExit(main()) diff --git a/backend/tests/test_placement_case_suite.py b/backend/tests/test_placement_case_suite.py index 0ff93c85a..a8fcf2b4e 100644 --- a/backend/tests/test_placement_case_suite.py +++ b/backend/tests/test_placement_case_suite.py @@ -187,3 +187,157 @@ def test_manifest_copy_is_untouched_by_mutations() -> None: original = copy.deepcopy(payload) payload["cases"][0]["expectations"]["decision"] = "reject" assert json.loads(MANIFEST.read_text()) == original + + + +def test_missing_cases_key_fails_with_manifest_path(tmp_path: Path) -> None: + """A manifest without a cases key cannot silently disable the gate.""" + payload = _absolutized_payload() + del payload["cases"] + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + message = str(excinfo.value) + assert str(drift) in message + assert "cases" in message + + +def test_empty_cases_list_fails_at_the_cli(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + payload = _absolutized_payload() + payload["cases"] = [] + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + out = tmp_path / "report.json" + assert cs.main(["--manifest", str(drift), "--output", str(out)]) == 1 + assert "cases" in capsys.readouterr().err + assert not out.exists() + + +def test_duplicate_case_ids_fail_with_field(tmp_path: Path) -> None: + payload = _absolutized_payload() + payload["cases"][1]["id"] = payload["cases"][0]["id"] + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + message = str(excinfo.value) + assert "cases[1].id" in message + assert payload["cases"][0]["id"] in message + + +@pytest.mark.parametrize("kind", ["positve", "", 3, None]) +def test_malformed_case_kinds_fail_with_field(tmp_path: Path, kind: object) -> None: + payload = _absolutized_payload() + payload["cases"][0]["kind"] = kind + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert "cases[0].kind" in str(excinfo.value) + + +@pytest.mark.parametrize( + ("mangle", "why"), + [ + (lambda case: case.pop("expectations"), "absent"), + (lambda case: case.update(expectations={}), "empty"), + (lambda case: case.update(expectations={"decision": "manual_review"}), "decision only"), + (lambda case: case["expectations"].update(spot="the good spot"), "spot wrong type"), + (lambda case: case["expectations"].update(checks=["gas_clearance"]), "checks wrong type"), + ], + ids=["absent", "empty", "decision-only", "spot-wrong-type", "checks-wrong-type"], +) +def test_unusable_positive_expectations_fail_with_field( + tmp_path: Path, mangle, why: str +) -> None: + payload = _absolutized_payload() + mangle(payload["cases"][0]) + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert "cases[0].expectations" in str(excinfo.value), why + + +def test_negative_case_requires_declared_error(tmp_path: Path) -> None: + payload = _absolutized_payload() + payload["cases"][2].pop("expected_error") + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert "cases[2].expected_error" in str(excinfo.value) + + thin = _absolutized_payload() + thin["cases"][2]["expected_error"] = {"code": "placement_scene_unsupported"} + thin_manifest = tmp_path / "thin.json" + thin_manifest.write_text(json.dumps(thin)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(thin_manifest) + assert "cases[2].expected_error" in str(excinfo.value) + + +def _forging_client(monkeypatch: pytest.MonkeyPatch, mutate) -> None: + """Swap the suite's client for one that mutates every 200 response body, + simulating a solver/API regression in the echoed response.""" + inner = cs._build_client() + + class _ShimResponse: + def __init__(self, status_code: int, body: dict) -> None: + self.status_code = status_code + self.text = "" + self._body = body + + def json(self) -> dict: + return self._body + + class _ForgingClient: + def post(self, *args: object, **kwargs: object) -> _ShimResponse: + response = inner.post(*args, **kwargs) + assert response.status_code == 200 + body = response.json() + mutate(body) + return _ShimResponse(200, body) + + monkeypatch.setattr(cs, "_get_client", lambda: _ForgingClient()) + + +def test_forged_input_sha256_fails_the_report(monkeypatch: pytest.MonkeyPatch) -> None: + """A response whose echoed hash points at other bytes must not match.""" + _forging_client( + monkeypatch, + lambda body: body.setdefault("stats", {}).update(input_sha256="0" * 64), + ) + report = cs.run_suite(MANIFEST) + case = next(case for case in report["cases"] if case["kind"] == cs.KIND_POSITIVE) + assert case["status"] == cs.STATUS_MISMATCHED + assert case["input_sha_matches_file"] is False + assert any("input_sha256" in mismatch for mismatch in case["mismatches"]) + assert report["ok"] is False + + +def test_missing_input_sha256_fails_the_report(monkeypatch: pytest.MonkeyPatch) -> None: + _forging_client( + monkeypatch, + lambda body: body.setdefault("stats", {}).pop("input_sha256", None), + ) + report = cs.run_suite(MANIFEST) + case = next(case for case in report["cases"] if case["kind"] == cs.KIND_POSITIVE) + assert case["status"] == cs.STATUS_MISMATCHED + assert case["input_sha256"] is None + + +def test_forged_input_sha256_fails_the_cli( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + _forging_client( + monkeypatch, + lambda body: body["stats"].update(input_sha256="f" * 64), + ) + out = tmp_path / "report.json" + assert cs.main(["--output", str(out)]) == 1 + assert json.loads(out.read_text())["ok"] is False + err = capsys.readouterr().err + assert "status=mismatched" in err + assert "input_sha256" in err From 83d93f0242e75178876b06cdc1df94b2d4e19f8a Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:09:57 +0000 Subject: [PATCH 02/19] Integrate burst8-bs-replay-acceptance runner enforcement (producer 5f9e2957, start bdfbb3fc) --- backend/scripts/capture_placement_cases.py | 393 ++++++++- backend/tests/test_capture_placement_cases.py | 819 +++++++++++++++++- 2 files changed, 1141 insertions(+), 71 deletions(-) diff --git a/backend/scripts/capture_placement_cases.py b/backend/scripts/capture_placement_cases.py index 1418752d1..bc9506f49 100644 --- a/backend/scripts/capture_placement_cases.py +++ b/backend/scripts/capture_placement_cases.py @@ -34,15 +34,34 @@ Checks per case (any mismatch fails the suite with a nonzero exit): - The merged scene in the run's artifacts equals the independently computed - stamp + merge of that case's source — a substituted reference scene, another + stamp + merge of that case's source — the merge contract is re-derived here + (same-id precedence asserted explicitly, never delegated to the production + ``merge_object_lists`` helper) — so a substituted reference scene, another case's source, or a second merge is caught here. - The placement stage's typed refusal (code, pointer) for source cases the - bridge refuses; no placement artifact on refusal; the served device result - stays conservative (``manual_review``) with exactly one terminal event. -- When a case converts: the stage response equals a direct ``POST /v1/placements`` - on exactly the captured converted bytes, and the decision semantics match the - independently authored wire case's expectations (input hashes differ by - identity stamp, so hashes are never compared across the pair). + bridge refuses; no placement artifact on refusal; a positive record must + solve (``status == done`` plus a real placement artifact — a declared refusal + is still a blocker); a negative case that somehow solves is a blocker, never + a success; the served device result stays conservative (``manual_review``, + HTTP 200, the journey's own run id) with exactly one terminal event. +- When a case converts: the captured converted bytes are compared field by + field against the independently authored wire scene (identity stripped, + floats canonicalized) — covering the tap hazard, units, evidence-derived + coverage and chain ends — and the decision semantics match the wire case's + expectations. Both solver responses' ``stats.input_sha256`` must match their + own exact request bytes (the stage's against the captured converted bytes, + the direct response's against the bytes posted), so forging the same wrong + hash in both replies is caught even though they then agree with each other. + +Aggregate checks: ``crossCapture`` compares the run's two positive cases and +requires distinct capture ids with distinct *normalized* geometry — identity +fields stripped and the rest canonicalized, so different identity stamps or +stamped scene hashes alone never pass. ``finalizeReplay`` must be an idempotent +success on the same run id. A full run (no ``--cases`` selection) requires both +positive cases present and solved; a subset diagnostic run marks the aggregate +checks it cannot make ``not_applicable``; an empty or unknown selection cannot +pass. Every failed check contributes a blocker, and the receipt is ``ok`` only +when there are none. Findings this run pins (the reason the suite exits 1 on the committed fixtures): @@ -66,6 +85,7 @@ import copy import hashlib import json +import math import os import shutil import subprocess @@ -88,7 +108,6 @@ from server.orchestrator.stateio import get_json # noqa: E402 from server.orchestrator.stubs._common import json_bytes, write_files # noqa: E402 from server.placement import adapter, case_suite # noqa: E402 -from server.scene.merge import merge_object_lists # noqa: E402 RECEIPT_SCHEMA_VERSION = "hss-capture-placement-receipt/v0" TERMINAL_EVENTS = {"verdict_ready", "retake_request", "failed"} @@ -433,9 +452,137 @@ def _run_artifacts(harness: _Harness, run_id: str) -> dict[str, str]: return artifacts +# The merge contract, re-derived here without the production +# ``server.scene.merge.merge_object_lists`` helper: incoming objects append, and a +# detector or VLM scene object leaves only when an incoming object takes its id or +# its place (same class, wall_rect on the same wall, or the same centre within the +# merge tolerance). Re-deriving keeps the oracle independent: a coordinator bug and +# an oracle bug can no longer be the same bug. +_MERGE_SAME_PLACE_TOL_M = 0.3 + + +def _object_center_m(obj: dict[str, Any]) -> tuple[float, float, float] | None: + geometry = obj.get("geometry") or {} + if geometry.get("kind") == "point": + x, y, z = (float(v) for v in geometry["pos_m"]) + return x, y, z + if geometry.get("kind") == "wall_rect": + span = geometry.get("s_m") or [0.0, 0.0] + return ( + (float(span[0]) + float(span[1])) / 2, + (float(geometry.get("bottom_m", 0.0)) + float(geometry.get("top_m", 0.0))) / 2, + 0.0, + ) + return None + + +def _same_place(a: dict[str, Any], b: dict[str, Any]) -> bool: + if a.get("class") != b.get("class"): + return False + center_a, center_b = _object_center_m(a), _object_center_m(b) + if center_a is None or center_b is None: + return False + geometry_a, geometry_b = a.get("geometry") or {}, b.get("geometry") or {} + if geometry_a.get("kind") == "wall_rect" and geometry_b.get("kind") == "wall_rect": + if geometry_a.get("wall_id") != geometry_b.get("wall_id"): + return False + return abs(center_a[0] - center_b[0]) <= _MERGE_SAME_PLACE_TOL_M and abs(center_a[1] - center_b[1]) <= ( + _MERGE_SAME_PLACE_TOL_M + ) + return math.dist(center_a, center_b) <= _MERGE_SAME_PLACE_TOL_M + + +def _replaced_independently(prev: dict[str, Any], incoming: list[dict[str, Any]]) -> bool: + kind = (prev.get("source") or {}).get("kind") + if kind not in {"detector", "vlm"}: + return False + for obj in incoming: + if prev.get("id") and prev.get("id") == obj.get("id"): + return True + if _same_place(prev, obj): + return True + return False + + +def _assert_merge_precedence(merged: dict[str, Any], journey: dict[str, Any], mismatches: list[str]) -> None: + """Same-id precedence, asserted explicitly: every injected object appears in the + merged scene exactly once and the merged copy *is* the injected object.""" + entry = _REGISTRY[journey["captureId"]] + merged_objects = merged.get("objects") or [] + for incoming in entry["objects_3d"]: + object_id = incoming.get("id") + matches = [obj for obj in merged_objects if obj.get("id") == object_id] + if len(matches) != 1: + mismatches.append( + f"injected object {object_id!r} appears {len(matches)} times after the merge, not exactly once" + ) + elif matches[0] != incoming: + mismatches.append( + f"injected object {object_id!r} lost same-id precedence: the merged copy is not the injected object" + ) + + +def _canonical(value: Any) -> Any: + if value is None or isinstance(value, (bool, str)): + return value + if isinstance(value, (int, float)): + return round(float(value), 6) + if isinstance(value, list): + return [_canonical(item) for item in value] + if isinstance(value, dict): + return {key: _canonical(item) for key, item in sorted(value.items())} + return value + + +def _normalized_geometry(document: dict[str, Any]) -> dict[str, Any]: + """Identity-stripped, canonicalized geometry. Hashes of normalized geometry are + equal only when the geometry itself is equal: different identity stamps or + different stamped scene hashes prove nothing here.""" + stripped = { + key: value for key, value in document.items() if key not in {"scene_id", "provenance", "schema_version"} + } + return _canonical(stripped) + + +def _geometry_diffs(converted: Any, expected: Any, path: str = "", limit: int = 5) -> list[str]: + """The first structural differences between the converted scene and the + independently authored wire scene, as readable paths.""" + if isinstance(converted, dict) and isinstance(expected, dict): + diffs: list[str] = [] + for key in sorted(set(converted) | set(expected)): + if len(diffs) >= limit: + return diffs + if key not in converted: + diffs.append(f"{path}/{key}: missing from the converted scene") + elif key not in expected: + diffs.append(f"{path}/{key}: not in the wire scene") + else: + diffs.extend(_geometry_diffs(converted[key], expected[key], f"{path}/{key}", limit - len(diffs))) + return diffs + if isinstance(converted, list) and isinstance(expected, list): + diffs = [] + if len(converted) != len(expected): + diffs.append(f"{path}: {len(converted)} entries, wire scene has {len(expected)}") + else: + for index, (a, b) in enumerate(zip(converted, expected, strict=True)): + if len(diffs) >= limit: + return diffs + diffs.extend(_geometry_diffs(a, b, f"{path}/{index}", limit - len(diffs))) + return diffs + if _canonical(converted) != _canonical(expected): + return [f"{path}: {converted!r} != wire {expected!r}"] + return [] + + +# One metre in feet, by definition: every converted length is the source metres +# divided by exactly 0.3048. The unit sentinel re-derives one length end to end. +M_PER_FT = 0.3048 + + def _expected_merged_scene(journey: dict[str, Any]) -> dict[str, Any]: """The merged scene this case must produce, computed independently of the - pipeline: the stamped source plus the injected objects, merged once.""" + pipeline: the stamped source plus the injected objects, merged once under the + contract re-derived above.""" entry = _REGISTRY[journey["captureId"]] stamped = _stamped_scene( entry["record"], @@ -444,7 +591,77 @@ def _expected_merged_scene(journey: dict[str, Any]) -> dict[str, Any]: capture_id=journey["captureId"], run_id=journey["runId"], ) - return merge_object_lists(stamped, copy.deepcopy(entry["objects_3d"])) + incoming = copy.deepcopy(entry["objects_3d"]) + kept = [obj for obj in stamped.get("objects") or [] if not _replaced_independently(obj, incoming)] + expected = dict(stamped) + expected["objects"] = kept + incoming + return expected + + +def _check_converted_geometry( + case: dict[str, Any], + record: case_suite.CaseRecord, + *, + converted_bytes: bytes, + merged_bytes: bytes | None, + mismatches: list[str], +) -> None: + """The captured converted scene against independently derived facts: the unit + sentinel, the preserved chain ends, the tap hazard's once-only conversion, and + a field-by-field comparison with the authored wire scene.""" + converted = json.loads(converted_bytes) + merged = json.loads(merged_bytes) if merged_bytes else {} + + source_height_m = (merged.get("walls") or [{}])[0].get("height_m") + converted_height_ft = (converted.get("walls") or [{}])[0].get("height_ft") + units_ok = ( + source_height_m is not None + and converted_height_ft is not None + and abs(float(converted_height_ft) - float(source_height_m) / M_PER_FT) <= 1e-6 + ) + case["unitsSentinel"] = "matched" if units_ok else "mismatched" + if not units_ok: + mismatches.append("converted wall height is not the merged source height divided by 0.3048 (units sentinel)") + + chain_ends = (merged.get("chain") or {}).get("ends") or {} + converted_ends = (converted.get("coverage") or {}).get("ends") or {} + for side, end in sorted(chain_ends.items()): + converted_kind = (converted_ends.get(side) or {}).get("kind") + if converted_kind != end.get("kind"): + mismatches.append( + f"converted coverage end {side!r} is {converted_kind!r}, not the source chain's {end.get('kind')!r}" + ) + + if record.id == TAP_CASE_ID: + expected_span_ft = [value / M_PER_FT for value in TAP_HAZARD["geometry"]["s_m"]] + taps = [ + obj + for obj in converted.get("objects") or [] + if obj.get("type") == TAP_HAZARD["class"] + and obj.get("source") == TAP_HAZARD["source"]["kind"] + and len(obj.get("span_ft") or []) == 2 + and all(abs(a - b) <= 1e-6 for a, b in zip(obj["span_ft"], expected_span_ft, strict=True)) + ] + case["tapHazardConvertedCount"] = len(taps) + if len(taps) != 1: + mismatches.append( + f"the observed tap hazard appears {len(taps)} times in the converted scene at its expected feet " + "span, not exactly once" + ) + + if record.wire_path is None: + case["geometryComparison"] = "not_applicable" + return + wire = json.loads(record.wire_path.read_bytes()) + normalized = _normalized_geometry(converted) + diffs = _geometry_diffs(normalized, _normalized_geometry(wire)) + case["geometryComparison"] = "matched" if not diffs else "mismatched" + case["geometryDiffs"] = diffs + # Canonical normalized-geometry hash, identity stamps stripped: the + # cross-capture comparison fails same-geometry-different-identity runs. + case["mergedGeometrySha256"] = _sha256(json.dumps(normalized, sort_keys=True).encode()) + if diffs: + mismatches.append("converted scene differs from the independently authored wire scene: " + "; ".join(diffs[:3])) def _decision_bytes(payload: dict[str, Any]) -> str: @@ -513,27 +730,63 @@ def _collect_case(harness: _Harness, record: case_suite.CaseRecord, journey: dic case["hasPlacementArtifact"] = bool(placement_artifact_key) and deps.storage.exists(placement_artifact_key) if stage_status == "done" and case["hasPlacementArtifact"]: + if record.kind != case_suite.KIND_POSITIVE: + case["mismatches"].append( + f"{record.kind!r} case produced a successful placement; only a positive case may solve" + ) case["status"] = "solved" + converted_bytes = deps.storage.get_bytes(converted_key) if converted_key else None placement = json.loads(deps.storage.get_bytes(placement_artifact_key)) - case["convertedSha256"] = _sha256(deps.storage.get_bytes(converted_key)) + case["convertedSha256"] = _sha256(converted_bytes) if converted_bytes else None case["resultSha256"] = _sha256(deps.storage.get_bytes(placement_artifact_key)) case["solverPolicy"] = placement.get("policy") - direct = TestClient(adapter.load_api()["api"].app).post( - "/v1/placements", - content=deps.storage.get_bytes(converted_key), - headers={"Content-Type": "application/json"}, + + # Both solver responses must bind their own input hash to their own exact + # request bytes. Forging the same wrong hash in both replies leaves them in + # agreement with each other and still fails here. + stage_input_sha = (placement.get("stats") or {}).get("input_sha256") + case["stageInputShaMatchesCapturedBytes"] = stage_input_sha == case["convertedSha256"] + if not case["stageInputShaMatchesCapturedBytes"]: + case["mismatches"].append( + "the stage response's stats.input_sha256 does not match the sha256 of the captured converted bytes it solved" + ) + + _check_converted_geometry( + case, + record, + converted_bytes=converted_bytes or b"{}", + merged_bytes=merged_bytes, + mismatches=case["mismatches"], ) - if direct.status_code != 200: - case["errors"].append(f"direct_placements_http_{direct.status_code}") + + if converted_bytes is None: + case["errors"].append("the placement stage solved without publishing placement/scene.json") case["directComparison"] = "error" else: - case["directComparison"] = ( - "matched" if _decision_bytes(direct.json()) == _decision_bytes(placement) else "mismatched" + direct = TestClient(adapter.load_api()["api"].app).post( + "/v1/placements", + content=converted_bytes, + headers={"Content-Type": "application/json"}, ) - if case["directComparison"] == "mismatched": - case["mismatches"].append( - "direct POST /v1/placements on the captured bytes disagrees with the stage response" + if direct.status_code != 200: + case["errors"].append(f"direct_placements_http_{direct.status_code}") + case["directComparison"] = "error" + else: + direct_body = direct.json() + direct_sha = (direct_body.get("stats") or {}).get("input_sha256") + case["directInputShaMatchesPostedBytes"] = direct_sha == case["convertedSha256"] + if not case["directInputShaMatchesPostedBytes"]: + case["mismatches"].append( + "the direct solver response's stats.input_sha256 does not match the sha256 of the exact " + "request bytes" + ) + case["directComparison"] = ( + "matched" if _decision_bytes(direct_body) == _decision_bytes(placement) else "mismatched" ) + if case["directComparison"] == "mismatched": + case["mismatches"].append( + "direct POST /v1/placements on the captured bytes disagrees with the stage response" + ) if record.wire_path is not None: wire_mismatches = case_suite._evaluate_expectations(record.expectations, placement) case["wireSemanticComparison"] = "matched" if not wire_mismatches else "mismatched" @@ -562,10 +815,16 @@ def _collect_case(harness: _Harness, record: case_suite.CaseRecord, journey: dic f"refusal pointer {declared.get('pointer')!r} not found in the refusal message" ) elif record.kind == case_suite.KIND_POSITIVE: - case["mismatches"].append( - "positive case refused conversion, but the manifest declares no expected error for it " - "(the source does not pass the scene bridge as authored)" - ) + if declared: + case["mismatches"].append( + "positive case refused conversion; even with a declared error a positive record must solve " + "and publish a placement artifact" + ) + else: + case["mismatches"].append( + "positive case refused conversion, but the manifest declares no expected error for it " + "(the source does not pass the scene bridge as authored)" + ) case["convertedSha256"] = None case["resultSha256"] = None case["solverPolicy"] = None @@ -577,14 +836,18 @@ def _collect_case(harness: _Harness, record: case_suite.CaseRecord, journey: dic f"placement stage {stage_status!r} with error {stage_error!r}: neither solved nor refused" ) - # The device boundary: conservative outcome, one terminal event. + # The device boundary: conservative outcome, the journey's own identity, one + # terminal event. Criteria and the result stay separately stubbed, so this pins + # the served boundary, never a claim that the decision caused the device state. client = harness.client capture = client.get(f"/v1/captures/{journey['captureId']}").json() served = client.get(f"/v1/captures/{journey['captureId']}/result") + served_body = served.json() if served.status_code == 200 else {} case["deviceOutcome"] = { "captureStatus": capture.get("status"), "servedResultHttpStatus": served.status_code, - "servedResultKind": served.json().get("outcome", {}).get("kind") if served.status_code == 200 else None, + "servedResultKind": (served_body.get("outcome") or {}).get("kind"), + "servedResultRunId": served_body.get("runId"), } terminals = [event for event in journey["events"] if event["type"] in TERMINAL_EVENTS] case["terminalEventCount"] = len(terminals) @@ -594,6 +857,17 @@ def _collect_case(harness: _Harness, record: case_suite.CaseRecord, journey: dic case["mismatches"].append( f"capture status {case['deviceOutcome']['captureStatus']!r} is not the conservative manual_review" ) + if served.status_code != 200: + case["mismatches"].append(f"GET /result returned HTTP {served.status_code}, not 200") + elif case["deviceOutcome"]["servedResultKind"] != "manual_review": + case["mismatches"].append( + f"served result outcome {case['deviceOutcome']['servedResultKind']!r} is not the conservative manual_review" + ) + if case["deviceOutcome"]["servedResultRunId"] != journey.get("runId"): + case["mismatches"].append( + f"served result run id {case['deviceOutcome']['servedResultRunId']!r} does not match this journey's " + f"run id {journey.get('runId')!r}" + ) return case @@ -606,7 +880,10 @@ def run_cases( """Run the manifest's cases through the front door and return the receipt.""" _apply_dev_env(workdir / "vol") workdir.mkdir(parents=True, exist_ok=True) + if case_ids is not None and not case_ids: + raise SystemExit("empty case selection: name at least one case id, or run every case") records = case_suite.load_cases(manifest_path) + full_run = case_ids is None if case_ids: wanted = set(case_ids) records = [record for record in records if record.id in wanted] @@ -631,15 +908,40 @@ def run_cases( journeys.append(_walk_front_door(harness, record, packet_id=f"placement-case-{index + 1:02d}")) harness.drain() cases = [_collect_case(harness, record, journey) for record, journey in zip(records, journeys, strict=True)] - replay = _replay_finalize(harness, journeys[0]) if journeys else None + if journeys and journeys[0].get("captureId") is not None: + replay = _replay_finalize(harness, journeys[0]) + replay["applicable"] = True + elif journeys: + replay = { + "applicable": False, + "status": "not_applicable", + "reason": "the first journey never created a capture", + "ok": None, + } + else: + replay = {"applicable": False, "status": "not_applicable", "reason": "no case ran", "ok": None} finally: _remove_boundaries() - # Cross-capture: distinct captures in one app must carry distinct scenes. - merged = [case.get("mergedSceneSha256") for case in cases[:2]] + # Cross-capture: distinct captures in one app must carry distinct scenes — and + # distinct *normalized* geometry, not merely distinct identity stamps or + # stamped scene hashes. Only the two positive cases are compared. + positive_cases = [case for case in cases if case["kind"] == case_suite.KIND_POSITIVE] + pair = positive_cases[:2] + raw_hashes = [case.get("mergedSceneSha256") for case in pair] + geometry_hashes = [case.get("mergedGeometrySha256") for case in pair] + capture_ids = [case.get("captureId") for case in pair] + applicable = len(pair) == 2 and all(raw_hashes) and all(geometry_hashes) and all(capture_ids) cross_capture = { - "captureIds": [case.get("captureId") for case in cases[:2]], - "distinctScenes": len(merged) == 2 and merged[0] != merged[1], + "applicable": applicable, + "status": "evaluated" if applicable else "not_applicable", + "reason": None + if applicable + else "not_applicable: this run does not include two positive cases that each carried a merged scene", + "captureIds": capture_ids, + "distinctScenes": (raw_hashes[0] != raw_hashes[1]) if applicable else None, + "distinctNormalizedGeometry": (geometry_hashes[0] != geometry_hashes[1]) if applicable else None, + "mergedGeometrySha256": dict(zip([case["id"] for case in pair], geometry_hashes, strict=False)) if pair else {}, } blockers = [] @@ -658,6 +960,31 @@ def run_cases( blockers.append(f"{case['id']}: " + "; ".join(case["mismatches"])) if case["errors"]: blockers.append(f"{case['id']}: " + "; ".join(case["errors"])) + if cross_capture["applicable"]: + if not ( + cross_capture["distinctScenes"] + and cross_capture["distinctNormalizedGeometry"] + and len(set(cross_capture["captureIds"])) == 2 + ): + blockers.append( + "crossCapture: the two positive cases do not carry distinct capture ids with distinct normalized " + f"geometry (distinctScenes={cross_capture['distinctScenes']}, " + f"distinctNormalizedGeometry={cross_capture['distinctNormalizedGeometry']})" + ) + if replay is not None and replay.get("applicable", True) and replay.get("ok") is not True: + blockers.append( + "finalizeReplay: the finalize replay is not an idempotent success (HTTP 202, same run id, one terminal " + f"event); got {json.dumps(replay)}" + ) + if full_run: + if len(positive_cases) < 2: + blockers.append(f"full run requires both positive cases; this manifest ran {len(positive_cases)}") + unsolved = [case["id"] for case in positive_cases if case["status"] != "solved"] + if unsolved: + blockers.append( + "full run requires both positive cases to solve and publish placement artifacts; not solved: " + + ", ".join(unsolved) + ) statuses = [case["status"] for case in cases] receipt: dict[str, Any] = { diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index 2d109b024..28760438c 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -1,41 +1,21 @@ -"""Focused tests for the capture placement case runner. - -The runner (`scripts/capture_placement_cases.py`) replays every hash-pinned case -in `server/placement/tests/data/cases/manifest.json` through the real capture -front door — create, register, signed upload, commit, finalize, events, result — -over the documented dev settings (SD-73), injecting each case's synthetic source -scene **only** at the upstream `scene` stage boundary and an explicit object list -at the `objects_3d` boundary, with the actual capture/run/packet identity stamped -in the contract's identity fields. - -These tests pin the three claims that make the runner worth trusting: - -1. The journey evidence: the coordinator's merged scene equals the case's - stamped source merged once with the injected objects (never a reference - scene, another capture's scene, or a second merge); the observed tap hazard - lands exactly once; refusals are typed, leave no placement artifact, and end - in the conservative device outcome with exactly one terminal event; two - captures in one app carry distinct scenes; a finalize replay is idempotent. -2. The suite fails loudly on tampering: substituting the old default stub's - reference scene at the boundary, or removing the real placement stage, both - produce mismatches and a nonzero exit — the boundary checks are load-bearing, - not decorative. -3. The committed fixtures' pinned state (2026-10-15): both positive cases refuse - conversion at `/objects/0/geometry/standoff_m` (a stand-off meter anchor the - bridge does not synthesise, then the footprint hazard kind) with no expected - error declared in the manifest, and `generated-hazard-unsupported` refuses at - `/generated_regions` rather than its declared `/objects/1/evidence`. The - suite reports these as blockers and exits nonzero: a producer-contract - mismatch, recorded here as evidence rather than papered over. +"""Acceptance tests for the capture placement replay runner. + +The committed manifest's positive sources refuse conversion (the pinned blocker), +so the success path is exercised through a temporary manifest whose positive +sources, wire scenes, and expectations are authored by hand in this module — the +same geometry in metres and in feet, never produced by running the converter. """ from __future__ import annotations +import hashlib import importlib import json +import subprocess import sys from pathlib import Path +import httpx import pytest BACKEND = Path(__file__).resolve().parents[1] @@ -52,6 +32,18 @@ POSITIVE_IDS = ("hazard-right-ground-gap", "hazard-left-ground-gap") NEGATIVE_IDS = ("multiwall-source-unsupported", "generated-hazard-unsupported") +M_PER_FT = 0.3048 + + +def _ft(metres: float) -> float: + """Feet for a source metres value — the conversion the bridge must apply, + used here to author the wire scenes by hand, not by running the converter.""" + return metres / M_PER_FT + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + @pytest.fixture def receipt(tmp_path, monkeypatch): @@ -64,6 +56,507 @@ def _case(receipt: dict, case_id: str) -> dict: return next(case for case in receipt["cases"] if case["id"] == case_id) +# --------------------------------------------------------------------------- +# Hand-authored supported sources, wire scenes, and pinned expectations +# --------------------------------------------------------------------------- + + +def _supported_source_a() -> dict: + """A hand-authored positive source: one wall, a meter anchor point, a window, + a tap gas hazard, an observed ground pad, overhead, facing gap, coverage. + The solver has plausible spots but unsure evidence forces manual review.""" + return { + "format": "base-scene/v0", + "scene_id": "burst8-supported-a", + "provenance": { + "data": "synthetic", + "site": "burst8-supported-a", + "producer": {"path": "capture-placement-cases-test"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "meter", "plus_minus_m": 0.01}, + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [[-3.0, 0.0], [5.0, 0.0]], + "height_m": 3.0, + "plus_minus_m": 0.05, + "evidence": "observed", + } + ], + "chain": { + "wall_ids": ["side"], + "ends": {"left": {"kind": "unexplored"}, "right": {"kind": "limit", "note": "fence line"}}, + }, + "ground": { + "surface": {"kind": "plane", "normal": [0, 1, 0], "offset_m": 0}, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "pad", + "material": "concrete", + "polygon_m": [[0.2, 0.5], [2.0, 0.5], [2.0, 3.0], [0.2, 3.0]], + "evidence": "observed", + "plus_minus_m": 0.1, + } + ], + }, + "objects": [ + { + "id": "meter", + "class": "meter", + "geometry": {"kind": "point", "pos_m": [0.0, 1.5, 0.02]}, + "confidence": 1, + "plus_minus_m": 0.02, + "source": {"kind": "reconstruction"}, + "evidence": "observed", + }, + { + "id": "win", + "class": "window", + "geometry": {"kind": "wall_rect", "wall_id": "side", "s_m": [1.0, 2.0], "bottom_m": 1.0, "top_m": 2.2}, + "confidence": 0.9, + "plus_minus_m": 0.08, + "source": {"kind": "vlm"}, + "evidence": "observed", + }, + { + "id": "gas", + "class": "gas_meter", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [-0.6, -0.3], + "bottom_m": 0.3, + "top_m": 0.9, + }, + "confidence": 0.95, + "plus_minus_m": 0.05, + "source": {"kind": "tap"}, + "evidence": "observed", + "attrs": {"operable": True}, + }, + ], + "coverage": { + "method": "capture-placement-cases-test", + "sensitivity": [ + { + "surface": "wall", + "max_incidence_deg": 75, + "bands": [{"band": "wall", "s_m": [-2.0, 4.0], "out_m": 2.13}], + } + ], + }, + "overheads": [ + {"wall_id": "side", "s_m": [1.0, 2.0], "clearance_m": 2.1, "plus_minus_m": 0.1, "evidence": "observed"} + ], + "facing": [ + {"wall_id": "side", "s_m": [-1.0, 1.0], "depth_m": 8.0, "plus_minus_m": 0.3, "evidence": "observed"} + ], + } + + +def _supported_source_b() -> dict: + """A second, mirrored positive source: the gas hazard right of the meter, the + window left, observed ground on the left, the left end a real limit and the + right end unexplored — the opposite of source A on every one of those.""" + return { + "format": "base-scene/v0", + "scene_id": "burst8-supported-b", + "provenance": { + "data": "synthetic", + "site": "burst8-supported-b", + "producer": {"path": "capture-placement-cases-test"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "meter", "plus_minus_m": 0.01}, + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [[-5.0, 0.0], [3.0, 0.0]], + "height_m": 2.8, + "plus_minus_m": 0.06, + "evidence": "observed", + } + ], + "chain": { + "wall_ids": ["side"], + "ends": {"left": {"kind": "limit", "note": "hedge line"}, "right": {"kind": "unexplored"}}, + }, + "ground": { + "surface": {"kind": "plane", "normal": [0, 1, 0], "offset_m": 0}, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "pad", + "material": "gravel", + "polygon_m": [[-2.0, 0.5], [-0.6, 0.5], [-0.6, 3.0], [-2.0, 3.0]], + "evidence": "observed", + "plus_minus_m": 0.08, + } + ], + }, + "objects": [ + { + "id": "meter", + "class": "meter", + "geometry": {"kind": "point", "pos_m": [0.0, 1.4, 0.03]}, + "confidence": 1, + "plus_minus_m": 0.03, + "source": {"kind": "reconstruction"}, + "evidence": "observed", + }, + { + "id": "win", + "class": "window", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [-2.4, -1.4], + "bottom_m": 1.0, + "top_m": 2.0, + }, + "confidence": 0.9, + "plus_minus_m": 0.08, + "source": {"kind": "vlm"}, + "evidence": "observed", + }, + { + "id": "gas", + "class": "gas_meter", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [0.8, 1.3], + "bottom_m": 0.25, + "top_m": 0.85, + }, + "confidence": 0.95, + "plus_minus_m": 0.05, + "source": {"kind": "tap"}, + "evidence": "observed", + }, + ], + "coverage": { + "method": "capture-placement-cases-test", + "sensitivity": [ + { + "surface": "wall", + "max_incidence_deg": 75, + "bands": [{"band": "wall", "s_m": [-4.0, 2.0], "out_m": 2.5}], + } + ], + }, + "overheads": [ + {"wall_id": "side", "s_m": [-2.4, -1.4], "clearance_m": 2.0, "plus_minus_m": 0.12, "evidence": "observed"} + ], + "facing": [ + {"wall_id": "side", "s_m": [-1.5, 0.5], "depth_m": 7.0, "plus_minus_m": 0.25, "evidence": "observed"} + ], + } + + +def _wire_a(*, with_tap: bool = True) -> dict: + """The wire scene a converter must produce for source A, authored from the + source geometry with metres divided by 0.3048. The observed tap hazard is + injected by the runner for the reserved positive id, so the wire scene + includes it as a third object unless a caller excludes it.""" + wire = { + "meter": {"pos": [0.0, _ft(1.5), _ft(0.02)], "wall_id": "side", "plus_minus_ft": _ft(0.02)}, + "walls": [ + { + "id": "side", + "baseline": [[_ft(-3.0), 0.0], [_ft(5.0), 0.0]], + "height_ft": _ft(3.0), + "plus_minus_ft": _ft(0.05), + } + ], + "objects": [ + { + "type": "window", + "wall_id": "side", + "span_ft": [_ft(1.0), _ft(2.0)], + "bottom_ft": _ft(1.0), + "top_ft": _ft(2.2), + "conf": 0.9, + "plus_minus_ft": _ft(0.08), + "source": "vlm", + }, + { + "type": "gas_meter", + "wall_id": "side", + "span_ft": [_ft(-0.6), _ft(-0.3)], + "bottom_ft": _ft(0.3), + "top_ft": _ft(0.9), + "conf": 0.95, + "plus_minus_ft": _ft(0.05), + "source": "tap", + "attrs": {"operable": True}, + }, + ], + "ground": [ + { + "type": "concrete", + "polygon": [ + [_ft(0.2), _ft(0.5)], + [_ft(2.0), _ft(0.5)], + [_ft(2.0), _ft(3.0)], + [_ft(0.2), _ft(3.0)], + ], + "plus_minus_ft": _ft(0.1), + } + ], + "overheads": [ + {"wall_id": "side", "span_ft": [_ft(1.0), _ft(2.0)], "clearance_ft": _ft(2.1), "plus_minus_ft": _ft(0.1)} + ], + "facing": [ + {"wall_id": "side", "span_ft": [_ft(-1.0), _ft(1.0)], "depth_ft": _ft(8.0), "plus_minus_ft": _ft(0.3)} + ], + "coverage": { + "ends": {"left": {"kind": "unexplored"}, "right": {"kind": "limit", "note": "fence line"}}, + "observed": [{"band": "wall", "span_ft": [_ft(-2.0), _ft(4.0)], "out_ft": _ft(2.13)}], + }, + } + if with_tap: + wire["objects"].append( + { + "type": "vent", + "wall_id": "side", + "span_ft": [_ft(1.6), _ft(1.8)], + "bottom_ft": _ft(0.5), + "top_ft": _ft(2.0), + "conf": 0.9, + "plus_minus_ft": _ft(0.05), + "source": "tap", + } + ) + return wire + + +def _wire_b() -> dict: + """The wire scene for source B, authored the same way — from the metres.""" + return { + "meter": {"pos": [0.0, _ft(1.4), _ft(0.03)], "wall_id": "side", "plus_minus_ft": _ft(0.03)}, + "walls": [ + { + "id": "side", + "baseline": [[_ft(-5.0), 0.0], [_ft(3.0), 0.0]], + "height_ft": _ft(2.8), + "plus_minus_ft": _ft(0.06), + } + ], + "objects": [ + { + "type": "window", + "wall_id": "side", + "span_ft": [_ft(-2.4), _ft(-1.4)], + "bottom_ft": _ft(1.0), + "top_ft": _ft(2.0), + "conf": 0.9, + "plus_minus_ft": _ft(0.08), + "source": "vlm", + }, + { + "type": "gas_meter", + "wall_id": "side", + "span_ft": [_ft(0.8), _ft(1.3)], + "bottom_ft": _ft(0.25), + "top_ft": _ft(0.85), + "conf": 0.95, + "plus_minus_ft": _ft(0.05), + "source": "tap", + }, + ], + "ground": [ + { + "type": "gravel", + "polygon": [ + [_ft(-2.0), _ft(0.5)], + [_ft(-0.6), _ft(0.5)], + [_ft(-0.6), _ft(3.0)], + [_ft(-2.0), _ft(3.0)], + ], + "plus_minus_ft": _ft(0.08), + } + ], + "overheads": [ + {"wall_id": "side", "span_ft": [_ft(-2.4), _ft(-1.4)], "clearance_ft": _ft(2.0), "plus_minus_ft": _ft(0.12)} + ], + "facing": [ + {"wall_id": "side", "span_ft": [_ft(-1.5), _ft(0.5)], "depth_ft": _ft(7.0), "plus_minus_ft": _ft(0.25)} + ], + "coverage": { + "ends": {"left": {"kind": "limit", "note": "hedge line"}, "right": {"kind": "unexplored"}}, + "observed": [{"band": "wall", "span_ft": [_ft(-4.0), _ft(2.0)], "out_ft": _ft(2.5)}], + }, + } + + +def _expected_a() -> dict: + """Geometry facts, not policy: pinned from the solver's deterministic output + on exactly the merged scene this manifest produces (tap hazard included).""" + return { + "decision": "manual_review", + "spot": { + "wall_id": "side", + "segment": 0, + "span_ft": [9.56168, 12.145013], + "outcome": "unsure", + }, + "checks": { + "gas_clearance": {"outcome": "unsure", "unsure_cause": "unobserved"}, + "ground_surface": {"outcome": "unsure", "unsure_cause": "unobserved"}, + "opening_clearance": {"outcome": "unsure", "unsure_cause": "margin"}, + "wall_equipment_above": {"outcome": "pass", "unsure_cause": None}, + "route_length": {"outcome": "pass", "unsure_cause": None}, + }, + "missing_evidence": [ + {"kind": "band", "band": "ground", "span_ft": [-0.614621, 22.321313], "out_ft": 12.009634}, + {"kind": "past_end", "side": "left", "span_ft": [-9.84252, -9.84252]}, + ], + "ends": {"left": "unexplored", "right": "limit"}, + } + + +def _expected_b() -> dict: + return { + "decision": "manual_review", + "spot": { + "wall_id": "side", + "segment": 0, + "span_ft": [-13.457349, -10.874016], + "outcome": "unsure", + }, + "checks": { + "ground_surface": {"outcome": "unsure", "unsure_cause": "unobserved"}, + "gas_clearance": {"outcome": "unsure", "unsure_cause": "unobserved"}, + "opening_clearance": {"outcome": "unsure", "unsure_cause": "margin"}, + "meter_working_space": {"outcome": "pass", "unsure_cause": None}, + "wall_equipment_above": {"outcome": "unsure", "unsure_cause": "unobserved"}, + "route_length": {"outcome": "pass", "unsure_cause": None}, + }, + "missing_evidence": [ + {"kind": "band", "band": "ground", "span_ft": [-23.666497, -0.664867], "out_ft": 12.042482}, + {"kind": "past_end", "side": "right", "span_ft": [9.84252, 9.84252]}, + ], + "ends": {"left": "limit", "right": "unexplored"}, + } + + +def _write_supported_manifest(tmp_path: Path) -> Path: + """A temp manifest: two hand-authored positive cases (source, wire scene and + expectations written and sha-pinned like the committed fixtures) plus one + committed negative case referenced by absolute path. This manifest is the + one command's success path: both positives must solve and publish.""" + fixtures = tmp_path / "fixtures" + fixtures.mkdir(parents=True) + cases = [] + for case_id, source, wire, expected in ( + (runner.TAP_CASE_ID, _supported_source_a(), _wire_a(), _expected_a()), + ("hazard-left-wall-gap", _supported_source_b(), _wire_b(), _expected_b()), + ): + source_path = fixtures / f"{case_id}.source.json" + source_path.write_text(json.dumps(source, indent=1) + "\n") + wire_path = fixtures / f"{case_id}.wire.json" + wire_path.write_text(json.dumps(wire, indent=1, sort_keys=True) + "\n") + cases.append( + { + "id": case_id, + "kind": "positive", + "source": {"path": str(source_path), "sha256": _sha256_file(source_path)}, + "wire": {"path": str(wire_path), "sha256": _sha256_file(wire_path)}, + "expectations": expected, + "note": "Hand-authored pair: the same geometry in metres and in feet.", + } + ) + cases.append( + { + "id": "multiwall-source-unsupported", + "kind": "negative_source", + "source": { + "path": str(BACKEND / "server/placement/tests/data/cases/multiwall-source-unsupported.source.json"), + "sha256": _sha256_file( + BACKEND / "server/placement/tests/data/cases/multiwall-source-unsupported.source.json" + ), + }, + "wire": None, + "expected_error": {"code": "placement_scene_unsupported", "pointer": "/walls"}, + } + ) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "schema_version": "placement-case-manifest/v0", + "description": ["Temporary supported-source manifest for the runner acceptance tests."], + "cases": cases, + }, + indent=1, + ) + + "\n" + ) + return manifest_path + + +def _write_twin_manifest(tmp_path: Path) -> Path: + """Two positive cases sharing one source and one wire scene byte for byte: + different identity stamps only. Cross-capture must fail them on geometry.""" + fixtures = tmp_path / "fixtures" + fixtures.mkdir(parents=True) + source_path = fixtures / "twin.source.json" + source_path.write_text(json.dumps(_supported_source_a(), indent=1) + "\n") + wire_path = fixtures / "twin.wire.json" + wire_path.write_text(json.dumps(_wire_a(with_tap=False), indent=1, sort_keys=True) + "\n") + source_ref = {"path": str(source_path), "sha256": _sha256_file(source_path)} + wire_ref = {"path": str(wire_path), "sha256": _sha256_file(wire_path)} + cases = [ + { + "id": "twin-one", + "kind": "positive", + "source": source_ref, + "wire": wire_ref, + "expectations": {"decision": "manual_review"}, + }, + { + "id": "twin-two", + "kind": "positive", + "source": source_ref, + "wire": wire_ref, + "expectations": {"decision": "manual_review"}, + }, + ] + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "schema_version": "placement-case-manifest/v0", + "description": ["Twin-identity manifest: one geometry, two identities."], + "cases": cases, + }, + indent=1, + ) + + "\n" + ) + return manifest_path + + +# --------------------------------------------------------------------------- +# The committed manifest: refusal state, pinned and loud +# --------------------------------------------------------------------------- + + def test_every_case_walks_the_real_front_door_with_case_true_boundaries(receipt): """Each case's journey: injected boundaries hold, refusal is typed and leaves no placement artifact, the device stays conservative, one terminal event.""" @@ -82,22 +575,31 @@ def test_every_case_walks_the_real_front_door_with_case_true_boundaries(receipt) assert case["hasPlacementArtifact"] is False, "a refused placement must not publish an artifact" assert case["status"] == "refused" assert case["refusal"]["code"] == "placement_scene_unsupported" - assert case["deviceOutcome"] == { - "captureStatus": "manual_review", - "servedResultHttpStatus": 200, - "servedResultKind": "manual_review", - } + assert case["deviceOutcome"]["captureStatus"] == "manual_review" + assert case["deviceOutcome"]["servedResultHttpStatus"] == 200 + assert case["deviceOutcome"]["servedResultKind"] == "manual_review" + assert case["deviceOutcome"]["servedResultRunId"] == case["runId"], ( + "the served result must carry this journey's own run id" + ) assert case["terminalEventCount"] == 1 # The one nonempty objects_3d case: the observed tap hazard, merged exactly once. assert _case(receipt, runner.TAP_CASE_ID)["tapHazardMergeCount"] == 1 - # Two distinct captures in one app: no cross-capture scene reuse. - assert receipt["crossCapture"]["distinctScenes"] is True + # Both positives refused: no merged scenes, so the cross-capture comparison + # cannot run and must say so instead of passing vacuously. + cross = receipt["crossCapture"] + assert cross["status"] == "not_applicable" + assert cross["distinctScenes"] is None and cross["distinctNormalizedGeometry"] is None + assert cross["captureIds"] == [ + _case(receipt, "hazard-right-ground-gap")["captureId"], + _case(receipt, "hazard-left-ground-gap")["captureId"], + ] # A finalize replay is idempotent: same run id, still one terminal event. assert receipt["finalizeReplay"]["ok"] is True assert receipt["finalizeReplay"]["terminalEventCountAfter"] == 1 + assert receipt["finalizeReplay"]["applicable"] is True # The receipt never carries the token or any signed upload capability. assert runner.TOKEN not in json.dumps(receipt) @@ -121,7 +623,9 @@ def test_declared_refusals_match_their_declarations(receipt): def test_committed_positive_sources_do_not_convert(receipt): """The positive pairs' sources are refused by the bridge as authored — the - pinned blocker this suite reports instead of papering over.""" + pinned blocker this suite reports instead of papering over. A declared or + undeclared positive refusal is a blocker either way: only a positive case + that solves counts as success.""" for case_id in POSITIVE_IDS: refusal = _case(receipt, case_id)["refusal"] assert refusal["declared"] is False @@ -129,7 +633,9 @@ def test_committed_positive_sources_do_not_convert(receipt): "the meter anchor's stand-off is the first refusal the bridge raises" ) assert _case(receipt, case_id)["mismatches"], "an undeclared positive refusal must be a suite blocker" - assert receipt["blockers"], "the fixture state above must make the suite fail loudly" + blockers = "\n".join(receipt["blockers"]) + assert "full run requires both positive cases" in blockers + assert "hazard-right-ground-gap" in blockers and "hazard-left-ground-gap" in blockers assert receipt["ok"] is False @@ -202,3 +708,240 @@ def test_cli_runs_from_a_temporary_directory_and_exits_nonzero_while_blocked(tmp assert runner.TOKEN not in output.read_text() blockers = "\n".join(receipt["blockers"]) assert "standoff_m" in blockers and "/objects/1/evidence" in blockers + + +# --------------------------------------------------------------------------- +# The supported manifest: the success path and the faults that break it +# --------------------------------------------------------------------------- + + +def test_supported_sources_solve_end_to_end(tmp_path): + """Hand-authored positive sources convert, solve, and publish: the converted + bytes match the wire scene field by field, both solver responses bind their + input hash to their own exact request bytes, the semantics match the pinned + expectations, the device boundary stays conservative on the journey's own run + id, and the two positives carry distinct normalized geometry.""" + manifest = _write_supported_manifest(tmp_path) + receipt = runner.run_cases(manifest, tmp_path / "run") + assert receipt["ok"] is True, receipt["blockers"] + + positives = [case for case in receipt["cases"] if case["kind"] == "positive"] + assert len(positives) == 2 + for case in positives: + assert case["status"] == "solved" + assert case["sceneBoundaryMatchesCase"] is True + assert case["hasPlacementArtifact"] is True + assert case["stageInputShaMatchesCapturedBytes"] is True + assert case["directInputShaMatchesPostedBytes"] is True + assert case["directComparison"] == "matched" + assert case["wireSemanticComparison"] == "matched", case["wireSemanticMismatches"] + assert case["geometryComparison"] == "matched", case.get("geometryDiffs") + assert case["unitsSentinel"] == "matched" + assert case["deviceOutcome"]["servedResultHttpStatus"] == 200 + assert case["deviceOutcome"]["servedResultKind"] == "manual_review" + assert case["deviceOutcome"]["servedResultRunId"] == case["runId"] + assert case["terminalEventCount"] == 1 + assert case["mismatches"] == [] + + # The tap hazard converted exactly once, at its expected feet span. + assert _case(receipt, runner.TAP_CASE_ID)["tapHazardConvertedCount"] == 1 + + cross = receipt["crossCapture"] + assert cross["status"] == "evaluated" + assert cross["distinctScenes"] is True + assert cross["distinctNormalizedGeometry"] is True + assert len(set(cross["captureIds"])) == 2 + assert receipt["finalizeReplay"]["ok"] is True + + negative = _case(receipt, "multiwall-source-unsupported") + assert negative["status"] == "refused" and negative["mismatches"] == [] + + +def test_cli_on_the_supported_manifest_exits_zero(tmp_path): + """The one-command entry point on a manifest that can actually succeed.""" + manifest = _write_supported_manifest(tmp_path) + output = tmp_path / "receipt.json" + proc = subprocess.run( + [ + sys.executable, + str(BACKEND / "scripts" / "capture_placement_cases.py"), + "--manifest", + str(manifest), + "--output", + str(output), + "--workdir", + str(tmp_path / "run"), + ], + capture_output=True, + text=True, + cwd=tmp_path, + timeout=600, + ) + assert proc.returncode == 0, proc.stderr[-2000:] + receipt = json.loads(output.read_text()) + assert receipt["ok"] is True + assert receipt["crossCapture"]["distinctNormalizedGeometry"] is True + + +def test_cli_on_the_committed_manifest_exits_nonzero_in_a_subprocess(tmp_path): + """The failure path through the real argv/exit-code boundary too.""" + output = tmp_path / "receipt.json" + proc = subprocess.run( + [ + sys.executable, + str(BACKEND / "scripts" / "capture_placement_cases.py"), + "--output", + str(output), + "--workdir", + str(tmp_path / "run"), + ], + capture_output=True, + text=True, + cwd=tmp_path, + timeout=600, + ) + assert proc.returncode == 1 + assert "standoff_m" in proc.stderr + + +def test_case_selection_cannot_pass_silently(tmp_path): + """An empty selection and an unknown selection both refuse to run.""" + manifest = _write_supported_manifest(tmp_path) + with pytest.raises(SystemExit): + runner.run_cases(manifest, tmp_path / "run", case_ids=[]) + with pytest.raises(SystemExit): + runner.run_cases(manifest, tmp_path / "run", case_ids=["no-such-case"]) + + +def test_corrupting_the_converted_scene_fails_the_suite(tmp_path, monkeypatch): + """A converter that silently moves the gas hazard is caught by the + field-by-field comparison against the hand-authored wire scene — the staged + response and the direct POST both agree with each other and both are wrong.""" + + placement_adapter = importlib.import_module("server.placement.adapter") + real_convert = placement_adapter.convert_scene + + def corrupting_convert(document): + converted = real_convert(document) + for obj in converted.get("objects", []): + if obj.get("type") == "gas_meter": + obj["span_ft"] = [obj["span_ft"][0] + 0.5, obj["span_ft"][1] + 0.5] + return converted + + monkeypatch.setattr(placement_adapter, "convert_scene", corrupting_convert) + receipt = runner.run_cases(_write_supported_manifest(tmp_path), tmp_path / "run", case_ids=[runner.TAP_CASE_ID]) + case = receipt["cases"][0] + assert case["status"] == "solved" + assert case["geometryComparison"] == "mismatched" + assert any("wire scene" in mismatch for mismatch in case["mismatches"]) + assert receipt["ok"] is False + + +def test_forging_the_solver_input_hash_fails_the_suite(tmp_path, monkeypatch): + """A solver that stamps the same forged hash into both replies keeps them in + agreement with each other — and fails both hash bindings against the exact + request bytes.""" + + placement_adapter = importlib.import_module("server.placement.adapter") + modules = placement_adapter.load_api() + solver, api = modules["solver"], modules["api"] + real_solve = solver.solve + + def forging_solve(parsed, rules): + result = real_solve(parsed, rules) + result["stats"]["input_sha256"] = "f" * 64 + return result + + # The stage resolves solver.solve dynamically; the API route binds the name + # at import (`from solver import solve`). Patch both so both replies carry + # the same forged hash and still agree with each other. + monkeypatch.setattr(solver, "solve", forging_solve) + monkeypatch.setattr(api, "solve", forging_solve) + receipt = runner.run_cases(_write_supported_manifest(tmp_path), tmp_path / "run", case_ids=[runner.TAP_CASE_ID]) + case = receipt["cases"][0] + assert case["status"] == "solved" + assert case["stageInputShaMatchesCapturedBytes"] is False + assert case["directInputShaMatchesPostedBytes"] is False + assert case["directComparison"] == "matched", "the two forged responses must still agree with each other" + assert any("input_sha256" in mismatch for mismatch in case["mismatches"]) + assert receipt["ok"] is False + + +@pytest.mark.parametrize("mutation,field", [("kind", "servedResultKind"), ("runId", "servedResultRunId")]) +def test_a_served_result_that_lies_fails_the_suite(tmp_path, monkeypatch, mutation, field): + """The device boundary is checked against what is actually served: a result + endpoint that flips the outcome kind or the run id fails the suite.""" + + real_client = runner.TestClient + + class LyingClient(real_client): + def get(self, url, **kwargs): + response = super().get(url, **kwargs) + if str(url).endswith("/result") and response.status_code == 200: + body = response.json() + if mutation == "kind": + body["outcome"] = {"kind": "proposed", "reason_codes": ["forged"]} + else: + body["runId"] = "run_forged" + return httpx.Response(200, json=body) + return response + + monkeypatch.setattr(runner, "TestClient", LyingClient) + receipt = runner.run_cases(_write_supported_manifest(tmp_path), tmp_path / "run", case_ids=[runner.TAP_CASE_ID]) + case = receipt["cases"][0] + assert case["deviceOutcome"][field] == ("proposed" if mutation == "kind" else "run_forged") + assert case["mismatches"], "a lying served result must be a case mismatch" + assert receipt["ok"] is False + + +def test_a_failed_finalize_replay_blocks_the_suite(tmp_path, monkeypatch): + """A finalize replay that is no longer idempotent is an aggregate blocker.""" + + def failed_replay(harness, journey): + return { + "caseId": journey["caseId"], + "replayHttpStatus": 202, + "sameRunId": False, + "terminalEventCountAfter": 2, + "ok": False, + "applicable": True, + } + + monkeypatch.setattr(runner, "_replay_finalize", failed_replay) + receipt = runner.run_cases( + _write_supported_manifest(tmp_path), tmp_path / "run", case_ids=["multiwall-source-unsupported"] + ) + assert receipt["finalizeReplay"]["ok"] is False + assert any("finalizeReplay" in blocker for blocker in receipt["blockers"]) + assert receipt["ok"] is False + + +def test_one_geometry_under_two_identities_fails_cross_capture(tmp_path): + """The same source and wire under two case ids produce different identity + stamps (distinct raw merged-scene hashes) but identical geometry — the + cross-capture check must fail on normalized geometry, not pass on stamps.""" + receipt = runner.run_cases(_write_twin_manifest(tmp_path), tmp_path / "run") + cross = receipt["crossCapture"] + assert cross["status"] == "evaluated" + assert cross["distinctScenes"] is True, "identity stamps alone differ" + assert cross["distinctNormalizedGeometry"] is False, "the geometry is byte-identical" + assert any("crossCapture" in blocker for blocker in receipt["blockers"]) + assert receipt["ok"] is False + + +def test_a_declared_negative_that_solves_fails_the_suite(tmp_path, monkeypatch): + """If the converter someday accepts a declared negative's source, the case + must fail loudly — a negative case that solves is never a success.""" + + placement_adapter = importlib.import_module("server.placement.adapter") + real_convert = placement_adapter.convert_scene + valid_wire = real_convert(_supported_source_a()) + + monkeypatch.setattr(placement_adapter, "convert_scene", lambda document: json.loads(json.dumps(valid_wire))) + receipt = runner.run_cases( + _write_supported_manifest(tmp_path), tmp_path / "run", case_ids=["multiwall-source-unsupported"] + ) + case = receipt["cases"][0] + assert case["status"] == "solved" + assert any("produced a successful placement" in mismatch for mismatch in case["mismatches"]) + assert receipt["ok"] is False From a73c5c082375b6bd191e4f32f97d0b76118b2261 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:09:57 +0000 Subject: [PATCH 03/19] Integrate burst8-bs-supported-case-pairs packet (producer e7ad2abc, start bdfbb3fc) --- .../footprint-hazard-unsupported.source.json | 234 ++++++++++++++++ ...ed-object-evidence-unsupported.source.json | 183 +++++++++++++ .../generated-regions-unsupported.source.json | 78 ++++++ .../hazard-left-ground-gap.source.json | 233 ++++++++++++++++ .../hazard-left-ground-gap.wire.json | 153 +++++++++++ .../hazard-right-ground-gap.source.json | 255 +++++++++++++++++ .../hazard-right-ground-gap.wire.json | 166 +++++++++++ .../tests/data/capture_cases_v1/manifest.json | 257 ++++++++++++++++++ .../multiwall-source-unsupported.source.json | 201 ++++++++++++++ ...ndoff-meter-anchor-unsupported.source.json | 91 +++++++ backend/tests/test_capture_case_pairs.py | 231 ++++++++++++++++ 11 files changed, 2082 insertions(+) create mode 100644 backend/server/placement/tests/data/capture_cases_v1/footprint-hazard-unsupported.source.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/generated-object-evidence-unsupported.source.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/generated-regions-unsupported.source.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.source.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.wire.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.source.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.wire.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/manifest.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json create mode 100644 backend/server/placement/tests/data/capture_cases_v1/standoff-meter-anchor-unsupported.source.json create mode 100644 backend/tests/test_capture_case_pairs.py diff --git a/backend/server/placement/tests/data/capture_cases_v1/footprint-hazard-unsupported.source.json b/backend/server/placement/tests/data/capture_cases_v1/footprint-hazard-unsupported.source.json new file mode 100644 index 000000000..eca8d1dd1 --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/footprint-hazard-unsupported.source.json @@ -0,0 +1,234 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst7-hazard-right-ground-gap", + "provenance": { + "data": "synthetic", + "site": "burst7-hazard-right-ground-gap", + "producer": { + "path": "hand-authored", + "version": "burst7" + }, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "notes": [ + "Authored for the burst7 placement case suite. It pairs with hazard-right-ground-gap.wire.json by construction (the same wall, hazard, coverage gap and ends, described once in meters and once in feet by hand), not by running any converter." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": { + "element_id": "obj-meter", + "plus_minus_m": 0.09 + } + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [ + [ + -0.6096, + 0.0 + ], + [ + 6.096, + 0.0 + ] + ], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": [ + "side" + ], + "ends": { + "left": { + "kind": "limit", + "s_m": -0.6096, + "note": "fence at the left property line" + }, + "right": { + "kind": "unexplored", + "s_m": 6.096, + "note": "wall continues right, not walked" + } + } + }, + "ground": { + "surface": { + "kind": "plane", + "normal": [ + 0.0, + 1.0, + 0.0 + ], + "offset_m": 0.0 + }, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [ + [ + -0.6096, + 0.0 + ], + [ + 6.096, + 0.0 + ], + [ + 6.096, + 1.2192 + ], + [ + -0.6096, + 1.2192 + ] + ], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [ + [ + -0.6096, + 1.2192 + ], + [ + 6.096, + 1.2192 + ], + [ + 6.096, + 4.2672 + ], + [ + -0.6096, + 4.2672 + ] + ], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + -0.15, + 0.15 + ], + "bottom_m": 0.15, + "top_m": 1.2 + }, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + }, + { + "id": "obj-gas", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "footprint", + "polygon_m": [ + [ + 1.0668, + 0.0 + ], + [ + 1.3716, + 0.0 + ], + [ + 1.3716, + 0.3048 + ], + [ + 1.0668, + 0.3048 + ] + ], + "bottom_m": 0.15, + "top_m": 0.76 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": { + "min_views": 1, + "max_incidence_deg": { + "wall": 60, + "ground": 60 + } + }, + "bands": [ + { + "wall_id": "side", + "band": "wall", + "s_m": [ + -0.6096, + 6.096 + ], + "out_m": 2.0 + }, + { + "wall_id": "side", + "band": "ground", + "s_m": [ + -0.6096, + 0.6096 + ], + "out_m": 4.2672 + }, + { + "wall_id": "side", + "band": "ground", + "s_m": [ + 2.7432, + 3.3528 + ], + "out_m": 4.2672 + }, + { + "wall_id": "side", + "band": "overhead", + "s_m": [ + -0.6096, + 6.096 + ], + "out_m": 3.0 + }, + { + "wall_id": "side", + "band": "facing", + "s_m": [ + -0.6096, + 6.096 + ], + "out_m": 4.2672 + } + ] + } +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/generated-object-evidence-unsupported.source.json b/backend/server/placement/tests/data/capture_cases_v1/generated-object-evidence-unsupported.source.json new file mode 100644 index 000000000..0607d8096 --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/generated-object-evidence-unsupported.source.json @@ -0,0 +1,183 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst8-generated-object-unsupported", + "provenance": { + "data": "synthetic", + "producer": { + "path": "hand-authored", + "version": "burst8" + }, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "site": "burst8-generated-object-unsupported", + "notes": [ + "Authored for the burst8 supported-case packet: one straight wall with an observed on-wall meter anchor and one object the producer completed rather than observed (evidence generated), with no generated_regions, no keyframes and band coverage, so the object's evidence is the scene's first unsupported feature and the bridge must refuse it at /objects/1/evidence." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": { + "element_id": "obj-meter", + "plus_minus_m": 0.09 + } + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [ + [ + -0.4572, + 0.0 + ], + [ + 3.048, + 0.0 + ] + ], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": [ + "side" + ], + "ends": { + "left": { + "kind": "limit", + "s_m": -0.4572 + }, + "right": { + "kind": "limit", + "s_m": 3.048 + } + } + }, + "ground": { + "surface": { + "kind": "plane", + "normal": [ + 0.0, + 1.0, + 0.0 + ], + "offset_m": 0.0 + }, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [ + [ + -0.4572, + 0.0 + ], + [ + 3.048, + 0.0 + ], + [ + 3.048, + 0.9144 + ], + [ + -0.4572, + 0.9144 + ] + ], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [ + [ + -0.4572, + 0.9144 + ], + [ + 3.048, + 0.9144 + ], + [ + 3.048, + 3.9624 + ], + [ + -0.4572, + 3.9624 + ] + ], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + -0.1524, + 0.1524 + ], + "bottom_m": 0.1524, + "top_m": 1.3716 + }, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + }, + { + "id": "obj-window-model", + "class": "window", + "wall_id": "side", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + 0.9144, + 1.524 + ], + "bottom_m": 0.9144, + "top_m": 2.1336 + }, + "confidence": 0.8, + "plus_minus_m": 0.09, + "source": { + "kind": "vlm" + }, + "evidence": "generated" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": { + "min_views": 1, + "max_incidence_deg": { + "wall": 60, + "ground": 60 + } + }, + "bands": [ + { + "wall_id": "side", + "band": "wall", + "s_m": [ + -0.4572, + 3.048 + ], + "out_m": 2.0 + } + ] + } +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/generated-regions-unsupported.source.json b/backend/server/placement/tests/data/capture_cases_v1/generated-regions-unsupported.source.json new file mode 100644 index 000000000..b30a6b7d0 --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/generated-regions-unsupported.source.json @@ -0,0 +1,78 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst7-generated-hazard-unsupported", + "provenance": { + "data": "synthetic", + "site": "burst7-generated-hazard-unsupported", + "producer": {"path": "hand-authored", "version": "burst7"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "notes": [ + "Negative case for the burst7 placement case suite: a schema-valid source whose second object is a gas meter a generative model produced (evidence generated). The future bridge (W7) must refuse it with code placement_scene_unsupported at pointer /objects/1/evidence: generated geometry is never evidence. This suite never runs that bridge; the case stays labelled awaiting conversion." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "obj-meter", "plus_minus_m": 0.09} + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [[-3.048, 0.0], [4.572, 0.0]], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": ["side"], + "ends": { + "left": {"kind": "limit", "s_m": -3.048, "note": "corner post"}, + "right": {"kind": "unexplored", "s_m": 4.572, "note": "wall continues right, not walked"} + } + }, + "ground": { + "surface": {"kind": "plane", "normal": [0.0, 1.0, 0.0], "offset_m": 0.0}, + "plus_minus_m": 0.05 + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": {"kind": "wall_rect", "wall_id": "side", "s_m": [-0.15, 0.15], "bottom_m": 0.15, "top_m": 1.2, "standoff_m": 0.1}, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + }, + { + "id": "obj-gas-generated", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "footprint", + "polygon_m": [[1.2192, 0.0], [1.524, 0.0], [1.524, 0.3048], [1.2192, 0.3048]], + "bottom_m": 0.15, + "top_m": 0.76 + }, + "confidence": 0.4, + "plus_minus_m": 0.3, + "source": {"kind": "generated", "model": "inpaint-demo"}, + "evidence": "generated" + } + ], + "generated_regions": [ + { + "id": "gen-hazard", + "geometry": {"kind": "box", "center_m": [1.3716, 0.5, 0.45], "size_m": [0.6, 0.9, 0.5], "yaw_deg": 0.0}, + "method": "hole-filling", + "note": "The hazard above was completed, not observed." + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": {"min_views": 1, "max_incidence_deg": {"wall": 60, "ground": 60}} + } +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.source.json b/backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.source.json new file mode 100644 index 000000000..52c5d353f --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.source.json @@ -0,0 +1,233 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst8-hazard-left-ground-gap", + "provenance": { + "data": "synthetic", + "producer": { + "path": "hand-authored", + "version": "burst8" + }, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "site": "burst8-hazard-left-ground-gap", + "notes": [ + "Authored for the burst8 supported-case packet. It pairs with hazard-left-ground-gap.wire.json by construction (the same wall, on-wall meter anchor, observed wall_rect hazard, coverage gaps and ends, described once in meters and once in feet by hand), not by running any converter." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": { + "element_id": "obj-meter", + "plus_minus_m": 0.09 + } + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [ + [ + -3.048, + 0.0 + ], + [ + 1.0668, + 0.0 + ] + ], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": [ + "side" + ], + "ends": { + "left": { + "kind": "unexplored", + "s_m": -3.048, + "note": "wall continues left, not walked" + }, + "right": { + "kind": "limit", + "s_m": 1.0668, + "note": "gate at the right property line" + } + } + }, + "ground": { + "surface": { + "kind": "plane", + "normal": [ + 0.0, + 1.0, + 0.0 + ], + "offset_m": 0.0 + }, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [ + [ + -3.048, + 0.0 + ], + [ + 1.0668, + 0.0 + ], + [ + 1.0668, + 0.762 + ], + [ + -3.048, + 0.762 + ] + ], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [ + [ + -3.048, + 0.762 + ], + [ + 1.0668, + 0.762 + ], + [ + 1.0668, + 3.6576 + ], + [ + -3.048, + 3.6576 + ] + ], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + -0.1524, + 0.1524 + ], + "bottom_m": 0.2286, + "top_m": 1.2192 + }, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + }, + { + "id": "obj-gas", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + -1.2192, + -0.9144 + ], + "bottom_m": 0.3048, + "top_m": 0.9144 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": { + "min_views": 1, + "max_incidence_deg": { + "wall": 60, + "ground": 60 + } + }, + "bands": [ + { + "wall_id": "side", + "band": "wall", + "s_m": [ + -3.048, + 1.0668 + ], + "out_m": 2.1336 + }, + { + "wall_id": "side", + "band": "ground", + "s_m": [ + -0.3048, + 1.0668 + ], + "out_m": 3.6576 + }, + { + "wall_id": "side", + "band": "ground", + "s_m": [ + -3.048, + -2.8956 + ], + "out_m": 3.6576 + }, + { + "wall_id": "side", + "band": "overhead", + "s_m": [ + -3.048, + 1.0668 + ], + "out_m": 2.4384 + }, + { + "wall_id": "side", + "band": "facing", + "s_m": [ + -3.048, + 1.0668 + ], + "out_m": 3.6576 + } + ] + }, + "facing": [ + { + "wall_id": "side", + "s_m": [ + -3.048, + 1.0668 + ], + "depth_m": 3.6576, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ] +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.wire.json b/backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.wire.json new file mode 100644 index 000000000..2a97fc86d --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/hazard-left-ground-gap.wire.json @@ -0,0 +1,153 @@ +{ + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 2.375, + 0.0 + ], + "wall_id": "side", + "plus_minus_ft": 0.295276 + }, + "walls": [ + { + "id": "side", + "baseline": [ + [ + -10.0, + 0.0 + ], + [ + 3.5, + 0.0 + ] + ], + "height_ft": 9.0, + "plus_minus_ft": 0.295276 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "side", + "span_ft": [ + -4.0, + -3.0 + ], + "bottom_ft": 1.0, + "top_ft": 3.0, + "source": "tap", + "conf": 0.9, + "plus_minus_ft": 0.295276 + } + ], + "ground": [ + { + "type": "mulch", + "polygon": [ + [ + -10.0, + 0.0 + ], + [ + 3.5, + 0.0 + ], + [ + 3.5, + 2.5 + ], + [ + -10.0, + 2.5 + ] + ] + }, + { + "type": "lawn", + "polygon": [ + [ + -10.0, + 2.5 + ], + [ + 3.5, + 2.5 + ], + [ + 3.5, + 12.0 + ], + [ + -10.0, + 12.0 + ] + ] + } + ], + "facing": [ + { + "wall_id": "side", + "span_ft": [ + -10.0, + 3.5 + ], + "depth_ft": 12.0, + "plus_minus_ft": 0.295276 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "unexplored", + "note": "wall continues left, not walked" + }, + "right": { + "kind": "limit", + "note": "gate at the right property line" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -10.0, + 3.5 + ], + "out_ft": 7.0 + }, + { + "band": "ground", + "span_ft": [ + -1.0, + 3.5 + ], + "out_ft": 12.0 + }, + { + "band": "ground", + "span_ft": [ + -10.0, + -9.5 + ], + "out_ft": 12.0 + }, + { + "band": "overhead", + "span_ft": [ + -10.0, + 3.5 + ], + "out_ft": 8.0 + }, + { + "band": "facing", + "span_ft": [ + -10.0, + 3.5 + ], + "out_ft": 12.0 + } + ] + } +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.source.json b/backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.source.json new file mode 100644 index 000000000..9d2de25de --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.source.json @@ -0,0 +1,255 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst8-hazard-right-ground-gap", + "provenance": { + "data": "synthetic", + "producer": { + "path": "hand-authored", + "version": "burst8" + }, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "site": "burst8-hazard-right-ground-gap", + "notes": [ + "Authored for the burst8 supported-case packet. It pairs with hazard-right-ground-gap.wire.json by construction (the same wall, on-wall meter anchor, observed wall_rect hazards, coverage gaps and ends, described once in meters and once in feet by hand), not by running any converter.", + "The vent obj-tap-vent carries the exact measured geometry of the replay runner's TAP_HAZARD (scripts/capture_placement_cases.py), so the coordinator's one-time same-id merge replaces it in place and the merged scene keeps exactly one vent." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": { + "element_id": "obj-meter", + "plus_minus_m": 0.09 + } + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [ + [ + -0.4572, + 0.0 + ], + [ + 3.048, + 0.0 + ] + ], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": [ + "side" + ], + "ends": { + "left": { + "kind": "limit", + "s_m": -0.4572, + "note": "fence at the left property line" + }, + "right": { + "kind": "unexplored", + "s_m": 3.048, + "note": "wall continues right, not walked" + } + } + }, + "ground": { + "surface": { + "kind": "plane", + "normal": [ + 0.0, + 1.0, + 0.0 + ], + "offset_m": 0.0 + }, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [ + [ + -0.4572, + 0.0 + ], + [ + 3.048, + 0.0 + ], + [ + 3.048, + 0.9144 + ], + [ + -0.4572, + 0.9144 + ] + ], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [ + [ + -0.4572, + 0.9144 + ], + [ + 3.048, + 0.9144 + ], + [ + 3.048, + 3.9624 + ], + [ + -0.4572, + 3.9624 + ] + ], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + -0.1524, + 0.1524 + ], + "bottom_m": 0.1524, + "top_m": 1.3716 + }, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + }, + { + "id": "obj-gas", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + 0.9144, + 1.2192 + ], + "bottom_m": 0.3048, + "top_m": 0.9144 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + }, + { + "id": "obj-tap-vent", + "class": "vent", + "wall_id": "side", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + 1.6, + 1.8 + ], + "bottom_m": 0.5, + "top_m": 2.0 + }, + "confidence": 0.9, + "plus_minus_m": 0.05, + "source": { + "kind": "tap" + }, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": { + "min_views": 1, + "max_incidence_deg": { + "wall": 60, + "ground": 60 + } + }, + "bands": [ + { + "wall_id": "side", + "band": "wall", + "s_m": [ + -0.4572, + 3.048 + ], + "out_m": 2.0 + }, + { + "wall_id": "side", + "band": "ground", + "s_m": [ + -0.4572, + 0.9144 + ], + "out_m": 3.9624 + }, + { + "wall_id": "side", + "band": "ground", + "s_m": [ + 2.5908, + 3.048 + ], + "out_m": 3.9624 + }, + { + "wall_id": "side", + "band": "overhead", + "s_m": [ + -0.4572, + 3.048 + ], + "out_m": 2.7432 + }, + { + "wall_id": "side", + "band": "facing", + "s_m": [ + -0.4572, + 3.048 + ], + "out_m": 3.9624 + } + ] + }, + "facing": [ + { + "wall_id": "side", + "s_m": [ + -0.4572, + 3.048 + ], + "depth_m": 3.9624, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ] +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.wire.json b/backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.wire.json new file mode 100644 index 000000000..ca742ee84 --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/hazard-right-ground-gap.wire.json @@ -0,0 +1,166 @@ +{ + "schema_version": "1.0", + "meter": { + "pos": [ + 0.0, + 2.5, + 0.0 + ], + "wall_id": "side", + "plus_minus_ft": 0.295276 + }, + "walls": [ + { + "id": "side", + "baseline": [ + [ + -1.5, + 0.0 + ], + [ + 10.0, + 0.0 + ] + ], + "height_ft": 9.0, + "plus_minus_ft": 0.295276 + } + ], + "objects": [ + { + "type": "gas_meter", + "wall_id": "side", + "span_ft": [ + 3.0, + 4.0 + ], + "bottom_ft": 1.0, + "top_ft": 3.0, + "source": "tap", + "conf": 0.9, + "plus_minus_ft": 0.295276 + }, + { + "type": "vent", + "wall_id": "side", + "span_ft": [ + 5.249344, + 5.905512 + ], + "bottom_ft": 1.64042, + "top_ft": 6.56168, + "source": "tap", + "conf": 0.9, + "plus_minus_ft": 0.164042 + } + ], + "ground": [ + { + "type": "mulch", + "polygon": [ + [ + -1.5, + 0.0 + ], + [ + 10.0, + 0.0 + ], + [ + 10.0, + 3.0 + ], + [ + -1.5, + 3.0 + ] + ] + }, + { + "type": "lawn", + "polygon": [ + [ + -1.5, + 3.0 + ], + [ + 10.0, + 3.0 + ], + [ + 10.0, + 13.0 + ], + [ + -1.5, + 13.0 + ] + ] + } + ], + "facing": [ + { + "wall_id": "side", + "span_ft": [ + -1.5, + 10.0 + ], + "depth_ft": 13.0, + "plus_minus_ft": 0.295276 + } + ], + "coverage": { + "ends": { + "left": { + "kind": "limit", + "note": "fence at the left property line" + }, + "right": { + "kind": "unexplored", + "note": "wall continues right, not walked" + } + }, + "observed": [ + { + "band": "wall", + "span_ft": [ + -1.5, + 10.0 + ], + "out_ft": 6.56168 + }, + { + "band": "ground", + "span_ft": [ + -1.5, + 3.0 + ], + "out_ft": 13.0 + }, + { + "band": "ground", + "span_ft": [ + 8.5, + 10.0 + ], + "out_ft": 13.0 + }, + { + "band": "overhead", + "span_ft": [ + -1.5, + 10.0 + ], + "out_ft": 9.0 + }, + { + "band": "facing", + "span_ft": [ + -1.5, + 10.0 + ], + "out_ft": 13.0 + } + ] + } +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/manifest.json b/backend/server/placement/tests/data/capture_cases_v1/manifest.json new file mode 100644 index 000000000..2a5f278d5 --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/manifest.json @@ -0,0 +1,257 @@ +{ + "schema_version": "placement-case-manifest/v0", + "description": [ + "Burst8 supported-case packet. Each positive case pairs a schema-valid synthetic base-scene/v0 source (metres, hand-authored) with an independently authored root-wire scene (feet, hand-derived as metres divided by 0.3048 and rounded to six decimals) — the same wall, on-wall meter anchor, observed wall_rect hazard, level ground, coverage gap, and chain ends, described twice by construction, never produced by running the converter.", + "Positive cases run the unchanged top-level solver through POST /v1/placements on the wire scene and on the bridge's conversion of the source scene; the two responses must agree except stats.elapsed_ms (timing) and stats.input_sha256 plus response.input_sha256 (the posted bytes differ: the wire file carries a schema_version envelope field the bridge does not emit). They must also match these expectations, derived from the authored dimensions: the spot is the last failing start pushed to the 3 ft gas-clearance line (gas edge + 3.0), gas_clearance is unsure by margin (measured exactly 3.0 ft against 0.590552 ft of uncertainty), the ground gap makes ground_surface unsure-unobserved, and the unexplored end within reach asks for a walk past it. The merged ground band's six-decimal extent is the buffer union over the authored gaps and the clearance radii of rules.yaml; its span and out_ft are pinned as the serialized geometric consequence of the authored dimensions.", + "Negative cases are source-only: the bridge must refuse them with the declared error code and pointer. multiwall-source-unsupported and generated-object-evidence-unsupported are newly authored to be clean of every other unsupported feature, so the declared refusal is the scene's first one. The other three preserve W7's documented bridge limitations: standoff-meter-anchor-unsupported is a byte-identical copy of the W7 hazard-right-ground-gap source, generated-regions-unsupported a byte-identical copy of the W7 generated-hazard source, and footprint-hazard-unsupported is the W7 right source with only the meter's standoff_m field removed so the footprint hazard is the first unsupported feature left (its hash pins the derived bytes).", + "Every fixture path is relative to this manifest's directory and pinned by sha256; a missing or drifted file fails the suite with its path." + ], + "cases": [ + { + "id": "hazard-right-ground-gap", + "kind": "positive", + "source": { + "path": "hazard-right-ground-gap.source.json", + "sha256": "57451d8af6e01ff5d1d40ee5911dbfceeefa1e87fab2cc9f3284533785bba059" + }, + "wire": { + "path": "hazard-right-ground-gap.wire.json", + "sha256": "06ea13384eb8dfcff5025284d2314645568af395679128f08600c065523d9bf3" + }, + "expectations": { + "decision": "manual_review", + "spot": { + "wall_id": "side", + "segment": 0, + "span_ft": [ + 7.0, + 9.583333 + ], + "outcome": "unsure" + }, + "checks": { + "gas_clearance": { + "outcome": "unsure", + "unsure_cause": "margin" + }, + "ground_surface": { + "outcome": "unsure", + "unsure_cause": "unobserved" + }, + "meter_working_space": { + "outcome": "pass" + }, + "wall_equipment_above": { + "outcome": "pass" + }, + "route_length": { + "outcome": "pass" + } + }, + "missing_evidence": [ + { + "kind": "band", + "band": "ground", + "span_ft": [ + -3.307693, + 8.495 + ], + "out_ft": 12.141026 + }, + { + "kind": "past_end", + "side": "right", + "span_ft": [ + 10.0, + 10.0 + ] + } + ], + "sweep_failing_runs": [ + { + "start_ft": [ + 2.846457, + 5.725066 + ], + "failing": [ + "gas_clearance", + "wall_equipment_above" + ] + }, + { + "start_ft": [ + 5.74147, + 6.069554 + ], + "failing": [ + "gas_clearance" + ] + } + ], + "ends": { + "left": "limit", + "right": "unexplored" + } + }, + "note": "Geometry facts, not policy: the wall runs s [-1.5, 10.0] ft with a limit left end and an unexplored right end; the gas meter at s [3.0, 4.0] ft pushes the best spot to the 3 ft clearance line (s0 = 4.0 + 3.0 = 7.0, measured 3.0 ft +- 0.590552, unsure by margin, not by observation); every start closer to the meter either overlaps the working space or the meter (estimate-failing) and every start past the clearance line has unsure ground too but a longer route. The wall_rect vent obj-tap-vent (the replay runner's TAP_HAZARD geometry) sits at s [5.249344, 5.905512] ft +- 0.164042: the chosen spot clears it (gap 1.094488 ft), while the pinned sweep runs overlap the gas meter and the vent — the second run ends exactly at the vent's tolerance line 5.905512 + 0.164042 = 6.069554. Ground is observed only at [-1.5, 3.0] and [8.5, 10.0] ft out 13, so the spot's ground is unsure-unobserved and the response must request the ground band and a walk past the right end." + }, + { + "id": "hazard-left-ground-gap", + "kind": "positive", + "source": { + "path": "hazard-left-ground-gap.source.json", + "sha256": "8000f6c9d0ae47f6203c0a3a42e61ff3e9ee891bc33c2396c80cec591f0d9e5b" + }, + "wire": { + "path": "hazard-left-ground-gap.wire.json", + "sha256": "6e8b6e001f55e99a17d202ce64f39297554cafaa9e32f0c3a33cd6ab35813839" + }, + "expectations": { + "decision": "manual_review", + "spot": { + "wall_id": "side", + "segment": 0, + "span_ft": [ + -9.583333, + -7.0 + ], + "outcome": "unsure" + }, + "checks": { + "gas_clearance": { + "outcome": "unsure", + "unsure_cause": "margin" + }, + "ground_surface": { + "outcome": "unsure", + "unsure_cause": "unobserved" + }, + "meter_working_space": { + "outcome": "pass" + }, + "opening_clearance": { + "outcome": "unsure", + "unsure_cause": "unobserved" + }, + "route_length": { + "outcome": "pass" + } + }, + "missing_evidence": [ + { + "kind": "band", + "band": "ground", + "span_ft": [ + -10.0, + -1.004999 + ], + "out_ft": 12.141026 + }, + { + "kind": "past_end", + "side": "left", + "span_ft": [ + -10.0, + -10.0 + ] + } + ], + "sweep_failing_runs": [ + { + "start_ft": [ + -8.975557, + -4.458333 + ], + "failing": [ + "gas_clearance" + ] + }, + { + "start_ft": [ + -0.590552, + 0.621391 + ], + "failing": [ + "meter_working_space" + ] + } + ], + "ends": { + "left": "unexplored", + "right": "limit" + } + }, + "note": "Geometry facts, not policy: the wall runs s [-10.0, 3.5] ft with an unexplored left end and a limit right end; the gas meter at s [-4.0, -3.0] ft pushes the best spot left onto the 3 ft clearance line (s1 = -4.0 - 3.0 = -7.0, s0 = -9.583333, measured 3.0 ft +- 0.590552, unsure by margin), and the wall's left end at -10.0 leaves no room for a fully clear gas spot so every left candidate keeps the gas unsure. Ground is observed only at [-10.0, -9.5] and [-1.0, 3.5] ft out 12, so the spot's ground is unsure-unobserved; the right stretch near the meter is blocked by the working space (the pinned meter_working_space run starts exactly at -(0.09 m + 0.09 m) = -0.590552 ft) and the wall's short right end blocks the rest. The response must request the ground band and a walk past the left end." + }, + { + "id": "multiwall-source-unsupported", + "kind": "negative_source", + "source": { + "path": "multiwall-source-unsupported.source.json", + "sha256": "3e13bace8381dd56bdaaa6e5e874cca488c8ea08617eb09c8bf4e19b8c75de05" + }, + "wire": null, + "expected_error": { + "code": "placement_scene_unsupported", + "pointer": "/walls" + }, + "note": "Awaiting conversion: an L of two house walls with everything else inside the bridge's bounded scope (observed on-wall meter anchor, observed wall_rect hazard, level ground, band coverage, contract-valid chain ends), so the two-wall chain is the scene's first unsupported feature and the bridge must decline it with code placement_scene_unsupported at pointer /walls. This suite never runs the missing bridge, and this case is never reported as a successful conversion." + }, + { + "id": "generated-object-evidence-unsupported", + "kind": "negative_source", + "source": { + "path": "generated-object-evidence-unsupported.source.json", + "sha256": "29fab430e733033655580d510121b4496e68bc264e23a6b56dfa0890bc7e315e" + }, + "wire": null, + "expected_error": { + "code": "placement_scene_unsupported", + "pointer": "/objects/1/evidence" + }, + "note": "Awaiting conversion: one straight wall with an observed on-wall meter anchor and one window object the producer completed rather than observed (evidence generated), with no generated_regions, no keyframes and band coverage, so the object's evidence is the scene's first unsupported feature and the bridge must decline it with code placement_scene_unsupported at pointer /objects/1/evidence. This suite never runs the missing bridge, and this case is never reported as a successful conversion." + }, + { + "id": "standoff-meter-anchor-unsupported", + "kind": "negative_source", + "source": { + "path": "standoff-meter-anchor-unsupported.source.json", + "sha256": "670624f98e6366c15d8d0154e9d5f98338cbe65a3d31f204475619fe22c9f714" + }, + "wire": null, + "expected_error": { + "code": "placement_scene_unsupported", + "pointer": "/objects/0/geometry/standoff_m" + }, + "note": "Awaiting conversion: a byte-identical copy of the W7 hazard-right-ground-gap source, preserved unchanged so the declared refusal is the scene's actual first one — the stand-off meter anchor the bridge does not synthesise, at /objects/0/geometry/standoff_m. The suite never runs the missing bridge, and this case is never reported as a successful conversion." + }, + { + "id": "footprint-hazard-unsupported", + "kind": "negative_source", + "source": { + "path": "footprint-hazard-unsupported.source.json", + "sha256": "953b0cb6d9fd32904f1aa84df7b4ecc62bcb6683cd5d1c0ea21bb10dd61b7184" + }, + "wire": null, + "expected_error": { + "code": "placement_scene_unsupported", + "pointer": "/objects/1/geometry/kind" + }, + "note": "Awaiting conversion: the W7 hazard-right-ground-gap source with only the meter anchor's standoff_m field removed (the scene is otherwise byte-for-byte the same document re-serialized), so the stand-off refusal no longer fires first and the footprint gas_meter hazard is the first unsupported feature left. The bridge must decline it with code placement_scene_unsupported at pointer /objects/1/geometry/kind. The suite never runs the missing bridge, and this case is never reported as a successful conversion." + }, + { + "id": "generated-regions-unsupported", + "kind": "negative_source", + "source": { + "path": "generated-regions-unsupported.source.json", + "sha256": "a9b9a35f49f8811e7dd1fbccb9b1eead6de0418494192fa49ee51c5fe603ce34" + }, + "wire": null, + "expected_error": { + "code": "placement_scene_unsupported", + "pointer": "/generated_regions" + }, + "note": "Awaiting conversion: a byte-identical copy of the W7 generated-hazard source, preserved unchanged and declared at its actual first refusal /generated_regions (generated regions are unknown space, not observed geometry) rather than the /objects/1/evidence pointer its W7 manifest entry declared. The suite never runs the missing bridge, and this case is never reported as a successful conversion." + } + ] +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json b/backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json new file mode 100644 index 000000000..18f8ab413 --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/multiwall-source-unsupported.source.json @@ -0,0 +1,201 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst8-multiwall-unsupported", + "provenance": { + "data": "synthetic", + "producer": { + "path": "hand-authored", + "version": "burst8" + }, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "site": "burst8-multiwall-unsupported", + "notes": [ + "Authored for the burst8 supported-case packet: an L of two house walls, everything else inside the bridge's bounded scope (observed on-wall meter anchor, observed wall_rect hazard, level ground, band coverage). The chain has two walls, so the bridge must refuse it at /walls with placement_scene_unsupported." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": { + "element_id": "obj-meter", + "plus_minus_m": 0.09 + } + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [ + [ + -0.4572, + 0.0 + ], + [ + 1.524, + 0.0 + ] + ], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + }, + { + "id": "rear", + "role": "house", + "baseline_m": [ + [ + -0.4572, + 0.0 + ], + [ + 0.9144, + 0.0 + ] + ], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": [ + "side", + "rear" + ], + "ends": { + "left": { + "kind": "limit", + "s_m": -0.4572 + }, + "right": { + "kind": "limit", + "s_m": 1.524 + } + } + }, + "ground": { + "surface": { + "kind": "plane", + "normal": [ + 0.0, + 1.0, + 0.0 + ], + "offset_m": 0.0 + }, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [ + [ + -0.4572, + 0.0 + ], + [ + 3.048, + 0.0 + ], + [ + 3.048, + 0.9144 + ], + [ + -0.4572, + 0.9144 + ] + ], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [ + [ + -0.4572, + 0.9144 + ], + [ + 3.048, + 0.9144 + ], + [ + 3.048, + 3.9624 + ], + [ + -0.4572, + 3.9624 + ] + ], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + -0.1524, + 0.1524 + ], + "bottom_m": 0.1524, + "top_m": 1.3716 + }, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + }, + { + "id": "obj-gas", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "wall_rect", + "wall_id": "side", + "s_m": [ + 0.3048, + 0.6096 + ], + "bottom_m": 0.3048, + "top_m": 0.9144 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": { + "kind": "tap" + }, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": { + "min_views": 1, + "max_incidence_deg": { + "wall": 60, + "ground": 60 + } + }, + "bands": [ + { + "wall_id": "side", + "band": "wall", + "s_m": [ + -0.4572, + 1.524 + ], + "out_m": 2.0 + } + ] + } +} diff --git a/backend/server/placement/tests/data/capture_cases_v1/standoff-meter-anchor-unsupported.source.json b/backend/server/placement/tests/data/capture_cases_v1/standoff-meter-anchor-unsupported.source.json new file mode 100644 index 000000000..287d6133d --- /dev/null +++ b/backend/server/placement/tests/data/capture_cases_v1/standoff-meter-anchor-unsupported.source.json @@ -0,0 +1,91 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst7-hazard-right-ground-gap", + "provenance": { + "data": "synthetic", + "site": "burst7-hazard-right-ground-gap", + "producer": {"path": "hand-authored", "version": "burst7"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "notes": [ + "Authored for the burst7 placement case suite. It pairs with hazard-right-ground-gap.wire.json by construction (the same wall, hazard, coverage gap and ends, described once in meters and once in feet by hand), not by running any converter." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "obj-meter", "plus_minus_m": 0.09} + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [[-0.6096, 0.0], [6.096, 0.0]], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": ["side"], + "ends": { + "left": {"kind": "limit", "s_m": -0.6096, "note": "fence at the left property line"}, + "right": {"kind": "unexplored", "s_m": 6.096, "note": "wall continues right, not walked"} + } + }, + "ground": { + "surface": {"kind": "plane", "normal": [0.0, 1.0, 0.0], "offset_m": 0.0}, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [[-0.6096, 0.0], [6.096, 0.0], [6.096, 1.2192], [-0.6096, 1.2192]], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [[-0.6096, 1.2192], [6.096, 1.2192], [6.096, 4.2672], [-0.6096, 4.2672]], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": {"kind": "wall_rect", "wall_id": "side", "s_m": [-0.15, 0.15], "bottom_m": 0.15, "top_m": 1.2, "standoff_m": 0.1}, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + }, + { + "id": "obj-gas", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "footprint", + "polygon_m": [[1.0668, 0.0], [1.3716, 0.0], [1.3716, 0.3048], [1.0668, 0.3048]], + "bottom_m": 0.15, + "top_m": 0.76 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": {"min_views": 1, "max_incidence_deg": {"wall": 60, "ground": 60}}, + "bands": [ + {"wall_id": "side", "band": "wall", "s_m": [-0.6096, 6.096], "out_m": 2.0}, + {"wall_id": "side", "band": "ground", "s_m": [-0.6096, 0.6096], "out_m": 4.2672}, + {"wall_id": "side", "band": "ground", "s_m": [2.7432, 3.3528], "out_m": 4.2672}, + {"wall_id": "side", "band": "overhead", "s_m": [-0.6096, 6.096], "out_m": 3.0}, + {"wall_id": "side", "band": "facing", "s_m": [-0.6096, 6.096], "out_m": 4.2672} + ] + } +} diff --git a/backend/tests/test_capture_case_pairs.py b/backend/tests/test_capture_case_pairs.py new file mode 100644 index 000000000..b6ea1182c --- /dev/null +++ b/backend/tests/test_capture_case_pairs.py @@ -0,0 +1,231 @@ +"""The burst8 supported-case packet: independently authored source/wire pairs. + +The packet at ``server/placement/tests/data/capture_cases_v1`` is a separate +suite from the W7 default manifest. Its positives are hand-authored twice — +once in metres (source) and once in feet (wire), derived by hand rather than +by running any converter — so the pair comparison below is a real check of the +W7 bridge, and its negatives pin the bridge's documented refusals, including +three cases that preserve W7's limitations byte-for-byte or byte-derived. + +These tests reuse the manifest loader and suite runner (hash pinning, input +hashes, expectation evaluation) and add what the default suite does not do: +comparing each positive source's bridge conversion against the authored wire +scene, and running the solver over both byte streams and requiring identical +responses apart from the request-identity fields. +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from server.placement import case_suite as cs +from server.placement.scene_bridge import SceneBridgeError, convert_scene + +PLACEMENT_DATA = Path(__file__).parents[1] / "server" / "placement" / "tests" / "data" +PACKET_DIR = PLACEMENT_DATA / "capture_cases_v1" +MANIFEST = PACKET_DIR / "manifest.json" + +POSITIVE_IDS = ("hazard-right-ground-gap", "hazard-left-ground-gap") +NEGATIVE_IDS = ( + "multiwall-source-unsupported", + "generated-object-evidence-unsupported", + "standoff-meter-anchor-unsupported", + "footprint-hazard-unsupported", + "generated-regions-unsupported", +) + +# The wire scenes round metres-derived feet to six decimals; the bridge emits +# full float precision. The largest possible gap is half of the last kept +# decimal, 5e-7 ft — about 0.15 micrometres, far inside any survey tolerance. +WIRE_ROUNDING_TOLERANCE_FT = 5e-7 + + +def _payload() -> dict: + return json.loads(MANIFEST.read_text()) + + +def _case(payload: dict, case_id: str) -> dict: + return next(case for case in payload["cases"] if case["id"] == case_id) + + +def _positive_docs(case_id: str) -> tuple[dict, dict]: + case = _case(_payload(), case_id) + source = json.loads((PACKET_DIR / case["source"]["path"]).read_text()) + wire = json.loads((PACKET_DIR / case["wire"]["path"]).read_text()) + return source, wire + + +@pytest.mark.parametrize("case_id", POSITIVE_IDS) +def test_positive_source_is_schema_valid(case_id: str) -> None: + source, _ = _positive_docs(case_id) + validator = cs._schema_validator(cs.SOURCE_SCHEMA) + assert cs._schema_errors(validator, source) == [] + + +def _differences(converted: dict, wire: dict, path: str = "", tolerance: float | None = None) -> list[str]: + """Structural diffs first, then float gaps within the declared tolerance.""" + if tolerance is None: + tolerance = WIRE_ROUNDING_TOLERANCE_FT + if isinstance(converted, dict) and isinstance(wire, dict): + diffs = [] + for key in sorted(set(converted) | set(wire)): + if key not in converted or key not in wire: + # The bridge omits empty lists; the wire writes [] (declared). + if wire.get(key) == [] and converted.get(key) is None: + continue + diffs.append(f"{path}/{key}: key present on one side only") + else: + diffs.extend(_differences(converted[key], wire[key], f"{path}/{key}", tolerance)) + return diffs + if isinstance(converted, list) and isinstance(wire, list): + if len(converted) != len(wire): + return [f"{path}: list length {len(converted)} != {len(wire)}"] + diffs = [] + for index, (left, right) in enumerate(zip(converted, wire, strict=False)): + diffs.extend(_differences(left, right, f"{path}[{index}]", tolerance)) + return diffs + if ( + isinstance(converted, (int, float)) + and isinstance(wire, (int, float)) + and not isinstance(converted, bool) + and not isinstance(wire, bool) + ): + if abs(converted - wire) > tolerance: + return [f"{path}: {converted} != {wire} beyond {tolerance}"] + return [] + if converted != wire: + return [f"{path}: {converted!r} != {wire!r}"] + return [] + + +@pytest.mark.parametrize("case_id", POSITIVE_IDS) +def test_positive_bridge_conversion_matches_authored_wire(case_id: str) -> None: + source, wire = _positive_docs(case_id) + converted = convert_scene(source) + wire_without_envelope = {key: value for key, value in wire.items() if key != "schema_version"} + diffs = _differences(converted, wire_without_envelope) + assert diffs == [] + + +@pytest.fixture(scope="module") +def packet_report() -> dict: + return cs.run_suite(MANIFEST) + + +def test_packet_summary(packet_report: dict) -> None: + assert packet_report["schema_version"] == cs.REPORT_SCHEMA_VERSION + assert packet_report["summary"] == { + "cases": 7, + "positive": 2, + "negative_source": 5, + "matched": 2, + "mismatched": 0, + "awaiting_conversion": 5, + "errored": 0, + } + assert packet_report["ok"] is True + + +@pytest.mark.parametrize("case_id", POSITIVE_IDS) +def test_positive_case_solved_and_matched(packet_report: dict, case_id: str) -> None: + case = next(case for case in packet_report["cases"] if case["id"] == case_id) + assert case["status"] == cs.STATUS_MATCHED, case + assert case["executed"] is True + assert case["executions_agree"] is True + assert case["input_sha_matches_file"] is True + assert case["decision"] == "manual_review" + + +def test_solver_agrees_between_wire_and_converted_bytes() -> None: + """The wire scene and the bridge's conversion of the source must produce + the same solver response apart from request identity.""" + client = cs._get_client() + for case_id in POSITIVE_IDS: + case = _case(_payload(), case_id) + source_bytes = (PACKET_DIR / case["source"]["path"]).read_bytes() + wire_bytes = (PACKET_DIR / case["wire"]["path"]).read_bytes() + wire_body = client.post("/v1/placements", content=wire_bytes, headers={"Content-Type": "application/json"}) + assert wire_body.status_code == 200 + converted = convert_scene(json.loads(source_bytes)) + converted_bytes = (json.dumps({"schema_version": "1.0", **converted}, indent=2) + "\n").encode() + source_body = client.post( + "/v1/placements", content=converted_bytes, headers={"Content-Type": "application/json"} + ) + assert source_body.status_code == 200 + + def identity(body: dict) -> dict: + scrubbed = copy.deepcopy(body) + scrubbed["stats"].pop("elapsed_ms", None) + scrubbed["stats"].pop("input_sha256", None) + scrubbed.pop("input_sha256", None) + return scrubbed + + # The wire's six-decimal rounding propagates at most a few 1e-6 ft into + # derived solver outputs; any real geometry disagreement would move a + # candidate by the 1/96 ft sweep step. 1e-4 ft (0.03 mm) separates them. + diffs = _differences(identity(wire_body.json()), identity(source_body.json()), tolerance=1e-4) + assert diffs == [], (case_id, diffs) + + +@pytest.mark.parametrize("case_id", NEGATIVE_IDS) +def test_negative_refuses_where_declared(case_id: str) -> None: + case = _case(_payload(), case_id) + source = json.loads((PACKET_DIR / case["source"]["path"]).read_text()) + with pytest.raises(SceneBridgeError) as excinfo: + convert_scene(source) + assert excinfo.value.code == case["expected_error"]["code"], case_id + assert excinfo.value.pointer == case["expected_error"]["pointer"], case_id + + +def test_w7_preserved_copies_are_byte_identical() -> None: + """The two byte-preserved W7 limitation sources must match the W7 fixtures + they document, hash for hash.""" + w7_dir = PLACEMENT_DATA / "cases" + for packet_name, w7_name in ( + ("standoff-meter-anchor-unsupported.source.json", "hazard-right-ground-gap.source.json"), + ("generated-regions-unsupported.source.json", "generated-hazard-unsupported.source.json"), + ): + packet_bytes = (PACKET_DIR / packet_name).read_bytes() + w7_bytes = (w7_dir / w7_name).read_bytes() + assert packet_bytes == w7_bytes, packet_name + + +def test_tampered_spot_expectation_is_caught(tmp_path: Path) -> None: + payload = _packet_payload_absolutized() + _case(payload, POSITIVE_IDS[0])["expectations"]["spot"]["span_ft"] = [0.0, 1.0] + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + report = cs.run_suite(drift) + case = next(item for item in report["cases"] if item["id"] == POSITIVE_IDS[0]) + assert case["status"] == cs.STATUS_MISMATCHED + assert any("spot" in mismatch for mismatch in case["mismatches"]) + assert report["ok"] is False + + +def test_drifted_pair_wire_fails_with_path(tmp_path: Path) -> None: + payload = _packet_payload_absolutized() + case = _case(payload, POSITIVE_IDS[0]) + fixture = tmp_path / "drifted.wire.json" + fixture.write_bytes(json.dumps(json.loads((PACKET_DIR / case["wire"]["path"]).read_text())).encode() + b" ") + case["wire"]["sha256"] = "0" * 64 + case["wire"]["path"] = str(fixture) + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert str(fixture) in str(excinfo.value) + + +def _packet_payload_absolutized() -> dict: + """Packet manifest with fixture paths made absolute so a copy anywhere + still resolves the committed fixtures.""" + payload = json.loads(MANIFEST.read_text()) + for case in payload["cases"]: + case["source"]["path"] = str(PACKET_DIR / case["source"]["path"]) + if case.get("wire"): + case["wire"]["path"] = str(PACKET_DIR / case["wire"]["path"]) + return payload From b4d28f43a4d8f8664ffd7ce07e5714b3ea18bc6d Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:12:09 +0000 Subject: [PATCH 04/19] Reconcile twin-identity manifest with the hardened case loader The case-integrity fix rejects decision-only positive expectations, so the twin-identity manifest now pins the coverage chain ends as well; the cross-capture refusal still comes from normalized geometry, not the loader. --- backend/tests/test_capture_placement_cases.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index 28760438c..e03937c23 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -527,14 +527,17 @@ def _write_twin_manifest(tmp_path: Path) -> Path: "kind": "positive", "source": source_ref, "wire": wire_ref, - "expectations": {"decision": "manual_review"}, + # The hardened loader (case-integrity fix) rejects decision-only + # positives, so the twins pin the coverage chain ends too — the + # cross-capture refusal must still come from geometry, not the loader. + "expectations": {"decision": "manual_review", "ends": {"left": "unexplored", "right": "limit"}}, }, { "id": "twin-two", "kind": "positive", "source": source_ref, "wire": wire_ref, - "expectations": {"decision": "manual_review"}, + "expectations": {"decision": "manual_review", "ends": {"left": "unexplored", "right": "limit"}}, }, ] manifest_path = tmp_path / "manifest.json" From 250c04332b96b7fac754ed28f663aad0be60aac3 Mon Sep 17 00:00:00 2001 From: Obvious Date: Fri, 16 Oct 2026 00:29:26 +0000 Subject: [PATCH 05/19] Reconcile packet acceptance with the real merge and bridge contracts - Deliver the tap hazard at the objects_3d boundary only when the case's source does not already carry it: the merge contract replaces only detector/VLM twins, so a same-id re-delivery of a tap-sourced hazard duplicates the id and is refused as an inconsistent scene. - Evaluate wire-pinned expectations within the packet's declared 1e-4 ft derived-output tolerance (the shared evaluator stays exact by default). - Record the integrated producer SHAs and a distinct solver decision in the receipt; default the replay CLI to the supported packet manifest. --- backend/scripts/capture_placement_cases.py | 56 ++++++++++++++++++---- backend/server/placement/case_suite.py | 35 ++++++++++++-- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/backend/scripts/capture_placement_cases.py b/backend/scripts/capture_placement_cases.py index bc9506f49..46f2be36b 100644 --- a/backend/scripts/capture_placement_cases.py +++ b/backend/scripts/capture_placement_cases.py @@ -24,9 +24,12 @@ - The ``objects_3d`` stage boundary gets an explicit, successful object list for every case: an empty list where the source scene already carries the hazards, and one nonempty observed tap hazard case whose single hazard the coordinator's - one-time merge must place into the merged scene exactly once. The default - objects stubs never run on this path: a capture with no registered injection - fails loudly instead of fabricating a hazard. + one-time merge must place into the merged scene exactly once — delivered only + when the case's source does not already carry that hazard id, because the merge + contract replaces only detector/VLM twins and a same-id re-delivery of a + tap-sourced hazard would duplicate the id and be refused. The default objects + stubs never run on this path: a capture with no registered injection fails + loudly instead of fabricating a hazard. - Every other unavailable vision stage stays explicitly stubbed (prewarm off, external vision calls off). ``placement`` is ``stubs_exempt``: the real conversion and solver run behind the door when a case reaches them. @@ -128,6 +131,23 @@ # The one case whose objects_3d boundary carries a nonempty observed tap hazard. TAP_CASE_ID = "hazard-right-ground-gap" +# Expectations pinned on the wire scene, evaluated against the journey's real +# converted-path result: the wire's six-decimal rounding (declared by the packet as +# 5e-7 ft) propagates through the real conversion and solver into derived feet +# values, so those compare within the packet's declared 1e-4 ft derived-output +# tolerance. Scene geometry itself is still compared exactly, field by field (see +# _check_converted_geometry), and the case-suite evaluator stays exact by default +# (tol_ft defaults to 0.0). +_WIRE_DERIVED_TOLERANCE_FT = 1e-4 + +# The supported case packet is the default replay target; the legacy W7 cases +# manifest stays available explicitly via --manifest and keeps its own pinned +# refusal state, reported as blockers rather than mistaken for acceptance. +DEFAULT_MANIFEST = ( + Path(__file__).resolve().parent.parent + / "server/placement/tests/data/capture_cases_v1/manifest.json" +) + # An observed tap hazard, hand-authored for this runner: a wall_rect vent on the # case's wall, away from the case's pinned decision geometry. It exists to prove # the coordinator's one-time merge, never to fabricate a conflicting hazard. @@ -347,11 +367,21 @@ def _walk_front_door( journey["captureId"] = capture_id # Register the case at both boundaries before finalize can start the pipeline. + # The tap hazard is delivered only where the upstream source does not already + # carry it: the merge contract replaces only detector/VLM twins, so a same-id + # re-delivery of a source-carried (tap-sourced) hazard is never replaced — it + # duplicates the id and the merged scene is refused as inconsistent. A + # source-carried hazard still reaches the one-time tap checks through the real + # merge and real conversion below. + source_object_ids = { + obj.get("id") for obj in json.loads(record.source_path.read_text()).get("objects") or [] + } + inject_tap = record.id == TAP_CASE_ID and TAP_HAZARD["id"] not in source_object_ids _REGISTRY[capture_id] = { "record": record, "packet_id": packet_id, "packet_sha256": journey["packetSha256"], - "objects_3d": [copy.deepcopy(TAP_HAZARD)] if record.id == TAP_CASE_ID else [], + "objects_3d": [copy.deepcopy(TAP_HAZARD)] if inject_tap else [], } files = [] @@ -740,6 +770,9 @@ def _collect_case(harness: _Harness, record: case_suite.CaseRecord, journey: dic case["convertedSha256"] = _sha256(converted_bytes) if converted_bytes else None case["resultSha256"] = _sha256(deps.storage.get_bytes(placement_artifact_key)) case["solverPolicy"] = placement.get("policy") + # The solver's own decision, kept distinct from the device's served result + # kind (deviceOutcome) so the receipt shows both sides of the door. + case["solverDecision"] = placement.get("decision") # Both solver responses must bind their own input hash to their own exact # request bytes. Forging the same wrong hash in both replies leaves them in @@ -788,7 +821,9 @@ def _collect_case(harness: _Harness, record: case_suite.CaseRecord, journey: dic "direct POST /v1/placements on the captured bytes disagrees with the stage response" ) if record.wire_path is not None: - wire_mismatches = case_suite._evaluate_expectations(record.expectations, placement) + wire_mismatches = case_suite._evaluate_expectations( + record.expectations, placement, tol_ft=_WIRE_DERIVED_TOLERANCE_FT + ) case["wireSemanticComparison"] = "matched" if not wire_mismatches else "mismatched" case["wireSemanticMismatches"] = wire_mismatches if wire_mismatches: @@ -828,6 +863,7 @@ def _collect_case(harness: _Harness, record: case_suite.CaseRecord, journey: dic case["convertedSha256"] = None case["resultSha256"] = None case["solverPolicy"] = None + case["solverDecision"] = None case["directComparison"] = "not_applicable" case["wireSemanticComparison"] = "not_applicable" else: @@ -992,8 +1028,12 @@ def run_cases( "generatedAt": datetime.now(UTC).isoformat(), "codeSha": code_sha, "producers": { - "caseSuiteBranch": "obv/burst7-bs-placement-cases@ffd476ddb3a4c6f0b21683f8c353d4280f05dca5", - "sceneStageBranch": "obv/burst7-bs-capture-scene-stage@38a5735abf6c48982f8131b6dc6814afb90e28f3", + # The integrated producer SHAs on this branch (base + imports), so the + # receipt names exactly which producer code ran, not just branches. + "base": "obv/burst7-bs-capture-placement-runner@bdfbb3fc649df8187675b880ba8df83a803fd119", + "caseIntegrityFix": "115b6ce313e51cede71496f267b65b8cd78aa118 (obv/burst7-bs-placement-cases, PR #271)", + "replayAcceptance": "5f9e29576556f635757eea0605d5a23d29ce1cc5 (obv/burst8-bs-replay-acceptance, PR #275)", + "supportedCasePairs": "e7ad2abc15d94691305bdb558a752ee535cf374e (obv/burst8-bs-supported-case-pairs, PR #276)", }, "devSettings": dict(TEMPLATE_ENV), "stubbedStages": { @@ -1028,7 +1068,7 @@ def _redaction_check(receipt: dict[str, Any]) -> str | None: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Replay the placement case suite through the real capture front door.") - parser.add_argument("--manifest", type=Path, default=case_suite.DEFAULT_MANIFEST, help="Path to the case manifest.") + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST, help="Path to the case manifest.") parser.add_argument("--output", type=Path, required=True, help="Where to write the JSON receipt.") parser.add_argument("--workdir", type=Path, help="run directory (default: a fresh temp dir)") parser.add_argument("--cases", help="comma-separated case ids to run (default: every case)") diff --git a/backend/server/placement/case_suite.py b/backend/server/placement/case_suite.py index f24dda085..d8dd19614 100644 --- a/backend/server/placement/case_suite.py +++ b/backend/server/placement/case_suite.py @@ -267,7 +267,34 @@ def as_authored(path: Path) -> str: } -def _evaluate_expectations(expectations: dict[str, Any], body: dict[str, Any]) -> list[str]: +def _numbers_match(actual: Any, expected: Any, tol_ft: float) -> bool: + """Equality with a declared tolerance for derived feet values: floats and + float lists compare within ``tol_ft``, everything else exactly.""" + if isinstance(expected, list): + return ( + isinstance(actual, list) + and len(actual) == len(expected) + and all(_numbers_match(a, e, tol_ft) for a, e in zip(actual, expected, strict=True)) + ) + if ( + isinstance(expected, (int, float)) + and not isinstance(expected, bool) + and isinstance(actual, (int, float)) + and not isinstance(actual, bool) + ): + return abs(float(actual) - float(expected)) <= tol_ft + return actual == expected + + +def _evaluate_expectations( + expectations: dict[str, Any], body: dict[str, Any], tol_ft: float = 0.0 +) -> list[str]: + """Evaluate declared expectations against a placement body. + + ``tol_ft`` widens only numeric feet comparisons (``spot`` values, + ``missing_evidence`` entries, ``sweep_failing_runs`` start points) to the + declared derived-output tolerance; the default 0.0 keeps the suite exact. + """ mismatches: list[str] = [] expected_decision = expectations.get("decision") if expected_decision is not None and body.get("decision") != expected_decision: @@ -279,7 +306,7 @@ def _evaluate_expectations(expectations: dict[str, Any], body: dict[str, Any]) - mismatches.append(f"spot is None but expected {expected_spot!r}") else: for key, value in expected_spot.items(): - if spot.get(key) != value: + if not _numbers_match(spot.get(key), value, tol_ft): mismatches.append(f"spot.{key} {spot.get(key)!r} != expected {value!r}") checks_by_id = {check["id"]: check for check in body.get("checks", [])} for check_id, expected in (expectations.get("checks") or {}).items(): @@ -297,7 +324,7 @@ def _evaluate_expectations(expectations: dict[str, Any], body: dict[str, Any]) - ) for expected in expectations.get("missing_evidence") or []: found = any( - all(actual.get(key) == value for key, value in expected.items()) + all(_numbers_match(actual.get(key), value, tol_ft) for key, value in expected.items()) for actual in body.get("missing_evidence", []) ) if not found: @@ -305,7 +332,7 @@ def _evaluate_expectations(expectations: dict[str, Any], body: dict[str, Any]) - sweep = body.get("sweep", []) for expected in expectations.get("sweep_failing_runs") or []: found = any( - run.get("start_ft") == expected.get("start_ft") + _numbers_match(run.get("start_ft"), expected.get("start_ft"), tol_ft) and set(expected.get("failing") or []).issubset(run.get("failing") or []) for run in sweep ) From 3dbe72bbb3836324377e63061d718301e5c3eb44 Mon Sep 17 00:00:00 2001 From: Obvious Date: Fri, 16 Oct 2026 00:33:05 +0000 Subject: [PATCH 06/19] Add end-to-end CLI acceptance tests for the supported packet The real replay CLI runs as a subprocess from a temporary cwd with fresh work directory and state: the packet default solves both positives through real conversion/merge/placement with all receipt cross-checks, deliberately corrupted fixtures and unexpected positive refusals exit nonzero, and the legacy cases manifest stays available explicitly with its own pinned refusal state reported as blockers. --- .../test_capture_replay_cli_acceptance.py | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 backend/tests/test_capture_replay_cli_acceptance.py diff --git a/backend/tests/test_capture_replay_cli_acceptance.py b/backend/tests/test_capture_replay_cli_acceptance.py new file mode 100644 index 000000000..1705c30aa --- /dev/null +++ b/backend/tests/test_capture_replay_cli_acceptance.py @@ -0,0 +1,173 @@ +"""End-to-end acceptance of the replay CLI against the supported packet. + +Runs the actual ``scripts/capture_placement_cases.py`` CLI as a subprocess from a +temporary cwd, with a fresh work directory and state, over the default supported +``capture_cases_v1`` manifest: both positive captures solve via the real +conversion, merge and placement path, every declared negative refuses with no +placement artifact, and the receipt's cross-checks all hold. A deliberately +corrupted fixture and an unexpected positive refusal both exit nonzero; the +legacy W7 cases manifest stays available explicitly and keeps its own pinned +refusal state reported as blockers, never mistaken for acceptance. +""" + +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +BACKEND_ROOT = Path(__file__).resolve().parents[1] +RUNNER = BACKEND_ROOT / "scripts" / "capture_placement_cases.py" +PACKET_DIR = BACKEND_ROOT / "server/placement/tests/data/capture_cases_v1" +LEGACY_MANIFEST = BACKEND_ROOT / "server/placement/tests/data/cases/manifest.json" + +POSITIVES = ("hazard-right-ground-gap", "hazard-left-ground-gap") +NEGATIVES = ( + "multiwall-source-unsupported", + "generated-object-evidence-unsupported", + "standoff-meter-anchor-unsupported", + "footprint-hazard-unsupported", + "generated-regions-unsupported", +) + + +def _run(root: Path, *extra: str) -> subprocess.CompletedProcess: + """The real CLI, as a subprocess, from a temporary cwd with fresh state.""" + return subprocess.run( + [ + sys.executable, + str(RUNNER), + "--output", + str(root / "receipt.json"), + "--workdir", + str(root / "work"), + *extra, + ], + cwd=root, + capture_output=True, + text=True, + timeout=600, + ) + + +def _receipt(root: Path) -> dict: + receipt = json.loads((root / "receipt.json").read_text()) + assert receipt["ok"] is True, receipt["blockers"] + return receipt + + +def test_the_supported_packet_solves_end_to_end_through_the_real_cli(tmp_path: Path) -> None: + result = _run(tmp_path) + assert result.returncode == 0, (result.stdout + result.stderr)[-3000:] + receipt = _receipt(tmp_path) + + summary = receipt["summary"] + assert summary == {"cases": 7, "solved": 2, "refused": 5, "refusedDeclared": 5, "mismatched": 0, "errored": 0} + assert len(receipt["codeSha"]) == 40 + + by_id = {case["id"]: case for case in receipt["cases"]} + for positive in POSITIVES: + case = by_id[positive] + assert case["status"] == "solved" + assert case["mismatches"] == [] + # Real conversion against the independently authored wire, field by field. + assert case["geometryComparison"] == "matched" + # Input-hash binding on both solver paths, direct endpoint parity. + assert case["stageInputShaMatchesCapturedBytes"] is True + assert case["directInputShaMatchesPostedBytes"] is True + assert case["directComparison"] == "matched" + assert case["wireSemanticComparison"] == "matched" + # Device and solver decisions, in distinct fields, both conservative. + assert case["solverDecision"] == "manual_review" + assert case["solverPolicy"]["id"] == "demo" + assert case["solverPolicy"]["auto_approve"] is True + assert case["deviceOutcome"]["captureStatus"] == "manual_review" + assert case["deviceOutcome"]["servedResultKind"] == "manual_review" + assert case["deviceOutcome"]["servedResultRunId"] + assert case["terminalEventCount"] == 1 + assert case["hasPlacementArtifact"] is True + for key in ("sourceSha256", "convertedSha256", "resultSha256"): + assert len(case[key]) == 64 + # One-time tap replacement: the tap hazard survives the real merge and the + # real conversion exactly once. + tap = by_id["hazard-right-ground-gap"] + assert tap["tapHazardMergeCount"] == 1 + assert tap["tapHazardConvertedCount"] == 1 + + for negative in NEGATIVES: + case = by_id[negative] + assert case["status"] == "refused" + assert case["hasPlacementArtifact"] is False + refusal = case["refusal"] + assert refusal["declared"] is True + assert refusal["codeMatchesDeclaration"] is True + assert refusal["pointerMatchesDeclaration"] is True + assert case["deviceOutcome"]["servedResultKind"] == "manual_review" + assert case["terminalEventCount"] == 1 + + cross = receipt["crossCapture"] + assert cross["distinctScenes"] is True + assert cross["distinctNormalizedGeometry"] is True + assert receipt["finalizeReplay"]["ok"] is True + + producers = receipt["producers"] + for producer in ("base", "caseIntegrityFix", "replayAcceptance", "supportedCasePairs"): + assert producers[producer] + + +def test_the_replay_cli_exits_nonzero_after_deliberate_fixture_corruption(tmp_path: Path) -> None: + packet = tmp_path / "packet" + shutil.copytree(PACKET_DIR, packet) + source = packet / "hazard-left-ground-gap.source.json" + document = json.loads(source.read_text()) + document["objects"][0]["plus_minus_m"] = 0.5 + source.write_text(json.dumps(document, indent=2) + "\n") + # The manifest's pinned sha256 is deliberately left behind: the drift is the + # corruption, and the run must refuse to build on drifted evidence. + + result = _run(tmp_path, "--manifest", str(packet / "manifest.json")) + assert result.returncode != 0 + output = result.stdout + result.stderr + assert "drifted fixture" in output and "!= pinned" in output + + +def test_the_replay_cli_exits_nonzero_when_a_positive_unexpectedly_refuses(tmp_path: Path) -> None: + packet = tmp_path / "packet" + shutil.copytree(PACKET_DIR, packet) + source = packet / "hazard-left-ground-gap.source.json" + document = json.loads(source.read_text()) + # The W7 refusal shape: a stand-off meter anchor the bridge does not support. + document["objects"][0]["geometry"]["standoff_m"] = 0.1 + source.write_text(json.dumps(document, indent=2) + "\n") + manifest_path = packet / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + pinned_sha = hashlib.sha256(source.read_bytes()).hexdigest() + for case in manifest["cases"]: + if case["id"] == "hazard-left-ground-gap": + case["source"]["sha256"] = pinned_sha + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + result = _run(tmp_path, "--manifest", str(manifest_path)) + assert result.returncode != 0 + receipt = json.loads((tmp_path / "receipt.json").read_text()) + assert receipt["ok"] is False + assert any("hazard-left-ground-gap" in blocker and "refused" in blocker for blocker in receipt["blockers"]) + + +def test_the_legacy_cases_manifest_stays_available_explicitly(tmp_path: Path) -> None: + result = _run(tmp_path, "--manifest", str(LEGACY_MANIFEST)) + assert result.returncode != 0 + receipt = json.loads((tmp_path / "receipt.json").read_text()) + # The legacy manifest's pinned refusal state is reported as blockers — the + # original limitation, not an acceptance criterion. + assert receipt["ok"] is False + assert receipt["summary"]["solved"] == 0 + assert receipt["blockers"] + + +@pytest.mark.parametrize("path", [RUNNER, PACKET_DIR / "manifest.json", LEGACY_MANIFEST]) +def test_the_replay_targets_exist(path: Path) -> None: + assert path.exists() From ba7eba2228adf50cecbca5a34be89ec13e7849ae Mon Sep 17 00:00:00 2001 From: Obvious Date: Fri, 16 Oct 2026 00:41:38 +0000 Subject: [PATCH 07/19] Keep legacy-manifest CLI tests on the explicit legacy manifest The replay CLI's default manifest is now the supported packet, so the two producer tests pinning the committed legacy refusal state pass that manifest explicitly; their intent (committed legacy fixtures stay bridge-incompatible, nonzero exit) is unchanged. Also reformat test_placement_case_suite.py, unformatted since the #271 integrity-fix import (bad040cc), so the full lint battery is clean. --- backend/scripts/capture_placement_cases.py | 9 ++------- backend/server/placement/case_suite.py | 11 +++++------ backend/tests/test_capture_placement_cases.py | 15 ++++++++++++++- backend/tests/test_placement_case_suite.py | 5 +---- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/backend/scripts/capture_placement_cases.py b/backend/scripts/capture_placement_cases.py index 46f2be36b..e83ef5525 100644 --- a/backend/scripts/capture_placement_cases.py +++ b/backend/scripts/capture_placement_cases.py @@ -143,10 +143,7 @@ # The supported case packet is the default replay target; the legacy W7 cases # manifest stays available explicitly via --manifest and keeps its own pinned # refusal state, reported as blockers rather than mistaken for acceptance. -DEFAULT_MANIFEST = ( - Path(__file__).resolve().parent.parent - / "server/placement/tests/data/capture_cases_v1/manifest.json" -) +DEFAULT_MANIFEST = Path(__file__).resolve().parent.parent / "server/placement/tests/data/capture_cases_v1/manifest.json" # An observed tap hazard, hand-authored for this runner: a wall_rect vent on the # case's wall, away from the case's pinned decision geometry. It exists to prove @@ -373,9 +370,7 @@ def _walk_front_door( # duplicates the id and the merged scene is refused as inconsistent. A # source-carried hazard still reaches the one-time tap checks through the real # merge and real conversion below. - source_object_ids = { - obj.get("id") for obj in json.loads(record.source_path.read_text()).get("objects") or [] - } + source_object_ids = {obj.get("id") for obj in json.loads(record.source_path.read_text()).get("objects") or []} inject_tap = record.id == TAP_CASE_ID and TAP_HAZARD["id"] not in source_object_ids _REGISTRY[capture_id] = { "record": record, diff --git a/backend/server/placement/case_suite.py b/backend/server/placement/case_suite.py index d8dd19614..ebd7af6dc 100644 --- a/backend/server/placement/case_suite.py +++ b/backend/server/placement/case_suite.py @@ -128,8 +128,10 @@ def _expectations_problem(expectations: Any) -> str | None: return "must be an object of pinned expectations" for key in EXPECTATION_KEYS: if key in expectations and not isinstance(expectations[key], EXPECTATION_SHAPES[key]): - shape = "a string" if EXPECTATION_SHAPES[key] is str else ( - "an object" if EXPECTATION_SHAPES[key] is dict else "an array" + shape = ( + "a string" + if EXPECTATION_SHAPES[key] is str + else ("an object" if EXPECTATION_SHAPES[key] is dict else "an array") ) return f".{key} must be {shape} (got {type(expectations[key]).__name__})" if not isinstance(expectations.get("decision"), str) or not expectations.get("decision"): @@ -286,9 +288,7 @@ def _numbers_match(actual: Any, expected: Any, tol_ft: float) -> bool: return actual == expected -def _evaluate_expectations( - expectations: dict[str, Any], body: dict[str, Any], tol_ft: float = 0.0 -) -> list[str]: +def _evaluate_expectations(expectations: dict[str, Any], body: dict[str, Any], tol_ft: float = 0.0) -> list[str]: """Evaluate declared expectations against a placement body. ``tol_ft`` widens only numeric feet comparisons (``spot`` values, @@ -554,6 +554,5 @@ def main(argv: list[str] | None = None) -> int: return 0 if report["ok"] else 1 - if __name__ == "__main__": raise SystemExit(main()) diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index e03937c23..65f5435b6 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -703,7 +703,18 @@ def test_cli_runs_from_a_temporary_directory_and_exits_nonzero_while_blocked(tmp """The one-command entry point: receipt on disk, nonzero exit while the committed fixtures stay bridge-incompatible, no secrets in the file.""" output = tmp_path / "receipt.json" - exit_code = runner.main(["--output", str(output), "--workdir", str(tmp_path / "run")]) + # The committed legacy manifest, passed explicitly since the CLI's default + # moved to the supported packet (its refusal state stays pinned here). + exit_code = runner.main( + [ + "--manifest", + str(runner.case_suite.DEFAULT_MANIFEST), + "--output", + str(output), + "--workdir", + str(tmp_path / "run"), + ] + ) assert exit_code == 1 receipt = json.loads(output.read_text()) assert receipt["schemaVersion"] == runner.RECEIPT_SCHEMA_VERSION @@ -793,6 +804,8 @@ def test_cli_on_the_committed_manifest_exits_nonzero_in_a_subprocess(tmp_path): [ sys.executable, str(BACKEND / "scripts" / "capture_placement_cases.py"), + "--manifest", + str(runner.case_suite.DEFAULT_MANIFEST), "--output", str(output), "--workdir", diff --git a/backend/tests/test_placement_case_suite.py b/backend/tests/test_placement_case_suite.py index a8fcf2b4e..e9e3ec33b 100644 --- a/backend/tests/test_placement_case_suite.py +++ b/backend/tests/test_placement_case_suite.py @@ -189,7 +189,6 @@ def test_manifest_copy_is_untouched_by_mutations() -> None: assert json.loads(MANIFEST.read_text()) == original - def test_missing_cases_key_fails_with_manifest_path(tmp_path: Path) -> None: """A manifest without a cases key cannot silently disable the gate.""" payload = _absolutized_payload() @@ -248,9 +247,7 @@ def test_malformed_case_kinds_fail_with_field(tmp_path: Path, kind: object) -> N ], ids=["absent", "empty", "decision-only", "spot-wrong-type", "checks-wrong-type"], ) -def test_unusable_positive_expectations_fail_with_field( - tmp_path: Path, mangle, why: str -) -> None: +def test_unusable_positive_expectations_fail_with_field(tmp_path: Path, mangle, why: str) -> None: payload = _absolutized_payload() mangle(payload["cases"][0]) drift = tmp_path / "manifest.json" From 596ed9a9bf1e8e59d31b9d2abf5c00087707ce5e Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:18:53 +0000 Subject: [PATCH 08/19] fix(backend): import the #258/#262 receipt fixes; a crashed run or replay can never report ok Scoped imports of two completed upstream review fixes, with their regression tests, onto the burst8 integration branch: - #258 at da0b638 (capture_flow_e2e.py): a run that crashes before any stage code exists reported ok:true beside its errorDetail and exited 0. The except branch now records runner_exception and _finish treats any recorded detail as failure (ok = not errors and not detail). Tests: rerun into the receipt's recorded workdir and a missing --packet both exit 1 with ok:false. - #262 at e2aa8a24 (replay_capture.py): run_replay's except branch stored errorDetail but appended no error code, and a pre-replay receipt without the replay block passed the fixture check vacuously and died on KeyError with a green report. The replay now validates the replay block up front with a named missing_replay_block error, appends exception: on any crash, flags receipt paths outside the committed manifest as fixture drift, and drops the dead _content_type. The later receipt metadata this branch already carries (idempotencyKeys/files/packetSha256 emission) is preserved. Evidence: all five imported tests fail against the pre-fix scripts (5 failed) and pass with them (14 passed across both suites). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- backend/scripts/capture_flow_e2e.py | 8 +- backend/scripts/replay_capture.py | 49 +++++++----- backend/tests/test_capture_flow_e2e_runner.py | 27 +++++++ backend/tests/test_replay_capture.py | 79 +++++++++++++++++++ 4 files changed, 143 insertions(+), 20 deletions(-) diff --git a/backend/scripts/capture_flow_e2e.py b/backend/scripts/capture_flow_e2e.py index 3240b1bdd..136962187 100644 --- a/backend/scripts/capture_flow_e2e.py +++ b/backend/scripts/capture_flow_e2e.py @@ -118,7 +118,8 @@ def run_capture_flow( expect_status: str = "processing", ) -> dict: """Run the journey and return the receipt dict. Never raises: failures land - in ``receipt["errorCodes"]`` with ``receipt["ok"] is False``.""" + in ``receipt["errorCodes"]`` with ``receipt["ok"] is False`` — including an + exception before any stage, which becomes the ``runner_exception`` code.""" started_at = datetime.now(UTC).isoformat() packet_dir = packet_dir or DEFAULT_PACKET receipt: dict = { @@ -296,6 +297,7 @@ def record(stage: str, **fields) -> None: if expect_status == "processing" and not validate_ok: errors.append("events_missing_validate_stage") except Exception as exc: # noqa: BLE001 - the receipt must survive any failure + errors.append("runner_exception") return _finish(receipt, errors, f"exception: {type(exc).__name__}: {exc}") return _finish(receipt, errors, None) @@ -304,7 +306,9 @@ def _finish(receipt: dict, errors: list[str], detail: str | None) -> dict: if detail: receipt["errorDetail"] = detail receipt["finishedAt"] = datetime.now(UTC).isoformat() - receipt["ok"] = not errors + # A recorded detail without an error code is still a failure: a run that + # crashed before any stage could record a code must never report ok. + receipt["ok"] = not errors and not detail return receipt diff --git a/backend/scripts/replay_capture.py b/backend/scripts/replay_capture.py index 0b81dc9a5..65fae0dc1 100644 --- a/backend/scripts/replay_capture.py +++ b/backend/scripts/replay_capture.py @@ -101,11 +101,20 @@ def fire(self, step: str) -> None: raise SimulatedTimeout(step) -def load_receipt(path: Path) -> dict: - receipt = json.loads(path.read_text()) +def receipt_problem(receipt: object) -> str | None: + """A named reason the saved receipt cannot be replayed, or None. + + The replay block the runner writes (files, idempotencyKeys, packetSha256) + is the replay's ground truth; a receipt without it was written by a + pre-replay runner and must be rejected, never replayed with guessed + hashes. + """ if not isinstance(receipt, dict) or receipt.get("runner") != "capture_flow_e2e": - raise SystemExit(f"{path} is not a capture_flow_e2e receipt") - return receipt + return "not_a_capture_flow_e2e_receipt" + missing = [key for key in ("files", "idempotencyKeys", "packetSha256") if not receipt.get(key)] + if missing: + return f"missing_replay_block:{','.join(missing)}" + return None def check_fixture(packet_dir: Path, receipt: dict, errors: list[str]) -> dict | None: @@ -126,6 +135,10 @@ def check_fixture(packet_dir: Path, receipt: dict, errors: list[str]) -> dict | drift.append(path) if entry.get("sha256") and entry["sha256"] != digest: drift.append(f"{path}:packet-manifest") + manifest_paths = {entry["path"] for entry in packet["files"]} + for path in saved_hashes: + if path not in manifest_paths: + drift.append(f"{path}:not-in-manifest") saved_sha = receipt.get("packetSha256") actual_sha = hashlib.sha256(packet_json.read_bytes()).hexdigest() if saved_sha and saved_sha != actual_sha: @@ -135,14 +148,6 @@ def check_fixture(packet_dir: Path, receipt: dict, errors: list[str]) -> dict | return {"packet": packet, "packetSha256": actual_sha, "drift": drift} -def _content_type(role: str) -> str: - if role == "stream": - return "application/gzip" - if role in {"features", "depth", "confidence", "mesh"}: - return "application/octet-stream" - return "image/jpeg" - - class _ReplayApp: """The real app under replay: dev handoff on, everything else local.""" @@ -215,19 +220,17 @@ def run_replay( faults: dict[str, int] | None = None, ) -> dict: """Replay the saved receipt and return the report. Never raises: failures - land in ``report["errorCodes"]`` with ``report["ok"] is False``.""" + land in ``report["errorCodes"]`` with ``report["ok"] is False`` — including + receipt problems and any unexpected exception.""" started_at = datetime.now(UTC).isoformat() - receipt = load_receipt(receipt_path) - saved_dir = receipt.get("packetDir") - packet_dir = packet_dir or (Path(saved_dir) if saved_dir and Path(saved_dir).is_dir() else DEFAULT_PACKET) report: dict = { "replay": "replay_capture", "startedAt": started_at, "receiptPath": str(receipt_path), - "packetDir": str(packet_dir), + "packetDir": None, "workdir": str(workdir), "faults": dict(faults or {}), - "idempotencyKeys": receipt.get("idempotencyKeys") or {}, + "idempotencyKeys": {}, "captureId": None, "runId": None, "steps": [], @@ -246,6 +249,15 @@ def record(step: str, **fields) -> None: app: _ReplayApp | None = None try: + receipt = json.loads(Path(receipt_path).read_text()) + problem = receipt_problem(receipt) + if problem: + errors.append(problem) + return _finish(report, errors, f"{receipt_path}: {problem} — written by a runner without the replay block") + saved_dir = receipt.get("packetDir") + packet_dir = packet_dir or (Path(saved_dir) if saved_dir and Path(saved_dir).is_dir() else DEFAULT_PACKET) + report["packetDir"] = str(packet_dir) + report["idempotencyKeys"] = receipt.get("idempotencyKeys") or {} fixture = check_fixture(packet_dir, receipt, errors) record( "fixture", files=len((fixture or {}).get("packet", {}).get("files", [])), drift=(fixture or {}).get("drift") @@ -477,6 +489,7 @@ def _finalize_again(): if terminals != ["verdict_ready"]: errors.append(f"terminal_events:{terminals}") except Exception as exc: # noqa: BLE001 - the report must survive any failure + errors.append(f"exception:{type(exc).__name__}") return _finish(report, errors, f"exception: {type(exc).__name__}: {exc}") finally: if app is not None: diff --git a/backend/tests/test_capture_flow_e2e_runner.py b/backend/tests/test_capture_flow_e2e_runner.py index 830752963..10ef1bab3 100644 --- a/backend/tests/test_capture_flow_e2e_runner.py +++ b/backend/tests/test_capture_flow_e2e_runner.py @@ -93,3 +93,30 @@ def test_cli_runs_from_a_clean_directory_and_exits_zero(runner, tmp_path) -> Non assert completed.returncode == 0, completed.stdout + completed.stderr receipt = _read_receipt(workdir / "receipt.json") assert receipt["ok"] is True and receipt["finalCaptureStatus"] == "processing" + + +def test_rerun_into_a_used_workdir_fails_closed(runner, tmp_path) -> None: + """The replay action the receipt invites — rerun with its recorded --workdir — + hits the already-present packet copy, so the crash must exit 1, not report + success with the errorDetail tucked beside ok:true.""" + used = tmp_path / "replay" + assert runner.run_capture_flow(used)["ok"] is True + + receipt = runner.run_capture_flow(used) + assert receipt["ok"] is False + assert "runner_exception" in receipt["errorCodes"] + assert receipt["errorDetail"] + + code = runner.main(["--workdir", str(used)]) + assert code == 1 + stored = _read_receipt(used / "receipt.json") + assert stored["ok"] is False and "runner_exception" in stored["errorCodes"] + + +def test_missing_packet_dir_fails_closed(runner, tmp_path) -> None: + code = runner.main(["--packet", str(tmp_path / "no-such-packet"), "--workdir", str(tmp_path / "w")]) + assert code == 1 + stored = _read_receipt(tmp_path / "w" / "receipt.json") + assert stored["ok"] is False + assert "runner_exception" in stored["errorCodes"] + assert stored["errorDetail"] diff --git a/backend/tests/test_replay_capture.py b/backend/tests/test_replay_capture.py index 9120bdeaf..15f9e3360 100644 --- a/backend/tests/test_replay_capture.py +++ b/backend/tests/test_replay_capture.py @@ -157,3 +157,82 @@ def test_cli_replays_a_saved_receipt_and_exits_zero(replay, saved_receipt, tmp_p stored = json.loads((tmp_path / "cli" / "replay-receipt.json").read_text()) assert stored["ok"] is True assert stored["finalCaptureStatus"] in replay.TERMINAL_STATUSES + + +def test_replay_rejects_a_pre_replay_receipt_missing_the_replay_block(replay, saved_receipt, tmp_path) -> None: + """A receipt written before the replay block exists must fail loudly. + + The runner's replay block (files, idempotencyKeys, packetSha256) is what + makes the receipt replayable; a pre-PR receipt has none of it and must be + rejected with a named error, not replayed with guessed hashes — and the + CLI must exit non-zero instead of reporting a green run. + """ + receipt = json.loads(saved_receipt.read_text()) + for key in ("files", "idempotencyKeys", "packetSha256"): + receipt.pop(key, None) + stripped = tmp_path / "stripped" / "receipt.json" + stripped.parent.mkdir(parents=True) + stripped.write_text(json.dumps(receipt)) + + report = replay.run_replay(stripped, tmp_path / "replay") + + assert report["ok"] is False, report + assert any(code.startswith("missing_replay_block") for code in report["errorCodes"]), report["errorCodes"] + assert report["captureId"] is None + + env = {**os.environ, "PYTHONPATH": str(ROOT)} + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "replay_capture.py"), + "--receipt", + str(stripped), + "--workdir", + str(tmp_path / "cli"), + ], + capture_output=True, + text=True, + env=env, + ) + assert completed.returncode == 1, completed.stdout + completed.stderr + stored = json.loads((tmp_path / "cli" / "replay-receipt.json").read_text()) + assert stored["ok"] is False + + +def test_a_crash_mid_replay_reports_ok_false(replay, saved_receipt, tmp_path) -> None: + """No exception path may leave the report green: a crash is a failure. + + The receipt's files block is malformed (an entry without path/sha256), so + the replay dies before the first error code would be appended. The report + must still come back ok=False with an exception error code — and the CLI + must exit 1. + """ + receipt = json.loads(saved_receipt.read_text()) + receipt["files"] = [{"path": "x.bin", "sha256": "0" * 64}, {"broken": True}] + broken = tmp_path / "broken" / "receipt.json" + broken.parent.mkdir(parents=True) + broken.write_text(json.dumps(receipt)) + + report = replay.run_replay(broken, tmp_path / "replay") + + assert report["ok"] is False, report + assert any(code.startswith("exception:") for code in report["errorCodes"]), report["errorCodes"] + assert report["captureId"] is None + + +def test_fixture_check_flags_receipt_files_outside_the_manifest(replay) -> None: + """A receipt claiming files the packet fixture never had is drift. + + The old comparison only looked at paths present in both sets, so an empty + or disjoint receipt files list passed vacuously. Every receipt path must + exist in the committed manifest. + """ + errors: list[str] = [] + + fixture = replay.check_fixture( + replay.DEFAULT_PACKET, {"files": [{"path": "not/in/this-packet.bin", "sha256": "0" * 64}]}, errors + ) + + assert fixture is not None and fixture["drift"], fixture + assert errors and errors[0].startswith("fixture_drift:"), errors + assert "not/in/this-packet.bin:not-in-manifest" in fixture["drift"] From 54a5a5b07f4a7f1edc06fc97993d2120038728c7 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:21:44 +0000 Subject: [PATCH 09/19] fix(backend): restore the run_cases dev env; the flags must not leak into the pytest process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scoped port of the #274 review fix at c255a235 (the one piece the burst8 integrated copy lacked): _apply_dev_env returns a restore callable and run_cases restores the prior environment in a finally block, so in-process runs no longer leak STATE_BACKEND/STORAGE_BACKEND/VOLUME_ROOT/UPLOAD_URL_HMAC_SECRET into the rest of the pytest process; a test pins the full environ equality across a run. The rest of c255a235 (replay and cross-capture blockers, identity-normalized geometry hashing, the mutation tests) is already present in this branch's stronger #275-integrated form — distinctScenes AND distinctNormalizedGeometry AND distinct captureIds, applicable-aware replay blocking, full-run positive solve requirements — and is retained unchanged; the supported default manifest is not weakened. Evidence: the new test fails against the pre-port script (env leaked) and passes after; the placement-cases/pairs/suite suites pass (60 passed). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- backend/scripts/capture_placement_cases.py | 306 ++++++++++-------- backend/tests/test_capture_placement_cases.py | 9 + 2 files changed, 175 insertions(+), 140 deletions(-) diff --git a/backend/scripts/capture_placement_cases.py b/backend/scripts/capture_placement_cases.py index e83ef5525..4a5d5a971 100644 --- a/backend/scripts/capture_placement_cases.py +++ b/backend/scripts/capture_placement_cases.py @@ -94,6 +94,7 @@ import subprocess import sys import tempfile +from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -169,19 +170,35 @@ def _sha256(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def _apply_dev_env(volume_root: Path) -> None: +def _apply_dev_env(volume_root: Path) -> Callable[[], None]: """The finalize handoff builds its service template from process env, so the dev flags must be environment, not only Settings fields — including the volume settings `services.current()` validates before the app's own state and - storage are bound in.""" - for key, value in { + storage are bound in. + + Returns a callable restoring the prior environment: run_cases runs in-process + inside the test suite, and leaked dev flags would make any later test that + asserts env-derived defaults order-dependent. + """ + applied = { **TEMPLATE_ENV, "VOLUME_ROOT": str(volume_root), "CAPTURE_API_PUBLIC_URL": "http://testserver", "UPLOAD_URL_HMAC_SECRET": "h" * 32, - }.items(): + } + saved: dict[str, str | None] = {key: os.environ.get(key) for key in applied} + for key, value in applied.items(): os.environ[key] = value + def _restore() -> None: + for key, value in saved.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + return _restore + def _stamped_scene( record: case_suite.CaseRecord, *, packet_id: str, packet_sha: str, capture_id: str, run_id: str @@ -909,148 +926,157 @@ def run_cases( case_ids: list[str] | None = None, ) -> dict[str, Any]: """Run the manifest's cases through the front door and return the receipt.""" - _apply_dev_env(workdir / "vol") - workdir.mkdir(parents=True, exist_ok=True) - if case_ids is not None and not case_ids: - raise SystemExit("empty case selection: name at least one case id, or run every case") - records = case_suite.load_cases(manifest_path) - full_run = case_ids is None - if case_ids: - wanted = set(case_ids) - records = [record for record in records if record.id in wanted] - if len(records) != len(wanted): - raise SystemExit(f"unknown case ids: {sorted(wanted - {record.id for record in records})}") + restore_env = _apply_dev_env(workdir / "vol") try: - code_sha = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=Path(__file__).resolve().parents[2], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - except (OSError, subprocess.CalledProcessError): - code_sha = "unknown" - - harness = _Harness(workdir / "app") - _install_boundaries() - try: - journeys = [] - for index, record in enumerate(records): - journeys.append(_walk_front_door(harness, record, packet_id=f"placement-case-{index + 1:02d}")) - harness.drain() - cases = [_collect_case(harness, record, journey) for record, journey in zip(records, journeys, strict=True)] - if journeys and journeys[0].get("captureId") is not None: - replay = _replay_finalize(harness, journeys[0]) - replay["applicable"] = True - elif journeys: - replay = { - "applicable": False, - "status": "not_applicable", - "reason": "the first journey never created a capture", - "ok": None, - } - else: - replay = {"applicable": False, "status": "not_applicable", "reason": "no case ran", "ok": None} - finally: - _remove_boundaries() - - # Cross-capture: distinct captures in one app must carry distinct scenes — and - # distinct *normalized* geometry, not merely distinct identity stamps or - # stamped scene hashes. Only the two positive cases are compared. - positive_cases = [case for case in cases if case["kind"] == case_suite.KIND_POSITIVE] - pair = positive_cases[:2] - raw_hashes = [case.get("mergedSceneSha256") for case in pair] - geometry_hashes = [case.get("mergedGeometrySha256") for case in pair] - capture_ids = [case.get("captureId") for case in pair] - applicable = len(pair) == 2 and all(raw_hashes) and all(geometry_hashes) and all(capture_ids) - cross_capture = { - "applicable": applicable, - "status": "evaluated" if applicable else "not_applicable", - "reason": None - if applicable - else "not_applicable: this run does not include two positive cases that each carried a merged scene", - "captureIds": capture_ids, - "distinctScenes": (raw_hashes[0] != raw_hashes[1]) if applicable else None, - "distinctNormalizedGeometry": (geometry_hashes[0] != geometry_hashes[1]) if applicable else None, - "mergedGeometrySha256": dict(zip([case["id"] for case in pair], geometry_hashes, strict=False)) if pair else {}, - } - - blockers = [] - for case in cases: - if case["status"] == "refused" and not case.get("refusal", {}).get("declared"): - blockers.append( - f"{case['id']}: refused at {case['refusal']['code']} — the manifest declares no expected error " - f"({case['refusal']['message'][:160]})" - ) - for key in ("codeMatchesDeclaration", "pointerMatchesDeclaration"): - if case.get("refusal", {}).get(key) is False: + workdir.mkdir(parents=True, exist_ok=True) + if case_ids is not None and not case_ids: + raise SystemExit("empty case selection: name at least one case id, or run every case") + records = case_suite.load_cases(manifest_path) + full_run = case_ids is None + if case_ids: + wanted = set(case_ids) + records = [record for record in records if record.id in wanted] + if len(records) != len(wanted): + raise SystemExit(f"unknown case ids: {sorted(wanted - {record.id for record in records})}") + try: + code_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + code_sha = "unknown" + + harness = _Harness(workdir / "app") + _install_boundaries() + try: + journeys = [] + for index, record in enumerate(records): + journeys.append(_walk_front_door(harness, record, packet_id=f"placement-case-{index + 1:02d}")) + harness.drain() + cases = [_collect_case(harness, record, journey) for record, journey in zip(records, journeys, strict=True)] + if journeys and journeys[0].get("captureId") is not None: + replay = _replay_finalize(harness, journeys[0]) + replay["applicable"] = True + elif journeys: + replay = { + "applicable": False, + "status": "not_applicable", + "reason": "the first journey never created a capture", + "ok": None, + } + else: + replay = {"applicable": False, "status": "not_applicable", "reason": "no case ran", "ok": None} + finally: + _remove_boundaries() + + # Cross-capture: distinct captures in one app must carry distinct scenes — and + # distinct *normalized* geometry, not merely distinct identity stamps or + # stamped scene hashes. Only the two positive cases are compared. + positive_cases = [case for case in cases if case["kind"] == case_suite.KIND_POSITIVE] + pair = positive_cases[:2] + raw_hashes = [case.get("mergedSceneSha256") for case in pair] + geometry_hashes = [case.get("mergedGeometrySha256") for case in pair] + capture_ids = [case.get("captureId") for case in pair] + applicable = len(pair) == 2 and all(raw_hashes) and all(geometry_hashes) and all(capture_ids) + cross_capture = { + "applicable": applicable, + "status": "evaluated" if applicable else "not_applicable", + "reason": None + if applicable + else "not_applicable: this run does not include two positive cases that each carried a merged scene", + "captureIds": capture_ids, + "distinctScenes": (raw_hashes[0] != raw_hashes[1]) if applicable else None, + "distinctNormalizedGeometry": (geometry_hashes[0] != geometry_hashes[1]) if applicable else None, + "mergedGeometrySha256": dict(zip([case["id"] for case in pair], geometry_hashes, strict=False)) + if pair + else {}, + } + + blockers = [] + for case in cases: + if case["status"] == "refused" and not case.get("refusal", {}).get("declared"): blockers.append( - f"{case['id']}: refusal {key.replace('MatchesDeclaration', '')} does not match the declaration" + f"{case['id']}: refused at {case['refusal']['code']} — the manifest declares no expected error " + f"({case['refusal']['message'][:160]})" ) - if case["mismatches"]: - blockers.append(f"{case['id']}: " + "; ".join(case["mismatches"])) - if case["errors"]: - blockers.append(f"{case['id']}: " + "; ".join(case["errors"])) - if cross_capture["applicable"]: - if not ( - cross_capture["distinctScenes"] - and cross_capture["distinctNormalizedGeometry"] - and len(set(cross_capture["captureIds"])) == 2 - ): - blockers.append( - "crossCapture: the two positive cases do not carry distinct capture ids with distinct normalized " - f"geometry (distinctScenes={cross_capture['distinctScenes']}, " - f"distinctNormalizedGeometry={cross_capture['distinctNormalizedGeometry']})" - ) - if replay is not None and replay.get("applicable", True) and replay.get("ok") is not True: - blockers.append( - "finalizeReplay: the finalize replay is not an idempotent success (HTTP 202, same run id, one terminal " - f"event); got {json.dumps(replay)}" - ) - if full_run: - if len(positive_cases) < 2: - blockers.append(f"full run requires both positive cases; this manifest ran {len(positive_cases)}") - unsolved = [case["id"] for case in positive_cases if case["status"] != "solved"] - if unsolved: + for key in ("codeMatchesDeclaration", "pointerMatchesDeclaration"): + if case.get("refusal", {}).get(key) is False: + blockers.append( + f"{case['id']}: refusal {key.replace('MatchesDeclaration', '')} does not match the declaration" + ) + if case["mismatches"]: + blockers.append(f"{case['id']}: " + "; ".join(case["mismatches"])) + if case["errors"]: + blockers.append(f"{case['id']}: " + "; ".join(case["errors"])) + if cross_capture["applicable"]: + if not ( + cross_capture["distinctScenes"] + and cross_capture["distinctNormalizedGeometry"] + and len(set(cross_capture["captureIds"])) == 2 + ): + blockers.append( + "crossCapture: the two positive cases do not carry distinct capture ids with distinct normalized " + f"geometry (distinctScenes={cross_capture['distinctScenes']}, " + f"distinctNormalizedGeometry={cross_capture['distinctNormalizedGeometry']})" + ) + if replay is not None and replay.get("applicable", True) and replay.get("ok") is not True: blockers.append( - "full run requires both positive cases to solve and publish placement artifacts; not solved: " - + ", ".join(unsolved) + "finalizeReplay: the finalize replay is not an idempotent success (HTTP 202, same run id, one terminal " + f"event); got {json.dumps(replay)}" ) + if full_run: + if len(positive_cases) < 2: + blockers.append(f"full run requires both positive cases; this manifest ran {len(positive_cases)}") + unsolved = [case["id"] for case in positive_cases if case["status"] != "solved"] + if unsolved: + blockers.append( + "full run requires both positive cases to solve and publish placement artifacts; not solved: " + + ", ".join(unsolved) + ) - statuses = [case["status"] for case in cases] - receipt: dict[str, Any] = { - "schemaVersion": RECEIPT_SCHEMA_VERSION, - "generatedAt": datetime.now(UTC).isoformat(), - "codeSha": code_sha, - "producers": { - # The integrated producer SHAs on this branch (base + imports), so the - # receipt names exactly which producer code ran, not just branches. - "base": "obv/burst7-bs-capture-placement-runner@bdfbb3fc649df8187675b880ba8df83a803fd119", - "caseIntegrityFix": "115b6ce313e51cede71496f267b65b8cd78aa118 (obv/burst7-bs-placement-cases, PR #271)", - "replayAcceptance": "5f9e29576556f635757eea0605d5a23d29ce1cc5 (obv/burst8-bs-replay-acceptance, PR #275)", - "supportedCasePairs": "e7ad2abc15d94691305bdb558a752ee535cf374e (obv/burst8-bs-supported-case-pairs, PR #276)", - }, - "devSettings": dict(TEMPLATE_ENV), - "stubbedStages": { - name: f"server.orchestrator.stubs.{STAGES[name].module}" for name in STAGES if not STAGES[name].stubs_exempt - }, - "realStages": [name for name in STAGES if STAGES[name].stubs_exempt], - "tapHazardCase": TAP_CASE_ID, - "cases": cases, - "crossCapture": cross_capture, - "finalizeReplay": replay, - "blockers": blockers, - } - receipt["summary"] = { - "cases": len(cases), - "solved": statuses.count("solved"), - "refused": statuses.count("refused"), - "refusedDeclared": sum(1 for case in cases if case["status"] == "refused" and case["refusal"].get("declared")), - "mismatched": sum(1 for case in cases if case["mismatches"]), - "errored": statuses.count("error"), - } - receipt["ok"] = not blockers - return receipt + statuses = [case["status"] for case in cases] + receipt: dict[str, Any] = { + "schemaVersion": RECEIPT_SCHEMA_VERSION, + "generatedAt": datetime.now(UTC).isoformat(), + "codeSha": code_sha, + "producers": { + # The integrated producer SHAs on this branch (base + imports), so the + # receipt names exactly which producer code ran, not just branches. + "base": "obv/burst7-bs-capture-placement-runner@bdfbb3fc649df8187675b880ba8df83a803fd119", + "caseIntegrityFix": "115b6ce313e51cede71496f267b65b8cd78aa118 (obv/burst7-bs-placement-cases, PR #271)", + "replayAcceptance": "5f9e29576556f635757eea0605d5a23d29ce1cc5 (obv/burst8-bs-replay-acceptance, PR #275)", + "supportedCasePairs": "e7ad2abc15d94691305bdb558a752ee535cf374e (obv/burst8-bs-supported-case-pairs, PR #276)", + }, + "devSettings": dict(TEMPLATE_ENV), + "stubbedStages": { + name: f"server.orchestrator.stubs.{STAGES[name].module}" + for name in STAGES + if not STAGES[name].stubs_exempt + }, + "realStages": [name for name in STAGES if STAGES[name].stubs_exempt], + "tapHazardCase": TAP_CASE_ID, + "cases": cases, + "crossCapture": cross_capture, + "finalizeReplay": replay, + "blockers": blockers, + } + receipt["summary"] = { + "cases": len(cases), + "solved": statuses.count("solved"), + "refused": statuses.count("refused"), + "refusedDeclared": sum( + 1 for case in cases if case["status"] == "refused" and case["refusal"].get("declared") + ), + "mismatched": sum(1 for case in cases if case["mismatches"]), + "errored": statuses.count("error"), + } + receipt["ok"] = not blockers + return receipt + finally: + restore_env() def _redaction_check(receipt: dict[str, Any]) -> str | None: diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index 65f5435b6..d9d75e835 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -11,6 +11,7 @@ import hashlib import importlib import json +import os import subprocess import sys from pathlib import Path @@ -961,3 +962,11 @@ def test_a_declared_negative_that_solves_fails_the_suite(tmp_path, monkeypatch): assert case["status"] == "solved" assert any("produced a successful placement" in mismatch for mismatch in case["mismatches"]) assert receipt["ok"] is False + + +def test_run_cases_restores_the_process_environment(tmp_path): + """The dev flags must not leak into the rest of the pytest process: any later + test asserting env-derived defaults would become order-dependent.""" + before = dict(os.environ) + runner.run_cases(MANIFEST, tmp_path / "run", case_ids=["multiwall-source-unsupported"]) + assert dict(os.environ) == before From b9032a7fe82fd4ac0550aa146b2123b788640a6c Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:26:41 +0000 Subject: [PATCH 10/19] fix(backend): sensitivity coverage bands are validated but are not observations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scene_bridge._coverage appended every coverage.sensitivity[].bands entry to the converted observed list, so a what-if sensitivity variant presented itself as ground-truth coverage the device saw and a sensitivity-only scene showed as a covered wall. Sensitivity bands keep their full validation (span, wall, out conversion, refusal pointers unchanged) but no longer convert into observations; a sensitivity-only scene still converts to ends only — a coverage gap, not a covered wall. The shared base_scene fixture carries a distinct primary band beside the sensitivity variant, pinning that only the primary band converts; new tests pin the sensitivity-only gap semantics and that a malformed sensitivity band still refuses with its own pointer. Evidence: the happy-path and sensitivity-only tests fail against the pre-fix bridge (2 failed) and pass after; the scene bridge and placement-stage suites pass (28 passed). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- backend/server/placement/scene_bridge.py | 18 +++++--- .../placement/tests/test_scene_bridge.py | 42 ++++++++++++++++++- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/backend/server/placement/scene_bridge.py b/backend/server/placement/scene_bridge.py index 4056fab70..aada91a63 100644 --- a/backend/server/placement/scene_bridge.py +++ b/backend/server/placement/scene_bridge.py @@ -54,9 +54,11 @@ - `overheads` and `facing` — observed entries on the wall, converted with the lengths. `object_id` cross-references have no wire field and are dropped. - `coverage` — the chain ends (`limit`/`unexplored` with their notes) and the explicit - bands from `coverage.bands` and every `coverage.sensitivity[].bands` entry: band name, - `span_ft` and `out_ft` convert with the lengths; `views` and `best_incidence_deg` have - no wire field and are dropped. A coverage without bands converts to ends only, which + primary bands from `coverage.bands`: band name, `span_ft` and `out_ft` convert with + the lengths; `views` and `best_incidence_deg` have no wire field and are dropped. + Sensitivity bands (`coverage.sensitivity[].bands`) are validated the same way but are + not observations — they are what-if variants, not the ground-truth coverage the + device saw. A coverage without bands converts to ends only, which the wire parser reads as nothing observed — an absent band stays unobserved. Refusals — every refusal raises `SceneBridgeError`, a `ValueError` carrying a `code` and @@ -583,7 +585,7 @@ def end_out(end, side: str) -> dict[str, Any]: observed: list[dict[str, Any]] = [] - def band_out(band, pointer: str) -> None: + def band_out(band, pointer: str) -> dict[str, Any]: if band.wall_id is not None and band.wall_id != wall_id: _invalid(f"the band names unknown wall {band.wall_id!r}", f"{pointer}/wall_id") record: dict[str, Any] = { @@ -592,12 +594,16 @@ def band_out(band, pointer: str) -> None: } if band.out_m is not None: record["out_ft"] = _ft(band.out_m) - observed.append(record) + return record for index, band in enumerate(scene.coverage.bands or []): - band_out(band, f"/coverage/bands/{index}") + observed.append(band_out(band, f"/coverage/bands/{index}")) for sensitivity_index, sensitivity in enumerate(scene.coverage.sensitivity or []): for band_index, band in enumerate(sensitivity.bands): + # Sensitivity variants are validated but are not observations: the wire's + # observed list is the ground-truth coverage the device saw, and a what-if + # sensitivity band is not that. A sensitivity-only scene still converts to + # ends only — no observation is invented. band_out(band, f"/coverage/sensitivity/{sensitivity_index}/bands/{band_index}") if observed: diff --git a/backend/server/placement/tests/test_scene_bridge.py b/backend/server/placement/tests/test_scene_bridge.py index 22320b9d3..21904e1e2 100644 --- a/backend/server/placement/tests/test_scene_bridge.py +++ b/backend/server/placement/tests/test_scene_bridge.py @@ -122,6 +122,7 @@ def base_scene() -> dict[str, Any]: ], "coverage": { "method": "scene-bridge-test", + "bands": [{"band": "wall", "s_m": [-1.0, 1.0]}], "sensitivity": [ { "surface": "wall", @@ -220,7 +221,9 @@ def test_happy_path_fields_in_independently_computed_feet() -> None: "right": {"kind": "limit", "note": "fence line"}, } assert converted["coverage"]["observed"] == [ - {"band": "wall", "span_ft": [-2.0 / 0.3048, 4.0 / 0.3048], "out_ft": 2.13 / 0.3048} + # The primary band only: the sensitivity variant below is validated but is + # not an observation, and must not appear in the converted coverage. + {"band": "wall", "span_ft": [-1.0 / 0.3048, 1.0 / 0.3048]} ] @@ -336,6 +339,43 @@ def test_meter_wall_rect_geometry_places_the_meter_at_its_span_centre_mid_height assert parsed.point_at(0.0)[0] == pytest.approx(0.0, abs=1e-6) +def test_sensitivity_only_coverage_converts_to_ends_only() -> None: + """A sensitivity band is a what-if variant, not a device observation: a scene + whose coverage carries only sensitivity bands must convert to ends only, with + no invented observed list — that is a coverage gap, not a covered wall.""" + document = base_scene() + document["coverage"] = { + "method": "scene-bridge-test", + "sensitivity": [ + { + "surface": "wall", + "max_incidence_deg": 75, + "bands": [{"band": "wall", "s_m": [-2.0, 4.0], "out_m": 2.13}], + } + ], + } + + converted = convert_scene(document) + + assert "observed" not in converted["coverage"] + assert converted["coverage"]["ends"] == { + "left": {"kind": "unexplored"}, + "right": {"kind": "limit", "note": "fence line"}, + } + + +def test_a_malformed_sensitivity_band_still_refuses() -> None: + """Not being an observation does not mean unvalidated: a reversed sensitivity + span still refuses, and the pointer names the sensitivity band.""" + document = base_scene() + document["coverage"]["sensitivity"][0]["bands"][0]["s_m"] = [4.0, -2.0] + + with pytest.raises(SceneBridgeError) as excinfo: + convert_scene(document) + + assert excinfo.value.pointer == "/coverage/sensitivity/0/bands/0/s_m" + + def test_absent_coverage_band_stays_unobserved_and_unexplored_ends_are_kept() -> None: document = base_scene() document["coverage"] = {"method": "scene-bridge-test"} From a99320846b6839d37851f2ad13d0699e9136a5fd Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:31:03 +0000 Subject: [PATCH 11/19] fix(backend): materialize missing stage records so an older unfinished run cannot hang MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unfinished run written by an older build can lack stage records entirely: _status_map treats a missing record as pending, but _spawn_one refuses to start a stage with no record, so every tick spawned nothing and the run hung forever between pending and running. tick now materializes only the missing pending records (same shape the API writes) before spawning: idempotent, existing records keep their attempts/failures/status, and terminal runs never reach it because tick returns before spawning. Evidence: the new test — a run with no stage records plus a pre-existing done validate record — fails against the pre-fix coordinator (run stuck running) and passes after; the handoff/run/placement/round2 suites pass (71 passed). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- backend/server/orchestrator/coordinator.py | 17 +++++++++++ .../server/orchestrator/tests/test_handoff.py | 29 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/backend/server/orchestrator/coordinator.py b/backend/server/orchestrator/coordinator.py index b66e3b4d8..3fb84cccd 100644 --- a/backend/server/orchestrator/coordinator.py +++ b/backend/server/orchestrator/coordinator.py @@ -229,6 +229,7 @@ def tick(self, run_id: str) -> StageStatus: def _spawn_ready(self, run: RunRecord, control: RunControl) -> list[str]: spawned: list[str] = [] + self._materialize_missing_records(run) status = self._status_map(run.run_id) names = ready_stages(status) if "criteria" in names and not control.merge_done: @@ -741,6 +742,22 @@ def _status_map(self, run_id: str) -> dict[StageName, StageStatus]: found = {rec.stage: rec.status for rec in self._rows(run_id)} return {name: found.get(name, "pending") for name in STAGE_NAMES} + def _materialize_missing_records(self, run: RunRecord) -> None: + """Recovery for runs written by older builds: a run can lack stage records + entirely (records materialize as the API schedules stages). The status map + treats a missing record as pending, but _spawn_one refuses to start a stage + with no record — so such a run hangs forever between ticks. Idempotently + create only the missing pending records: existing records keep their + attempts, failures and artifacts, and terminal runs never reach here + because tick returns before spawning. + """ + now = self.s.clock.iso() + for name in STAGE_NAMES: + if self.s.state.get_stage(run.run_id, name) is None: + self.s.state.upsert_stage( + StageRecord(run_id=run.run_id, stage=name, status="pending", attempt=1, heartbeat=now) + ) + def _snapshot(self, run_id: str) -> str: parts = [f"{rec.stage}:{rec.status}:{rec.attempt}:{rec.failures}:{rec.respawns}" for rec in self._rows(run_id)] run = self._run(run_id) diff --git a/backend/server/orchestrator/tests/test_handoff.py b/backend/server/orchestrator/tests/test_handoff.py index 00cd0acb1..0e67d02c1 100644 --- a/backend/server/orchestrator/tests/test_handoff.py +++ b/backend/server/orchestrator/tests/test_handoff.py @@ -346,3 +346,32 @@ def spawn(run_id: str) -> None: control = get_json(services.state, f"run_control:{run.run_id}") assert control["coordinator_respawns"] == 1 assert services.state.stages(run.run_id) == [] + +def test_a_run_without_stage_records_still_progresses(tmp_path) -> None: + """An older unfinished run can lack stage records entirely: the status map + treats a missing record as pending, but spawning refused to start a stage + with no record, so the run hung forever between ticks. Recovery must + materialize the missing pending records idempotently — existing records + keep their attempt and status — and the run must reach a terminal state.""" + services, _uri, digest = build_services(tmp_path, "memory") + deadline = (services.clock.now() + timedelta(hours=1)).isoformat() + run = _run(services, digest, deadline=deadline) + # The run exists but no stage record was ever written for it. + assert services.state.stages(run.run_id) == [] + validate = StageRecord( + run_id=run.run_id, stage="validate", status="done", attempt=2, heartbeat=services.clock.iso() + ) + services.state.upsert_stage(validate) + + RunCoordinator(services).tick(run.run_id) + + rows = {rec.stage: rec for rec in services.state.stages(run.run_id)} + assert {rec.stage for rec in rows.values()} == set(STAGE_NAMES) + assert rows["validate"].status == "done" and rows["validate"].attempt == 2, "existing record preserved" + status = "running" + for _ in range(40): + status = RunCoordinator(services).tick(run.run_id) + if status not in {"pending", "running"}: + break + assert status in {"done", "failed"}, f"run must terminate, not hang: {status}" + From c4632e3a6a46f0c717af3023e3bf37a8e9e82b44 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:37:45 +0000 Subject: [PATCH 12/19] fix(backend): placement refuses objects_3d artifacts without an explicit objects list A format-correct objects_3d manifest whose objects field is missing, null, or not a list is a malformed manifest, not evidence of a hazard-free scene. The placement stage now refuses such artifacts with placement_objects_3d_invalid before any placement artifact is written; an explicit empty list remains a valid, deliberate "no hazards". Evidence: the journey test publishes a format-correct artifact with a null and a missing objects list through the real capture stage and asserts the refusal code, no placement artifact, and a receipt blocker; it fails against the pre-fix adapter and passes after. The focused placement, orchestrator, and runner suites pass (258 passed). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> EOF2 fix(backend): supported capture fixtures carry their observed coverage as primary bands The supported sources held their only coverage band under the sensitivity what-if wrapper while the independently authored wire scenes expect that band in coverage.observed. That only passed while the bridge wrongly promoted sensitivity bands to observations; since b9032a7f sensitivity variants are validated but are not observations, so the converted scene lacked /coverage/observed and both supported positives failed the receipt check. Each fixture now publishes the observed band as a primary coverage band (the ground truth the device saw) and keeps the sensitivity variant as the what-if it is. The supported end-to-end and supported-CLI tests fail at the previous head and pass after; the focused suites pass (258 passed). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- backend/server/placement/adapter.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/backend/server/placement/adapter.py b/backend/server/placement/adapter.py index 6c2f09075..d8aaeefe5 100644 --- a/backend/server/placement/adapter.py +++ b/backend/server/placement/adapter.py @@ -111,6 +111,18 @@ def __init__(self, message: str) -> None: self.code = "placement_objects_3d_unavailable" +class PlacementObjectsFormatError(PlacementObjectsError): + """The run's objects_3d artifact does not carry an explicit objects list. + + Not retryable: a malformed manifest is a refusal, not an empty hazard set — + an empty list is a valid "no hazards", a missing or null list is not. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + self.code = "placement_objects_3d_invalid" + + def _top_level_server_dir() -> Path: return _REPO_ROOT / "server" @@ -236,6 +248,14 @@ def _objects_3d_evidence(ctx: StageContext) -> None: fmt = document.get("format") if isinstance(document, dict) else None if fmt != "hss-objects-3d/v0": raise PlacementObjectsError(f'{_OBJECTS_3D} must carry format "hss-objects-3d/v0", got {fmt!r}') + # An empty list is a valid "no hazards"; a missing or null list is a malformed + # manifest and must refuse, not decode as "no hazards". + objects = document.get("objects") if isinstance(document, dict) else None + if not isinstance(objects, list): + raise PlacementObjectsFormatError( + f'{_OBJECTS_3D} must carry an explicit "objects" list (an empty list is valid), got ' + f'{"null" if objects is None else type(objects).__name__!r}' + ) def _source_scene_bytes(ctx: StageContext) -> bytes: From e2376df8386c0c7257fd71e6ccc014c4ed18479a Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:37:45 +0000 Subject: [PATCH 13/19] fix(backend): supported capture fixtures carry their observed coverage as primary bands The supported sources held their only coverage band under the sensitivity what-if wrapper while the independently authored wire scenes expect that band in coverage.observed. That only passed while the bridge wrongly promoted sensitivity bands to observations; since b9032a7f sensitivity variants are validated but are not observations, so the converted scene lacked /coverage/observed and both supported positives failed the receipt check. Each fixture now publishes the observed band as a primary coverage band (the ground truth the device saw) and keeps the sensitivity variant as the what-if it is. The supported end-to-end and supported-CLI tests fail at the previous head and pass after; the focused suites pass (258 passed). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- backend/tests/test_capture_placement_cases.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index d9d75e835..5a2cd1b09 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -26,7 +26,7 @@ import capture_placement_cases as runner # noqa: E402 from server.eval.sceneio import validate # noqa: E402 -from server.orchestrator.stubs._common import write_files # noqa: E402 +from server.orchestrator.stubs._common import json_bytes, write_files # noqa: E402 MANIFEST = BACKEND / "server" / "placement" / "tests" / "data" / "cases" / "manifest.json" @@ -146,6 +146,7 @@ def _supported_source_a() -> dict: ], "coverage": { "method": "capture-placement-cases-test", + "bands": [{"band": "wall", "s_m": [-2.0, 4.0], "out_m": 2.13}], "sensitivity": [ { "surface": "wall", @@ -252,6 +253,7 @@ def _supported_source_b() -> dict: ], "coverage": { "method": "capture-placement-cases-test", + "bands": [{"band": "wall", "s_m": [-4.0, 2.0], "out_m": 2.5}], "sensitivity": [ { "surface": "wall", @@ -970,3 +972,29 @@ def test_run_cases_restores_the_process_environment(tmp_path): before = dict(os.environ) runner.run_cases(MANIFEST, tmp_path / "run", case_ids=["multiwall-source-unsupported"]) assert dict(os.environ) == before + +@pytest.mark.parametrize("objects_value", [pytest.param(None, id="null-objects"), pytest.param("omit", id="missing-objects")]) +def test_a_format_correct_objects_3d_without_an_objects_list_refuses(tmp_path, monkeypatch, objects_value): + """Through the actual capture stage: a format-correct objects_3d artifact whose + objects list is missing or null is a malformed manifest and must refuse + placement with placement_objects_3d_invalid — never decode as an empty, + hazard-free scene — and no placement artifact may exist (the refusal is not a + fallback success).""" + body = {"format": "hss-objects-3d/v0"} + if objects_value != "omit": + body["objects"] = None + + def malformed_objects_3d(ctx): + return write_files(ctx, {"objects/objects_3d.json": json_bytes(body)}) + + monkeypatch.setattr(runner, "_injected_objects_3d_run", malformed_objects_3d) + receipt = runner.run_cases(_write_supported_manifest(tmp_path), tmp_path / "run", case_ids=[runner.TAP_CASE_ID]) + case = receipt["cases"][0] + + assert case["status"] == "refused", case + assert case["refusal"]["code"] == "placement_objects_3d_invalid", case["refusal"] + assert case["hasPlacementArtifact"] is False, "a refused placement must not publish an artifact" + assert case["solverDecision"] is None + assert receipt["summary"]["solved"] == 0 + assert receipt["ok"] is False, "an undeclared positive-case refusal is a blocker" + From 13277524fbf533751a964cebee9430a99219cb38 Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:51:00 +0000 Subject: [PATCH 14/19] fix(backend): stage-record recovery claims are atomic, so a racing coordinator cannot reset a running stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recovery for an old run's missing stage records was a check-then-upsert: two coordinators can resume the same run without an exclusive claim (the sweeper, _resume_if_stale, and the worker tick), so one could read a missing record, pause, watch the other create it and a worker start it, then overwrite that running record with pending, attempt 1, and no worker id — losing the first worker and launching a second one against the same stage. state gains create_stage_if_absent: MemoryState takes a dedicated stage lock across the deciding read and the insert; ModalDictState reuses put_if_absent, whose Modal Dict skip_if_exists is the atomic cross-process claim, under a per-(run, stage) lock for the plain-dict fallback. The coordinator materializes missing records through it. Also format-only cleanups in files pushed earlier. Evidence: five new tests — coordinator-level (the racing record survives materialize with its attempt and worker id) and backend-level claims on both state backends, including eight threads claiming one record — fail against the pre-fix coordinator and backends, pass after; the focused common, orchestrator, placement, and runner suites pass (296 passed). Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- .../server/common/backends/state_memory.py | 16 +++++ .../common/backends/state_modal_dict.py | 27 +++++++ backend/server/common/state.py | 4 ++ .../common/tests/test_stage_record_claim.py | 72 +++++++++++++++++++ backend/server/orchestrator/coordinator.py | 11 +-- .../server/orchestrator/tests/test_handoff.py | 40 +++++++++++ backend/server/placement/adapter.py | 2 +- backend/tests/test_capture_placement_cases.py | 6 +- 8 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 backend/server/common/tests/test_stage_record_claim.py diff --git a/backend/server/common/backends/state_memory.py b/backend/server/common/backends/state_memory.py index 4cc6ef244..bda4c4496 100644 --- a/backend/server/common/backends/state_memory.py +++ b/backend/server/common/backends/state_memory.py @@ -49,6 +49,9 @@ def __init__(self) -> None: self._spend: dict[str, float] = {} self._open_by_owner: dict[str, list[str]] = {} self._claim_lock = threading.Lock() + # Dedicated lock for stage-record creation (the append_event lesson: never + # share one lock across unrelated write paths — lock-ordering deadlock). + self._stage_lock = threading.Lock() def get_capture(self, capture_id: str) -> CaptureRecord | None: rec = self._captures.get(capture_id) @@ -179,6 +182,19 @@ def list_runs(self, *, status: set[StageStatus], started_before: datetime | None def upsert_stage(self, rec: StageRecord) -> None: self._stages[(rec.run_id, rec.stage)] = rec.model_copy(deep=True) + def create_stage_if_absent(self, rec: StageRecord) -> StageRecord: + """Atomic insert-if-absent: the deciding read and the insert share one lock, + so a record another coordinator created and started inside the race window + is returned untouched instead of being reset to pending. Returns the record + now stored.""" + key = (rec.run_id, rec.stage) + with self._stage_lock: + existing = self._stages.get(key) + if existing is not None: + return existing.model_copy(deep=True) + self._stages[key] = rec.model_copy(deep=True) + return rec.model_copy(deep=True) + def get_stage(self, run_id: str, stage: StageName) -> StageRecord | None: rec = self._stages.get((run_id, stage)) return None if rec is None else rec.model_copy(deep=True) diff --git a/backend/server/common/backends/state_modal_dict.py b/backend/server/common/backends/state_modal_dict.py index c6e002f43..c908360e2 100644 --- a/backend/server/common/backends/state_modal_dict.py +++ b/backend/server/common/backends/state_modal_dict.py @@ -24,6 +24,20 @@ _APPEND_GUARD = threading.Lock() _APPEND_LOCKS: dict[int, threading.Lock] = {} +_STAGE_GUARD = threading.Lock() +_STAGE_LOCKS: dict[tuple[str, str], threading.Lock] = {} + + +def _stage_lock(key: tuple[str, str]) -> threading.Lock: + """One lock per (run, stage) key, so concurrent recovery claims serialize.""" + + with _STAGE_GUARD: + lock = _STAGE_LOCKS.get(key) + if lock is None: + lock = threading.Lock() + _STAGE_LOCKS[key] = lock + return lock + def _append_lock(mapping: object) -> threading.Lock: """One lock per backing map, so two states on the same dict share it.""" @@ -485,6 +499,19 @@ def list_runs(self, *, status: set[StageStatus], started_before: datetime | None def upsert_stage(self, rec: StageRecord) -> None: self._kv[f"stage:{rec.run_id}:{rec.stage}"] = rec.model_dump(mode="json") + def create_stage_if_absent(self, rec: StageRecord) -> StageRecord: + """Insert-if-absent across processes: Modal Dict ``skip_if_exists`` is the + atomic claim, so two coordinators resuming the same old run cannot both + create the record — the loser reads back the winner's record instead of + resetting a running stage to pending. The per-key lock covers the plain-dict + fallback, where the check-then-set is only in-process.""" + + key = f"stage:{rec.run_id}:{rec.stage}" + with _stage_lock((rec.run_id, rec.stage)): + self.put_if_absent(key, rec.model_dump(mode="json")) + stored = self._model(key, StageRecord) + return stored if stored is not None else rec.model_copy(deep=True) + def get_stage(self, run_id: str, stage: StageName) -> StageRecord | None: return self._model(f"stage:{run_id}:{stage}", StageRecord) diff --git a/backend/server/common/state.py b/backend/server/common/state.py index ce0d2a152..75fcc6632 100644 --- a/backend/server/common/state.py +++ b/backend/server/common/state.py @@ -68,6 +68,10 @@ def list_runs(self, *, status: set[StageStatus], started_before: datetime | None def upsert_stage(self, rec: StageRecord) -> None: ... + def create_stage_if_absent(self, rec: StageRecord) -> StageRecord: + """Atomic insert-if-absent: recovery must never reset a record another + coordinator materialized and started first. Returns the record now stored.""" + def get_stage(self, run_id: str, stage: StageName) -> StageRecord | None: ... def stages(self, run_id: str) -> list[StageRecord]: ... diff --git a/backend/server/common/tests/test_stage_record_claim.py b/backend/server/common/tests/test_stage_record_claim.py new file mode 100644 index 000000000..f24181a6f --- /dev/null +++ b/backend/server/common/tests/test_stage_record_claim.py @@ -0,0 +1,72 @@ +"""Stage-record recovery claims are atomic: a record another coordinator +materialized and started inside the race window is returned untouched, never +reset to pending with its attempt and worker identity stripped.""" + +from __future__ import annotations + +import threading + +from server.common.backends.state_memory import MemoryState +from server.common.backends.state_modal_dict import ModalDictState +from server.contracts.jobs import StageRecord + + +def _rec(run_id: str, stage: str, worker: str | None = None) -> StageRecord: + return StageRecord( + run_id=run_id, + stage=stage, + status="pending", + attempt=1, + heartbeat="2026-10-16T00:00:00Z", + worker_call_id=worker, + ) + + +def test_memory_claim_returns_the_existing_record_untouched() -> None: + state = MemoryState() + first = state.create_stage_if_absent(_rec("run", "poses", worker="left")) + assert first.worker_call_id == "left" + + second = state.create_stage_if_absent(_rec("run", "poses", worker="right")) + + assert second.worker_call_id == "left", "the loser must read back the winner" + stored = state.get_stage("run", "poses") + assert stored.worker_call_id == "left" and stored.attempt == 1 + + +def test_memory_claim_inserts_when_absent() -> None: + state = MemoryState() + created = state.create_stage_if_absent(_rec("run", "poses")) + assert created.status == "pending" and created.attempt == 1 + assert state.get_stage("run", "poses") == created + + +def test_modal_claim_two_states_share_one_map() -> None: + mapping: dict[str, object] = {} + left, right = ModalDictState(mapping), ModalDictState(mapping) + winner = left.create_stage_if_absent(_rec("run", "poses", worker="left")) + loser = right.create_stage_if_absent(_rec("run", "poses", worker="right")) + assert winner.worker_call_id == "left" + assert loser.worker_call_id == "left", "the loser must read back the winner" + assert right.get_stage("run", "poses").worker_call_id == "left" + + +def test_modal_claim_many_threads_one_winner() -> None: + state = ModalDictState() + results: list[str] = [] + guard = threading.Lock() + + def claim(worker: str) -> None: + rec = state.create_stage_if_absent(_rec("run", "poses", worker=worker)) + with guard: + results.append(rec.worker_call_id) + + threads = [threading.Thread(target=claim, args=(f"w{i}",)) for i in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(set(results)) == 1, f"every claimant must read back the same winner: {results}" + stored = state.get_stage("run", "poses") + assert stored.worker_call_id == results[0] and stored.attempt == 1 diff --git a/backend/server/orchestrator/coordinator.py b/backend/server/orchestrator/coordinator.py index 3fb84cccd..2d4d85b17 100644 --- a/backend/server/orchestrator/coordinator.py +++ b/backend/server/orchestrator/coordinator.py @@ -753,10 +753,13 @@ def _materialize_missing_records(self, run: RunRecord) -> None: """ now = self.s.clock.iso() for name in STAGE_NAMES: - if self.s.state.get_stage(run.run_id, name) is None: - self.s.state.upsert_stage( - StageRecord(run_id=run.run_id, stage=name, status="pending", attempt=1, heartbeat=now) - ) + # Atomic insert-if-absent: two coordinators can resume the same old run + # (sweeper, _resume_if_stale, worker tick) without an exclusive claim. A + # plain check-then-upsert would reset a record the other coordinator + # created and a worker started inside the race window. + self.s.state.create_stage_if_absent( # type: ignore[attr-defined] + StageRecord(run_id=run.run_id, stage=name, status="pending", attempt=1, heartbeat=now) + ) def _snapshot(self, run_id: str) -> str: parts = [f"{rec.stage}:{rec.status}:{rec.attempt}:{rec.failures}:{rec.respawns}" for rec in self._rows(run_id)] diff --git a/backend/server/orchestrator/tests/test_handoff.py b/backend/server/orchestrator/tests/test_handoff.py index 0e67d02c1..089e9fa07 100644 --- a/backend/server/orchestrator/tests/test_handoff.py +++ b/backend/server/orchestrator/tests/test_handoff.py @@ -4,6 +4,7 @@ from datetime import timedelta +from server.common.backends.state_memory import MemoryState from server.contracts.jobs import RunRecord, StageRecord, StageResult from server.orchestrator.coordinator import RunControl, RunCoordinator from server.orchestrator.graph import STAGE_NAMES @@ -347,6 +348,7 @@ def spawn(run_id: str) -> None: assert control["coordinator_respawns"] == 1 assert services.state.stages(run.run_id) == [] + def test_a_run_without_stage_records_still_progresses(tmp_path) -> None: """An older unfinished run can lack stage records entirely: the status map treats a missing record as pending, but spawning refused to start a stage @@ -375,3 +377,41 @@ def test_a_run_without_stage_records_still_progresses(tmp_path) -> None: break assert status in {"done", "failed"}, f"run must terminate, not hang: {status}" + +def test_materialize_yields_to_a_record_that_raced_in(tmp_path, monkeypatch) -> None: + """Two coordinators can resume the same old run without an exclusive claim + (the sweeper, _resume_if_stale, and the worker tick all recover). If another + coordinator creates a record and a worker starts it inside the recovery race + window, the atomic claim must return that record untouched — never reset a + running stage to pending, attempt 1, no worker.""" + services, _uri, digest = build_services(tmp_path, "memory") + deadline = (services.clock.now() + timedelta(hours=1)).isoformat() + run = _run(services, digest, deadline=deadline) + assert services.state.stages(run.run_id) == [] + + real_create = MemoryState.create_stage_if_absent + + def racing_create(self, rec): + if rec.stage == "poses" and self.get_stage(rec.run_id, rec.stage) is None: + # The other coordinator wins the race: it creates the record and a + # worker starts it before our claim lands. + self.upsert_stage( + StageRecord( + run_id=rec.run_id, + stage="poses", + status="running", + attempt=2, + heartbeat=rec.heartbeat, + worker_call_id="other-coord", + ) + ) + return real_create(self, rec) + + monkeypatch.setattr(MemoryState, "create_stage_if_absent", racing_create) + + RunCoordinator(services).tick(run.run_id) + + poses = services.state.get_stage(run.run_id, "poses") + assert poses is not None + assert poses.status == "running", poses + assert poses.attempt == 2 and poses.worker_call_id == "other-coord", poses diff --git a/backend/server/placement/adapter.py b/backend/server/placement/adapter.py index d8aaeefe5..c6defdace 100644 --- a/backend/server/placement/adapter.py +++ b/backend/server/placement/adapter.py @@ -254,7 +254,7 @@ def _objects_3d_evidence(ctx: StageContext) -> None: if not isinstance(objects, list): raise PlacementObjectsFormatError( f'{_OBJECTS_3D} must carry an explicit "objects" list (an empty list is valid), got ' - f'{"null" if objects is None else type(objects).__name__!r}' + f"{'null' if objects is None else type(objects).__name__!r}" ) diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index 5a2cd1b09..c072dc1d4 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -973,7 +973,10 @@ def test_run_cases_restores_the_process_environment(tmp_path): runner.run_cases(MANIFEST, tmp_path / "run", case_ids=["multiwall-source-unsupported"]) assert dict(os.environ) == before -@pytest.mark.parametrize("objects_value", [pytest.param(None, id="null-objects"), pytest.param("omit", id="missing-objects")]) + +@pytest.mark.parametrize( + "objects_value", [pytest.param(None, id="null-objects"), pytest.param("omit", id="missing-objects")] +) def test_a_format_correct_objects_3d_without_an_objects_list_refuses(tmp_path, monkeypatch, objects_value): """Through the actual capture stage: a format-correct objects_3d artifact whose objects list is missing or null is a malformed manifest and must refuse @@ -997,4 +1000,3 @@ def malformed_objects_3d(ctx): assert case["solverDecision"] is None assert receipt["summary"]["solved"] == 0 assert receipt["ok"] is False, "an undeclared positive-case refusal is a blocker" - From c3e479bf3cdd2ad79c25a68ffba896dff91246b5 Mon Sep 17 00:00:00 2001 From: Obvious Date: Fri, 16 Oct 2026 05:43:36 +0000 Subject: [PATCH 15/19] fix(backend): the legacy manifest's cross-capture now evaluates on the merged runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alpha's _collect_case publishes mergedGeometrySha256 for every case, so the legacy manifest's two refused positives now carry geometry hashes and the cross-capture comparison runs instead of skipping — and finds distinct scenes with distinct normalized geometry. The pinned expectation follows the merged data; the anti-vacuous intent (never pass without a real comparison) still holds, now in its stronger branch. Co-authored-by: Sam Gu --- backend/tests/test_capture_placement_cases.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index c072dc1d4..99d632cf8 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -592,11 +592,13 @@ def test_every_case_walks_the_real_front_door_with_case_true_boundaries(receipt) # The one nonempty objects_3d case: the observed tap hazard, merged exactly once. assert _case(receipt, runner.TAP_CASE_ID)["tapHazardMergeCount"] == 1 - # Both positives refused: no merged scenes, so the cross-capture comparison - # cannot run and must say so instead of passing vacuously. + # Both positives refuse, but the merged runtime persists their merged scenes + # and their geometry hashes, so the cross-capture comparison actually runs — + # and must not pass vacuously: the two scenes and their normalized geometry + # are distinct. cross = receipt["crossCapture"] - assert cross["status"] == "not_applicable" - assert cross["distinctScenes"] is None and cross["distinctNormalizedGeometry"] is None + assert cross["status"] == "evaluated" + assert cross["distinctScenes"] is True and cross["distinctNormalizedGeometry"] is True assert cross["captureIds"] == [ _case(receipt, "hazard-right-ground-gap")["captureId"], _case(receipt, "hazard-left-ground-gap")["captureId"], From 31ca9ef8535f89ce9a2d7cef78bf0f37f6502d15 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 16 Oct 2026 02:19:52 -0500 Subject: [PATCH 16/19] fix(backend): claim only the stage records the tick found missing _materialize_missing_records claimed all twelve stages on every tick. With ModalDictState each claim is a conditional remote write plus a read-back, so a run whose records all existed paid 24 sequential round-trips per tick. The tick has already read the run's rows; claim only the stages absent from that read and keep the atomic insert for those. Co-Authored-By: Claude Opus 5.5 --- backend/server/orchestrator/coordinator.py | 13 ++++++-- .../server/orchestrator/tests/test_handoff.py | 33 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/backend/server/orchestrator/coordinator.py b/backend/server/orchestrator/coordinator.py index 0568d21cd..4fe3a5f3b 100644 --- a/backend/server/orchestrator/coordinator.py +++ b/backend/server/orchestrator/coordinator.py @@ -765,16 +765,25 @@ def _materialize_missing_records(self, run: RunRecord) -> None: create only the missing pending records: existing records keep their attempts, failures and artifacts, and terminal runs never reach here because tick returns before spawning. + + Only stages absent from the rows this tick already read are claimed: with + ModalDictState each claim is a conditional remote write plus a read-back, + so claiming every stage cost two round-trips per stage on every tick. """ + present = {rec.stage for rec in self._rows(run.run_id)} + missing = [name for name in STAGE_NAMES if name not in present] + if not missing: + return now = self.s.clock.iso() - for name in STAGE_NAMES: + for name in missing: # Atomic insert-if-absent: two coordinators can resume the same old run # (sweeper, _resume_if_stale, worker tick) without an exclusive claim. A # plain check-then-upsert would reset a record the other coordinator # created and a worker started inside the race window. - self.s.state.create_stage_if_absent( # type: ignore[attr-defined] + stored = self.s.state.create_stage_if_absent( # type: ignore[attr-defined] StageRecord(run_id=run.run_id, stage=name, status="pending", attempt=1, heartbeat=now) ) + self._remember_stage(stored) def _snapshot(self, run_id: str) -> str: parts = [f"{rec.stage}:{rec.status}:{rec.attempt}:{rec.failures}:{rec.respawns}" for rec in self._rows(run_id)] diff --git a/backend/server/orchestrator/tests/test_handoff.py b/backend/server/orchestrator/tests/test_handoff.py index 089e9fa07..3c4216d5f 100644 --- a/backend/server/orchestrator/tests/test_handoff.py +++ b/backend/server/orchestrator/tests/test_handoff.py @@ -415,3 +415,36 @@ def racing_create(self, rec): assert poses is not None assert poses.status == "running", poses assert poses.attempt == 2 and poses.worker_call_id == "other-coord", poses + + +def test_materialize_claims_only_the_missing_records(tmp_path, monkeypatch) -> None: + """With ModalDictState each claim is a conditional remote write plus a read-back, + so claiming all stages on every tick costs two round-trips per stage even when + every record exists. The tick already read the run's records; only the stages + missing from that read may be claimed.""" + services, _uri, digest = build_services(tmp_path, "memory") + deadline = (services.clock.now() + timedelta(hours=1)).isoformat() + run = _run(services, digest, deadline=deadline) + now = services.clock.iso() + for name in STAGE_NAMES: + if name != "poses": + services.state.upsert_stage( + StageRecord(run_id=run.run_id, stage=name, status="pending", attempt=1, heartbeat=now) + ) + + claimed: list[str] = [] + real_create = MemoryState.create_stage_if_absent + + def counting_create(self, rec): + claimed.append(rec.stage) + return real_create(self, rec) + + monkeypatch.setattr(MemoryState, "create_stage_if_absent", counting_create) + + RunCoordinator(services).tick(run.run_id) + assert claimed == ["poses"], claimed + assert services.state.get_stage(run.run_id, "poses") is not None + + claimed.clear() + RunCoordinator(services).tick(run.run_id) + assert claimed == [], "a tick whose run has every record must not claim any" From df1b2b52930cd272badff2c11a3d862d7156be62 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 16 Oct 2026 02:19:52 -0500 Subject: [PATCH 17/19] fix(backend): one stage-claim lock instead of a lock per run and stage _STAGE_LOCKS kept a lock for every (run_id, stage) a worker ever claimed and never dropped them, so long-lived workers grew it without bound. The lock only guards the plain-dict fallback's check-then-set (a real Modal Dict claims atomically with skip_if_exists), and claims now happen only for missing records, so one module lock is enough. Co-Authored-By: Claude Opus 5.5 --- .../common/backends/state_modal_dict.py | 23 +++++++------------ .../common/tests/test_stage_record_claim.py | 22 ++++++++++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/backend/server/common/backends/state_modal_dict.py b/backend/server/common/backends/state_modal_dict.py index efec53c57..b26e115cf 100644 --- a/backend/server/common/backends/state_modal_dict.py +++ b/backend/server/common/backends/state_modal_dict.py @@ -24,19 +24,12 @@ _APPEND_GUARD = threading.Lock() _APPEND_LOCKS: dict[int, threading.Lock] = {} -_STAGE_GUARD = threading.Lock() -_STAGE_LOCKS: dict[tuple[str, str], threading.Lock] = {} - - -def _stage_lock(key: tuple[str, str]) -> threading.Lock: - """One lock per (run, stage) key, so concurrent recovery claims serialize.""" - - with _STAGE_GUARD: - lock = _STAGE_LOCKS.get(key) - if lock is None: - lock = threading.Lock() - _STAGE_LOCKS[key] = lock - return lock +# Serializes create_stage_if_absent on the plain-dict fallback, where put_if_absent +# is an in-process check-then-set; a real Modal Dict claims atomically on its own. +# One lock, not one per (run, stage): a per-key registry kept a lock for every run +# a long-lived worker ever touched. Claims only happen for records missing from +# the coordinator's read, so the shared lock serializes only that recovery path. +_STAGE_CLAIM_LOCK = threading.Lock() def _append_lock(mapping: object) -> threading.Lock: @@ -511,11 +504,11 @@ def create_stage_if_absent(self, rec: StageRecord) -> StageRecord: """Insert-if-absent across processes: Modal Dict ``skip_if_exists`` is the atomic claim, so two coordinators resuming the same old run cannot both create the record — the loser reads back the winner's record instead of - resetting a running stage to pending. The per-key lock covers the plain-dict + resetting a running stage to pending. The module lock covers the plain-dict fallback, where the check-then-set is only in-process.""" key = f"stage:{rec.run_id}:{rec.stage}" - with _stage_lock((rec.run_id, rec.stage)): + with _STAGE_CLAIM_LOCK: self.put_if_absent(key, rec.model_dump(mode="json")) stored = self._model(key, StageRecord) return stored if stored is not None else rec.model_copy(deep=True) diff --git a/backend/server/common/tests/test_stage_record_claim.py b/backend/server/common/tests/test_stage_record_claim.py index f24181a6f..e631f025d 100644 --- a/backend/server/common/tests/test_stage_record_claim.py +++ b/backend/server/common/tests/test_stage_record_claim.py @@ -70,3 +70,25 @@ def claim(worker: str) -> None: assert len(set(results)) == 1, f"every claimant must read back the same winner: {results}" stored = state.get_stage("run", "poses") assert stored.worker_call_id == results[0] and stored.attempt == 1 + + +def test_modal_claims_leave_no_per_run_state_in_the_module() -> None: + """A long-lived worker claims records for many runs. Module-level lock state + must stay bounded: a lock kept per (run, stage) key grows without limit.""" + from server.common.backends import state_modal_dict + + state = ModalDictState() + state.create_stage_if_absent(_rec("warm", "poses")) + + def sizes() -> dict[str, int]: + return { + name: len(value) + for name, value in vars(state_modal_dict).items() + if isinstance(value, dict) and not name.startswith("__") + } + + before = sizes() + for index in range(200): + state.create_stage_if_absent(_rec(f"run-{index}", "poses")) + + assert sizes() == before From 3daefe42c3b88c30b279fa5533fcd4142148b2cc Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 16 Oct 2026 04:11:50 -0500 Subject: [PATCH 18/19] test(backend): the tap case merges exactly one vent on every committed manifest capture_cases_v1/hazard-right-ground-gap.source.json already carries obj-tap-vent, identical to the runner's TAP_HAZARD. merge_object_lists keeps a tap-sourced object and appends the incoming list, so injecting the vent there too (as alpha's runner still does) merges two vents and the supported positive stops solving. This branch already injects only when the source lacks the id; that decision moves into _objects_3d_for so a test can call it. The new test runs the real merge on each committed manifest's tap case and requires exactly one vent equal to TAP_HAZARD. Making the injection unconditional fails the capture-cases-v1 case; the legacy case, whose source has no vent, still gets the injected one. Co-Authored-By: Claude Opus 5.5 --- backend/scripts/capture_placement_cases.py | 26 ++++++++++++------- backend/tests/test_capture_placement_cases.py | 22 ++++++++++++++++ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/backend/scripts/capture_placement_cases.py b/backend/scripts/capture_placement_cases.py index 18f8e5dfc..d035879c2 100644 --- a/backend/scripts/capture_placement_cases.py +++ b/backend/scripts/capture_placement_cases.py @@ -221,6 +221,22 @@ def _scene_bytes(document: dict[str, Any]) -> bytes: return json.dumps(document, sort_keys=True).encode() +def _objects_3d_for(record: case_suite.CaseRecord) -> list[dict[str, Any]]: + """The objects_3d list this case's boundary delivers. + + The tap hazard is delivered only where the upstream source does not already + carry it: the merge contract replaces only detector/VLM twins, so a same-id + re-delivery of a source-carried (tap-sourced) hazard is never replaced — it + duplicates the id and the merged scene is refused as inconsistent. A + source-carried hazard still reaches the one-time tap checks through the real + merge and real conversion. + """ + if record.id != TAP_CASE_ID: + return [] + source_object_ids = {obj.get("id") for obj in json.loads(record.source_path.read_text()).get("objects") or []} + return [] if TAP_HAZARD["id"] in source_object_ids else [copy.deepcopy(TAP_HAZARD)] + + def _injected_scene_run(ctx: StageContext) -> StageResult: """The scene stage boundary: this capture's stamped source scene, or a loud refusal. The default stub scene never runs here.""" @@ -381,19 +397,11 @@ def _walk_front_door( journey["captureId"] = capture_id # Register the case at both boundaries before finalize can start the pipeline. - # The tap hazard is delivered only where the upstream source does not already - # carry it: the merge contract replaces only detector/VLM twins, so a same-id - # re-delivery of a source-carried (tap-sourced) hazard is never replaced — it - # duplicates the id and the merged scene is refused as inconsistent. A - # source-carried hazard still reaches the one-time tap checks through the real - # merge and real conversion below. - source_object_ids = {obj.get("id") for obj in json.loads(record.source_path.read_text()).get("objects") or []} - inject_tap = record.id == TAP_CASE_ID and TAP_HAZARD["id"] not in source_object_ids _REGISTRY[capture_id] = { "record": record, "packet_id": packet_id, "packet_sha256": journey["packetSha256"], - "objects_3d": [copy.deepcopy(TAP_HAZARD)] if inject_tap else [], + "objects_3d": _objects_3d_for(record), } files = [] diff --git a/backend/tests/test_capture_placement_cases.py b/backend/tests/test_capture_placement_cases.py index 24ab8e6e0..7f5c0ab63 100644 --- a/backend/tests/test_capture_placement_cases.py +++ b/backend/tests/test_capture_placement_cases.py @@ -27,6 +27,7 @@ from server.eval.sceneio import validate # noqa: E402 from server.orchestrator.stubs._common import json_bytes, write_files # noqa: E402 +from server.scene.merge import merge_object_lists # noqa: E402 MANIFEST = BACKEND / "server" / "placement" / "tests" / "data" / "cases" / "manifest.json" @@ -1013,6 +1014,27 @@ def test_run_cases_restores_the_process_environment(tmp_path): assert dict(os.environ) == before +@pytest.mark.parametrize( + "manifest", + [ + pytest.param(MANIFEST, id="legacy-cases"), + pytest.param(runner.DEFAULT_MANIFEST, id="capture-cases-v1"), + ], +) +def test_the_tap_case_merges_exactly_one_vent_on_every_committed_manifest(manifest): + """The source fixture and the runner's injected objects_3d must agree on who + supplies the observed tap vent. The merge keeps a tap-sourced source object + and appends the incoming one, so a fixture that already carries the vent plus + an injected copy merges two vents and the supported positive stops solving. + Whichever side supplies it, the merged scene holds exactly the runner's + TAP_HAZARD once, so the converted-span check measures the right object.""" + record = next(record for record in runner.case_suite.load_cases(manifest) if record.id == runner.TAP_CASE_ID) + source_objects = json.loads(record.source_path.read_text())["objects"] + merged = merge_object_lists({"objects": source_objects}, runner._objects_3d_for(record)) + vents = [obj for obj in merged["objects"] if obj["id"] == runner.TAP_HAZARD["id"]] + assert vents == [runner.TAP_HAZARD] + + @pytest.mark.parametrize( "objects_value", [pytest.param(None, id="null-objects"), pytest.param("omit", id="missing-objects")] ) From 2e469fc4bdff5aa33d27f17d25779fef32fa9449 Mon Sep 17 00:00:00 2001 From: SamGu-NRX Date: Fri, 16 Oct 2026 04:19:15 -0500 Subject: [PATCH 19/19] fix(backend): a late correction does not reopen a run without stage records After the alpha merge, test_confirmation_updates_the_reads_record failed: #278's _recompose_after_correction flips a done run back to running, and this branch's _materialize_missing_records then creates pending records for every absent stage. On a done run with no stage records, reads re-ran and overwrote the correction just persisted. Neither parent fails alone. _materialize_missing_records assumes terminal runs never reach it. Only a run that has a record for every stage is reopened now; any other run stays closed, and a warning is logged. start_run writes every stage record, so current runs still recompose. Co-Authored-By: Claude Opus 5.5 --- backend/server/orchestrator/api.py | 14 ++++++++++++++ .../orchestrator/tests/test_greptile_followup.py | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/backend/server/orchestrator/api.py b/backend/server/orchestrator/api.py index 8f5990498..0a382e53b 100644 --- a/backend/server/orchestrator/api.py +++ b/backend/server/orchestrator/api.py @@ -168,6 +168,20 @@ def _recompose_after_correction(capture_id: str, run: RunRecord) -> None: services = current() if run.status != "done": return + present = {rec.stage for rec in services.state.stages(run.run_id)} # type: ignore[attr-defined] + if not present.issuperset(STAGE_NAMES): + # A finished run without a record for every stage predates per-stage + # records. Reopening it would let the coordinator's missing-record + # recovery re-run its upstream stages, and the re-run reads stage would + # overwrite the correction just persisted. Keep the run closed instead. + log_event( + "confirmation.recompose_skipped", + level="warning", + msg="finished run has no record for every stage", + capture_id=capture_id, + run_id=run.run_id, + ) + return now = services.clock.iso() for stage in ("criteria", "result"): rec = services.state.get_stage(run.run_id, stage) # type: ignore[attr-defined] diff --git a/backend/server/orchestrator/tests/test_greptile_followup.py b/backend/server/orchestrator/tests/test_greptile_followup.py index 039a3a4bd..d74b700dc 100644 --- a/backend/server/orchestrator/tests/test_greptile_followup.py +++ b/backend/server/orchestrator/tests/test_greptile_followup.py @@ -142,6 +142,10 @@ def test_confirmation_updates_the_reads_record(tmp_path: Path) -> None: assert stored.meter.status == "auto_fill" assert stored.confirm is not None assert stored.confirm.answer == "12345" + # This done run has no stage records, so the correction must not reopen it: + # missing-record recovery would re-run reads and overwrite the answer. + assert services.state.stages("run_reads") == [] + assert services.state.get_run("run_reads").status == "done" def test_new_attempt_clears_entered_at(tmp_path: Path) -> None: