diff --git a/.gitignore b/.gitignore index 66bcedbbe..31be9eb25 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,7 @@ server/prompts/ # Experiment outputs, local-only notes and secrets data/ +# except the committed placement case-suite fixtures +!backend/server/placement/tests/data/ .local/ .env.* diff --git a/backend/server/placement/case_suite.py b/backend/server/placement/case_suite.py new file mode 100644 index 000000000..525fd768e --- /dev/null +++ b/backend/server/placement/case_suite.py @@ -0,0 +1,533 @@ +"""Offline batch evaluator for the placement boundary. + +Runs the case manifest in ``server/placement/tests/data/cases/`` against the +unchanged top-level solver, driven through ``POST /v1/placements`` on the +placement adapter. It exists so placement work is judged on independently +authored geometry instead of repeated references to one scene. + +What it does: + +- Loads the manifest and verifies every fixture exists and matches its pinned + sha256. Missing or drifted files fail with their paths. +- Validates every source fixture against the committed base-scene/v0 contract. +- Positive cases: posts the wire scene to the placement endpoint twice, + verifies the two executions agree on everything except ``stats.elapsed_ms``, + validates the response against the result schema, and evaluates the + manifest's explicit expectations about the chosen spot, the per-check + outcomes and unsure causes, the missing-evidence requests, the failing sweep + runs and the chain ends. Decision codes are expectations too, but every + pinned decision here is a manual review forced by geometry (no fully passing + candidate exists), so it holds under the demo and strict policies alike. +- Negative cases are source-only: the W7 bridge that would convert sources + does not exist in this packet, so they are reported as awaiting conversion + with the error code and pointer the bridge must eventually produce. They are + never executed and never reported as a successful conversion. + +The solver's policy object is recorded verbatim in the report (id, version, +``rules_sha256``, notice) so any drift in rule sources is visible, while the +expectations stay geometry-only. + +Usage:: + + uv run --frozen python -m server.placement.case_suite --output=/tmp/report.json + +Exit code 0 means every case matched or is legitimately awaiting conversion; +1 means a mismatch, an execution error, or a load problem (which is also +printed to stderr with the offending paths). +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from fastapi.testclient import TestClient +from jsonschema import Draft202012Validator + +from server.placement import adapter + +MANIFEST_SCHEMA_VERSION = "placement-case-manifest/v0" +REPORT_SCHEMA_VERSION = "hss-placement-case-report/v0" +ELAPSED_KEY = "elapsed_ms" +KIND_POSITIVE = "positive" +KIND_NEGATIVE = "negative_source" +STATUS_MATCHED = "matched" +STATUS_MISMATCHED = "mismatched" +STATUS_AWAITING_CONVERSION = "awaiting_conversion" +STATUS_ERRORED = "error" +DEFAULT_MANIFEST = Path(__file__).parent / "tests" / "data" / "cases" / "manifest.json" +_REPO_ROOT = Path(__file__).parents[3] +RESULT_SCHEMA = _REPO_ROOT / "server" / "schemas" / "result.schema.json" +SOURCE_SCHEMA = _REPO_ROOT / "backend" / "server" / "contracts" / "schemas" / "scene.v0.schema.json" +# Recognized expectation keys and the container shape each must have. +EXPECTATION_KEYS = ("decision", "spot", "checks", "missing_evidence", "sweep_failing_runs", "ends") +EXPECTATION_SHAPES = { + "decision": str, + "spot": dict, + "checks": dict, + "missing_evidence": list, + "sweep_failing_runs": list, + "ends": dict, +} +# A positive case's expectations are meaningful only when they pin the decision +# and at least one geometric expectation that can actually mismatch. +GEOMETRIC_EXPECTATION_KEYS = ("spot", "checks", "missing_evidence", "sweep_failing_runs", "ends") + + +class CaseSuiteError(RuntimeError): + """Raised when the manifest or its fixtures are missing or drifted.""" + + +@dataclass(frozen=True) +class CaseRecord: + """One manifest case with its fixtures resolved to absolute paths.""" + + id: str + kind: str + source_path: Path + wire_path: Path | None + expected_error: dict[str, str] | None + expectations: dict[str, Any] + note: str | None = None + + +def _file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _check_fixture(name: str, fixture: dict[str, Any], base: Path, problems: list[str]) -> Path: + rel = fixture.get("path") + expected_sha = fixture.get("sha256") + if not isinstance(rel, str) or not rel: + problems.append(f"case fixture has no usable path: {fixture!r}") + raise CaseSuiteError(_problems_message(problems)) + path = base / rel + if not path.is_file(): + problems.append(f"missing fixture file for {name!r}: {path}") + raise CaseSuiteError(_problems_message(problems)) + if not isinstance(expected_sha, str) or not expected_sha: + problems.append(f"fixture {path} has no pinned sha256") + raise CaseSuiteError(_problems_message(problems)) + actual = _file_sha256(path) + if actual != expected_sha: + problems.append(f"drifted fixture for {name!r}: {path} sha256 {actual} != pinned {expected_sha}") + raise CaseSuiteError(_problems_message(problems)) + return path + + +def _expectations_problem(expectations: Any) -> str | None: + """Return a named problem when a positive case's expectations could not + fail a solver regression, or ``None`` when they can.""" + if not isinstance(expectations, dict): + return "must be an object of pinned expectations" + for key in EXPECTATION_KEYS: + if key in expectations and not isinstance(expectations[key], EXPECTATION_SHAPES[key]): + shape = ( + "a string" + if EXPECTATION_SHAPES[key] is str + else ("an object" if EXPECTATION_SHAPES[key] is dict else "an array") + ) + return f".{key} must be {shape} (got {type(expectations[key]).__name__})" + if not isinstance(expectations.get("decision"), str) or not expectations.get("decision"): + return "must pin a nonempty 'decision' string" + geometric = [ + key + for key in GEOMETRIC_EXPECTATION_KEYS + if isinstance(expectations.get(key), EXPECTATION_SHAPES[key]) and expectations[key] + ] + if not geometric: + return ( + "must pin at least one geometric expectation with content " + f"(one of {', '.join(GEOMETRIC_EXPECTATION_KEYS)}), so a solver change can fail the case" + ) + return None + + +def _problems_message(problems: list[str]) -> str: + return "case suite fixtures failed to load:\n" + "\n".join(f"- {p}" for p in problems) + + +def load_cases(path: Path) -> list[CaseRecord]: + """Load the manifest at ``path`` and verify its fixtures. + + The loader is the gate against a silently weakened manifest: it requires a + nonempty ``cases`` list of unique nonempty ids with a recognized ``kind`` + (``positive`` or ``negative_source``), the fixture records each kind needs + (positive: ``source`` plus ``wire``; negative: ``source``), meaningful + positive expectations (a pinned ``decision`` plus at least one geometric + expectation of the right shape) and a negative ``expected_error`` with + nonempty ``code`` and ``pointer`` strings. Fixture existence and pinned + sha256s are checked before any record is returned. Every rejection names + the manifest path and the offending field (``cases[N].field``) instead of + guessing a default. + + Raises :class:`CaseSuiteError` with the manifest path and field on any + malformed entry, and naming every missing or drifted fixture path. + """ + problems: list[str] = [] + if not path.is_file(): + raise CaseSuiteError(_problems_message([f"missing case manifest: {path}"])) + manifest = json.loads(path.read_text()) + if not isinstance(manifest, dict): + raise CaseSuiteError(f"case manifest {path}: top level must be a JSON object") + if manifest.get("schema_version") != MANIFEST_SCHEMA_VERSION: + problems.append(f"{path}: schema_version {manifest.get('schema_version')!r} != {MANIFEST_SCHEMA_VERSION!r}") + raise CaseSuiteError(_problems_message(problems)) + raw_cases = manifest.get("cases") + if not isinstance(raw_cases, list) or not raw_cases: + raise CaseSuiteError( + f"case manifest {path}: field 'cases' must be a nonempty list of case objects " + f"(got {'nothing' if raw_cases is None else type(raw_cases).__name__})" + ) + base = path.parent + records: list[CaseRecord] = [] + seen_ids: set[str] = set() + for index, raw in enumerate(raw_cases): + field = f"cases[{index}]" + if not isinstance(raw, dict): + raise CaseSuiteError(f"case manifest {path}: field {field} must be an object (got {type(raw).__name__})") + name = raw.get("id") + if not isinstance(name, str) or not name: + raise CaseSuiteError(f"case manifest {path}: field {field}.id must be a nonempty string (got {name!r})") + if name in seen_ids: + raise CaseSuiteError(f"case manifest {path}: field {field}.id duplicates id {name!r} from an earlier case") + seen_ids.add(name) + kind = raw.get("kind") + if kind not in (KIND_POSITIVE, KIND_NEGATIVE): + raise CaseSuiteError( + f"case manifest {path}: field {field}.kind {kind!r} is not a recognized case kind " + f"(expected {KIND_POSITIVE!r} or {KIND_NEGATIVE!r})" + ) + source = raw.get("source") + if not isinstance(source, dict): + raise CaseSuiteError( + f"case manifest {path}: field {field}.source must be an object with a path and a pinned sha256" + ) + source_path = _check_fixture(name, source, base, problems) + expectations: dict[str, Any] = {} + expected_error: dict[str, str] | None = None + if kind == KIND_POSITIVE: + wire = raw.get("wire") + if not isinstance(wire, dict): + raise CaseSuiteError( + f"case manifest {path}: field {field}.wire must be an object with a path and a pinned " + "sha256 (a positive case is executed, so its wire fixture is mandatory)" + ) + wire_path = _check_fixture(name, wire, base, problems) + expectations = raw.get("expectations") + problem = _expectations_problem(expectations) + if problem is not None: + raise CaseSuiteError(f"case manifest {path}: field {field}.expectations {problem}") + expectations = dict(expectations) + else: + wire_path = None + expected_error = raw.get("expected_error") + if ( + not isinstance(expected_error, dict) + or not isinstance(expected_error.get("code"), str) + or not expected_error.get("code") + or not isinstance(expected_error.get("pointer"), str) + or not expected_error.get("pointer") + ): + raise CaseSuiteError( + f"case manifest {path}: field {field}.expected_error must be an object with a nonempty " + "'code' and 'pointer' string (the error the future converter must produce)" + ) + records.append( + CaseRecord( + id=name, + kind=kind, + source_path=source_path, + wire_path=wire_path, + expected_error=expected_error, + expectations=expectations, + note=raw.get("note"), + ) + ) + return records + + +def _case_paths(record: CaseRecord, manifest_dir: Path) -> dict[str, Any]: + """Fixture paths as authored in the manifest, made relative when they live + beside it; copies of the manifest elsewhere keep their absolute paths.""" + + def as_authored(path: Path) -> str: + try: + return path.relative_to(manifest_dir).as_posix() + except ValueError: + return path.as_posix() + + return { + "source_path": as_authored(record.source_path), + "wire_path": as_authored(record.wire_path) if record.wire_path else None, + } + + +def _evaluate_expectations(expectations: dict[str, Any], body: dict[str, Any]) -> list[str]: + mismatches: list[str] = [] + expected_decision = expectations.get("decision") + if expected_decision is not None and body.get("decision") != expected_decision: + mismatches.append(f"decision {body.get('decision')!r} != expected {expected_decision!r}") + expected_spot = expectations.get("spot") + if expected_spot is not None: + spot = body.get("spot") + if spot is None: + mismatches.append(f"spot is None but expected {expected_spot!r}") + else: + for key, value in expected_spot.items(): + if spot.get(key) != value: + mismatches.append(f"spot.{key} {spot.get(key)!r} != expected {value!r}") + checks_by_id = {check["id"]: check for check in body.get("checks", [])} + for check_id, expected in (expectations.get("checks") or {}).items(): + check = checks_by_id.get(check_id) + if check is None: + mismatches.append(f"check {check_id!r} missing from response") + continue + if check.get("outcome") != expected.get("outcome"): + mismatches.append( + f"check {check_id!r} outcome {check.get('outcome')!r} != expected {expected.get('outcome')!r}" + ) + if "unsure_cause" in expected and check.get("unsure_cause") != expected["unsure_cause"]: + mismatches.append( + f"check {check_id!r} unsure_cause {check.get('unsure_cause')!r} != expected {expected['unsure_cause']!r}" + ) + for expected in expectations.get("missing_evidence") or []: + found = any( + all(actual.get(key) == value for key, value in expected.items()) + for actual in body.get("missing_evidence", []) + ) + if not found: + mismatches.append(f"missing_evidence entry not found: {expected!r}") + sweep = body.get("sweep", []) + for expected in expectations.get("sweep_failing_runs") or []: + found = any( + run.get("start_ft") == expected.get("start_ft") + and set(expected.get("failing") or []).issubset(run.get("failing") or []) + for run in sweep + ) + if not found: + mismatches.append(f"failing sweep run not found: {expected!r}") + expected_ends = expectations.get("ends") or {} + for side, kind in expected_ends.items(): + actual_kind = (body.get("ends") or {}).get(side, {}).get("kind") + if actual_kind != kind: + mismatches.append(f"end {side!r} kind {actual_kind!r} != expected {kind!r}") + return mismatches + + +def _record_check(check: dict[str, Any]) -> dict[str, Any]: + keys = ("id", "outcome", "unsure_cause", "measured_ft", "plus_minus_ft", "threshold_ft", "subject") + return {key: check.get(key) for key in keys if check.get(key) is not None} + + +def _schema_validator(path: Path) -> Draft202012Validator: + return Draft202012Validator(json.loads(path.read_text())) + + +def _schema_errors(validator: Draft202012Validator, document: Any) -> list[str]: + return [ + f"schema error at {list(error.absolute_path)}: {error.message}" + for error in list(validator.iter_errors(document))[:10] + ] + + +def _positive_case( + record: CaseRecord, + client: Any, + result_validator: Draft202012Validator, + manifest_dir: Path, +) -> dict[str, Any]: + case: dict[str, Any] = { + "id": record.id, + "kind": record.kind, + "executed": False, + "executions_agree": None, + "mismatches": [], + "errors": [], + **_case_paths(record, manifest_dir), + } + if record.note: + case["note"] = record.note + wire_bytes = record.wire_path.read_bytes() + responses = [] + for _ in range(2): + response = client.post("/v1/placements", content=wire_bytes, headers={"Content-Type": "application/json"}) + if response.status_code != 200: + case["errors"].append( + { + "code": "http_error", + "status_code": response.status_code, + "message": response.text, + } + ) + case["status"] = STATUS_ERRORED + return case + responses.append(response.json()) + first, second = responses + case["executed"] = True + case["executions_agree"] = _agree_without_elapsed(first, second) + stats = first.get("stats", {}) + case["input_sha256"] = stats.get("input_sha256") + posted_sha = _file_sha256(record.wire_path) + case["input_sha_matches_file"] = case["input_sha256"] == posted_sha + if not case["input_sha_matches_file"]: + # A response tied to different bytes must not pass, no matter how well + # its placement fields happen to satisfy the expectations. + case["mismatches"].append( + f"stats.input_sha256 {case['input_sha256']!r} does not match the sha256 of the " + f"posted wire bytes ({posted_sha})" + ) + case["policy"] = first.get("policy") + case["decision"] = first.get("decision") + case["spot"] = first.get("spot") + case["checks"] = [_record_check(check) for check in first.get("checks", [])] + case["missing_evidence"] = first.get("missing_evidence", []) + case["ends"] = first.get("ends") + case["sweep_failing"] = [ + {key: run.get(key) for key in ("start_ft", "outcome", "failing", "unsure")} + for run in first.get("sweep", []) + if run.get("outcome") != "pass" + ] + case["objects_not_used"] = first.get("objects_not_used", []) + schema_errors = _schema_errors(result_validator, first) + if schema_errors: + case["errors"].append({"code": "result_schema_invalid", "message": "; ".join(schema_errors)}) + if not case["executions_agree"]: + case["mismatches"].append("the two direct executions disagree beyond stats.elapsed_ms") + case["mismatches"].extend(_evaluate_expectations(record.expectations, first)) + case["status"] = STATUS_MISMATCHED if case["mismatches"] or case["errors"] else STATUS_MATCHED + return case + + +def _agree_without_elapsed(first: dict[str, Any], second: dict[str, Any]) -> bool: + def without_elapsed(body: dict[str, Any]) -> dict[str, Any]: + stats = dict(body.get("stats") or {}) + stats.pop(ELAPSED_KEY, None) + return {**body, "stats": stats} + + return json.dumps(without_elapsed(first), sort_keys=True) == json.dumps(without_elapsed(second), sort_keys=True) + + +def _negative_case(record: CaseRecord, manifest_dir: Path) -> dict[str, Any]: + case: dict[str, Any] = { + "id": record.id, + "kind": record.kind, + "status": STATUS_AWAITING_CONVERSION, + "executed": False, + "expected_error": record.expected_error, + "mismatches": [], + "errors": [], + **_case_paths(record, manifest_dir), + } + if record.note: + case["note"] = record.note + return case + + +def run_suite(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]: + """Run every case in the manifest and return the full report dict.""" + records = load_cases(manifest_path) + manifest_dir = manifest_path.parent + source_validator = _schema_validator(SOURCE_SCHEMA) + result_validator = _schema_validator(RESULT_SCHEMA) + cases: list[dict[str, Any]] = [] + for record in records: + schema_errors = _schema_errors(source_validator, json.loads(record.source_path.read_text())) + if record.kind == KIND_POSITIVE: + case = _positive_case(record, _get_client(), result_validator, manifest_dir) + if schema_errors: + case["errors"].append({"code": "source_schema_invalid", "message": "; ".join(schema_errors)}) + case["status"] = STATUS_ERRORED + elif record.kind == KIND_NEGATIVE: + case = _negative_case(record, manifest_dir) + if schema_errors: + case["errors"].append({"code": "source_schema_invalid", "message": "; ".join(schema_errors)}) + case["status"] = STATUS_ERRORED + else: + case = { + "id": record.id, + "kind": record.kind, + "status": STATUS_ERRORED, + "executed": False, + "errors": [{"code": "unknown_case_kind", "message": f"kind {record.kind!r} is not supported"}], + "mismatches": [], + **_case_paths(record, manifest_dir), + } + cases.append(case) + report: dict[str, Any] = { + "schema_version": REPORT_SCHEMA_VERSION, + "generated_at": datetime.now(UTC).isoformat(), + "manifest": { + "path": manifest_path.as_posix(), + "sha256": _file_sha256(manifest_path), + }, + "cases": cases, + } + report["summary"] = _summary(cases) + report["ok"] = report["summary"]["mismatched"] == 0 and report["summary"]["errored"] == 0 + return report + + +def _summary(cases: list[dict[str, Any]]) -> dict[str, int]: + statuses = [case["status"] for case in cases] + return { + "cases": len(cases), + "positive": sum(1 for case in cases if case["kind"] == KIND_POSITIVE), + "negative_source": sum(1 for case in cases if case["kind"] == KIND_NEGATIVE), + "matched": statuses.count(STATUS_MATCHED), + "mismatched": statuses.count(STATUS_MISMATCHED), + "awaiting_conversion": statuses.count(STATUS_AWAITING_CONVERSION), + "errored": statuses.count(STATUS_ERRORED), + } + + +def _build_client() -> Any: + modules = adapter.load_api() + return TestClient(modules["api"].app) + + +_client: Any = None + + +def _get_client() -> Any: + global _client + if _client is None: + _client = _build_client() + return _client + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run the offline placement case suite.") + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST, help="Path to the case manifest.") + parser.add_argument("--output", type=Path, required=True, help="Where to write the JSON report.") + args = parser.parse_args(argv) + try: + report = run_suite(args.manifest) + except CaseSuiteError as error: + print(error, file=sys.stderr) + return 1 + args.output.write_text(json.dumps(report, indent=2) + "\n") + summary = report["summary"] + print( + f"cases={summary['cases']} matched={summary['matched']} " + f"awaiting_conversion={summary['awaiting_conversion']} " + f"mismatched={summary['mismatched']} errored={summary['errored']}" + ) + for case in report["cases"]: + if case["status"] not in (STATUS_MATCHED, STATUS_AWAITING_CONVERSION): + print(f"case {case['id']}: status={case['status']}", file=sys.stderr) + for mismatch in case["mismatches"]: + print(f" mismatch: {mismatch}", file=sys.stderr) + for error in case["errors"]: + print(f" error: {error}", file=sys.stderr) + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/server/placement/tests/data/cases/generated-hazard-unsupported.source.json b/backend/server/placement/tests/data/cases/generated-hazard-unsupported.source.json new file mode 100644 index 000000000..b30a6b7d0 --- /dev/null +++ b/backend/server/placement/tests/data/cases/generated-hazard-unsupported.source.json @@ -0,0 +1,78 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst7-generated-hazard-unsupported", + "provenance": { + "data": "synthetic", + "site": "burst7-generated-hazard-unsupported", + "producer": {"path": "hand-authored", "version": "burst7"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "notes": [ + "Negative case for the burst7 placement case suite: a schema-valid source whose second object is a gas meter a generative model produced (evidence generated). The future bridge (W7) must refuse it with code placement_scene_unsupported at pointer /objects/1/evidence: generated geometry is never evidence. This suite never runs that bridge; the case stays labelled awaiting conversion." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "obj-meter", "plus_minus_m": 0.09} + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [[-3.048, 0.0], [4.572, 0.0]], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": ["side"], + "ends": { + "left": {"kind": "limit", "s_m": -3.048, "note": "corner post"}, + "right": {"kind": "unexplored", "s_m": 4.572, "note": "wall continues right, not walked"} + } + }, + "ground": { + "surface": {"kind": "plane", "normal": [0.0, 1.0, 0.0], "offset_m": 0.0}, + "plus_minus_m": 0.05 + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": {"kind": "wall_rect", "wall_id": "side", "s_m": [-0.15, 0.15], "bottom_m": 0.15, "top_m": 1.2, "standoff_m": 0.1}, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + }, + { + "id": "obj-gas-generated", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "footprint", + "polygon_m": [[1.2192, 0.0], [1.524, 0.0], [1.524, 0.3048], [1.2192, 0.3048]], + "bottom_m": 0.15, + "top_m": 0.76 + }, + "confidence": 0.4, + "plus_minus_m": 0.3, + "source": {"kind": "generated", "model": "inpaint-demo"}, + "evidence": "generated" + } + ], + "generated_regions": [ + { + "id": "gen-hazard", + "geometry": {"kind": "box", "center_m": [1.3716, 0.5, 0.45], "size_m": [0.6, 0.9, 0.5], "yaw_deg": 0.0}, + "method": "hole-filling", + "note": "The hazard above was completed, not observed." + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": {"min_views": 1, "max_incidence_deg": {"wall": 60, "ground": 60}} + } +} diff --git a/backend/server/placement/tests/data/cases/hazard-left-ground-gap.source.json b/backend/server/placement/tests/data/cases/hazard-left-ground-gap.source.json new file mode 100644 index 000000000..3e4fe711b --- /dev/null +++ b/backend/server/placement/tests/data/cases/hazard-left-ground-gap.source.json @@ -0,0 +1,105 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst7-hazard-left-ground-gap", + "provenance": { + "data": "synthetic", + "site": "burst7-hazard-left-ground-gap", + "producer": {"path": "hand-authored", "version": "burst7"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "notes": [ + "Authored for the burst7 placement case suite. It pairs with hazard-left-ground-gap.wire.json by construction (the same wall, two hazards, coverage gap and ends, described once in meters and once in feet by hand), not by running any converter." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "obj-meter", "plus_minus_m": 0.09} + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [[-6.096, 0.0], [3.6576, 0.0]], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": ["side"], + "ends": { + "left": {"kind": "unexplored", "s_m": -6.096, "note": "wall continues left round the corner"}, + "right": {"kind": "limit", "s_m": 3.6576, "note": "driveway cut at the right end"} + } + }, + "ground": { + "surface": {"kind": "plane", "normal": [0.0, 1.0, 0.0], "offset_m": 0.0}, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [[-6.096, 0.0], [3.6576, 0.0], [3.6576, 1.2192], [-6.096, 1.2192]], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [[-6.096, 1.2192], [3.6576, 1.2192], [3.6576, 4.2672], [-6.096, 4.2672]], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": {"kind": "wall_rect", "wall_id": "side", "s_m": [-0.15, 0.15], "bottom_m": 0.15, "top_m": 1.2, "standoff_m": 0.1}, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + }, + { + "id": "obj-gas-near", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "footprint", + "polygon_m": [[-1.8288, 0.0], [-1.524, 0.0], [-1.524, 0.3048], [-1.8288, 0.3048]], + "bottom_m": 0.15, + "top_m": 0.76 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + }, + { + "id": "obj-gas-far", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "footprint", + "polygon_m": [[-3.9624, 0.0], [-3.6576, 0.0], [-3.6576, 0.3048], [-3.9624, 0.3048]], + "bottom_m": 0.15, + "top_m": 0.76 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": {"min_views": 1, "max_incidence_deg": {"wall": 60, "ground": 60}}, + "bands": [ + {"wall_id": "side", "band": "wall", "s_m": [-6.096, 3.6576], "out_m": 2.0}, + {"wall_id": "side", "band": "ground", "s_m": [-4.2672, 0.6096], "out_m": 4.2672}, + {"wall_id": "side", "band": "overhead", "s_m": [-6.096, 3.6576], "out_m": 3.0}, + {"wall_id": "side", "band": "facing", "s_m": [-6.096, 3.6576], "out_m": 4.2672} + ] + } +} diff --git a/backend/server/placement/tests/data/cases/hazard-left-ground-gap.wire.json b/backend/server/placement/tests/data/cases/hazard-left-ground-gap.wire.json new file mode 100644 index 000000000..623438e3b --- /dev/null +++ b/backend/server/placement/tests/data/cases/hazard-left-ground-gap.wire.json @@ -0,0 +1,31 @@ +{ + "schema_version": "1.0", + "meter": {"pos": [0.0, 0.5, 0.0], "wall_id": "side", "plus_minus_ft": 0.3}, + "walls": [ + {"id": "side", "baseline": [[-20.0, 0.0], [12.0, 0.0]], "height_ft": 9.0, "source": "tap"} + ], + "objects": [ + {"type": "gas_meter", "wall_id": "side", "span_ft": [-6.0, -5.0], "bottom_ft": 0.5, "top_ft": 2.5, + "source": "tap", "plus_minus_ft": 0.3, + "footprint": [[-6.0, 0.0], [-5.0, 0.0], [-5.0, 1.0], [-6.0, 1.0]]}, + {"type": "gas_meter", "wall_id": "side", "span_ft": [-13.0, -12.0], "bottom_ft": 0.5, "top_ft": 2.5, + "source": "tap", "plus_minus_ft": 0.3, + "footprint": [[-13.0, 0.0], [-12.0, 0.0], [-12.0, 1.0], [-13.0, 1.0]]} + ], + "ground": [ + {"type": "mulch", "polygon": [[-20.0, 0.0], [12.0, 0.0], [12.0, 4.0], [-20.0, 4.0]]}, + {"type": "lawn", "polygon": [[-20.0, 4.0], [12.0, 4.0], [12.0, 14.0], [-20.0, 14.0]]} + ], + "overheads": [], + "facing": [{"wall_id": "side", "span_ft": [-20.0, 12.0], "depth_ft": 14.0, "plus_minus_ft": 0.5}], + "coverage": { + "ends": {"left": {"kind": "unexplored", "note": "wall continues left round the corner"}, + "right": {"kind": "limit", "note": "driveway cut at the right end"}}, + "observed": [ + {"band": "wall", "span_ft": [-20.0, 12.0]}, + {"band": "ground", "span_ft": [-14.0, 2.0], "out_ft": 14.0}, + {"band": "overhead", "span_ft": [-20.0, 12.0]}, + {"band": "facing", "span_ft": [-20.0, 12.0]} + ] + } +} diff --git a/backend/server/placement/tests/data/cases/hazard-right-ground-gap.source.json b/backend/server/placement/tests/data/cases/hazard-right-ground-gap.source.json new file mode 100644 index 000000000..287d6133d --- /dev/null +++ b/backend/server/placement/tests/data/cases/hazard-right-ground-gap.source.json @@ -0,0 +1,91 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst7-hazard-right-ground-gap", + "provenance": { + "data": "synthetic", + "site": "burst7-hazard-right-ground-gap", + "producer": {"path": "hand-authored", "version": "burst7"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "notes": [ + "Authored for the burst7 placement case suite. It pairs with hazard-right-ground-gap.wire.json by construction (the same wall, hazard, coverage gap and ends, described once in meters and once in feet by hand), not by running any converter." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "obj-meter", "plus_minus_m": 0.09} + }, + "walls": [ + { + "id": "side", + "role": "house", + "baseline_m": [[-0.6096, 0.0], [6.096, 0.0]], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": ["side"], + "ends": { + "left": {"kind": "limit", "s_m": -0.6096, "note": "fence at the left property line"}, + "right": {"kind": "unexplored", "s_m": 6.096, "note": "wall continues right, not walked"} + } + }, + "ground": { + "surface": {"kind": "plane", "normal": [0.0, 1.0, 0.0], "offset_m": 0.0}, + "plus_minus_m": 0.05, + "regions": [ + { + "id": "mulch", + "material": "mulch", + "polygon_m": [[-0.6096, 0.0], [6.096, 0.0], [6.096, 1.2192], [-0.6096, 1.2192]], + "evidence": "observed" + }, + { + "id": "lawn", + "material": "lawn", + "polygon_m": [[-0.6096, 1.2192], [6.096, 1.2192], [6.096, 4.2672], [-0.6096, 4.2672]], + "evidence": "observed" + } + ] + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": {"kind": "wall_rect", "wall_id": "side", "s_m": [-0.15, 0.15], "bottom_m": 0.15, "top_m": 1.2, "standoff_m": 0.1}, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + }, + { + "id": "obj-gas", + "class": "gas_meter", + "wall_id": "side", + "geometry": { + "kind": "footprint", + "polygon_m": [[1.0668, 0.0], [1.3716, 0.0], [1.3716, 0.3048], [1.0668, 0.3048]], + "bottom_m": 0.15, + "top_m": 0.76 + }, + "confidence": 0.9, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": {"min_views": 1, "max_incidence_deg": {"wall": 60, "ground": 60}}, + "bands": [ + {"wall_id": "side", "band": "wall", "s_m": [-0.6096, 6.096], "out_m": 2.0}, + {"wall_id": "side", "band": "ground", "s_m": [-0.6096, 0.6096], "out_m": 4.2672}, + {"wall_id": "side", "band": "ground", "s_m": [2.7432, 3.3528], "out_m": 4.2672}, + {"wall_id": "side", "band": "overhead", "s_m": [-0.6096, 6.096], "out_m": 3.0}, + {"wall_id": "side", "band": "facing", "s_m": [-0.6096, 6.096], "out_m": 4.2672} + ] + } +} diff --git a/backend/server/placement/tests/data/cases/hazard-right-ground-gap.wire.json b/backend/server/placement/tests/data/cases/hazard-right-ground-gap.wire.json new file mode 100644 index 000000000..e1e9886e4 --- /dev/null +++ b/backend/server/placement/tests/data/cases/hazard-right-ground-gap.wire.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0", + "meter": {"pos": [0.0, 0.5, 0.0], "wall_id": "side", "plus_minus_ft": 0.3}, + "walls": [ + {"id": "side", "baseline": [[-2.0, 0.0], [20.0, 0.0]], "height_ft": 9.0, "source": "tap"} + ], + "objects": [ + {"type": "gas_meter", "wall_id": "side", "span_ft": [3.5, 4.5], "bottom_ft": 0.5, "top_ft": 2.5, + "source": "tap", "plus_minus_ft": 0.3, + "footprint": [[3.5, 0.0], [4.5, 0.0], [4.5, 1.0], [3.5, 1.0]]} + ], + "ground": [ + {"type": "mulch", "polygon": [[-2.0, 0.0], [20.0, 0.0], [20.0, 4.0], [-2.0, 4.0]]}, + {"type": "lawn", "polygon": [[-2.0, 4.0], [20.0, 4.0], [20.0, 14.0], [-2.0, 14.0]]} + ], + "overheads": [], + "facing": [{"wall_id": "side", "span_ft": [-2.0, 20.0], "depth_ft": 14.0, "plus_minus_ft": 0.5}], + "coverage": { + "ends": {"left": {"kind": "limit", "note": "fence at the left property line"}, + "right": {"kind": "unexplored", "note": "wall continues right, not walked"}}, + "observed": [ + {"band": "wall", "span_ft": [-2.0, 20.0]}, + {"band": "ground", "span_ft": [-2.0, 2.0], "out_ft": 14.0}, + {"band": "ground", "span_ft": [9.0, 11.0], "out_ft": 14.0}, + {"band": "overhead", "span_ft": [-2.0, 20.0]}, + {"band": "facing", "span_ft": [-2.0, 20.0]} + ] + } +} diff --git a/backend/server/placement/tests/data/cases/manifest.json b/backend/server/placement/tests/data/cases/manifest.json new file mode 100644 index 000000000..9ec8c3b7b --- /dev/null +++ b/backend/server/placement/tests/data/cases/manifest.json @@ -0,0 +1,188 @@ +{ + "schema_version": "placement-case-manifest/v0", + "description": [ + "Placement-boundary case suite. Each positive case pairs a schema-valid synthetic base-scene/v0 source with an independently authored root-wire scene; both describe the same wall, a nearby gas hazard, a stretch of ground nobody observed, and an unexplored end. The pair is authored by hand on both sides (the same geometry in meters and in feet), never produced by running the converter, which does not exist yet.", + "Positive cases run the unchanged top-level solver through POST /v1/placements on the wire scene and compare the whole response, except stats.elapsed_ms, between two direct executions, against these expectations. Negative cases are source-only: the W7 bridge that would convert sources must refuse them with the declared error code and pointer, so they stay labelled awaiting conversion until that bridge consumes them.", + "Every fixture path is relative to this manifest's directory and pinned by sha256; a missing or drifted file fails the suite with its path." + ], + "cases": [ + { + "id": "hazard-right-ground-gap", + "kind": "positive", + "source": { + "path": "hazard-right-ground-gap.source.json", + "sha256": "670624f98e6366c15d8d0154e9d5f98338cbe65a3d31f204475619fe22c9f714" + }, + "wire": { + "path": "hazard-right-ground-gap.wire.json", + "sha256": "7658044cc2d6bf30b1b6f39542adc0ad5f26228a58bca8562757779be407d7f5" + }, + "expectations": { + "decision": "manual_review", + "spot": { + "wall_id": "side", + "segment": 0, + "span_ft": [ + 7.5, + 10.083333 + ], + "outcome": "unsure" + }, + "checks": { + "gas_clearance": { + "outcome": "unsure", + "unsure_cause": "margin" + }, + "ground_surface": { + "outcome": "unsure", + "unsure_cause": "unobserved" + }, + "route_length": { + "outcome": "pass" + }, + "facing_gap": { + "outcome": "pass" + } + }, + "missing_evidence": [ + { + "kind": "band", + "band": "ground", + "span_ft": [ + -4.427701, + 20.0 + ], + "out_ft": 13.761035 + }, + { + "kind": "past_end", + "side": "right", + "span_ft": [ + 20.0, + 20.0 + ] + } + ], + "sweep_failing_runs": [ + { + "start_ft": [ + 2.708333, + 5.541667 + ], + "failing": [ + "gas_clearance" + ] + } + ], + "ends": { + "left": "limit", + "right": "unexplored" + } + }, + "note": "Geometry facts, not policy: the gas meter at s [3.5, 4.5] ft leaves the nearest non-failing spot just past it (measured 3.0 ft +- 2.21, unsure by margin, not by observation); the ground band is observed only at [-2, 2] and [9, 11] ft, so the chosen spot's ground is unsure-unobserved and the response must request the ground band; the right end is unexplored within reach, so the response must ask for a walk past it; the left end is a real limit." + }, + { + "id": "hazard-left-ground-gap", + "kind": "positive", + "source": { + "path": "hazard-left-ground-gap.source.json", + "sha256": "fb299f305040d1f1651127a63c5f050c5f8c76be3204831e113d66db79fec32b" + }, + "wire": { + "path": "hazard-left-ground-gap.wire.json", + "sha256": "8b9b9b7383eba0315fd2591dd962390bc5e0dca12c3bf3b48e65020f6a90a760" + }, + "expectations": { + "decision": "manual_review", + "spot": { + "wall_id": "side", + "segment": 0, + "span_ft": [ + 2.708333, + 5.291667 + ], + "outcome": "unsure" + }, + "checks": { + "gas_clearance": { + "outcome": "unsure", + "unsure_cause": "unobserved" + }, + "ground_surface": { + "outcome": "unsure", + "unsure_cause": "unobserved" + }, + "opening_clearance": { + "outcome": "pass" + }, + "route_length": { + "outcome": "pass" + } + }, + "missing_evidence": [ + { + "kind": "band", + "band": "ground", + "span_ft": [ + 2.004999, + 16.451777 + ], + "out_ft": 12.993443 + }, + { + "kind": "past_end", + "side": "left", + "span_ft": [ + -20.0, + -20.0 + ] + } + ], + "sweep_failing_runs": [ + { + "start_ft": [ + -6.958333, + -5.291667 + ], + "failing": [ + "gas_clearance" + ] + } + ], + "ends": { + "left": "unexplored", + "right": "limit" + } + }, + "note": "Geometry facts, not policy: two gas meters block the observed left stretch (their failing sweep runs are pinned), the ground is observed only to s 2 ft so the nearest non-failing spot sits right of the meter with unsure-unobserved ground and an unobserved-caused gas request, and the left end is unexplored within reach, so the response must ask for a walk past it; the right end is a real limit." + }, + { + "id": "multiwall-source-unsupported", + "kind": "negative_source", + "source": { + "path": "multiwall-source-unsupported.source.json", + "sha256": "c1b3d9dd4b0c7e6926b3f25ce728a6c0e26f969c6f73494eff5202c983736eea" + }, + "wire": null, + "expected_error": { + "code": "placement_scene_unsupported", + "pointer": "/walls" + }, + "note": "Awaiting conversion: the source is schema-valid, so this is a content refusal, not a parse failure. The W7 bridge must decline a two-wall chain with code placement_scene_unsupported at pointer /walls. This suite never runs the missing bridge, and this case is never reported as a successful conversion." + }, + { + "id": "generated-hazard-unsupported", + "kind": "negative_source", + "source": { + "path": "generated-hazard-unsupported.source.json", + "sha256": "a9b9a35f49f8811e7dd1fbccb9b1eead6de0418494192fa49ee51c5fe603ce34" + }, + "wire": null, + "expected_error": { + "code": "placement_scene_unsupported", + "pointer": "/objects/1/evidence" + }, + "note": "Awaiting conversion: the second object's hazard is generated, never evidence, so the W7 bridge must decline it with code placement_scene_unsupported at pointer /objects/1/evidence. This suite never runs the missing bridge, and this case is never reported as a successful conversion." + } + ] +} diff --git a/backend/server/placement/tests/data/cases/multiwall-source-unsupported.source.json b/backend/server/placement/tests/data/cases/multiwall-source-unsupported.source.json new file mode 100644 index 000000000..c9f4ff012 --- /dev/null +++ b/backend/server/placement/tests/data/cases/multiwall-source-unsupported.source.json @@ -0,0 +1,66 @@ +{ + "format": "base-scene/v0", + "scene_id": "burst7-multiwall-unsupported", + "provenance": { + "data": "synthetic", + "site": "burst7-multiwall-unsupported", + "producer": {"path": "hand-authored", "version": "burst7"}, + "scale_source": "native_metric", + "gravity_source": "ground_truth", + "notes": [ + "Negative case for the burst7 placement case suite: a schema-valid source whose chain holds two walls. The future bridge (W7) must refuse it with code placement_scene_unsupported at pointer /walls. This suite never runs that bridge; the case stays labelled awaiting conversion." + ] + }, + "frame": { + "units": "m", + "convention": "meter_anchored/v0", + "anchor": {"element_id": "obj-meter", "plus_minus_m": 0.09} + }, + "walls": [ + { + "id": "front", + "role": "house", + "baseline_m": [[0.0, 0.0], [6.096, 0.0]], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + }, + { + "id": "side", + "role": "house", + "baseline_m": [[6.096, 0.0], [6.096, -3.6576]], + "height_m": 2.7432, + "plus_minus_m": 0.09, + "evidence": "observed" + } + ], + "chain": { + "wall_ids": ["front", "side"], + "corners": [ + {"id": "corner-1", "left_wall": "front", "right_wall": "side", "point_m": [6.096, 0.0], "turn_deg": -90.0, "plus_minus_m": 0.09, "evidence": "observed"} + ], + "ends": { + "left": {"kind": "limit", "s_m": 0.0, "note": "shed blocks the left end"}, + "right": {"kind": "limit", "s_m": 9.7536, "note": "fence at the right property line"} + } + }, + "ground": { + "surface": {"kind": "plane", "normal": [0.0, 1.0, 0.0], "offset_m": 0.0}, + "plus_minus_m": 0.05 + }, + "objects": [ + { + "id": "obj-meter", + "class": "meter", + "geometry": {"kind": "wall_rect", "wall_id": "front", "s_m": [0.3, 0.6], "bottom_m": 0.15, "top_m": 1.2, "standoff_m": 0.1}, + "confidence": 0.99, + "plus_minus_m": 0.09, + "source": {"kind": "tap"}, + "evidence": "observed" + } + ], + "coverage": { + "method": "synthetic-authored", + "rule": {"min_views": 1, "max_incidence_deg": {"wall": 60, "ground": 60}} + } +} diff --git a/backend/tests/test_placement_case_suite.py b/backend/tests/test_placement_case_suite.py new file mode 100644 index 000000000..e9e3ec33b --- /dev/null +++ b/backend/tests/test_placement_case_suite.py @@ -0,0 +1,340 @@ +"""Focused tests for the offline placement case suite and its manifest. + +These tests exercise the suite machinery, not solver geometry twice: the +manifest's pinned expectations are evaluated once per full-suite run and the +individual tests assert on the resulting report plus the failure paths of the +loader. Expectation mutations are checked against copies of the manifest. +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from server.placement import case_suite as cs + +MANIFEST = cs.DEFAULT_MANIFEST + + +@pytest.fixture(scope="module") +def report() -> dict: + return cs.run_suite(MANIFEST) + + +def test_load_cases_interface() -> None: + records = cs.load_cases(MANIFEST) + ids = [record.id for record in records] + assert ids == [ + "hazard-right-ground-gap", + "hazard-left-ground-gap", + "multiwall-source-unsupported", + "generated-hazard-unsupported", + ] + by_id = {record.id: record for record in records} + assert all(record.source_path.is_file() for record in records) + for positive in ("hazard-right-ground-gap", "hazard-left-ground-gap"): + assert by_id[positive].kind == cs.KIND_POSITIVE + assert by_id[positive].wire_path is not None and by_id[positive].wire_path.is_file() + assert by_id[positive].expectations["decision"] == "manual_review" + assert by_id[positive].expectations["spot"]["span_ft"] + for negative in ("multiwall-source-unsupported", "generated-hazard-unsupported"): + assert by_id[negative].kind == cs.KIND_NEGATIVE + assert by_id[negative].wire_path is None + assert by_id[negative].expected_error == { + "code": "placement_scene_unsupported", + "pointer": "/walls" if negative.startswith("multiwall") else "/objects/1/evidence", + } + + +def test_positive_cases_match_report(report: dict) -> None: + assert report["schema_version"] == cs.REPORT_SCHEMA_VERSION + assert report["summary"]["awaiting_conversion"] == 2 + assert report["summary"]["matched"] == 2 + assert report["summary"]["mismatched"] == 0 + for case in report["cases"]: + if case["kind"] != cs.KIND_POSITIVE: + continue + assert case["status"] == cs.STATUS_MATCHED, case + assert case["executed"] is True + assert case["executions_agree"] is True + assert case["input_sha_matches_file"] is True + assert case["decision"] == "manual_review" + assert case["policy"]["rules_sha256"] + # The wire scene's own hash is echoed back by the solver. + wire_bytes = Path(cs.DEFAULT_MANIFEST).parent / case["wire_path"] + assert case["input_sha256"] == cs._file_sha256(wire_bytes) + + +def test_negative_cases_never_claim_conversion(report: dict) -> None: + for case in report["cases"]: + if case["kind"] != cs.KIND_NEGATIVE: + continue + assert case["status"] == cs.STATUS_AWAITING_CONVERSION + assert case["executed"] is False + assert "awaiting conversion" in (case["note"] or "").lower() + assert case["expected_error"]["code"] == "placement_scene_unsupported" + statuses = {case["status"] for case in report["cases"]} + assert "converted" not in statuses + + +def _absolutized_payload() -> dict: + """Manifest payload with fixture paths made absolute, so a copy of the + manifest living anywhere still resolves the committed fixtures.""" + payload = json.loads(MANIFEST.read_text()) + cases_dir = MANIFEST.parent + for case in payload["cases"]: + case["source"]["path"] = str(cases_dir / case["source"]["path"]) + if case.get("wire"): + case["wire"]["path"] = str(cases_dir / case["wire"]["path"]) + return payload + + +def test_missing_fixture_fails_with_path(tmp_path: Path) -> None: + payload = _absolutized_payload() + missing = tmp_path / "does-not-exist.wire.json" + payload["cases"][0]["wire"]["path"] = str(missing) + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert str(missing) in str(excinfo.value) + + +def test_drifted_fixture_fails_with_path(tmp_path: Path) -> None: + payload = _absolutized_payload() + fixture = tmp_path / "hazard-right-ground-gap.wire.json" + fixture.write_bytes(b"{}") # same path name, different bytes, wrong hash + payload["cases"][0]["wire"]["sha256"] = "0" * 64 + payload["cases"][0]["wire"]["path"] = str(fixture) + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert str(fixture) in str(excinfo.value) + + +def _run_mutated(tmp_path: Path, mutate) -> dict: + """Run the suite against a mutated manifest copy placed in tmp.""" + payload = _absolutized_payload() + mutate(payload) + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + return cs.run_suite(drift) + + +def test_changed_hazard_expectation_fails(tmp_path: Path) -> None: + report = _run_mutated( + tmp_path, + lambda payload: payload["cases"][0]["expectations"]["checks"].update( + {"gas_clearance": {"outcome": "fail", "unsure_cause": "margin"}} + ), + ) + case = report["cases"][0] + assert case["status"] == cs.STATUS_MISMATCHED + assert any("gas_clearance" in mismatch for mismatch in case["mismatches"]) + assert report["ok"] is False + + +def test_changed_coverage_expectation_fails(tmp_path: Path) -> None: + report = _run_mutated( + tmp_path, + lambda payload: payload["cases"][0]["expectations"]["missing_evidence"][0].update({"out_ft": 9.0}), + ) + case = report["cases"][0] + assert case["status"] == cs.STATUS_MISMATCHED + assert any("missing_evidence" in mismatch for mismatch in case["mismatches"]) + assert report["ok"] is False + + +def test_cli_writes_report_from_clean_tmp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + monkeypatch.chdir(tmp_path) + out = tmp_path / "report.json" + exit_code = cs.main(["--output", str(out)]) + assert exit_code == 0 + assert out.is_file() + written = json.loads(out.read_text()) + assert written["schema_version"] == cs.REPORT_SCHEMA_VERSION + assert written["summary"]["matched"] == 2 + assert "matched=2" in capsys.readouterr().out + + +def test_two_executions_agree_only_elapsed_differs() -> None: + """Direct in-process check: same scene twice differs only in elapsed_ms.""" + modules = cs.adapter.load_api() + from fastapi.testclient import TestClient + + wire = MANIFEST.parent / "hazard-left-ground-gap.wire.json" + with TestClient(modules["api"].app) as client: + bodies = [ + client.post( + "/v1/placements", content=wire.read_bytes(), headers={"Content-Type": "application/json"} + ).json() + for _ in range(2) + ] + assert bodies[0]["stats"]["elapsed_ms"] is not None + for body in bodies: + body["stats"] = {k: v for k, v in body["stats"].items() if k != "elapsed_ms"} + assert json.dumps(bodies[0], sort_keys=True) == json.dumps(bodies[1], sort_keys=True) + + +def test_manifest_copy_is_untouched_by_mutations() -> None: + payload = json.loads(MANIFEST.read_text()) + original = copy.deepcopy(payload) + payload["cases"][0]["expectations"]["decision"] = "reject" + assert json.loads(MANIFEST.read_text()) == original + + +def test_missing_cases_key_fails_with_manifest_path(tmp_path: Path) -> None: + """A manifest without a cases key cannot silently disable the gate.""" + payload = _absolutized_payload() + del payload["cases"] + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + message = str(excinfo.value) + assert str(drift) in message + assert "cases" in message + + +def test_empty_cases_list_fails_at_the_cli(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + payload = _absolutized_payload() + payload["cases"] = [] + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + out = tmp_path / "report.json" + assert cs.main(["--manifest", str(drift), "--output", str(out)]) == 1 + assert "cases" in capsys.readouterr().err + assert not out.exists() + + +def test_duplicate_case_ids_fail_with_field(tmp_path: Path) -> None: + payload = _absolutized_payload() + payload["cases"][1]["id"] = payload["cases"][0]["id"] + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + message = str(excinfo.value) + assert "cases[1].id" in message + assert payload["cases"][0]["id"] in message + + +@pytest.mark.parametrize("kind", ["positve", "", 3, None]) +def test_malformed_case_kinds_fail_with_field(tmp_path: Path, kind: object) -> None: + payload = _absolutized_payload() + payload["cases"][0]["kind"] = kind + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert "cases[0].kind" in str(excinfo.value) + + +@pytest.mark.parametrize( + ("mangle", "why"), + [ + (lambda case: case.pop("expectations"), "absent"), + (lambda case: case.update(expectations={}), "empty"), + (lambda case: case.update(expectations={"decision": "manual_review"}), "decision only"), + (lambda case: case["expectations"].update(spot="the good spot"), "spot wrong type"), + (lambda case: case["expectations"].update(checks=["gas_clearance"]), "checks wrong type"), + ], + ids=["absent", "empty", "decision-only", "spot-wrong-type", "checks-wrong-type"], +) +def test_unusable_positive_expectations_fail_with_field(tmp_path: Path, mangle, why: str) -> None: + payload = _absolutized_payload() + mangle(payload["cases"][0]) + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert "cases[0].expectations" in str(excinfo.value), why + + +def test_negative_case_requires_declared_error(tmp_path: Path) -> None: + payload = _absolutized_payload() + payload["cases"][2].pop("expected_error") + drift = tmp_path / "manifest.json" + drift.write_text(json.dumps(payload)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(drift) + assert "cases[2].expected_error" in str(excinfo.value) + + thin = _absolutized_payload() + thin["cases"][2]["expected_error"] = {"code": "placement_scene_unsupported"} + thin_manifest = tmp_path / "thin.json" + thin_manifest.write_text(json.dumps(thin)) + with pytest.raises(cs.CaseSuiteError) as excinfo: + cs.load_cases(thin_manifest) + assert "cases[2].expected_error" in str(excinfo.value) + + +def _forging_client(monkeypatch: pytest.MonkeyPatch, mutate) -> None: + """Swap the suite's client for one that mutates every 200 response body, + simulating a solver/API regression in the echoed response.""" + inner = cs._build_client() + + class _ShimResponse: + def __init__(self, status_code: int, body: dict) -> None: + self.status_code = status_code + self.text = "" + self._body = body + + def json(self) -> dict: + return self._body + + class _ForgingClient: + def post(self, *args: object, **kwargs: object) -> _ShimResponse: + response = inner.post(*args, **kwargs) + assert response.status_code == 200 + body = response.json() + mutate(body) + return _ShimResponse(200, body) + + monkeypatch.setattr(cs, "_get_client", lambda: _ForgingClient()) + + +def test_forged_input_sha256_fails_the_report(monkeypatch: pytest.MonkeyPatch) -> None: + """A response whose echoed hash points at other bytes must not match.""" + _forging_client( + monkeypatch, + lambda body: body.setdefault("stats", {}).update(input_sha256="0" * 64), + ) + report = cs.run_suite(MANIFEST) + case = next(case for case in report["cases"] if case["kind"] == cs.KIND_POSITIVE) + assert case["status"] == cs.STATUS_MISMATCHED + assert case["input_sha_matches_file"] is False + assert any("input_sha256" in mismatch for mismatch in case["mismatches"]) + assert report["ok"] is False + + +def test_missing_input_sha256_fails_the_report(monkeypatch: pytest.MonkeyPatch) -> None: + _forging_client( + monkeypatch, + lambda body: body.setdefault("stats", {}).pop("input_sha256", None), + ) + report = cs.run_suite(MANIFEST) + case = next(case for case in report["cases"] if case["kind"] == cs.KIND_POSITIVE) + assert case["status"] == cs.STATUS_MISMATCHED + assert case["input_sha256"] is None + + +def test_forged_input_sha256_fails_the_cli( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + _forging_client( + monkeypatch, + lambda body: body["stats"].update(input_sha256="f" * 64), + ) + out = tmp_path / "report.json" + assert cs.main(["--output", str(out)]) == 1 + assert json.loads(out.read_text())["ok"] is False + err = capsys.readouterr().err + assert "status=mismatched" in err + assert "input_sha256" in err