From ab2d086a2f2b177a1f7ae1f2964bfd25834bc6bb Mon Sep 17 00:00:00 2001 From: Sam Gu <127461594+SamGu-NRX@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:34:19 +0000 Subject: [PATCH] test(backend): guard the capture-journey fixture with explicit manifest entries The committed packet check ran with --no-files, so the keyframes and streams under data/fixtures/packets/mini-arkit were never re-verified: a file that left the repository passed make validate and only surfaced as a raw FileNotFoundError inside the capture journey test. The manifest now names the packet and all eleven files its packet.json references, and validate_fixtures runs a guard that fails loudly, naming the missing or drifted input, when the manifest, packet.json and disk disagree. Co-authored-by: obvious-autobuild[bot] <262744130+obvious-autobuild[bot]@users.noreply.github.com> --- backend/data/fixtures/manifest.json | 121 ++++++++++++++++++ backend/scripts/validate_fixtures.py | 75 +++++++++++ .../tests/test_capture_fixture_manifest.py | 101 +++++++++++++++ 3 files changed, 297 insertions(+) create mode 100644 backend/tests/test_capture_fixture_manifest.py diff --git a/backend/data/fixtures/manifest.json b/backend/data/fixtures/manifest.json index b845144f1..06081c76f 100644 --- a/backend/data/fixtures/manifest.json +++ b/backend/data/fixtures/manifest.json @@ -585,6 +585,127 @@ "committed": true, "licence": "synthetic" }, + { + "id": "mini-arkit-file-keyframes-k00001-features-bin", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00001.features.bin", + "cache_path": null, + "source": null, + "bytes": 4240, + "sha256": "7710dcbc1ad6d99132a3cdf2c45f49cbae908eff3398c8edd779955e37962bf8", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-keyframes-k00001-jpg", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00001.jpg", + "cache_path": null, + "source": null, + "bytes": 714, + "sha256": "753a386b150b92e235357144459d2109ed480bed805fbb0ac4ca989c42bc6d79", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-keyframes-k00002-features-bin", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00002.features.bin", + "cache_path": null, + "source": null, + "bytes": 3740, + "sha256": "ee6e18a01b94f594c8f17cc92e6f3e3bcef5347190b02a6a67fce8b10dc1f8b9", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-keyframes-k00002-jpg", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00002.jpg", + "cache_path": null, + "source": null, + "bytes": 713, + "sha256": "80beae7b3ea40a8951f3001ea9e87a5bb342e9d4a402037bda815f953c3a3c8e", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-keyframes-k00003-jpg", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00003.jpg", + "cache_path": null, + "source": null, + "bytes": 714, + "sha256": "a53e442431e7041303f024d5432e133dac2e3dc5726cbed37653b3d656a8b143", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-keyframes-k00004-jpg", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00004.jpg", + "cache_path": null, + "source": null, + "bytes": 714, + "sha256": "98a40bb14e16042551263230a3474a2fa2ea688f2262ca2b84b6becc7dc67f65", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-streams-arkit-poses-csv-gz", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/streams/arkit_poses.csv.gz", + "cache_path": null, + "source": null, + "bytes": 73, + "sha256": "8b95dff9754f3567bd2b8d03906aa24c97c7d97d0e881970a8ddb0dd55d3dc3e", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-streams-heading-csv-gz", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/streams/heading.csv.gz", + "cache_path": null, + "source": null, + "bytes": 76, + "sha256": "4a9c16d18e22eeeb85c024b6fa0db32b9a22dd89c48e10bb14c989f183696a58", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-streams-imu-raw-csv-gz", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/streams/imu_raw.csv.gz", + "cache_path": null, + "source": null, + "bytes": 44, + "sha256": "564ece90c04a9c78f7690ccae56c032dd4f1fb904e64b847697f66125bfd4b60", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-streams-location-csv-gz", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/streams/location.csv.gz", + "cache_path": null, + "source": null, + "bytes": 107, + "sha256": "fc12d4b107cc3e7f7a903375c90310bcb2565fd3b50785b6bf3696282f83cc30", + "committed": true, + "licence": "synthetic" + }, + { + "id": "mini-arkit-file-streams-motion-csv-gz", + "kind": "file", + "repo_path": "data/fixtures/packets/mini-arkit/streams/motion.csv.gz", + "cache_path": null, + "source": null, + "bytes": 95, + "sha256": "d21ce4423410a0b1c5178c268dbbe615090e1dd9521affd4c31a34f830b4ec54", + "committed": true, + "licence": "synthetic" + }, { "id": "score-gt.C1.fp3__eth3d-facade.demo.labels", "kind": "score", diff --git a/backend/scripts/validate_fixtures.py b/backend/scripts/validate_fixtures.py index 0e76ff6c5..166dc3f7f 100644 --- a/backend/scripts/validate_fixtures.py +++ b/backend/scripts/validate_fixtures.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import importlib.util import json import os @@ -33,6 +34,74 @@ def _labels() -> list[Path]: return sorted(FIXTURES.glob("labels/*.json")) +CAPTURE_JOURNEY_PACKET_ID = "mini-arkit" + + +def capture_journey_errors(fixtures_dir: Path) -> list[str]: + """Named missing-input failures for the packet the capture journey test drives. + + The committed packet check runs with --no-files, so the keyframes and streams + under data/fixtures/packets/mini-arkit were never re-verified: a file that left + the repository passed `make validate` and only surfaced as a raw FileNotFoundError + inside tests/test_capture_fixture_flow.py. The manifest now names the packet and + every file it references; this guard fails loudly, by name, when they disagree + with what is on disk. + """ + + document = json.loads((fixtures_dir / "manifest.json").read_text()) + items = {item["id"]: item for item in document["items"]} + packet_item = items.get(CAPTURE_JOURNEY_PACKET_ID) + if packet_item is None or packet_item.get("kind") != "packet_dir": + return [f"{CAPTURE_JOURNEY_PACKET_ID}: no packet_dir item in the fixture manifest"] + + packet_dir = fixtures_dir / Path(packet_item["repo_path"]).relative_to("data/fixtures") + packet_path = packet_dir / "packet.json" + errors: list[str] = [] + packet_files: dict[str, int] = {} + if not packet_path.is_file(): + return [f"{CAPTURE_JOURNEY_PACKET_ID}: missing input {packet_item['repo_path']}/packet.json"] + packet = json.loads(packet_path.read_text()) + for index, entry in enumerate(packet["files"]): + packet_files[entry["path"]] = index + + def _describe(item: dict) -> str: + return f"manifest item {item['id']}" + + # The packet_dir entry itself pins packet.json; keep that hash guarded too. + raw = packet_path.read_bytes() + if len(raw) != int(packet_item["bytes"]) or hashlib.sha256(raw).hexdigest() != packet_item["sha256"]: + errors.append(f"{CAPTURE_JOURNEY_PACKET_ID}: packet.json does not match its {_describe(packet_item)}") + + named: dict[str, dict] = {} + prefix = packet_item["repo_path"] + "/" + for item in document["items"]: + repo_path = item.get("repo_path") + if item is packet_item or not repo_path or not str(repo_path).startswith(prefix): + continue + relative = str(repo_path)[len(prefix) :] + named[relative] = item + path = fixtures_dir / Path(repo_path).relative_to("data/fixtures") + if not path.is_file(): + errors.append( + f"{CAPTURE_JOURNEY_PACKET_ID}: missing input {repo_path} ({_describe(item)}, " + f"packet.json /files/{packet_files[relative]})" + if relative in packet_files + else f"{CAPTURE_JOURNEY_PACKET_ID}: missing input {repo_path} ({_describe(item)})" + ) + continue + data = path.read_bytes() + if len(data) != int(item["bytes"]) or hashlib.sha256(data).hexdigest() != item["sha256"]: + errors.append(f"{CAPTURE_JOURNEY_PACKET_ID}: {repo_path} does not match {_describe(item)}") + + for relative, index in sorted(packet_files.items(), key=lambda pair: pair[1]): + if relative not in named: + errors.append( + f"{CAPTURE_JOURNEY_PACKET_ID}: packet.json /files/{index} references {relative}, " + "which the fixture manifest does not name" + ) + return errors + + def _packet_manifests() -> list[Path]: """Every committed manifest. `mini-arkit` is named explicitly (SD-43), never by glob.""" @@ -67,6 +136,12 @@ def _validate_committed() -> int: ) if code != 0: return code + journey_errors = capture_journey_errors(FIXTURES) + if journey_errors: + print("capture-journey fixture guard failed:", file=sys.stderr) + for error in journey_errors: + print(" " + error, file=sys.stderr) + return 1 print( "validate ok " f"scenes={len(_scenes())} verdicts={len(_verdicts())} " diff --git a/backend/tests/test_capture_fixture_manifest.py b/backend/tests/test_capture_fixture_manifest.py new file mode 100644 index 000000000..7a5688231 --- /dev/null +++ b/backend/tests/test_capture_fixture_manifest.py @@ -0,0 +1,101 @@ +"""The manifest guard over the capture journey's ``mini-arkit`` fixture. + +The committed packet check runs with ``--no-files``, so the keyframes and streams +the journey test copies could disappear from the repository and ``make validate`` +would still pass, leaving a raw ``FileNotFoundError`` inside +``test_capture_fixture_flow`` as the only symptom. These tests pin the seam: +the committed fixture passes the guard, and an incomplete temporary copy fails +with the missing input named. +""" + +from __future__ import annotations + +import importlib.util +import json +import shutil +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[1] +SCRIPT = REPO / "scripts" / "validate_fixtures.py" +FIXTURES = REPO / "data" / "fixtures" + + +def _guard(): + spec = importlib.util.spec_from_file_location("validate_fixtures", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.capture_journey_errors + + +@pytest.fixture +def guard(): + return _guard() + + +@pytest.fixture +def copied_fixtures(tmp_path: Path) -> Path: + """A temporary copy of just the manifest and the capture-journey packet.""" + root = tmp_path / "fixtures" + (root / "packets").mkdir(parents=True) + shutil.copy(FIXTURES / "manifest.json", root / "manifest.json") + shutil.copytree(FIXTURES / "packets" / "mini-arkit", root / "packets" / "mini-arkit") + return root + + +def test_committed_capture_fixture_passes_the_manifest_guard(guard) -> None: + assert guard(FIXTURES) == [] + + +def test_missing_referenced_file_fails_by_name(guard, copied_fixtures: Path) -> None: + (copied_fixtures / "packets" / "mini-arkit" / "keyframes" / "k00003.jpg").unlink() + + errors = guard(copied_fixtures) + + assert errors, "an incomplete copy must not pass the guard" + joined = "\n".join(errors) + assert "missing input" in joined + assert "keyframes/k00003.jpg" in joined + assert "packet.json /files/4" in joined + + +def test_absent_stream_fails_by_name(guard, copied_fixtures: Path) -> None: + (copied_fixtures / "packets" / "mini-arkit" / "streams" / "imu_raw.csv.gz").unlink() + + errors = guard(copied_fixtures) + + joined = "\n".join(errors) + assert "missing input" in joined + assert "streams/imu_raw.csv.gz" in joined + + +def test_packet_file_not_named_by_the_manifest_fails_by_name(guard, copied_fixtures: Path) -> None: + packet_path = copied_fixtures / "packets" / "mini-arkit" / "packet.json" + packet = json.loads(packet_path.read_text()) + entry = dict(packet["files"][0]) + entry["path"] = "keyframes/k00005.jpg" + entry["bytes"] = 5 + entry["sha256"] = "0" * 64 + packet["files"].append(entry) + packet_path.write_text(json.dumps(packet, indent=2)) + + errors = guard(copied_fixtures) + + joined = "\n".join(errors) + assert "keyframes/k00005.jpg" in joined + assert "does not name" in joined + + +def test_tampered_file_fails_by_name(guard, copied_fixtures: Path) -> None: + target = copied_fixtures / "packets" / "mini-arkit" / "keyframes" / "k00001.features.bin" + data = bytearray(target.read_bytes()) + data[0] ^= 0xFF + target.write_bytes(bytes(data)) + + errors = guard(copied_fixtures) + + joined = "\n".join(errors) + assert "does not match" in joined + assert "keyframes/k00001.features.bin" in joined