Skip to content

[obvious-fleet] Backend: make the capture-journey fixture explicit and fail loudly when it disappears #260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SamGu-NRX merged 1 commit into from
Oct 16, 2026
Merged

[obvious-fleet] Backend: make the capture-journey fixture explicit and fail loudly when it disappears #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions backend/data/fixtures/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,127 @@
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-keyframes-k00001-features-bin",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00001.features.bin",
"cache_path": null,
"source": null,
"bytes": 4240,
"sha256": "7710dcbc1ad6d99132a3cdf2c45f49cbae908eff3398c8edd779955e37962bf8",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-keyframes-k00001-jpg",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00001.jpg",
"cache_path": null,
"source": null,
"bytes": 714,
"sha256": "753a386b150b92e235357144459d2109ed480bed805fbb0ac4ca989c42bc6d79",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-keyframes-k00002-features-bin",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00002.features.bin",
"cache_path": null,
"source": null,
"bytes": 3740,
"sha256": "ee6e18a01b94f594c8f17cc92e6f3e3bcef5347190b02a6a67fce8b10dc1f8b9",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-keyframes-k00002-jpg",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00002.jpg",
"cache_path": null,
"source": null,
"bytes": 713,
"sha256": "80beae7b3ea40a8951f3001ea9e87a5bb342e9d4a402037bda815f953c3a3c8e",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-keyframes-k00003-jpg",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00003.jpg",
"cache_path": null,
"source": null,
"bytes": 714,
"sha256": "a53e442431e7041303f024d5432e133dac2e3dc5726cbed37653b3d656a8b143",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-keyframes-k00004-jpg",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/keyframes/k00004.jpg",
"cache_path": null,
"source": null,
"bytes": 714,
"sha256": "98a40bb14e16042551263230a3474a2fa2ea688f2262ca2b84b6becc7dc67f65",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-streams-arkit-poses-csv-gz",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/streams/arkit_poses.csv.gz",
"cache_path": null,
"source": null,
"bytes": 73,
"sha256": "8b95dff9754f3567bd2b8d03906aa24c97c7d97d0e881970a8ddb0dd55d3dc3e",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-streams-heading-csv-gz",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/streams/heading.csv.gz",
"cache_path": null,
"source": null,
"bytes": 76,
"sha256": "4a9c16d18e22eeeb85c024b6fa0db32b9a22dd89c48e10bb14c989f183696a58",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-streams-imu-raw-csv-gz",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/streams/imu_raw.csv.gz",
"cache_path": null,
"source": null,
"bytes": 44,
"sha256": "564ece90c04a9c78f7690ccae56c032dd4f1fb904e64b847697f66125bfd4b60",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-streams-location-csv-gz",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/streams/location.csv.gz",
"cache_path": null,
"source": null,
"bytes": 107,
"sha256": "fc12d4b107cc3e7f7a903375c90310bcb2565fd3b50785b6bf3696282f83cc30",
"committed": true,
"licence": "synthetic"
},
{
"id": "mini-arkit-file-streams-motion-csv-gz",
"kind": "file",
"repo_path": "data/fixtures/packets/mini-arkit/streams/motion.csv.gz",
"cache_path": null,
"source": null,
"bytes": 95,
"sha256": "d21ce4423410a0b1c5178c268dbbe615090e1dd9521affd4c31a34f830b4ec54",
"committed": true,
"licence": "synthetic"
},
{
"id": "score-gt.C1.fp3__eth3d-facade.demo.labels",
"kind": "score",
Expand Down
75 changes: 75 additions & 0 deletions backend/scripts/validate_fixtures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from __future__ import annotations

import argparse
import hashlib
import importlib.util
import json
import os
Expand Down Expand Up @@ -33,6 +34,74 @@ def _labels() -> list[Path]:
return sorted(FIXTURES.glob("labels/*.json"))


CAPTURE_JOURNEY_PACKET_ID = "mini-arkit"


def capture_journey_errors(fixtures_dir: Path) -> list[str]:
"""Named missing-input failures for the packet the capture journey test drives.

The committed packet check runs with --no-files, so the keyframes and streams
under data/fixtures/packets/mini-arkit were never re-verified: a file that left
the repository passed `make validate` and only surfaced as a raw FileNotFoundError
inside tests/test_capture_fixture_flow.py. The manifest now names the packet and
every file it references; this guard fails loudly, by name, when they disagree
with what is on disk.
"""

document = json.loads((fixtures_dir / "manifest.json").read_text())
items = {item["id"]: item for item in document["items"]}
packet_item = items.get(CAPTURE_JOURNEY_PACKET_ID)
if packet_item is None or packet_item.get("kind") != "packet_dir":
return [f"{CAPTURE_JOURNEY_PACKET_ID}: no packet_dir item in the fixture manifest"]

packet_dir = fixtures_dir / Path(packet_item["repo_path"]).relative_to("data/fixtures")
packet_path = packet_dir / "packet.json"
errors: list[str] = []
packet_files: dict[str, int] = {}
if not packet_path.is_file():
return [f"{CAPTURE_JOURNEY_PACKET_ID}: missing input {packet_item['repo_path']}/packet.json"]
packet = json.loads(packet_path.read_text())
for index, entry in enumerate(packet["files"]):
packet_files[entry["path"]] = index
Comment on lines +64 to +65

@greptile-apps greptile-apps Bot Oct 16, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Stale packet details pass validation

capture_journey_errors checks file bytes against manifest.json, but keeps only paths from packet.json. If a contributor updates a fixture file and its manifest hash without updating packet.json, the guard passes even though the packet declares the old size or hash. The earlier --no-files check cannot catch this either.

Compare each packet entry’s bytes and sha256 with the matching manifest entry, and add a test for this disagreement.

Prompt To Fix With AI 
This is a comment left during a code review.
Path: backend/scripts/validate_fixtures.py
Line: 64-65

Comment:
**Stale packet details pass validation**

`capture_journey_errors` checks file bytes against `manifest.json`, but keeps only paths from `packet.json`. If a contributor updates a fixture file and its manifest hash without updating `packet.json`, the guard passes even though the packet declares the old size or hash. The earlier `--no-files` check cannot catch this either.

Compare each packet entry’s `bytes` and `sha256` with the matching manifest entry, and add a test for this disagreement.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Fix in Claude Code Fix in Codex


def _describe(item: dict) -> str:
return f"manifest item {item['id']}"

# The packet_dir entry itself pins packet.json; keep that hash guarded too.
raw = packet_path.read_bytes()
if len(raw) != int(packet_item["bytes"]) or hashlib.sha256(raw).hexdigest() != packet_item["sha256"]:
errors.append(f"{CAPTURE_JOURNEY_PACKET_ID}: packet.json does not match its {_describe(packet_item)}")

named: dict[str, dict] = {}
prefix = packet_item["repo_path"] + "/"
for item in document["items"]:
repo_path = item.get("repo_path")
if item is packet_item or not repo_path or not str(repo_path).startswith(prefix):
continue
relative = str(repo_path)[len(prefix) :]
named[relative] = item
path = fixtures_dir / Path(repo_path).relative_to("data/fixtures")
if not path.is_file():
errors.append(
f"{CAPTURE_JOURNEY_PACKET_ID}: missing input {repo_path} ({_describe(item)}, "
f"packet.json /files/{packet_files[relative]})"
if relative in packet_files
else f"{CAPTURE_JOURNEY_PACKET_ID}: missing input {repo_path} ({_describe(item)})"
)
continue
data = path.read_bytes()
if len(data) != int(item["bytes"]) or hashlib.sha256(data).hexdigest() != item["sha256"]:
errors.append(f"{CAPTURE_JOURNEY_PACKET_ID}: {repo_path} does not match {_describe(item)}")

for relative, index in sorted(packet_files.items(), key=lambda pair: pair[1]):
if relative not in named:
errors.append(
f"{CAPTURE_JOURNEY_PACKET_ID}: packet.json /files/{index} references {relative}, "
"which the fixture manifest does not name"
)
return errors


def _packet_manifests() -> list[Path]:
"""Every committed manifest. `mini-arkit` is named explicitly (SD-43), never by glob."""

Expand Down Expand Up @@ -67,6 +136,12 @@ def _validate_committed() -> int:
)
if code != 0:
return code
journey_errors = capture_journey_errors(FIXTURES)
if journey_errors:
print("capture-journey fixture guard failed:", file=sys.stderr)
for error in journey_errors:
print(" " + error, file=sys.stderr)
return 1
print(
"validate ok "
f"scenes={len(_scenes())} verdicts={len(_verdicts())} "
Expand Down
101 changes: 101 additions & 0 deletions backend/tests/test_capture_fixture_manifest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
"""The manifest guard over the capture journey's ``mini-arkit`` fixture.

The committed packet check runs with ``--no-files``, so the keyframes and streams
the journey test copies could disappear from the repository and ``make validate``
would still pass, leaving a raw ``FileNotFoundError`` inside
``test_capture_fixture_flow`` as the only symptom. These tests pin the seam:
the committed fixture passes the guard, and an incomplete temporary copy fails
with the missing input named.
"""

from __future__ import annotations

import importlib.util
import json
import shutil
from pathlib import Path

import pytest

REPO = Path(__file__).resolve().parents[1]
SCRIPT = REPO / "scripts" / "validate_fixtures.py"
FIXTURES = REPO / "data" / "fixtures"


def _guard():
spec = importlib.util.spec_from_file_location("validate_fixtures", SCRIPT)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.capture_journey_errors


@pytest.fixture
def guard():
return _guard()


@pytest.fixture
def copied_fixtures(tmp_path: Path) -> Path:
"""A temporary copy of just the manifest and the capture-journey packet."""
root = tmp_path / "fixtures"
(root / "packets").mkdir(parents=True)
shutil.copy(FIXTURES / "manifest.json", root / "manifest.json")
shutil.copytree(FIXTURES / "packets" / "mini-arkit", root / "packets" / "mini-arkit")
return root


def test_committed_capture_fixture_passes_the_manifest_guard(guard) -> None:
assert guard(FIXTURES) == []


def test_missing_referenced_file_fails_by_name(guard, copied_fixtures: Path) -> None:
(copied_fixtures / "packets" / "mini-arkit" / "keyframes" / "k00003.jpg").unlink()

errors = guard(copied_fixtures)

assert errors, "an incomplete copy must not pass the guard"
joined = "\n".join(errors)
assert "missing input" in joined
assert "keyframes/k00003.jpg" in joined
assert "packet.json /files/4" in joined


def test_absent_stream_fails_by_name(guard, copied_fixtures: Path) -> None:
(copied_fixtures / "packets" / "mini-arkit" / "streams" / "imu_raw.csv.gz").unlink()

errors = guard(copied_fixtures)

joined = "\n".join(errors)
assert "missing input" in joined
assert "streams/imu_raw.csv.gz" in joined


def test_packet_file_not_named_by_the_manifest_fails_by_name(guard, copied_fixtures: Path) -> None:
packet_path = copied_fixtures / "packets" / "mini-arkit" / "packet.json"
packet = json.loads(packet_path.read_text())
entry = dict(packet["files"][0])
entry["path"] = "keyframes/k00005.jpg"
entry["bytes"] = 5
entry["sha256"] = "0" * 64
packet["files"].append(entry)
packet_path.write_text(json.dumps(packet, indent=2))

errors = guard(copied_fixtures)

joined = "\n".join(errors)
assert "keyframes/k00005.jpg" in joined
assert "does not name" in joined


def test_tampered_file_fails_by_name(guard, copied_fixtures: Path) -> None:
target = copied_fixtures / "packets" / "mini-arkit" / "keyframes" / "k00001.features.bin"
data = bytearray(target.read_bytes())
data[0] ^= 0xFF
target.write_bytes(bytes(data))

errors = guard(copied_fixtures)

joined = "\n".join(errors)
assert "does not match" in joined
assert "keyframes/k00001.features.bin" in joined