verification: independent end-to-end checks (S4)

.github/workflows/verification.yml

@@ -0,0 +1,42 @@
+name: Verification
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "verification/**"
+      - ".github/workflows/verification.yml"
+  pull_request:
+    paths:
+      - "verification/**"
+      - ".github/workflows/verification.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: verification-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  verification-checks:
+    name: Verification checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: verification
+    steps:
+      # The default depth of 1 is enough: tests/test_gitref.py only reads HEAD, and a detached
+      # worktree of HEAD works in a shallow clone.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      # Reads tool.uv.required-version from verification/pyproject.toml to pick the uv version.
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          working-directory: verification
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q

verification/Makefile

@@ -0,0 +1,46 @@
+# Independent verification. Every target runs from this folder; reports go to
+# ~/house-scanning-data/reports/ (outside git) because runs can show dataset frames.
+
+REF ?= origin/t3/ios-mvf
+SERVER_REF ?= origin/t3/server
+# Set SERVER_URL to use a running server (the hosted API) instead of starting one from SERVER_REF.
+SERVER_URL ?=
+SERVER = $(if $(SERVER_URL),--server-url $(SERVER_URL),--server-ref $(SERVER_REF))
+REPLAY ?= $(HOME)/house-scanning-data/replays/advio-20-0040-0075
+ARGS ?=
+
+.PHONY: test sim sim-app sim-probe e2e scoreboard
+
+test:
+	uv run ruff check .
+	uv run ruff format --check .
+	uv run pytest -q
+
+# Build REF for the Simulator, run it with ARGS (e.g. ARGS="--replay <dir> --autopilot"),
+# screenshot every STATE.
+sim:
+	uv run python -m hsverify.simrun --ref $(REF) $(ARGS)
+
+# The app at REF on the real ADVIO replay (C3), driven by its autopilot, uploading to the server
+# started from SERVER_REF (or at SERVER_URL): a screenshot per STATE in ~/house-scanning-data/reports/sim/. Fails
+# unless the run reaches the result state; the scan bundle is copied into the report as scan.zip.
+sim-app:
+	uv run python -m hsverify.simrun --ref $(REF) --replay $(REPLAY) --autopilot \
+		$(SERVER) --extra-arg=-autopilotHold --extra-arg=3 \
+		--idle 60 --timeout 900 --require result --require-export --label replay $(ARGS)
+
+# Every case in e2e/cases/ plus the ETH3D facade scene through the server's API at SERVER_REF;
+# each result is validated against server/schemas/result.schema.json at the same ref.
+e2e:
+	uv run python -m hsverify.e2e $(SERVER) --scene scenes/eth3d-facade/case.json $(ARGS)
+
+# Self-test of the runner against a fixture app that logs a known STATE sequence.
+sim-probe:
+	uv run python -m hsverify.simrun --ref HEAD --project verification/fixtures/state-probe/StateProbe.xcodeproj \
+		--scheme StateProbe --until probe_done --extra-arg=-probeArg --label probe
+
+# Every open PR with its CI, and every overnight-plan metric with its evidence or gap. Writes
+# SCOREBOARD.md and scoreboard/scoreboard.json. ARGS=--slow also runs the slow test probes;
+# ARGS="--slow S2-1" runs only that one.
+scoreboard:
+	uv run python -m hsverify.scoreboard $(ARGS)

verification/README.md

@@ -0,0 +1,36 @@
+# Verification (S4)
+
+**Question.** Do the app and server branches do what the plan says? Pass: the app at
+`t3/ios-mvf`'s head reaches a result from the real ADVIO replay and keeps its scan; every server
+answer at `t3/server`'s head validates against the result schema and never passes without the
+coverage its checks need; hostile inputs are refused, or answered with a valid result, within
+10 s and 500 MB of server growth; each plan metric has evidence at the current head.
+
+**Run**, from `verification/`, with uv (and Xcode 26 with an iOS Simulator for `sim-app`):
+
+```
+make sim-app      # REF=, SERVER_REF= or SERVER_URL=
+make e2e          # ARGS="--app-export <sim report>"
+make scoreboard   # writes SCOREBOARD.md
+make test
+```
+
+Reports go to `~/house-scanning-data/reports/`, outside git, because replay frames come from
+non-commercial datasets. What each check asserts is in the docstrings of `hsverify/e2e.py`,
+`resultcheck.py` and `simrun.py`.
+
+**Result.** At `t3/server` `3baa338`, with the app's upload from `c820cf1` (#10's freeze, synthetic
+replay, hosted server), 49 of 50 scenes pass (45 answered and checked; slowest real scene 0.48 s).
+The ETH3D scene fails: after its one requested capture the server leaves pool unsure with no
+request or reason.
+
+**What it changed.** S2 fixed seven defects it found:
+- unseen ground passing the pool check;
+- a short reach cutoff;
+- unbounded input cost;
+- incomplete photo requests;
+- repeated requests past a wall end;
+- ground past a real end ignored;
+- a facing gap called too close to call.
+
+The friction audit (#61) became team decision 8.