Research handoff: live survey findings and tested prototypes (#18, conflicts resolved)

.github/labels.json

@@ -24,6 +24,11 @@
     "color": "b60205",
     "description": "Not ready to merge. State what must change before it is ready."
   },
+  {
+    "name": "full-ui",
+    "color": "5319e7",
+    "description": "iOS CI also runs the every-state accessibility audit (about 10 more minutes)."
+  },
   {
     "name": "size:XS",
     "color": "0e8a16",

.github/workflows/evals.yml

@@ -0,0 +1,42 @@
+name: Evals
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "experiments/evals/**"
+      - ".github/workflows/evals.yml"
+  pull_request:
+    paths:
+      - "experiments/evals/**"
+      - ".github/workflows/evals.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: evals-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  evals-checks:
+    name: Evals checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: experiments/evals
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      # Reads tool.uv.required-version from experiments/evals/pyproject.toml.
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          working-directory: experiments/evals
+      # The datasets are non-commercial and never in git, so CI runs the metric code's unit tests
+      # (hand-computed cases), lint and format; the result tables are regenerated locally.
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q

.github/workflows/ios.yml

@@ -2,35 +2,57 @@ name: iOS
 
 on:
   push:
-    branches: [main]
+    branches: [main, t3/ios-mvf]
     paths:
       - "ios/**"
+      - "!ios/**/*.md"
       - ".github/workflows/ios.yml"
   pull_request:
+    # ready_for_review: drafts are skipped below, so run once a draft is marked ready.
+    # labeled: adding full-ui reruns the pull request with the accessibility audit.
+    types: [opened, synchronize, reopened, ready_for_review, labeled]
     paths:
       - "ios/**"
+      - "!ios/**/*.md"
       - ".github/workflows/ios.yml"
+  # Runs every test, the audit included, on any branch.
+  workflow_dispatch:
 
 permissions:
   contents: read
 
 concurrency:
-  group: ios-${{ github.ref }}
+  # Adding any label other than full-ui gets a group of its own, so it cannot cancel a real run.
+  group: ios-${{ github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'full-ui' && format('-{0}', github.run_id) || '' }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   ios-build:
     name: iOS build
+    # A free account runs 5 macOS jobs at once, shared by every open pull request, so drafts and
+    # labels other than full-ui don't take a slot.
+    if: >-
+      github.event_name != 'pull_request' ||
+      (!github.event.pull_request.draft &&
+       (github.event.action != 'labeled' || github.event.label.name == 'full-ui'))
     # macos-26 (arm64) is the newest GA macOS image. The xcode-27 image is still a preview.
     runs-on: macos-26
-    timeout-minutes: 30
+    # Beta run 36302099002 took 43 minutes before the spot-confirmation and review checks
+    # were added. Leave room for those checks without dropping the every-state audit.
+    timeout-minutes: 60
     env:
       # Newest stable Xcode on macos-26 per images/macos/macos-26-arm64-Readme.md (image 20260907).
       DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer
+      # iPhone 17 on iOS 26.5 ships with Xcode 26.6 on macos-26 (images/macos/macos-26-arm64-Readme.md).
+      SIM_NAME: iPhone 17
+      SIM_OS: "26.5"
       # Homebrew only installs the latest XcodeGen. The release zip gives an exact version,
       # and the drift check below needs the same version the committed project was made with.
       XCODEGEN_VERSION: 2.46.0
       XCODEGEN_SHA256: 4d9e34b62172d645eed6457cac13fc222569974098ef4ee9c3368bedf0196806
+      # The every-state accessibility audit relaunches the app 65 times (about 10 min). Pull
+      # requests skip it unless labelled full-ui; pushes and manual runs always include it.
+      FULL_UI: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'full-ui') }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -45,6 +67,45 @@ jobs:
           xcodebuild -version
           swift --version
 
+      # Booting takes 2 to 3 minutes and needs nothing from the build, so it runs alongside the
+      # steps below; the UI test step waits for it.
+      - name: Boot the Simulator in the background
+        run: |
+          runtime="com.apple.CoreSimulator.SimRuntime.iOS-${SIM_OS//./-}"
+          udid="$(xcrun simctl list devices available -j \
+            | jq -r --arg runtime "$runtime" --arg name "$SIM_NAME" \
+              '(.devices[$runtime] // [])[] | select(.name == $name) | .udid' | head -n 1)"
+          if [ -z "$udid" ]; then
+            echo "::error::No $SIM_NAME on iOS $SIM_OS on this runner image. Pick one from the image readme."
+            xcrun simctl list devices available
+            exit 1
+          fi
+          echo "SIM_UDID=$udid" >> "$GITHUB_ENV"
+          nohup xcrun simctl bootstatus "$udid" -b > "$RUNNER_TEMP/simulator-boot.log" 2>&1 &
+
+      # The package's drift test compares the vendored server schemas with the server's own copies:
+      # the checkout's files once that branch merges, else the t3/server branch, which the shallow
+      # checkout leaves out. One of the two is required:
+      # a failed fetch fails the run, and HOUSESCAN_REQUIRE_UPSTREAM makes the tests fail rather
+      # than skip, so CI can't go green without comparing. The checkout counts only when it holds
+      # every compared file: a folder alone isn't enough (main has a server/ without schemas).
+      - name: Fetch the server contract
+        run: |
+          fetch_unless_present() {
+            local branch="$1"; shift
+            local missing=0
+            for file in "$@"; do [ -f "$file" ] || missing=1; done
+            if [ "$missing" = 0 ]; then
+              echo "$* are in the checkout; the drift tests read them."
+              return
+            fi
+            if ! git fetch --no-tags --depth=1 origin "+refs/heads/$branch:refs/remotes/origin/$branch"; then
+              echo "::error::Neither the checkout nor $branch holds $*, so the vendored copies can't be compared."
+              exit 1
+            fi
+          }
+          fetch_unless_present t3/server server/schemas/scene.schema.json server/schemas/result.schema.json server/tests/fixtures/example-scene.json
+
       - name: Install XcodeGen
         run: |
           curl -fsSL -o "$RUNNER_TEMP/xcodegen.zip" \
@@ -69,7 +130,44 @@ jobs:
             exit 1
           fi
 
+      - name: Test HouseScanKit
+        # The capture logic is a plain Swift package; its tests run on macOS without a Simulator.
+        env:
+          HOUSESCAN_REQUIRE_UPSTREAM: "1"
+        run: swift test --package-path ios/HouseScanKit -Xswiftc -warnings-as-errors
+
+      # CI builds once from clean, so the index Xcode keeps for editing is wasted work.
       - name: Build
         run: >-
           xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug
-          -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO build
+          -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO build
+
+      - name: Wait for the Simulator
+        timeout-minutes: 10
+        run: |
+          xcrun simctl bootstatus "$SIM_UDID" -b
+          echo "Booted $SIM_NAME on iOS $SIM_OS ($SIM_UDID)."
+
+      - name: UI test the whole flow in the Simulator
+        # Replays the committed synthetic wall (ios/HouseScanUITests/Fixtures/synthetic-wall) with
+        # -autopilot and the bundled sample result.
+        run: |
+          skip=()
+          if [ "$FULL_UI" != "true" ]; then
+            skip=(-skip-testing:HouseScanUITests/ScreenStatesUITests/testEveryStatePassesTheAccessibilityAudit)
+            echo "::notice::Skipped the every-state accessibility audit (about 10 min). Add the full-ui label to run it on this pull request."
+          fi
+          xcodebuild -project ios/HouseScan.xcodeproj -scheme HouseScan -configuration Debug \
+            -destination "id=$SIM_UDID" \
+            -resultBundlePath "$RUNNER_TEMP/HouseScanUITests.xcresult" \
+            -only-testing:HouseScanUITests "${skip[@]}" \
+            COMPILER_INDEX_STORE_ENABLE=NO test
+
+      - name: Keep UI test screenshots
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: ui-test-results
+          path: ${{ runner.temp }}/HouseScanUITests.xcresult
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/measure-lab.yml

@@ -0,0 +1,97 @@
+name: Measure lab
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "experiments/measure-lab/**"
+      - ".github/workflows/measure-lab.yml"
+  pull_request:
+    paths:
+      - "experiments/measure-lab/**"
+      - ".github/workflows/measure-lab.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: measure-lab-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  geometry-linux:
+    name: Measure lab geometry (Linux)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    container:
+      # Swift 6.2 is the oldest toolchain the tests compile with (their raw-identifier names need
+      # it); the app job's Xcode 26.6 ships Swift 6.3. Pinned by digest so a retag can't change
+      # the compiler under the tests.
+      image: swift:6.2-noble@sha256:9bea530093ffff8cf6c259991715ee843fe4d0f932e612f7f4b79cca6e00db87
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Show toolchain
+        run: swift --version
+
+      - name: Test
+        working-directory: experiments/measure-lab/Geometry
+        run: swift test
+
+  app-build:
+    name: Measure lab app build
+    # Same image and Xcode as .github/workflows/ios.yml.
+    runs-on: macos-26
+    timeout-minutes: 30
+    env:
+      DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer
+      XCODEGEN_VERSION: 2.46.0
+      XCODEGEN_SHA256: 4d9e34b62172d645eed6457cac13fc222569974098ef4ee9c3368bedf0196806
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Show toolchain
+        run: |
+          if [ ! -d "$DEVELOPER_DIR" ]; then
+            echo "::error::$DEVELOPER_DIR is not on this runner image. Pick an installed Xcode from the image readme."
+            exit 1
+          fi
+          xcodebuild -version
+          swift --version
+
+      - name: Install XcodeGen
+        run: |
+          curl -fsSL -o "$RUNNER_TEMP/xcodegen.zip" \
+            "https://github.com/yonaskolb/XcodeGen/releases/download/${XCODEGEN_VERSION}/xcodegen.zip"
+          echo "${XCODEGEN_SHA256}  $RUNNER_TEMP/xcodegen.zip" | shasum -a 256 -c -
+          unzip -q "$RUNNER_TEMP/xcodegen.zip" -d "$RUNNER_TEMP"
+          echo "$RUNNER_TEMP/xcodegen/bin" >> "$GITHUB_PATH"
+
+      - name: Generate project
+        run: xcodegen generate --spec experiments/measure-lab/project.yml
+
+      - name: Check committed project matches project.yml
+        run: |
+          if ! git diff --exit-code -- experiments/measure-lab; then
+            echo "::error::MeasureLab.xcodeproj differs from project.yml. Run 'make project' in experiments/measure-lab (XcodeGen ${XCODEGEN_VERSION}) and commit the result."
+            exit 1
+          fi
+          untracked="$(git ls-files --others --exclude-standard -- experiments/measure-lab)"
+          if [ -n "$untracked" ]; then
+            echo "::error::xcodegen generate wrote files that are not committed. Run 'make project' and commit them."
+            echo "$untracked"
+            exit 1
+          fi
+
+      - name: Test geometry on macOS
+        working-directory: experiments/measure-lab/Geometry
+        run: swift test
+
+      - name: Build
+        run: >-
+          xcodebuild -project experiments/measure-lab/MeasureLab.xcodeproj -scheme MeasureLab
+          -configuration Debug -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO build

.github/workflows/meter-closeup.yml

@@ -0,0 +1,45 @@
+name: Meter close-up experiment
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "experiments/meter-closeup/**"
+      - ".github/workflows/meter-closeup.yml"
+  pull_request:
+    paths:
+      - "experiments/meter-closeup/**"
+      - ".github/workflows/meter-closeup.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: meter-closeup-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  checks:
+    name: Meter close-up checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: experiments/meter-closeup
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          working-directory: experiments/meter-closeup
+      # The unit tests cover the metric code only; the Vision reader needs macOS and the photos.
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q
+      # Looks for any transcribed meter identifier in the tracked files. The key stays in a
+      # repository secret; without it the step fails with an explanation.
+      - run: uv run python -m meter_eval.leakcheck
+        env:
+          METER_HMAC_KEY: ${{ secrets.METER_HMAC_KEY }}

.github/workflows/panel-label.yml

@@ -0,0 +1,46 @@
+name: Panel label experiment
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "experiments/panel-label/**"
+      - ".github/workflows/panel-label.yml"
+  pull_request:
+    paths:
+      - "experiments/panel-label/**"
+      - ".github/workflows/panel-label.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: panel-label-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  checks:
+    name: Panel label checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: experiments/panel-label
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          working-directory: experiments/panel-label
+      - run: uv sync --locked
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q
+      # Photos of real panels can show addresses and inspection stickers; they stay out of git.
+      - name: No photos committed
+        run: |
+          if git ls-files . | grep -iE '\.(jpe?g|png|heic|webp|tiff?)$'; then
+            echo "Photos must stay in ~/house-scanning-data/panel, not in the repository." >&2
+            exit 1
+          fi