observer-protocol · BTCBoyd · Aug 9, 2026 · Aug 9, 2026 · Aug 9, 2026
diff --git a/.github/workflows/reproduction.yml b/.github/workflows/reproduction.yml
@@ -45,8 +45,38 @@ jobs:
           node-version: 22.22.3
 
       - name: Reproduce ${{ matrix.package }}
+        env:
+          REPRO_RESULT_FILE: ${{ runner.temp }}/result.txt
         run: bash scripts/verify-reproduction.sh packages/${{ matrix.package }}
 
+      # if: always() ON PURPOSE. A job that failed still measured something, and a job that died
+      # without writing anything must be visible as "no result" rather than absent. Uploading only
+      # on success would make the population job blind to exactly the runs worth counting.
+      - name: Record the outcome for the population assertion
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: repro-result-${{ matrix.package }}
+          path: ${{ runner.temp }}/result.txt
+          if-no-files-found: ignore
+          retention-days: 7
+
+  # ─── HOW MANY PACKAGES WERE ACTUALLY CHECKED? ────────────────────────────────────────────────────
+  # The matrix above reports whether each job PASSED. It cannot report how many packages were
+  # MEASURED, because a skip passes. Measured 2026-08-09: five green jobs, two of them skips.
+  population:
+    needs: reproduce
+    if: always()          # the count matters most on the runs where something went red
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          path: results
+          pattern: repro-result-*
+      - name: Assert the measured population
+        run: bash scripts/assert-reproduction-population.sh results
+
   # The publish guard is exercised here too, so its own failure modes stay live rather than being
   # discovered at publish time by whoever is publishing.
   guard-self-test:
@@ -60,11 +90,40 @@ jobs:
       # The guard refuses a branch with no upstream BEFORE it reaches any later check, and
       # actions/checkout does not configure upstream tracking. Without this the self-tests below
       # would go red for the wrong reason and read as "the guard is broken".
-      - name: Give HEAD an upstream, so the guard reaches the checks under test
+      #
+      # ─── THIS STEP USED TO CHECK OUT origin/main, WHICH TESTED THE WRONG TREE ────────────────────
+      # Measured 2026-08-09 on run 31296563830. The workflow STEPS come from the pull request, but
+      # `git checkout -B guard-selftest origin/main` replaced the working tree with main's, so every
+      # assertion below ran against main's copy of scripts/. The steps were the PR's and the code
+      # under test was not.
+      #
+      # That is invisible while a PR does not touch the guard, and it inverts the moment one does:
+      # a pull request that BROKE refuse-dirty-publish.sh would have its self-tests pass, because
+      # they would be exercising the unbroken copy on main. It surfaced only because this PR added
+      # a file that main does not have, so the step failed with "No such file or directory" instead
+      # of quietly passing.
+      #
+      # Checking out the head of the branch under test gives HEAD an upstream AND keeps the tree
+      # that is actually being reviewed. On a push to main the two are the same thing.
+      - name: Give HEAD an upstream, WITHOUT discarding the tree under test
         run: |
-          git fetch origin main
-          git checkout -B guard-selftest origin/main
-          git branch --set-upstream-to=origin/main guard-selftest
+          BRANCH="${{ github.head_ref || github.ref_name }}"
+          git fetch origin "$BRANCH"
+          git checkout -B guard-selftest "origin/$BRANCH"
+          git branch --set-upstream-to="origin/$BRANCH" guard-selftest
+          echo "self-tests are running against origin/$BRANCH at $(git rev-parse --short HEAD)"
+
+      # The tree under test must be THIS branch's, not whatever the previous step happened to leave.
+      # Without this line the failure above is silent again the next time someone edits that step.
+      - name: The checked-out tree must be the one under review
+        run: |
+          WANT="${{ github.event.pull_request.head.sha || github.sha }}"
+          GOT="$(git rev-parse HEAD)"
+          if [ "$WANT" != "$GOT" ]; then
+            echo "::error::self-tests are running against $GOT but the tree under review is $WANT — they would be testing the wrong code"
+            exit 1
+          fi
+          echo "tree under test confirmed: $GOT"
 
       - name: The guard must refuse an already-published version FOR THAT REASON
         working-directory: packages/x402-op-authorize
@@ -102,3 +161,55 @@ jobs:
             exit 1
           fi
           echo "negative control fired, naming the instrument"
+
+      # ─── THE POPULATION ASSERTION MUST BE ABLE TO FAIL ──────────────────────────────────────────
+      # A coverage check that cannot go red is worth less than no coverage check, because it reads
+      # as coverage. These cases are synthetic on purpose: they exercise the assertion itself rather
+      # than the state of this repo today, so the assertion stays honest as the real population moves.
+      - name: The population assertion must fail when everything is skipped
+        run: |
+          FLOOR="$(grep -vE '^\s*#|^\s*$' scripts/reproduction-floor.txt | head -1 | tr -d '[:space:]')"
+          mkdir -p /tmp/allskip
+          for d in packages/*/; do
+            p="$(basename "${d%/}")"
+            mkdir -p "/tmp/allskip/repro-result-$p"
+            echo "skipped $p@0.0.0-synthetic" > "/tmp/allskip/repro-result-$p/result.txt"
+          done
+          OUT="$(bash scripts/assert-reproduction-population.sh /tmp/allskip 2>&1 || true)"
+          echo "$OUT"
+          # ASSERT THE REASON. A bare "it exited non-zero" would also be satisfied by the missing-result
+          # branch, so this requires the floor comparison specifically, quoting the floor it read.
+          if ! printf '%s' "$OUT" | grep -q "only 0 package(s) were actually measured against the registry; the floor is ${FLOOR}"; then
+            echo "::error::a matrix where every package is unpublished did not trip the floor. The coverage check cannot fail, so its green means nothing."
+            exit 1
+          fi
+          echo "floor tripped at 0 measured, as intended"
+
+      - name: The population assertion must fail when a package reports nothing
+        run: |
+          mkdir -p /tmp/missing
+          FIRST=""
+          for d in packages/*/; do
+            p="$(basename "${d%/}")"
+            if [ -z "$FIRST" ]; then FIRST="$p"; continue; fi     # this one deliberately reports nothing
+            mkdir -p "/tmp/missing/repro-result-$p"
+            echo "measured $p@0.0.0-synthetic" > "/tmp/missing/repro-result-$p/result.txt"
+          done
+          OUT="$(bash scripts/assert-reproduction-population.sh /tmp/missing 2>&1 || true)"
+          echo "$OUT"
+          if ! printf '%s' "$OUT" | grep -q "1 package(s) in packages/ produced no reproduction result"; then
+            echo "::error::a package that reported nothing was not counted as missing — an omitted or crashed job would read as covered"
+            exit 1
+          fi
+          echo "missing-result branch fired for $FIRST, as intended"
+
+      - name: The population assertion must PASS when the floor is met
+        run: |
+          mkdir -p /tmp/ok
+          for d in packages/*/; do
+            p="$(basename "${d%/}")"
+            mkdir -p "/tmp/ok/repro-result-$p"
+            echo "measured $p@0.0.0-synthetic" > "/tmp/ok/repro-result-$p/result.txt"
+          done
+          # Without this the two cases above are satisfied by an assertion that always fails.
+          bash scripts/assert-reproduction-population.sh /tmp/ok
diff --git a/scripts/assert-reproduction-population.sh b/scripts/assert-reproduction-population.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+# HOW MANY PACKAGES WERE ACTUALLY MEASURED? Asserted here, because the matrix cannot answer it.
+#
+# The reproduction matrix reports whether each job PASSED. It has never reported how many packages
+# were CHECKED, and those are different facts: a skip passes. Measured 2026-08-09, five of seven
+# jobs were green and two of those were skips, so "7 jobs, 0 failures" described a run that had
+# compared five packages against the registry and two against nothing.
+#
+# This asserts three things, each able to fail on its own:
+#
+#   1. EVERY package directory produced a result. A job that was omitted from the matrix, or that
+#      died before it could record anything, is counted as "no result" rather than passed. The
+#      expected population is read from packages/ rather than from the matrix, so a package added
+#      to the repo and forgotten in the workflow fails here instead of being silently uncovered.
+#   2. The measured count is at or above the floor declared in scripts/reproduction-floor.txt.
+#   3. The floor file itself is present and parseable. Fail closed: an unreadable floor is not
+#      permission to skip the check, and a missing file must not read as a floor of zero.
+#
+# Usage: assert-reproduction-population.sh <results-dir>
+#   <results-dir> holds one file per package, each containing "<state> <name>@<version>",
+#   where <state> is measured | skipped | inconclusive.
+set -uo pipefail
+
+DIR="${1:?usage: assert-reproduction-population.sh <results-dir>}"
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+FLOOR_FILE="${ROOT}/scripts/reproduction-floor.txt"
+
+# ─── 3. THE FLOOR MUST BE READABLE ───────────────────────────────────────────────────────────────
+if [ ! -f "${FLOOR_FILE}" ]; then
+  echo "REFUSING: ${FLOOR_FILE} is missing. A floor that cannot be read is not a floor of zero." >&2
+  exit 1
+fi
+FLOOR="$(grep -vE '^\s*#|^\s*$' "${FLOOR_FILE}" | head -1 | tr -d '[:space:]')"
+if ! printf '%s' "${FLOOR}" | grep -qE '^[0-9]+$'; then
+  echo "REFUSING: ${FLOOR_FILE} does not contain a bare integer (read: '${FLOOR}')." >&2
+  exit 1
+fi
+
+# ─── 1. EVERY PACKAGE MUST HAVE REPORTED ─────────────────────────────────────────────────────────
+EXPECTED=()
+for d in "${ROOT}"/packages/*/; do
+  [ -f "${d}package.json" ] || continue
+  EXPECTED+=("$(basename "${d%/}")")
+done
+if [ "${#EXPECTED[@]}" -eq 0 ]; then
+  echo "REFUSING: no package directories found under ${ROOT}/packages. An empty population passes" >&2
+  echo "          every check trivially, which is the failure this line exists to prevent." >&2
+  exit 1
+fi
+
+MEASURED=0; SKIPPED=0; INCONCLUSIVE=0; MISSING=0
+echo "REPRODUCTION POPULATION"
+echo
+for pkg in "${EXPECTED[@]}"; do
+  # The result file may arrive at <dir>/<pkg>/result.txt or <dir>/repro-result-<pkg>/result.txt
+  # depending on how artifacts were laid down. Find it rather than assuming one shape.
+  f="$(find "${DIR}" -type f -path "*${pkg}*" -name '*.txt' 2>/dev/null | head -1)"
+  if [ -z "${f}" ] || [ ! -s "${f}" ]; then
+    printf '  %-26s NO RESULT\n' "${pkg}"
+    MISSING=$((MISSING + 1)); continue
+  fi
+  line="$(head -1 "${f}")"
+  state="${line%% *}"
+  case "${state}" in
+    measured)     MEASURED=$((MEASURED + 1)) ;;
+    skipped)      SKIPPED=$((SKIPPED + 1)) ;;
+    inconclusive) INCONCLUSIVE=$((INCONCLUSIVE + 1)) ;;
+    *)            printf '  %-26s UNRECOGNISED STATE: %s\n' "${pkg}" "${line}"; MISSING=$((MISSING + 1)); continue ;;
+  esac
+  printf '  %-26s %s\n' "${pkg}" "${line}"
+done
+
+echo
+echo "  packages in repo : ${#EXPECTED[@]}"
+echo "  measured         : ${MEASURED}   (floor ${FLOOR})"
+echo "  skipped          : ${SKIPPED}"
+echo "  inconclusive     : ${INCONCLUSIVE}"
+echo "  no result        : ${MISSING}"
+echo
+
+FAILED=0
+
+if [ "${MISSING}" -gt 0 ]; then
+  cat >&2 <<MISSING_EOF
+  FAIL: ${MISSING} package(s) in packages/ produced no reproduction result.
+
+  Either the matrix in .github/workflows/reproduction.yml does not list them, or their job died
+  before recording an outcome. Both mean this run does not know their state, and not knowing is
+  not a pass.
+
+MISSING_EOF
+  FAILED=1
+fi
+
+if [ "${MEASURED}" -lt "${FLOOR}" ]; then
+  cat >&2 <<FLOOR_EOF
+  FAIL: only ${MEASURED} package(s) were actually measured against the registry; the floor is ${FLOOR}.
+
+  A skip is green. That is why this check exists: coverage can fall without any job going red, and
+  it did — a version bump moved a package to an unpublished number and took the reproduction check
+  with it, silently.
+
+  If the drop is intended, lower the number in scripts/reproduction-floor.txt and record there which
+  package left and why. Accepting less coverage is a decision someone makes, not a side effect.
+
+FLOOR_EOF
+  FAILED=1
+fi
+
+if [ "${FAILED}" -ne 0 ]; then exit 1; fi
+
+echo "  OK: ${MEASURED} package(s) measured, at or above the declared floor of ${FLOOR}."
+exit 0
diff --git a/scripts/reproduction-floor.txt b/scripts/reproduction-floor.txt
@@ -0,0 +1,29 @@
+# HOW MANY PACKAGES MUST ACTUALLY BE MEASURED FOR A GREEN CI RUN TO MEAN ANYTHING.
+#
+# scripts/verify-reproduction.sh exits 0 for a package whose declared version is not published:
+# there is no published artifact to compare against, so there is nothing to do. That is correct,
+# and it is also invisible. At job level a skip and a reproduction are the same green tick.
+#
+# Measured 2026-08-09 on run 31289030963: seven matrix jobs, five green, and TWO of those five
+# were skips. Coverage had dropped from six packages to five and no signal said so. The drop was
+# caused by a CORRECT fix (x402-op-authorize was bumped off the 0.4.0 it had drifted from), which
+# is the point: nothing went wrong, and coverage fell anyway.
+#
+# This number is the floor. It is a DECISION, not a measurement, and it lives in a committed file
+# so that lowering it is an edit someone has to make and a reviewer has to see. If a version bump
+# takes another package below this line, CI goes red and the choice to accept less coverage gets
+# made deliberately instead of by side effect.
+#
+# RAISE IT when a package is published and starts being measured.
+# LOWER IT only as an explicit, reviewed decision, and say here which package left and why.
+#
+# Current population, measured 2026-08-09:
+#   measured (5) : fireblocks-op-authorize@0.1.0, l402-op-authorize@0.4.0, mppx-op-account@0.3.1,
+#                  ows-op-verify@0.3.0, wdk-op-policy@0.4.1
+#   skipped  (2) : ap2-op-authorize@0.1.0 (never published),
+#                  x402-op-authorize@0.4.1-rc.0 (bumped off a drifted 0.4.0, not yet published)
+#
+# l402-op-authorize counts as MEASURED even though its job is red. The comparison ran and returned
+# an answer. A red measurement is coverage; a skip is not.
+
+5
diff --git a/scripts/verify-reproduction.sh b/scripts/verify-reproduction.sh
@@ -14,9 +14,33 @@
 #
 # Together the two are total: a tree either declares an unpublished version (publishable, skipped
 # here) or a published one (must reproduce it). There is no third state.
+#
+# BUT A SKIP IS NOT FREE, AND THAT IS WHY THIS SCRIPT RECORDS ITS OUTCOME.
+# A skip exits 0, so the job goes green, and at job level a skip is indistinguishable from a
+# reproduction. Measured 2026-08-09: of seven matrix jobs, five were green and TWO of those five were
+# skips — x402-op-authorize (bumped to the unpublished 0.4.1-rc.0) and ap2-op-authorize (never
+# published). The bump was correct and was the fix for a real defect. Its side effect was to move the
+# one package this check was written for out of the check's reach, silently, while CI stayed green.
+#
+# So each run records measured/skipped/inconclusive, and scripts/assert-reproduction-population.sh
+# asserts the population afterwards. The number of packages CHECKED is a separate fact from the number
+# that PASSED, and only the second was ever visible.
 set -uo pipefail
 PKG_DIR="${1:?usage: verify-reproduction.sh <package-dir>}"
-cd "$PKG_DIR" || exit 1
+
+NAME=""; VER=""
+
+# Record the outcome for the population assertion. Written at every exit path, including the failing
+# ones: a job that dies without writing this file is counted as "no result" rather than as a pass, so
+# a crashed or omitted job cannot be mistaken for a measured one.
+record() {                              # $1 = measured | skipped | inconclusive
+  if [ -n "${REPRO_RESULT_FILE:-}" ]; then
+    printf '%s %s@%s\n' "$1" "${NAME:-unknown}" "${VER:-unknown}" > "${REPRO_RESULT_FILE}"
+  fi
+  return 0
+}
+
+cd "$PKG_DIR" || { record inconclusive; exit 1; }
 NAME="$(node -p 'require("./package.json").name')"
 VER="$(node -p 'require("./package.json").version')"
 
@@ -25,22 +49,24 @@ VER="$(node -p 'require("./package.json").version')"
 OUT="$(npm view "${NAME}@${VER}" version --json 2>/dev/null)"; RC=$?
 if [ $RC -ne 0 ] && printf '%s' "$OUT" | grep -q '"code": *"E404"'; then
   echo "SKIP  ${NAME}@${VER} — not published, nothing to reproduce"
+  record skipped
   exit 0
 fi
 if [ $RC -ne 0 ] || ! printf '%s' "$OUT" | grep -q "\"${VER}\""; then
   echo "FAIL  ${NAME}@${VER} — could not establish whether this version is published (fail-closed)" >&2
+  record inconclusive
   exit 1
 fi
 
 WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
 ( cd "$WORK" && npm pack "${NAME}@${VER}" >/dev/null 2>&1 ) || {
-  echo "FAIL  ${NAME}@${VER} — could not fetch the published tarball (fail-closed)" >&2; exit 1; }
+  echo "FAIL  ${NAME}@${VER} — could not fetch the published tarball (fail-closed)" >&2; record inconclusive; exit 1; }
 PUBLISHED="$(ls "$WORK"/*.tgz | head -1)"
 
-npm ci --silent >/dev/null 2>&1 || { echo "FAIL  ${NAME}@${VER} — npm ci failed" >&2; exit 1; }
-npm run build --silent >/dev/null 2>&1 || { echo "FAIL  ${NAME}@${VER} — build failed" >&2; exit 1; }
+npm ci --silent >/dev/null 2>&1 || { echo "FAIL  ${NAME}@${VER} — npm ci failed" >&2; record inconclusive; exit 1; }
+npm run build --silent >/dev/null 2>&1 || { echo "FAIL  ${NAME}@${VER} — build failed" >&2; record inconclusive; exit 1; }
 REBUILT="$(npm pack --silent 2>/dev/null | tail -1)"
-[ -f "$REBUILT" ] || { echo "FAIL  ${NAME}@${VER} — npm pack produced nothing" >&2; exit 1; }
+[ -f "$REBUILT" ] || { echo "FAIL  ${NAME}@${VER} — npm pack produced nothing" >&2; record inconclusive; exit 1; }
 
 # WHAT IS COMPARED, AND WHY IT IS NOT THE WHOLE TARBALL.
 #
@@ -67,6 +93,8 @@ if ! diff -rq "$WORK/a/package/dist" "$WORK/b/package/dist" >/dev/null 2>&1; the
   echo "      restore the tree." >&2
   diff -rq "$WORK/a/package/dist" "$WORK/b/package/dist" 2>&1 \
     | sed "s|$WORK/a/package/dist|published:|g; s|$WORK/b/package/dist|rebuilt:|g" | sed 's/^/      /' >&2
+  # MEASURED, not inconclusive: the comparison ran and returned an answer. The answer is "no".
+  record measured
   exit 1
 fi
 
@@ -80,4 +108,5 @@ if [ -n "$OTHER" ]; then
 else
   echo "OK    ${NAME}@${VER} — reproduces the published tarball byte-for-byte"
 fi
+record measured
 exit 0