CI-4: make every allow-fail unit blocking or remove it; Containerfile reproducibility replaces the Guix job

.github/workflows/chapel-ci.yml

@@ -141,18 +141,17 @@ jobs:
           path: src/zig_ffi/zig-out/lib/
 
   # Job 3: Build Rust with chapel feature (links against Zig FFI stubs).
-  # Marked continue-on-error: true alongside its needed dependency zig-ffi
-  # (#133) — when the upstream Zig FFI build is allowed to fail, this job's
-  # "Download FFI library" step has no artifact to consume and red-fires.
-  # Re-enable strict gating once #133's Chapel + FFI rehabilitation lands.
+  # Blocking since the CI-4 ruling (#416). It was allow-fail while its
+  # dependency zig-ffi was; zig-ffi has been strict since #133 closed, and
+  # this job passed on main after #455 (run 42100322359) and on every PR run
+  # checked up to 2026-11-09.
   rust-chapel-feature:
     name: Rust Build with Chapel Feature
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     needs: zig-ffi
     env:
       ECHIDNA_CHAPEL_EXPECT_MODE: stubs
-    continue-on-error: true  # allow-fail: issue=#416 expires=2026-11-30
     steps:
       - uses: actions/checkout@v7.0.1
 
@@ -180,16 +179,19 @@ jobs:
   #
   # Cargo is the final static consumer: Zig's archive intentionally leaves
   # chapel_* references for libechidna_chapel.a + the compiler's runtime.
-  # No PIC rebuild is needed for this executable link. The existing #416
-  # waiver is unchanged; promotion requires the exact-source hosted result.
+  # No PIC rebuild is needed for this executable link.
+  #
+  # Blocking since the CI-4 ruling (#416). The real link landed with #455
+  # (build.rs, build_support/chapel.rs), and this job passed on main
+  # (run 42100322359, job 127501960139) and on every PR run checked up to
+  # 2026-11-09. It also consumes chapel-build's chapel-lib artifact.
   rust-chapel-real:
-    name: Rust Build — Real Chapel Library (allow-fail, L2.3+ gate)
+    name: Rust Build — Real Chapel Library
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     needs: [chapel-build, zig-ffi]
     env:
       ECHIDNA_CHAPEL_EXPECT_MODE: real
-    continue-on-error: true  # allow-fail: issue=#416 expires=2026-11-30
     steps:
       - uses: actions/checkout@v7.0.1
 

.github/workflows/container-ci.yml

@@ -6,12 +6,23 @@
 #
 # container-ci.yml — Container build verification.
 #
-# Two jobs:
-#   container-build   — builds the minimal ECHIDNA image on every PR/push;
-#                       hard gate.
-#   tier3-container   — builds each Wave-3 prover image weekly (Sunday 06:00 UTC)
-#                       and on any change to the .containerization/** tree;
-#                       allow-fail per image (proprietary builds excluded).
+# Jobs, all blocking:
+#   container-build         — builds and verifies the minimal ECHIDNA image.
+#   container-rebuild       — builds the same image again on an independent
+#                             runner.
+#   container-reproducible  — requires both builds to have the same image ID.
+#                             This is the reproducibility check that replaced
+#                             the T1 Guix manifest job (owner ruling, CI-4):
+#                             the Containerfile is the reproducibility path.
+#   tier3-container         — builds each Wave-3 prover image weekly (Sunday
+#                             06:00 UTC); proprietary builds excluded.
+#
+# Both minimal builds use --no-cache and --timestamp 0, so creation times
+# and file mtimes cannot make two builds of the same inputs differ. A
+# mismatch means an input floats (an unpinned package, a download without
+# a digest, a non-deterministic build step). The compare job prints the
+# layer digests of both builds, and the deepest paths whose digests differ
+# (each build prints a per-path fingerprint, scripts/ci/fs-fingerprint.sh).
 
 name: Container Build Verification
 
@@ -24,6 +35,15 @@
     paths:
       - 'Containerfile'
       - 'scripts/ci/install-lean.sh'
+      - 'scripts/ci/fs-fingerprint.sh'
+      - 'scripts/ci/fs-fingerprint-drift.sh'
+      - 'scripts/ci/test-fs-fingerprint.sh'
+      # Everything the Containerfile copies into the image (lines 26-42).
+      - 'build.rs'
+      - 'build_support/**'
+      - 'crates/**'
+      - 'benches/**'
+      - 'src/interfaces/**'
       - '.containerization/**'
       - 'Cargo.toml'
       - 'Cargo.lock'
@@ -33,6 +53,15 @@
     paths:
       - 'Containerfile'
       - 'scripts/ci/install-lean.sh'
+      - 'scripts/ci/fs-fingerprint.sh'
+      - 'scripts/ci/fs-fingerprint-drift.sh'
+      - 'scripts/ci/test-fs-fingerprint.sh'
+      # Everything the Containerfile copies into the image (lines 26-42).
+      - 'build.rs'
+      - 'build_support/**'
+      - 'crates/**'
+      - 'benches/**'
+      - 'src/interfaces/**'
       - '.containerization/**'
       - 'Cargo.toml'
       - 'Cargo.lock'
@@ -43,6 +72,7 @@
     # Distinct from live-provers.yml Tier-3 run (Sunday 05:00 UTC)
     # so the container build does not race the live-prover tests.
     - cron: '0 6 * * 0'
+  workflow_dispatch:
 
 # Cause-B mitigation (#77): de-duplicate superseded builds on the same
 # ref. cancel-in-progress is false so an in-flight image build/publish
@@ -60,11 +90,19 @@
     name: Build & verify container image
     runs-on: ubuntu-latest
     timeout-minutes: 90
+    outputs:
+      image-id: ${{ steps.identity.outputs.image-id }}
+      layers: ${{ steps.identity.outputs.layers }}
 
     steps:
       - name: Checkout
         uses: actions/checkout@v7.0.1
 
+      # The fingerprint and drift scripts are exercised on every run, not only
+      # when two builds differ.
+      - name: Test filesystem fingerprints
+        run: bash scripts/ci/test-fs-fingerprint.sh
+
       - name: Install Podman
         run: |
           sudo apt-get update -q
@@ -76,8 +114,31 @@
             -f Containerfile \
             -t echidna:ci-test \
             --no-cache \
+            --timestamp 0 \
             .
 
+      - name: Record image identity
+        id: identity
+        run: |
+          set -euo pipefail
+          echo "image-id=$(podman image inspect echidna:ci-test --format '{{.Id}}')" >> "$GITHUB_OUTPUT"
+          echo "layers=$(podman image inspect echidna:ci-test --format '{{json .RootFS.Layers}}')" >> "$GITHUB_OUTPUT"
+
+      # Per-path digests of the image filesystem, printed between markers so
+      # container-reproducible can name the drifting paths, not only the layer.
+      - name: Fingerprint the image filesystem
+        run: |
+          set -euo pipefail
+          cid=$(podman create echidna:ci-test)
+          podman export "$cid" | bash scripts/ci/fs-fingerprint.sh 12 > "$RUNNER_TEMP/fs-fingerprint"
+          podman rm "$cid" >/dev/null
+          echo "fs-fingerprint: $(wc -l < "$RUNNER_TEMP/fs-fingerprint") paths"
+          echo "::group::fs-fingerprint (depth 12)"
+          echo "FS-FINGERPRINT-BEGIN"
+          cat "$RUNNER_TEMP/fs-fingerprint"
+          echo "FS-FINGERPRINT-END"
+          echo "::endgroup::"
+
       - name: Verify image metadata
         run: |
           echo "=== Image labels ==="
@@ -105,24 +166,130 @@
         if: always()
         run: podman rmi echidna:ci-test 2>/dev/null || true
 
-  # ── Job 2: Wave-3 Tier-3 prover images — weekly, allow-fail per image ─────
+  # ── Containerfile reproducibility: an independent rebuild ─────────────────
+  container-rebuild:
+    name: Rebuild container image (independent runner)
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    outputs:
+      image-id: ${{ steps.identity.outputs.image-id }}
+      layers: ${{ steps.identity.outputs.layers }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+
+      - name: Install Podman
+        run: |
+          sudo apt-get update -q
+          sudo apt-get install -y -q podman
+
+      - name: Rebuild minimal container image
+        run: |
+          podman build \
+            -f Containerfile \
+            -t echidna:ci-rebuild \
+            --no-cache \
+            --timestamp 0 \
+            .
+
+      - name: Record image identity
+        id: identity
+        run: |
+          set -euo pipefail
+          echo "image-id=$(podman image inspect echidna:ci-rebuild --format '{{.Id}}')" >> "$GITHUB_OUTPUT"
+          echo "layers=$(podman image inspect echidna:ci-rebuild --format '{{json .RootFS.Layers}}')" >> "$GITHUB_OUTPUT"
+
+      # Per-path digests of the image filesystem, printed between markers so
+      # container-reproducible can name the drifting paths, not only the layer.
+      - name: Fingerprint the image filesystem
+        run: |
+          set -euo pipefail
+          cid=$(podman create echidna:ci-rebuild)
+          podman export "$cid" | bash scripts/ci/fs-fingerprint.sh 12 > "$RUNNER_TEMP/fs-fingerprint"
+          podman rm "$cid" >/dev/null
+          echo "fs-fingerprint: $(wc -l < "$RUNNER_TEMP/fs-fingerprint") paths"
+          echo "::group::fs-fingerprint (depth 12)"
+          echo "FS-FINGERPRINT-BEGIN"
+          cat "$RUNNER_TEMP/fs-fingerprint"
+          echo "FS-FINGERPRINT-END"
+          echo "::endgroup::"
+
+  container-reproducible:
+    name: Containerfile reproducibility (same image ID twice)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [container-build, container-rebuild]
+    permissions:
+      actions: read     # read the two build jobs' logs for their fs-fingerprints
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+
+      - name: Compare the two builds
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ID_A: ${{ needs.container-build.outputs.image-id }}
+          ID_B: ${{ needs.container-rebuild.outputs.image-id }}
+          LAYERS_A: ${{ needs.container-build.outputs.layers }}
+          LAYERS_B: ${{ needs.container-rebuild.outputs.layers }}
+        run: |
+          set -euo pipefail
+          if [ -z "$ID_A" ] || [ -z "$ID_B" ]; then echo "::error::a build reported no image ID"; exit 1; fi
+          {
+            echo "## Containerfile reproducibility"
+            echo "build:   $ID_A"
+            echo "rebuild: $ID_B"
+          } >> "$GITHUB_STEP_SUMMARY"
+          if [ "$ID_A" = "$ID_B" ]; then
+            echo "OK: two independent --no-cache --timestamp 0 builds gave image $ID_A"
+            exit 0
+          fi
+          echo "::error::The Containerfile is not reproducible: two builds of the same commit gave $ID_A and $ID_B."
+          paste <(jq -r '.[]' <<<"$LAYERS_A") <(jq -r '.[]' <<<"$LAYERS_B") \
+            | awk -F '\t' '{ printf "layer %d: %s %s %s\n", NR, ($1 == $2 ? "same" : "DIFFERS"), $1, $2 }' \
+            | tee -a "$GITHUB_STEP_SUMMARY"
+          # Name the drift: each build job printed per-path digests of its
+          # image between markers (scripts/ci/fs-fingerprint.sh).
+          fingerprint() {
+            local id
+            id=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs" \
+                   --paginate --jq ".jobs[] | select(.name == \"$1\") | .id" | head -n 1)
+            [ -n "$id" ] || { echo "::warning::no job named '$1' in this run attempt"; return 1; }
+            gh api "repos/$GITHUB_REPOSITORY/actions/jobs/$id/logs" \
+              | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' \
+              | sed -n '/^FS-FINGERPRINT-BEGIN$/,/^FS-FINGERPRINT-END$/p' \
+              | grep -E '^[0-9a-f]{64}  '
+          }
+          if fingerprint "Build & verify container image" > "$RUNNER_TEMP/fp-a" \
+             && fingerprint "Rebuild container image (independent runner)" > "$RUNNER_TEMP/fp-b"; then
+            {
+              echo "### Paths that differ between the two builds (deepest only)"
+              echo '```'
+              bash scripts/ci/fs-fingerprint-drift.sh "$RUNNER_TEMP/fp-a" "$RUNNER_TEMP/fp-b" | head -n 200
+              echo '```'
+            } | tee -a "$GITHUB_STEP_SUMMARY"
+          else
+            echo "::warning::could not read both fs-fingerprints; only the layer comparison is available"
+          fi
+          exit 1
+
+  # ── Wave-3 Tier-3 prover images — weekly, blocking per image ──────────────
   #
-  # Runs on the weekly schedule and whenever any .containerization/** file
-  # changes.  Each prover builds in its own matrix cell; allow-fail is set
-  # at cell level so one broken prover does not mask the others.
+  # Runs on the weekly schedule. Each prover builds in its own matrix cell
+  # (fail-fast off), so one broken prover does not mask the others. Blocking
+  # since the CI-4 ruling (#416): the scheduled run on main of 2026-11-08
+  # (42453284210) passed all eight cells, stub-sentinel detection included.
   #
   # Imandra is excluded: it is proprietary and requires a licence token not
   # available in public CI.
   tier3-container:
     name: Tier-3 / ${{ matrix.prover }}
     runs-on: ubuntu-latest
     timeout-minutes: 90
-    # Run on schedule or when the .containerization tree changes on main.
-    # Not a merge gate — these are informational weekly builds.
-    if: >-
-      github.event_name == 'schedule' ||
-      (github.event_name == 'push' && contains(github.event.head_commit.modified, '.containerization/'))
-    continue-on-error: true  # allow-fail: issue=#416 expires=2026-11-30
+    # Weekly and on manual dispatch. A red cell is a real failure, but these
+    # builds are too slow to gate every PR.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     strategy:
       fail-fast: false
       matrix: