From 5b641c9bc8fb0cbfcdd1df2f24eb556419afdfa8 Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:36:34 -0700
Subject: [PATCH 1/3] ci: route merge-queue heavy jobs to Latchkey behind a
 fallback guard

Merge-queue runs send the heavy test jobs to Latchkey runners when
vars.LATCHKEY_HEAVY is 'on': backend on medium, frontend and service on
small. PR runs stay on GitHub-hosted runners.

A held Latchkey job sits queued with no error, so routing never trusts
Latchkey blind. A probe job must be picked up first. A fresh hold or a
spent budget keeps the run on GitHub. A heavy job held after routing
trips a backstop, which records a hold and cancels the run, so the queue
entry fails instead of stalling.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VD3azQJZrdMmw8g51iscWD
---
 .github/workflows/tests.yml               | 166 +++++++++++++----
 docs/runbooks/ci-heavy-runner.md          | 137 ++++++++++----
 scripts/__tests__/latchkey-route.test.mjs | 138 ++++++++++++++
 scripts/latchkey-route.mjs                | 209 ++++++++++++++++++++++
 4 files changed, 586 insertions(+), 64 deletions(-)
 create mode 100644 scripts/__tests__/latchkey-route.test.mjs
 create mode 100644 scripts/latchkey-route.mjs

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a6e01dee17..84be4e4b2b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -22,6 +22,19 @@ on:
   schedule:
     - cron: '0 8 * * *'
   workflow_dispatch:
+    inputs:
+      latchkey:
+        description: Route this run's heavy jobs through the Latchkey path (rehearsal; the merge queue uses vars.LATCHKEY_HEAVY)
+        type: boolean
+        default: false
+      latchkey_probe_label:
+        description: Runner label for the probe; a label with no runners rehearses a held job
+        type: string
+        default: ''
+      latchkey_budget_eqmin:
+        description: Override the free-minute budget; 0 rehearses spent minutes
+        type: string
+        default: ''
 
 permissions:
   contents: read
@@ -48,21 +61,120 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # Heavy jobs (backend, frontend, service tests) run on Latchkey runners on
+  # merge-queue runs when vars.LATCHKEY_HEAVY is 'on': those runners don't
+  # count against the org's 20-concurrent-job cap that made the queue wait
+  # (D37). Every other run, PRs included, stays on GitHub-hosted runners, which
+  # are free on this public repo. Unsetting LATCHKEY_HEAVY turns it all off.
+  #
+  # A Latchkey job that is held (free minutes spent, or an outage) sits
+  # "queued" on GitHub with no error, so the route never trusts Latchkey
+  # blind: it falls back to GitHub-hosted on a recent hold or a spent budget,
+  # and otherwise waits for this probe to be picked up first. The probe is
+  # deliberately outside the gate's needs: a held probe must not hold the
+  # required check. See docs/runbooks/ci-heavy-runner.md.
+  latchkey-probe:
+    name: Latchkey probe
+    if: ${{ (github.event_name == 'merge_group' && vars.LATCHKEY_HEAVY == 'on') || (github.event_name == 'workflow_dispatch' && inputs.latchkey) }}
+    runs-on: ${{ inputs.latchkey_probe_label || vars.LATCHKEY_LIGHT_LABEL || 'latchkey-small' }}
+    timeout-minutes: 2
+    steps:
+      - run: echo "picked up by $RUNNER_NAME"
+
+  latchkey-route:
+    name: Latchkey route
+    if: ${{ (github.event_name == 'merge_group' && vars.LATCHKEY_HEAVY == 'on') || (github.event_name == 'workflow_dispatch' && inputs.latchkey) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 6
+    permissions:
+      actions: read
+      contents: read
+    outputs:
+      backend: ${{ steps.route.outputs.backend }}
+      frontend: ${{ steps.route.outputs.frontend }}
+      service: ${{ steps.route.outputs.service }}
+      use: ${{ steps.route.outputs.use }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: scripts/latchkey-route.mjs
+          sparse-checkout-cone-mode: false
+      - id: route
+        run: node scripts/latchkey-route.mjs route
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUDGET_START: ${{ vars.LATCHKEY_BUDGET_START }}
+          BUDGET_EQMIN: ${{ inputs.latchkey_budget_eqmin || vars.LATCHKEY_BUDGET_EQMIN }}
+          EQ_PER_RUN: ${{ vars.LATCHKEY_EQ_PER_RUN }}
+          PICKUP_SEC: ${{ vars.LATCHKEY_PICKUP_SEC }}
+          BACKEND_LABEL: ${{ vars.LATCHKEY_BACKEND_LABEL }}
+          LIGHT_LABEL: ${{ vars.LATCHKEY_LIGHT_LABEL }}
+      - if: ${{ steps.route.outputs.hold == 'true' }}
+        run: echo "probe not picked up in run ${{ github.run_id }}" > latchkey-hold.txt
+      - if: ${{ steps.route.outputs.hold == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: latchkey-hold
+          path: latchkey-hold.txt
+          retention-days: 1
+          overwrite: true
+      # One artifact per Latchkey run is the budget's counter.
+      - if: ${{ steps.route.outputs.use == 'true' }}
+        run: echo "${{ github.run_id }}" > latchkey-run.txt
+      - if: ${{ steps.route.outputs.use == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: latchkey-run
+          path: latchkey-run.txt
+          retention-days: 35
+          overwrite: true
+
+  # Once the heavy jobs are routed to Latchkey, a held one would leave the
+  # required check pending. This cancels the run instead, so the queue entry
+  # fails loudly, and records a hold so the next runs stay on GitHub.
+  latchkey-backstop:
+    name: Latchkey backstop
+    needs: latchkey-route
+    if: ${{ needs.latchkey-route.outputs.use == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 12
+    permissions:
+      actions: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: scripts/latchkey-route.mjs
+          sparse-checkout-cone-mode: false
+      - id: watch
+        run: node scripts/latchkey-route.mjs backstop
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - if: ${{ steps.watch.outputs.tripped == 'true' }}
+        run: echo "heavy job held in run ${{ github.run_id }}" > latchkey-hold.txt
+      - if: ${{ steps.watch.outputs.tripped == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: latchkey-hold
+          path: latchkey-hold.txt
+          retention-days: 1
+          overwrite: true
+      - if: ${{ steps.watch.outputs.tripped == 'true' }}
+        run: gh run cancel ${{ github.run_id }} --repo ${{ github.repository }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+
   backend-tests:
     name: Backend tests (${{ matrix.shard }}/3)
     strategy:
       fail-fast: false
       matrix:
         shard: [1, 2, 3]
-    # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey
-    # runners, which don't count against the org's 20-concurrent-job cap).
-    # Unset means GitHub-hosted, so rolling back is deleting one variable. The
-    # job names, and so the required check, don't change either way. A fork
-    # PR's own pull_request run stays on GitHub. A merge_group run uses
-    # HEAVY_RUNNER even when the group holds a reviewed fork change: queue
-    # admission (a maintainer queues a gated PR) is the trust decision, not
-    # this expression. See docs/runbooks/ci-heavy-runner.md.
-    runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }}
+    # Routed by latchkey-route (backend on medium, the rest on small; see
+    # the route job). Skipped routing leaves the output empty: GitHub-hosted.
+    needs: latchkey-route
+    if: ${{ !cancelled() }}
+    runs-on: ${{ needs.latchkey-route.outputs.backend || 'ubuntu-latest' }}
     env:
       JWT_SECRET: test-jwt-secret
       SENDGRID_API_KEY: fake-api-key
@@ -109,15 +221,10 @@ jobs:
 
   frontend-tests:
     name: Frontend tests
-    # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey
-    # runners, which don't count against the org's 20-concurrent-job cap).
-    # Unset means GitHub-hosted, so rolling back is deleting one variable. The
-    # job names, and so the required check, don't change either way. A fork
-    # PR's own pull_request run stays on GitHub. A merge_group run uses
-    # HEAVY_RUNNER even when the group holds a reviewed fork change: queue
-    # admission (a maintainer queues a gated PR) is the trust decision, not
-    # this expression. See docs/runbooks/ci-heavy-runner.md.
-    runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }}
+    # Routed by latchkey-route, as backend-tests.
+    needs: latchkey-route
+    if: ${{ !cancelled() }}
+    runs-on: ${{ needs.latchkey-route.outputs.frontend || 'ubuntu-latest' }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -174,6 +281,7 @@ jobs:
       - run: npm run verify:moltbot-tools
       - run: npm run verify:litellm-patch-runner
       - run: npm run verify:readme-invariants
+      - run: node --test scripts/__tests__/latchkey-route.test.mjs
       # These node:test files lived only in the retired all-in-one job.
       - name: Run agent service tests
         run: |
@@ -190,7 +298,8 @@ jobs:
   test-gate:
     name: Test & Coverage
     if: always()
-    needs: [backend-tests, backend-checks, frontend-tests, tool-tests, service-test, chart-lint]
+    # latchkey-probe is left out on purpose: see the comment above it.
+    needs: [latchkey-route, latchkey-backstop, backend-tests, backend-checks, frontend-tests, tool-tests, service-test, chart-lint]
     runs-on: ubuntu-latest
     env:
       NEEDS_JSON: ${{ toJSON(needs) }}
@@ -199,14 +308,16 @@ jobs:
       - name: Verify gate covers every suite
         run: |
           diff -u \
-            <(awk '/^jobs:$/ { jobs=1; next } jobs && /^  [[:alnum:]_-]+:$/ { id=$1; sub(/:$/, "", id); if (id != "test-gate") print id }' .github/workflows/tests.yml | sort) \
+            <(awk '/^jobs:$/ { jobs=1; next } jobs && /^  [[:alnum:]_-]+:$/ { id=$1; sub(/:$/, "", id); if (id != "test-gate" && id != "latchkey-probe") print id }' .github/workflows/tests.yml | sort) \
             <(printf '%s\n' "$NEEDS_JSON" | jq -r 'keys[]' | sort)
       - name: Verify every check passed
         run: |
           # Suite jobs run unconditionally; a skipped result is a failed gate.
+          # Only the Latchkey routing jobs may be skipped (off on PR runs).
           # Revisit this predicate if a later PR adds path-based job skipping.
           echo "$NEEDS_JSON" | jq -e '
-            [.[].result] | all(. == "success")
+            to_entries | all(.value.result == "success"
+              or ((.key | startswith("latchkey-")) and .value.result == "skipped"))
           '
 
   service-test:
@@ -215,15 +326,10 @@ jobs:
       fail-fast: false
       matrix:
         shard: [1, 2]
-    # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey
-    # runners, which don't count against the org's 20-concurrent-job cap).
-    # Unset means GitHub-hosted, so rolling back is deleting one variable. The
-    # job names, and so the required check, don't change either way. A fork
-    # PR's own pull_request run stays on GitHub. A merge_group run uses
-    # HEAVY_RUNNER even when the group holds a reviewed fork change: queue
-    # admission (a maintainer queues a gated PR) is the trust decision, not
-    # this expression. See docs/runbooks/ci-heavy-runner.md.
-    runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }}
+    # Routed by latchkey-route, as backend-tests.
+    needs: latchkey-route
+    if: ${{ !cancelled() }}
+    runs-on: ${{ needs.latchkey-route.outputs.service || 'ubuntu-latest' }}
 
     services:
       mongo:
diff --git a/docs/runbooks/ci-heavy-runner.md b/docs/runbooks/ci-heavy-runner.md
index 2e6eb79d88..8d15e5b34f 100644
--- a/docs/runbooks/ci-heavy-runner.md
+++ b/docs/runbooks/ci-heavy-runner.md
@@ -1,48 +1,113 @@
-# CI heavy jobs on Latchkey runners (HEAVY_RUNNER)
+# CI heavy jobs on Latchkey runners
 
 The heavy jobs in `.github/workflows/tests.yml` (the three backend shards,
-frontend tests and the Tier 1 service tests) run on whatever runner label the
-repository variable `HEAVY_RUNNER` names, and on `ubuntu-latest` when it is
-unset. The plan (D54, Sam 2026-10-24) is to set it to `latchkey-medium` once
-the PR adding it merges: Latchkey runners don't count against the org's
-20-concurrent-job cap on GitHub-hosted runners, which was what made the merge
-queue wait.
+frontend tests and the two Tier 1 service shards) can run on Latchkey runners,
+which don't count against the org's 20-concurrent-job cap on GitHub-hosted
+runners. That cap is what made the merge queue wait (D37).
 
-Everything else, including tool-tests, the guards, backend checks and chart
-lint, stays on GitHub. Job names don't change either way, so the required check
-("Test & Coverage") and merge-queue runs are unaffected.
+They do so only on **merge-queue runs**, and only while the repository
+variable `LATCHKEY_HEAVY` is `on`. Every other run, PRs included, stays on
+GitHub-hosted runners, which are free on this public repo. Backend runs on
+`latchkey-medium` and frontend and service on `latchkey-small` (sizes below).
+
+## How a run is routed
+
+A held Latchkey job (free minutes spent, the spend limit reached, or an
+outage) does not fail. It sits "queued" on GitHub with no error, and a
+required check that waits on it leaves the merge queue stalled with nothing
+red anywhere. So nothing is sent to Latchkey blind. The logic is
+`scripts/latchkey-route.mjs`, with tests in
+`scripts/__tests__/latchkey-route.test.mjs`.
+
+1. **Latchkey probe** asks for a `latchkey-small` runner and only echoes. It
+   is not in the gate's needs, so a held probe never holds the required check.
+2. **Latchkey route** (GitHub-hosted) picks GitHub-hosted for the heavy jobs
+   when:
+   - a `latchkey-hold` artifact was recorded in the last 60 minutes;
+   - the budget is spent, or `LATCHKEY_BUDGET_START` is unset; or
+   - the probe isn't picked up within 180 s. This case also records a hold.
+
+   Otherwise it sends the heavy jobs to Latchkey and records a `latchkey-run`
+   artifact, which is the budget's counter. Any API error falls back to
+   GitHub-hosted.
+3. **Latchkey backstop** (GitHub-hosted) watches the heavy jobs once they are
+   routed. If one is still queued after 300 s, it records a hold and cancels
+   the run. The queue entry then fails loudly instead of waiting, and later
+   runs stay on GitHub until the hold expires. The press re-queues the PR as
+   for any dequeue.
+
+The budget is an estimate: Latchkey runs since `LATCHKEY_BUDGET_START` times
+`LATCHKEY_EQ_PER_RUN` (default 32 small-equivalent minutes), stopped at 80%
+of `LATCHKEY_BUDGET_EQMIN` (default 10000). The 20% left is margin for the
+estimate and for runs already in flight. Check it against the measured
+figure below, and raise `LATCHKEY_EQ_PER_RUN` if the measured figure is higher.
+
+## Variables
+
+| variable | meaning | default |
+|---|---|---|
+| `LATCHKEY_HEAVY` | `on` routes merge-queue runs through Latchkey; anything else is off | off |
+| `LATCHKEY_BUDGET_START` | ISO start of the free-minute period; unset keeps everything on GitHub | none |
+| `LATCHKEY_BUDGET_EQMIN` | free small-equivalent minutes in the period | 10000 |
+| `LATCHKEY_EQ_PER_RUN` | estimated small-equivalent minutes per routed run | 32 |
+| `LATCHKEY_PICKUP_SEC` | how long the route waits for the probe | 180 |
+| `LATCHKEY_BACKEND_LABEL` / `LATCHKEY_LIGHT_LABEL` | runner sizes | `latchkey-medium` / `latchkey-small` |
+
+## Turn it on, and off
+
+```bash
+gh variable set LATCHKEY_BUDGET_START --body 2026-10-26T00:00:00Z --repo Team-Commonly/commonly
+gh variable set LATCHKEY_HEAVY --body on --repo Team-Commonly/commonly
+# off: runs that start afterwards are entirely GitHub-hosted
+gh variable delete LATCHKEY_HEAVY --repo Team-Commonly/commonly
+```
+
+Runs already routed keep their labels. If one is held, the backstop cancels it.
+
+## Rehearse the fallback
+
+Dispatch Tests with `latchkey: true`. This works whether or not
+`LATCHKEY_HEAVY` is set, because dispatch is maintainer-only.
+
+- **A held job:** set `latchkey_probe_label` to a label no runner serves. The
+  route waits out the pickup time, records a hold and runs the heavy jobs on
+  GitHub. Cancel the run afterwards, since the probe stays queued. Then delete
+  the hold artifact, or wait an hour for it to expire.
+- **Spent minutes:** set `latchkey_budget_eqmin` to `0`. The route falls back
+  without waiting.
+- **The real path:** no overrides. This spends about 30 small-equivalent
+  minutes.
+
+## Measure the burn
+
+Latchkey bills per job, rounded up to the minute, at 1× for small and 2× for
+medium. Sum `ceil(job minutes) × size` over the Latchkey-labelled jobs from
+the Actions jobs API, per day. The figure matched Latchkey's own meter within
+a few percent on the 10-24 trial. Compare it to the budget estimate above.
 
 ## Trust boundary
 
-- A fork PR's own `pull_request` run always uses `ubuntu-latest`. Unreviewed
-  fork code does not get a Latchkey runner from opening a PR.
-- A `merge_group` run uses `HEAVY_RUNNER`, including a group that holds a
-  reviewed fork change. The expression does not inspect where queued PRs came
+- Every `pull_request` run uses `ubuntu-latest`, fork or not. The routing
+  jobs are skipped on PR runs.
+- A `merge_group` run can use Latchkey, including a group that holds a
+  reviewed fork change. The route does not inspect where queued PRs came
   from.
 - So queue admission is the trust decision: a maintainer puts a PR in the
   queue only after its gate passed. A green test run, or this expression, does
   not establish that trust. GitHub doesn't enforce the seat review either; the
   ruleset requires only a green merge queue, so the review gate is operating
   process.
-- That is acceptable here because the Tests workflow grants only
-  `contents: read`, has no `pull_request_target` trigger, and exposes test
-  fixtures, not production secrets. Latchkey documents single-job VMs
+- That is acceptable here because the test jobs get only `contents: read`,
+  the workflow has no `pull_request_target` trigger, and the jobs see test
+  fixtures, not production secrets. Two routing jobs get more, and both run
+  on GitHub-hosted runners: the route gets `actions: read`, to read the run's
+  jobs and the hold and budget artifacts, and the backstop gets
+  `actions: write`, to cancel its own run. A queued change can rewrite those
+  jobs like any other part of the workflow, which is why queue admission, not
+  this file, is the trust decision. Latchkey documents single-job VMs
   destroyed after each job (vendor-published, not independently attested).
 
-## Flip back to GitHub-hosted
-
-Do this if Latchkey is down, jobs sit queued with no runner, or the free
-minutes run out:
-
-```bash
-gh variable delete HEAVY_RUNNER --repo Team-Commonly/commonly
-```
-
-Runs that start after the delete go to `ubuntu-latest`. Runs already queued
-keep the label they were given, so cancel and re-run those. To flip back to
-Latchkey: `gh variable set HEAVY_RUNNER --body latchkey-medium --repo Team-Commonly/commonly`.
-
-## Why medium, and the Mongo ulimit
+## Why these sizes, and the Mongo ulimit
 
 Measured 2026-10-24 on dispatch-only trial runs:
 
@@ -60,6 +125,10 @@ Measured 2026-10-24 on dispatch-only trial runs:
 tool-tests stays on GitHub: its CLI PATH test fails on Latchkey, and its Jest
 process doesn't exit.
 
-Medium jobs count double against the free budget (14k minutes a month, plus a
-10k bonus to 2026-11-23). Check usage on the Latchkey dashboard before the
-bonus ends.
+Frontend took 2.8 minutes on small against 1.9 on medium, and the service
+tests 5.5 minutes on small, unsharded. Medium counts double against the free
+minutes and is rounded up per job, so a bigger size pays off only when it at
+least halves a job's time: it does for the backend shards (about 18
+small-equivalent minutes a run on medium against 36 on small) and does not for
+frontend or service. Each figure is one trial run, so re-measure before
+changing sizes.
diff --git a/scripts/__tests__/latchkey-route.test.mjs b/scripts/__tests__/latchkey-route.test.mjs
new file mode 100644
index 0000000000..ab2d9eceb0
--- /dev/null
+++ b/scripts/__tests__/latchkey-route.test.mjs
@@ -0,0 +1,138 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  PROBE_JOB, backstop, budgetDecision, holdActive, probeState, route, runsSince, stuckJobs,
+} from '../latchkey-route.mjs';
+
+const NOW = Date.parse('2026-10-26T12:00:00Z');
+const ago = (sec) => new Date(NOW - sec * 1000).toISOString();
+const job = (name, status, labels, createdSecAgo = 0) => ({ name, status, labels, created_at: ago(createdSecAgo) });
+
+test('a hold is fresh only inside its window and never when expired', () => {
+  assert.equal(holdActive([{ created_at: ago(30 * 60), expired: false }], NOW, 60), true);
+  assert.equal(holdActive([{ created_at: ago(61 * 60), expired: false }], NOW, 60), false);
+  assert.equal(holdActive([{ created_at: ago(60), expired: true }], NOW, 60), false);
+  assert.equal(holdActive([], NOW, 60), false);
+});
+
+test('the budget counts runs since the start and stops at 80% of the free minutes', () => {
+  const start = '2026-10-26T00:00:00Z';
+  const artifacts = [{ created_at: '2026-10-25T23:59:00Z' }, { created_at: '2026-10-26T00:00:00Z' }, { created_at: ago(10) }];
+  assert.equal(runsSince(artifacts, start), 2);
+  assert.deepEqual(budgetDecision({ runs: 249, eqPerRun: 32, budgetEqMin: 10000 }), { used: 7968, limit: 8000, ok: true });
+  assert.equal(budgetDecision({ runs: 250, eqPerRun: 32, budgetEqMin: 10000 }).ok, false);
+  // A zero budget is the rehearsal for spent minutes.
+  assert.equal(budgetDecision({ runs: 0, eqPerRun: 32, budgetEqMin: 0 }).ok, false);
+});
+
+test('probe state reads the probe job only', () => {
+  assert.equal(probeState([]), 'missing');
+  assert.equal(probeState([job(PROBE_JOB, 'queued', ['latchkey-small'])]), 'queued');
+  assert.equal(probeState([job(PROBE_JOB, 'in_progress', ['latchkey-small'])]), 'started');
+  assert.equal(probeState([job(PROBE_JOB, 'completed', ['latchkey-small'])]), 'started');
+});
+
+test('only heavy Latchkey jobs queued past the limit count as stuck', () => {
+  const jobs = [
+    job('Backend tests (1/3)', 'queued', ['latchkey-medium'], 400),
+    job('Backend tests (2/3)', 'queued', ['latchkey-medium'], 100),
+    job('Frontend tests', 'in_progress', ['latchkey-small'], 400),
+    job(PROBE_JOB, 'queued', ['latchkey-small'], 900),
+    job('Chart Lint', 'queued', ['ubuntu-latest'], 900),
+  ];
+  assert.deepEqual(stuckJobs(jobs, NOW, 300).map((j) => j.name), ['Backend tests (1/3)']);
+});
+
+// The loops run against a fake API and a fake clock that each wait advances.
+const harness = async (envOverrides, jobsByPoll, artifactsByName = {}) => {
+  const dir = await mkdtemp(join(tmpdir(), 'latchkey-route-'));
+  const saved = { ...process.env };
+  Object.assign(process.env, {
+    GITHUB_REPOSITORY: 'o/r', GITHUB_RUN_ID: '1', GITHUB_RUN_ATTEMPT: '1', GH_TOKEN: 'x',
+    GITHUB_OUTPUT: join(dir, 'out'), GITHUB_STEP_SUMMARY: join(dir, 'summary'),
+    BUDGET_START: '2026-10-26T00:00:00Z', ...envOverrides,
+  });
+  let clock = NOW;
+  let poll = 0;
+  const realFetch = globalThis.fetch;
+  globalThis.fetch = async (url) => {
+    const path = String(url);
+    if (path.includes('/jobs?')) {
+      const jobs = jobsByPoll[Math.min(poll, jobsByPoll.length - 1)];
+      poll += 1;
+      if (jobs instanceof Error) return { ok: false, status: 502, json: async () => ({}) };
+      return { ok: true, json: async () => ({ jobs }) };
+    }
+    const name = new URL(path).searchParams.get('name');
+    return { ok: true, json: async () => ({ artifacts: artifactsByName[name] || [] }) };
+  };
+  return {
+    run: (fn) => fn({ now: () => clock, wait: async (ms) => { clock += ms; } }),
+    out: async () => Object.fromEntries((await readFile(join(dir, 'out'), 'utf8')).trim().split('\n').map((l) => l.split('='))),
+    restore: () => { globalThis.fetch = realFetch; process.env = saved; return rm(dir, { recursive: true, force: true }); },
+  };
+};
+
+test('route sends heavy jobs to Latchkey once the probe is picked up', async () => {
+  const h = await harness({}, [[job(PROBE_JOB, 'queued', ['latchkey-small'])], [job(PROBE_JOB, 'in_progress', ['latchkey-small'])]]);
+  await h.run(route);
+  assert.deepEqual(await h.out(), { use: 'true', backend: 'latchkey-medium', frontend: 'latchkey-small', service: 'latchkey-small', hold: 'false' });
+  await h.restore();
+});
+
+test('route falls back and records a hold when the probe is never picked up', async () => {
+  const h = await harness({ PICKUP_SEC: '60' }, [[job(PROBE_JOB, 'queued', ['latchkey-norunner'])]]);
+  await h.run(route);
+  assert.deepEqual(await h.out(), { use: 'false', backend: 'ubuntu-latest', frontend: 'ubuntu-latest', service: 'ubuntu-latest', hold: 'true' });
+  await h.restore();
+});
+
+test('route falls back without probing when a hold is fresh or the budget is spent', async () => {
+  let h = await harness({}, [[]], { 'latchkey-hold': [{ created_at: ago(120), expired: false }] });
+  await h.run(route);
+  assert.equal((await h.out()).use, 'false');
+  assert.equal((await h.out()).hold, 'false');
+  await h.restore();
+  h = await harness({ BUDGET_EQMIN: '0' }, [[job(PROBE_JOB, 'in_progress', ['latchkey-small'])]]);
+  await h.run(route);
+  assert.equal((await h.out()).use, 'false');
+  await h.restore();
+});
+
+test('route falls back on an API error and when the budget start is unset', async () => {
+  let h = await harness({}, [new Error('502')]);
+  await h.run(route);
+  assert.equal((await h.out()).use, 'false');
+  await h.restore();
+  h = await harness({ BUDGET_START: '' }, [[job(PROBE_JOB, 'in_progress', ['latchkey-small'])]]);
+  await h.run(route);
+  assert.equal((await h.out()).use, 'false');
+  await h.restore();
+});
+
+test('backstop trips on a held heavy job and passes once every heavy job starts', async () => {
+  let h = await harness({ STUCK_SEC: '300' }, [[job('Backend tests (1/3)', 'queued', ['latchkey-medium'], 301)]]);
+  await h.run(backstop);
+  assert.equal((await h.out()).tripped, 'true');
+  await h.restore();
+  const running = [job('Backend tests (1/3)', 'in_progress', ['latchkey-medium']), job('Frontend tests', 'completed', ['latchkey-small'])];
+  h = await harness({}, [[job('Backend tests (1/3)', 'queued', ['latchkey-medium'])], running, running]);
+  await h.run(backstop);
+  assert.equal((await h.out()).tripped, 'false');
+  await h.restore();
+});
+
+test('backstop retries failed reads, and cancels only if the run stays unreadable', async () => {
+  const running = [job('Backend tests (1/3)', 'in_progress', ['latchkey-medium'])];
+  let h = await harness({}, [new Error('502'), running, running]);
+  await h.run(backstop);
+  assert.equal((await h.out()).tripped, 'false');
+  await h.restore();
+  h = await harness({ STUCK_SEC: '30' }, [new Error('502')]);
+  await h.run(backstop);
+  assert.equal((await h.out()).tripped, 'true');
+  await h.restore();
+});
diff --git a/scripts/latchkey-route.mjs b/scripts/latchkey-route.mjs
new file mode 100644
index 0000000000..10a54d2f11
--- /dev/null
+++ b/scripts/latchkey-route.mjs
@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+// Routes the Tests workflow's heavy jobs to Latchkey runners, with a fallback
+// to GitHub-hosted runners that never leaves the merge queue waiting on a
+// held job. See docs/runbooks/ci-heavy-runner.md.
+//
+//   route     run by the "Latchkey route" job. Picks GitHub-hosted when a
+//             hold is fresh or the budget is spent, otherwise waits for the
+//             "Latchkey probe" job to be picked up. A probe still queued after
+//             PICKUP_SEC means Latchkey is holding jobs: the heavy jobs go to
+//             GitHub-hosted and the job records a hold for later runs.
+//   backstop  run by the "Latchkey backstop" job once the heavy jobs are
+//             routed. A heavy job still queued after STUCK_SEC trips it: the
+//             job records a hold and cancels the run, so a held job fails the
+//             required check instead of leaving it pending.
+//
+// Any API error routes to GitHub-hosted. Falling back is never a failure.
+import { appendFileSync } from 'node:fs';
+import { pathToFileURL } from 'node:url';
+
+export const PROBE_JOB = 'Latchkey probe';
+export const HOLD_ARTIFACT = 'latchkey-hold';
+export const RUN_ARTIFACT = 'latchkey-run';
+export const BUDGET_FRACTION = 0.8;
+const GITHUB_LABEL = 'ubuntu-latest';
+
+const isLatchkey = (job) => (job.labels || []).some((label) => label.startsWith('latchkey-'));
+const started = (job) => job.status === 'in_progress' || job.status === 'completed';
+const ageSec = (iso, now) => (now - Date.parse(iso)) / 1000;
+
+// A hold is fresh for holdMin minutes after it was recorded.
+export const holdActive = (artifacts, now, holdMin) => artifacts.some((a) => !a.expired
+  && ageSec(a.created_at, now) < holdMin * 60);
+
+// Latchkey runs recorded since the budget period started.
+export const runsSince = (artifacts, startIso) => artifacts
+  .filter((a) => Date.parse(a.created_at) >= Date.parse(startIso)).length;
+
+// The guard stops at BUDGET_FRACTION of the free minutes, leaving the rest as
+// margin for the estimate and for runs already in flight.
+export const budgetDecision = ({ runs, eqPerRun, budgetEqMin }) => {
+  const used = runs * eqPerRun;
+  const limit = budgetEqMin * BUDGET_FRACTION;
+  return { used, limit, ok: used < limit };
+};
+
+// Heavy Latchkey jobs (every Latchkey-labelled job except the probe) still
+// queued after stuckSec.
+export const stuckJobs = (jobs, now, stuckSec) => jobs
+  .filter((job) => isLatchkey(job) && job.name !== PROBE_JOB && !started(job)
+    && ageSec(job.created_at, now) > stuckSec);
+
+export const heavyJobs = (jobs) => jobs.filter((job) => isLatchkey(job) && job.name !== PROBE_JOB);
+
+export const probeState = (jobs) => {
+  const probe = jobs.find((job) => job.name === PROBE_JOB);
+  if (!probe) return 'missing';
+  return started(probe) ? 'started' : 'queued';
+};
+
+const env = (name, fallback) => {
+  const value = (process.env[name] || '').trim();
+  return value || fallback;
+};
+
+const api = async (path) => {
+  const response = await fetch(`https://api.github.com/repos/${process.env.GITHUB_REPOSITORY}/${path}`, {
+    headers: {
+      Authorization: `Bearer ${process.env.GH_TOKEN}`,
+      Accept: 'application/vnd.github+json',
+      'User-Agent': 'commonly-latchkey-route',
+    },
+  });
+  if (!response.ok) throw new Error(`GET ${path}: HTTP ${response.status}`);
+  return response.json();
+};
+
+const listArtifacts = async (name, startIso) => {
+  const out = [];
+  for (let page = 1; page <= 20; page += 1) {
+    // eslint-disable-next-line no-await-in-loop
+    const body = await api(`actions/artifacts?name=${name}&per_page=100&page=${page}`);
+    const rows = body.artifacts || [];
+    out.push(...rows);
+    // Newest first, so a page with nothing inside the window ends the scan.
+    if (rows.length < 100 || (startIso && !rows.some((a) => Date.parse(a.created_at) >= Date.parse(startIso)))) break;
+  }
+  return out;
+};
+
+const runJobs = async () => {
+  const body = await api(`actions/runs/${process.env.GITHUB_RUN_ID}/attempts/${process.env.GITHUB_RUN_ATTEMPT || 1}/jobs?per_page=100`);
+  return body.jobs || [];
+};
+
+const sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); });
+
+const output = (values) => {
+  const lines = Object.entries(values).map(([key, value]) => `${key}=${value}`);
+  if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`);
+  console.log(lines.join('\n'));
+};
+
+const summary = (text) => {
+  if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${text}\n`);
+  console.log(text);
+};
+
+const labels = (use) => ({
+  use,
+  backend: use ? env('BACKEND_LABEL', 'latchkey-medium') : GITHUB_LABEL,
+  frontend: use ? env('LIGHT_LABEL', 'latchkey-small') : GITHUB_LABEL,
+  service: use ? env('LIGHT_LABEL', 'latchkey-small') : GITHUB_LABEL,
+});
+
+export const route = async ({ now = () => Date.now(), wait = sleep } = {}) => {
+  const fallback = (reason, hold = false) => {
+    summary(`Heavy jobs on GitHub-hosted runners: ${reason}.`);
+    output({ ...labels(false), hold });
+  };
+  try {
+    const holds = await listArtifacts(HOLD_ARTIFACT);
+    const holdMin = Number(env('HOLD_MIN', '60'));
+    if (holdActive(holds, now(), holdMin)) return fallback(`a Latchkey hold was recorded in the last ${holdMin} min`);
+
+    const start = env('BUDGET_START', '');
+    if (!start || Number.isNaN(Date.parse(start))) return fallback('LATCHKEY_BUDGET_START is not set');
+    const runs = runsSince(await listArtifacts(RUN_ARTIFACT, start), start);
+    const budget = budgetDecision({
+      runs,
+      eqPerRun: Number(env('EQ_PER_RUN', '32')),
+      budgetEqMin: Number(env('BUDGET_EQMIN', '10000')),
+    });
+    summary(`Latchkey budget since ${start}: ${runs} runs, about ${budget.used} of ${budget.limit} small-equivalent minutes allowed.`);
+    if (!budget.ok) return fallback('the Latchkey budget is spent');
+
+    const pickupSec = Number(env('PICKUP_SEC', '180'));
+    const deadline = now() + pickupSec * 1000;
+    for (;;) {
+      // eslint-disable-next-line no-await-in-loop
+      const state = probeState(await runJobs());
+      if (state === 'started') {
+        summary('Heavy jobs on Latchkey runners: the probe was picked up.');
+        return output({ ...labels(true), hold: false });
+      }
+      if (now() >= deadline) {
+        return fallback(`the Latchkey probe was not picked up within ${pickupSec} s (probe ${state})`, true);
+      }
+      // eslint-disable-next-line no-await-in-loop
+      await wait(10000);
+    }
+  } catch (error) {
+    return fallback(`routing failed (${error.message})`);
+  }
+};
+
+export const backstop = async ({ now = () => Date.now(), wait = sleep } = {}) => {
+  const stuckSec = Number(env('STUCK_SEC', '300'));
+  // Long enough for the heavy jobs to be queued and for any of them to pass
+  // stuckSec; past it there is nothing left to watch.
+  const deadline = now() + (stuckSec + 300) * 1000;
+  let stable = 0;
+  let lastError = null;
+  for (;;) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      const jobs = await runJobs();
+      lastError = null;
+      const stuck = stuckJobs(jobs, now(), stuckSec);
+      if (stuck.length) {
+        summary(`Latchkey is holding ${stuck.map((job) => job.name).join(', ')}: recording a hold and cancelling the run.`);
+        return output({ tripped: true });
+      }
+      const heavy = heavyJobs(jobs);
+      // Two polls in a row with every heavy job started, so a matrix still
+      // being expanded is not mistaken for a finished pickup.
+      stable = heavy.length && heavy.every(started) ? stable + 1 : 0;
+      if (stable >= 2) {
+        summary(`All ${heavy.length} heavy Latchkey jobs were picked up.`);
+        return output({ tripped: false });
+      }
+    } catch (error) {
+      // A failed read is retried until the deadline.
+      lastError = error;
+      stable = 0;
+    }
+    if (now() >= deadline) {
+      // Unreadable to the end: cancel rather than risk a held job leaving the
+      // required check pending.
+      if (lastError) {
+        summary(`Backstop could not read the run (${lastError.message}): recording a hold and cancelling the run.`);
+        return output({ tripped: true });
+      }
+      summary('No heavy Latchkey job left to watch.');
+      return output({ tripped: false });
+    }
+    // eslint-disable-next-line no-await-in-loop
+    await wait(15000);
+  }
+};
+
+if (import.meta.url === pathToFileURL(process.argv[1] || '').href) {
+  const mode = process.argv[2];
+  if (mode === 'route') await route();
+  else if (mode === 'backstop') await backstop();
+  else {
+    console.error('usage: latchkey-route.mjs route|backstop');
+    process.exit(2);
+  }
+}

From 400422ee3f7cae9b0969dcbd522e915102a4ca82 Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:58:59 -0700
Subject: [PATCH 2/3] ci: raise the Latchkey per-run estimate to the measured
 48 eq-min

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VD3azQJZrdMmw8g51iscWD
---
 docs/runbooks/ci-heavy-runner.md | 4 ++--
 scripts/latchkey-route.mjs       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/runbooks/ci-heavy-runner.md b/docs/runbooks/ci-heavy-runner.md
index 8d15e5b34f..b196bcaa91 100644
--- a/docs/runbooks/ci-heavy-runner.md
+++ b/docs/runbooks/ci-heavy-runner.md
@@ -37,7 +37,7 @@ red anywhere. So nothing is sent to Latchkey blind. The logic is
    for any dequeue.
 
 The budget is an estimate: Latchkey runs since `LATCHKEY_BUDGET_START` times
-`LATCHKEY_EQ_PER_RUN` (default 32 small-equivalent minutes), stopped at 80%
+`LATCHKEY_EQ_PER_RUN` (default 48 small-equivalent minutes, measured on the 10-25 rehearsal), stopped at 80%
 of `LATCHKEY_BUDGET_EQMIN` (default 10000). The 20% left is margin for the
 estimate and for runs already in flight. Check it against the measured
 figure below, and raise `LATCHKEY_EQ_PER_RUN` if the measured figure is higher.
@@ -49,7 +49,7 @@ figure below, and raise `LATCHKEY_EQ_PER_RUN` if the measured figure is higher.
 | `LATCHKEY_HEAVY` | `on` routes merge-queue runs through Latchkey; anything else is off | off |
 | `LATCHKEY_BUDGET_START` | ISO start of the free-minute period; unset keeps everything on GitHub | none |
 | `LATCHKEY_BUDGET_EQMIN` | free small-equivalent minutes in the period | 10000 |
-| `LATCHKEY_EQ_PER_RUN` | estimated small-equivalent minutes per routed run | 32 |
+| `LATCHKEY_EQ_PER_RUN` | estimated small-equivalent minutes per routed run | 48 |
 | `LATCHKEY_PICKUP_SEC` | how long the route waits for the probe | 180 |
 | `LATCHKEY_BACKEND_LABEL` / `LATCHKEY_LIGHT_LABEL` | runner sizes | `latchkey-medium` / `latchkey-small` |
 
diff --git a/scripts/latchkey-route.mjs b/scripts/latchkey-route.mjs
index 10a54d2f11..47ecc8704d 100644
--- a/scripts/latchkey-route.mjs
+++ b/scripts/latchkey-route.mjs
@@ -127,7 +127,7 @@ export const route = async ({ now = () => Date.now(), wait = sleep } = {}) => {
     const runs = runsSince(await listArtifacts(RUN_ARTIFACT, start), start);
     const budget = budgetDecision({
       runs,
-      eqPerRun: Number(env('EQ_PER_RUN', '32')),
+      eqPerRun: Number(env('EQ_PER_RUN', '48')),
       budgetEqMin: Number(env('BUDGET_EQMIN', '10000')),
     });
     summary(`Latchkey budget since ${start}: ${runs} runs, about ${budget.used} of ${budget.limit} small-equivalent minutes allowed.`);

From d3dec55e9fa2541d2ea117ef40485e86a04a5f78 Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:18:42 -0700
Subject: [PATCH 3/3] ci(latchkey): backstop waits for every expected heavy job
 before passing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VD3azQJZrdMmw8g51iscWD
---
 .github/workflows/tests.yml               |  3 +++
 scripts/__tests__/latchkey-route.test.mjs | 15 +++++++++++++--
 scripts/latchkey-route.mjs                |  9 ++++++---
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 84be4e4b2b..3d77ca1cf9 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -150,6 +150,9 @@ jobs:
         run: node scripts/latchkey-route.mjs backstop
         env:
           GH_TOKEN: ${{ github.token }}
+          # Backend shards + frontend + service shards. Keep in step with the
+          # matrices below, or the backstop passes before every heavy job is listed.
+          EXPECTED_HEAVY: '6'
       - if: ${{ steps.watch.outputs.tripped == 'true' }}
         run: echo "heavy job held in run ${{ github.run_id }}" > latchkey-hold.txt
       - if: ${{ steps.watch.outputs.tripped == 'true' }}
diff --git a/scripts/__tests__/latchkey-route.test.mjs b/scripts/__tests__/latchkey-route.test.mjs
index ab2d9eceb0..11e35bf955 100644
--- a/scripts/__tests__/latchkey-route.test.mjs
+++ b/scripts/__tests__/latchkey-route.test.mjs
@@ -119,7 +119,7 @@ test('backstop trips on a held heavy job and passes once every heavy job starts'
   assert.equal((await h.out()).tripped, 'true');
   await h.restore();
   const running = [job('Backend tests (1/3)', 'in_progress', ['latchkey-medium']), job('Frontend tests', 'completed', ['latchkey-small'])];
-  h = await harness({}, [[job('Backend tests (1/3)', 'queued', ['latchkey-medium'])], running, running]);
+  h = await harness({ EXPECTED_HEAVY: '2' }, [[job('Backend tests (1/3)', 'queued', ['latchkey-medium'])], running, running]);
   await h.run(backstop);
   assert.equal((await h.out()).tripped, 'false');
   await h.restore();
@@ -127,7 +127,7 @@ test('backstop trips on a held heavy job and passes once every heavy job starts'
 
 test('backstop retries failed reads, and cancels only if the run stays unreadable', async () => {
   const running = [job('Backend tests (1/3)', 'in_progress', ['latchkey-medium'])];
-  let h = await harness({}, [new Error('502'), running, running]);
+  let h = await harness({ EXPECTED_HEAVY: '1' }, [new Error('502'), running, running]);
   await h.run(backstop);
   assert.equal((await h.out()).tripped, 'false');
   await h.restore();
@@ -136,3 +136,14 @@ test('backstop retries failed reads, and cancels only if the run stays unreadabl
   assert.equal((await h.out()).tripped, 'true');
   await h.restore();
 });
+
+test('backstop keeps watching until every expected heavy job is listed', async () => {
+  // Two polls see only the first shard started; the second shard is listed
+  // later and is still queued past the limit, so the backstop must trip.
+  const first = [job('Backend tests (1/3)', 'in_progress', ['latchkey-medium'])];
+  const later = [...first, job('Backend tests (2/3)', 'queued', ['latchkey-medium'], 301)];
+  const h = await harness({ EXPECTED_HEAVY: '2', STUCK_SEC: '300' }, [first, first, first, later]);
+  await h.run(backstop);
+  assert.equal((await h.out()).tripped, 'true');
+  await h.restore();
+});
diff --git a/scripts/latchkey-route.mjs b/scripts/latchkey-route.mjs
index 47ecc8704d..9c09638835 100644
--- a/scripts/latchkey-route.mjs
+++ b/scripts/latchkey-route.mjs
@@ -155,6 +155,9 @@ export const route = async ({ now = () => Date.now(), wait = sleep } = {}) => {
 
 export const backstop = async ({ now = () => Date.now(), wait = sleep } = {}) => {
   const stuckSec = Number(env('STUCK_SEC', '300'));
+  // Matrix jobs are listed as they expand, so a poll can see only some of
+  // them. The backstop passes only once this many heavy jobs have started.
+  const expected = Number(env('EXPECTED_HEAVY', '6'));
   // Long enough for the heavy jobs to be queued and for any of them to pass
   // stuckSec; past it there is nothing left to watch.
   const deadline = now() + (stuckSec + 300) * 1000;
@@ -171,9 +174,9 @@ export const backstop = async ({ now = () => Date.now(), wait = sleep } = {}) =>
         return output({ tripped: true });
       }
       const heavy = heavyJobs(jobs);
-      // Two polls in a row with every heavy job started, so a matrix still
-      // being expanded is not mistaken for a finished pickup.
-      stable = heavy.length && heavy.every(started) ? stable + 1 : 0;
+      // Two polls in a row with every expected heavy job listed and started,
+      // so a matrix still being expanded is not mistaken for a finished pickup.
+      stable = heavy.length >= expected && heavy.every(started) ? stable + 1 : 0;
       if (stable >= 2) {
         summary(`All ${heavy.length} heavy Latchkey jobs were picked up.`);
         return output({ tripped: false });