From cbe18a44019156110549413251ad7f5bb06314c8 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:14:42 -0700 Subject: [PATCH 1/4] ci(tests): heavy jobs run on vars.HEAVY_RUNNER, default ubuntu-latest (D37) backend-tests (3 shards), frontend-tests and service-test take runs-on from the repo variable HEAVY_RUNNER and fall back to ubuntu-latest when it is unset, so this commit changes nothing until the variable is set and rolling back is deleting it. Job ids and names are unchanged, so test-gate's coverage diff and the required check "Test & Coverage" are unaffected, and merge_group runs read the same variable. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V7T2fgquLRaqaYiGLC6Dey --- .github/workflows/tests.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c337865e40..136ecb9905 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,11 @@ jobs: fail-fast: false matrix: shard: [1, 2, 3] - runs-on: ubuntu-latest + # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey + # runners, which don't count against the org's 20-concurrent-job cap). + # Unset means GitHub-hosted, so rolling back is deleting one variable. The + # job names, and so the required check, don't change either way. + runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} env: JWT_SECRET: test-jwt-secret SENDGRID_API_KEY: fake-api-key @@ -99,7 +103,11 @@ jobs: frontend-tests: name: Frontend tests - runs-on: ubuntu-latest + # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey + # runners, which don't count against the org's 20-concurrent-job cap). + # Unset means GitHub-hosted, so rolling back is deleting one variable. The + # job names, and so the required check, don't change either way. + runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -192,7 +200,11 @@ jobs: service-test: name: Service Tests (Tier 1 — real DBs) - runs-on: ubuntu-latest + # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey + # runners, which don't count against the org's 20-concurrent-job cap). + # Unset means GitHub-hosted, so rolling back is deleting one variable. The + # job names, and so the required check, don't change either way. + runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} services: mongo: From 5d2f6abb4a28265c985fd7ca15339b7c6055fd74 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:56:02 -0700 Subject: [PATCH 2/4] ci(tests): raise mongo service nofile ulimit (needed on Latchkey) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V7T2fgquLRaqaYiGLC6Dey --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 136ecb9905..cbcba94045 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -210,7 +210,11 @@ jobs: mongo: image: mongo:7 ports: ["27017:27017"] + # Latchkey's docker default nofile is low enough that mongod hits + # EMFILE during index builds and WT_PANICs (trial 2, 10-24); a no-op + # on GitHub-hosted runners. options: >- + --ulimit nofile=64000:64000 --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'" --health-interval 10s --health-timeout 5s From 70027551be2cad5aa31cd0b247c5dd46ffb4f29a Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:35:59 -0700 Subject: [PATCH 3/4] ci(tests): fork PRs stay on GitHub; runbook for HEAVY_RUNNER and the flip-back Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V7T2fgquLRaqaYiGLC6Dey --- .github/workflows/tests.yml | 15 ++++++---- docs/runbooks/ci-heavy-runner.md | 47 ++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 docs/runbooks/ci-heavy-runner.md diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cbcba94045..dbc21228a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,8 +57,9 @@ jobs: # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey # runners, which don't count against the org's 20-concurrent-job cap). # Unset means GitHub-hosted, so rolling back is deleting one variable. The - # job names, and so the required check, don't change either way. - runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} + # job names, and so the required check, don't change either way. A fork PR + # always stays on GitHub: untrusted code doesn't get a third-party runner. + runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }} env: JWT_SECRET: test-jwt-secret SENDGRID_API_KEY: fake-api-key @@ -106,8 +107,9 @@ jobs: # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey # runners, which don't count against the org's 20-concurrent-job cap). # Unset means GitHub-hosted, so rolling back is deleting one variable. The - # job names, and so the required check, don't change either way. - runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} + # job names, and so the required check, don't change either way. A fork PR + # always stays on GitHub: untrusted code doesn't get a third-party runner. + runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -203,8 +205,9 @@ jobs: # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey # runners, which don't count against the org's 20-concurrent-job cap). # Unset means GitHub-hosted, so rolling back is deleting one variable. The - # job names, and so the required check, don't change either way. - runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} + # job names, and so the required check, don't change either way. A fork PR + # always stays on GitHub: untrusted code doesn't get a third-party runner. + runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }} services: mongo: diff --git a/docs/runbooks/ci-heavy-runner.md b/docs/runbooks/ci-heavy-runner.md new file mode 100644 index 0000000000..ab8a068d14 --- /dev/null +++ b/docs/runbooks/ci-heavy-runner.md @@ -0,0 +1,47 @@ +# CI heavy jobs on Latchkey runners (HEAVY_RUNNER) + +The heavy jobs in `.github/workflows/tests.yml` (the three backend shards, +frontend tests and the Tier 1 service tests) run on whatever runner label the +repository variable `HEAVY_RUNNER` names, and on `ubuntu-latest` when it is +unset. Since 2026-10-24 it is set to `latchkey-medium`: Latchkey runners don't +count against the org's 20-concurrent-job cap on GitHub-hosted runners, which +was what made the merge queue wait. + +Everything else, including tool-tests, the guards, backend checks and chart +lint, stays on GitHub. Job names don't change either way, so the required check +("Test & Coverage") and merge-queue runs are unaffected. A PR from a fork +always runs on GitHub, so untrusted code never reaches a third-party runner. + +## Flip back to GitHub-hosted + +Do this if Latchkey is down, jobs sit queued with no runner, or the free +minutes run out: + +```bash +gh variable delete HEAVY_RUNNER --repo Team-Commonly/commonly +``` + +Runs that start after the delete go to `ubuntu-latest`. Runs already queued +keep the label they were given, so cancel and re-run those. To flip back to +Latchkey: `gh variable set HEAVY_RUNNER --body latchkey-medium --repo Team-Commonly/commonly`. + +## Why medium, and the Mongo ulimit + +Measured 2026-10-24 on dispatch-only trial runs: + +- **latchkey-small:** backend shards took 6.0, 12.7 and 16.6 minutes, against + 2.5 to 2.9 on GitHub. Slower overall, and more weighted minutes per run than + medium (about 40 against about 26). +- **latchkey-medium:** backend shards took about 3 minutes. +- **Mongo on Latchkey:** the `mongo:7` service container died with EMFILE ("Too + many open files") during index builds, then WT_PANIC, because the runner's + docker default `nofile` is low. `--ulimit nofile=64000:64000` on the service + fixes it; with it, the service tests passed 670 of 670 in 5.5 minutes. The + flag is a no-op on GitHub-hosted runners. + +tool-tests stays on GitHub: its CLI PATH test fails on Latchkey, and its Jest +process doesn't exit. + +Medium jobs count double against the free budget (14k minutes a month, plus a +10k bonus to 2026-11-23). Check usage on the Latchkey dashboard before the +bonus ends. From 6da2ef616d971608462643678322c4c2c0e0a7b0 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:43:50 -0700 Subject: [PATCH 4/4] ci(tests): state the HEAVY_RUNNER trust boundary accurately (fork PR runs vs merge_group) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V7T2fgquLRaqaYiGLC6Dey --- .github/workflows/tests.yml | 25 +++++++++++++++++-------- docs/runbooks/ci-heavy-runner.md | 30 ++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dbc21228a3..a2ca894cee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,8 +57,11 @@ jobs: # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey # runners, which don't count against the org's 20-concurrent-job cap). # Unset means GitHub-hosted, so rolling back is deleting one variable. The - # job names, and so the required check, don't change either way. A fork PR - # always stays on GitHub: untrusted code doesn't get a third-party runner. + # job names, and so the required check, don't change either way. A fork + # PR's own pull_request run stays on GitHub. A merge_group run uses + # HEAVY_RUNNER even when the group holds a reviewed fork change: queue + # admission (a maintainer queues a gated PR) is the trust decision, not + # this expression. See docs/runbooks/ci-heavy-runner.md. runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }} env: JWT_SECRET: test-jwt-secret @@ -107,8 +110,11 @@ jobs: # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey # runners, which don't count against the org's 20-concurrent-job cap). # Unset means GitHub-hosted, so rolling back is deleting one variable. The - # job names, and so the required check, don't change either way. A fork PR - # always stays on GitHub: untrusted code doesn't get a third-party runner. + # job names, and so the required check, don't change either way. A fork + # PR's own pull_request run stays on GitHub. A merge_group run uses + # HEAVY_RUNNER even when the group holds a reviewed fork change: queue + # admission (a maintainer queues a gated PR) is the trust decision, not + # this expression. See docs/runbooks/ci-heavy-runner.md. runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 @@ -205,8 +211,11 @@ jobs: # Heavy jobs run where the repo variable HEAVY_RUNNER points (D37: Latchkey # runners, which don't count against the org's 20-concurrent-job cap). # Unset means GitHub-hosted, so rolling back is deleting one variable. The - # job names, and so the required check, don't change either way. A fork PR - # always stays on GitHub: untrusted code doesn't get a third-party runner. + # job names, and so the required check, don't change either way. A fork + # PR's own pull_request run stays on GitHub. A merge_group run uses + # HEAVY_RUNNER even when the group holds a reviewed fork change: queue + # admission (a maintainer queues a gated PR) is the trust decision, not + # this expression. See docs/runbooks/ci-heavy-runner.md. runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && vars.HEAVY_RUNNER || 'ubuntu-latest' }} services: @@ -214,8 +223,8 @@ jobs: image: mongo:7 ports: ["27017:27017"] # Latchkey's docker default nofile is low enough that mongod hits - # EMFILE during index builds and WT_PANICs (trial 2, 10-24); a no-op - # on GitHub-hosted runners. + # EMFILE during index builds and WT_PANICs (trial 2, 10-24). It applies + # on GitHub-hosted runners too, where the service tests pass with it. options: >- --ulimit nofile=64000:64000 --health-cmd "mongosh --eval 'db.runCommand(\"ping\").ok'" diff --git a/docs/runbooks/ci-heavy-runner.md b/docs/runbooks/ci-heavy-runner.md index ab8a068d14..2e6eb79d88 100644 --- a/docs/runbooks/ci-heavy-runner.md +++ b/docs/runbooks/ci-heavy-runner.md @@ -3,14 +3,31 @@ The heavy jobs in `.github/workflows/tests.yml` (the three backend shards, frontend tests and the Tier 1 service tests) run on whatever runner label the repository variable `HEAVY_RUNNER` names, and on `ubuntu-latest` when it is -unset. Since 2026-10-24 it is set to `latchkey-medium`: Latchkey runners don't -count against the org's 20-concurrent-job cap on GitHub-hosted runners, which -was what made the merge queue wait. +unset. The plan (D54, Sam 2026-10-24) is to set it to `latchkey-medium` once +the PR adding it merges: Latchkey runners don't count against the org's +20-concurrent-job cap on GitHub-hosted runners, which was what made the merge +queue wait. Everything else, including tool-tests, the guards, backend checks and chart lint, stays on GitHub. Job names don't change either way, so the required check -("Test & Coverage") and merge-queue runs are unaffected. A PR from a fork -always runs on GitHub, so untrusted code never reaches a third-party runner. +("Test & Coverage") and merge-queue runs are unaffected. + +## Trust boundary + +- A fork PR's own `pull_request` run always uses `ubuntu-latest`. Unreviewed + fork code does not get a Latchkey runner from opening a PR. +- A `merge_group` run uses `HEAVY_RUNNER`, including a group that holds a + reviewed fork change. The expression does not inspect where queued PRs came + from. +- So queue admission is the trust decision: a maintainer puts a PR in the + queue only after its gate passed. A green test run, or this expression, does + not establish that trust. GitHub doesn't enforce the seat review either; the + ruleset requires only a green merge queue, so the review gate is operating + process. +- That is acceptable here because the Tests workflow grants only + `contents: read`, has no `pull_request_target` trigger, and exposes test + fixtures, not production secrets. Latchkey documents single-job VMs + destroyed after each job (vendor-published, not independently attested). ## Flip back to GitHub-hosted @@ -37,7 +54,8 @@ Measured 2026-10-24 on dispatch-only trial runs: many open files") during index builds, then WT_PANIC, because the runner's docker default `nofile` is low. `--ulimit nofile=64000:64000` on the service fixes it; with it, the service tests passed 670 of 670 in 5.5 minutes. The - flag is a no-op on GitHub-hosted runners. + limit applies on GitHub-hosted runners too (it is not gated on the + variable); the service tests passed there with it on this PR's CI. tool-tests stays on GitHub: its CLI PATH test fails on Latchkey, and its Jest process doesn't exit.