From 2cf1581c64f7e27b0441d38b5cd03f5700403dfa Mon Sep 17 00:00:00 2001 From: mkreyman Date: Wed, 21 Oct 2026 17:46:02 -0600 Subject: [PATCH 1/2] CI: self-hosted jobs run on the fleet-ci pool (beelink, minis, x1) minis and x1 now serve this repo too (mkreyman/infra decisions/0021). Each carries the same CI-only Postgres on 127.0.0.1:5433 (the same pgvector digest) that every self-hosted job here already targets, and a developer's databases on 5432, which none of them touch. The GitHub-hosted jobs are unchanged. --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55759777..4851f61a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ env: # mix.exs `~> 1.18`); only the OTP line moves forward to a working, current patch. OTP_VERSION: "28.5.0.2" -# SELF-HOSTED SPLIT (runners on beelink). lint / test / +# SELF-HOSTED SPLIT (runners on the fleet-ci pool: beelink, minis, x1). lint / test / # security / dialyzer / scale-test moved; the rest # stay on ubuntu-latest for specific reasons, not by omission. Measured job times # (2026-07-30) are recorded per entry, because the whole point of the split is @@ -130,7 +130,7 @@ env: jobs: lint: name: Lint - runs-on: [self-hosted, beelink] + runs-on: [self-hosted, fleet-ci] # FORK-PR GUARD (#881): a fork PR never runs on the shared self-hosted pool — its job # code runs as the github-runner uid every pool slot shares, with read access to other # slots' live JIT config and secrets. On a fork PR this job is SKIPPED (a skipped job @@ -300,7 +300,7 @@ jobs: test: name: Test - runs-on: [self-hosted, beelink] + runs-on: [self-hosted, fleet-ci] # Fork-PR guard — see the Lint job's comment (#881). if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository # NO `services:` block on a FIXED host port on self-hosted (scale-test's container binds a @@ -324,8 +324,8 @@ jobs: # Safe as a non-integer: this job does not pass `mix test --partitions`, which is # the only thing that validates the variable as a bare 1..N. MIX_TEST_PARTITION: ci${{ github.run_id }} - # The CI-only Postgres, NOT the instance on 5432 -- which, now that the runners - # are back on beelink, holds Mark's development databases. + # The CI-only Postgres, NOT the instance on 5432 -- which, on every fleet-ci host + # (beelink, minis, x1), holds a developer's databases. TEST_DB_PORT: "5433" # The psql steps below use libpq directly; PGPORT is what points THEM at the # same instance the Ecto repos use. Setting only one of the two is the bug @@ -510,7 +510,7 @@ jobs: retrieval-eval: name: Retrieval Eval - runs-on: [self-hosted, beelink] + runs-on: [self-hosted, fleet-ci] # Fork-PR guard — see the Lint job's comment (#881). if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository # MIX_ENV=dev on purpose (the workflow-level default is test): `mix @@ -741,7 +741,7 @@ jobs: security: name: Security - runs-on: [self-hosted, beelink] + runs-on: [self-hosted, fleet-ci] # Fork-PR guard — see the Lint job's comment (#881). if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository steps: @@ -831,7 +831,7 @@ jobs: dialyzer: name: Dialyzer - runs-on: [self-hosted, beelink] + runs-on: [self-hosted, fleet-ci] # Fork-PR guard — see the Lint job's comment (#881). if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository steps: @@ -934,7 +934,7 @@ jobs: # included GitHub-hosted minutes. It keeps its OWN Postgres service container, on a host # port Docker picks, so its committed corpus, its role bootstrap and its index builds # never touch the CI Postgres the Test job runs against. - runs-on: [self-hosted, beelink] + runs-on: [self-hosted, fleet-ci] timeout-minutes: 30 if: > (github.event_name == 'schedule' || @@ -944,7 +944,7 @@ jobs: github.event.pull_request.head.repo.full_name == github.repository) services: postgres: - # Pinned by DIGEST to the image beelink's CI Postgres already runs (pgvector 0.8.7, + # Pinned by DIGEST to the image every fleet-ci host's CI Postgres already runs (pgvector 0.8.7, # tag 0.8.7-pg16-trixie), so the job never pulls a floating tag from Docker Hub: a # present digest is not re-pulled. Above the 0.8.4 floor (#965), so the insert/VACUUM # race is gone. Moved with the host instances under mkreyman/infra#280 (#977); the @@ -1304,7 +1304,7 @@ jobs: # other's ANALYZE results any more than two containers could. What a shared cluster # DOES share is CPU, shared_buffers and disk — which affects timing, not the plan # CHOICE these tests assert on, and the matrix runs serially on one runner anyway. - runs-on: [self-hosted, beelink] + runs-on: [self-hosted, fleet-ci] # MANUAL DISPATCH, PLUS THE WEEKLY CRON ONLY. Not every-PR (the 80k seed per file is far # too slow for every push) and no longer every night. The enforced pre-merge step for a # Theme-2/3/4 (query/index) change is `gh workflow run CI --ref ` or the local From 7e9cd75d57065c2a81bd0764dc041a2fce3473e6 Mon Sep 17 00:00:00 2001 From: mkreyman Date: Wed, 21 Oct 2026 17:52:12 -0600 Subject: [PATCH 2/2] CI comments and docs describe the three-host pool Review fixes: scale-test's tmpfs note says its memory is a developer's RAM on minis and x1, outside ci.slice and unseen by the CPU-only gate; scale-nightly legs can run at once across hosts, and a leftover database is on the host the job page names; CLAUDE.md and the scale runbook stop saying one runner. --- .github/workflows/ci.yml | 31 +++++++++++-------- CLAUDE.md | 3 +- docs/runbooks/knowledge_scale_verification.md | 7 +++-- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4851f61a..57f3ae10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: schedule: # Nightly at 02:00 UTC — the ordinary jobs (lint / test / retrieval-eval / security / # dialyzer / scale-test / pgbouncer-e2e). Explicitly NOT scale-nightly: that matrix is - # gated to manual dispatch and would otherwise hold the one self-hosted runner + # gated to manual dispatch and would otherwise hold self-hosted runner slots # for hours every night, queued ahead of ordinary PR checks. - cron: "0 2 * * *" # Manual trigger so the scale-nightly matrix can be RUN and validated on a branch BEFORE @@ -60,16 +60,16 @@ env: # pgbouncer-e2e 1.4-2.4 min, EVERY push and PR. Needs pgbouncer in front of # Postgres; the port topology has to be reworked for a shared # host DB. Also GATES deploy, so it cannot simply be dropped. -# scale-test MOVED to beelink 2026-09-26: 3.1-5.1 min on EVERY push and PR +# scale-test MOVED self-hosted 2026-09-26: 3.1-5.1 min on EVERY push and PR # had spent the account's included minutes. It brings its own # Postgres service container on a Docker-picked host port. # scale-nightly ZERO billed hosted minutes: it moved self-hosted (the port is # DONE -- see the per-matrix-file database note on the job) and # it no longer runs nightly. ~143 min of wall clock per full # 14-file matrix (one outlier, streaming_export, is 40 min on -# its own) -- MEASURED ON BEELINK, 24 cores. The host is now -# nuc at 4 cores, so treat 143 as a floor, not an estimate, -# until it is re-measured. Now only on an explicit dispatch plus any +# its own) -- MEASURED ON BEELINK, 24 cores. It now runs on any +# fleet-ci host (beelink 24 cores, minis 16, x1 24), legs in +# parallel across slots. Now only on an explicit dispatch plus any # manual dispatch -- so ~620 min/month (143 x 52/12) of self-hosted time in # place of the ~4,300 billed min/month it used to be, when it # was the largest hosted line item by a wide margin. @@ -930,7 +930,7 @@ jobs: # under the fork-PR guard. This ensures the scale suite (which commits real rows and runs # ANALYZE) is executed in CI and that the DBConnection.OwnershipError or # other regressions are caught before they reach production. - # On beelink since 2026-09-26: 3-5 minutes on every push and PR had spent the account's + # Self-hosted since 2026-09-26: 3-5 minutes on every push and PR had spent the account's # included GitHub-hosted minutes. It keeps its OWN Postgres service container, on a host # port Docker picks, so its committed corpus, its role bootstrap and its index builds # never touch the CI Postgres the Test job runs against. @@ -948,7 +948,8 @@ jobs: # tag 0.8.7-pg16-trixie), so the job never pulls a floating tag from Docker Hub: a # present digest is not re-pulled. Above the 0.8.4 floor (#965), so the insert/VACUUM # race is gone. Moved with the host instances under mkreyman/infra#280 (#977); the - # digest was pre-pulled on beelink (2026-10-13). + # digest was pre-pulled on beelink (2026-10-13), and minis and x1 hold it as the image + # their own CI Postgres runs. image: pgvector/pgvector@sha256:40d828eda66737aaacb934387a337babf1455ebd56e9bdd13fc08e211257c972 env: POSTGRES_USER: postgres @@ -961,9 +962,11 @@ jobs: # The data directory is a tmpfs, not the image's declared VOLUME. The runner removes a # service container without -v, so the anonymous volume outlived every run: 221 of them, # 476.7 GB, on beelink by 2026-10-07 (#946). A tmpfs gets no volume, so it dies with the - # container. Its pages are beelink's RAM (or swap) for the whole run, shared with every - # OTHER repo's CI runner on the host (home_care_billing's among them) and beelink's own - # Docker Postgres; this repo's jobs there run one at a time. The cap bounds that: a run's + # container. Its pages are the host's RAM (or swap) for the whole run, shared with every + # OTHER repo's CI runner on the host (home_care_billing's among them) and its Docker + # Postgres instances. On minis and x1 that is a developer's workstation RAM (28 and + # 59 GB): the pool's load gate admits a job on CPU headroom, never memory, and a service + # container sits outside ci.slice's MemoryMax. The cap bounds that: a run's # end-of-run data dir, WAL included, measured ~2.17 GB as a leaked volume, so a run that # outgrows 4g fails on ENOSPC instead of growing without limit. The report step at the # end of this job prints the container's PEAK memory, which bounds the tmpfs high-water @@ -1297,13 +1300,15 @@ jobs: # hosted job combined. Moved self-hosted, and off the nightly cron — see above. # # The isolation used to come from a FRESH Postgres service container per matrix job. - # On the self-hosted box there is one long-lived Postgres, so isolation comes from a + # On a self-hosted host there is one long-lived CI Postgres, so isolation comes from a # PER-MATRIX-FILE DATABASE instead (see "Derive this matrix file's database" below). # That is equivalent for the property that matters: planner statistics live in the # per-database pg_statistic/pg_class, so two databases in one cluster cannot see each # other's ANALYZE results any more than two containers could. What a shared cluster # DOES share is CPU, shared_buffers and disk — which affects timing, not the plan - # CHOICE these tests assert on, and the matrix runs serially on one runner anyway. + # CHOICE these tests assert on. Legs for different files can run at once on different + # slots and hosts (beelink, minis, x1); each has its own database either way, and a leg's + # leftover database is on the host its job page names. runs-on: [self-hosted, fleet-ci] # MANUAL DISPATCH, PLUS THE WEEKLY CRON ONLY. Not every-PR (the 80k seed per file is far # too slow for every push) and no longer every night. The enforced pre-merge step for a @@ -1328,7 +1333,7 @@ jobs: env: # scale-nightly derives its own database per matrix file (loopctl_test_sn_) # and drops it again at the end, so it never shared a name with anything. What it - # DID share, once the runners moved back to beelink, is the instance: without + # DID share, on every self-hosted host, is the instance: without # these it would create and seed multi-hundred-thousand-row scale databases inside # Mark's development Postgres. Both variables are needed -- TEST_DB_PORT for the # Ecto repos, PGPORT for the psql steps. diff --git a/CLAUDE.md b/CLAUDE.md index fefd6dac..c475f039 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -471,7 +471,8 @@ matters rather than from a number here: gh api repos/:owner/:repo/branches/master/protection/required_status_checks --jq .contexts Why: every update restarts the whole suite on a new head, and the required set takes minutes — -five of its contexts run on the single self-hosted runner, the rest on `ubuntu-latest`. The +five of its contexts run on the self-hosted `fleet-ci` pool (beelink, minis, x1), the rest +on `ubuntu-latest`. The saving is real but is not the whole of a PR's CI bill: review-fix pushes restart it too, and `ci.yml`'s `concurrency` group is keyed per REF with `cancel-in-progress` off for master, so a push cancels that branch's own in-flight run and never another PR's. Concurrent PRs do not diff --git a/docs/runbooks/knowledge_scale_verification.md b/docs/runbooks/knowledge_scale_verification.md index 398e586b..ca472582 100644 --- a/docs/runbooks/knowledge_scale_verification.md +++ b/docs/runbooks/knowledge_scale_verification.md @@ -128,10 +128,11 @@ It runs in two places: 02:00 one the rest of CI uses, and not any other. So plan drift that arrives with no PR (a Postgres/pgvector upgrade, an ANALYZE shift) is NOT caught automatically: dispatch the matrix yourself after any such infrastructure change. It is a **matrix — one isolated job per scale file**, each - on its own database (`loopctl_test_sn_`, derived in the job; beelink has one - long-lived Postgres, and planner stats live in the per-database `pg_statistic`) so a + on its own database (`loopctl_test_sn_`, derived in the job; each fleet-ci host (beelink, minis, x1) + has one long-lived CI Postgres, and planner stats live in the per-database `pg_statistic`) so a committed-row corpus from one file can't pollute another's global planner stats. A - failed drop step leaves that database behind — reclaim it by hand. + failed drop step leaves that database behind on the host the job page names — reclaim it + there by hand. Each leg runs `SCALE_TESTS=true SCALE_NIGHTLY=true mix test --only scale_nightly `. The matrix covers **every `:scale_nightly`-tagged file** (the US-27.1 seed-floor check, the HNSW ANN / keyset / shared-assertion plan gates, and the US-27.8 per-endpoint index-usage,