From a22e90754ac8a8aa6d353a6e2e20ca928920a79d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:39:01 -0700 Subject: [PATCH 01/14] ci: use local caches on self-hosted runners Remove dev-shell uploads and the Nix-only x86 warming job. Keep hosted Rust warming and use mbx's local backend for self-hosted jobs. Co-Authored-By: GPT-6 --- .github/actions/rust-cache/action.yml | 4 ++- .github/workflows/cache.yml | 41 ++++----------------------- sh/gh/setup.test.ts | 11 ++++++- 3 files changed, 18 insertions(+), 38 deletions(-) diff --git a/.github/actions/rust-cache/action.yml b/.github/actions/rust-cache/action.yml index 43727027141..979c9401bae 100644 --- a/.github/actions/rust-cache/action.yml +++ b/.github/actions/rust-cache/action.yml @@ -1,5 +1,5 @@ name: Rust cache -description: Install mbx, put its Cargo shim on PATH, and restore the build cache the default branch warms. +description: Install mbx and its Cargo shim, using local storage on self-hosted runners or GitHub cache elsewhere. inputs: scope: @@ -17,6 +17,8 @@ runs: - uses: jdx/mr-boxington-action@7234d3dd1a6ca8f6c381eea8e4dfb03f18fcf777 # v1.3.0 with: version: 1.10.0 + # Self-hosted runners keep their mbx store locally between jobs. + backend: ${{ runner.environment == 'self-hosted' && 'local' || 'github' }} # `github-cache-mode` is left at its `target` default. That transports # Cargo's pruned target tree, the same payload Swatinem carried, with # mbx's per-compilation store on top. The `objects` closure is portable diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index b00f9aede95..0e752bc8bab 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -1,7 +1,6 @@ name: Cache -# Populates the Rust build cache that pull request jobs restore, and the dev -# shell closure that every nix job substitutes. +# Populates the Rust build cache that pull request and merge queue jobs restore. # # Actions scopes cache reads to the current branch plus the default branch, so # only caches written here on `main` are reachable from a pull request. This @@ -11,8 +10,7 @@ name: Cache # The Rust store is ARM64 only, because every job that restores it runs there. # Each store is about 6 GB against the repository's 10 GB budget, and Actions # evicts least recently used first, so an x86 store nobody on a pull request -# read was enough to push out the one they all do. The x86 leg still pushes its -# dev shell, which the x86 jobs substitute. +# read was enough to push out the one they all do. # # This runs the unscoped suite rather than the diff-scoped one: the point is to # leave artifacts for whatever a future PR happens to touch, not to check `main` @@ -50,16 +48,8 @@ concurrency: jobs: warm: - name: Warm (${{ matrix.runner }}) - strategy: - fail-fast: false - matrix: - include: - - runner: ubuntu-24.04-arm - rust: true - - runner: ubuntu-latest - rust: false - runs-on: ${{ matrix.runner }} + name: Warm + runs-on: ubuntu-24.04-arm timeout-minutes: 60 steps: @@ -81,30 +71,10 @@ jobs: extra-substituters = https://kixelated.cachix.org extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - # cachix.yml only fires on release tags and only pushes the tagged - # package, so the dev shell itself was never cached. Anything in it that - # nixpkgs doesn't serve prebuilt (uniffi-bindgen-go, a - # buildRustPackage from a git tag) was therefore compiled from source by - # every job that entered the shell. Push it once here so pull requests - # substitute it instead. - - uses: cachix/cachix-action@38b082610b782e7e93e209c35fd730d399dee866 # v17 - with: - name: kixelated - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - - - name: Cache the dev shell - run: | - # `--profile` realises the shell's full build closure, which is what - # the runners actually need; `nix build .#devShells...` alone leaves - # out the build-time inputs. - nix develop --profile /tmp/devshell --command true - cachix push kixelated /tmp/devshell - # The single writer. `save-on-dispatch` is what lets the # `workflow_dispatch` above prime a cold store; the action otherwise saves # only on pushes to the default branch. - name: Rust cache - if: matrix.rust uses: ./.github/actions/rust-cache with: save-on-dispatch: true @@ -119,14 +89,13 @@ jobs: # pull request restoring a store days stale. The pull request gates are # what report those failures. - name: Check - if: matrix.rust continue-on-error: true run: nix develop --command just ci check --all env: MOQ_STRICT: 1 - name: Test - if: ${{ matrix.rust && !cancelled() }} + if: ${{ !cancelled() }} continue-on-error: true run: nix develop --command just ci test --all env: diff --git a/sh/gh/setup.test.ts b/sh/gh/setup.test.ts index 66d7e5e8c8f..11298390d7a 100644 --- a/sh/gh/setup.test.ts +++ b/sh/gh/setup.test.ts @@ -1,4 +1,4 @@ -// The hosted runner still owns timing and action execution. This checks its setup gates. +// Runner timing and action execution stay in CI. These tests check the setup gates. import { expect, test } from "bun:test"; type Step = { @@ -48,3 +48,12 @@ test("scope queries support both merge groups and release PRs without path filte expect(workflow.on.pull_request).toBeDefined(); expect(workflow.on.pull_request).not.toHaveProperty("paths"); }); + +test("self-hosted runners use the local Rust cache while hosted runners restore GitHub cache", async () => { + const action = Bun.YAML.parse(await Bun.file(".github/actions/rust-cache/action.yml").text()); + const setup = action.runs.steps.find((step: Step) => step.uses?.startsWith("jdx/mr-boxington-action@")); + const expression = setup.with.backend.replace(/^\$\{\{\s*|\s*\}\}$/g, ""); + const backend = new Function("runner", `return (${expression});`); + expect(backend({ environment: "self-hosted" })).toBe("local"); + expect(backend({ environment: "github-hosted" })).toBe("github"); +}); From aaedc05c39bec2b3ea20cdb4c2082e7a6ee8fd80 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:03:32 -0700 Subject: [PATCH 02/14] ci: make the merge-queue runner configurable Use the MOQ_QUEUE_RUNNER label for merge-group jobs when configured, retain hosted PR jobs, and use the VM's existing Nix installation. Co-Authored-By: GPT-6 --- .github/workflows/check.yml | 9 +++++---- sh/gh/setup.test.ts | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 98fce3d41ea..90a3b610c6e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -27,7 +27,8 @@ jobs: # Skip the actual work on close/merge: this run exists only to cancel the # superseded in-flight build via the concurrency group above. if: github.event.action != 'closed' - runs-on: ubuntu-24.04-arm + # Leave the variable unset until the local runner is online and verified. + runs-on: ${{ github.event_name == 'merge_group' && vars.MOQ_QUEUE_RUNNER || 'ubuntu-24.04-arm' }} # Backstop a hang instead of holding the runner for the 6-hour default. timeout-minutes: 60 @@ -50,7 +51,7 @@ jobs: CI_BASE: ${{ github.event.merge_group.base_sha }} - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - if: steps.scope.outputs.needed == 'true' + if: steps.scope.outputs.needed == 'true' && runner.environment != 'self-hosted' with: determinate: false # Trust the flake's cachix substituter. `nixConfig` in flake.nix is @@ -95,7 +96,7 @@ jobs: test: name: Test if: github.event.action != 'closed' - runs-on: ubuntu-24.04-arm + runs-on: ${{ github.event_name == 'merge_group' && vars.MOQ_QUEUE_RUNNER || 'ubuntu-24.04-arm' }} timeout-minutes: 60 steps: @@ -117,7 +118,7 @@ jobs: CI_BASE: ${{ github.event.merge_group.base_sha }} - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - if: steps.scope.outputs.needed == 'true' + if: steps.scope.outputs.needed == 'true' && runner.environment != 'self-hosted' with: determinate: false # Trust the flake's cachix substituter. `nixConfig` in flake.nix is diff --git a/sh/gh/setup.test.ts b/sh/gh/setup.test.ts index 11298390d7a..3b748e9ff4b 100644 --- a/sh/gh/setup.test.ts +++ b/sh/gh/setup.test.ts @@ -10,7 +10,7 @@ type Step = { env?: Record; "continue-on-error"?: boolean; }; -type Job = { name: string; steps: Step[] }; +type Job = { name: string; steps: Step[]; "runs-on": string }; const workflow = Bun.YAML.parse(await Bun.file(".github/workflows/check.yml").text()) as { on: Record; jobs: Record; @@ -38,9 +38,20 @@ for (const name of ["check", "test"]) { expect(setup.at(-1)?.name).toBe(job.name); for (const step of setup) { const output = step.uses === "./.github/actions/rust-cache" ? "rust" : "needed"; - expect(step.if).toBe(`steps.scope.outputs.${output} == 'true'`); + const hosted = step.uses?.startsWith("DeterminateSystems/nix-installer-action@") + ? " && runner.environment != 'self-hosted'" + : ""; + expect(step.if).toBe(`steps.scope.outputs.${output} == 'true'${hosted}`); } }); + + test(`${name} uses the configured runner only for merge groups`, () => { + const expression = workflow.jobs[name]["runs-on"].replace(/^\$\{\{\s*|\s*\}\}$/g, ""); + const runner = new Function("github", "vars", `return (${expression});`); + expect(runner({ event_name: "merge_group" }, { MOQ_QUEUE_RUNNER: "moq-queue" })).toBe("moq-queue"); + expect(runner({ event_name: "merge_group" }, {})).toBe("ubuntu-24.04-arm"); + expect(runner({ event_name: "pull_request" }, { MOQ_QUEUE_RUNNER: "moq-queue" })).toBe("ubuntu-24.04-arm"); + }); } test("scope queries support both merge groups and release PRs without path filters", () => { From f6fddbf1ae62db26215aca23d1d2eab986dfce64 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:07:34 -0700 Subject: [PATCH 03/14] ci: refuse non-queue jobs on persistent local caches Keep pull requests on hosted runners and fail before cache setup if a non-merge-group job selects a persistent self-hosted runner. Co-Authored-By: GPT-6 --- .github/actions/rust-cache/action.yml | 11 +++++++++-- sh/gh/setup.test.ts | 16 ++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/actions/rust-cache/action.yml b/.github/actions/rust-cache/action.yml index 979c9401bae..1fa6d81343e 100644 --- a/.github/actions/rust-cache/action.yml +++ b/.github/actions/rust-cache/action.yml @@ -12,13 +12,20 @@ inputs: runs: using: composite steps: + - name: Require merge queue on self-hosted runners + if: runner.environment == 'self-hosted' && github.event_name != 'merge_group' + shell: bash + run: | + echo "::error::Persistent self-hosted caches require a merge_group job." >&2 + exit 1 + # The pinned action saves in a post step guarded by success(), so failed # builds cannot publish a cache even when dispatch saving is enabled. - uses: jdx/mr-boxington-action@7234d3dd1a6ca8f6c381eea8e4dfb03f18fcf777 # v1.3.0 with: version: 1.10.0 - # Self-hosted runners keep their mbx store locally between jobs. - backend: ${{ runner.environment == 'self-hosted' && 'local' || 'github' }} + # Only reviewed queue jobs share the persistent local store. + backend: ${{ runner.environment == 'self-hosted' && github.event_name == 'merge_group' && 'local' || 'github' }} # `github-cache-mode` is left at its `target` default. That transports # Cargo's pruned target tree, the same payload Swatinem carried, with # mbx's per-compilation store on top. The `objects` closure is portable diff --git a/sh/gh/setup.test.ts b/sh/gh/setup.test.ts index 3b748e9ff4b..fd78a1b2d16 100644 --- a/sh/gh/setup.test.ts +++ b/sh/gh/setup.test.ts @@ -60,11 +60,19 @@ test("scope queries support both merge groups and release PRs without path filte expect(workflow.on.pull_request).not.toHaveProperty("paths"); }); -test("self-hosted runners use the local Rust cache while hosted runners restore GitHub cache", async () => { +test("only self-hosted merge groups use the local Rust cache", async () => { const action = Bun.YAML.parse(await Bun.file(".github/actions/rust-cache/action.yml").text()); const setup = action.runs.steps.find((step: Step) => step.uses?.startsWith("jdx/mr-boxington-action@")); const expression = setup.with.backend.replace(/^\$\{\{\s*|\s*\}\}$/g, ""); - const backend = new Function("runner", `return (${expression});`); - expect(backend({ environment: "self-hosted" })).toBe("local"); - expect(backend({ environment: "github-hosted" })).toBe("github"); + const backend = new Function("runner", "github", `return (${expression});`); + const guard = new Function("runner", "github", `return (${action.runs.steps[0].if});`); + for (const event_name of ["merge_group", "pull_request", "pull_request_target", "push", "workflow_dispatch"]) { + expect(backend({ environment: "self-hosted" }, { event_name })).toBe( + event_name === "merge_group" ? "local" : "github", + ); + expect(guard({ environment: "self-hosted" }, { event_name })).toBe(event_name !== "merge_group"); + expect(backend({ environment: "github-hosted" }, { event_name })).toBe("github"); + expect(guard({ environment: "github-hosted" }, { event_name })).toBe(false); + } + expect(action.runs.steps[0].run).toContain("exit 1"); }); From 33738fcf44c2283e19ff4e202daf702cd94ac1b5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:37:49 -0700 Subject: [PATCH 04/14] ci: provide a reusable persistent runner VM Expose a NixOS module with configurable repository, labels, uplink, resources, and guest configuration. Keep Nix and mbx caches on the guest disk and isolate the VM from host and private networks. Build the image and test startup, isolation, and cache persistence in CI. Co-Authored-By: GPT-6 --- .github/justfile | 4 + .github/workflows/runner-vm.yml | 35 +++++ flake.nix | 6 +- nix/modules/github-runner-vm.nix | 243 +++++++++++++++++++++++++++++++ nix/runner/README.md | 72 +++++++++ nix/runner/checks.nix | 23 +++ nix/runner/guest.nix | 137 +++++++++++++++++ nix/runner/stop.py | 18 +++ nix/runner/test.nix | 146 +++++++++++++++++++ sh/gh/alert.sh | 2 +- sh/gh/alert.test.ts | 2 +- 11 files changed, 685 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/runner-vm.yml create mode 100644 nix/modules/github-runner-vm.nix create mode 100644 nix/runner/README.md create mode 100644 nix/runner/checks.nix create mode 100644 nix/runner/guest.nix create mode 100644 nix/runner/stop.py create mode 100644 nix/runner/test.nix diff --git a/.github/justfile b/.github/justfile index d4777d96aee..4d84f1293cc 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,5 +1,9 @@ set working-directory := '..' +# Build and test the persistent NixOS runner VM on x86_64 Linux. +runner-vm: + nix build --no-link .#checks.x86_64-linux.github-runner-vm .#checks.x86_64-linux.github-runner-vm-image --print-build-logs + # Lint GitHub Actions workflows, their alert coverage, and that every step runs a recipe. check: actionlint diff --git a/.github/workflows/runner-vm.yml b/.github/workflows/runner-vm.yml new file mode 100644 index 00000000000..bcef31c8842 --- /dev/null +++ b/.github/workflows/runner-vm.yml @@ -0,0 +1,35 @@ +name: Runner VM + +on: + pull_request: + paths: + - "nix/**" + - "flake.nix" + - "flake.lock" + - ".github/justfile" + - ".github/workflows/runner-vm.yml" + push: + branches: [main] + paths: + - "nix/**" + - "flake.nix" + - "flake.lock" + - ".github/justfile" + - ".github/workflows/runner-vm.yml" + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + - run: just gh runner-vm diff --git a/flake.nix b/flake.nix index 8a8c51cc2d3..b877bfc6f16 100644 --- a/flake.nix +++ b/flake.nix @@ -53,6 +53,7 @@ { nixosModules = { moq-relay = import ./nix/modules/moq-relay.nix; + github-runner-vm = import ./nix/modules/github-runner-vm.nix; }; overlays.default = import ./nix/overlay.nix { inherit crane; }; @@ -646,7 +647,10 @@ test -f "${overlayPkgs.moq-boy.src}/rs/moq-video/src/frame/nv12_resize.ptx" touch "$out" ''; - }; + } + // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( + import ./nix/runner/checks.nix { inherit pkgs; } + ); } ); } diff --git a/nix/modules/github-runner-vm.nix b/nix/modules/github-runner-vm.nix new file mode 100644 index 00000000000..2e6d514a89d --- /dev/null +++ b/nix/modules/github-runner-vm.nix @@ -0,0 +1,243 @@ +{ + config, + lib, + pkgs, + utils, + ... +}: + +let + cfg = config.services.github-runner-vm; + tap = "tap-${cfg.name}"; + user = cfg.name; + guest = import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + (import ../runner/guest.nix { inherit (cfg) runner resources; }) + cfg.guestConfig + ]; + }; + device = "${utils.escapeSystemdPath "/sys/subsystem/net/devices/${tap}"}.device"; + qemu = pkgs.qemu_kvm; + stop = pkgs.writeShellScript "github-runner-vm-stop" '' + set -euo pipefail + ${ + pkgs.python3.withPackages (p: [ p.qemu-qmp ]) + }/bin/python3 ${../runner/stop.py} "$RUNTIME_DIRECTORY/qmp" + ${pkgs.coreutils}/bin/tail --pid="$MAINPID" -f /dev/null + ''; +in +{ + options.services.github-runner-vm = { + enable = lib.mkEnableOption "a GitHub Actions runner in a persistent KVM guest"; + name = lib.mkOption { + type = lib.types.strMatching "[a-z][a-z0-9-]{0,9}"; + default = "github-ci"; + description = "Name of the host service, state directory, and VM user."; + }; + runner = { + url = lib.mkOption { + type = lib.types.str; + description = "GitHub repository URL registered to the runner."; + }; + name = lib.mkOption { + type = lib.types.str; + default = "${config.networking.hostName}-${cfg.name}"; + description = "Runner name advertised to GitHub."; + }; + labels = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Additional labels advertised to GitHub."; + }; + }; + network.uplink = lib.mkOption { + type = lib.types.strMatching "[a-zA-Z0-9_.:-]{1,15}"; + description = "Host interface used for public IPv4 egress."; + }; + resources = { + cores = lib.mkOption { + type = lib.types.ints.positive; + default = 4; + description = "Virtual CPUs and maximum host CPU usage in cores."; + }; + memory = lib.mkOption { + type = lib.types.ints.positive; + default = 8192; + description = "Guest memory in MiB, with 2 GiB extra allowed for QEMU."; + }; + disk = lib.mkOption { + type = lib.types.ints.positive; + default = 128; + description = "Initial persistent disk capacity in GiB."; + }; + }; + tokenFile = lib.mkOption { + type = lib.types.str; + default = "/var/lib/secrets/github-runner-vm.token"; + description = "Host path to the runtime GitHub registration token."; + }; + guestConfig = lib.mkOption { + type = lib.types.deferredModule; + default = { }; + description = "Extra NixOS guest configuration, including SSH keys and binary caches."; + }; + guest = lib.mkOption { + type = lib.types.raw; + readOnly = true; + description = "Evaluated guest system for deployment as a NixOS flake configuration."; + }; + }; + + config = lib.mkIf cfg.enable { + services.github-runner-vm.guest = guest; + assertions = [ + { + assertion = pkgs.stdenv.hostPlatform.system == "x86_64-linux"; + message = "services.github-runner-vm supports x86_64-linux KVM hosts."; + } + ]; + users.groups.${user} = { }; + users.users.${user} = { + isSystemUser = true; + group = user; + extraGroups = [ "kvm" ]; + }; + + networking.networkmanager.unmanaged = [ "interface-name:${tap}" ]; + systemd.network = { + enable = true; + # When NetworkManager owns the uplink, it also owns online readiness. + # The TAP cannot gain carrier until QEMU starts. + wait-online.enable = lib.mkIf config.networking.networkmanager.enable false; + netdevs."30-github-runner" = { + netdevConfig = { + Kind = "tap"; + Name = tap; + }; + tapConfig.User = user; + }; + networks."30-github-runner" = { + matchConfig.Name = tap; + address = [ "10.231.0.1/30" ]; + linkConfig.RequiredForOnline = "no"; + networkConfig = { + LinkLocalAddressing = "no"; + ConfigureWithoutCarrier = true; + }; + }; + }; + + networking.nat = { + enable = true; + externalInterface = cfg.network.uplink; + internalInterfaces = [ tap ]; + }; + networking.firewall.filterForward = true; + networking.nftables = { + enable = true; + tables.github-runner-vm = { + family = "inet"; + content = '' + chain input { + type filter hook input priority -10; policy accept; + iifname "${tap}" ct state established,related accept + iifname "${tap}" counter drop + } + chain forward { + type filter hook forward priority -10; policy accept; + iifname "${tap}" meta nfproto ipv6 counter drop + iifname "${tap}" ip saddr != 10.231.0.2 counter drop + iifname "${tap}" ip daddr { + 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, + 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, + 198.18.0.0/15, 224.0.0.0/4, 240.0.0.0/4 + } counter drop + iifname "${tap}" oifname != "${cfg.network.uplink}" counter drop + oifname "${tap}" ct state established,related accept + oifname "${tap}" counter drop + } + ''; + }; + }; + + systemd.services.${cfg.name} = { + description = "Persistent GitHub Actions runner VM"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + bindsTo = [ + "nftables.service" + device + ]; + after = [ + "network-online.target" + "nftables.service" + device + ]; + path = [ qemu ]; + preStart = '' + if ! test -e /dev/kvm; then + echo "KVM is unavailable. Enable virtualization in firmware and load the CPU KVM module." >&2 + exit 1 + fi + if ! test -r /dev/kvm || ! test -w /dev/kvm; then + echo "The ${user} user needs read and write access to /dev/kvm through the kvm group." >&2 + exit 1 + fi + if ! test -e "$STATE_DIRECTORY/root.qcow2"; then + cp --reflink=auto ${guest.config.system.build.image}/nixos.qcow2 "$STATE_DIRECTORY/root.qcow2.tmp" + chmod 0600 "$STATE_DIRECTORY/root.qcow2.tmp" + qemu-img resize "$STATE_DIRECTORY/root.qcow2.tmp" ${toString cfg.resources.disk}G + mv "$STATE_DIRECTORY/root.qcow2.tmp" "$STATE_DIRECTORY/root.qcow2" + fi + ''; + script = '' + exec ${qemu}/bin/qemu-system-x86_64 \ + -enable-kvm -cpu host -smp ${toString cfg.resources.cores} -m ${toString cfg.resources.memory} \ + -drive file="$STATE_DIRECTORY/root.qcow2",format=qcow2,if=virtio,cache=none \ + -netdev tap,id=net,ifname=${tap},script=no,downscript=no \ + -device virtio-net-pci,netdev=net,mac=02:00:00:00:23:02 \ + -device virtio-rng-pci \ + -fw_cfg name=opt/io.systemd.credentials/runner-token,file="$CREDENTIALS_DIRECTORY/runner-token" \ + -qmp unix:"$RUNTIME_DIRECTORY/qmp",server=on,wait=off \ + -nographic + ''; + serviceConfig = { + User = user; + Group = user; + StateDirectory = cfg.name; + StateDirectoryMode = "0700"; + RuntimeDirectory = cfg.name; + RuntimeDirectoryMode = "0700"; + LoadCredential = [ "runner-token:${cfg.tokenFile}" ]; + ExecStop = stop; + TimeoutStopSec = 120; + ProtectHome = true; + ProtectSystem = "strict"; + PrivateTmp = true; + NoNewPrivileges = true; + CapabilityBoundingSet = ""; + DevicePolicy = "closed"; + DeviceAllow = [ + "/dev/kvm rw" + "/dev/net/tun rw" + ]; + # QEMU queries the TAP's MTU through an IP socket; guest packets use the + # TAP descriptor, so denying host IP traffic does not block guest NAT. + RestrictAddressFamilies = [ + "AF_UNIX" + "AF_INET" + "AF_INET6" + "AF_NETLINK" + ]; + IPAddressDeny = "any"; + Nice = 10; + CPUWeight = 10; + IOWeight = 10; + CPUQuota = "${toString (cfg.resources.cores * 100)}%"; + MemoryMax = "${toString (cfg.resources.memory + 2048)}M"; + TasksMax = 1024; + }; + }; + }; +} diff --git a/nix/runner/README.md b/nix/runner/README.md new file mode 100644 index 00000000000..108fe2d11ad --- /dev/null +++ b/nix/runner/README.md @@ -0,0 +1,72 @@ +# Persistent GitHub Actions runner VM + +The `github-runner-vm` NixOS module runs a repository's runner in a persistent +KVM guest. The guest keeps its Nix store, mbx cache, checkout, and runner state +across jobs and reboots. It has no host filesystem mounts or tailnet access. +The firewall permits public IPv4 egress and blocks the host and private networks. + +Import `moq.nixosModules.github-runner-vm` into an x86\_64 NixOS host: + +```nix +services.github-runner-vm = { + enable = true; + name = "queue-ci"; + runner = { + url = "https://github.com/example/project"; + name = "build-host-queue"; + labels = [ "merge-queue" ]; + }; + network.uplink = "eth0"; + tokenFile = "/var/lib/secrets/queue-runner.token"; + resources = { cores = 8; memory = 16384; disk = 192; }; + guestConfig = { + users.users.root.openssh.authorizedKeys.keyFiles = [ ./deploy.pub ]; + nix.settings = { + extra-substituters = [ "https://kixelated.cachix.org" ]; + extra-trusted-public-keys = [ + "kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk=" + ]; + }; + }; +}; +``` + +Enable virtualization in firmware and load the CPU's KVM module before activation. +`/dev/kvm` must exist. Reserve `10.231.0.0/30` for the private VM link. The host is +`10.231.0.1`; the guest is `10.231.0.2`. NetworkManager owns the host uplink when +enabled; otherwise configure that uplink in networkd. The TAP receives its address +before QEMU attaches, so guest startup does not depend on TAP carrier. + +Mint a repository registration token with `gh api -X POST +repos/OWNER/REPO/actions/runners/registration-token --jq .token`, and install it +at `tokenFile` with mode `0600` in a root-only directory. Runtime credentials reach +the guest through QEMU firmware configuration; secret contents never enter the +Nix store. Keep the expired token file after registration. The guest retains its +own long-lived credential; wiping its state requires a fresh registration token. +Restart the host VM service after replacing the token. + +The host service and disk are named by `name`: the example uses `queue-ci.service` +and `/var/lib/queue-ci/root.qcow2`. Its disk is created once. Changing the configured +image does not replace existing state, and changing `resources.disk` does not +resize an existing disk. For guest updates, expose the evaluated guest through +the host flake: + +```nix +nixosConfigurations.queue-ci = + nixosConfigurations.build-host.config.services.github-runner-vm.guest; +``` + +Then deploy with `nixos-rebuild switch --flake .#queue-ci --target-host +root@10.231.0.2`, using the dedicated deployment key. Host rebuilds update the VM +launcher. Back up the deployment private key separately. + +The runner's home is `/var/cache/github-runner-home`; mbx uses +`/var/cache/github-runner-home/.cache/mbx`. Workflows must select mbx's local backend +on this runner. There is no cache-warming service or cache server. Persistent job +state is shared with subsequent jobs, so route only trusted work to the runner. +For MoQ, set `MOQ_QUEUE_RUNNER` to the runner label after a representative CI run +passes; pull requests stay on hosted runners. + +`just gh runner-vm` builds a guest image and runs NixOS integration tests for +startup without carrier, public egress, private-network isolation, and Nix/mbx +persistence across runner and guest restarts. diff --git a/nix/runner/checks.nix b/nix/runner/checks.nix new file mode 100644 index 00000000000..e54f51e111e --- /dev/null +++ b/nix/runner/checks.nix @@ -0,0 +1,23 @@ +{ pkgs }: +let + guestModule = import ./guest.nix { + runner = { + url = "https://github.com/example/queue"; + name = "test-queue"; + labels = [ "queue" ]; + }; + resources = { + cores = 2; + memory = 1024; + disk = 16; + }; + }; + guest = import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ guestModule ]; + }; +in +{ + github-runner-vm = import ./test.nix { inherit pkgs guestModule; }; + github-runner-vm-image = guest.config.system.build.image; +} diff --git a/nix/runner/guest.nix b/nix/runner/guest.nix new file mode 100644 index 00000000000..431882aa174 --- /dev/null +++ b/nix/runner/guest.nix @@ -0,0 +1,137 @@ +{ runner, resources }: +{ + config, + lib, + pkgs, + modulesPath, + ... +}: + +{ + imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; + + networking = { + hostName = "github-runner"; + useDHCP = false; + usePredictableInterfaceNames = false; + interfaces.eth0.ipv4.addresses = [ + { + address = "10.231.0.2"; + prefixLength = 30; + } + ]; + defaultGateway = "10.231.0.1"; + nameservers = [ + "1.1.1.1" + "8.8.8.8" + ]; + enableIPv6 = false; + }; + + boot = { + loader.grub.device = "/dev/vda"; + kernelParams = [ "console=ttyS0" ]; + kernelModules = [ "qemu_fw_cfg" ]; + growPartition = true; + }; + fileSystems."/" = { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + autoResize = true; + }; + system.stateVersion = "26.05"; + + system.build.image = import (modulesPath + "/../lib/make-disk-image.nix") { + inherit config lib pkgs; + format = "qcow2"; + partitionTableType = "legacy"; + diskSize = "auto"; + additionalSpace = "1G"; + copyChannel = false; + }; + + services.openssh = { + enable = true; + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + PermitRootLogin = "prohibit-password"; + }; + }; + programs.nix-ld.enable = true; + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + max-jobs = 2; + cores = resources.cores; + trusted-users = [ "root" ]; + }; + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 30d"; + }; + }; + + # Only a short-lived registration token enters the VM. The host's deployment + # key and private services are never mounted in the guest. + systemd.services.runner-token = { + wantedBy = [ "multi-user.target" ]; + before = [ "github-runner-runner.service" ]; + serviceConfig.Type = "oneshot"; + script = '' + install -D -m 0600 \ + /sys/firmware/qemu_fw_cfg/by_name/opt/io.systemd.credentials/runner-token/raw \ + /var/lib/secrets/github-runner.token + ''; + }; + + services.github-runners.runner = { + enable = true; + url = runner.url; + name = runner.name; + extraLabels = runner.labels; + tokenFile = "/var/lib/secrets/github-runner.token"; + tokenType = "registration"; + replace = true; + workDir = "/var/cache/github-runner-work"; + extraPackages = with pkgs; [ + curl + just + openssh + unzip + which + ]; + extraEnvironment = { + HOME = "/var/cache/github-runner-home"; + MBX_CACHE_DIR = "/var/cache/github-runner-home/.cache/mbx"; + CARGO_BUILD_JOBS = toString resources.cores; + }; + serviceOverrides = { + CacheDirectory = [ + "github-runner-home" + "github-runner-work" + ]; + CacheDirectoryMode = "0700"; + BindPaths = lib.mkForce [ ]; + ProtectHome = true; + RestrictNamespaces = false; + PrivateUsers = false; + SystemCallFilter = lib.mkForce [ ]; + TasksMax = 16384; + }; + }; + systemd.services.github-runner-runner = { + requires = [ "runner-token.service" ]; + after = [ "runner-token.service" ]; + }; + + environment.systemPackages = with pkgs; [ + git + just + ]; +} diff --git a/nix/runner/stop.py b/nix/runner/stop.py new file mode 100644 index 00000000000..c18f5b7b0ac --- /dev/null +++ b/nix/runner/stop.py @@ -0,0 +1,18 @@ +"""Ask QEMU to shut down the guest before systemd stops the VM process.""" + +import asyncio +import sys + +from qemu.qmp import QMPClient + + +async def main(): + client = QMPClient("github-runner-vm") + await client.connect(sys.argv[1]) + try: + await client.execute("system_powerdown") + finally: + await client.disconnect() + + +asyncio.run(main()) diff --git a/nix/runner/test.nix b/nix/runner/test.nix new file mode 100644 index 00000000000..104d893c39b --- /dev/null +++ b/nix/runner/test.nix @@ -0,0 +1,146 @@ +{ pkgs, guestModule }: + +pkgs.testers.runNixOSTest { + name = "github-runner-vm"; + nodes = { + startup = { lib, ... }: { + imports = [ ../modules/github-runner-vm.nix ]; + services.github-runner-vm = { + enable = true; + name = "queue-ci"; + runner.url = "https://github.com/example/queue"; + network.uplink = "uplink0"; + guestConfig.system.build.image = lib.mkForce pkgs.emptyDirectory; + }; + systemd.services.queue-ci.enable = false; + networking.networkmanager.enable = true; + networking.networkmanager.unmanaged = [ "interface-name:eth0" ]; + networking.networkmanager.ensureProfiles.profiles.uplink = { + connection = { + id = "uplink"; + type = "ethernet"; + interface-name = "eth1"; + }; + ipv4 = { + method = "manual"; + address1 = "192.168.1.4/24"; + }; + ipv6.method = "disabled"; + }; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; + networking.interfaces.eth1.ipv6.addresses = lib.mkForce [ ]; + networking.useDHCP = false; + virtualisation.memorySize = 768; + systemd.targets.network-online.wantedBy = [ "multi-user.target" ]; + }; + router = { lib, ... }: { + imports = [ ../modules/github-runner-vm.nix ]; + services.github-runner-vm = { + enable = true; + name = "queue-ci"; + runner.url = "https://github.com/example/queue"; + network.uplink = "uplink0"; + guestConfig.system.build.image = lib.mkForce pkgs.emptyDirectory; + }; + systemd.services.queue-ci.enable = false; + # The test driver supplies the two network links instead of QEMU's TAP. + systemd.network.netdevs = lib.mkForce { }; + networking.networkmanager.enable = false; + # The test driver configures the uplink outside networkd. + systemd.network.wait-online.enable = false; + networking.useDHCP = false; + virtualisation.vlans = [ + 1 + 2 + ]; + networking.firewall.allowedTCPPorts = [ 8080 ]; + environment.systemPackages = [ pkgs.curl ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; + }; + guest = { lib, ... }: { + imports = [ guestModule ]; + virtualisation.memorySize = 1024; + virtualisation.writableStoreUseTmpfs = false; + boot.growPartition = lib.mkForce false; + networking.interfaces.eth1.ipv4.addresses = [ + { + address = "10.231.0.2"; + prefixLength = 30; + } + ]; + networking.interfaces.eth0.ipv4.addresses = lib.mkForce [ ]; + virtualisation.qemu.options = [ + "-fw_cfg name=opt/io.systemd.credentials/runner-token,file=${pkgs.writeText "runner-token" "fixture"}" + ]; + systemd.services.github-runner-runner.serviceConfig = { + ExecStartPre = lib.mkForce [ + "+${pkgs.coreutils}/bin/touch /var/lib/github-runner/runner/.current-token" + ]; + ExecStart = lib.mkForce ( + pkgs.writeShellScript "cache-probe" '' + mkdir -p "$MBX_CACHE_DIR" + if test -f "$MBX_CACHE_DIR/marker"; then + touch "$MBX_CACHE_DIR/reused" + fi + touch "$MBX_CACHE_DIR/marker" + exec ${pkgs.coreutils}/bin/sleep infinity + '' + ); + }; + environment.systemPackages = [ pkgs.curl ]; + }; + internet = { + virtualisation.vlans = [ 2 ]; + networking.firewall.allowedTCPPorts = [ 8080 ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; + }; + }; + testScript = '' + start_all() + with subtest("the carrier-less TAP is configured without blocking the host uplink"): + startup.wait_for_unit("network-online.target") + startup.succeed("systemctl is-active NetworkManager-wait-online.service") + startup.fail("systemctl is-enabled systemd-networkd-wait-online.service") + startup.succeed("test $(cat /sys/class/net/tap-queue-ci/carrier) = 0") + startup.succeed("ip -4 -o address show dev tap-queue-ci | grep -F '10.231.0.1/30'") + + router.wait_for_unit("nftables.service") + router.wait_for_unit("http.service") + internet.wait_for_unit("http.service") + guest.wait_for_unit("github-runner-runner.service") + router.wait_for_open_port(8080) + internet.wait_for_open_port(8080) + + router.succeed("ip addr flush dev eth1; ip link set eth1 down; ip link set eth1 name tap-queue-ci; ip link set tap-queue-ci up") + router.wait_until_succeeds("ip -4 -o address show dev tap-queue-ci | grep -F '10.231.0.1/30'") + router.succeed("ip link set eth2 down; ip link set eth2 name uplink0; ip addr flush dev uplink0; ip addr add 192.168.2.1/24 dev uplink0; ip link set uplink0 up") + internet.succeed("ip addr flush dev eth1; ip addr add 192.168.2.2/24 dev eth1; ip addr add 203.0.113.1/32 dev eth1; ip addr add 100.100.100.1/32 dev eth1") + router.succeed("ip route add 203.0.113.1/32 dev uplink0; ip route add 100.100.100.1/32 dev uplink0") + guest.succeed("ip route replace default via 10.231.0.1 dev eth1") + + with subtest("public traffic works, host, LAN and tailnet access are blocked"): + router.succeed("curl --fail http://192.168.2.2:8080/") + guest.succeed("curl --fail --connect-timeout 5 http://203.0.113.1:8080/") + for address in ["10.231.0.1", "192.168.2.2", "100.100.100.1"]: + guest.fail(f"curl --fail --connect-timeout 1 http://{address}:8080/") + router.succeed("nft list table inet github-runner-vm") + + with subtest("local Nix and mbx storage survive runner and VM restarts"): + guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/marker") + guest.succeed("echo cached > /tmp/nix-cache-marker") + store_path = guest.succeed("nix-store --add /tmp/nix-cache-marker").strip() + guest.succeed("systemctl restart github-runner-runner") + guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/reused") + guest.shutdown() + guest.start() + guest.wait_for_unit("github-runner-runner.service") + guest.succeed("test -f /var/cache/github-runner-home/.cache/mbx/reused") + guest.succeed(f"test -f {store_path}") + ''; +} diff --git a/sh/gh/alert.sh b/sh/gh/alert.sh index 2bf7998b823..c6783b32e94 100755 --- a/sh/gh/alert.sh +++ b/sh/gh/alert.sh @@ -116,7 +116,7 @@ const files = (await Bun.stdin.text()).split("\n").filter(Boolean); // only by these needs no entry. const UNWATCHED = new Set(["pull_request", "pull_request_target", "merge_group", "workflow_call"]); // These push workflows validate branches or warm caches; none publishes. -const CI_ONLY = new Set(["Cache", "OBS", "Platform", "Quest"]); +const CI_ONLY = new Set(["Cache", "OBS", "Platform", "Quest", "Runner VM"]); const names = []; for (const file of files) { const doc = Bun.YAML.parse(await Bun.file(file).text()); diff --git a/sh/gh/alert.test.ts b/sh/gh/alert.test.ts index 6888fb03dca..d0382723854 100644 --- a/sh/gh/alert.test.ts +++ b/sh/gh/alert.test.ts @@ -46,7 +46,7 @@ test("cron workflows only alert on scheduled failures", async () => { }); test("ordinary CI never subscribes to Discord alerts", () => { - for (const name of ["Check", "Cache", "OBS", "Platform", "Quest"]) { + for (const name of ["Check", "Cache", "OBS", "Platform", "Quest", "Runner VM"]) { expect(watched).not.toContain(name); } }); From afaf35f67bc65c9a32a8b25eefb1ec2982975bbb Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:41:22 -0700 Subject: [PATCH 05/14] docs: reference the evaluated host through flake self Co-Authored-By: GPT-6 --- nix/runner/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/runner/README.md b/nix/runner/README.md index 108fe2d11ad..be1baf91247 100644 --- a/nix/runner/README.md +++ b/nix/runner/README.md @@ -48,12 +48,12 @@ Restart the host VM service after replacing the token. The host service and disk are named by `name`: the example uses `queue-ci.service` and `/var/lib/queue-ci/root.qcow2`. Its disk is created once. Changing the configured image does not replace existing state, and changing `resources.disk` does not -resize an existing disk. For guest updates, expose the evaluated guest through -the host flake: +resize an existing disk. For guest updates, expose the evaluated guest in the +host flake's outputs (`{ self, ... }:`): ```nix nixosConfigurations.queue-ci = - nixosConfigurations.build-host.config.services.github-runner-vm.guest; + self.nixosConfigurations.build-host.config.services.github-runner-vm.guest; ``` Then deploy with `nixos-rebuild switch --flake .#queue-ci --target-host From fb6c0696f7f208879265ee8c5117ff66da011b9a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:44:50 -0700 Subject: [PATCH 06/14] ci: allow the runner to lock 64 MiB for ring tests Match the locked-memory budget used by hosted CI and verify the actual limit inherited by the guest runner process in the integration suite. Co-Authored-By: GPT-6 --- nix/runner/guest.nix | 2 ++ nix/runner/test.nix | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/nix/runner/guest.nix b/nix/runner/guest.nix index 431882aa174..308021cf0bf 100644 --- a/nix/runner/guest.nix +++ b/nix/runner/guest.nix @@ -123,6 +123,8 @@ PrivateUsers = false; SystemCallFilter = lib.mkForce [ ]; TasksMax = 16384; + # Concurrent io_uring tests share the runner service's locked-memory budget. + LimitMEMLOCK = "64M"; }; }; systemd.services.github-runner-runner = { diff --git a/nix/runner/test.nix b/nix/runner/test.nix index 104d893c39b..1376f899fc8 100644 --- a/nix/runner/test.nix +++ b/nix/runner/test.nix @@ -82,6 +82,7 @@ pkgs.testers.runNixOSTest { ExecStart = lib.mkForce ( pkgs.writeShellScript "cache-probe" '' mkdir -p "$MBX_CACHE_DIR" + ulimit -l > "$MBX_CACHE_DIR/memlock" if test -f "$MBX_CACHE_DIR/marker"; then touch "$MBX_CACHE_DIR/reused" fi @@ -131,6 +132,10 @@ pkgs.testers.runNixOSTest { guest.fail(f"curl --fail --connect-timeout 1 http://{address}:8080/") router.succeed("nft list table inet github-runner-vm") + with subtest("the runner inherits the hosted CI locked-memory budget"): + guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/memlock") + guest.succeed("test $(cat /var/cache/github-runner-home/.cache/mbx/memlock) = 65536") + with subtest("local Nix and mbx storage survive runner and VM restarts"): guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/marker") guest.succeed("echo cached > /tmp/nix-cache-marker") From c5ab968737924c990e151ed765cfed9a5ff97e96 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:45:16 -0700 Subject: [PATCH 07/14] style(ci): format the runner integration test Co-Authored-By: GPT-6 --- nix/runner/test.nix | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nix/runner/test.nix b/nix/runner/test.nix index 1376f899fc8..0e8c18dafb3 100644 --- a/nix/runner/test.nix +++ b/nix/runner/test.nix @@ -81,13 +81,13 @@ pkgs.testers.runNixOSTest { ]; ExecStart = lib.mkForce ( pkgs.writeShellScript "cache-probe" '' - mkdir -p "$MBX_CACHE_DIR" - ulimit -l > "$MBX_CACHE_DIR/memlock" - if test -f "$MBX_CACHE_DIR/marker"; then - touch "$MBX_CACHE_DIR/reused" - fi - touch "$MBX_CACHE_DIR/marker" - exec ${pkgs.coreutils}/bin/sleep infinity + mkdir -p "$MBX_CACHE_DIR" + ulimit -l > "$MBX_CACHE_DIR/memlock" + if test -f "$MBX_CACHE_DIR/marker"; then + touch "$MBX_CACHE_DIR/reused" + fi + touch "$MBX_CACHE_DIR/marker" + exec ${pkgs.coreutils}/bin/sleep infinity '' ); }; From a9b7f84c7685e7889626bae206f03118c4d253bb Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:09:29 -0700 Subject: [PATCH 08/14] ci: support isolated runner VMs for multiple repositories Configure named instances with separate users, disks, runtime credentials, and /30 links. Reject duplicate network IDs. Verify both guests reach public endpoints while host, private networks, and sibling guest access remain blocked; preserve seeded local caches across runner and VM restarts. Co-Authored-By: GPT-6 --- .github/justfile | 2 +- nix/modules/github-runner-vm.nix | 486 +++++++++++++++++-------------- nix/runner/README.md | 39 ++- nix/runner/checks.nix | 5 + nix/runner/config-test.nix | 45 +++ nix/runner/guest.nix | 10 +- nix/runner/test.nix | 114 ++++++-- 7 files changed, 438 insertions(+), 263 deletions(-) create mode 100644 nix/runner/config-test.nix diff --git a/.github/justfile b/.github/justfile index 4d84f1293cc..a5b90b330ec 100644 --- a/.github/justfile +++ b/.github/justfile @@ -2,7 +2,7 @@ set working-directory := '..' # Build and test the persistent NixOS runner VM on x86_64 Linux. runner-vm: - nix build --no-link .#checks.x86_64-linux.github-runner-vm .#checks.x86_64-linux.github-runner-vm-image --print-build-logs + nix build --no-link .#checks.x86_64-linux.github-runner-vm .#checks.x86_64-linux.github-runner-vm-image .#checks.x86_64-linux.github-runner-vm-config --print-build-logs # Lint GitHub Actions workflows, their alert coverage, and that every step runs a recipe. check: diff --git a/nix/modules/github-runner-vm.nix b/nix/modules/github-runner-vm.nix index 2e6d514a89d..8a48aae104d 100644 --- a/nix/modules/github-runner-vm.nix +++ b/nix/modules/github-runner-vm.nix @@ -5,239 +5,285 @@ utils, ... }: - let - cfg = config.services.github-runner-vm; - tap = "tap-${cfg.name}"; - user = cfg.name; - guest = import (pkgs.path + "/nixos/lib/eval-config.nix") { - system = pkgs.stdenv.hostPlatform.system; - modules = [ - (import ../runner/guest.nix { inherit (cfg) runner resources; }) - cfg.guestConfig - ]; - }; - device = "${utils.escapeSystemdPath "/sys/subsystem/net/devices/${tap}"}.device"; - qemu = pkgs.qemu_kvm; - stop = pkgs.writeShellScript "github-runner-vm-stop" '' - set -euo pipefail - ${ - pkgs.python3.withPackages (p: [ p.qemu-qmp ]) - }/bin/python3 ${../runner/stop.py} "$RUNTIME_DIRECTORY/qmp" - ${pkgs.coreutils}/bin/tail --pid="$MAINPID" -f /dev/null - ''; -in -{ - options.services.github-runner-vm = { - enable = lib.mkEnableOption "a GitHub Actions runner in a persistent KVM guest"; - name = lib.mkOption { - type = lib.types.strMatching "[a-z][a-z0-9-]{0,9}"; - default = "github-ci"; - description = "Name of the host service, state directory, and VM user."; - }; - runner = { - url = lib.mkOption { - type = lib.types.str; - description = "GitHub repository URL registered to the runner."; - }; - name = lib.mkOption { - type = lib.types.str; - default = "${config.networking.hostName}-${cfg.name}"; - description = "Runner name advertised to GitHub."; - }; - labels = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - description = "Additional labels advertised to GitHub."; - }; - }; - network.uplink = lib.mkOption { - type = lib.types.strMatching "[a-zA-Z0-9_.:-]{1,15}"; - description = "Host interface used for public IPv4 egress."; + hostConfig = config; + enabled = lib.filterAttrs (_: cfg: cfg.enable) config.services.github-runner-vms; + ids = map (cfg: cfg.network.id) (lib.attrValues enabled); + guestFor = + cfg: + import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + (import ../runner/guest.nix { + inherit (cfg) runner resources; + network = { + hostAddress = "10.231.${toString cfg.network.id}.1"; + guestAddress = "10.231.${toString cfg.network.id}.2"; + }; + }) + cfg.guestConfig + ]; }; - resources = { - cores = lib.mkOption { - type = lib.types.ints.positive; - default = 4; - description = "Virtual CPUs and maximum host CPU usage in cores."; - }; - memory = lib.mkOption { - type = lib.types.ints.positive; - default = 8192; - description = "Guest memory in MiB, with 2 GiB extra allowed for QEMU."; - }; - disk = lib.mkOption { - type = lib.types.ints.positive; - default = 128; - description = "Initial persistent disk capacity in GiB."; + render = + name: cfg: + let + tap = "tap-${name}"; + user = name; + prefix = "10.231.${toString cfg.network.id}"; + guest = cfg.guest; + device = "${utils.escapeSystemdPath "/sys/subsystem/net/devices/${tap}"}.device"; + qemu = pkgs.qemu_kvm; + stop = pkgs.writeShellScript "github-runner-vm-stop" '' + set -euo pipefail + ${ + pkgs.python3.withPackages (p: [ p.qemu-qmp ]) + }/bin/python3 ${../runner/stop.py} "$RUNTIME_DIRECTORY/qmp" + ${pkgs.coreutils}/bin/tail --pid="$MAINPID" -f /dev/null + ''; + in + { + assertions = [ + { + assertion = builtins.match "[a-z][a-z0-9-]{0,9}" name != null; + message = "Runner VM names must start with a lowercase letter and contain at most 10 lowercase letters, digits, or hyphens."; + } + { + assertion = pkgs.stdenv.hostPlatform.system == "x86_64-linux"; + message = "services.github-runner-vms supports x86_64-linux KVM hosts."; + } + ]; + users.groups.${user} = { }; + users.users.${user} = { + isSystemUser = true; + group = user; + extraGroups = [ "kvm" ]; }; - }; - tokenFile = lib.mkOption { - type = lib.types.str; - default = "/var/lib/secrets/github-runner-vm.token"; - description = "Host path to the runtime GitHub registration token."; - }; - guestConfig = lib.mkOption { - type = lib.types.deferredModule; - default = { }; - description = "Extra NixOS guest configuration, including SSH keys and binary caches."; - }; - guest = lib.mkOption { - type = lib.types.raw; - readOnly = true; - description = "Evaluated guest system for deployment as a NixOS flake configuration."; - }; - }; - - config = lib.mkIf cfg.enable { - services.github-runner-vm.guest = guest; - assertions = [ - { - assertion = pkgs.stdenv.hostPlatform.system == "x86_64-linux"; - message = "services.github-runner-vm supports x86_64-linux KVM hosts."; - } - ]; - users.groups.${user} = { }; - users.users.${user} = { - isSystemUser = true; - group = user; - extraGroups = [ "kvm" ]; - }; - networking.networkmanager.unmanaged = [ "interface-name:${tap}" ]; - systemd.network = { - enable = true; - # When NetworkManager owns the uplink, it also owns online readiness. - # The TAP cannot gain carrier until QEMU starts. - wait-online.enable = lib.mkIf config.networking.networkmanager.enable false; - netdevs."30-github-runner" = { - netdevConfig = { - Kind = "tap"; - Name = tap; + networking.networkmanager.unmanaged = [ "interface-name:${tap}" ]; + systemd.network = { + enable = true; + # When NetworkManager owns the uplink, it also owns online readiness. + # The TAP cannot gain carrier until QEMU starts. + wait-online.enable = lib.mkIf config.networking.networkmanager.enable false; + netdevs."30-${name}" = { + netdevConfig = { + Kind = "tap"; + Name = tap; + }; + tapConfig.User = user; }; - tapConfig.User = user; - }; - networks."30-github-runner" = { - matchConfig.Name = tap; - address = [ "10.231.0.1/30" ]; - linkConfig.RequiredForOnline = "no"; - networkConfig = { - LinkLocalAddressing = "no"; - ConfigureWithoutCarrier = true; + networks."30-${name}" = { + matchConfig.Name = tap; + address = [ "${prefix}.1/30" ]; + linkConfig.RequiredForOnline = "no"; + networkConfig = { + LinkLocalAddressing = "no"; + ConfigureWithoutCarrier = true; + }; }; }; - }; - networking.nat = { - enable = true; - externalInterface = cfg.network.uplink; - internalInterfaces = [ tap ]; - }; - networking.firewall.filterForward = true; - networking.nftables = { - enable = true; - tables.github-runner-vm = { - family = "inet"; - content = '' - chain input { - type filter hook input priority -10; policy accept; - iifname "${tap}" ct state established,related accept - iifname "${tap}" counter drop - } - chain forward { - type filter hook forward priority -10; policy accept; - iifname "${tap}" meta nfproto ipv6 counter drop - iifname "${tap}" ip saddr != 10.231.0.2 counter drop - iifname "${tap}" ip daddr { - 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, - 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, - 198.18.0.0/15, 224.0.0.0/4, 240.0.0.0/4 - } counter drop - iifname "${tap}" oifname != "${cfg.network.uplink}" counter drop - oifname "${tap}" ct state established,related accept - oifname "${tap}" counter drop - } - ''; + networking.nat = { + enable = true; + internalInterfaces = [ tap ]; + }; + networking.firewall.filterForward = true; + networking.nftables = { + enable = true; + tables."github-runner-${name}" = { + family = "inet"; + content = '' + chain input { + type filter hook input priority -10; policy accept; + iifname "${tap}" ct state established,related accept + iifname "${tap}" counter drop + } + chain forward { + type filter hook forward priority -10; policy accept; + iifname "${tap}" meta nfproto ipv6 counter drop + iifname "${tap}" ip saddr != ${prefix}.2 counter drop + iifname "${tap}" ip daddr { + 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, + 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, + 198.18.0.0/15, 224.0.0.0/4, 240.0.0.0/4 + } counter drop + iifname "${tap}" oifname != "${cfg.network.uplink}" counter drop + oifname "${tap}" ct state established,related accept + oifname "${tap}" counter drop + } + ''; + }; }; - }; - systemd.services.${cfg.name} = { - description = "Persistent GitHub Actions runner VM"; - wantedBy = [ "multi-user.target" ]; - wants = [ "network-online.target" ]; - bindsTo = [ - "nftables.service" - device - ]; - after = [ - "network-online.target" - "nftables.service" - device - ]; - path = [ qemu ]; - preStart = '' - if ! test -e /dev/kvm; then - echo "KVM is unavailable. Enable virtualization in firmware and load the CPU KVM module." >&2 - exit 1 - fi - if ! test -r /dev/kvm || ! test -w /dev/kvm; then - echo "The ${user} user needs read and write access to /dev/kvm through the kvm group." >&2 - exit 1 - fi - if ! test -e "$STATE_DIRECTORY/root.qcow2"; then - cp --reflink=auto ${guest.config.system.build.image}/nixos.qcow2 "$STATE_DIRECTORY/root.qcow2.tmp" - chmod 0600 "$STATE_DIRECTORY/root.qcow2.tmp" - qemu-img resize "$STATE_DIRECTORY/root.qcow2.tmp" ${toString cfg.resources.disk}G - mv "$STATE_DIRECTORY/root.qcow2.tmp" "$STATE_DIRECTORY/root.qcow2" - fi - ''; - script = '' - exec ${qemu}/bin/qemu-system-x86_64 \ - -enable-kvm -cpu host -smp ${toString cfg.resources.cores} -m ${toString cfg.resources.memory} \ - -drive file="$STATE_DIRECTORY/root.qcow2",format=qcow2,if=virtio,cache=none \ - -netdev tap,id=net,ifname=${tap},script=no,downscript=no \ - -device virtio-net-pci,netdev=net,mac=02:00:00:00:23:02 \ - -device virtio-rng-pci \ - -fw_cfg name=opt/io.systemd.credentials/runner-token,file="$CREDENTIALS_DIRECTORY/runner-token" \ - -qmp unix:"$RUNTIME_DIRECTORY/qmp",server=on,wait=off \ - -nographic - ''; - serviceConfig = { - User = user; - Group = user; - StateDirectory = cfg.name; - StateDirectoryMode = "0700"; - RuntimeDirectory = cfg.name; - RuntimeDirectoryMode = "0700"; - LoadCredential = [ "runner-token:${cfg.tokenFile}" ]; - ExecStop = stop; - TimeoutStopSec = 120; - ProtectHome = true; - ProtectSystem = "strict"; - PrivateTmp = true; - NoNewPrivileges = true; - CapabilityBoundingSet = ""; - DevicePolicy = "closed"; - DeviceAllow = [ - "/dev/kvm rw" - "/dev/net/tun rw" + systemd.services.${name} = { + description = "Persistent GitHub Actions runner VM"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + bindsTo = [ + "nftables.service" + device ]; - # QEMU queries the TAP's MTU through an IP socket; guest packets use the - # TAP descriptor, so denying host IP traffic does not block guest NAT. - RestrictAddressFamilies = [ - "AF_UNIX" - "AF_INET" - "AF_INET6" - "AF_NETLINK" + after = [ + "network-online.target" + "nftables.service" + device ]; - IPAddressDeny = "any"; - Nice = 10; - CPUWeight = 10; - IOWeight = 10; - CPUQuota = "${toString (cfg.resources.cores * 100)}%"; - MemoryMax = "${toString (cfg.resources.memory + 2048)}M"; - TasksMax = 1024; + path = [ qemu ]; + preStart = '' + if ! test -e /dev/kvm; then + echo "KVM is unavailable. Enable virtualization in firmware and load the CPU KVM module." >&2 + exit 1 + fi + if ! test -r /dev/kvm || ! test -w /dev/kvm; then + echo "The ${user} user needs read and write access to /dev/kvm through the kvm group." >&2 + exit 1 + fi + if ! test -e "$STATE_DIRECTORY/root.qcow2"; then + cp --reflink=auto ${guest.config.system.build.image}/nixos.qcow2 "$STATE_DIRECTORY/root.qcow2.tmp" + chmod 0600 "$STATE_DIRECTORY/root.qcow2.tmp" + qemu-img resize "$STATE_DIRECTORY/root.qcow2.tmp" ${toString cfg.resources.disk}G + mv "$STATE_DIRECTORY/root.qcow2.tmp" "$STATE_DIRECTORY/root.qcow2" + fi + ''; + script = '' + exec ${qemu}/bin/qemu-system-x86_64 \ + -enable-kvm -cpu host -smp ${toString cfg.resources.cores} -m ${toString cfg.resources.memory} \ + -drive file="$STATE_DIRECTORY/root.qcow2",format=qcow2,if=virtio,cache=none \ + -netdev tap,id=net,ifname=${tap},script=no,downscript=no \ + -device virtio-net-pci,netdev=net,mac=02:00:00:00:23:02 \ + -device virtio-rng-pci \ + -fw_cfg name=opt/io.systemd.credentials/runner-token,file="$CREDENTIALS_DIRECTORY/runner-token" \ + -qmp unix:"$RUNTIME_DIRECTORY/qmp",server=on,wait=off \ + -nographic + ''; + serviceConfig = { + User = user; + Group = user; + StateDirectory = name; + StateDirectoryMode = "0700"; + RuntimeDirectory = name; + RuntimeDirectoryMode = "0700"; + LoadCredential = [ "runner-token:${cfg.tokenFile}" ]; + ExecStop = stop; + TimeoutStopSec = 120; + ProtectHome = true; + ProtectSystem = "strict"; + PrivateTmp = true; + NoNewPrivileges = true; + CapabilityBoundingSet = ""; + DevicePolicy = "closed"; + DeviceAllow = [ + "/dev/kvm rw" + "/dev/net/tun rw" + ]; + # QEMU queries the TAP's MTU through an IP socket; guest packets use the + # TAP descriptor, so denying host IP traffic does not block guest NAT. + RestrictAddressFamilies = [ + "AF_UNIX" + "AF_INET" + "AF_INET6" + "AF_NETLINK" + ]; + IPAddressDeny = "any"; + Nice = 10; + CPUWeight = 10; + IOWeight = 10; + CPUQuota = "${toString (cfg.resources.cores * 100)}%"; + MemoryMax = "${toString (cfg.resources.memory + 2048)}M"; + TasksMax = 1024; + }; }; + }; +in +{ + options.services.github-runner-vms = lib.mkOption { + default = { }; + description = "Persistent GitHub Actions runner VMs isolated from the host and each other."; + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, config, ... }: { + options = { + enable = lib.mkEnableOption "a GitHub Actions runner in a persistent KVM guest"; + runner = { + url = lib.mkOption { + type = lib.types.str; + description = "GitHub repository URL registered to the runner."; + }; + name = lib.mkOption { + type = lib.types.str; + default = "${hostConfig.networking.hostName}-${name}"; + description = "Runner name advertised to GitHub."; + }; + labels = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Additional labels advertised to GitHub."; + }; + }; + network.uplink = lib.mkOption { + type = lib.types.strMatching "[a-zA-Z0-9_.:-]{1,15}"; + description = "Host interface used for public IPv4 egress."; + }; + network.id = lib.mkOption { + type = lib.types.ints.u8; + default = 0; + description = "Unique link number using the private 10.231.ID.0/30 subnet."; + }; + resources = { + cores = lib.mkOption { + type = lib.types.ints.positive; + default = 4; + description = "Virtual CPUs and maximum host CPU usage in cores."; + }; + memory = lib.mkOption { + type = lib.types.ints.positive; + default = 8192; + description = "Guest memory in MiB, with 2 GiB extra allowed for QEMU."; + }; + disk = lib.mkOption { + type = lib.types.ints.positive; + default = 128; + description = "Initial persistent disk capacity in GiB."; + }; + }; + tokenFile = lib.mkOption { + type = lib.types.str; + default = "/var/lib/secrets/github-runner-${name}.token"; + description = "Host path to the runtime GitHub registration token."; + }; + guestConfig = lib.mkOption { + type = lib.types.deferredModule; + default = { }; + description = "Extra NixOS guest configuration, including SSH keys and binary caches."; + }; + guest = lib.mkOption { + type = lib.types.raw; + readOnly = true; + description = "Evaluated guest system for deployment as a NixOS flake configuration."; + }; + + }; + config.guest = lib.mkIf config.enable (guestFor config); + } + ) + ); }; + config = + let + instances = lib.mapAttrsToList render enabled; + in + { + assertions = [ + { + assertion = lib.length ids == lib.length (lib.unique ids); + message = "Each enabled services.github-runner-vms instance needs a unique network.id."; + } + ] + ++ lib.concatMap (instance: instance.assertions) instances; + users.users = lib.mkMerge (map (instance: instance.users.users) instances); + users.groups = lib.mkMerge (map (instance: instance.users.groups) instances); + networking = lib.mkMerge (map (instance: instance.networking) instances); + systemd = lib.mkMerge (map (instance: instance.systemd) instances); + }; } diff --git a/nix/runner/README.md b/nix/runner/README.md index be1baf91247..557f9371ba4 100644 --- a/nix/runner/README.md +++ b/nix/runner/README.md @@ -1,22 +1,21 @@ # Persistent GitHub Actions runner VM -The `github-runner-vm` NixOS module runs a repository's runner in a persistent -KVM guest. The guest keeps its Nix store, mbx cache, checkout, and runner state +The `github-runner-vm` NixOS module runs each repository's runner in its own +persistent KVM guest. The guest keeps its Nix store, mbx cache, checkout, and runner state across jobs and reboots. It has no host filesystem mounts or tailnet access. The firewall permits public IPv4 egress and blocks the host and private networks. Import `moq.nixosModules.github-runner-vm` into an x86\_64 NixOS host: ```nix -services.github-runner-vm = { +services.github-runner-vms.queue-ci = { enable = true; - name = "queue-ci"; runner = { url = "https://github.com/example/project"; name = "build-host-queue"; labels = [ "merge-queue" ]; }; - network.uplink = "eth0"; + network = { id = 0; uplink = "eth0"; }; tokenFile = "/var/lib/secrets/queue-runner.token"; resources = { cores = 8; memory = 16384; disk = 192; }; guestConfig = { @@ -31,9 +30,27 @@ services.github-runner-vm = { }; ``` +Add another named instance for another repository, with its own `network.id`, +registration token, and guest configuration. Every instance gets a separate user, +disk, runtime credentials, and TAP interface; no cache data is shared between +instances. For example: + +```nix +services.github-runner-vms.other-ci = { + enable = true; + runner.url = "https://github.com/example/another-project"; + network = { id = 1; uplink = "eth0"; }; + guestConfig.users.users.root.openssh.authorizedKeys.keyFiles = [ ./deploy.pub ]; +}; +``` + +The default token path is `/var/lib/secrets/github-runner-NAME.token`. +Renaming an instance selects a new disk; it does not migrate its existing state. + Enable virtualization in firmware and load the CPU's KVM module before activation. -`/dev/kvm` must exist. Reserve `10.231.0.0/30` for the private VM link. The host is -`10.231.0.1`; the guest is `10.231.0.2`. NetworkManager owns the host uplink when +`/dev/kvm` must exist. Each instance needs a unique `network.id` from 0 to 255. +It uses `10.231.ID.0/30`: the host is `.1` and the guest is `.2`. Reserve these +subnets on the host. Duplicate IDs are rejected during evaluation. NetworkManager owns the host uplink when enabled; otherwise configure that uplink in networkd. The TAP receives its address before QEMU attaches, so guest startup does not depend on TAP carrier. @@ -45,7 +62,7 @@ Nix store. Keep the expired token file after registration. The guest retains its own long-lived credential; wiping its state requires a fresh registration token. Restart the host VM service after replacing the token. -The host service and disk are named by `name`: the example uses `queue-ci.service` +The host service and disk are named by the instance attribute: the example uses `queue-ci.service` and `/var/lib/queue-ci/root.qcow2`. Its disk is created once. Changing the configured image does not replace existing state, and changing `resources.disk` does not resize an existing disk. For guest updates, expose the evaluated guest in the @@ -53,7 +70,7 @@ host flake's outputs (`{ self, ... }:`): ```nix nixosConfigurations.queue-ci = - self.nixosConfigurations.build-host.config.services.github-runner-vm.guest; + self.nixosConfigurations.build-host.config.services.github-runner-vms.queue-ci.guest; ``` Then deploy with `nixos-rebuild switch --flake .#queue-ci --target-host @@ -68,5 +85,5 @@ For MoQ, set `MOQ_QUEUE_RUNNER` to the runner label after a representative CI ru passes; pull requests stay on hosted runners. `just gh runner-vm` builds a guest image and runs NixOS integration tests for -startup without carrier, public egress, private-network isolation, and Nix/mbx -persistence across runner and guest restarts. +startup without carrier, public egress for multiple guests, private-network and +sibling-guest isolation, the runner's locked-memory budget, and Nix/mbx persistence across runner and guest restarts. diff --git a/nix/runner/checks.nix b/nix/runner/checks.nix index e54f51e111e..132e95c9d6b 100644 --- a/nix/runner/checks.nix +++ b/nix/runner/checks.nix @@ -6,6 +6,10 @@ let name = "test-queue"; labels = [ "queue" ]; }; + network = { + hostAddress = "10.231.0.1"; + guestAddress = "10.231.0.2"; + }; resources = { cores = 2; memory = 1024; @@ -20,4 +24,5 @@ in { github-runner-vm = import ./test.nix { inherit pkgs guestModule; }; github-runner-vm-image = guest.config.system.build.image; + github-runner-vm-config = import ./config-test.nix { inherit pkgs; }; } diff --git a/nix/runner/config-test.nix b/nix/runner/config-test.nix new file mode 100644 index 00000000000..0291ddcbf2e --- /dev/null +++ b/nix/runner/config-test.nix @@ -0,0 +1,45 @@ +{ pkgs }: +let + evaluate = + ids: + import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + ../modules/github-runner-vm.nix + { + boot.isContainer = true; + networking.useHostResolvConf = false; + system.stateVersion = "26.05"; + fileSystems."/" = { + device = "none"; + fsType = "tmpfs"; + }; + services.github-runner-vms = pkgs.lib.mapAttrs (_: id: { + enable = true; + runner.url = "https://github.com/example/project"; + network = { + inherit id; + uplink = "eth0"; + }; + }) ids; + } + ]; + }; + valid = evaluate { + first = 0; + second = 1; + }; + duplicate = evaluate { + first = 0; + second = 0; + }; +in +assert pkgs.lib.assertMsg (pkgs.lib.all (item: item.assertion) valid.config.assertions) ( + pkgs.lib.concatMapStringsSep "\n" (item: item.message) ( + builtins.filter (item: !item.assertion) valid.config.assertions + ) +); +assert pkgs.lib.any (item: !item.assertion) duplicate.config.assertions; +pkgs.runCommand "github-runner-vm-config-test" { } '' + touch "$out" +'' diff --git a/nix/runner/guest.nix b/nix/runner/guest.nix index 308021cf0bf..b72ccdd7091 100644 --- a/nix/runner/guest.nix +++ b/nix/runner/guest.nix @@ -1,4 +1,8 @@ -{ runner, resources }: +{ + runner, + resources, + network, +}: { config, lib, @@ -16,11 +20,11 @@ usePredictableInterfaceNames = false; interfaces.eth0.ipv4.addresses = [ { - address = "10.231.0.2"; + address = network.guestAddress; prefixLength = 30; } ]; - defaultGateway = "10.231.0.1"; + defaultGateway = network.hostAddress; nameservers = [ "1.1.1.1" "8.8.8.8" diff --git a/nix/runner/test.nix b/nix/runner/test.nix index 0e8c18dafb3..299565c56b5 100644 --- a/nix/runner/test.nix +++ b/nix/runner/test.nix @@ -1,18 +1,33 @@ { pkgs, guestModule }: +let + hostModule = { lib, ... }: { + imports = [ ../modules/github-runner-vm.nix ]; + services.github-runner-vms = + lib.mapAttrs + (name: id: { + enable = true; + runner.url = "https://github.com/example/${name}"; + network = { + inherit id; + uplink = "uplink0"; + }; + guestConfig.system.build.image = lib.mkForce pkgs.emptyDirectory; + }) + { + queue-ci = 0; + other-ci = 1; + }; + systemd.services = lib.genAttrs [ "queue-ci" "other-ci" ] (_: { + enable = false; + }); + }; +in pkgs.testers.runNixOSTest { name = "github-runner-vm"; nodes = { startup = { lib, ... }: { - imports = [ ../modules/github-runner-vm.nix ]; - services.github-runner-vm = { - enable = true; - name = "queue-ci"; - runner.url = "https://github.com/example/queue"; - network.uplink = "uplink0"; - guestConfig.system.build.image = lib.mkForce pkgs.emptyDirectory; - }; - systemd.services.queue-ci.enable = false; + imports = [ hostModule ]; networking.networkmanager.enable = true; networking.networkmanager.unmanaged = [ "interface-name:eth0" ]; networking.networkmanager.ensureProfiles.profiles.uplink = { @@ -34,16 +49,8 @@ pkgs.testers.runNixOSTest { systemd.targets.network-online.wantedBy = [ "multi-user.target" ]; }; router = { lib, ... }: { - imports = [ ../modules/github-runner-vm.nix ]; - services.github-runner-vm = { - enable = true; - name = "queue-ci"; - runner.url = "https://github.com/example/queue"; - network.uplink = "uplink0"; - guestConfig.system.build.image = lib.mkForce pkgs.emptyDirectory; - }; - systemd.services.queue-ci.enable = false; - # The test driver supplies the two network links instead of QEMU's TAP. + imports = [ hostModule ]; + # The test driver supplies the three network links instead of QEMU's TAP. systemd.network.netdevs = lib.mkForce { }; networking.networkmanager.enable = false; # The test driver configures the uplink outside networkd. @@ -52,6 +59,7 @@ pkgs.testers.runNixOSTest { virtualisation.vlans = [ 1 2 + 3 ]; networking.firewall.allowedTCPPorts = [ 8080 ]; environment.systemPackages = [ pkgs.curl ]; @@ -62,6 +70,11 @@ pkgs.testers.runNixOSTest { }; guest = { lib, ... }: { imports = [ guestModule ]; + networking.firewall.allowedTCPPorts = [ 8080 ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; virtualisation.memorySize = 1024; virtualisation.writableStoreUseTmpfs = false; boot.growPartition = lib.mkForce false; @@ -75,24 +88,54 @@ pkgs.testers.runNixOSTest { virtualisation.qemu.options = [ "-fw_cfg name=opt/io.systemd.credentials/runner-token,file=${pkgs.writeText "runner-token" "fixture"}" ]; + systemd.services.cache-seed-fixture = { + requiredBy = [ "github-runner-runner.service" ]; + before = [ "github-runner-runner.service" ]; + unitConfig.ConditionPathExists = "!/var/lib/cache-seeded"; + serviceConfig.Type = "oneshot"; + script = '' + install -d -m 0700 /var/cache/github-runner-home + echo seeded > /var/cache/github-runner-home/seeded + chmod 0600 /var/cache/github-runner-home/seeded + touch /var/lib/cache-seeded + ''; + }; systemd.services.github-runner-runner.serviceConfig = { ExecStartPre = lib.mkForce [ "+${pkgs.coreutils}/bin/touch /var/lib/github-runner/runner/.current-token" ]; ExecStart = lib.mkForce ( pkgs.writeShellScript "cache-probe" '' - mkdir -p "$MBX_CACHE_DIR" + test "$(cat "$HOME/seeded")" = seeded + mkdir -p "$MBX_CACHE_DIR" ulimit -l > "$MBX_CACHE_DIR/memlock" - if test -f "$MBX_CACHE_DIR/marker"; then - touch "$MBX_CACHE_DIR/reused" - fi - touch "$MBX_CACHE_DIR/marker" - exec ${pkgs.coreutils}/bin/sleep infinity + if test -f "$MBX_CACHE_DIR/marker"; then + touch "$MBX_CACHE_DIR/reused" + fi + touch "$MBX_CACHE_DIR/marker" + exec ${pkgs.coreutils}/bin/sleep infinity '' ); }; environment.systemPackages = [ pkgs.curl ]; }; + other = { + virtualisation.vlans = [ 3 ]; + networking.useDHCP = false; + networking.interfaces.eth1.ipv4.addresses = [ + { + address = "10.231.1.2"; + prefixLength = 30; + } + ]; + networking.defaultGateway = "10.231.1.1"; + networking.firewall.allowedTCPPorts = [ 8080 ]; + environment.systemPackages = [ pkgs.curl ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; + }; internet = { virtualisation.vlans = [ 2 ]; networking.firewall.allowedTCPPorts = [ 8080 ]; @@ -110,27 +153,42 @@ pkgs.testers.runNixOSTest { startup.fail("systemctl is-enabled systemd-networkd-wait-online.service") startup.succeed("test $(cat /sys/class/net/tap-queue-ci/carrier) = 0") startup.succeed("ip -4 -o address show dev tap-queue-ci | grep -F '10.231.0.1/30'") + startup.succeed("test $(cat /sys/class/net/tap-other-ci/carrier) = 0") + startup.succeed("ip -4 -o address show dev tap-other-ci | grep -F '10.231.1.1/30'") router.wait_for_unit("nftables.service") router.wait_for_unit("http.service") internet.wait_for_unit("http.service") + other.wait_for_unit("http.service") guest.wait_for_unit("github-runner-runner.service") + guest.wait_for_unit("http.service") router.wait_for_open_port(8080) internet.wait_for_open_port(8080) + other.wait_for_open_port(8080) + guest.wait_for_open_port(8080) router.succeed("ip addr flush dev eth1; ip link set eth1 down; ip link set eth1 name tap-queue-ci; ip link set tap-queue-ci up") router.wait_until_succeeds("ip -4 -o address show dev tap-queue-ci | grep -F '10.231.0.1/30'") + router.succeed("ip addr flush dev eth3; ip link set eth3 down; ip link set eth3 name tap-other-ci; ip link set tap-other-ci up") + router.wait_until_succeeds("ip -4 -o address show dev tap-other-ci | grep -F '10.231.1.1/30'") router.succeed("ip link set eth2 down; ip link set eth2 name uplink0; ip addr flush dev uplink0; ip addr add 192.168.2.1/24 dev uplink0; ip link set uplink0 up") internet.succeed("ip addr flush dev eth1; ip addr add 192.168.2.2/24 dev eth1; ip addr add 203.0.113.1/32 dev eth1; ip addr add 100.100.100.1/32 dev eth1") router.succeed("ip route add 203.0.113.1/32 dev uplink0; ip route add 100.100.100.1/32 dev uplink0") guest.succeed("ip route replace default via 10.231.0.1 dev eth1") - with subtest("public traffic works, host, LAN and tailnet access are blocked"): + with subtest("both guests reach public networks while host and sibling guests stay isolated"): + router.succeed("curl --fail http://10.231.1.2:8080/") + router.succeed("curl --fail http://10.231.0.2:8080/") + other.succeed("ip route replace default via 10.231.1.1 dev eth1") + other.succeed("curl --fail --connect-timeout 5 http://203.0.113.1:8080/") + other.fail("curl --fail --connect-timeout 1 http://10.231.0.1:8080/") + other.fail("curl --fail --connect-timeout 1 http://10.231.0.2:8080/") router.succeed("curl --fail http://192.168.2.2:8080/") guest.succeed("curl --fail --connect-timeout 5 http://203.0.113.1:8080/") - for address in ["10.231.0.1", "192.168.2.2", "100.100.100.1"]: + for address in ["10.231.0.1", "10.231.1.2", "192.168.2.2", "100.100.100.1"]: guest.fail(f"curl --fail --connect-timeout 1 http://{address}:8080/") - router.succeed("nft list table inet github-runner-vm") + router.succeed("nft list table inet github-runner-queue-ci") + router.succeed("nft list table inet github-runner-other-ci") with subtest("the runner inherits the hosted CI locked-memory budget"): guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/memlock") From 05e88f2697d68bbfa3d630cdc8d7834216f23c22 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:33:34 -0700 Subject: [PATCH 09/14] fix(ci): reject spoofed guest replies and conflicting host NAT Reproduce a sibling reply spoof with reverse-path filtering disabled, then validate guest sources before accepting established host traffic. Reject host NAT settings that would silently break runner egress. Co-Authored-By: GPT-6 --- nix/modules/github-runner-vm.nix | 13 ++++++++++++ nix/runner/README.md | 7 ++++++- nix/runner/config-test.nix | 14 ++++++++++--- nix/runner/spoof-test.py | 36 ++++++++++++++++++++++++++++++++ nix/runner/test.nix | 17 +++++++++++++++ 5 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 nix/runner/spoof-test.py diff --git a/nix/modules/github-runner-vm.nix b/nix/modules/github-runner-vm.nix index 8a48aae104d..082d5af4a1a 100644 --- a/nix/modules/github-runner-vm.nix +++ b/nix/modules/github-runner-vm.nix @@ -51,6 +51,16 @@ let assertion = pkgs.stdenv.hostPlatform.system == "x86_64-linux"; message = "services.github-runner-vms supports x86_64-linux KVM hosts."; } + { + assertion = + config.networking.nat.externalInterface == null + || config.networking.nat.externalInterface == cfg.network.uplink; + message = "Runner VM ${name} needs networking.nat.externalInterface unset or equal to its network.uplink."; + } + { + assertion = config.networking.nat.externalIP == null; + message = "Runner VMs require masquerading; networking.nat.externalIP must be unset."; + } ]; users.groups.${user} = { }; users.users.${user} = { @@ -95,6 +105,9 @@ let content = '' chain input { type filter hook input priority -10; policy accept; + # Conntrack is shared across TAPs, so validate the sender first. + iifname "${tap}" meta nfproto ipv6 counter drop + iifname "${tap}" ip saddr != ${prefix}.2 counter drop iifname "${tap}" ct state established,related accept iifname "${tap}" counter drop } diff --git a/nix/runner/README.md b/nix/runner/README.md index 557f9371ba4..be9270da4af 100644 --- a/nix/runner/README.md +++ b/nix/runner/README.md @@ -53,6 +53,9 @@ It uses `10.231.ID.0/30`: the host is `.1` and the guest is `.2`. Reserve these subnets on the host. Duplicate IDs are rejected during evaluation. NetworkManager owns the host uplink when enabled; otherwise configure that uplink in networkd. The TAP receives its address before QEMU attaches, so guest startup does not depend on TAP carrier. +The module uses the host's NixOS NAT service. Leave `networking.nat.externalInterface` +unset, or set it to the same uplink as every runner. Fixed `networking.nat.externalIP` +is unsupported. Conflicting settings are rejected during evaluation. Mint a repository registration token with `gh api -X POST repos/OWNER/REPO/actions/runners/registration-token --jq .token`, and install it @@ -86,4 +89,6 @@ passes; pull requests stay on hosted runners. `just gh runner-vm` builds a guest image and runs NixOS integration tests for startup without carrier, public egress for multiple guests, private-network and -sibling-guest isolation, the runner's locked-memory budget, and Nix/mbx persistence across runner and guest restarts. +sibling-guest isolation including spoofed established replies, compatible host NAT +settings, the runner's locked-memory budget, and Nix/mbx persistence across runner +and guest restarts. diff --git a/nix/runner/config-test.nix b/nix/runner/config-test.nix index 0291ddcbf2e..d4d50203bd1 100644 --- a/nix/runner/config-test.nix +++ b/nix/runner/config-test.nix @@ -1,7 +1,7 @@ { pkgs }: let evaluate = - ids: + ids: nat: import (pkgs.path + "/nixos/lib/eval-config.nix") { system = pkgs.stdenv.hostPlatform.system; modules = [ @@ -9,6 +9,7 @@ let { boot.isContainer = true; networking.useHostResolvConf = false; + networking.nat = nat; system.stateVersion = "26.05"; fileSystems."/" = { device = "none"; @@ -25,21 +26,28 @@ let } ]; }; - valid = evaluate { + ids = { first = 0; second = 1; }; + valid = evaluate ids { }; + matchingNat = evaluate ids { externalInterface = "eth0"; }; + wrongUplink = evaluate ids { externalInterface = "eth1"; }; + fixedAddress = evaluate ids { externalIP = "203.0.113.1"; }; duplicate = evaluate { first = 0; second = 0; - }; + } { }; in assert pkgs.lib.assertMsg (pkgs.lib.all (item: item.assertion) valid.config.assertions) ( pkgs.lib.concatMapStringsSep "\n" (item: item.message) ( builtins.filter (item: !item.assertion) valid.config.assertions ) ); +assert pkgs.lib.all (item: item.assertion) matchingNat.config.assertions; assert pkgs.lib.any (item: !item.assertion) duplicate.config.assertions; +assert pkgs.lib.any (item: !item.assertion) wrongUplink.config.assertions; +assert pkgs.lib.any (item: !item.assertion) fixedAddress.config.assertions; pkgs.runCommand "github-runner-vm-config-test" { } '' touch "$out" '' diff --git a/nix/runner/spoof-test.py b/nix/runner/spoof-test.py new file mode 100644 index 00000000000..c87d4471d19 --- /dev/null +++ b/nix/runner/spoof-test.py @@ -0,0 +1,36 @@ +"""Probe whether one guest can impersonate a peer in a host UDP flow.""" + +from pathlib import Path +import socket +import sys + + +role = sys.argv[1] +host = ("10.231.1.1", 45678) +peer = ("10.231.1.2", 45679) + +if role == "peer": + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.bind(("0.0.0.0", peer[1])) + Path("/tmp/udp-peer-ready").touch() + data, sender = connection.recvfrom(32) + assert data == b"probe" + data, _ = connection.recvfrom(32) + assert data == b"finish" + connection.sendto(b"finish", sender) +elif role == "receiver": + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.bind(host) + connection.sendto(b"probe", peer) + Path("/tmp/udp-receiver-ready").touch() + data, _ = connection.recvfrom(32) + Path("/tmp/udp-result").write_text(data.decode("ascii")) +elif role == "attack": + from scapy.all import IP, UDP, Raw, send + + send(IP(src=peer[0], dst=host[0]) / UDP(sport=peer[1], dport=host[1]) / Raw(b"spoofed"), verbose=False) +elif role == "finish": + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.sendto(b"finish", ("127.0.0.1", peer[1])) +else: + raise ValueError(f"Unsupported role: {role}") diff --git a/nix/runner/test.nix b/nix/runner/test.nix index 299565c56b5..df455782870 100644 --- a/nix/runner/test.nix +++ b/nix/runner/test.nix @@ -53,6 +53,8 @@ pkgs.testers.runNixOSTest { # The test driver supplies the three network links instead of QEMU's TAP. systemd.network.netdevs = lib.mkForce { }; networking.networkmanager.enable = false; + # The module's isolation must also hold on hosts using asymmetric routing. + networking.firewall.checkReversePath = false; # The test driver configures the uplink outside networkd. systemd.network.wait-online.enable = false; networking.useDHCP = false; @@ -130,6 +132,7 @@ pkgs.testers.runNixOSTest { ]; networking.defaultGateway = "10.231.1.1"; networking.firewall.allowedTCPPorts = [ 8080 ]; + networking.firewall.allowedUDPPorts = [ 45679 ]; environment.systemPackages = [ pkgs.curl ]; systemd.services.http = { wantedBy = [ "multi-user.target" ]; @@ -190,6 +193,20 @@ pkgs.testers.runNixOSTest { router.succeed("nft list table inet github-runner-queue-ci") router.succeed("nft list table inet github-runner-other-ci") + with subtest("a guest cannot impersonate a sibling's established reply to the host"): + other.succeed("systemd-run --unit=udp-peer --service-type=exec ${pkgs.python3}/bin/python3 ${./spoof-test.py} peer") + other.wait_for_file("/tmp/udp-peer-ready") + router.succeed("systemd-run --unit=udp-receiver --service-type=exec ${pkgs.python3}/bin/python3 ${./spoof-test.py} receiver") + router.wait_for_file("/tmp/udp-receiver-ready") + guest.succeed("${ + pkgs.python3.withPackages (p: [ p.scapy ]) + }/bin/python3 ${./spoof-test.py} attack") + # This response traverses the attacker's same transmit queue after its spoof. + guest.succeed("curl --fail http://203.0.113.1:8080/") + other.succeed("${pkgs.python3}/bin/python3 ${./spoof-test.py} finish") + router.wait_for_file("/tmp/udp-result") + router.succeed("test $(cat /tmp/udp-result) = finish") + with subtest("the runner inherits the hosted CI locked-memory budget"): guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/memlock") guest.succeed("test $(cat /var/cache/github-runner-home/.cache/mbx/memlock) = 65536") From 6d80a01f73c85200d0dd5baa5a742573594a19b6 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:36:22 -0700 Subject: [PATCH 10/14] docs(ci): clarify hosted and persistent queue cache behavior Co-Authored-By: GPT-6 --- .github/workflows/check.yml | 8 +++----- nix/runner/README.md | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5a54ca04923..38a96394926 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -72,11 +72,9 @@ jobs: - uses: DeterminateSystems/flake-checker-action@786422608c7bded2bbc9741ad9f91356842bf520 # main if: steps.scope.outputs.needed == 'true' - # cache.yml warms the default-branch store on `main`. Queue and release PR - # runs restore it; the action refuses to publish from them, and Actions - # would scope such an entry to its own ref anyway, where no later run - # could read it while it ate the repository's 10 GB budget. A selection - # of lints alone compiles no Rust, so it skips the restore. + # Hosted jobs restore the ARM64 cache warmed by cache.yml; self-hosted + # queue jobs reuse their local mbx store. Cache setup is needed only for + # selected work that compiles Rust. - name: Rust cache if: steps.scope.outputs.rust == 'true' uses: ./.github/actions/rust-cache diff --git a/nix/runner/README.md b/nix/runner/README.md index be9270da4af..da175533163 100644 --- a/nix/runner/README.md +++ b/nix/runner/README.md @@ -85,7 +85,7 @@ The runner's home is `/var/cache/github-runner-home`; mbx uses on this runner. There is no cache-warming service or cache server. Persistent job state is shared with subsequent jobs, so route only trusted work to the runner. For MoQ, set `MOQ_QUEUE_RUNNER` to the runner label after a representative CI run -passes; pull requests stay on hosted runners. +passes; release pull requests use hosted runners and main pull requests skip cloud CI. `just gh runner-vm` builds a guest image and runs NixOS integration tests for startup without carrier, public egress for multiple guests, private-network and From 08a049c3ae006951deb444974020cab7c1c5f64a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:07:58 -0700 Subject: [PATCH 11/14] test(ci): cover local and hosted R2 cache selection Co-Authored-By: GPT-6 --- sh/gh/setup.test.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sh/gh/setup.test.ts b/sh/gh/setup.test.ts index 630aab6024e..b0b6339767f 100644 --- a/sh/gh/setup.test.ts +++ b/sh/gh/setup.test.ts @@ -77,19 +77,29 @@ test("main PRs skip both jobs while the queue and release PRs run them", () => { test("self-hosted queue jobs use local caches without R2 credentials", async () => { const action = Bun.YAML.parse(await Bun.file(".github/actions/rust-cache/action.yml").text()); const setup = action.runs.steps.find((step: Step) => step.uses?.startsWith("jdx/mr-boxington-action@")); - const expression = setup.with.backend.replaceAll("inputs.access-key-id", 'inputs["access-key-id"]').replaceAll("inputs.secret-access-key", 'inputs["secret-access-key"]').replace(/^\$\{\{\s*|\s*\}\}$/g, ""); + const expression = setup.with.backend + .replaceAll("inputs.access-key-id", 'inputs["access-key-id"]') + .replaceAll("inputs.secret-access-key", 'inputs["secret-access-key"]') + .replace(/^\$\{\{\s*|\s*\}\}$/g, ""); const backend = new Function("runner", "inputs", `return (${expression});`); const guard = new Function("runner", "github", `return (${action.runs.steps[0].if});`); for (const event_name of ["merge_group", "pull_request", "pull_request_target", "push", "workflow_dispatch"]) { expect(guard({ environment: "self-hosted" }, { event_name })).toBe(event_name !== "merge_group"); expect(guard({ environment: "github-hosted" }, { event_name })).toBe(false); } - for (const credentials of [{ "access-key-id": "key", "secret-access-key": "secret" }, { "access-key-id": "", "secret-access-key": "" }]) { + for (const credentials of [ + { "access-key-id": "key", "secret-access-key": "secret" }, + { "access-key-id": "", "secret-access-key": "" }, + ]) { expect(backend({ environment: "self-hosted" }, credentials)).toBe("local"); - expect(backend({ environment: "github-hosted" }, credentials)).toBe(credentials["access-key-id"] ? "remote" : "local"); + expect(backend({ environment: "github-hosted" }, credentials)).toBe( + credentials["access-key-id"] ? "remote" : "local", + ); } for (const name of ["Cache credentials", "Check the cache"]) { - expect(action.runs.steps.find((step: Step) => step.name === name).if).toStartWith("runner.environment != 'self-hosted'"); + expect(action.runs.steps.find((step: Step) => step.name === name).if).toStartWith( + "runner.environment != 'self-hosted'", + ); } expect(action.runs.steps[0].run).toContain("exit 1"); }); From 6f39914c5a2167a3900888029d21b8a5a8cc402f Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:20:27 -0700 Subject: [PATCH 12/14] fix(ci): restart persistent runner VMs after unexpected exits Co-Authored-By: GPT-6 --- nix/modules/github-runner-vm.nix | 1 + nix/runner/config-test.nix | 3 +++ 2 files changed, 4 insertions(+) diff --git a/nix/modules/github-runner-vm.nix b/nix/modules/github-runner-vm.nix index 082d5af4a1a..2dfa8bbda77 100644 --- a/nix/modules/github-runner-vm.nix +++ b/nix/modules/github-runner-vm.nix @@ -178,6 +178,7 @@ let RuntimeDirectoryMode = "0700"; LoadCredential = [ "runner-token:${cfg.tokenFile}" ]; ExecStop = stop; + Restart = "always"; TimeoutStopSec = 120; ProtectHome = true; ProtectSystem = "strict"; diff --git a/nix/runner/config-test.nix b/nix/runner/config-test.nix index d4d50203bd1..8670853552e 100644 --- a/nix/runner/config-test.nix +++ b/nix/runner/config-test.nix @@ -45,6 +45,9 @@ assert pkgs.lib.assertMsg (pkgs.lib.all (item: item.assertion) valid.config.asse ) ); assert pkgs.lib.all (item: item.assertion) matchingNat.config.assertions; +assert pkgs.lib.all ( + name: valid.config.systemd.services.${name}.serviceConfig.Restart == "always" +) (builtins.attrNames ids); assert pkgs.lib.any (item: !item.assertion) duplicate.config.assertions; assert pkgs.lib.any (item: !item.assertion) wrongUplink.config.assertions; assert pkgs.lib.any (item: !item.assertion) fixedAddress.config.assertions; From 7fe7b1f8b07153c435fd285731dc0c69d0ba144c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:26:12 -0700 Subject: [PATCH 13/14] docs(ci): require external workflow approvals before public runner registration Co-Authored-By: GPT-6 --- nix/runner/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nix/runner/README.md b/nix/runner/README.md index da175533163..6c6f5e5306f 100644 --- a/nix/runner/README.md +++ b/nix/runner/README.md @@ -57,6 +57,13 @@ The module uses the host's NixOS NAT service. Leave `networking.nat.externalInte unset, or set it to the same uplink as every runner. Fixed `networking.nat.externalIP` is unsupported. Conflicting settings are rejected during evaluation. +For a public repository, require approval for **all external contributors' workflows** +before registering a persistent runner. A contributor can change `runs-on` in a +pull request, so a workflow condition is not an assignment boundary. Review the +workflow and its code before approving it: approval authorizes execution against +the persistent guest and its caches. Restricting approval only to first-time +contributors does not provide this boundary. See [GitHub's approval policy](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository). + Mint a repository registration token with `gh api -X POST repos/OWNER/REPO/actions/runners/registration-token --jq .token`, and install it at `tokenFile` with mode `0600` in a root-only directory. Runtime credentials reach From 0d89b96e59f1699a6cb6326848b314c506d9fbff Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:09:24 -0700 Subject: [PATCH 14/14] fix(ci): preserve host networking around runner VMs Keep forwarding restrictions scoped to runner TAPs and retain wait-online for required networkd links. Cover mixed network managers and unrelated transit traffic, and observe spoof drops directly in the integration test. Co-Authored-By: GPT-6 --- nix/modules/github-runner-vm.nix | 15 +++++++++++---- nix/runner/config-test.nix | 14 ++++++++++++++ nix/runner/test.nix | 30 +++++++++++++++++++++++++----- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/nix/modules/github-runner-vm.nix b/nix/modules/github-runner-vm.nix index 2dfa8bbda77..a2c022e038e 100644 --- a/nix/modules/github-runner-vm.nix +++ b/nix/modules/github-runner-vm.nix @@ -72,9 +72,17 @@ let networking.networkmanager.unmanaged = [ "interface-name:${tap}" ]; systemd.network = { enable = true; - # When NetworkManager owns the uplink, it also owns online readiness. - # The TAP cannot gain carrier until QEMU starts. - wait-online.enable = lib.mkIf config.networking.networkmanager.enable false; + # TAP-only networkd has no link that can satisfy wait-online. Preserve + # readiness tracking as soon as the host has another required link. + wait-online.enable = lib.mkDefault ( + lib.any ( + network: + !(lib.elem (network.linkConfig.RequiredForOnline or null) [ + false + "no" + ]) + ) (lib.attrValues config.systemd.network.networks) + ); netdevs."30-${name}" = { netdevConfig = { Kind = "tap"; @@ -97,7 +105,6 @@ let enable = true; internalInterfaces = [ tap ]; }; - networking.firewall.filterForward = true; networking.nftables = { enable = true; tables."github-runner-${name}" = { diff --git a/nix/runner/config-test.nix b/nix/runner/config-test.nix index 8670853552e..05d9fedf296 100644 --- a/nix/runner/config-test.nix +++ b/nix/runner/config-test.nix @@ -31,6 +31,17 @@ let second = 1; }; valid = evaluate ids { }; + mixedNetworking = valid.extendModules { + modules = [ + { + networking.networkmanager.enable = true; + systemd.network.networks.secondary = { + matchConfig.Name = "eth1"; + linkConfig.RequiredForOnline = "routable"; + }; + } + ]; + }; matchingNat = evaluate ids { externalInterface = "eth0"; }; wrongUplink = evaluate ids { externalInterface = "eth1"; }; fixedAddress = evaluate ids { externalIP = "203.0.113.1"; }; @@ -45,6 +56,9 @@ assert pkgs.lib.assertMsg (pkgs.lib.all (item: item.assertion) valid.config.asse ) ); assert pkgs.lib.all (item: item.assertion) matchingNat.config.assertions; +assert !valid.config.networking.firewall.filterForward; +assert mixedNetworking.config.systemd.network.wait-online.enable; +assert !valid.config.systemd.network.wait-online.enable; assert pkgs.lib.all ( name: valid.config.systemd.services.${name}.serviceConfig.Restart == "always" ) (builtins.attrNames ids); diff --git a/nix/runner/test.nix b/nix/runner/test.nix index df455782870..cd9741492ac 100644 --- a/nix/runner/test.nix +++ b/nix/runner/test.nix @@ -29,7 +29,19 @@ pkgs.testers.runNixOSTest { startup = { lib, ... }: { imports = [ hostModule ]; networking.networkmanager.enable = true; - networking.networkmanager.unmanaged = [ "interface-name:eth0" ]; + networking.networkmanager.unmanaged = [ + "interface-name:eth0" + "interface-name:eth2" + ]; + virtualisation.vlans = [ + 1 + 2 + ]; + systemd.network.networks."10-secondary" = { + matchConfig.Name = "eth2"; + address = [ "192.168.2.250/24" ]; + linkConfig.RequiredForOnline = "routable"; + }; networking.networkmanager.ensureProfiles.profiles.uplink = { connection = { id = "uplink"; @@ -38,7 +50,7 @@ pkgs.testers.runNixOSTest { }; ipv4 = { method = "manual"; - address1 = "192.168.1.4/24"; + address1 = "192.168.1.250/24"; }; ipv6.method = "disabled"; }; @@ -153,7 +165,8 @@ pkgs.testers.runNixOSTest { with subtest("the carrier-less TAP is configured without blocking the host uplink"): startup.wait_for_unit("network-online.target") startup.succeed("systemctl is-active NetworkManager-wait-online.service") - startup.fail("systemctl is-enabled systemd-networkd-wait-online.service") + startup.succeed("systemctl is-active systemd-networkd-wait-online.service") + startup.succeed("ip -4 -o address show dev eth2 | grep -F '192.168.2.250/24'") startup.succeed("test $(cat /sys/class/net/tap-queue-ci/carrier) = 0") startup.succeed("ip -4 -o address show dev tap-queue-ci | grep -F '10.231.0.1/30'") startup.succeed("test $(cat /sys/class/net/tap-other-ci/carrier) = 0") @@ -179,6 +192,13 @@ pkgs.testers.runNixOSTest { router.succeed("ip route add 203.0.113.1/32 dev uplink0; ip route add 100.100.100.1/32 dev uplink0") guest.succeed("ip route replace default via 10.231.0.1 dev eth1") + with subtest("unrelated host forwarding remains available"): + router.succeed("ip netns add transit; ip link add transit0 type veth peer name transit1 netns transit") + router.succeed("ip addr add 10.232.0.1/30 dev transit0; ip link set transit0 up") + router.succeed("ip -n transit addr add 10.232.0.2/30 dev transit1; ip -n transit link set transit1 up; ip -n transit route add default via 10.232.0.1") + internet.succeed("ip route add 10.232.0.0/30 via 192.168.2.1") + router.succeed("ip netns exec transit ${pkgs.curl}/bin/curl --fail --connect-timeout 5 http://203.0.113.1:8080/") + with subtest("both guests reach public networks while host and sibling guests stay isolated"): router.succeed("curl --fail http://10.231.1.2:8080/") router.succeed("curl --fail http://10.231.0.2:8080/") @@ -201,8 +221,8 @@ pkgs.testers.runNixOSTest { guest.succeed("${ pkgs.python3.withPackages (p: [ p.scapy ]) }/bin/python3 ${./spoof-test.py} attack") - # This response traverses the attacker's same transmit queue after its spoof. - guest.succeed("curl --fail http://203.0.113.1:8080/") + # Observe the spoof being dropped before releasing the legitimate reply. + router.wait_until_succeeds("nft list chain inet github-runner-queue-ci input | grep -E 'ip saddr != 10.231.0.2 counter packets [1-9]'") other.succeed("${pkgs.python3}/bin/python3 ${./spoof-test.py} finish") router.wait_for_file("/tmp/udp-result") router.succeed("test $(cat /tmp/udp-result) = finish")