diff --git a/.github/actions/rust-cache/action.yml b/.github/actions/rust-cache/action.yml index 6580c8d4b14..2fb3a234d34 100644 --- a/.github/actions/rust-cache/action.yml +++ b/.github/actions/rust-cache/action.yml @@ -1,5 +1,5 @@ name: Rust cache -description: Install mbx, put its Cargo shim on PATH, and back it with the R2 build cache that `main` and its merge queue write. +description: Install mbx and its Cargo shim, using local storage in runner VMs and R2 on hosted runners. inputs: access-key-id: @@ -12,6 +12,13 @@ inputs: runs: using: composite steps: + - name: Require merge queue on self-hosted runners + if: runner.environment == 'self-hosted' && github.event_name != 'merge_group' + shell: bash + run: | + echo "::error::Persistent self-hosted caches require a merge_group job." >&2 + exit 1 + # Runs on `main` and in its merge queue write; everything else reads. A # queued pull request is about to become `main`, so its code is trusted # like `main`'s, and publishing from the queue keeps the cache current @@ -28,6 +35,7 @@ runs: # building on would leave every reader cold with nothing to say why. Half # a pair is a broken secret on any job, never a fork's empty one. - name: Cache credentials + if: runner.environment != 'self-hosted' shell: bash env: KEY_ID: ${{ inputs.access-key-id }} @@ -70,7 +78,7 @@ runs: - uses: jdx/mr-boxington-action@47ce49b32025d93a7a2f4570815afaa2b7092240 # v1.7.2 with: version: 1.25.0 - backend: ${{ inputs.access-key-id != '' && inputs.secret-access-key != '' && 'remote' || 'local' }} + backend: ${{ runner.environment != 'self-hosted' && inputs.access-key-id != '' && inputs.secret-access-key != '' && 'remote' || 'local' }} remote-url: s3://mbx-moq-dev namespace: ${{ runner.os }}-${{ runner.arch }} @@ -80,7 +88,7 @@ runs: # come out read-write, or nothing would publish and every reader would # drift cold unnoticed. - name: Check the cache - if: inputs.access-key-id != '' && inputs.secret-access-key != '' + if: runner.environment != 'self-hosted' && inputs.access-key-id != '' && inputs.secret-access-key != '' shell: bash run: | report=$(mbx doctor --json || true) diff --git a/.github/justfile b/.github/justfile index d4777d96aee..a5b90b330ec 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,5 +1,9 @@ set working-directory := '..' +# Build and test the persistent NixOS runner VM on x86_64 Linux. +runner-vm: + nix build --no-link .#checks.x86_64-linux.github-runner-vm .#checks.x86_64-linux.github-runner-vm-image .#checks.x86_64-linux.github-runner-vm-config --print-build-logs + # Lint GitHub Actions workflows, their alert coverage, and that every step runs a recipe. check: actionlint diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml deleted file mode 100644 index 043fd041a6a..00000000000 --- a/.github/workflows/cache.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Cache - -# Pushes the dev shell closure that every nix job substitutes from nix.moq.dev. -# -# The Nix signing key and the CI bucket's write key live in the `cache-write` -# Environment, which pull requests cannot deploy to (see infra/README.md). -# -# The Rust build cache is not warmed here: `main` and its merge queue publish -# what they compile as they run (see .github/actions/rust-cache). - -permissions: - contents: read - -on: - push: - branches: [main] - # Only the flake decides the dev shell, so don't spend a runner otherwise. - paths: - - "flake.nix" - - "flake.lock" - # Patches and lockfiles the dev shell builds from. - - "nix/**" - - ".github/workflows/cache.yml" - # Priming the shell by hand, e.g. after a failed push. - workflow_dispatch: - -# Queue rather than cancel, so a run always finishes. Actions keeps only the -# newest pending run per group, so merges that land mid-run coalesce into one. -concurrency: - group: cache-${{ github.ref }} - cancel-in-progress: false - -jobs: - warm: - name: Warm (${{ matrix.runner }}) - strategy: - fail-fast: false - matrix: - runner: [ubuntu-24.04-arm, ubuntu-latest] - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - environment: cache-write - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/nix - - # Anything in the dev shell that nixpkgs doesn't serve prebuilt - # (uniffi-bindgen-go, a buildRustPackage from a git tag) would otherwise - # compile from source in every job that enters it. Only paths missing - # from the bucket upload, so this also restores whatever its lifecycle - # rule expired. - - name: Cache the dev shell - env: - NIX_SIGNING_KEY: ${{ secrets.NIX_SIGNING_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.NIX_R2_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.NIX_R2_SECRET_ACCESS_KEY }} - run: | - # `--profile` realises the shell's full build closure, which is what - # the runners actually need; `nix build .#devShells...` alone leaves - # out the build-time inputs. - nix develop --profile /tmp/devshell --command true - nix develop --command just infra nix push nix-ci-moq-dev /tmp/devshell diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e63b2aa3da5..31872302fea 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -33,7 +33,8 @@ jobs: # A close/merge run exists only to cancel the superseded in-flight build via # the concurrency group above. if: github.event.action != 'closed' && (github.event_name != 'pull_request' || github.base_ref == 'release') - runs-on: ubuntu-24.04-arm + # Leave the variable unset until the local runner is online and verified. + runs-on: ${{ github.event_name == 'merge_group' && vars.MOQ_QUEUE_RUNNER || 'ubuntu-24.04-arm' }} # `main` and its merge queue write the Rust cache; see .github/actions/rust-cache. environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/gh-readonly-queue/main/')) && 'cache-write' || '' }} # Backstop a hang instead of holding the runner for the 6-hour default. @@ -58,7 +59,7 @@ jobs: CI_BASE: ${{ github.base_ref == 'release' && '--all' || github.event.merge_group.base_sha }} - uses: ./.github/actions/nix - if: steps.scope.outputs.needed == 'true' + if: steps.scope.outputs.needed == 'true' && runner.environment != 'self-hosted' - uses: DeterminateSystems/flake-checker-action@786422608c7bded2bbc9741ad9f91356842bf520 # main if: steps.scope.outputs.needed == 'true' @@ -88,7 +89,8 @@ jobs: test: name: Test if: github.event.action != 'closed' && (github.event_name != 'pull_request' || github.base_ref == 'release') - runs-on: ubuntu-24.04-arm + # Leave the variable unset until the local runner is online and verified. + runs-on: ${{ github.event_name == 'merge_group' && vars.MOQ_QUEUE_RUNNER || 'ubuntu-24.04-arm' }} # `main` and its merge queue write the Rust cache; see .github/actions/rust-cache. environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/gh-readonly-queue/main/')) && 'cache-write' || '' }} timeout-minutes: 60 @@ -112,7 +114,7 @@ jobs: CI_BASE: ${{ github.base_ref == 'release' && '--all' || github.event.merge_group.base_sha }} - uses: ./.github/actions/nix - if: steps.scope.outputs.needed == 'true' + if: steps.scope.outputs.needed == 'true' && runner.environment != 'self-hosted' # Reads and writes like the `check` job above. - name: Rust cache diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 51a9c7f30dc..88c2f2dc12e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -48,6 +48,9 @@ concurrency: cancel-in-progress: true jobs: + runner-vm: + uses: ./.github/workflows/runner-vm.yml + # The path-filtered gates main PRs skip; release PRs still run them. x11: uses: ./.github/workflows/x11.yml diff --git a/.github/workflows/runner-vm.yml b/.github/workflows/runner-vm.yml new file mode 100644 index 00000000000..7fb353058dc --- /dev/null +++ b/.github/workflows/runner-vm.yml @@ -0,0 +1,35 @@ +name: Runner VM + +on: + workflow_call: + pull_request: + branches: [release] + types: [opened, synchronize, reopened, edited, closed] + paths: + - "nix/**" + - "flake.nix" + - "flake.lock" + - ".github/justfile" + - ".github/workflows/runner-vm.yml" + +permissions: + contents: read + +concurrency: + group: runner-vm-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + if: github.event.action != 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + - run: just gh runner-vm diff --git a/flake.nix b/flake.nix index 5929983ccaf..7c2406df250 100644 --- a/flake.nix +++ b/flake.nix @@ -58,6 +58,7 @@ { nixosModules = { moq-relay = import ./nix/modules/moq-relay.nix; + github-runner-vm = import ./nix/modules/github-runner-vm.nix; }; overlays.default = import ./nix/overlay.nix { inherit crane; }; @@ -661,7 +662,10 @@ test -f "${overlayPkgs.moq-boy.src}/rs/moq-video/src/frame/nv12_resize.ptx" touch "$out" ''; - }; + } + // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( + import ./nix/runner/checks.nix { inherit pkgs; } + ); } ); } diff --git a/nix/modules/github-runner-vm.nix b/nix/modules/github-runner-vm.nix new file mode 100644 index 00000000000..a2c022e038e --- /dev/null +++ b/nix/modules/github-runner-vm.nix @@ -0,0 +1,310 @@ +{ + config, + lib, + pkgs, + utils, + ... +}: +let + hostConfig = config; + enabled = lib.filterAttrs (_: cfg: cfg.enable) config.services.github-runner-vms; + ids = map (cfg: cfg.network.id) (lib.attrValues enabled); + guestFor = + cfg: + import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + (import ../runner/guest.nix { + inherit (cfg) runner resources; + network = { + hostAddress = "10.231.${toString cfg.network.id}.1"; + guestAddress = "10.231.${toString cfg.network.id}.2"; + }; + }) + cfg.guestConfig + ]; + }; + render = + name: cfg: + let + tap = "tap-${name}"; + user = name; + prefix = "10.231.${toString cfg.network.id}"; + guest = cfg.guest; + device = "${utils.escapeSystemdPath "/sys/subsystem/net/devices/${tap}"}.device"; + qemu = pkgs.qemu_kvm; + stop = pkgs.writeShellScript "github-runner-vm-stop" '' + set -euo pipefail + ${ + pkgs.python3.withPackages (p: [ p.qemu-qmp ]) + }/bin/python3 ${../runner/stop.py} "$RUNTIME_DIRECTORY/qmp" + ${pkgs.coreutils}/bin/tail --pid="$MAINPID" -f /dev/null + ''; + in + { + assertions = [ + { + assertion = builtins.match "[a-z][a-z0-9-]{0,9}" name != null; + message = "Runner VM names must start with a lowercase letter and contain at most 10 lowercase letters, digits, or hyphens."; + } + { + assertion = pkgs.stdenv.hostPlatform.system == "x86_64-linux"; + message = "services.github-runner-vms supports x86_64-linux KVM hosts."; + } + { + assertion = + config.networking.nat.externalInterface == null + || config.networking.nat.externalInterface == cfg.network.uplink; + message = "Runner VM ${name} needs networking.nat.externalInterface unset or equal to its network.uplink."; + } + { + assertion = config.networking.nat.externalIP == null; + message = "Runner VMs require masquerading; networking.nat.externalIP must be unset."; + } + ]; + users.groups.${user} = { }; + users.users.${user} = { + isSystemUser = true; + group = user; + extraGroups = [ "kvm" ]; + }; + + networking.networkmanager.unmanaged = [ "interface-name:${tap}" ]; + systemd.network = { + enable = true; + # TAP-only networkd has no link that can satisfy wait-online. Preserve + # readiness tracking as soon as the host has another required link. + wait-online.enable = lib.mkDefault ( + lib.any ( + network: + !(lib.elem (network.linkConfig.RequiredForOnline or null) [ + false + "no" + ]) + ) (lib.attrValues config.systemd.network.networks) + ); + netdevs."30-${name}" = { + netdevConfig = { + Kind = "tap"; + Name = tap; + }; + tapConfig.User = user; + }; + networks."30-${name}" = { + matchConfig.Name = tap; + address = [ "${prefix}.1/30" ]; + linkConfig.RequiredForOnline = "no"; + networkConfig = { + LinkLocalAddressing = "no"; + ConfigureWithoutCarrier = true; + }; + }; + }; + + networking.nat = { + enable = true; + internalInterfaces = [ tap ]; + }; + networking.nftables = { + enable = true; + tables."github-runner-${name}" = { + family = "inet"; + content = '' + chain input { + type filter hook input priority -10; policy accept; + # Conntrack is shared across TAPs, so validate the sender first. + iifname "${tap}" meta nfproto ipv6 counter drop + iifname "${tap}" ip saddr != ${prefix}.2 counter drop + iifname "${tap}" ct state established,related accept + iifname "${tap}" counter drop + } + chain forward { + type filter hook forward priority -10; policy accept; + iifname "${tap}" meta nfproto ipv6 counter drop + iifname "${tap}" ip saddr != ${prefix}.2 counter drop + iifname "${tap}" ip daddr { + 0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, + 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16, + 198.18.0.0/15, 224.0.0.0/4, 240.0.0.0/4 + } counter drop + iifname "${tap}" oifname != "${cfg.network.uplink}" counter drop + oifname "${tap}" ct state established,related accept + oifname "${tap}" counter drop + } + ''; + }; + }; + + systemd.services.${name} = { + description = "Persistent GitHub Actions runner VM"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + bindsTo = [ + "nftables.service" + device + ]; + after = [ + "network-online.target" + "nftables.service" + device + ]; + path = [ qemu ]; + preStart = '' + if ! test -e /dev/kvm; then + echo "KVM is unavailable. Enable virtualization in firmware and load the CPU KVM module." >&2 + exit 1 + fi + if ! test -r /dev/kvm || ! test -w /dev/kvm; then + echo "The ${user} user needs read and write access to /dev/kvm through the kvm group." >&2 + exit 1 + fi + if ! test -e "$STATE_DIRECTORY/root.qcow2"; then + cp --reflink=auto ${guest.config.system.build.image}/nixos.qcow2 "$STATE_DIRECTORY/root.qcow2.tmp" + chmod 0600 "$STATE_DIRECTORY/root.qcow2.tmp" + qemu-img resize "$STATE_DIRECTORY/root.qcow2.tmp" ${toString cfg.resources.disk}G + mv "$STATE_DIRECTORY/root.qcow2.tmp" "$STATE_DIRECTORY/root.qcow2" + fi + ''; + script = '' + exec ${qemu}/bin/qemu-system-x86_64 \ + -enable-kvm -cpu host -smp ${toString cfg.resources.cores} -m ${toString cfg.resources.memory} \ + -drive file="$STATE_DIRECTORY/root.qcow2",format=qcow2,if=virtio,cache=none \ + -netdev tap,id=net,ifname=${tap},script=no,downscript=no \ + -device virtio-net-pci,netdev=net,mac=02:00:00:00:23:02 \ + -device virtio-rng-pci \ + -fw_cfg name=opt/io.systemd.credentials/runner-token,file="$CREDENTIALS_DIRECTORY/runner-token" \ + -qmp unix:"$RUNTIME_DIRECTORY/qmp",server=on,wait=off \ + -nographic + ''; + serviceConfig = { + User = user; + Group = user; + StateDirectory = name; + StateDirectoryMode = "0700"; + RuntimeDirectory = name; + RuntimeDirectoryMode = "0700"; + LoadCredential = [ "runner-token:${cfg.tokenFile}" ]; + ExecStop = stop; + Restart = "always"; + TimeoutStopSec = 120; + ProtectHome = true; + ProtectSystem = "strict"; + PrivateTmp = true; + NoNewPrivileges = true; + CapabilityBoundingSet = ""; + DevicePolicy = "closed"; + DeviceAllow = [ + "/dev/kvm rw" + "/dev/net/tun rw" + ]; + # QEMU queries the TAP's MTU through an IP socket; guest packets use the + # TAP descriptor, so denying host IP traffic does not block guest NAT. + RestrictAddressFamilies = [ + "AF_UNIX" + "AF_INET" + "AF_INET6" + "AF_NETLINK" + ]; + IPAddressDeny = "any"; + Nice = 10; + CPUWeight = 10; + IOWeight = 10; + CPUQuota = "${toString (cfg.resources.cores * 100)}%"; + MemoryMax = "${toString (cfg.resources.memory + 2048)}M"; + TasksMax = 1024; + }; + }; + + }; +in +{ + options.services.github-runner-vms = lib.mkOption { + default = { }; + description = "Persistent GitHub Actions runner VMs isolated from the host and each other."; + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, config, ... }: { + options = { + enable = lib.mkEnableOption "a GitHub Actions runner in a persistent KVM guest"; + runner = { + url = lib.mkOption { + type = lib.types.str; + description = "GitHub repository URL registered to the runner."; + }; + name = lib.mkOption { + type = lib.types.str; + default = "${hostConfig.networking.hostName}-${name}"; + description = "Runner name advertised to GitHub."; + }; + labels = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Additional labels advertised to GitHub."; + }; + }; + network.uplink = lib.mkOption { + type = lib.types.strMatching "[a-zA-Z0-9_.:-]{1,15}"; + description = "Host interface used for public IPv4 egress."; + }; + network.id = lib.mkOption { + type = lib.types.ints.u8; + default = 0; + description = "Unique link number using the private 10.231.ID.0/30 subnet."; + }; + resources = { + cores = lib.mkOption { + type = lib.types.ints.positive; + default = 4; + description = "Virtual CPUs and maximum host CPU usage in cores."; + }; + memory = lib.mkOption { + type = lib.types.ints.positive; + default = 8192; + description = "Guest memory in MiB, with 2 GiB extra allowed for QEMU."; + }; + disk = lib.mkOption { + type = lib.types.ints.positive; + default = 128; + description = "Initial persistent disk capacity in GiB."; + }; + }; + tokenFile = lib.mkOption { + type = lib.types.str; + default = "/var/lib/secrets/github-runner-${name}.token"; + description = "Host path to the runtime GitHub registration token."; + }; + guestConfig = lib.mkOption { + type = lib.types.deferredModule; + default = { }; + description = "Extra NixOS guest configuration, including SSH keys and binary caches."; + }; + guest = lib.mkOption { + type = lib.types.raw; + readOnly = true; + description = "Evaluated guest system for deployment as a NixOS flake configuration."; + }; + + }; + config.guest = lib.mkIf config.enable (guestFor config); + } + ) + ); + }; + config = + let + instances = lib.mapAttrsToList render enabled; + in + { + assertions = [ + { + assertion = lib.length ids == lib.length (lib.unique ids); + message = "Each enabled services.github-runner-vms instance needs a unique network.id."; + } + ] + ++ lib.concatMap (instance: instance.assertions) instances; + users.users = lib.mkMerge (map (instance: instance.users.users) instances); + users.groups = lib.mkMerge (map (instance: instance.users.groups) instances); + networking = lib.mkMerge (map (instance: instance.networking) instances); + systemd = lib.mkMerge (map (instance: instance.systemd) instances); + }; +} diff --git a/nix/runner/README.md b/nix/runner/README.md new file mode 100644 index 00000000000..f59cc18073b --- /dev/null +++ b/nix/runner/README.md @@ -0,0 +1,101 @@ +# Persistent GitHub Actions runner VM + +The `github-runner-vm` NixOS module runs each repository's runner in its own +persistent KVM guest. The guest keeps its Nix store, mbx cache, checkout, and runner state +across jobs and reboots. It has no host filesystem mounts or tailnet access. +The firewall permits public IPv4 egress and blocks the host and private networks. + +Import `moq.nixosModules.github-runner-vm` into an x86\_64 NixOS host: + +```nix +services.github-runner-vms.queue-ci = { + enable = true; + runner = { + url = "https://github.com/example/project"; + name = "build-host-queue"; + labels = [ "merge-queue" ]; + }; + network = { id = 0; uplink = "eth0"; }; + tokenFile = "/var/lib/secrets/queue-runner.token"; + resources = { cores = 8; memory = 16384; disk = 192; }; + guestConfig = { + users.users.root.openssh.authorizedKeys.keyFiles = [ ./deploy.pub ]; + nix.settings = { + extra-substituters = [ "https://nix.moq.dev" ]; + extra-trusted-public-keys = [ + "nix.moq.dev-1:BZ0DtCypDFv1ESX3hxuB42yc0vitSA1YHojSmVwB5bk=" + ]; + }; + }; +}; +``` + +Add another named instance for another repository, with its own `network.id`, +registration token, and guest configuration. Every instance gets a separate user, +disk, runtime credentials, and TAP interface; no cache data is shared between +instances. For example: + +```nix +services.github-runner-vms.other-ci = { + enable = true; + runner.url = "https://github.com/example/another-project"; + network = { id = 1; uplink = "eth0"; }; + guestConfig.users.users.root.openssh.authorizedKeys.keyFiles = [ ./deploy.pub ]; +}; +``` + +The default token path is `/var/lib/secrets/github-runner-NAME.token`. +Renaming an instance selects a new disk; it does not migrate its existing state. + +Enable virtualization in firmware and load the CPU's KVM module before activation. +`/dev/kvm` must exist. Each instance needs a unique `network.id` from 0 to 255. +It uses `10.231.ID.0/30`: the host is `.1` and the guest is `.2`. Reserve these +subnets on the host. Duplicate IDs are rejected during evaluation. NetworkManager owns the host uplink when +enabled; otherwise configure that uplink in networkd. The TAP receives its address +before QEMU attaches, so guest startup does not depend on TAP carrier. +The module uses the host's NixOS NAT service. Leave `networking.nat.externalInterface` +unset, or set it to the same uplink as every runner. Fixed `networking.nat.externalIP` +is unsupported. Conflicting settings are rejected during evaluation. + +For a public repository, require approval for **all external contributors' workflows** +before registering a persistent runner. A contributor can change `runs-on` in a +pull request, so a workflow condition is not an assignment boundary. Review the +workflow and its code before approving it: approval authorizes execution against +the persistent guest and its caches. Restricting approval only to first-time +contributors does not provide this boundary. See [GitHub's approval policy](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository). + +Mint a repository registration token with `gh api -X POST +repos/OWNER/REPO/actions/runners/registration-token --jq .token`, and install it +at `tokenFile` with mode `0600` in a root-only directory. Runtime credentials reach +the guest through QEMU firmware configuration; secret contents never enter the +Nix store. Keep the expired token file after registration. The guest retains its +own long-lived credential; wiping its state requires a fresh registration token. +Restart the host VM service after replacing the token. + +The host service and disk are named by the instance attribute: the example uses `queue-ci.service` +and `/var/lib/queue-ci/root.qcow2`. Its disk is created once. Changing the configured +image does not replace existing state, and changing `resources.disk` does not +resize an existing disk. For guest updates, expose the evaluated guest in the +host flake's outputs (`{ self, ... }:`): + +```nix +nixosConfigurations.queue-ci = + self.nixosConfigurations.build-host.config.services.github-runner-vms.queue-ci.guest; +``` + +Then deploy with `nixos-rebuild switch --flake .#queue-ci --target-host +root@10.231.0.2`, using the dedicated deployment key. Host rebuilds update the VM +launcher. Back up the deployment private key separately. + +The runner's home is `/var/cache/github-runner-home`; mbx uses +`/var/cache/github-runner-home/.cache/mbx`. Workflows must select mbx's local backend +on this runner. There is no cache-warming service or cache server. Persistent job +state is shared with subsequent jobs, so route only trusted work to the runner. +For MoQ, set `MOQ_QUEUE_RUNNER` to the runner label after a representative CI run +passes; release pull requests use hosted runners and main pull requests skip cloud CI. + +`just gh runner-vm` builds a guest image and runs NixOS integration tests for +startup without carrier, public egress for multiple guests, private-network and +sibling-guest isolation including spoofed established replies, compatible host NAT +settings, the runner's locked-memory budget, and Nix/mbx persistence across runner +and guest restarts. diff --git a/nix/runner/checks.nix b/nix/runner/checks.nix new file mode 100644 index 00000000000..132e95c9d6b --- /dev/null +++ b/nix/runner/checks.nix @@ -0,0 +1,28 @@ +{ pkgs }: +let + guestModule = import ./guest.nix { + runner = { + url = "https://github.com/example/queue"; + name = "test-queue"; + labels = [ "queue" ]; + }; + network = { + hostAddress = "10.231.0.1"; + guestAddress = "10.231.0.2"; + }; + resources = { + cores = 2; + memory = 1024; + disk = 16; + }; + }; + guest = import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ guestModule ]; + }; +in +{ + github-runner-vm = import ./test.nix { inherit pkgs guestModule; }; + github-runner-vm-image = guest.config.system.build.image; + github-runner-vm-config = import ./config-test.nix { inherit pkgs; }; +} diff --git a/nix/runner/config-test.nix b/nix/runner/config-test.nix new file mode 100644 index 00000000000..05d9fedf296 --- /dev/null +++ b/nix/runner/config-test.nix @@ -0,0 +1,70 @@ +{ pkgs }: +let + evaluate = + ids: nat: + import (pkgs.path + "/nixos/lib/eval-config.nix") { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + ../modules/github-runner-vm.nix + { + boot.isContainer = true; + networking.useHostResolvConf = false; + networking.nat = nat; + system.stateVersion = "26.05"; + fileSystems."/" = { + device = "none"; + fsType = "tmpfs"; + }; + services.github-runner-vms = pkgs.lib.mapAttrs (_: id: { + enable = true; + runner.url = "https://github.com/example/project"; + network = { + inherit id; + uplink = "eth0"; + }; + }) ids; + } + ]; + }; + ids = { + first = 0; + second = 1; + }; + valid = evaluate ids { }; + mixedNetworking = valid.extendModules { + modules = [ + { + networking.networkmanager.enable = true; + systemd.network.networks.secondary = { + matchConfig.Name = "eth1"; + linkConfig.RequiredForOnline = "routable"; + }; + } + ]; + }; + matchingNat = evaluate ids { externalInterface = "eth0"; }; + wrongUplink = evaluate ids { externalInterface = "eth1"; }; + fixedAddress = evaluate ids { externalIP = "203.0.113.1"; }; + duplicate = evaluate { + first = 0; + second = 0; + } { }; +in +assert pkgs.lib.assertMsg (pkgs.lib.all (item: item.assertion) valid.config.assertions) ( + pkgs.lib.concatMapStringsSep "\n" (item: item.message) ( + builtins.filter (item: !item.assertion) valid.config.assertions + ) +); +assert pkgs.lib.all (item: item.assertion) matchingNat.config.assertions; +assert !valid.config.networking.firewall.filterForward; +assert mixedNetworking.config.systemd.network.wait-online.enable; +assert !valid.config.systemd.network.wait-online.enable; +assert pkgs.lib.all ( + name: valid.config.systemd.services.${name}.serviceConfig.Restart == "always" +) (builtins.attrNames ids); +assert pkgs.lib.any (item: !item.assertion) duplicate.config.assertions; +assert pkgs.lib.any (item: !item.assertion) wrongUplink.config.assertions; +assert pkgs.lib.any (item: !item.assertion) fixedAddress.config.assertions; +pkgs.runCommand "github-runner-vm-config-test" { } '' + touch "$out" +'' diff --git a/nix/runner/guest.nix b/nix/runner/guest.nix new file mode 100644 index 00000000000..b72ccdd7091 --- /dev/null +++ b/nix/runner/guest.nix @@ -0,0 +1,143 @@ +{ + runner, + resources, + network, +}: +{ + config, + lib, + pkgs, + modulesPath, + ... +}: + +{ + imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; + + networking = { + hostName = "github-runner"; + useDHCP = false; + usePredictableInterfaceNames = false; + interfaces.eth0.ipv4.addresses = [ + { + address = network.guestAddress; + prefixLength = 30; + } + ]; + defaultGateway = network.hostAddress; + nameservers = [ + "1.1.1.1" + "8.8.8.8" + ]; + enableIPv6 = false; + }; + + boot = { + loader.grub.device = "/dev/vda"; + kernelParams = [ "console=ttyS0" ]; + kernelModules = [ "qemu_fw_cfg" ]; + growPartition = true; + }; + fileSystems."/" = { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + autoResize = true; + }; + system.stateVersion = "26.05"; + + system.build.image = import (modulesPath + "/../lib/make-disk-image.nix") { + inherit config lib pkgs; + format = "qcow2"; + partitionTableType = "legacy"; + diskSize = "auto"; + additionalSpace = "1G"; + copyChannel = false; + }; + + services.openssh = { + enable = true; + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + PermitRootLogin = "prohibit-password"; + }; + }; + programs.nix-ld.enable = true; + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + max-jobs = 2; + cores = resources.cores; + trusted-users = [ "root" ]; + }; + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 30d"; + }; + }; + + # Only a short-lived registration token enters the VM. The host's deployment + # key and private services are never mounted in the guest. + systemd.services.runner-token = { + wantedBy = [ "multi-user.target" ]; + before = [ "github-runner-runner.service" ]; + serviceConfig.Type = "oneshot"; + script = '' + install -D -m 0600 \ + /sys/firmware/qemu_fw_cfg/by_name/opt/io.systemd.credentials/runner-token/raw \ + /var/lib/secrets/github-runner.token + ''; + }; + + services.github-runners.runner = { + enable = true; + url = runner.url; + name = runner.name; + extraLabels = runner.labels; + tokenFile = "/var/lib/secrets/github-runner.token"; + tokenType = "registration"; + replace = true; + workDir = "/var/cache/github-runner-work"; + extraPackages = with pkgs; [ + curl + just + openssh + unzip + which + ]; + extraEnvironment = { + HOME = "/var/cache/github-runner-home"; + MBX_CACHE_DIR = "/var/cache/github-runner-home/.cache/mbx"; + CARGO_BUILD_JOBS = toString resources.cores; + }; + serviceOverrides = { + CacheDirectory = [ + "github-runner-home" + "github-runner-work" + ]; + CacheDirectoryMode = "0700"; + BindPaths = lib.mkForce [ ]; + ProtectHome = true; + RestrictNamespaces = false; + PrivateUsers = false; + SystemCallFilter = lib.mkForce [ ]; + TasksMax = 16384; + # Concurrent io_uring tests share the runner service's locked-memory budget. + LimitMEMLOCK = "64M"; + }; + }; + systemd.services.github-runner-runner = { + requires = [ "runner-token.service" ]; + after = [ "runner-token.service" ]; + }; + + environment.systemPackages = with pkgs; [ + git + just + ]; +} diff --git a/nix/runner/spoof-test.py b/nix/runner/spoof-test.py new file mode 100644 index 00000000000..c87d4471d19 --- /dev/null +++ b/nix/runner/spoof-test.py @@ -0,0 +1,36 @@ +"""Probe whether one guest can impersonate a peer in a host UDP flow.""" + +from pathlib import Path +import socket +import sys + + +role = sys.argv[1] +host = ("10.231.1.1", 45678) +peer = ("10.231.1.2", 45679) + +if role == "peer": + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.bind(("0.0.0.0", peer[1])) + Path("/tmp/udp-peer-ready").touch() + data, sender = connection.recvfrom(32) + assert data == b"probe" + data, _ = connection.recvfrom(32) + assert data == b"finish" + connection.sendto(b"finish", sender) +elif role == "receiver": + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.bind(host) + connection.sendto(b"probe", peer) + Path("/tmp/udp-receiver-ready").touch() + data, _ = connection.recvfrom(32) + Path("/tmp/udp-result").write_text(data.decode("ascii")) +elif role == "attack": + from scapy.all import IP, UDP, Raw, send + + send(IP(src=peer[0], dst=host[0]) / UDP(sport=peer[1], dport=host[1]) / Raw(b"spoofed"), verbose=False) +elif role == "finish": + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as connection: + connection.sendto(b"finish", ("127.0.0.1", peer[1])) +else: + raise ValueError(f"Unsupported role: {role}") diff --git a/nix/runner/stop.py b/nix/runner/stop.py new file mode 100644 index 00000000000..c18f5b7b0ac --- /dev/null +++ b/nix/runner/stop.py @@ -0,0 +1,18 @@ +"""Ask QEMU to shut down the guest before systemd stops the VM process.""" + +import asyncio +import sys + +from qemu.qmp import QMPClient + + +async def main(): + client = QMPClient("github-runner-vm") + await client.connect(sys.argv[1]) + try: + await client.execute("system_powerdown") + finally: + await client.disconnect() + + +asyncio.run(main()) diff --git a/nix/runner/test.nix b/nix/runner/test.nix new file mode 100644 index 00000000000..cd9741492ac --- /dev/null +++ b/nix/runner/test.nix @@ -0,0 +1,246 @@ +{ pkgs, guestModule }: + +let + hostModule = { lib, ... }: { + imports = [ ../modules/github-runner-vm.nix ]; + services.github-runner-vms = + lib.mapAttrs + (name: id: { + enable = true; + runner.url = "https://github.com/example/${name}"; + network = { + inherit id; + uplink = "uplink0"; + }; + guestConfig.system.build.image = lib.mkForce pkgs.emptyDirectory; + }) + { + queue-ci = 0; + other-ci = 1; + }; + systemd.services = lib.genAttrs [ "queue-ci" "other-ci" ] (_: { + enable = false; + }); + }; +in +pkgs.testers.runNixOSTest { + name = "github-runner-vm"; + nodes = { + startup = { lib, ... }: { + imports = [ hostModule ]; + networking.networkmanager.enable = true; + networking.networkmanager.unmanaged = [ + "interface-name:eth0" + "interface-name:eth2" + ]; + virtualisation.vlans = [ + 1 + 2 + ]; + systemd.network.networks."10-secondary" = { + matchConfig.Name = "eth2"; + address = [ "192.168.2.250/24" ]; + linkConfig.RequiredForOnline = "routable"; + }; + networking.networkmanager.ensureProfiles.profiles.uplink = { + connection = { + id = "uplink"; + type = "ethernet"; + interface-name = "eth1"; + }; + ipv4 = { + method = "manual"; + address1 = "192.168.1.250/24"; + }; + ipv6.method = "disabled"; + }; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ ]; + networking.interfaces.eth1.ipv6.addresses = lib.mkForce [ ]; + networking.useDHCP = false; + virtualisation.memorySize = 768; + systemd.targets.network-online.wantedBy = [ "multi-user.target" ]; + }; + router = { lib, ... }: { + imports = [ hostModule ]; + # The test driver supplies the three network links instead of QEMU's TAP. + systemd.network.netdevs = lib.mkForce { }; + networking.networkmanager.enable = false; + # The module's isolation must also hold on hosts using asymmetric routing. + networking.firewall.checkReversePath = false; + # The test driver configures the uplink outside networkd. + systemd.network.wait-online.enable = false; + networking.useDHCP = false; + virtualisation.vlans = [ + 1 + 2 + 3 + ]; + networking.firewall.allowedTCPPorts = [ 8080 ]; + environment.systemPackages = [ pkgs.curl ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; + }; + guest = { lib, ... }: { + imports = [ guestModule ]; + networking.firewall.allowedTCPPorts = [ 8080 ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; + virtualisation.memorySize = 1024; + virtualisation.writableStoreUseTmpfs = false; + boot.growPartition = lib.mkForce false; + networking.interfaces.eth1.ipv4.addresses = [ + { + address = "10.231.0.2"; + prefixLength = 30; + } + ]; + networking.interfaces.eth0.ipv4.addresses = lib.mkForce [ ]; + virtualisation.qemu.options = [ + "-fw_cfg name=opt/io.systemd.credentials/runner-token,file=${pkgs.writeText "runner-token" "fixture"}" + ]; + systemd.services.cache-seed-fixture = { + requiredBy = [ "github-runner-runner.service" ]; + before = [ "github-runner-runner.service" ]; + unitConfig.ConditionPathExists = "!/var/lib/cache-seeded"; + serviceConfig.Type = "oneshot"; + script = '' + install -d -m 0700 /var/cache/github-runner-home + echo seeded > /var/cache/github-runner-home/seeded + chmod 0600 /var/cache/github-runner-home/seeded + touch /var/lib/cache-seeded + ''; + }; + systemd.services.github-runner-runner.serviceConfig = { + ExecStartPre = lib.mkForce [ + "+${pkgs.coreutils}/bin/touch /var/lib/github-runner/runner/.current-token" + ]; + ExecStart = lib.mkForce ( + pkgs.writeShellScript "cache-probe" '' + test "$(cat "$HOME/seeded")" = seeded + mkdir -p "$MBX_CACHE_DIR" + ulimit -l > "$MBX_CACHE_DIR/memlock" + if test -f "$MBX_CACHE_DIR/marker"; then + touch "$MBX_CACHE_DIR/reused" + fi + touch "$MBX_CACHE_DIR/marker" + exec ${pkgs.coreutils}/bin/sleep infinity + '' + ); + }; + environment.systemPackages = [ pkgs.curl ]; + }; + other = { + virtualisation.vlans = [ 3 ]; + networking.useDHCP = false; + networking.interfaces.eth1.ipv4.addresses = [ + { + address = "10.231.1.2"; + prefixLength = 30; + } + ]; + networking.defaultGateway = "10.231.1.1"; + networking.firewall.allowedTCPPorts = [ 8080 ]; + networking.firewall.allowedUDPPorts = [ 45679 ]; + environment.systemPackages = [ pkgs.curl ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; + }; + internet = { + virtualisation.vlans = [ 2 ]; + networking.firewall.allowedTCPPorts = [ 8080 ]; + systemd.services.http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = "${pkgs.python3}/bin/python3 -m http.server 8080"; + }; + }; + }; + testScript = '' + start_all() + with subtest("the carrier-less TAP is configured without blocking the host uplink"): + startup.wait_for_unit("network-online.target") + startup.succeed("systemctl is-active NetworkManager-wait-online.service") + startup.succeed("systemctl is-active systemd-networkd-wait-online.service") + startup.succeed("ip -4 -o address show dev eth2 | grep -F '192.168.2.250/24'") + startup.succeed("test $(cat /sys/class/net/tap-queue-ci/carrier) = 0") + startup.succeed("ip -4 -o address show dev tap-queue-ci | grep -F '10.231.0.1/30'") + startup.succeed("test $(cat /sys/class/net/tap-other-ci/carrier) = 0") + startup.succeed("ip -4 -o address show dev tap-other-ci | grep -F '10.231.1.1/30'") + + router.wait_for_unit("nftables.service") + router.wait_for_unit("http.service") + internet.wait_for_unit("http.service") + other.wait_for_unit("http.service") + guest.wait_for_unit("github-runner-runner.service") + guest.wait_for_unit("http.service") + router.wait_for_open_port(8080) + internet.wait_for_open_port(8080) + other.wait_for_open_port(8080) + guest.wait_for_open_port(8080) + + router.succeed("ip addr flush dev eth1; ip link set eth1 down; ip link set eth1 name tap-queue-ci; ip link set tap-queue-ci up") + router.wait_until_succeeds("ip -4 -o address show dev tap-queue-ci | grep -F '10.231.0.1/30'") + router.succeed("ip addr flush dev eth3; ip link set eth3 down; ip link set eth3 name tap-other-ci; ip link set tap-other-ci up") + router.wait_until_succeeds("ip -4 -o address show dev tap-other-ci | grep -F '10.231.1.1/30'") + router.succeed("ip link set eth2 down; ip link set eth2 name uplink0; ip addr flush dev uplink0; ip addr add 192.168.2.1/24 dev uplink0; ip link set uplink0 up") + internet.succeed("ip addr flush dev eth1; ip addr add 192.168.2.2/24 dev eth1; ip addr add 203.0.113.1/32 dev eth1; ip addr add 100.100.100.1/32 dev eth1") + router.succeed("ip route add 203.0.113.1/32 dev uplink0; ip route add 100.100.100.1/32 dev uplink0") + guest.succeed("ip route replace default via 10.231.0.1 dev eth1") + + with subtest("unrelated host forwarding remains available"): + router.succeed("ip netns add transit; ip link add transit0 type veth peer name transit1 netns transit") + router.succeed("ip addr add 10.232.0.1/30 dev transit0; ip link set transit0 up") + router.succeed("ip -n transit addr add 10.232.0.2/30 dev transit1; ip -n transit link set transit1 up; ip -n transit route add default via 10.232.0.1") + internet.succeed("ip route add 10.232.0.0/30 via 192.168.2.1") + router.succeed("ip netns exec transit ${pkgs.curl}/bin/curl --fail --connect-timeout 5 http://203.0.113.1:8080/") + + with subtest("both guests reach public networks while host and sibling guests stay isolated"): + router.succeed("curl --fail http://10.231.1.2:8080/") + router.succeed("curl --fail http://10.231.0.2:8080/") + other.succeed("ip route replace default via 10.231.1.1 dev eth1") + other.succeed("curl --fail --connect-timeout 5 http://203.0.113.1:8080/") + other.fail("curl --fail --connect-timeout 1 http://10.231.0.1:8080/") + other.fail("curl --fail --connect-timeout 1 http://10.231.0.2:8080/") + router.succeed("curl --fail http://192.168.2.2:8080/") + guest.succeed("curl --fail --connect-timeout 5 http://203.0.113.1:8080/") + for address in ["10.231.0.1", "10.231.1.2", "192.168.2.2", "100.100.100.1"]: + guest.fail(f"curl --fail --connect-timeout 1 http://{address}:8080/") + router.succeed("nft list table inet github-runner-queue-ci") + router.succeed("nft list table inet github-runner-other-ci") + + with subtest("a guest cannot impersonate a sibling's established reply to the host"): + other.succeed("systemd-run --unit=udp-peer --service-type=exec ${pkgs.python3}/bin/python3 ${./spoof-test.py} peer") + other.wait_for_file("/tmp/udp-peer-ready") + router.succeed("systemd-run --unit=udp-receiver --service-type=exec ${pkgs.python3}/bin/python3 ${./spoof-test.py} receiver") + router.wait_for_file("/tmp/udp-receiver-ready") + guest.succeed("${ + pkgs.python3.withPackages (p: [ p.scapy ]) + }/bin/python3 ${./spoof-test.py} attack") + # Observe the spoof being dropped before releasing the legitimate reply. + router.wait_until_succeeds("nft list chain inet github-runner-queue-ci input | grep -E 'ip saddr != 10.231.0.2 counter packets [1-9]'") + other.succeed("${pkgs.python3}/bin/python3 ${./spoof-test.py} finish") + router.wait_for_file("/tmp/udp-result") + router.succeed("test $(cat /tmp/udp-result) = finish") + + with subtest("the runner inherits the hosted CI locked-memory budget"): + guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/memlock") + guest.succeed("test $(cat /var/cache/github-runner-home/.cache/mbx/memlock) = 65536") + + with subtest("local Nix and mbx storage survive runner and VM restarts"): + guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/marker") + guest.succeed("echo cached > /tmp/nix-cache-marker") + store_path = guest.succeed("nix-store --add /tmp/nix-cache-marker").strip() + guest.succeed("systemctl restart github-runner-runner") + guest.wait_for_file("/var/cache/github-runner-home/.cache/mbx/reused") + guest.shutdown() + guest.start() + guest.wait_for_unit("github-runner-runner.service") + guest.succeed("test -f /var/cache/github-runner-home/.cache/mbx/reused") + guest.succeed(f"test -f {store_path}") + ''; +} diff --git a/sh/gh/queue.test.ts b/sh/gh/queue.test.ts index 9a4d1b50346..12c3cb14538 100644 --- a/sh/gh/queue.test.ts +++ b/sh/gh/queue.test.ts @@ -63,7 +63,7 @@ assert.equal(nightly.jobs.platform.uses, "./.github/workflows/platform.yml"); assert.deepEqual(platform.on.push.branches, ["release"]); // Nightly covers what main PRs skip. -for (const name of ["x11", "android", "cpp", "wasm", "audio-quality", "oss-scanner"]) { +for (const name of ["x11", "android", "cpp", "wasm", "audio-quality", "oss-scanner", "runner-vm"]) { assert.ok( Object.values(nightly.jobs).some( (job) => (job as { uses?: string }).uses === `./.github/workflows/${name}.yml`, diff --git a/sh/gh/setup.test.ts b/sh/gh/setup.test.ts index 815af5a3a91..40eef296824 100644 --- a/sh/gh/setup.test.ts +++ b/sh/gh/setup.test.ts @@ -1,4 +1,4 @@ -// The hosted runner still owns timing and action execution. This checks its setup gates. +// Runner timing and action execution stay in CI. These tests check the setup gates. import { expect, test } from "bun:test"; type Step = { @@ -10,7 +10,7 @@ type Step = { env?: Record; "continue-on-error"?: boolean; }; -type Job = { name: string; if?: string; steps: Step[] }; +type Job = { name: string; if?: string; steps: Step[]; "runs-on": string }; const workflow = Bun.YAML.parse(await Bun.file(".github/workflows/check.yml").text()) as { on: Record; jobs: Record; @@ -35,14 +35,25 @@ for (const name of ["check", "test"]) { // No expensive action or command may bypass the query, including future additions. // The Rust cache is the largest restore, so it also waits for selected work that compiles Rust. const setup = job.steps.slice(scopeIndex + 1); - expect(setup.some((step) => step.uses?.startsWith("DeterminateSystems/nix-installer-action@"))).toBe(true); + expect(setup.some((step) => step.uses === "./.github/actions/nix")).toBe(true); expect(setup.some((step) => step.uses === "./.github/actions/rust-cache")).toBe(true); expect(setup.at(-1)?.name).toBe(job.name); for (const step of setup) { const output = step.uses === "./.github/actions/rust-cache" ? "rust" : "needed"; - expect(step.if).toBe(`steps.scope.outputs.${output} == 'true'`); + const hosted = step.uses === "./.github/actions/nix" + ? " && runner.environment != 'self-hosted'" + : ""; + expect(step.if).toBe(`steps.scope.outputs.${output} == 'true'${hosted}`); } }); + + test(`${name} uses the configured runner only for merge groups`, () => { + const expression = workflow.jobs[name]["runs-on"].replace(/^\$\{\{\s*|\s*\}\}$/g, ""); + const runner = new Function("github", "vars", `return (${expression});`); + expect(runner({ event_name: "merge_group" }, { MOQ_QUEUE_RUNNER: "moq-queue" })).toBe("moq-queue"); + expect(runner({ event_name: "merge_group" }, {})).toBe("ubuntu-24.04-arm"); + expect(runner({ event_name: "pull_request" }, { MOQ_QUEUE_RUNNER: "moq-queue" })).toBe("ubuntu-24.04-arm"); + }); } test("scope queries support both merge groups and release PRs without path filters", () => { @@ -62,3 +73,33 @@ test("main PRs skip both jobs while the queue and release PRs run them", () => { ); } }); + +test("self-hosted queue jobs use local caches without R2 credentials", async () => { + const action = Bun.YAML.parse(await Bun.file(".github/actions/rust-cache/action.yml").text()); + const setup = action.runs.steps.find((step: Step) => step.uses?.startsWith("jdx/mr-boxington-action@")); + const expression = setup.with.backend + .replaceAll("inputs.access-key-id", 'inputs["access-key-id"]') + .replaceAll("inputs.secret-access-key", 'inputs["secret-access-key"]') + .replace(/^\$\{\{\s*|\s*\}\}$/g, ""); + const backend = new Function("runner", "inputs", `return (${expression});`); + const guard = new Function("runner", "github", `return (${action.runs.steps[0].if});`); + for (const event_name of ["merge_group", "pull_request", "pull_request_target", "push", "workflow_dispatch"]) { + expect(guard({ environment: "self-hosted" }, { event_name })).toBe(event_name !== "merge_group"); + expect(guard({ environment: "github-hosted" }, { event_name })).toBe(false); + } + for (const credentials of [ + { "access-key-id": "key", "secret-access-key": "secret" }, + { "access-key-id": "", "secret-access-key": "" }, + ]) { + expect(backend({ environment: "self-hosted" }, credentials)).toBe("local"); + expect(backend({ environment: "github-hosted" }, credentials)).toBe( + credentials["access-key-id"] ? "remote" : "local", + ); + } + for (const name of ["Cache credentials", "Check the cache"]) { + expect(action.runs.steps.find((step: Step) => step.name === name).if).toStartWith( + "runner.environment != 'self-hosted'", + ); + } + expect(action.runs.steps[0].run).toContain("exit 1"); +});