From eed010930d4f9c395cacd59569a909db1e0057fc Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 6 Nov 2026 19:03:32 -0800 Subject: [PATCH] feat(computer): the QEMU launcher boots Windows guests and shares a thin disk's base The launcher boots a disk as the guest its folder's guest file names, linux when there's none. A windows guest, which has no virtio drivers, boots from secure-boot UEFI with Microsoft's keys and a fresh swtpm TPM 2.0 on every boot, its disk and seed on AHCI, an e1000e NIC, a 1920x1080 display and, under KVM, Hyper-V enlightenments; it boots on amd64 alone. The launcher image adds swtpm, and the release's smoke test checks for it and the secure-boot firmware. A local machine image of a thin disk (more than one layer: a base in disk/base.img below disk/disk.img and its guest file) copies the base in a step of its own, so every machine on one base shares a single copy of it. Part of FD-5311 (W1). The windows OS type stays refused until composing a Windows computer lands with it. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 3 +- src/agent_env/artifact/artifacts/computer.py | 3 + .../computers/disk/linux/launcher/Dockerfile | 8 +-- .../computers/disk/linux/launcher/launch | 63 ++++++++++++++----- .../sandbox_providers/local_docker/disk.py | 18 ++++-- tst/unit/env/envs/test_disk_recipe.py | 56 ++++++++++++++++- 6 files changed, 124 insertions(+), 27 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index a2ea9007b..f90e4eb5a 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -136,7 +136,8 @@ "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", "qemu-system-x86_64 --version && qemu-system-aarch64 --version" " && test -f /usr/share/OVMF/OVMF_CODE_4M.fd && test -f /usr/share/AAVMF/AAVMF_CODE.fd" - " && command -v socat zstd"], + " && test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd" + " && test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd && command -v socat zstd swtpm"], } diff --git a/src/agent_env/artifact/artifacts/computer.py b/src/agent_env/artifact/artifacts/computer.py index 3587557f1..a06a49ed9 100644 --- a/src/agent_env/artifact/artifacts/computer.py +++ b/src/agent_env/artifact/artifacts/computer.py @@ -39,6 +39,9 @@ LOCAL_IMAGE_PREFIX = "local:" #: The file a disk form's image holds, as KubeVirt's containerDisk has it, owned by the uid its QEMU runs as. DISK_FILE, DISK_OWNER = "disk/disk.img", 107 +#: A thin disk's base, in a layer of its own below ``DISK_FILE``'s, and the guest its launcher boots it as (``windows``), +#: in ``DISK_FILE``'s layer. +DISK_BASE_FILE, DISK_GUEST_FILE = "disk/base.img", "disk/guest" #: What a setting holds: its default's type, which a value an env or a deploy gives must share. SettingValue = str | int | float | bool diff --git a/src/agent_env/computers/disk/linux/launcher/Dockerfile b/src/agent_env/computers/disk/linux/launcher/Dockerfile index 8a3d108a9..5d66a74da 100644 --- a/src/agent_env/computers/disk/linux/launcher/Dockerfile +++ b/src/agent_env/computers/disk/linux/launcher/Dockerfile @@ -1,10 +1,10 @@ -# The QEMU launcher: boots a Linux computer's disk form, mounted or copied at /disk/disk.img, with its settings in a -# NoCloud seed (see launch). It boots either arch's disk, with KVM where /dev/kvm is passed in and the arch is its own, -# behind socat on each forwarded port, and expands a zstd-compressed disk. +# The QEMU launcher: boots a computer's disk form, mounted or copied at /disk/disk.img, with its settings in a NoCloud +# seed (see launch). It boots either arch's disk, with KVM where /dev/kvm is passed in and the arch is its own, behind +# socat on each forwarded port, and expands a zstd-compressed disk; a windows guest gets swtpm's TPM. FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - qemu-system-x86 qemu-system-arm qemu-utils ovmf qemu-efi-aarch64 dosfstools mtools socat zstd \ + qemu-system-x86 qemu-system-arm qemu-utils ovmf qemu-efi-aarch64 dosfstools mtools socat zstd swtpm \ && rm -rf /var/lib/apt/lists/* COPY launch seed /usr/local/bin/ ENTRYPOINT ["/usr/local/bin/launch"] diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index 3bfdf66bf..95b5322fd 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -1,8 +1,11 @@ #!/bin/bash -# launch: boot a Linux computer's disk form under QEMU until it powers off, on a copy-on-write overlay so the disk stays -# as it is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from UEFI firmware. The -# guest's console is this process's output. -# VM_DISK the disk, /disk/disk.img by default; one ending .zst is expanded onto this machine's disk first +# launch: boot a computer's disk form under QEMU until it powers off, on a copy-on-write overlay so the disk stays as it +# is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from UEFI firmware. The guest's +# console is this process's output. The guest is what the guest file beside the disk names, linux when there's none; a +# windows guest, which has no virtio drivers, boots from secure-boot UEFI with Microsoft's keys and a fresh TPM 2.0, +# its disk and seed on AHCI, an e1000e NIC, a 1920x1080 display and, under KVM, Hyper-V enlightenments. +# VM_DISK the disk, /disk/disk.img by default, which may be backed by another beside it; one ending .zst is +# expanded onto this machine's disk first # VM_ENV_ the computer's variable NAME, its value base64'd, for its NoCloud seed (CIDATA; see seed) # VM_PORTS guest TCP ports, each served on the same port here by socat, whose listen backlog takes a burst of # connections, in front of QEMU's forward on 127.0.0.1 at the port plus 10000, whose backlog is 1; @@ -39,6 +42,8 @@ fi here=$(dpkg --print-architecture) arch=${VM_ARCH:-$here} disk=${VM_DISK:-/disk/disk.img} +guest=linux +[ -s "$(dirname "$disk")/guest" ] && guest=$(cat "$(dirname "$disk")/guest") work=$state/boot mkdir -p "$control" rm -f "$control/qemu.pid" "$control/reset" "$control/next.env" @@ -50,11 +55,24 @@ elif [ -n "${VM_REQUIRE_KVM:-}" ]; then "($([ -e /dev/kvm ] && echo "/dev/kvm isn't writable" || echo "there's no /dev/kvm"))" >&2 exit 3 fi -case "$arch" in - amd64) machine=(qemu-system-x86_64 -machine "q35,accel=$accel" - -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd) ;; - arm64) machine=(qemu-system-aarch64 -machine "virt,accel=$accel" - -drive if=pflash,format=raw,readonly=on,file="$work/code.fd") ;; +case "$guest/$arch" in + linux/*) ;; + windows/amd64) [ "$accel" = kvm ] && cpu=host,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_time ;; + windows/*) + echo "launch: a windows guest boots on amd64 alone, not $arch" >&2 + exit 2 ;; + *) + echo "launch: no guest $guest" >&2 + exit 2 ;; +esac +case "$guest/$arch" in + windows/amd64) machine=(qemu-system-x86_64 -machine "q35,smm=on,accel=$accel" + -global driver=cfi.pflash01,property=secure,value=on + -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd) ;; + */amd64) machine=(qemu-system-x86_64 -machine "q35,accel=$accel" + -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd) ;; + */arm64) machine=(qemu-system-aarch64 -machine "virt,accel=$accel" + -drive if=pflash,format=raw,readonly=on,file="$work/code.fd") ;; *) echo "launch: no architecture $arch" >&2 exit 2 ;; @@ -85,17 +103,28 @@ boot() { seed "$work/seed" mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null mcopy -i "$work/seed.img" "$work/seed/"* ::/ - if [ "$arch" = amd64 ]; then - cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd" + if [ "$guest" = windows ]; then + cp /usr/share/OVMF/OVMF_VARS_4M.ms.fd "$work/vars.fd" + mkdir "$work/tpm" + swtpm socket --tpm2 --tpmstate dir="$work/tpm" --ctrl type=unixio,path="$work/tpm.sock" --daemon --terminate + devices=(-chardev socket,id=tpm,path="$work/tpm.sock" -tpmdev emulator,id=tpm,chardev=tpm + -device tpm-tis,tpmdev=tpm + -drive file="$work/disk.qcow2",format=qcow2,if=none,id=disk -device ide-hd,drive=disk,bus=ide.0 + -drive file="$work/seed.img",format=raw,if=none,id=seed -device ide-hd,drive=seed,bus=ide.1 + -device e1000e,netdev=net -vga none -device VGA,xres=1920,yres=1080) else - cp /usr/share/AAVMF/AAVMF_CODE.fd "$work/code.fd" - truncate -s 64M "$work/code.fd" "$work/vars.fd" + if [ "$arch" = amd64 ]; then + cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd" + else + cp /usr/share/AAVMF/AAVMF_CODE.fd "$work/code.fd" + truncate -s 64M "$work/code.fd" "$work/vars.fd" + fi + devices=(-drive file="$work/disk.qcow2",format=qcow2,if=virtio + -drive file="$work/seed.img",format=raw,if=virtio,readonly=on + -device virtio-net-pci,netdev=net -device virtio-rng-pci) fi "${machine[@]}" -cpu "$cpu" -smp "${VM_CPUS:-2}" -m "${VM_MEMORY_MB:-4096}" \ - -drive if=pflash,format=raw,file="$work/vars.fd" \ - -drive file="$work/disk.qcow2",format=qcow2,if=virtio \ - -drive file="$work/seed.img",format=raw,if=virtio,readonly=on \ - -netdev "user,id=net$forwards" -device virtio-net-pci,netdev=net -device virtio-rng-pci \ + -drive if=pflash,format=raw,file="$work/vars.fd" -netdev "user,id=net$forwards" "${devices[@]}" \ -display none -serial stdio -no-reboot & echo "$!" > "$control/qemu.pid" } diff --git a/src/agent_env/providers/sandbox_providers/local_docker/disk.py b/src/agent_env/providers/sandbox_providers/local_docker/disk.py index 4f387f691..20048e2ce 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/disk.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/disk.py @@ -10,7 +10,7 @@ import tempfile from pathlib import Path, PurePosixPath -from agent_env.artifact.artifacts.computer import DISK_FILE +from agent_env.artifact.artifacts.computer import DISK_BASE_FILE, DISK_FILE, DISK_GUEST_FILE from agent_env.artifact.artifacts.docker_image import docker_login from agent_env.config import get_config from agent_env.config.paths import cache_root @@ -30,15 +30,25 @@ def kvm_platforms() -> frozenset[str]: async def machine_image(disk: str) -> str: - """An image of the launcher holding ``disk``'s ``DISK_FILE`` for this machine's platform, built unless held.""" + """An image of the launcher holding ``disk``'s files for this machine's platform, built unless held. A disk of more + than one layer is a thin one on ``DISK_BASE_FILE``, which gets a step of its own, so every machine on that base + shares one copy of it.""" launcher = await asyncio.to_thread(_launcher) tag = "agentenv-vm:" + hashlib.sha256(f"{launcher} {disk}".encode()).hexdigest()[:16] if not await asyncio.to_thread(present, tag): - folder = f"/{PurePosixPath(DISK_FILE).parent}/" - await asyncio.to_thread(_build, f"FROM {launcher}\nCOPY --from={disk} {folder} {folder}\n", tag) + layers = int(await asyncio.to_thread(docker, "image", "inspect", "-f", "{{len .RootFS.Layers}}", disk)) + await asyncio.to_thread(_build, _machine_dockerfile(launcher, disk, layers), tag) return tag +def _machine_dockerfile(launcher: str, disk: str, layers: int) -> str: + folder = f"/{PurePosixPath(DISK_FILE).parent}/" + if layers == 1: + return f"FROM {launcher}\nCOPY --from={disk} {folder} {folder}\n" + return (f"FROM {launcher}\nCOPY --from={disk} /{DISK_BASE_FILE} /{DISK_BASE_FILE}\n" + f"COPY --from={disk} /{DISK_FILE} /{DISK_GUEST_FILE} {folder}\n") + + def launcher_args(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> list[str]: """The ``docker run`` arguments that pass the launcher KVM and its ``launcher_variables``.""" return ["--device", KVM, *(arg for name, value in launcher_variables(ports, env, cpu, memory).items() diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index 7aeeadf21..379298a53 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -150,7 +150,8 @@ def test_the_launcher_gets_kvm_the_vms_ports_and_size_and_its_environment_under_ "qemu-img": 'echo "qemu-img $*" >> "$CALLS"', "mkfs.vfat": ":", "mcopy": ":", - "cp": 'touch "${@: -1}"', + "cp": 'echo "cp $*" >> "$CALLS"; touch "${@: -1}"', + "swtpm": 'echo "swtpm $*" >> "$CALLS"', "zstd": 'echo "zstd $*" >> "$CALLS"; sleep "${ZSTD_TAKES_S:-0}"; touch "${@: -1}"', "socat": 'echo "socat $*" >> "$CALLS"; sleep "${SOCAT_EXITS_S:-30}"; exit 7', **{qemu: f'echo "{qemu} $* seed=$(grep -vc "^#" "$VM_STATE/boot/seed/user-data")' @@ -276,6 +277,41 @@ def test_the_launcher_expands_a_compressed_disk_onto_its_state_folder_and_boots_ assert f"-b {tmp_path}/state/disk.img -F qcow2" in _logged(calls, "qemu-img")[0] +def test_a_windows_disk_boots_from_secure_boot_on_ahci_and_e1000e_with_a_fresh_tpm_each_time(launcher, tmp_path): + """Windows has no virtio drivers of its own and installs only with secure boot and a TPM 2.0.""" + launch, calls = launcher + disk = tmp_path / "disk" / "disk.img" + disk.parent.mkdir() + (disk.parent / "guest").write_text("windows\n") + running = launch(VM_DISK=str(disk), VM_PORTS="18765") + (qemu,) = _logged(calls, "qemu-system-x86_64") + + reset = launch("reset", input="") + + assert reset.wait(10) == 0, reset.stderr.read() + work = tmp_path / "state" / "boot" + assert "-machine q35,smm=on,accel=" in qemu and "file=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd " in qemu + assert "-device tpm-tis,tpmdev=tpm " in qemu and "-device ide-hd,drive=disk,bus=ide.0 " in qemu + assert "-device e1000e,netdev=net " in qemu and "-device VGA,xres=1920,yres=1080 " in qemu and "virtio" not in qemu + assert _logged(calls, "swtpm", 2) == 2 * [f"swtpm socket --tpm2 --tpmstate dir={work}/tpm --ctrl" + f" type=unixio,path={work}/tpm.sock --daemon --terminate"] + assert f"cp /usr/share/OVMF/OVMF_VARS_4M.ms.fd {work}/vars.fd" in _logged(calls, "cp") + running.terminate() + assert running.wait(10) == 143 + + +@pytest.mark.parametrize("guest, arch, why", [("windows", "arm64", "a windows guest boots on amd64 alone, not arm64"), + ("plan9", "amd64", "no guest plan9")]) +def test_the_launcher_refuses_a_guest_it_cant_boot(launcher, tmp_path, guest, arch, why): + launch, calls = launcher + (tmp_path / "guest").write_text(guest) + + refused = launch(VM_DISK=str(tmp_path / "disk.img"), VM_ARCH=arch) + + assert refused.wait(10) == 2 and why in refused.stderr.read() + assert not calls.exists() or "qemu" not in calls.read_text() + + def test_a_reset_boots_the_computer_afresh_with_its_new_environment_while_its_ports_stay_up(launcher): launch, calls = launcher running = launch(VM_PORTS="18765", VM_ENV_A="YQ==") @@ -427,6 +463,24 @@ async def test_a_disk_isnt_booted_where_docker_cant_pass_kvm(launched, monkeypat assert launched == [] and LocalSandboxProvider.disk_image_platforms() == frozenset() +@pytest.mark.asyncio +@pytest.mark.parametrize("layers, dockerfile", [ + ("1", "FROM launcher:1\nCOPY --from=disk:1 /disk/ /disk/\n"), + ("2", "FROM launcher:1\nCOPY --from=disk:1 /disk/base.img /disk/base.img\n" + "COPY --from=disk:1 /disk/disk.img /disk/guest /disk/\n")], ids=["one-layer", "thin-on-a-base"]) +async def test_a_machine_copies_a_thin_disks_base_in_a_step_of_its_own_so_machines_on_one_base_share_it( + monkeypatch, layers, dockerfile): + built = [] + monkeypatch.setattr(local_disk, "_launcher", lambda: "launcher:1") + monkeypatch.setattr(local_disk, "present", lambda tag: False) + monkeypatch.setattr(local_disk, "docker", lambda *args: layers) + monkeypatch.setattr(local_disk, "_build", lambda text, tag: built.append(text)) + + await local_disk.machine_image("disk:1") + + assert built == [dockerfile] + + def test_a_disk_is_exported_compressed_into_the_cache_once_for_each_ref(tmp_path, monkeypatch): """By this machine's Docker, signed in to the image store as a boot's pull is: the disk copied out of a container of its image, then compressed in the launcher."""