feat(computer): the QEMU launcher boots Windows guests and shares a thin disk's base

.github/scripts/infra_build.py

@@ -136,7 +136,8 @@
     "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",
                                "qemu-system-x86_64 --version && qemu-system-aarch64 --version"
                                " && test -f /usr/share/OVMF/OVMF_CODE_4M.fd && test -f /usr/share/AAVMF/AAVMF_CODE.fd"
-                               " && command -v socat zstd"],
+                               " && test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd"
+                               " && test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd && command -v socat zstd swtpm"],
 }
 
 

src/agent_env/artifact/artifacts/computer.py

@@ -39,6 +39,9 @@
 LOCAL_IMAGE_PREFIX = "local:"
 #: The file a disk form's image holds, as KubeVirt's containerDisk has it, owned by the uid its QEMU runs as.
 DISK_FILE, DISK_OWNER = "disk/disk.img", 107
+#: A thin disk's base, in a layer of its own below ``DISK_FILE``'s, and the guest its launcher boots it as (``windows``),
+#: in ``DISK_FILE``'s layer.
+DISK_BASE_FILE, DISK_GUEST_FILE = "disk/base.img", "disk/guest"
 #: What a setting holds: its default's type, which a value an env or a deploy gives must share.
 SettingValue = str | int | float | bool
 

src/agent_env/computers/disk/linux/launcher/Dockerfile

@@ -1,10 +1,10 @@
-# The QEMU launcher: boots a Linux computer's disk form, mounted or copied at /disk/disk.img, with its settings in a
-# NoCloud seed (see launch). It boots either arch's disk, with KVM where /dev/kvm is passed in and the arch is its own,
-# behind socat on each forwarded port, and expands a zstd-compressed disk.
+# The QEMU launcher: boots a computer's disk form, mounted or copied at /disk/disk.img, with its settings in a NoCloud
+# seed (see launch). It boots either arch's disk, with KVM where /dev/kvm is passed in and the arch is its own, behind
+# socat on each forwarded port, and expands a zstd-compressed disk; a windows guest gets swtpm's TPM.
 FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
-        qemu-system-x86 qemu-system-arm qemu-utils ovmf qemu-efi-aarch64 dosfstools mtools socat zstd \
+        qemu-system-x86 qemu-system-arm qemu-utils ovmf qemu-efi-aarch64 dosfstools mtools socat zstd swtpm \
     && rm -rf /var/lib/apt/lists/*
 COPY launch seed /usr/local/bin/
 ENTRYPOINT ["/usr/local/bin/launch"]

src/agent_env/computers/disk/linux/launcher/launch

@@ -1,8 +1,11 @@
 #!/bin/bash
-# launch: boot a Linux computer's disk form under QEMU until it powers off, on a copy-on-write overlay so the disk stays
-# as it is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from UEFI firmware. The
-# guest's console is this process's output.
-#   VM_DISK        the disk, /disk/disk.img by default; one ending .zst is expanded onto this machine's disk first
+# launch: boot a computer's disk form under QEMU until it powers off, on a copy-on-write overlay so the disk stays as it
+# is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from UEFI firmware. The guest's
+# console is this process's output. The guest is what the guest file beside the disk names, linux when there's none; a
+# windows guest, which has no virtio drivers, boots from secure-boot UEFI with Microsoft's keys and a fresh TPM 2.0,
+# its disk and seed on AHCI, an e1000e NIC, a 1920x1080 display and, under KVM, Hyper-V enlightenments.
+#   VM_DISK        the disk, /disk/disk.img by default, which may be backed by another beside it; one ending .zst is
+#                  expanded onto this machine's disk first
 #   VM_ENV_<NAME>  the computer's variable NAME, its value base64'd, for its NoCloud seed (CIDATA; see seed)
 #   VM_PORTS       guest TCP ports, each served on the same port here by socat, whose listen backlog takes a burst of
 #                  connections, in front of QEMU's forward on 127.0.0.1 at the port plus 10000, whose backlog is 1;
@@ -39,6 +42,8 @@ fi
 here=$(dpkg --print-architecture)
 arch=${VM_ARCH:-$here}
 disk=${VM_DISK:-/disk/disk.img}
+guest=linux
+[ -s "$(dirname "$disk")/guest" ] && guest=$(cat "$(dirname "$disk")/guest")
 work=$state/boot
 mkdir -p "$control"
 rm -f "$control/qemu.pid" "$control/reset" "$control/next.env"
@@ -50,11 +55,24 @@ elif [ -n "${VM_REQUIRE_KVM:-}" ]; then
          "($([ -e /dev/kvm ] && echo "/dev/kvm isn't writable" || echo "there's no /dev/kvm"))" >&2
     exit 3
 fi
-case "$arch" in
-    amd64) machine=(qemu-system-x86_64 -machine "q35,accel=$accel"
-                    -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd) ;;
-    arm64) machine=(qemu-system-aarch64 -machine "virt,accel=$accel"
-                    -drive if=pflash,format=raw,readonly=on,file="$work/code.fd") ;;
+case "$guest/$arch" in
+    linux/*) ;;
+    windows/amd64) [ "$accel" = kvm ] && cpu=host,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_time ;;
+    windows/*)
+        echo "launch: a windows guest boots on amd64 alone, not $arch" >&2
+        exit 2 ;;
+    *)
+        echo "launch: no guest $guest" >&2
+        exit 2 ;;
+esac
+case "$guest/$arch" in
+    windows/amd64) machine=(qemu-system-x86_64 -machine "q35,smm=on,accel=$accel"
+                            -global driver=cfi.pflash01,property=secure,value=on
+                            -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd) ;;
+    */amd64) machine=(qemu-system-x86_64 -machine "q35,accel=$accel"
+                      -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd) ;;
+    */arm64) machine=(qemu-system-aarch64 -machine "virt,accel=$accel"
+                      -drive if=pflash,format=raw,readonly=on,file="$work/code.fd") ;;
     *)
         echo "launch: no architecture $arch" >&2
         exit 2 ;;
@@ -85,17 +103,28 @@ boot() {
     seed "$work/seed"
     mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null
     mcopy -i "$work/seed.img" "$work/seed/"* ::/
-    if [ "$arch" = amd64 ]; then
-        cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd"
+    if [ "$guest" = windows ]; then
+        cp /usr/share/OVMF/OVMF_VARS_4M.ms.fd "$work/vars.fd"
+        mkdir "$work/tpm"
+        swtpm socket --tpm2 --tpmstate dir="$work/tpm" --ctrl type=unixio,path="$work/tpm.sock" --daemon --terminate
+        devices=(-chardev socket,id=tpm,path="$work/tpm.sock" -tpmdev emulator,id=tpm,chardev=tpm
+                 -device tpm-tis,tpmdev=tpm
+                 -drive file="$work/disk.qcow2",format=qcow2,if=none,id=disk -device ide-hd,drive=disk,bus=ide.0
+                 -drive file="$work/seed.img",format=raw,if=none,id=seed -device ide-hd,drive=seed,bus=ide.1
+                 -device e1000e,netdev=net -vga none -device VGA,xres=1920,yres=1080)
     else
-        cp /usr/share/AAVMF/AAVMF_CODE.fd "$work/code.fd"
-        truncate -s 64M "$work/code.fd" "$work/vars.fd"
+        if [ "$arch" = amd64 ]; then
+            cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd"
+        else
+            cp /usr/share/AAVMF/AAVMF_CODE.fd "$work/code.fd"
+            truncate -s 64M "$work/code.fd" "$work/vars.fd"
+        fi
+        devices=(-drive file="$work/disk.qcow2",format=qcow2,if=virtio
+                 -drive file="$work/seed.img",format=raw,if=virtio,readonly=on
+                 -device virtio-net-pci,netdev=net -device virtio-rng-pci)
     fi
     "${machine[@]}" -cpu "$cpu" -smp "${VM_CPUS:-2}" -m "${VM_MEMORY_MB:-4096}" \
-        -drive if=pflash,format=raw,file="$work/vars.fd" \
-        -drive file="$work/disk.qcow2",format=qcow2,if=virtio \
-        -drive file="$work/seed.img",format=raw,if=virtio,readonly=on \
-        -netdev "user,id=net$forwards" -device virtio-net-pci,netdev=net -device virtio-rng-pci \
+        -drive if=pflash,format=raw,file="$work/vars.fd" -netdev "user,id=net$forwards" "${devices[@]}" \
         -display none -serial stdio -no-reboot &
     echo "$!" > "$control/qemu.pid"
 }

src/agent_env/providers/sandbox_providers/local_docker/disk.py

@@ -10,7 +10,7 @@
 import tempfile
 from pathlib import Path, PurePosixPath
 
-from agent_env.artifact.artifacts.computer import DISK_FILE
+from agent_env.artifact.artifacts.computer import DISK_BASE_FILE, DISK_FILE, DISK_GUEST_FILE
 from agent_env.artifact.artifacts.docker_image import docker_login
 from agent_env.config import get_config
 from agent_env.config.paths import cache_root
@@ -30,15 +30,25 @@ def kvm_platforms() -> frozenset[str]:
 
 
 async def machine_image(disk: str) -> str:
-    """An image of the launcher holding ``disk``'s ``DISK_FILE`` for this machine's platform, built unless held."""
+    """An image of the launcher holding ``disk``'s files for this machine's platform, built unless held. A disk of more
+    than one layer is a thin one on ``DISK_BASE_FILE``, which gets a step of its own, so every machine on that base
+    shares one copy of it."""
     launcher = await asyncio.to_thread(_launcher)
     tag = "agentenv-vm:" + hashlib.sha256(f"{launcher} {disk}".encode()).hexdigest()[:16]
     if not await asyncio.to_thread(present, tag):
-        folder = f"/{PurePosixPath(DISK_FILE).parent}/"
-        await asyncio.to_thread(_build, f"FROM {launcher}\nCOPY --from={disk} {folder} {folder}\n", tag)
+        layers = int(await asyncio.to_thread(docker, "image", "inspect", "-f", "{{len .RootFS.Layers}}", disk))
+        await asyncio.to_thread(_build, _machine_dockerfile(launcher, disk, layers), tag)
     return tag
 
 
+def _machine_dockerfile(launcher: str, disk: str, layers: int) -> str:
+    folder = f"/{PurePosixPath(DISK_FILE).parent}/"
+    if layers == 1:
+        return f"FROM {launcher}\nCOPY --from={disk} {folder} {folder}\n"
+    return (f"FROM {launcher}\nCOPY --from={disk} /{DISK_BASE_FILE} /{DISK_BASE_FILE}\n"
+            f"COPY --from={disk} /{DISK_FILE} /{DISK_GUEST_FILE} {folder}\n")
+
+
 def launcher_args(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> list[str]:
     """The ``docker run`` arguments that pass the launcher KVM and its ``launcher_variables``."""
     return ["--device", KVM, *(arg for name, value in launcher_variables(ports, env, cpu, memory).items()

tst/unit/env/envs/test_disk_recipe.py

@@ -150,7 +150,8 @@ def test_the_launcher_gets_kvm_the_vms_ports_and_size_and_its_environment_under_
     "qemu-img": 'echo "qemu-img $*" >> "$CALLS"',
     "mkfs.vfat": ":",
     "mcopy": ":",
-    "cp": 'touch "${@: -1}"',
+    "cp": 'echo "cp $*" >> "$CALLS"; touch "${@: -1}"',
+    "swtpm": 'echo "swtpm $*" >> "$CALLS"',
     "zstd": 'echo "zstd $*" >> "$CALLS"; sleep "${ZSTD_TAKES_S:-0}"; touch "${@: -1}"',
     "socat": 'echo "socat $*" >> "$CALLS"; sleep "${SOCAT_EXITS_S:-30}"; exit 7',
     **{qemu: f'echo "{qemu} $* seed=$(grep -vc "^#" "$VM_STATE/boot/seed/user-data")'
@@ -276,6 +277,41 @@ def test_the_launcher_expands_a_compressed_disk_onto_its_state_folder_and_boots_
     assert f"-b {tmp_path}/state/disk.img -F qcow2" in _logged(calls, "qemu-img")[0]
 
 
+def test_a_windows_disk_boots_from_secure_boot_on_ahci_and_e1000e_with_a_fresh_tpm_each_time(launcher, tmp_path):
+    """Windows has no virtio drivers of its own and installs only with secure boot and a TPM 2.0."""
+    launch, calls = launcher
+    disk = tmp_path / "disk" / "disk.img"
+    disk.parent.mkdir()
+    (disk.parent / "guest").write_text("windows\n")
+    running = launch(VM_DISK=str(disk), VM_PORTS="18765")
+    (qemu,) = _logged(calls, "qemu-system-x86_64")
+
+    reset = launch("reset", input="")
+
+    assert reset.wait(10) == 0, reset.stderr.read()
+    work = tmp_path / "state" / "boot"
+    assert "-machine q35,smm=on,accel=" in qemu and "file=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd " in qemu
+    assert "-device tpm-tis,tpmdev=tpm " in qemu and "-device ide-hd,drive=disk,bus=ide.0 " in qemu
+    assert "-device e1000e,netdev=net " in qemu and "-device VGA,xres=1920,yres=1080 " in qemu and "virtio" not in qemu
+    assert _logged(calls, "swtpm", 2) == 2 * [f"swtpm socket --tpm2 --tpmstate dir={work}/tpm --ctrl"
+                                              f" type=unixio,path={work}/tpm.sock --daemon --terminate"]
+    assert f"cp /usr/share/OVMF/OVMF_VARS_4M.ms.fd {work}/vars.fd" in _logged(calls, "cp")
+    running.terminate()
+    assert running.wait(10) == 143
+
+
+@pytest.mark.parametrize("guest, arch, why", [("windows", "arm64", "a windows guest boots on amd64 alone, not arm64"),
+                                              ("plan9", "amd64", "no guest plan9")])
+def test_the_launcher_refuses_a_guest_it_cant_boot(launcher, tmp_path, guest, arch, why):
+    launch, calls = launcher
+    (tmp_path / "guest").write_text(guest)
+
+    refused = launch(VM_DISK=str(tmp_path / "disk.img"), VM_ARCH=arch)
+
+    assert refused.wait(10) == 2 and why in refused.stderr.read()
+    assert not calls.exists() or "qemu" not in calls.read_text()
+
+
 def test_a_reset_boots_the_computer_afresh_with_its_new_environment_while_its_ports_stay_up(launcher):
     launch, calls = launcher
     running = launch(VM_PORTS="18765", VM_ENV_A="YQ==")
@@ -427,6 +463,24 @@ async def test_a_disk_isnt_booted_where_docker_cant_pass_kvm(launched, monkeypat
     assert launched == [] and LocalSandboxProvider.disk_image_platforms() == frozenset()
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("layers, dockerfile", [
+    ("1", "FROM launcher:1\nCOPY --from=disk:1 /disk/ /disk/\n"),
+    ("2", "FROM launcher:1\nCOPY --from=disk:1 /disk/base.img /disk/base.img\n"
+          "COPY --from=disk:1 /disk/disk.img /disk/guest /disk/\n")], ids=["one-layer", "thin-on-a-base"])
+async def test_a_machine_copies_a_thin_disks_base_in_a_step_of_its_own_so_machines_on_one_base_share_it(
+        monkeypatch, layers, dockerfile):
+    built = []
+    monkeypatch.setattr(local_disk, "_launcher", lambda: "launcher:1")
+    monkeypatch.setattr(local_disk, "present", lambda tag: False)
+    monkeypatch.setattr(local_disk, "docker", lambda *args: layers)
+    monkeypatch.setattr(local_disk, "_build", lambda text, tag: built.append(text))
+
+    await local_disk.machine_image("disk:1")
+
+    assert built == [dockerfile]
+
+
 def test_a_disk_is_exported_compressed_into_the_cache_once_for_each_ref(tmp_path, monkeypatch):
     """By this machine's Docker, signed in to the image store as a boot's pull is: the disk copied out of a container of
     its image, then compressed in the launcher."""