perf(computer): trim Windows disks and keep background app installs out of them

src/agent_env/computers/disk/linux/launcher/launch

@@ -3,8 +3,9 @@
 # is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from UEFI firmware. The guest's
 # console is this process's output. The guest is what the guest file beside the disk names, linux when there's none; a
 # windows guest, which has no virtio drivers, boots from secure-boot UEFI with Microsoft's keys and a fresh TPM 2.0,
-# its disk on AHCI, its seed on a removable USB drive, since Windows mounts a FAT filesystem with no partition table on
-# removable media alone, an e1000e NIC, a 1920x1080 display and, under KVM, Hyper-V enlightenments.
+# its disk on AHCI, an SSD to Windows, whose TRIM frees the overlay's clusters, its seed on a removable USB drive, since
+# Windows mounts a FAT filesystem with no partition table on removable media alone, an e1000e NIC, a 1920x1080 display
+# and, under KVM, Hyper-V enlightenments.
 #   VM_DISK        the disk, /disk/disk.img by default, which may be backed by another beside it; one ending .zst is
 #                  expanded onto this machine's disk first
 #   VM_ENV_<NAME>  the computer's variable NAME, its value base64'd, for its NoCloud seed (CIDATA; see seed)
@@ -137,7 +138,8 @@ boot() {
         swtpm socket --tpm2 --tpmstate dir="$work/tpm" --ctrl type=unixio,path="$work/tpm.sock" --daemon --terminate
         devices=(-chardev socket,id=tpm,path="$work/tpm.sock" -tpmdev emulator,id=tpm,chardev=tpm
                  -device tpm-tis,tpmdev=tpm
-                 -drive file="$booted",format=qcow2,if=none,id=disk -device ide-hd,drive=disk,bus=ide.0,bootindex=1
+                 -drive file="$booted",format=qcow2,if=none,id=disk,discard=unmap,detect-zeroes=unmap
+                 -device ide-hd,drive=disk,bus=ide.0,bootindex=1,rotation_rate=1
                  -device qemu-xhci,id=usb
                  -drive file="$work/seed.img",format=raw,if=none,id=seed -device usb-storage,bus=usb.0,drive=seed,removable=on
                  -device e1000e,netdev=net -vga none -device VGA,xres=1920,yres=1080)

src/agent_env/computers/disk/windows/compact.ps1

@@ -0,0 +1,11 @@
+# compact.ps1: run as agentadmin over ssh as a Windows disk is about to shut down to be packed, a base or a composed
+# layer. It removes what Windows keeps that a computer doesn't need (update downloads, temp files, the delivery
+# optimization cache), then trims the free space, which the launcher's disk, attached with discards, passes to its image,
+# so packing it keeps only what Windows uses.
+$ErrorActionPreference = 'Continue'
+$ProgressPreference = 'SilentlyContinue'
+$caches = @("$env:SystemRoot\SoftwareDistribution\Download", "$env:SystemRoot\Temp") +
+    (Get-ChildItem C:\Users -Directory -Force | ForEach-Object { "$($_.FullName)\AppData\Local\Temp" })
+foreach ($cache in $caches) { Get-ChildItem -Force $cache -ErrorAction SilentlyContinue | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue }
+Delete-DeliveryOptimizationCache -Force -ErrorAction SilentlyContinue
+Optimize-Volume -DriveLetter C -ReTrim

src/agent_env/computers/disk/windows/setup.ps1

@@ -1,9 +1,10 @@
 # setup.ps1: a Windows base readied for agent-env, once, elevated, at agentadmin's first sign-in. OpenSSH comes first
 # (agentadmin by agent-env's key alone), which is how agent-env composes the base and reads this setup's log when it
 # fails (setup-failed). Then nothing of Windows' own is left in an agent's way: no updates, lock screen, notifications,
-# widgets, Copilot or first-run pages, and no page file, which would grow every layer composed on the base. The pinned
-# Python, with Tk, is what layers' venvs and the probe's fixture run on; agentenv-session starts the session's services,
-# windowless, at the session user's sign-in. Then the session user signs in at every boot.
+# widgets, Copilot, first-run pages or apps Windows installs in the background, and no page file, all of which would grow
+# every layer composed on the base. The pinned Python, with Tk, is what layers' venvs and the probe's fixture run on;
+# agentenv-session starts the session's services, windowless, at the session user's sign-in. Then the session user signs
+# in at every boot.
 param([string]$Media, [string]$User, [string]$UserPassword)
 $ErrorActionPreference = 'Stop'
 $ProgressPreference = 'SilentlyContinue'
@@ -66,6 +67,18 @@ try {
     [gc]::Collect()
     reg.exe unload HKU\AgentenvDefault | Out-Null
 
+    # Windows' background installs: Edge's updater, which brings the Copilot app, and the new Outlook and Dev Home, which
+    # Windows installs after a first sign-in. Each would land in every composed layer and run at every boot.
+    Set-Values 'HKLM\SOFTWARE\Policies\Microsoft\EdgeUpdate' @{ UpdateDefault = 0; InstallDefault = 0; AutoUpdateCheckPeriodMinutes = 0 }
+    foreach ($service in 'edgeupdate', 'edgeupdatem') {
+        Stop-Service $service -Force -ErrorAction SilentlyContinue
+        Set-Service $service -StartupType Disabled -ErrorAction SilentlyContinue
+    }
+    Get-ScheduledTask -TaskName 'MicrosoftEdgeUpdate*' -ErrorAction SilentlyContinue | Disable-ScheduledTask | Out-Null
+    Remove-Item -Recurse -Force -ErrorAction SilentlyContinue `
+        'HKLM:\SOFTWARE\Microsoft\WindowsUpdate\Orchestrator\UScheduler_Oobe\OutlookUpdate',
+        'HKLM:\SOFTWARE\Microsoft\WindowsUpdate\Orchestrator\UScheduler_Oobe\DevHomeUpdate'
+
     powercfg /change standby-timeout-ac 0
     powercfg /change monitor-timeout-ac 0
     powercfg /hibernate off

src/agent_env/env/envs/computer/utils/windows_disk.py

@@ -55,6 +55,8 @@
 INSTALL_TIMEOUT_S, COMPOSE_TIMEOUT_S, SETTLE_S = 3600, 3600, 60
 #: How long a guest may take to power off once told.
 SHUTDOWN_TIMEOUT_S = 300
+#: How long trimming a disk's free space may take.
+COMPACT_TIMEOUT_S = 1800
 #: How often an install looks for setup's end, a compose for the base's ssh, and a shutdown for the guest's.
 INSTALL_POLL_S, BOOT_POLL_S, SHUTDOWN_POLL_S = 15, 5, 2
 CPUS, MEMORY_MB = 4, 8192
@@ -176,9 +178,11 @@ def _installed(work: Path, iso: Path, keys: Path, disk_gb: int, user: str, say:
         say(f"installed and signed in as {user} after {time.monotonic() - started:.0f} s; settling")
         time.sleep(SETTLE_S)
         facts = json.loads(ssh(port, keys, powershell(_FACTS)).stdout.strip().splitlines()[-1])
+        _compacted(port, keys, say)
         _shut_down(container, port, keys)
     finally:
         subprocess.run(["docker", "rm", "-f", container], capture_output=True)
+    _converted(out, launcher())
     shutil.move(out / "disk.img", work / "base.img")
     return facts
 
@@ -191,7 +195,8 @@ def compose(base_: Base, build: Path, image: str, labels: dict[str, str], say: C
         out.mkdir()
         name = f"agentenv-windows-compose-{image.rpartition(':')[2]}"
         say(f"booting its base in {name}")
-        container = _launched(name, {"VM_KEEP": "/out/disk.img"}, ["-v", f"{out}:/out"], asyncio.run(machine_image(base_.image)))
+        machine = asyncio.run(machine_image(base_.image))
+        container = _launched(name, {"VM_KEEP": "/out/disk.img"}, ["-v", f"{out}:/out"], machine)
         try:
             port, deadline = _ssh_port(container), time.monotonic() + COMPOSE_TIMEOUT_S
             while ssh(port, base_.keys, "echo ready", check=False, timeout=60).returncode != 0:
@@ -205,9 +210,11 @@ def compose(base_: Base, build: Path, image: str, labels: dict[str, str], say: C
             if done.returncode != 0:
                 raise RuntimeError(f"provisioning exited {done.returncode}: {(done.stderr + done.stdout).strip()[-3000:]}")
             ssh(port, base_.keys, f'rmdir /s /q "{BUILD}"')
+            _compacted(port, base_.keys, say)
             _shut_down(container, port, base_.keys)
         finally:
             subprocess.run(["docker", "rm", "-f", container], capture_output=True)
+        _converted(out, machine, backing=f"/{DISK_BASE_FILE}")
         subprocess.run(["docker", "run", "--rm", "-v", f"{out}:/out", "--entrypoint", "qemu-img", launcher(), "rebase",
                         "-u", "-b", PureWindowsPath(DISK_BASE_FILE).name, "-F", "qcow2", "/out/disk.img"],
                        check=True, capture_output=True)
@@ -327,6 +334,24 @@ def _signed_in(port: int, keys: Path, user: str) -> bool:
                      re.MULTILINE | re.IGNORECASE) is not None
 
 
+def _compacted(port: int, keys: Path, say: Callable[[str], None]) -> None:
+    """What the guest keeps that a computer doesn't need removed and its free space trimmed (``compact.ps1``), so its
+    disk's image, rewritten after it shuts down, keeps only what Windows uses."""
+    say("trimming its free space")
+    done = ssh(port, keys, powershell((RECIPE / "compact.ps1").read_text()), check=False, timeout=COMPACT_TIMEOUT_S)
+    if done.returncode != 0:
+        say(f"trimming exited {done.returncode}, so its image keeps what it held: {(done.stderr + done.stdout)[-300:]}")
+
+
+def _converted(folder: Path, image: str, backing: str | None = None) -> None:
+    """``folder``'s ``disk.img`` rewritten by ``qemu-img convert`` in ``image``, which holds its backing file, keeping
+    only the clusters it uses: a base's whole, a layer's over ``backing``, the backing file ``image`` holds."""
+    over = ["-B", backing, "-F", "qcow2"] if backing else []
+    subprocess.run(["docker", "run", "--rm", "-v", f"{folder}:/out", "--entrypoint", "qemu-img", image, "convert", "-O",
+                    "qcow2", *over, "/out/disk.img", "/out/compact.img"], check=True, capture_output=True)
+    (folder / "compact.img").replace(folder / "disk.img")
+
+
 def _launched(name: str, variables: dict[str, str], mounts: list[str], image: str) -> str:
     """A launcher container of ``image``, named ``name``, booting with ``variables`` and KVM, its guest's ssh published
     on a port of this machine's loopback; any earlier one of the name removed first."""

tst/unit/env/envs/test_computer_windows.py

@@ -244,3 +244,22 @@ def test_a_windows_computers_display_is_1920x1080_for_now(local_stores, cli_rout
         "computer_settings.resolution: a Windows computer's display is 1920x1080 for now, not 1280x800"]
     assert gate._display(base, {"width": 1280, "height": 800}, None) == [
         "display: a Windows computer's display is 1920x1080 for now"]
+
+
+@pytest.mark.parametrize("backing, over", [(None, []), ("/disk/base.img", ["-B", "/disk/base.img", "-F", "qcow2"])],
+                         ids=["a-base", "a-layer-over-its-base"])
+def test_a_disk_is_rewritten_keeping_only_the_clusters_it_uses(monkeypatch, tmp_path, backing, over):
+    ran = []
+
+    def run(argv, **kwargs):
+        ran.append(argv)
+        (tmp_path / "compact.img").write_text("compacted")
+
+    (tmp_path / "disk.img").write_text("as Windows left it")
+    monkeypatch.setattr(windows_disk.subprocess, "run", run)
+
+    windows_disk._converted(tmp_path, "machine:1", backing=backing)
+
+    assert ran == [["docker", "run", "--rm", "-v", f"{tmp_path}:/out", "--entrypoint", "qemu-img", "machine:1", "convert",
+                    "-O", "qcow2", *over, "/out/disk.img", "/out/compact.img"]]
+    assert (tmp_path / "disk.img").read_text() == "compacted" and not (tmp_path / "compact.img").exists()

tst/unit/env/envs/test_disk_recipe.py

@@ -291,7 +291,8 @@ def test_a_windows_disk_boots_from_secure_boot_on_ahci_and_e1000e_with_a_fresh_t
     assert reset.wait(10) == 0, reset.stderr.read()
     work = tmp_path / "state" / "boot"
     assert "-machine q35,smm=on,accel=" in qemu and "file=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd " in qemu
-    assert "-device tpm-tis,tpmdev=tpm " in qemu and "-device ide-hd,drive=disk,bus=ide.0,bootindex=1 " in qemu
+    assert "-device tpm-tis,tpmdev=tpm " in qemu and "-device ide-hd,drive=disk,bus=ide.0,bootindex=1,rotation_rate=1 " in qemu
+    assert ",id=disk,discard=unmap,detect-zeroes=unmap " in qemu
     assert "-device e1000e,netdev=net " in qemu and "-device VGA,xres=1920,yres=1080 " in qemu and "virtio" not in qemu
     assert "-device usb-storage,bus=usb.0,drive=seed,removable=on " in qemu
     assert _logged(calls, "swtpm", 2) == 2 * [f"swtpm socket --tpm2 --tpmstate dir={work}/tpm --ctrl"
@@ -351,7 +352,7 @@ def test_a_windows_install_boots_its_iso_first_pressing_enter_and_finds_its_answ
     (qemu,) = _logged(calls, "qemu-system-x86_64")
     assert ("-drive file=/in/win11.iso,format=raw,if=none,id=media,media=cdrom,readonly=on"
             " -device ide-cd,drive=media,bus=ide.2,bootindex=0 ") in qemu
-    assert "-device ide-hd,drive=disk,bus=ide.0,bootindex=1 " in qemu
+    assert "-device ide-hd,drive=disk,bus=ide.0,bootindex=1,rotation_rate=1 " in qemu
     assert "-device usb-storage,bus=usb.0,drive=files,removable=on " in qemu
     assert _logged(calls, "socat - UNIX-CONNECT:")
     assert _logged(calls, "mcopy -s")[0].startswith(f"mcopy -s -i {tmp_path}/state/boot/files.img {answers}/")