diff --git a/.github/workflows/build-static.yml b/.github/workflows/build-static.yml new file mode 100644 index 0000000..0102654 --- /dev/null +++ b/.github/workflows/build-static.yml @@ -0,0 +1,96 @@ +name: build-static + +# Reproducible musl static builds for `clone` + its guest-side helpers. +# +# The workflow never runs on our own hosts: GitHub Actions runners carry the +# toolchain (rustup targets + cross-rs toolchain container), so developer +# giga machines stay clean of rust toolchain installs entirely. +# +# Targets: +# x86_64-unknown-linux-musl static x86_64 host builds +# aarch64-unknown-linux-musl static arm64 thin-Dom0 host builds (current phone EL2 kernel 5.10) +# arm-unknown-linux-musleabihf static arm32 (experimental; the VMM core is x86-only so this will +# stay red until the arm32 port lands in-tree) + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + workflow_dispatch: + +jobs: + musl-static: + name: static ${{ matrix.target }} + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-musl + experimental: false + - target: aarch64-unknown-linux-musl + experimental: false + - target: arm-unknown-linux-musleabihf + experimental: true + + steps: + - name: Checkout ${{ github.repository }} @ ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Install Rust stable + ${{ matrix.target }} target + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache cargo registry + target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-v1-${{ matrix.target }} + restore-keys: ${{ runner.os }}-cargo-v1- + + - name: cross-rs setup + uses: taiki-e/install-action@v2 + with: + tool: cross + + - name: cargo build (static musl) — workspace + if: matrix.target != 'aarch64-unknown-linux-musl' + run: | + cross build --release --target ${{ matrix.target }} --workspace + + - name: cargo build (static musl) — aarch64, plain cargo + rust-lld + # cross's user-mode emulation of aarch64 build scripts dies against + # the host glibc (gettid GLIBC_2.30 relocation). Plain cargo compiles + # build scripts on the host and links the final binaries against the + # musl libc that ships inside the rustup target — self-contained, + # no cross gcc or qemu needed. + if: matrix.target == 'aarch64-unknown-linux-musl' + run: | + cargo build --release --target aarch64-unknown-linux-musl --workspace \ + --config 'target.aarch64-unknown-linux-musl.linker="rust-lld"' + + - name: static-link sanity check (file + ldd) + run: | + for b in clone clone-init clone-agent; do + f=target/${{ matrix.target }}/release/$b + if [ -f "$f" ]; then + echo "== $b ==" + file "$f" + if command -v ldd >/dev/null; then ldd "$f" || true; fi + else + echo "MISSING expected binary: $f" + fi + done + + - name: Upload artifact — ${{ matrix.target }} + uses: actions/upload-artifact@v4 + with: + name: clone-${{ matrix.target }} + path: target/${{ matrix.target }}/release/clone* + if-no-files-found: warn diff --git a/crates/clone-init/src/main.rs b/crates/clone-init/src/main.rs index 5aab9ac..6406694 100644 --- a/crates/clone-init/src/main.rs +++ b/crates/clone-init/src/main.rs @@ -36,8 +36,10 @@ fn msg(s: &str) { } /// Write a string directly to COM1 (I/O port 0x3F8) using port I/O. -/// Requires iopl(3) to have been called first. -#[cfg(target_os = "linux")] +/// Requires iopl(3) to have been called first. x86-only: port I/O has no +/// arm64 equivalent (guest earlycon there is PL011 MMIO, reachable via +/// /dev/console once the kernel has it mapped). +#[cfg(all(target_os = "linux", any(target_arch = "x86_64", target_arch = "x86")))] fn serial_write(s: &str) { for &b in s.as_bytes() { // Wait for THR empty (LSR bit 5) @@ -57,18 +59,20 @@ fn serial_write(s: &str) { } } -#[cfg(not(target_os = "linux"))] +#[cfg(not(all(target_os = "linux", any(target_arch = "x86_64", target_arch = "x86"))))] fn serial_write(_s: &str) {} /// Call iopl(3) to enable I/O port access for direct serial writes. -#[cfg(target_os = "linux")] +/// x86-only syscall: on aarch64 (and any other arch) libc doesn't even +/// declare it — early serial out there goes through /dev/console instead. +#[cfg(all(target_os = "linux", any(target_arch = "x86_64", target_arch = "x86")))] fn enable_iopl() { unsafe { libc::iopl(3); } } -#[cfg(not(target_os = "linux"))] +#[cfg(not(all(target_os = "linux", any(target_arch = "x86_64", target_arch = "x86"))))] fn enable_iopl() {} fn main() { @@ -106,6 +110,22 @@ fn main() { if Path::new("/lib/modules/overlay.ko").exists() { load_module("/lib/modules/overlay.ko"); } + // Boot-critical modules for kernels where ext4/virtio are =m (Debian + // arm64). Dependency order matters: crc32c -> crc16 -> jbd2 -> mbcache + // -> ext4; virtio_mmio/virtio_blk have no deps (core virtio built-in). + for module in &[ + "/lib/modules/crc32c-cryptoapi.ko", + "/lib/modules/crc16.ko", + "/lib/modules/jbd2.ko", + "/lib/modules/mbcache.ko", + "/lib/modules/ext4.ko", + "/lib/modules/virtio_mmio.ko", + "/lib/modules/virtio_blk.ko", + ] { + if Path::new(module).exists() { + load_module(module); + } + } // Parse kernel command line let cmdline = read_cmdline(); diff --git a/src/boot/arm64.rs b/src/boot/arm64.rs new file mode 100644 index 0000000..d3a897b --- /dev/null +++ b/src/boot/arm64.rs @@ -0,0 +1,476 @@ +//! arm64 guest boot: Image loader + hand-built FDT + vCPU register contract. +//! +//! Direct port of the proven `dom0/kvmhost.c` boot path (worked against this +//! exact 5.10 kernel with a boot to real rootfs mounting): +//! - Image header: magic "ARM\x64" @0x38, text_offset u32 LE @0x04 +//! - kernel Image loaded whole at RAM_BASE + text_offset (default 0x80000) +//! - BSP enters at the Image load address with x0 = DTB physical address +//! - FDT v17 builder, no libfdt (token-faithful port of fdt_build()) +//! +//! All constants confirmed against this system's uapi headers +//! (/usr/include/linux/kvm.h, /usr/include/aarch64-linux-gnu/asm/kvm.h). + +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::{bail, Context, Result}; +use vm_memory::GuestAddress; + +use crate::memory::GuestMem; +use crate::virtio::{IRQ_BASE, MMIO_BASE, MMIO_STRIDE}; + +/// Guest RAM base — aarch64 Linux kernel expects RAM well above 0. +pub const RAM_BASE: u64 = 0x4000_0000; +/// Default Image text offset (header overrides). +pub const DEFAULT_TEXT_OFFSET: u64 = 0x8_0000; +/// DTB load address (kvmhost: image ends lower, 64 KiB DTB budget). +pub const DTB_ADDR: u64 = 0x4700_0000; +/// Initrd load address. +pub const INITRD_ADDR: u64 = 0x4800_0000; +/// Cap initrd size like kvmhost (0x10000000 - 0x800000). +pub const INITRD_MAX: u64 = 0x1000_0000 - 0x80_0000; +/// PL011 UART window the DTB advertises. The kernel's earlycon writes land +/// here as KVM_EXIT_MMIO; the vCPU run loop forwards them to host stdout. +pub const PL011_BASE: u64 = 0x0A00_0000; +pub const PL011_SIZE: u64 = 0x1000; +/// vGICv3 placement (kvmhost: dist@0x09000000, redist@0x080b0000). +pub const GIC_DIST_BASE: u64 = 0x0900_0000; +pub const GIC_REDIST_BASE: u64 = 0x080b_0000; + +/// DTB physical address — read by Vcpu::new to set x0 for the BSP. +pub static GUEST_DTB_ADDR: AtomicU64 = AtomicU64::new(DTB_ADDR); + +// --- KVM uapi constants (arm64) ------------------------------------------- + +/// KVM_DEV_TYPE_ARM_VGIC_V3 (enum value 7, /usr/include/linux/kvm.h). +pub const KVM_DEV_TYPE_ARM_VGIC_V3: u32 = 7; +/// KVM_DEV_ARM_VGIC_GRP_ADDR. +pub const KVM_DEV_ARM_VGIC_GRP_ADDR: u32 = 0; +/// KVM_DEV_ARM_VGIC_GRP_CTRL. +pub const KVM_DEV_ARM_VGIC_GRP_CTRL: u32 = 4; +/// KVM_DEV_ARM_VGIC_CTRL_INIT. +pub const KVM_DEV_ARM_VGIC_CTRL_INIT: u64 = 0; +/// KVM_VGIC_V3_ADDR_TYPE_DIST. +pub const KVM_VGIC_V3_ADDR_TYPE_DIST: u64 = 2; +/// KVM_VGIC_V3_ADDR_TYPE_REDIST. +pub const KVM_VGIC_V3_ADDR_TYPE_REDIST: u64 = 3; +/// KVM_ARM_TARGET_GENERIC_V8 (/usr/include/aarch64-linux-gnu/asm/kvm.h). +pub const KVM_ARM_TARGET_GENERIC_V8: u32 = 5; +/// KVM_ARM_VCPU_PSCI_0_2 bit index. +pub const KVM_ARM_VCPU_PSCI_0_2: u32 = 2; +/// KVM_ARM_VCPU_POWER_OFF bit index (start non-BSP vCPUs powered off so +/// guest PSCI CPU_ON works — see Vcpu::new). +pub const KVM_ARM_VCPU_POWER_OFF: u32 = 0; + +/// KVM_CREATE_DEVICE = _IOWR(0xAE, 0xe0, kvm_create_device) +/// (struct kvm_create_device = 3×u32 = 12 bytes) +pub const KVM_CREATE_DEVICE: u64 = 0xC00CAEE0; +/// KVM_SET_DEVICE_ATTR = _IOW(0xAE, 0xe1, kvm_device_attr) +pub const KVM_SET_DEVICE_ATTR: u64 = 0x4018AEE1; +/// KVM_GET_DEVICE_ATTR = _IOWR(0xAE, 0xe0, kvm_device_attr) — dir=2 (_IOC_READ) +pub const KVM_GET_DEVICE_ATTR: u64 = 0x8018AEE0; +/// KVM_DEV_ARM_VGIC_GRP_DIST_REGS. +pub const KVM_DEV_ARM_VGIC_GRP_DIST_REGS: u32 = 1; +/// KVM_SET_ONE_REG = _IOW(0xAE, 0xac, kvm_one_reg) +pub const KVM_SET_ONE_REG: u64 = 0x4010AEAC; +/// KVM_ARM_VCPU_INIT = _IOW(0xAE, 0xae, kvm_vcpu_init) +/// (struct kvm_vcpu_init = target u32 + features[7] u32 = 32 bytes) +/// NOTE: nr is 0xAE, so the number ends in …AEAE (verified against +/// /usr/include/linux/kvm.h and kvmhost.c — 0x4020AEEE is wrong). +pub const KVM_ARM_VCPU_INIT: u64 = 0x4020AEAE; + +/// KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | offset/4 +/// KVM_REG_ARM64=0x6000_0000_0000_0000, SIZE_U64=0x0030_0000_0000_0000, +/// ARM_CORE=0x0010<<16. +pub const fn core_reg_id(offset_u32: u64) -> u64 { + 0x6000_0000_0000_0000 | 0x0030_0000_0000_0000 | 0x0010_0000 | offset_u32 +} +/// id for regs.pc (offsetof = 31*8 + 8 = 256; CORE_REG divides by 4) +pub const REG_ID_PC: u64 = core_reg_id(256 / 4); +/// id for regs.regs[0] (x0) +pub const REG_ID_X0: u64 = core_reg_id(0); + +#[repr(C)] +#[derive(Default)] +pub struct KvmVcpuInit { + pub target: u32, + pub features: [u32; 7], +} + +#[repr(C)] +#[derive(Default)] +pub struct KvmCreateDevice { + pub dev_type: u32, + pub fd: i32, + pub flags: u32, +} + +#[repr(C)] +#[derive(Default)] +pub struct KvmDeviceAttr { + pub flags: u32, + pub group: u32, + pub attr: u64, + pub addr: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct KvmOneReg { + pub id: u64, + pub addr: u64, +} + +// --- FDT builder (token-faithful port of kvmhost.c fdt_build) -------------- + +const FDT_MAGIC: u32 = 0xd00d_feed; +const FDT_HDR: usize = 40; +const FDT_STRUCT_OFF: usize = 56; // after header + 16-byte reservation map + +struct FdtBuilder { + /// struct blob (starts at FDT_STRUCT_OFF in the final DTB) + st: Vec, + strings: Vec, +} + +impl FdtBuilder { + fn new() -> Self { + FdtBuilder { st: Vec::new(), strings: Vec::new() } + } + + fn put32(&mut self, v: u32) { + self.st.extend_from_slice(&v.to_be_bytes()); + } + + fn name_off(&mut self, name: &str) -> u32 { + let off = self.strings.len() as u32; + self.strings.extend_from_slice(name.as_bytes()); + self.strings.push(0); + off + } + + fn begin_node(&mut self, name: &str) { + self.put32(1); // FDT_BEGIN_NODE + let n = name.as_bytes(); + self.st.extend_from_slice(n); + self.st.push(0); + while self.st.len() % 4 != 0 { + self.st.push(0); + } + } + + fn end_node(&mut self) { + self.put32(2); // FDT_END_NODE + } + + fn end(&mut self) { + self.put32(9); // FDT_END + } + + fn prop(&mut self, name: &str, data: &[u8]) { + let name_off = self.name_off(name); + self.put32(3); // FDT_PROP + self.put32(data.len() as u32); + self.put32(name_off); + self.st.extend_from_slice(data); + while self.st.len() % 4 != 0 { + self.st.push(0); + } + } + + fn prop_str(&mut self, name: &str, s: &str) { + let mut v = s.as_bytes().to_vec(); + v.push(0); + self.prop(name, &v); + } + + /// NUL-separated list "a\0b\0" + fn prop_strlist(&mut self, name: &str, list: &[&str]) { + let mut v = Vec::new(); + for s in list { + v.extend_from_slice(s.as_bytes()); + v.push(0); + } + self.prop(name, &v); + } + + fn prop_u32(&mut self, name: &str, vals: &[u32]) { + let mut v = Vec::with_capacity(vals.len() * 4); + for x in vals { + v.extend_from_slice(&x.to_be_bytes()); + } + self.prop(name, &v); + } + + fn finish(self) -> Vec { + let strings_len = (self.strings.len() + 3) & !3; + let struct_len = self.st.len(); + let strings_off = (FDT_STRUCT_OFF + ((struct_len + 3) & !3)) as u32; + let total = strings_off + strings_len as u32; + + let mut dtb = vec![0u8; total as usize]; + let mut w = |off: usize, v: u32| dtb[off..off + 4].copy_from_slice(&v.to_be_bytes()); + w(0, FDT_MAGIC); + w(4, total); + w(8, FDT_STRUCT_OFF as u32); // off_dt_struct + w(12, strings_off); // off_dt_strings + w(16, FDT_HDR as u32); // off_mem_rsvmap + w(20, 17); // version + w(24, 16); // last_comp_version + w(28, 0); // boot_cpuid_phys + w(32, strings_len as u32); // size_dt_strings + w(36, struct_len as u32); // size_dt_struct + + let mut st = self.st; + while st.len() % 4 != 0 { + st.push(0); + } + // Struct and strings must land exactly at the header offsets inside + // the preallocated buffer — extend_from_slice() here appended them + // past the end of the blob, leaving all-zero bytes at off_dt_struct + // (guest spun silently before earlycon). + dtb[FDT_STRUCT_OFF..FDT_STRUCT_OFF + st.len()].copy_from_slice(&st); + let mut strings = self.strings; + while strings.len() % 4 != 0 { + strings.push(0); + } + dtb[strings_off as usize..strings_off as usize + strings.len()] + .copy_from_slice(&strings); + dtb + } +} + +/// Build the guest DTB. `ram_span` = usable RAM bytes starting at RAM_BASE. +/// `mmio_devices` = (mmio base, SPI number 0-based) for virtio-mmio nodes. +pub fn build_dtb( + ram_span: u64, + cmdline: &str, + num_cpus: u32, + initrd_size: u64, + mmio_devices: &[(u64, u32)], +) -> Vec { + let mut b = FdtBuilder::new(); + + let memreg: [u32; 4] = [0, RAM_BASE as u32, 0, ram_span as u32]; + // Each vCPU owns a 2-page (0x20000) GICR frame (RD + SGI bases). The DT + // must declare the full span the kernel will walk — undersizing it + // crashes gic_iterate_rdists when num_cpus > 1 (proven: python replica + // booted all the way to root-mount panic after 0x40000 with 2 CPUs). + let gicreg: [u32; 8] = [ + 0, GIC_DIST_BASE as u32, 0, 0x10000, + 0, GIC_REDIST_BASE as u32, 0, (0x20000u64 * num_cpus.max(1) as u64) as u32, + ]; + let uartreg: [u32; 4] = [0, PL011_BASE as u32, 0, PL011_SIZE as u32]; + + b.begin_node(""); // root + b.prop_u32("#address-cells", &[2]); + b.prop_u32("#size-cells", &[2]); + b.prop_str("model", "loki thin-dom0 vm"); + b.prop_str("compatible", "loki-dom0-vm"); + + b.begin_node("memory@40000000"); + b.prop_str("device_type", "memory"); + b.prop_u32("numa-node-id", &[0]); + b.prop_u32("reg", &memreg); + b.end_node(); + + // OF NUMA: single explicit node — stops the kernel's fake-node path + // ever leaving cpu_to_node(0) unassigned (which made + // node_zonelists(-1) OOB-read in nr_free_zone_pages). See kvmhost.c. + b.begin_node("distance-map"); + b.prop_str("compatible", "numa-distance-map-v1"); + b.prop_u32("distance-matrix", &[0, 0, 10]); + b.end_node(); + + b.begin_node("chosen"); + let bootargs = format!( + "console=ttyAMA0 earlycon=pl011,mmio32,{:#x} {} panic=-1", + PL011_BASE, cmdline + ); + b.prop_str("bootargs", bootargs.trim()); + b.prop_str("stdout-path", "pl011"); + if initrd_size > 0 { + b.prop_u32("linux,initrd-start", &[0, INITRD_ADDR as u32]); + b.prop_u32("linux,initrd-end", &[0, (INITRD_ADDR + initrd_size) as u32]); + } + b.end_node(); + + b.begin_node("cpus"); + b.prop_u32("#address-cells", &[1]); + b.prop_u32("#size-cells", &[0]); + for cpu in 0..num_cpus { + b.begin_node(&format!("cpu@{cpu}")); + b.prop_u32("reg", &[cpu]); + b.prop_u32("numa-node-id", &[0]); + b.prop_str("device_type", "cpu"); + b.prop_str("compatible", "arm,arm-v8"); + b.prop_str("enable-method", "psci"); + b.end_node(); + } + b.end_node(); + + // arch timer: PPIs 29(13)/30(14)/27(11)/26(10), cpu0, level-low + b.begin_node("timer"); + b.prop_str("compatible", "arm,armv8-timer"); + b.prop_u32("interrupt-parent", &[1]); + b.prop_u32( + "interrupts", + &[1, 13, 0x108, 1, 14, 0x108, 1, 11, 0x108, 1, 10, 0x108], + ); + b.end_node(); + + b.begin_node("psci"); + b.prop_str("compatible", "arm,psci-0.2"); + b.prop_str("method", "hvc"); + b.end_node(); + + b.begin_node("uartclk"); + b.prop_str("compatible", "fixed-clock"); + b.prop_u32("#clock-cells", &[0]); + b.prop_u32("phandle", &[2]); + b.prop_u32("clock-frequency", &[24000000]); + b.end_node(); + + b.begin_node("pl011@0A000000"); + b.prop_strlist("compatible", &["arm,pl011", "arm,primecell"]); + b.prop_u32("reg", &uartreg); + b.prop_u32("interrupt-parent", &[1]); + b.prop_u32("interrupts", &[0, 1, 4]); + b.prop_u32("clocks", &[2, 2]); + b.prop_strlist("clock-names", &["apb_pclk", "uartclk"]); + b.prop_str("status", "okay"); + b.end_node(); + + b.begin_node("gic@09000000"); + b.prop_str("compatible", "arm,gic-v3"); + b.prop_str("interrupt-controller", ""); + b.prop_u32("phandle", &[1]); + // 3 cells (type, number, flags) — with 1 cell the kernel parses the + // 3-cell interrupt specs as garbage and request_threaded_irq fails + // with -EINVAL ("arch_timer: No interrupt available" is the same + // breakage). kvmhost.c got away with 1 cell only because its boot + // never probed an IRQ-driven device. + b.prop_u32("#interrupt-cells", &[3]); + b.prop_u32("#address-cells", &[1]); + b.prop_u32("reg", &gicreg); + b.end_node(); + + for &(base, spi) in mmio_devices { + b.begin_node(&format!("virtio_mmio@{base:08x}")); + b.prop_str("compatible", "virtio,mmio"); + b.prop_u32("reg", &[0, base as u32, 0, MMIO_STRIDE as u32]); + b.prop_u32("interrupt-parent", &[1]); + b.prop_u32("interrupts", &[0, spi, 4]); + b.prop_str("status", "okay"); + b.end_node(); + } + + b.end_node(); // root + b.end(); + b.finish() +} + +// --- Loader ---------------------------------------------------------------- + +/// arm64 Image header magic at offset 0x38. +pub fn is_arm64_image(data: &[u8]) -> bool { + data.len() > 0x3C && &data[0x38..0x3C] == b"ARM\x64" +} + +/// Load kernel Image + initrd + DTB into guest memory. +/// `ram_span` = usable RAM bytes starting at RAM_BASE (the KVM slot size). +/// Returns the BSP entry address (Image load address). +pub fn load_image_boot( + mem: &GuestMem, + kernel_path: &str, + initrd_path: Option<&str>, + cmdline: &str, + num_cpus: u32, + ram_span: u64, + mmio_devices: &[(u64, u32)], +) -> Result { + let ram_span = ram_span.min(0xC000_0000 - RAM_BASE); + if ram_span < 0x0800_0000 { + bail!("arm64 guest RAM span too small: {ram_span:#x} (need ≥ 128MB above {RAM_BASE:#x})"); + } + + let kernel_data = std::fs::read(kernel_path) + .with_context(|| format!("Failed to read kernel Image: {kernel_path}"))?; + if !is_arm64_image(&kernel_data) { + bail!( + "not an arm64 kernel Image (missing 'ARM\\x64' magic at 0x38): {kernel_path}" + ); + } + // Modern Image header (booting.rst, matches kvmhost.c): + // text_offset u64@0x08 — deprecated, always 0 on modern kernels; + // image_size u64@0x10 — in-memory size of the decompressed Image; + // flags u64@0x18; magic @0x38. + let text_offset = if kernel_data.len() >= 16 { + let t = u64::from_le_bytes(kernel_data[8..16].try_into().unwrap()); + if t != 0 { + t + } else { + DEFAULT_TEXT_OFFSET + } + } else { + DEFAULT_TEXT_OFFSET + }; + if kernel_data.len() >= 24 { + let img_size = u64::from_le_bytes(kernel_data[16..24].try_into().unwrap()); + if img_size != 0 && img_size > ram_span - DEFAULT_TEXT_OFFSET { + bail!( + "Image in-memory size {img_size:#x} exceeds RAM span budget {:#x}", + ram_span - DEFAULT_TEXT_OFFSET + ); + } + } + let load = RAM_BASE + text_offset; + if load + kernel_data.len() as u64 >= DTB_ADDR { + bail!("kernel Image too large for the layout (overlaps DTB slot)"); + } + mem.write_at(load, &kernel_data) + .context("Failed to write kernel Image into guest memory")?; + + // Initrd + let mut initrd_size = 0u64; + if let Some(path) = initrd_path { + let initrd = std::fs::read(path) + .with_context(|| format!("Failed to read initrd: {path}"))?; + if initrd.len() as u64 > INITRD_MAX { + bail!("initrd too large: {} > {INITRD_MAX}", initrd.len()); + } + mem.write_at(INITRD_ADDR, &initrd) + .context("Failed to write initrd into guest memory")?; + initrd_size = initrd.len() as u64; + } + + // DTB + let dtb = build_dtb(ram_span, cmdline, num_cpus, initrd_size, mmio_devices); + if dtb.len() > 0x1_0000 { + bail!("DTB exceeds 64 KiB budget: {} bytes", dtb.len()); + } + mem.write_at(DTB_ADDR, &dtb) + .context("Failed to write DTB into guest memory")?; + GUEST_DTB_ADDR.store(DTB_ADDR, Ordering::Relaxed); + + tracing::info!( + "arm64 boot: image={:#x}+{:#x} dtb={:#x}({}B) initrd={:#x}({}B) cpus={} mmio_devs={}", + load, + kernel_data.len(), + DTB_ADDR, + dtb.len(), + INITRD_ADDR, + initrd_size, + num_cpus, + mmio_devices.len(), + ); + + Ok(GuestAddress(load)) +} + +/// virtio-mmio device table for the DTB: (base, SPI). Mirrors MmioBus layout: +/// base = MMIO_BASE + i*MMIO_STRIDE, irq = IRQ_BASE + i. +pub fn dtb_device_list(count: usize) -> Vec<(u64, u32)> { + (0..count) + .map(|i| (MMIO_BASE + (i as u64) * MMIO_STRIDE, IRQ_BASE + i as u32)) + .collect() +} diff --git a/src/boot/mod.rs b/src/boot/mod.rs index 96fd5ea..f432c08 100644 --- a/src/boot/mod.rs +++ b/src/boot/mod.rs @@ -17,6 +17,10 @@ const BOOT_PARAMS_ADDR: u64 = 0x7000; /// Kernel command line placed at 0x20000. const CMDLINE_ADDR: u64 = 0x2_0000; +/// arm64 guest boot (Image loader + hand-built FDT, ported from dom0/kvmhost.c). +#[cfg(target_arch = "aarch64")] +pub mod arm64; + /// Load a Linux kernel (bzImage or ELF) into guest memory. /// Returns the entry point address. pub fn load_kernel( @@ -81,6 +85,40 @@ pub fn load_kernel_with_pci( Ok(entry) } +/// Arch dispatch: aarch64 boots an Image + hand-built DTB (kvmhost path); +/// x86_64 keeps the bzImage/boot_params loader. `mmio_devices` is the +/// registered virtio-mmio (base, irq) table — used for DTB nodes on arm64. +/// `ram_size` = total guest RAM bytes; on aarch64 it is the span of the +/// single KVM slot registered at GPA 0x40000000 (see Vm::boot). +pub fn load_kernel_for_arch( + mem: &GuestMem, + kernel_path: &str, + initrd_path: Option<&str>, + cmdline: &str, + num_cpus: u32, + ram_size: u64, + pci_enabled: bool, + mmio_devices: &[(u64, u32)], +) -> Result { + #[cfg(target_arch = "aarch64")] + { + return arm64::load_image_boot( + mem, + kernel_path, + initrd_path, + cmdline, + num_cpus, + ram_size, + mmio_devices, + ); + } + #[cfg(not(target_arch = "aarch64"))] + { + let _ = mmio_devices; + load_kernel_with_pci(mem, kernel_path, initrd_path, cmdline, num_cpus, ram_size, pci_enabled) + } +} + /// Check if kernel is a bzImage (Linux boot protocol magic). fn is_bzimage(data: &[u8]) -> bool { // bzImage has magic "HdrS" at offset 0x202 in the setup header diff --git a/src/control/jailer.rs b/src/control/jailer.rs index a541ae4..1367449 100644 --- a/src/control/jailer.rs +++ b/src/control/jailer.rs @@ -45,8 +45,15 @@ fn default_allowed_syscalls() -> Vec { #[cfg(target_os = "linux")] mod linux_syscalls { - /// x86_64 syscall numbers the VMM needs. + /// x86_64 syscall numbers the VMM needs (plus arch-specific additions). pub fn default_allowed() -> Vec { + let mut v = base_allowed(); + v.extend(arch_specific()); + v + } + + /// Portable syscall base set (present on x86_64, arm64 and arm32). + pub fn base_allowed() -> Vec { vec![ // KVM ioctls libc::SYS_ioctl as i32, @@ -101,16 +108,13 @@ mod linux_syscalls { libc::SYS_set_robust_list as i32, libc::SYS_get_robust_list as i32, libc::SYS_prctl as i32, - libc::SYS_arch_prctl as i32, libc::SYS_clone as i32, libc::SYS_clone3 as i32, libc::SYS_wait4 as i32, // Epoll libc::SYS_epoll_create1 as i32, libc::SYS_epoll_ctl as i32, - libc::SYS_epoll_wait as i32, libc::SYS_epoll_pwait as i32, - libc::SYS_poll as i32, libc::SYS_eventfd2 as i32, libc::SYS_timerfd_create as i32, libc::SYS_timerfd_settime as i32, @@ -135,6 +139,22 @@ mod linux_syscalls { libc::SYS_chdir as i32, ] } + + /// Arch-specific additions (x86_64: arch_prctl; pre-epoll_pwait poll). + pub fn arch_specific() -> Vec { + let mut v = Vec::new(); + #[cfg(target_arch = "x86_64")] + v.push(libc::SYS_arch_prctl as i32); + #[cfg(target_arch = "x86_64")] + v.push(libc::SYS_epoll_wait as i32); + #[cfg(target_arch = "x86_64")] + v.push(libc::SYS_poll as i32); + #[cfg(target_arch = "arm")] + v.push(libc::SYS_epoll_wait as i32); + #[cfg(target_arch = "arm")] + v.push(libc::SYS_poll as i32); + v + } } // --------------------------------------------------------------------------- diff --git a/src/control/sync_server.rs b/src/control/sync_server.rs index 86ca122..74ab6f7 100644 --- a/src/control/sync_server.rs +++ b/src/control/sync_server.rs @@ -28,8 +28,10 @@ pub struct VmHandle { pub kvm_slot_size: u64, /// Shared vCPU pause coordination state. pub pause_state: Arc, - /// pthread_t handles for each vCPU thread (used for SIGUSR1 kick). - pub vcpu_threads: Vec, + /// vCPU thread IDs (pthread_t as usize — musl types pthread_t as a raw + /// pointer, which isn't Send; usize travels across threads fine and is + /// cast back to pthread_t at every libc call site). + pub vcpu_threads: Vec, /// Number of vCPUs. pub num_vcpus: u32, /// Global shutdown flag (shared with the VM run loop). @@ -234,11 +236,11 @@ fn pause_vcpus(vm: &VmHandle) -> Result<(), String> { // Kick each vCPU out of KVM_RUN (send twice to handle races) for &tid in &vm.vcpu_threads { - unsafe { libc::pthread_kill(tid, libc::SIGUSR1); } + unsafe { libc::pthread_kill(tid as libc::pthread_t, libc::SIGUSR1); } } std::thread::sleep(std::time::Duration::from_millis(50)); for &tid in &vm.vcpu_threads { - unsafe { libc::pthread_kill(tid, libc::SIGUSR1); } + unsafe { libc::pthread_kill(tid as libc::pthread_t, libc::SIGUSR1); } } // Wait for all vCPUs to park @@ -335,7 +337,7 @@ fn handle_shutdown(vm: &VmHandle) -> Response { // Kick vCPUs out of KVM_RUN so they notice the shutdown for &tid in &vm.vcpu_threads { unsafe { - libc::pthread_kill(tid, libc::SIGUSR1); + libc::pthread_kill(tid as libc::pthread_t, libc::SIGUSR1); } } @@ -474,8 +476,11 @@ fn handle_snapshot(vm: &VmHandle, output_path: &str) -> Response { .collect(); // Save in-kernel irqchip state (PIC master, PIC slave, IOAPIC) and PIT - let mut irqchip_states = Vec::new(); - let mut pit_bytes = Vec::new(); + // x86-only: aarch64 has no PIC/IOAPIC/PIT to save (vGIC state lives + // in-kernel and is regenerated on the fresh-boot path). + #[cfg(target_arch = "x86_64")] + let (mut irqchip_states, mut pit_bytes) = (Vec::new(), Vec::new()); + #[cfg(target_arch = "x86_64")] if let Some(ref vm_fd) = vm.vm_fd { use kvm_bindings::{KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE, KVM_IRQCHIP_IOAPIC, kvm_irqchip}; for chip_id in [KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE, KVM_IRQCHIP_IOAPIC] { @@ -507,6 +512,9 @@ fn handle_snapshot(vm: &VmHandle, output_path: &str) -> Response { } } + #[cfg(not(target_arch = "x86_64"))] + let (irqchip_states, pit_bytes): (Vec>, Vec) = (Vec::new(), Vec::new()); + DeviceStates { serial: None, virtio_configs: std::collections::HashMap::new(), @@ -517,10 +525,14 @@ fn handle_snapshot(vm: &VmHandle, output_path: &str) -> Response { }; // 3. Save kvmclock and template + // x86-only (KVM_GET_CLOCK is an x86 ioctl); arm64 guests use the arch timer. + #[cfg(target_arch = "x86_64")] let clock_ns = vm.vm_fd.as_ref() .and_then(|fd| fd.get_clock().ok()) .map(|c| c.clock) .unwrap_or(0); + #[cfg(not(target_arch = "x86_64"))] + let clock_ns: u64 = 0; let result = { let guest_mem = unsafe { diff --git a/src/main.rs b/src/main.rs index 62ae0ef..74fde21 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ #[cfg(target_os = "linux")] +mod sys; mod vmm; #[cfg(target_os = "linux")] mod memory; @@ -453,7 +454,15 @@ fn main() -> Result<()> { let init_binary = rootfs::find_init_binary()?; tracing::info!("Using init binary: {}", init_binary.display()); - let initrd_bytes = rootfs::generate_initrd(&init_binary)?; + // Guest kernel version from the --kernel filename + // (e.g. vmlinuz-7.3.3+deb14-arm64) so the initrd embeds + // modules for the GUEST, not the host. + let kernel_version = std::path::Path::new(&kernel) + .file_name() + .and_then(|f| f.to_str()) + .and_then(|f| f.strip_prefix("vmlinuz-")) + .map(str::to_string); + let initrd_bytes = rootfs::generate_initrd(&init_binary, kernel_version.as_deref())?; // Write generated initrd to a temp file let initrd_path = std::env::temp_dir().join(format!( diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 1a581e0..b0d692e 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -23,6 +23,10 @@ const GUEST_MEM_START: u64 = 0; pub struct GuestMem { ptr: *mut u8, size: u64, // total mmap'd size (contiguous userspace allocation) + /// GPA base of the backing slot. 0 on x86 (slot 0 at GPA 0). On arm64 + /// the single slot is registered at GPA 0x40000000 (kvmhost layout), so + /// all GPA→HVA translation subtracts this. + base: u64, /// For VMs > 3GB, the MMIO hole splits guest physical address space: /// GPA [0, hole_start) maps to userspace [ptr, ptr+hole_start) /// GPA [hole_end, ...) maps to userspace [ptr+hole_start, ...) @@ -34,7 +38,7 @@ pub struct GuestMem { impl GuestMem { /// Create a GuestMem from a raw pointer and size (no MMIO hole). pub fn from_raw(ptr: *mut u8, size: u64) -> Self { - Self { ptr, size, hole_start: 0, hole_end: 0 } + Self { ptr, size, base: 0, hole_start: 0, hole_end: 0 } } /// Create a GuestMem with an MMIO hole for large VMs. @@ -42,17 +46,26 @@ impl GuestMem { /// - GPA [0, hole_start) → userspace [ptr, ptr+hole_start) /// - GPA [hole_end, ...) → userspace [ptr+hole_start, ...) pub fn from_raw_with_hole(ptr: *mut u8, size: u64, hole_start: u64, hole_end: u64) -> Self { - Self { ptr, size, hole_start, hole_end } + Self { ptr, size, base: 0, hole_start, hole_end } + } + + /// Override the GPA base of the backing slot (arm64: 0x40000000). + pub fn set_base(&mut self, base: u64) { + self.base = base; + } + + pub fn base(&self) -> u64 { + self.base } /// Create a temporary borrow of existing guest memory. pub unsafe fn borrow_raw(ptr: *mut u8, size: u64) -> BorrowedGuestMem { - BorrowedGuestMem { inner: GuestMem { ptr, size, hole_start: 0, hole_end: 0 } } + BorrowedGuestMem { inner: GuestMem { ptr, size, base: 0, hole_start: 0, hole_end: 0 } } } /// Create a temporary borrow with MMIO hole info. pub unsafe fn borrow_raw_with_hole(ptr: *mut u8, size: u64, hole_start: u64, hole_end: u64) -> BorrowedGuestMem { - BorrowedGuestMem { inner: GuestMem { ptr, size, hole_start, hole_end } } + BorrowedGuestMem { inner: GuestMem { ptr, size, base: 0, hole_start, hole_end } } } pub fn as_ptr(&self) -> *mut u8 { @@ -76,28 +89,33 @@ impl GuestMem { } /// Translate a guest physical address (GPA) to a userspace pointer offset. - /// Accounts for the MMIO hole if present. + /// Accounts for the GPA base (arm64 slot at 0x40000000) and the MMIO hole + /// if present. fn gpa_to_offset(&self, gpa: u64) -> Result { + if gpa < self.base { + anyhow::bail!("GPA {gpa:#x} is below the guest memory base {:#x}", self.base); + } + let rel = gpa - self.base; if self.hole_start == 0 { // No hole — direct mapping - if gpa >= self.size { - anyhow::bail!("GPA {gpa:#x} exceeds memory size {:#x}", self.size); + if rel >= self.size { + anyhow::bail!("GPA {gpa:#x} exceeds memory size {:#x} (base {:#x})", self.base + self.size, self.base); } - return Ok(gpa as usize); + return Ok(rel as usize); } - if gpa < self.hole_start { + if rel < self.hole_start { // Below the hole — direct mapping - Ok(gpa as usize) - } else if gpa >= self.hole_end { + Ok(rel as usize) + } else if rel >= self.hole_end { // Above the hole — subtract the hole size - let offset = self.hole_start + (gpa - self.hole_end); + let offset = self.hole_start + (rel - self.hole_end); if offset >= self.size { anyhow::bail!("GPA {gpa:#x} (offset {offset:#x}) exceeds memory size {:#x}", self.size); } Ok(offset as usize) } else { - anyhow::bail!("GPA {gpa:#x} is in the MMIO hole ({:#x}..{:#x})", self.hole_start, self.hole_end); + anyhow::bail!("GPA {gpa:#x} is in the MMIO hole ({:#x}..{:#x})", self.base + self.hole_start, self.base + self.hole_end); } } @@ -215,6 +233,7 @@ pub fn create_guest_memory_with_hole(size: u64, hole_start: u64, hole_end: u64) Ok(GuestMem { ptr: ptr as *mut u8, size, + base: 0, hole_start, hole_end, }) diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 014e665..5497b9c 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -433,7 +433,7 @@ pub fn run_sender( // Resume vCPUs so they can exit crate::control::sync_server::resume_vcpus_pub(vm); for &tid in &vm.vcpu_threads { - unsafe { libc::pthread_kill(tid, libc::SIGUSR1); } + unsafe { libc::pthread_kill(tid as libc::pthread_t, libc::SIGUSR1); } } Ok(stats) @@ -453,10 +453,9 @@ pub fn run_receiver( use std::sync::atomic::AtomicBool; use std::sync::Arc; - use kvm_bindings::{ - kvm_pit_config, kvm_userspace_memory_region, - KVM_MEM_LOG_DIRTY_PAGES, KVM_PIT_SPEAKER_DUMMY, - }; + use kvm_bindings::{kvm_userspace_memory_region, KVM_MEM_LOG_DIRTY_PAGES}; + #[cfg(target_arch = "x86_64")] + use kvm_bindings::{kvm_pit_config, KVM_PIT_SPEAKER_DUMMY}; use kvm_ioctls::Kvm; let listener = TcpListener::bind(format!("0.0.0.0:{port}")) @@ -500,14 +499,17 @@ pub fn run_receiver( .context("Failed to set KVM memory region")?; } - // Create irqchip + PIT + GSI routing - vm_fd.create_irq_chip().context("Failed to create irqchip")?; - let pit_config = kvm_pit_config { - flags: KVM_PIT_SPEAKER_DUMMY, - ..Default::default() - }; - vm_fd.create_pit2(pit_config).context("Failed to create PIT")?; - crate::vmm::setup_gsi_routing(&vm_fd)?; + // Create irqchip + PIT + GSI routing (x86 only) + #[cfg(target_arch = "x86_64")] + { + vm_fd.create_irq_chip().context("Failed to create irqchip")?; + let pit_config = kvm_pit_config { + flags: KVM_PIT_SPEAKER_DUMMY, + ..Default::default() + }; + vm_fd.create_pit2(pit_config).context("Failed to create PIT")?; + crate::vmm::setup_gsi_routing(&vm_fd)?; + } // Send Ready write_msg(&mut stream, MSG_READY, &[])?; @@ -644,17 +646,17 @@ pub fn run_receiver( let shutdown_flag = Arc::new(AtomicBool::new(false)); let _agent_state = crate::vmm::agent_listener::start_listener(Arc::clone(&shutdown_flag), crate::vmm::agent_listener::AGENT_VSOCK_PORT_BASE); - let mut vcpu_threads: Vec = Vec::new(); + let mut vcpu_threads: Vec = Vec::new(); let mut ap_handles = Vec::new(); let mut all_vcpus: Vec = vcpus.drain(..).collect(); for vcpu in all_vcpus.drain(1..) { - let (tx, rx) = std::sync::mpsc::channel::(); + let (tx, rx) = std::sync::mpsc::channel::(); let handle = std::thread::Builder::new() .name(format!("vcpu-{}", vcpu.id())) .spawn(move || { - let _ = tx.send(unsafe { libc::pthread_self() }); + let _ = tx.send(unsafe { libc::pthread_self() } as usize); let mut vcpu = vcpu; if let Err(e) = vcpu.run_loop() { tracing::error!("vCPU {} exited with error: {e}", vcpu.id()); @@ -666,7 +668,7 @@ pub fn run_receiver( ap_handles.push(handle); } - let bsp_tid = unsafe { libc::pthread_self() }; + let bsp_tid = unsafe { libc::pthread_self() } as usize; vcpu_threads.insert(0, bsp_tid); let vm_handle = Arc::new(crate::control::sync_server::VmHandle { diff --git a/src/net/mod.rs b/src/net/mod.rs index 3ff763c..4b0d882 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -102,7 +102,7 @@ pub fn create_tap(name: &str) -> anyhow::Result { let flags: libc::c_short = IFF_TAP | IFF_NO_PI; ifr[16..18].copy_from_slice(&flags.to_ne_bytes()); - let ret = unsafe { libc::ioctl(fd, TUNSETIFF, ifr.as_ptr()) }; + let ret = unsafe { crate::sys::ioctl(fd, TUNSETIFF as u64, ifr.as_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { let err = std::io::Error::last_os_error(); unsafe { libc::close(fd); } @@ -160,7 +160,7 @@ pub fn configure_tap(fd: RawFd, ip: &str, netmask: &str) -> anyhow::Result<()> { // TUNGETIFF to retrieve the interface name. const TUNGETIFF: libc::c_ulong = 0x800454D2; let mut ifr = [0u8; 40]; - let ret = unsafe { libc::ioctl(fd, TUNGETIFF, ifr.as_mut_ptr()) }; + let ret = unsafe { crate::sys::ioctl(fd, TUNGETIFF as u64, ifr.as_mut_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { unsafe { libc::close(sock); } return Err(anyhow::anyhow!( @@ -194,7 +194,7 @@ pub fn configure_tap(fd: RawFd, ip: &str, netmask: &str) -> anyhow::Result<()> { // SIOCSIFADDR = 0x8916 const SIOCSIFADDR: libc::c_ulong = 0x8916; set_sockaddr_in(&mut ifr, &ip_addr); - let ret = unsafe { libc::ioctl(sock, SIOCSIFADDR, ifr.as_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCSIFADDR as u64, ifr.as_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { let err = std::io::Error::last_os_error(); unsafe { libc::close(sock); } @@ -204,7 +204,7 @@ pub fn configure_tap(fd: RawFd, ip: &str, netmask: &str) -> anyhow::Result<()> { // SIOCSIFNETMASK = 0x891C const SIOCSIFNETMASK: libc::c_ulong = 0x891C; set_sockaddr_in(&mut ifr, &netmask_addr); - let ret = unsafe { libc::ioctl(sock, SIOCSIFNETMASK, ifr.as_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCSIFNETMASK as u64, ifr.as_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { let err = std::io::Error::last_os_error(); unsafe { libc::close(sock); } @@ -216,7 +216,7 @@ pub fn configure_tap(fd: RawFd, ip: &str, netmask: &str) -> anyhow::Result<()> { const SIOCGIFFLAGS: libc::c_ulong = 0x8913; // First get current flags. - let ret = unsafe { libc::ioctl(sock, SIOCGIFFLAGS, ifr.as_mut_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCGIFFLAGS as u64, ifr.as_mut_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { let err = std::io::Error::last_os_error(); unsafe { libc::close(sock); } @@ -227,7 +227,7 @@ pub fn configure_tap(fd: RawFd, ip: &str, netmask: &str) -> anyhow::Result<()> { let new_flags = current_flags | (libc::IFF_UP as i16); ifr[16..18].copy_from_slice(&new_flags.to_ne_bytes()); - let ret = unsafe { libc::ioctl(sock, SIOCSIFFLAGS, ifr.as_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCSIFFLAGS as u64, ifr.as_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { let err = std::io::Error::last_os_error(); unsafe { libc::close(sock); } @@ -270,7 +270,7 @@ pub fn setup_bridge(bridge_name: &str, tap_name: &str) -> anyhow::Result<()> { // SIOCGIFINDEX = 0x8933 const SIOCGIFINDEX: libc::c_ulong = 0x8933; - let ret = unsafe { libc::ioctl(sock, SIOCGIFINDEX, ifr.as_mut_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCGIFINDEX as u64, ifr.as_mut_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { let err = std::io::Error::last_os_error(); unsafe { libc::close(sock); } @@ -287,7 +287,7 @@ pub fn setup_bridge(bridge_name: &str, tap_name: &str) -> anyhow::Result<()> { // SIOCBRADDIF = 0x89A2 const SIOCBRADDIF: libc::c_ulong = 0x89A2; - let ret = unsafe { libc::ioctl(sock, SIOCBRADDIF, br_ifr.as_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCBRADDIF as u64, br_ifr.as_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { let err = std::io::Error::last_os_error(); unsafe { libc::close(sock); } @@ -512,7 +512,7 @@ fn bring_interface_up(name: &str) -> anyhow::Result<()> { const SIOCGIFFLAGS: libc::c_ulong = 0x8913; const SIOCSIFFLAGS: libc::c_ulong = 0x8914; - let ret = unsafe { libc::ioctl(sock, SIOCGIFFLAGS, ifr.as_mut_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCGIFFLAGS as u64, ifr.as_mut_ptr() as *const _ as *mut libc::c_void) }; if ret < 0 { unsafe { libc::close(sock) }; anyhow::bail!("SIOCGIFFLAGS for {name} failed: {}", std::io::Error::last_os_error()); @@ -522,7 +522,7 @@ fn bring_interface_up(name: &str) -> anyhow::Result<()> { let new_flags = current_flags | (libc::IFF_UP as i16); ifr[16..18].copy_from_slice(&new_flags.to_ne_bytes()); - let ret = unsafe { libc::ioctl(sock, SIOCSIFFLAGS, ifr.as_ptr()) }; + let ret = unsafe { crate::sys::ioctl(sock, SIOCSIFFLAGS as u64, ifr.as_ptr() as *const _ as *mut libc::c_void) }; unsafe { libc::close(sock) }; if ret < 0 { diff --git a/src/pci/vfio.rs b/src/pci/vfio.rs index b23b30a..8988e96 100644 --- a/src/pci/vfio.rs +++ b/src/pci/vfio.rs @@ -143,20 +143,20 @@ impl VfioDevice { .context("Failed to open /dev/vfio/vfio")?; // Check API version - let version = unsafe { libc::ioctl(container_fd, VFIO_GET_API_VERSION as libc::c_ulong) }; + let version = unsafe { crate::sys::ioctl(container_fd, VFIO_GET_API_VERSION as u64, std::ptr::null_mut()) }; if version != VFIO_API_VERSION as i32 { anyhow::bail!("VFIO API version mismatch: got {version}, expected {VFIO_API_VERSION}"); } // Check IOMMU support let has_type1v2 = unsafe { - libc::ioctl(container_fd, VFIO_CHECK_EXTENSION as libc::c_ulong, VFIO_TYPE1V2_IOMMU) + crate::sys::ioctl_value(container_fd, VFIO_CHECK_EXTENSION as u64, VFIO_TYPE1V2_IOMMU as u64) }; let iommu_type = if has_type1v2 > 0 { VFIO_TYPE1V2_IOMMU } else { let has_type1 = unsafe { - libc::ioctl(container_fd, VFIO_CHECK_EXTENSION as libc::c_ulong, VFIO_TYPE1_IOMMU) + crate::sys::ioctl_value(container_fd, VFIO_CHECK_EXTENSION as u64, VFIO_TYPE1_IOMMU as u64) }; if has_type1 <= 0 { anyhow::bail!("No supported VFIO IOMMU type found"); @@ -172,7 +172,7 @@ impl VfioDevice { // Check group is viable let mut status = VfioGroupStatus { argsz: 8, flags: 0 }; let ret = unsafe { - libc::ioctl(group_fd, VFIO_GROUP_GET_STATUS as libc::c_ulong, &mut status) + crate::sys::ioctl(group_fd, VFIO_GROUP_GET_STATUS as u64, &mut status as *const _ as *mut libc::c_void) }; if ret < 0 { anyhow::bail!("VFIO_GROUP_GET_STATUS failed: {}", std::io::Error::last_os_error()); @@ -186,7 +186,7 @@ impl VfioDevice { // 4. Set container for group let ret = unsafe { - libc::ioctl(group_fd, VFIO_GROUP_SET_CONTAINER as libc::c_ulong, &container_fd) + crate::sys::ioctl(group_fd, VFIO_GROUP_SET_CONTAINER as u64, &container_fd as *const _ as *mut libc::c_void) }; if ret < 0 { anyhow::bail!("VFIO_GROUP_SET_CONTAINER failed: {}", std::io::Error::last_os_error()); @@ -194,7 +194,7 @@ impl VfioDevice { // 5. Set IOMMU type let ret = unsafe { - libc::ioctl(container_fd, VFIO_SET_IOMMU as libc::c_ulong, iommu_type) + crate::sys::ioctl_value(container_fd, VFIO_SET_IOMMU as u64, iommu_type as u64) }; if ret < 0 { anyhow::bail!("VFIO_SET_IOMMU failed: {}", std::io::Error::last_os_error()); @@ -203,7 +203,7 @@ impl VfioDevice { // 6. Get device fd let bdf_cstr = std::ffi::CString::new(bdf)?; let device_fd = unsafe { - libc::ioctl(group_fd, VFIO_GROUP_GET_DEVICE_FD as libc::c_ulong, bdf_cstr.as_ptr()) + crate::sys::ioctl(group_fd, VFIO_GROUP_GET_DEVICE_FD as u64, bdf_cstr.as_ptr() as *const _ as *mut libc::c_void) }; if device_fd < 0 { anyhow::bail!( @@ -220,7 +220,7 @@ impl VfioDevice { num_irqs: 0, }; let ret = unsafe { - libc::ioctl(device_fd, VFIO_DEVICE_GET_INFO as libc::c_ulong, &mut dev_info) + crate::sys::ioctl(device_fd, VFIO_DEVICE_GET_INFO as u64, &mut dev_info as *const _ as *mut libc::c_void) }; if ret < 0 { anyhow::bail!("VFIO_DEVICE_GET_INFO failed: {}", std::io::Error::last_os_error()); @@ -242,7 +242,7 @@ impl VfioDevice { offset: 0, }; let ret = unsafe { - libc::ioctl(device_fd, VFIO_DEVICE_GET_REGION_INFO as libc::c_ulong, &mut region) + crate::sys::ioctl(device_fd, VFIO_DEVICE_GET_REGION_INFO as u64, &mut region as *const _ as *mut libc::c_void) }; if ret < 0 { tracing::warn!("Failed to get region {i} info"); @@ -278,7 +278,7 @@ impl VfioDevice { }; let ret = unsafe { - libc::ioctl(self.container_fd, VFIO_IOMMU_MAP_DMA as libc::c_ulong, &dma_map) + crate::sys::ioctl(self.container_fd, VFIO_IOMMU_MAP_DMA as u64, &dma_map as *const _ as *mut libc::c_void) }; if ret < 0 { anyhow::bail!( @@ -410,7 +410,7 @@ impl VfioDevice { /// Reset the device. pub fn reset(&self) -> Result<()> { let ret = unsafe { - libc::ioctl(self.device_fd, VFIO_DEVICE_RESET as libc::c_ulong) + crate::sys::ioctl(self.device_fd, VFIO_DEVICE_RESET as u64, std::ptr::null_mut()) }; if ret < 0 { tracing::warn!("VFIO device reset failed: {}", std::io::Error::last_os_error()); diff --git a/src/rootfs.rs b/src/rootfs.rs index f0789ec..1b49461 100644 --- a/src/rootfs.rs +++ b/src/rootfs.rs @@ -111,12 +111,22 @@ pub fn find_agent_binary() -> Option { /// Searches `/lib/modules//` for vsock and overlay modules. /// These are placed at `/lib/modules/` (flat) in the initrd so clone-init /// can load them before switching root into the real rootfs. -fn embed_kernel_modules(cpio: &mut Vec) { - let uname = match std::process::Command::new("uname").arg("-r").output() { - Ok(o) if o.status.success() => { - String::from_utf8_lossy(&o.stdout).trim().to_string() - } - _ => return, +fn embed_kernel_modules(cpio: &mut Vec, kernel_version: Option<&str>) { + // Module tree must match the GUEST kernel, not the host running the + // VMM (on a phone Dom0, uname -r is the Android host kernel while the + // guest boots vmlinuz-). Prefer the version parsed + // from the --kernel filename; fall back to uname. + let uname = kernel_version.map(str::to_string).or_else(|| { + std::process::Command::new("uname") + .arg("-r") + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + }); + let uname = match uname { + Some(u) => u, + None => return, }; let mod_root = PathBuf::from(format!("/lib/modules/{uname}")); @@ -125,8 +135,21 @@ fn embed_kernel_modules(cpio: &mut Vec) { return; } - // Modules to embed: (subdir, filename) — loaded in order by clone-init + // Modules to embed: (subdir, filename) — loaded in dependency order by + // clone-init BEFORE waiting for /dev/vda. On Debian arm64 kernels, + // ext4/virtio_mmio/virtio_blk are modules (CONFIG_*_m) and the core + // virtio layer is built-in, so this exact chain makes the root + // mountable. let modules: &[(&str, &str)] = &[ + // ext4 dependency chain (crc32c -> crc16 -> jbd2 -> mbcache -> ext4) + ("kernel/crypto", "crc32c-cryptoapi.ko"), + ("kernel/lib/crc", "crc16.ko"), + ("kernel/fs/jbd2", "jbd2.ko"), + ("kernel/fs", "mbcache.ko"), + ("kernel/fs/ext4", "ext4.ko"), + // virtio transport + block (core virtio is built-in) + ("kernel/drivers/virtio", "virtio_mmio.ko"), + ("kernel/drivers/block", "virtio_blk.ko"), // vsock transport (needed for guest agent) ("kernel/net/vmw_vsock", "vsock.ko"), ("kernel/net/vmw_vsock", "vmw_vsock_virtio_transport_common.ko"), @@ -180,7 +203,7 @@ fn embed_kernel_modules(cpio: &mut Vec) { /// and optionally the clone-agent binary. /// /// Returns the initrd contents as a byte vector. -pub fn generate_initrd(init_binary: &Path) -> Result> { +pub fn generate_initrd(init_binary: &Path, kernel_version: Option<&str>) -> Result> { let init_data = fs::read(init_binary).with_context(|| format!("Failed to read {}", init_binary.display()))?; @@ -229,7 +252,7 @@ pub fn generate_initrd(init_binary: &Path) -> Result> { // Include vsock kernel modules so clone-init can load them before switching root. // These are required for the guest agent to connect back to the host VMM. // The kernel may have CONFIG_VIRTIO_VSOCKETS=m, so modules must be loaded explicitly. - embed_kernel_modules(&mut cpio); + embed_kernel_modules(&mut cpio, kernel_version); // Write trailer cpio_write_trailer(&mut cpio); diff --git a/src/sys.rs b/src/sys.rs new file mode 100644 index 0000000..71cdfe2 --- /dev/null +++ b/src/sys.rs @@ -0,0 +1,70 @@ +//! Kernel-ABI compatibility shims (`sys`). +//! +//! glibc's `ioctl` request is `c_ulong`; musl's `libc` crate types it +//! `c_int` (fixed 3-arg ABI), and a few wrappers (`renameat2`, `iopl`) are +//! absent from the musl or arm builds of `libc`. Every direct libc call in +//! this crate routes through here so one file owns the ABI knowledge. + +use libc::c_int; + +/// `ioctl(2)` with the request passed as the raw Linux u64 command value +/// and the argument coerced to its exact vararg slot (register-wide). +#[inline] +pub unsafe fn ioctl(fd: c_int, request: u64, arg: *mut libc::c_void) -> c_int { + #[cfg(target_env = "musl")] + let req = request as libc::c_int; + #[cfg(not(target_env = "musl"))] + let req = request as libc::c_ulong; + libc::ioctl(fd, req, arg) +} + +/// `renameat2(2)` — glibc wrapper only in `libc`; musl via raw syscall. +#[inline] +pub unsafe fn renameat2( + olddirfd: c_int, + oldpath: *const libc::c_char, + newdirfd: c_int, + newpath: *const libc::c_char, + flags: libc::c_uint, +) -> c_int { + #[cfg(target_env = "musl")] + { + libc::syscall( + libc::SYS_renameat2, + olddirfd, + oldpath, + newdirfd, + newpath, + flags, + ) as c_int + } + #[cfg(not(target_env = "musl"))] + { + libc::renameat2(olddirfd, oldpath, newdirfd, newpath, flags) + } +} + +/// `iopl(2)` — x86-only privileged I/O-port level; unavailable (and +/// meaningless) elsewhere. +#[inline] +pub unsafe fn iopl(level: libc::c_int) -> libc::c_int { + #[cfg(target_arch = "x86_64")] + { + libc::syscall(libc::SYS_iopl, level) as libc::c_int + } + #[cfg(target_arch = "x86")] + { + libc::syscall(172, level) as libc::c_int // SYS_iopl on i386 + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))] + { + let _ = level; + -libc::ENOSYS + } +} + +/// Value-arg ioctl: integer `arg` passed in the register slot verbatim. +#[inline] +pub unsafe fn ioctl_value(fd: c_int, request: u64, value: u64) -> c_int { + ioctl(fd, request, value as *mut libc::c_void) +} diff --git a/src/virtio/balloon.rs b/src/virtio/balloon.rs index 391423b..36470e8 100644 --- a/src/virtio/balloon.rs +++ b/src/virtio/balloon.rs @@ -78,6 +78,8 @@ pub struct VirtioBalloon { /// to host pointers for `madvise`. guest_mem_base: *mut u8, guest_mem_size: u64, + /// GPA of the start of the mapped region (0 on x86, RAM_BASE on arm64). + mem_base: u64, /// Whether the device has been activated by the driver. activated: bool, @@ -104,6 +106,7 @@ impl VirtioBalloon { acked_features: 0, guest_mem_base, guest_mem_size, + mem_base: 0, activated: false, config_interrupt_pending: false, } @@ -134,8 +137,10 @@ impl VirtioBalloon { let mut inflated = 0u32; for &pfn in pfns { - let guest_addr = pfn as u64 * BALLOON_PAGE_SIZE as u64; - if guest_addr + BALLOON_PAGE_SIZE as u64 > self.guest_mem_size { + // Guest PFNs are GPAs relative to mem_base (0 on x86, RAM_BASE + // on arm64 where guest RAM starts at 0x40000000). + let guest_addr = self.mem_base + pfn as u64 * BALLOON_PAGE_SIZE as u64; + if guest_addr + BALLOON_PAGE_SIZE as u64 > self.mem_base + self.guest_mem_size { tracing::warn!(pfn, "inflate: PFN out of range, skipping"); continue; } @@ -338,6 +343,9 @@ impl VirtioDevice for VirtioBalloon { } fn as_any_mut(&mut self) -> &mut dyn std::any::Any { self } + fn set_mem_base(&mut self, mem_base: u64) { + self.mem_base = mem_base; + } fn reset(&mut self) { self.config.num_pages.store(0, Ordering::Release); self.config.actual.store(0, Ordering::Release); diff --git a/src/virtio/fs.rs b/src/virtio/fs.rs index 197419b..40aa9bf 100644 --- a/src/virtio/fs.rs +++ b/src/virtio/fs.rs @@ -1777,7 +1777,7 @@ impl VirtioFs { Err(_) => return self.make_error_response(header.unique, -libc::EINVAL), }; let ret = unsafe { - libc::renameat2( + crate::sys::renameat2( libc::AT_FDCWD, c_old.as_ptr(), libc::AT_FDCWD, diff --git a/src/virtio/mmio.rs b/src/virtio/mmio.rs index 7022117..bd61782 100644 --- a/src/virtio/mmio.rs +++ b/src/virtio/mmio.rs @@ -115,6 +115,8 @@ pub struct MmioTransport { guest_mem: *mut u8, /// Guest memory size in bytes. guest_mem_size: u64, + /// GPA of the start of the mapped region (0 on x86, RAM_BASE on arm64). + mem_base: u64, /// External interrupt status from vhost call_fd poll thread. /// Set atomically by the poll thread, read and cleared by the transport. vhost_interrupt: Arc, @@ -173,6 +175,7 @@ impl MmioTransport { config_generation: 0, guest_mem, guest_mem_size, + mem_base: 0, vhost_interrupt: Arc::new(AtomicU32::new(0)), } } @@ -202,6 +205,18 @@ impl MmioTransport { .collect(); } + /// Set the GPA of the start of the mapped guest memory region + /// (arm64: RAM_BASE, since guest RAM starts at 0x40000000). Must be + /// called after set_guest_memory_with_hole. Propagates to all + /// virtqueues and to the device. + pub fn set_mem_base(&mut self, mem_base: u64) { + self.mem_base = mem_base; + for q in &mut self.virtqueues { + q.set_mem_base(mem_base); + } + self.device.set_mem_base(mem_base); + } + /// Reset the device to initial state. pub fn force_reset(&mut self) { self.handle_status_write(0); @@ -386,6 +401,11 @@ impl MmioTransport { } reg::QUEUE_NOTIFY => { let queue_idx = val as u16; + tracing::info!( + "virtio-mmio QUEUE_NOTIFY q={queue_idx} dev={:?} ready={}", + self.device.device_type(), + self.virtqueues.get(queue_idx as usize).map(|q| q.is_ready()).unwrap_or(false) + ); if let Err(e) = self.device.process_queue(queue_idx) { tracing::error!("Error processing queue {val}: {e}"); } @@ -731,6 +751,14 @@ impl MmioBus { } } + /// Set the GPA of the start of the mapped guest memory region + /// (0 on x86 where guest RAM starts at GPA 0; RAM_BASE on arm64). + pub fn set_mem_base(&mut self, mem_base: u64) { + for transport in &mut self.devices { + transport.set_mem_base(mem_base); + } + } + /// Register a virtio device on the bus. Returns the MMIO base address /// and IRQ number assigned to it. pub fn register(&mut self, device: Box) -> (u64, u32) { @@ -775,6 +803,9 @@ impl MmioBus { } else { None }; + tracing::info!( + "virtio-mmio write addr={addr:#x} off={offset:#x} had={had_interrupt} has={has_interrupt} inject={inject:?}" + ); (true, inject) } else { (false, None) diff --git a/src/virtio/mod.rs b/src/virtio/mod.rs index a353df8..b86cf92 100644 --- a/src/virtio/mod.rs +++ b/src/virtio/mod.rs @@ -122,6 +122,11 @@ pub trait VirtioDevice: Send { /// Called before activate() for VMs with >3GB RAM. fn set_memory_hole(&mut self, _hole_start: u64, _hole_end: u64) {} + /// Set the GPA of the start of the mapped guest memory region + /// (0 on x86 where guest RAM starts at GPA 0; RAM_BASE on arm64). + /// Devices that translate GPAs to host pointers must subtract this. + fn set_mem_base(&mut self, _mem_base: u64) {} + /// Reset the device to initial state. fn reset(&mut self); diff --git a/src/virtio/net.rs b/src/virtio/net.rs index b640845..67520ae 100644 --- a/src/virtio/net.rs +++ b/src/virtio/net.rs @@ -140,6 +140,8 @@ pub struct VirtioNet { queue_configs: Vec, guest_mem: *mut u8, guest_mem_size: u64, + /// GPA of the start of the mapped region (0 on x86, RAM_BASE on arm64). + mem_base: u64, hole_start: u64, hole_end: u64, } @@ -150,6 +152,10 @@ unsafe impl Send for VirtioNet {} impl VirtioNet { fn gpa_to_hva(&self, gpa: u64) -> u64 { let base = self.guest_mem as u64; + let gpa = match gpa.checked_sub(self.mem_base) { + Some(g) => g, + None => return base, + }; if self.hole_start == 0 || gpa < self.hole_start { base + gpa } else if gpa >= self.hole_end { @@ -177,6 +183,7 @@ impl VirtioNet { queue_configs: Vec::new(), guest_mem: std::ptr::null_mut(), guest_mem_size: 0, + mem_base: 0, hole_start: 0, hole_end: 0, } @@ -262,7 +269,7 @@ impl VirtioNet { self.vhost_fd = vhost_fd; // 2. VHOST_SET_OWNER - let ret = unsafe { libc::ioctl(vhost_fd, vhost::SET_OWNER) }; + let ret = unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_OWNER as u64, std::ptr::null_mut()) }; if ret < 0 { return Err(anyhow::anyhow!( "VHOST_SET_OWNER failed: {}", @@ -273,11 +280,7 @@ impl VirtioNet { // 3. VHOST_GET_FEATURES then SET intersection with driver-negotiated features let mut vhost_features: u64 = 0; if unsafe { - libc::ioctl( - vhost_fd, - vhost::GET_FEATURES, - &mut vhost_features as *mut u64, - ) + crate::sys::ioctl(vhost_fd, vhost::GET_FEATURES as u64, &mut vhost_features as *mut u64 as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( @@ -301,7 +304,7 @@ impl VirtioNet { features ); - if unsafe { libc::ioctl(vhost_fd, vhost::SET_FEATURES, &features as *const u64) } < 0 { + if unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_FEATURES as u64, &features as *const u64 as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( "VHOST_SET_FEATURES failed: {}", std::io::Error::last_os_error() @@ -329,7 +332,7 @@ impl VirtioNet { }, ], }; - unsafe { libc::ioctl(vhost_fd, vhost::SET_MEM_TABLE, &mem_table as *const vhost::Memory2) } + unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_MEM_TABLE as u64, &mem_table as *const vhost::Memory2 as *const _ as *mut libc::c_void) } } else { let mem_table = vhost::Memory { nregions: 1, @@ -341,10 +344,7 @@ impl VirtioNet { flags_padding: 0, }], }; - unsafe { libc::ioctl(vhost_fd, - vhost::SET_MEM_TABLE, - &mem_table as *const vhost::Memory, - ) } + unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_MEM_TABLE as u64, &mem_table as *const vhost::Memory as *const _ as *mut libc::c_void) } }; if ret < 0 { return Err(anyhow::anyhow!( @@ -378,7 +378,7 @@ impl VirtioNet { index: qi, num: qc.size as u32, }; - if unsafe { libc::ioctl(vhost_fd, vhost::SET_VRING_NUM, &state) } < 0 { + if unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_VRING_NUM as u64, &state as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( "VHOST_SET_VRING_NUM(q={qi}) failed: {}", std::io::Error::last_os_error() @@ -394,7 +394,7 @@ impl VirtioNet { avail_user_addr: self.gpa_to_hva(qc.avail_addr), log_guest_addr: 0, }; - if unsafe { libc::ioctl(vhost_fd, vhost::SET_VRING_ADDR, &addr) } < 0 { + if unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_VRING_ADDR as u64, &addr as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( "VHOST_SET_VRING_ADDR(q={qi}) failed: {}", std::io::Error::last_os_error() @@ -415,7 +415,7 @@ impl VirtioNet { index: qi, num: avail_idx, }; - if unsafe { libc::ioctl(vhost_fd, vhost::SET_VRING_BASE, &base) } < 0 { + if unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_VRING_BASE as u64, &base as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( "VHOST_SET_VRING_BASE(q={qi}, avail_idx={avail_idx}) failed: {}", std::io::Error::last_os_error() @@ -427,7 +427,7 @@ impl VirtioNet { index: qi, fd: kick_fd, }; - if unsafe { libc::ioctl(vhost_fd, vhost::SET_VRING_KICK, &kick) } < 0 { + if unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_VRING_KICK as u64, &kick as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( "VHOST_SET_VRING_KICK(q={qi}) failed: {}", std::io::Error::last_os_error() @@ -439,7 +439,7 @@ impl VirtioNet { index: qi, fd: call_fd, }; - if unsafe { libc::ioctl(vhost_fd, vhost::SET_VRING_CALL, &call) } < 0 { + if unsafe { crate::sys::ioctl(vhost_fd, vhost::SET_VRING_CALL as u64, &call as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( "VHOST_SET_VRING_CALL(q={qi}) failed: {}", std::io::Error::last_os_error() @@ -451,7 +451,7 @@ impl VirtioNet { index: qi, fd: self.tap_fd, }; - if unsafe { libc::ioctl(vhost_fd, vhost::NET_SET_BACKEND, &backend) } < 0 { + if unsafe { crate::sys::ioctl(vhost_fd, vhost::NET_SET_BACKEND as u64, &backend as *const _ as *mut libc::c_void) } < 0 { return Err(anyhow::anyhow!( "VHOST_NET_SET_BACKEND(q={qi}) failed: {}", std::io::Error::last_os_error() @@ -496,6 +496,11 @@ impl VirtioNet { /// GPA → host pointer, accounting for MMIO hole. fn gpa_to_ptr(&self, gpa: u64) -> *mut u8 { + // gpa is relative to mem_base (0 on x86, RAM_BASE on arm64). + let gpa = match gpa.checked_sub(self.mem_base) { + Some(g) => g, + None => return std::ptr::null_mut(), + }; let offset = if self.hole_start > 0 && gpa >= self.hole_end { self.hole_start + (gpa - self.hole_end) } else { @@ -737,6 +742,10 @@ impl VirtioDevice for VirtioNet { self.hole_end = hole_end; } + fn set_mem_base(&mut self, mem_base: u64) { + self.mem_base = mem_base; + } + fn activate(&mut self) -> anyhow::Result<()> { self.activated = true; diff --git a/src/virtio/queue.rs b/src/virtio/queue.rs index 8e0058f..4e310b0 100644 --- a/src/virtio/queue.rs +++ b/src/virtio/queue.rs @@ -59,8 +59,11 @@ pub struct Virtqueue { used_ring: u64, /// Last index we consumed from the available ring. last_avail_idx: u16, - /// Host pointer to guest physical address 0. + /// Host pointer to the start of the mapped guest memory region. + /// Corresponds to GPA `mem_base` (0 on x86, RAM_BASE on arm64). guest_mem: *mut u8, + /// GPA of the start of the mapped region (0 when guest RAM starts at 0). + mem_base: u64, /// Total size of the guest memory mmap region. guest_mem_size: u64, /// MMIO hole: GPA range [hole_start, hole_end) has no backing memory. @@ -83,6 +86,7 @@ impl Virtqueue { used_ring: 0, last_avail_idx: 0, guest_mem, + mem_base: 0, guest_mem_size, hole_start: 0, hole_end: 0, @@ -95,6 +99,11 @@ impl Virtqueue { self.hole_end = hole_end; } + /// Set the GPA of the start of the mapped region (arm64: RAM_BASE). + pub fn set_mem_base(&mut self, mem_base: u64) { + self.mem_base = mem_base; + } + /// Configure the three ring area addresses. pub fn configure(&mut self, desc_table: u64, avail_ring: u64, used_ring: u64) { self.desc_table = desc_table; @@ -140,6 +149,9 @@ impl Virtqueue { /// Translate a guest physical address to a userspace offset, /// accounting for the MMIO hole on large VMs. fn gpa_to_offset(&self, gpa: u64) -> Option { + // Translate within the mapped region: gpa is relative to the + // region start (mem_base), 0 on x86 where guest RAM starts at 0. + let gpa = gpa.checked_sub(self.mem_base)?; if self.hole_start == 0 { // No hole — direct mapping if gpa >= self.guest_mem_size { return None; } diff --git a/src/virtio/vsock.rs b/src/virtio/vsock.rs index 5d960f8..b20ac4e 100644 --- a/src/virtio/vsock.rs +++ b/src/virtio/vsock.rs @@ -103,6 +103,8 @@ pub struct VirtioVsock { queue_configs: Vec, guest_mem: *mut u8, guest_mem_size: u64, + /// GPA of the start of the mapped region (0 on x86, RAM_BASE on arm64). + mem_base: u64, hole_start: u64, hole_end: u64, @@ -191,6 +193,7 @@ impl VirtioVsock { queue_configs: Vec::new(), guest_mem: std::ptr::null_mut(), guest_mem_size: 0, + mem_base: 0, hole_start: 0, hole_end: 0, last_avail: [0; NUM_QUEUES], @@ -234,11 +237,24 @@ impl VirtioVsock { // --- GPA translation --- fn gpa_to_ptr(&self, gpa: u64) -> *mut u8 { + // Guest RAM starts at GPA mem_base (0 on x86, RAM_BASE on arm64); + // translate to an offset within the mapped region first. Bounds + // check so a bogus queue address can't segfault the VMM. + let gpa = match gpa.checked_sub(self.mem_base) { + Some(g) => g, + None => return std::ptr::null_mut(), + }; + if gpa >= self.guest_mem_size { + return std::ptr::null_mut(); + } let offset = if self.hole_start > 0 && gpa >= self.hole_end { self.hole_start + (gpa - self.hole_end) } else { gpa }; + if offset >= self.guest_mem_size { + return std::ptr::null_mut(); + } unsafe { self.guest_mem.add(offset as usize) } } @@ -635,12 +651,17 @@ impl VirtioDevice for VirtioVsock { // Initialize queue indices from guest memory. for (qi, qc) in queues.iter().enumerate() { if qi < NUM_QUEUES && !guest_mem.is_null() && qc.avail_addr != 0 { - let avail_idx = unsafe { - *(self.gpa_to_ptr(qc.avail_addr + 2) as *const u16) - }; - let used_idx = unsafe { - *(self.gpa_to_ptr(qc.used_addr + 2) as *const u16) - }; + let avail_ptr = self.gpa_to_ptr(qc.avail_addr + 2) as *const u16; + let used_ptr = self.gpa_to_ptr(qc.used_addr + 2) as *const u16; + if avail_ptr.is_null() || used_ptr.is_null() { + tracing::warn!( + "virtio-vsock: queue {qi} addresses out of range (avail={:#x} used={:#x}), skipping", + qc.avail_addr, qc.used_addr + ); + continue; + } + let avail_idx = unsafe { *avail_ptr }; + let used_idx = unsafe { *used_ptr }; self.last_used[qi] = used_idx; if qi == TX_QUEUE as usize { // TX: start from avail_idx to skip already-posted buffers. @@ -658,6 +679,10 @@ impl VirtioDevice for VirtioVsock { self.hole_end = hole_end; } + fn set_mem_base(&mut self, mem_base: u64) { + self.mem_base = mem_base; + } + fn activate(&mut self) -> anyhow::Result<()> { self.activated = true; tracing::info!("virtio-vsock: activated (guest_cid={}, userspace)", self.guest_cid); diff --git a/src/vmm/mod.rs b/src/vmm/mod.rs index 7ec2176..ca044c1 100644 --- a/src/vmm/mod.rs +++ b/src/vmm/mod.rs @@ -5,17 +5,21 @@ pub mod vcpu; use std::sync::{Arc, Mutex}; use anyhow::{Context, Result}; +#[cfg(target_arch = "x86_64")] use kvm_bindings::{ - kvm_pit_config, kvm_userspace_memory_region, + kvm_pit_config, kvm_irq_routing, kvm_irq_routing_entry, KVM_IRQ_ROUTING_IRQCHIP, KVM_IRQCHIP_IOAPIC, KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE, - KVM_PIT_SPEAKER_DUMMY, KVM_MEM_LOG_DIRTY_PAGES, + KVM_PIT_SPEAKER_DUMMY, }; +use kvm_bindings::{kvm_userspace_memory_region, KVM_MEM_LOG_DIRTY_PAGES}; use kvm_ioctls::{Kvm, VmFd}; use crate::boot; use crate::memory; +#[cfg(target_arch = "aarch64")] +use crate::boot::arm64; use std::os::unix::io::AsRawFd; use crate::virtio::balloon::VirtioBalloon; @@ -124,26 +128,72 @@ impl Vm { // 1. Set up guest memory with overcommit (MAP_NORESERVE) let mem_size = (self.config.mem_mb as u64) << 20; - let guard_size: u64 = 128 << 20; // 128MB guard past e820 end - let mmio_hole_start: u64 = 0xC000_0000; // 3 GB — MMIO hole starts here - let mmio_hole_end: u64 = 0x1_0000_0000; // 4 GB — MMIO hole ends here - let alloc_size = if mem_size <= mmio_hole_start { - mem_size + guard_size - } else { - mmio_hole_start + (mem_size - mmio_hole_start) + guard_size + // aarch64 (kvmhost layout): one contiguous slot at GPA 0x40000000 — + // the arm64 kernel Image is loaded at RAM_BASE+0x80000 and the DTB + // advertises memory@40000000, so nothing may live below RAM_BASE. + #[cfg(target_arch = "aarch64")] + let (alloc_size, guest_memory) = { + let gm = memory::create_guest_memory(mem_size) + .context("Failed to create guest memory")?; + let mut gm = gm; + gm.set_base(crate::boot::arm64::RAM_BASE); + (mem_size, gm) }; - let guest_memory = if mem_size <= mmio_hole_start { - memory::create_guest_memory(alloc_size) - .context("Failed to create guest memory")? - } else { - memory::create_guest_memory_with_hole(alloc_size, mmio_hole_start, mmio_hole_end) - .context("Failed to create guest memory")? + // x86_64: GPA 0 layout with the 3-4GB MMIO hole for VMs > 3GB. + #[cfg(target_arch = "x86_64")] + let (alloc_size, guest_memory) = { + let guard_size: u64 = 128 << 20; // 128MB guard past e820 end + let mmio_hole_start: u64 = 0xC000_0000; // 3 GB — MMIO hole starts here + let mmio_hole_end: u64 = 0x1_0000_0000; // 4 GB — MMIO hole ends here + + let alloc_size = if mem_size <= mmio_hole_start { + mem_size + guard_size + } else { + mmio_hole_start + (mem_size - mmio_hole_start) + guard_size + }; + + let gm = if mem_size <= mmio_hole_start { + memory::create_guest_memory(alloc_size) + .context("Failed to create guest memory")? + } else { + memory::create_guest_memory_with_hole(alloc_size, mmio_hole_start, mmio_hole_end) + .context("Failed to create guest memory")? + }; + (alloc_size, gm) }; // 2. Register memory region(s) with KVM - if !guest_memory.has_hole() { + #[cfg(target_arch = "aarch64")] + { + let mem_region = kvm_userspace_memory_region { + slot: 0, + guest_phys_addr: crate::boot::arm64::RAM_BASE, + memory_size: mem_size, + userspace_addr: guest_memory.as_ptr() as u64, + flags: KVM_MEM_LOG_DIRTY_PAGES, + }; + unsafe { + self.vm_fd + .set_user_memory_region(mem_region) + .context("Failed to set KVM memory region (arm64 @0x40000000)")?; + } + self.kvm_slot_size = mem_size; + tracing::info!( + "arm64 memory: slot @0x40000000, {}MB, slab at {:#x}", + mem_size >> 20, + guest_memory.as_ptr() as u64 + ); + } + + #[cfg(target_arch = "x86_64")] + { + let guard_size: u64 = 128 << 20; + let mmio_hole_start: u64 = 0xC000_0000; + let mmio_hole_end: u64 = 0x1_0000_0000; + + if !guest_memory.has_hole() { // Small VM: single KVM slot let mem_region = kvm_userspace_memory_region { slot: 0, @@ -157,7 +207,8 @@ impl Vm { .set_user_memory_region(mem_region) .context("Failed to set KVM memory region")?; } - } else { + self.kvm_slot_size = alloc_size; + } else { // Large VM: two KVM slots around the MMIO hole let above_hole = mem_size - mmio_hole_start; @@ -195,34 +246,43 @@ impl Vm { mmio_hole_end, above_hole >> 20, mem_size >> 20, ); + } } let t_memory = boot_start.elapsed(); - // 3. Create in-kernel irqchip (LAPIC + IOAPIC) -- must happen before vCPU creation - self.vm_fd - .create_irq_chip() - .context("Failed to create in-kernel irqchip (LAPIC + IOAPIC)")?; - - // 4. Create in-kernel PIT (i8254 timer) -- must happen before vCPU creation - // KVM_PIT_SPEAKER_DUMMY: handle port 0x61 in-kernel so the kernel's - // PIT verification loop (which reads port 0x61 bit 5) works correctly. - let pit_config = kvm_pit_config { - flags: KVM_PIT_SPEAKER_DUMMY, - ..Default::default() - }; - self.vm_fd - .create_pit2(pit_config) - .context("Failed to create in-kernel PIT (i8254)")?; + // 3. Create in-kernel irqchip + PIT + GSI routing — x86 only. + // On aarch64 the vGICv3 device is created after the vCPUs + // (kvm_arch_vcpu_precreate() refuses once vGIC is up). + #[cfg(target_arch = "x86_64")] + { + // 3. Create in-kernel irqchip (LAPIC + IOAPIC) -- must happen before vCPU creation + self.vm_fd + .create_irq_chip() + .context("Failed to create in-kernel irqchip (LAPIC + IOAPIC)")?; + + // 4. Create in-kernel PIT (i8254 timer) -- must happen before vCPU creation + // KVM_PIT_SPEAKER_DUMMY: handle port 0x61 in-kernel so the kernel's + // PIT verification loop (which reads port 0x61 bit 5) works correctly. + let pit_config = kvm_pit_config { + flags: KVM_PIT_SPEAKER_DUMMY, + ..Default::default() + }; + self.vm_fd + .create_pit2(pit_config) + .context("Failed to create in-kernel PIT (i8254)")?; - // 5. Set up explicit GSI routing (PIT/PIC/IOAPIC) - setup_gsi_routing(&self.vm_fd)?; + // 5. Set up explicit GSI routing (PIT/PIC/IOAPIC) + setup_gsi_routing(&self.vm_fd)?; + } let t_irqchip = boot_start.elapsed(); - tracing::info!("In-kernel irqchip (LAPIC + IOAPIC) and PIT created"); + tracing::info!("arch init: irqchip stage complete"); // 5. Register virtio devices with the MMIO bus let mut virtio_cmdline_params = Vec::new(); + // (base, irq) table for the arm64 DTB virtio-mmio nodes + let mut mmio_devices: Vec<(u64, u32)> = Vec::new(); { let mut mmio_bus = self.mmio_bus.lock().unwrap(); @@ -234,6 +294,7 @@ impl Vm { self.mem_size = mem_size; self.kvm_slot_size = alloc_size; let (base, irq) = mmio_bus.register(Box::new(balloon)); + mmio_devices.push((base, irq)); self.balloon_irq = Some(irq); virtio_cmdline_params.push(format!( "virtio_mmio.device=0x{:x}@0x{:x}:{}", @@ -249,6 +310,7 @@ impl Vm { self.vsock_dev_index = Some(dev_idx); self.vsock_irq = Some(predicted_irq); let (base, irq) = mmio_bus.register(Box::new(vsock)); + mmio_devices.push((base, irq)); debug_assert_eq!(irq, predicted_irq); virtio_cmdline_params.push(format!( "virtio_mmio.device=0x{:x}@0x{:x}:{}", @@ -265,6 +327,7 @@ impl Vm { match crate::virtio::block::VirtioBlock::open(block_path, false) { Ok(block) => { let (base, irq) = mmio_bus.register(Box::new(block)); + mmio_devices.push((base, irq)); virtio_cmdline_params.push(format!( "virtio_mmio.device=0x{:x}@0x{:x}:{}", MMIO_STRIDE, base, irq @@ -281,6 +344,7 @@ impl Vm { match crate::virtio::block::VirtioBlock::open(overlay_path, false) { Ok(block) => { let (base, irq) = mmio_bus.register(Box::new(block)); + mmio_devices.push((base, irq)); virtio_cmdline_params.push(format!( "virtio_mmio.device=0x{:x}@0x{:x}:{}", MMIO_STRIDE, base, irq @@ -308,6 +372,7 @@ impl Vm { if root_dir.is_dir() { let fs_dev = crate::virtio::fs::VirtioFs::new(root_dir, tag.clone()); let (base, irq) = mmio_bus.register(Box::new(fs_dev)); + mmio_devices.push((base, irq)); virtio_cmdline_params.push(format!( "virtio_mmio.device=0x{:x}@0x{:x}:{}", MMIO_STRIDE, base, irq @@ -338,6 +403,7 @@ impl Vm { let call_fds = net_dev.call_fds(); let dev_index = mmio_bus.device_count(); let (base, actual_irq) = mmio_bus.register(Box::new(net_dev)); + mmio_devices.push((base, actual_irq)); debug_assert_eq!(irq, actual_irq); virtio_cmdline_params.push(format!( "virtio_mmio.device=0x{:x}@0x{:x}:{}", @@ -356,6 +422,10 @@ impl Vm { { let mut mmio_bus = self.mmio_bus.lock().unwrap(); mmio_bus.set_guest_memory_with_hole(guest_memory.as_ptr(), guest_memory.size(), guest_memory.hole_start(), guest_memory.hole_end()); + // arm64: guest RAM starts at GPA RAM_BASE (0x40000000), so all + // GPA→HVA translations must subtract the region base. No-op on + // x86 where base() is 0. + mmio_bus.set_mem_base(guest_memory.base()); } // Build the final kernel command line with virtio_mmio.device parameters @@ -407,7 +477,7 @@ impl Vm { tracing::info!("Kernel command line: {cmdline}"); // 6. Load kernel into guest memory - let kernel_entry = boot::load_kernel_with_pci( + let kernel_entry = boot::load_kernel_for_arch( &guest_memory, &self.config.kernel_path, self.config.initrd_path.as_deref(), @@ -415,6 +485,7 @@ impl Vm { self.config.vcpus, mem_size, pci_enabled, + &mmio_devices, ) .context("Failed to load kernel")?; @@ -437,6 +508,86 @@ impl Vm { self.vcpus.push(vcpu); } + // 7b. aarch64: create + place + explicitly init the vGICv3 device. + // Order per kvmhost.c (proven on the 5.10 vendor kernel): + // vCPU FIRST (kvm_arch_vcpu_precreate refuses once vGIC is up), + // then device, DIST addr, REDIST addr, CTRL_INIT. + #[cfg(target_arch = "aarch64")] + { + let mut cd = arm64::KvmCreateDevice { + dev_type: arm64::KVM_DEV_TYPE_ARM_VGIC_V3, + fd: 0, + flags: 0, + }; + let ret = unsafe { + crate::sys::ioctl( + self.vm_fd.as_raw_fd(), + arm64::KVM_CREATE_DEVICE, + &mut cd as *mut _ as *mut libc::c_void, + ) + }; + if ret < 0 { + return Err(anyhow::anyhow!( + "KVM_CREATE_DEVICE(ARM_VGIC_V3) failed: {}", + std::io::Error::last_os_error() + )); + } + let vgic_fd = cd.fd; + crate::vmm::dbg_store_vgic_fd(vgic_fd); + + let set_attr = |group: u32, attr: u64, val: u64| -> Result<()> { + let mut attr_data = val; + let a = arm64::KvmDeviceAttr { + flags: 0, + group, + attr, + addr: &mut attr_data as *mut u64 as u64, + }; + let ret = unsafe { + crate::sys::ioctl( + vgic_fd, + arm64::KVM_SET_DEVICE_ATTR, + &a as *const _ as *mut libc::c_void, + ) + }; + if ret < 0 { + Err(anyhow::anyhow!( + "vGIC attr group={group} attr={attr} failed: {}", + std::io::Error::last_os_error() + )) + } else { + Ok(()) + } + }; + + set_attr( + arm64::KVM_DEV_ARM_VGIC_GRP_ADDR, + arm64::KVM_VGIC_V3_ADDR_TYPE_DIST, + arm64::GIC_DIST_BASE, + ) + .context("vgic-v3 DIST addr")?; + set_attr( + arm64::KVM_DEV_ARM_VGIC_GRP_ADDR, + arm64::KVM_VGIC_V3_ADDR_TYPE_REDIST, + arm64::GIC_REDIST_BASE, + ) + .context("vgic-v3 REDIST addr")?; + set_attr( + arm64::KVM_DEV_ARM_VGIC_GRP_CTRL, + arm64::KVM_DEV_ARM_VGIC_CTRL_INIT, + 0, + ) + .context("vgic-v3 CTRL_INIT")?; + // vgic fd intentionally leaked for the VM's lifetime (KVM owns state; + // closing early can tear the interrupt controller down). + tracing::info!( + "vGICv3 created: dist@{:#x} redist@{:#x} ({} vcpus)", + arm64::GIC_DIST_BASE, + arm64::GIC_REDIST_BASE, + self.config.vcpus + ); + } + self.guest_memory = Some(guest_memory); let t_total = boot_start.elapsed(); @@ -532,22 +683,26 @@ impl Vm { } // 3. Create in-kernel irqchip and PIT (required before restore) - self.vm_fd - .create_irq_chip() - .context("Failed to create in-kernel irqchip")?; - let pit_config = kvm_pit_config { - flags: KVM_PIT_SPEAKER_DUMMY, - ..Default::default() - }; - self.vm_fd - .create_pit2(pit_config) - .context("Failed to create in-kernel PIT")?; - setup_gsi_routing(&self.vm_fd)?; + #[cfg(target_arch = "x86_64")] + { + self.vm_fd + .create_irq_chip() + .context("Failed to create in-kernel irqchip")?; + let pit_config = kvm_pit_config { + flags: KVM_PIT_SPEAKER_DUMMY, + ..Default::default() + }; + self.vm_fd + .create_pit2(pit_config) + .context("Failed to create in-kernel PIT")?; + setup_gsi_routing(&self.vm_fd)?; + } // 3b. Restore VM state: PIT → clock → PICs → IOAPIC // (matches Firecracker's restore_state order exactly) // Restore PIT state from snapshot + #[cfg(target_arch = "x86_64")] if !template.device_states.pit.is_empty() { let expected = std::mem::size_of::(); if template.device_states.pit.len() == expected { @@ -568,6 +723,7 @@ impl Vm { } // Restore irqchip (PIC master, PIC slave, IOAPIC) from snapshot + #[cfg(target_arch = "x86_64")] if template.device_states.irqchip.len() == 3 { use kvm_bindings::kvm_irqchip; let expected_size = std::mem::size_of::(); @@ -683,12 +839,12 @@ impl Vm { flags: 1, ..Default::default() }; let ret = unsafe { - libc::ioctl(self.vm_fd.as_raw_fd(), 0x4040AE79u64 as libc::c_ulong, &ioeventfd) + crate::sys::ioctl(self.vm_fd.as_raw_fd(), 0x4040AE79u64 as u64, &ioeventfd as *const _ as *mut libc::c_void) }; if ret != 0 { ioeventfd.flags = 0; ioeventfd.datamatch = 0; - unsafe { libc::ioctl(self.vm_fd.as_raw_fd(), 0x4040AE79u64 as libc::c_ulong, &ioeventfd); } + unsafe { crate::sys::ioctl(self.vm_fd.as_raw_fd(), 0x4040AE79u64 as u64, &ioeventfd as *const _ as *mut libc::c_void); } } } self.net_tap_info = Some((tap_fd, dev_index, actual_irq, kick_evt)); @@ -703,6 +859,7 @@ impl Vm { // (virtio-fs registered above, before net) mmio_bus.set_guest_memory_with_hole(guest_memory.as_ptr(), guest_memory.size(), guest_memory.hole_start(), guest_memory.hole_end()); + mmio_bus.set_mem_base(guest_memory.base()); // Restore transport state from the template snapshot so devices are // in the same state the guest kernel expects (queue addresses, activated). @@ -777,6 +934,7 @@ impl Vm { Arc::clone(&self.serial), )?; // Notify guest this vCPU was paused (prevents soft lockup watchdog) + #[cfg(target_arch = "x86_64")] let _ = vcpu.fd().kvmclock_ctrl(); self.vcpus.push(vcpu); } @@ -791,8 +949,8 @@ impl Vm { transport.vhost_interrupt().fetch_or(1, std::sync::atomic::Ordering::Release); } drop(bus); - let _ = self.vm_fd.set_irq_line(irq, true); - let _ = self.vm_fd.set_irq_line(irq, false); + let _ = set_spi_line(&self.vm_fd, irq, true); + let _ = set_spi_line(&self.vm_fd, irq, false); } tracing::info!( @@ -836,8 +994,8 @@ impl Vm { // Raise IRQ 4 if the guest enabled data-available interrupts (IER bit 0) if serial.interrupt_enabled() { drop(serial); // release lock before ioctl - let _ = vm_fd_clone.set_irq_line(4, true); - let _ = vm_fd_clone.set_irq_line(4, false); + let _ = set_spi_line(&vm_fd_clone, 4, true); + let _ = set_spi_line(&vm_fd_clone, 4, false); } } }); @@ -893,8 +1051,8 @@ impl Vm { } } // Inject IRQ (edge-triggered) - let _ = vm_fd_clone.set_irq_line(irq, true); - let _ = vm_fd_clone.set_irq_line(irq, false); + let _ = set_spi_line(&vm_fd_clone, irq, true); + let _ = set_spi_line(&vm_fd_clone, irq, false); } } }); @@ -909,8 +1067,8 @@ impl Vm { let signal = Arc::new(crate::virtio::vsock::IrqSignal { interrupt_status, signal_fn: Box::new(move || { - let _ = vm_fd.set_irq_line(irq, true); - let _ = vm_fd.set_irq_line(irq, false); + let _ = set_spi_line(&vm_fd, irq, true); + let _ = set_spi_line(&vm_fd, irq, false); }), }); if let Some(vsock) = transport.device_mut().as_any_mut().downcast_mut::() { @@ -946,8 +1104,8 @@ impl Vm { // Signal guest interrupt vhost_int.fetch_or(1, std::sync::atomic::Ordering::Release); drop(bus); - let _ = vm_fd_clone.set_irq_line(irq, true); - let _ = vm_fd_clone.set_irq_line(irq, false); + let _ = set_spi_line(&vm_fd_clone, irq, true); + let _ = set_spi_line(&vm_fd_clone, irq, false); } } } @@ -1015,8 +1173,8 @@ impl Vm { } } if need_irq { - let _ = vm_fd_clone.set_irq_line(irq, true); - let _ = vm_fd_clone.set_irq_line(irq, false); + let _ = set_spi_line(&vm_fd_clone, irq, true); + let _ = set_spi_line(&vm_fd_clone, irq, false); } } }) @@ -1094,8 +1252,8 @@ impl Vm { transport.raise_config_change_interrupt(); } } - let _ = vm_fd_clone.set_irq_line(balloon_irq, true); - let _ = vm_fd_clone.set_irq_line(balloon_irq, false); + let _ = set_spi_line(&vm_fd_clone, balloon_irq, true); + let _ = set_spi_line(&vm_fd_clone, balloon_irq, false); } // Periodically refresh overcommit tracking (every 10 ticks = 10s) @@ -1118,7 +1276,7 @@ impl Vm { // Run all vCPUs — BSP on main thread, APs on spawned threads. let mut ap_handles = Vec::new(); - let mut vcpu_threads: Vec = Vec::new(); + let mut vcpu_threads: Vec = Vec::new(); // Drain vCPUs: take APs first (index 1+), then BSP (index 0) runs on main thread. let mut all_vcpus: Vec = self.vcpus.drain(..).collect(); @@ -1126,12 +1284,12 @@ impl Vm { // Spawn AP threads (vCPU 1, 2, ...) and capture their pthread_t handles for vcpu in all_vcpus.drain(1..) { // Use a channel to get the pthread_t from inside the spawned thread - let (tx, rx) = std::sync::mpsc::channel::(); + let (tx, rx) = std::sync::mpsc::channel::(); let handle = std::thread::Builder::new() .name(format!("vcpu-{}", vcpu.id())) .spawn(move || { // Send our pthread_t to the main thread - let _ = tx.send(unsafe { libc::pthread_self() }); + let _ = tx.send(unsafe { libc::pthread_self() } as usize); let mut vcpu = vcpu; if let Err(e) = vcpu.run_loop() { tracing::error!("vCPU {} exited with error: {e}", vcpu.id()); @@ -1146,7 +1304,7 @@ impl Vm { } // BSP (vCPU 0) runs on the main thread — store its pthread_t - let bsp_tid = unsafe { libc::pthread_self() }; + let bsp_tid = unsafe { libc::pthread_self() } as usize; // Insert BSP at front so vcpu_threads[0] = BSP vcpu_threads.insert(0, bsp_tid); @@ -1264,8 +1422,8 @@ impl Vm { serial.enqueue_input(buf[i]); if serial.interrupt_enabled() { drop(serial); - let _ = vm_fd_for_console.set_irq_line(4, true); - let _ = vm_fd_for_console.set_irq_line(4, false); + let _ = set_spi_line(&vm_fd_for_console, 4, true); + let _ = set_spi_line(&vm_fd_for_console, 4, false); } } } @@ -1320,6 +1478,7 @@ impl Vm { /// This matches what KVM sets up by default with create_irq_chip(), but /// being explicit ensures nothing is missed. GSI 0-7 go to both PIC master /// and IOAPIC, GSI 8-15 to PIC slave and IOAPIC, GSI 16-23 to IOAPIC only. +#[cfg(target_arch = "x86_64")] pub fn setup_gsi_routing(vm_fd: &VmFd) -> Result<()> { // Build routing entries: 24 IOAPIC + 8 PIC master + 8 PIC slave = 40 entries let mut entries: Vec = Vec::with_capacity(40); @@ -1373,3 +1532,83 @@ pub fn setup_gsi_routing(vm_fd: &VmFd) -> Result<()> { tracing::info!("GSI routing configured: 24 IOAPIC + 16 PIC entries"); Ok(()) } + +/// Inject a device IRQ line. x86: raw IRQ line. aarch64: KVM_IRQ_LINE +/// takes a composite word `type<<24 | vcpu<<16 | num` — SPI injection is +/// `KVM_ARM_IRQ_TYPE_SPI=1` and the number is the GIC **INTID** (SPI +/// number + 32), like kvmtool's `spi_irq + 32` and QEMU's +/// `irq + GIC_INTERNAL`. Passing the bare SPI number fails the kernel's +/// `irqid < 32` check with -EINVAL (verified on-device via strace: +/// ioctl(4, KVM_IRQ_LINE, ...) = -1 EINVAL — the block-device kick was +/// reaching clone but the injection was rejected, so virtio I/O hung). +/// Lines are edge-pulsed by the callers. +#[allow(unused_variables)] +fn set_spi_line(vm_fd: &VmFd, irq: u32, active: bool) { + #[cfg(target_arch = "aarch64")] + let irq = (1u32 << 24) | ((irq + 32) & 0xFFFF); + let ret = vm_fd.set_irq_line(irq, active); + if let Err(e) = ret { + tracing::warn!("set_irq_line({irq:#x}, {active}) failed: {e}"); + } + let _ = ret; +} + +// TEMP DEBUG: vgic fd stashed for post-injection state dumps. +static VGIC_FD: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1); +pub fn dbg_store_vgic_fd(fd: i32) { + VGIC_FD.store(fd, std::sync::atomic::Ordering::SeqCst); +} + +#[cfg(target_arch = "aarch64")] +fn dbg_vgic_read_reg(group: u32, attr: u64) -> u32 { + let vgic_fd = VGIC_FD.load(std::sync::atomic::Ordering::SeqCst); + if vgic_fd < 0 { + return 0xDEADBEEF; + } + let mut val: u32 = 0; + let a = crate::boot::arm64::KvmDeviceAttr { + flags: 0, + group, + attr, + addr: &mut val as *mut u32 as u64, + }; + let ret = unsafe { + crate::sys::ioctl( + vgic_fd, + crate::boot::arm64::KVM_GET_DEVICE_ATTR, + &a as *const _ as *mut libc::c_void, + ) + }; + if ret < 0 { + let err = std::io::Error::last_os_error(); + tracing::warn!("vgic GET_DEVICE_ATTR group={group} attr={attr:#x} failed: {err}"); + return 0xFFFF_FFFF; + } + val +} + +/// TEMP DEBUG: after injecting an SPI, dump the vGIC state that determines +/// whether the guest can possibly take it (CTLR, ISENABLER, ISPENDR for the +/// intid, and IROUTER). Called from the run loop for virtio kicks. +#[cfg(target_arch = "aarch64")] +pub fn dbg_vgic_dump_spi(intid: u32) { + if VGIC_FD.load(std::sync::atomic::Ordering::SeqCst) < 0 { + return; + } + let ena_off = 0x100u64 + ((intid / 32) * 4) as u64; + let pend_off = 0x200u64 + ((intid / 32) * 4) as u64; + let irouter_off = 0x6000u64 + ((intid - 32) * 8) as u64; + let ctlr = dbg_vgic_read_reg(1, 0x0); + let isena = dbg_vgic_read_reg(1, ena_off); + let ispend = dbg_vgic_read_reg(1, pend_off); + let irouter = dbg_vgic_read_reg(1, irouter_off); + let bit = intid % 32; + tracing::info!( + "vgic-dump intid={intid}: CTLR={ctlr:#x} ISENABLER[{ena_off:#x}]={isena:#x} (bit{bit}={}) ISPENDR[{pend_off:#x}]={ispend:#x} (bit{bit}={}) IROUTER[{irouter_off:#x}]={irouter:#x}", + isena & (1 << bit) != 0, + ispend & (1 << bit) != 0, + ); +} + +#[cfg(not(target_arch = "aarch64"))] +pub fn dbg_vgic_dump_spi(_intid: u32) {} diff --git a/src/vmm/vcpu.rs b/src/vmm/vcpu.rs index c64ba24..4c9f6b6 100644 --- a/src/vmm/vcpu.rs +++ b/src/vmm/vcpu.rs @@ -3,8 +3,13 @@ use std::sync::{Arc, Condvar, Mutex}; use std::time::Duration; use anyhow::{Context, Result}; +#[cfg(target_arch = "x86_64")] use kvm_bindings::{KVM_MAX_CPUID_ENTRIES, Msrs}; -// AsRawFd removed — no longer needed +// AsRawFd needed for the aarch64 ONE_REG path +#[cfg(target_arch = "aarch64")] +use std::os::unix::io::AsRawFd; +#[cfg(target_arch = "aarch64")] +use crate::boot::arm64; use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; use vm_memory::GuestAddress; @@ -144,6 +149,11 @@ impl Vcpu { .create_vcpu(id as u64) .context("Failed to create vCPU")?; + // x86: CPUID passthrough + MTRR/PAT MSRs + LAPIC + long-mode sregs. + // aarch64: KVM_ARM_VCPU_INIT (generic_v8 + PSCI 0.2); BSP enters at + // the Image load address with x0 = DTB (kvmhost.c contract). + #[cfg(target_arch = "x86_64")] + { // Set up CPUID -- pass through host CPUID with KVM filtering let mut cpuid = kvm .get_supported_cpuid(KVM_MAX_CPUID_ENTRIES) @@ -260,10 +270,74 @@ impl Vcpu { regs.rsi = boot_params_addr(); fd.set_regs(®s).context("Failed to set regs")?; } + } // end x86_64 branch + + #[cfg(target_arch = "aarch64")] + { + let mut vi = arm64::KvmVcpuInit { + target: arm64::KVM_ARM_TARGET_GENERIC_V8, + features: [0u32; 7], + }; + vi.features[0] |= 1u32 << arm64::KVM_ARM_VCPU_PSCI_0_2; + // Non-BSP vCPUs must start POWERED OFF, or the guest's PSCI + // CPU_ON returns ALREADY_ON (mapped to -EINVAL/-22) and SMP + // never comes up — CPU1 never boots. Verified on-device: with + // this bit, "smp: Brought up 1 node, 2 CPUs". + if id != 0 { + vi.features[0] |= 1u32 << arm64::KVM_ARM_VCPU_POWER_OFF; + } + let ret = unsafe { + crate::sys::ioctl( + fd.as_raw_fd(), + arm64::KVM_ARM_VCPU_INIT, + &mut vi as *mut _ as *mut libc::c_void, + ) + }; + if ret < 0 { + anyhow::bail!( + "KVM_ARM_VCPU_INIT failed: {}", + std::io::Error::last_os_error() + ); + } + // BSP enters at the Image load address with x0 = DTB phys addr. + if id == 0 { + Self::set_arm64_core_reg( + fd.as_raw_fd(), + arm64::REG_ID_X0, + arm64::GUEST_DTB_ADDR.load(Ordering::Relaxed), + )?; + Self::set_arm64_core_reg(fd.as_raw_fd(), arm64::REG_ID_PC, entry_addr.0)?; + } + } Ok(Self { id, fd, vm_fd: Arc::clone(vm_fd), mmio_bus, serial, cmos_index: 0, pm_regs: [0u8; 8], pause_state: None, pci_bus: None, deferred_tsc_deadline: None }) } + /// Write one aarch64 core register via KVM_SET_ONE_REG. + #[cfg(target_arch = "aarch64")] + fn set_arm64_core_reg(vcpu_fd: libc::c_int, reg_id: u64, value: u64) -> Result<()> { + let mut v = value; + let r = arm64::KvmOneReg { + id: reg_id, + addr: &mut v as *mut u64 as u64, + }; + let ret = unsafe { + crate::sys::ioctl( + vcpu_fd, + arm64::KVM_SET_ONE_REG, + &r as *const _ as *mut libc::c_void, + ) + }; + if ret < 0 { + Err(anyhow::anyhow!( + "KVM_SET_ONE_REG {reg_id:#x}={value:#x} failed: {}", + std::io::Error::last_os_error() + )) + } else { + Ok(()) + } + } + /// Set the shared pause state for snapshot coordination. pub fn set_pause_state(&mut self, state: Arc) { self.pause_state = Some(state); @@ -277,6 +351,7 @@ impl Vcpu { /// Create a vCPU and restore its register state from a template snapshot. /// /// Used by `clone fork` to resume from a CoW template without re-booting. + #[cfg(target_arch = "x86_64")] pub fn from_template( kvm: &Kvm, vm_fd: &Arc, @@ -469,6 +544,20 @@ impl Vcpu { Ok(Self { id, fd, vm_fd: Arc::clone(vm_fd), mmio_bus, serial, cmos_index: 0, pm_regs: [0u8; 8], pause_state: None, pci_bus: None, deferred_tsc_deadline }) } + /// aarch64: template/fork restore is x86-only for now — fresh boot + /// (the /lokiinit path) works. + #[cfg(not(target_arch = "x86_64"))] + pub fn from_template( + _kvm: &Kvm, + _vm_fd: &Arc, + _id: u32, + _vcpu_state: &crate::boot::template::VcpuState, + _mmio_bus: Arc>, + _serial: Arc>, + ) -> Result { + anyhow::bail!("template/fork restore not yet ported to this arch (x86-only)") + } + /// Serialize a KVM struct to bytes. fn to_bytes(val: &T) -> Vec { unsafe { @@ -480,6 +569,7 @@ impl Vcpu { } /// Capture complete vCPU state for snapshotting. + #[cfg(target_arch = "x86_64")] pub fn capture_state(&self) -> Result { let regs = self.fd.get_regs().context("get kvm_regs")?; let sregs = self.fd.get_sregs().context("get kvm_sregs")?; @@ -522,9 +612,16 @@ impl Vcpu { }) } + /// aarch64: snapshot capture is x86-only for now. + #[cfg(not(target_arch = "x86_64"))] + pub fn capture_state(&self) -> Result { + anyhow::bail!("snapshot capture not yet ported to this arch (x86-only)") + } + /// Capture all MSRs from the vCPU. Uses KVM's msrs_to_save list /// (like Firecracker) to capture ALL MSRs the vCPU needs, not just /// a hardcoded subset. + #[cfg(target_arch = "x86_64")] fn capture_msrs(&self) -> Result> { // Get the full list of MSRs KVM says need saving let kvm = kvm_ioctls::Kvm::new().context("open /dev/kvm for MSR list")?; @@ -611,6 +708,7 @@ impl Vcpu { // This must be the LAST thing before entering the guest to prevent // Restart the LAPIC timer to fire immediately (one-shot mode). // This wakes the guest from HLT and lets hrtimer_interrupt reprogram. + #[cfg(target_arch = "x86_64")] if self.deferred_tsc_deadline.take().is_some() { if let Ok(mut lapic) = self.fd.get_lapic() { // Force one-shot mode if snapshot had TSC-deadline @@ -630,6 +728,8 @@ impl Vcpu { let _ = self.fd.set_lapic(&lapic); } } + #[cfg(not(target_arch = "x86_64"))] + let _ = self.deferred_tsc_deadline.take(); let mut exit_count: u64 = 0; let mut serial_bytes: u64 = 0; @@ -638,6 +738,7 @@ impl Vcpu { loop { if SHUTDOWN_REQUESTED.load(Ordering::Relaxed) { self.serial.lock().unwrap().flush_output(); + #[cfg(target_arch = "x86_64")] if let Ok(regs) = self.fd.get_regs() { tracing::info!( "vCPU {} shutting down: RIP={:#x} RSP={:#x} RFLAGS={:#x} exits={} serial={}", @@ -646,6 +747,11 @@ impl Vcpu { } else { tracing::info!("vCPU {} shutting down (signal received)", self.id); } + #[cfg(not(target_arch = "x86_64"))] + tracing::info!( + "vCPU {} shutting down (signal received) exits={} serial={}", + self.id, exit_count, serial_bytes + ); break; } let action = match self.fd.run() { @@ -676,6 +782,26 @@ impl Vcpu { } VcpuExit::MmioRead(addr, data) => { let mut handled = false; + // aarch64: PL011 UART window. The Flag Register (0x18) + // must report TXFE|RXFE set (0x90): the kernel console + // path polls FR waiting for "TX FIFO empty" to be + // SET — returning 0 leaves it spinning forever after + // the first store (proven with a ctypes replica). + // Other PL011 reads return 0 (all-clear). + #[cfg(target_arch = "aarch64")] + if addr >= crate::boot::arm64::PL011_BASE + && addr < crate::boot::arm64::PL011_BASE + crate::boot::arm64::PL011_SIZE + { + data.fill(0x00); + if (addr - crate::boot::arm64::PL011_BASE) == 0x18 { + // FR: CTS|DSR|DCD|BUSY|RXFE|TXFF|RXFF|TXFE + // = TXFE (bit 7) | RXFE (bit 4) = 0x90 + if let Some(first) = data.first_mut() { + *first = 0x90; + } + } + handled = true; + } // Try PCI bus first (ECAM + BAR regions) if let Some(ref pci_bus) = self.pci_bus { let bus = pci_bus.lock().unwrap(); @@ -695,19 +821,52 @@ impl Vcpu { ExitAction::MmioRead { addr, len: data.len() } } VcpuExit::MmioWrite(addr, data) => { + // aarch64: PL011 UART window — forward bytes to the + // host console (guest earlycon/console=ttyAMA0). + // A store to DR may be a 32-bit access; the character + // is the low byte, so forward data[0] only. + #[cfg(target_arch = "aarch64")] + if addr >= crate::boot::arm64::PL011_BASE + && addr < crate::boot::arm64::PL011_BASE + crate::boot::arm64::PL011_SIZE + { + use std::io::Write; + let out = std::io::stdout(); + let mut lock = out.lock(); + let _ = lock.write_all(&data[..1]); + let _ = lock.flush(); + } let len = data.len(); let mut data_bytes = [0u8; 8]; let copy_len = len.min(8); data_bytes[..copy_len].copy_from_slice(&data[..copy_len]); ExitAction::MmioWrite { addr, data_bytes, len } } - VcpuExit::Debug(dbg_info) => ExitAction::Debug { pc: dbg_info.pc, dr6: dbg_info.dr6 }, - other => ExitAction::Unknown(format!("{:?}", other)), + VcpuExit::Debug(dbg_info) => { + // x86-only fields (pc/dr6); aarch64 has no DR registers. + #[cfg(target_arch = "x86_64")] + let action = ExitAction::Debug { pc: dbg_info.pc, dr6: dbg_info.dr6 }; + #[cfg(not(target_arch = "x86_64"))] + let action = ExitAction::Debug { pc: 0, dr6: 0 }; + action + } + #[cfg(target_arch = "aarch64")] + VcpuExit::SystemEvent(event_type, _data) => { + // Guest called PSCI SYSTEM_OFF/SYS_RESET (e.g. the + // panic handler with panic=-1, or a clean poweroff). + // Not an error — treat as a deliberate guest shutdown. + tracing::info!( + "vCPU {} guest PSCI system event type={}: guest requested stop", + self.id, event_type + ); + ExitAction::Shutdown + } + other => ExitAction::Unknown(format!("{:?}", other)), }, Err(e) => { if e.errno() == libc::EAGAIN || e.errno() == libc::EINTR { if SHUTDOWN_REQUESTED.load(Ordering::SeqCst) { self.serial.lock().unwrap().flush_output(); + #[cfg(target_arch = "x86_64")] if let Ok(regs) = self.fd.get_regs() { let cr2 = self.fd.get_sregs().map(|s| s.cr2).unwrap_or(0); tracing::info!( @@ -717,12 +876,18 @@ impl Vcpu { } else { tracing::info!("vCPU {} received shutdown signal", self.id); } + #[cfg(not(target_arch = "x86_64"))] + tracing::info!( + "vCPU {} signal shutdown: exits={} serial={}", + self.id, exit_count, serial_bytes + ); break; } // Check if this EINTR was from a pause request (SIGUSR1) self.check_pause(); continue; } + #[cfg(target_arch = "x86_64")] if let Ok(regs) = self.fd.get_regs() { if let Ok(sregs) = self.fd.get_sregs() { tracing::error!( @@ -810,8 +975,16 @@ impl Vcpu { tracing::warn!("MMIO write: unhandled addr={addr:#x}, len={len}"); } if let Some(irq) = irq { - let _ = self.vm_fd.set_irq_line(irq, true); - let _ = self.vm_fd.set_irq_line(irq, false); + // aarch64: virtio kicks must be injected as SPIs + // (KVM_IRQ_LINE word type<<24|num with + // KVM_ARM_IRQ_TYPE_SPI) — a raw irq number means + // IRQ_TYPE_CPU (a PPI/CPU line) and the guest + // virtio driver never sees its interrupt, so + // block I/O hangs forever. Same as the vsock/ + // balloon paths which already use set_spi_line. + crate::vmm::set_spi_line(&self.vm_fd, irq, true); + crate::vmm::set_spi_line(&self.vm_fd, irq, false); + crate::vmm::dbg_vgic_dump_spi(irq + 32); } } } @@ -856,11 +1029,13 @@ fn cmos_read(index: u8) -> u8 { /// Address where boot_params struct is placed in guest memory. /// Convention from Linux x86 boot protocol. +#[cfg(target_arch = "x86_64")] fn boot_params_addr() -> u64 { 0x7000 } /// Configure segment registers and page tables for 64-bit long mode. +#[cfg(target_arch = "x86_64")] fn setup_long_mode(sregs: &mut kvm_bindings::kvm_sregs) { // Set up page tables for identity mapping // PML4 at 0x9000, PDPT at 0xA000, PD at 0xB000