diff --git a/Makefile b/Makefile index 9c99a9e22..76825da81 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,12 @@ LINUX_INTEGRATION_KERNEL := kernel/vmlinuz-x86_64 else LINUX_INTEGRATION_KERNEL := kernel/vmlinux-$(KERNEL_ARCH) endif + +# Optional test-name filter for `make linux-integration`, e.g. +# make linux-integration FILTER="pod hotplug" +# Comma-separated; a test is kept if its name contains ANY of the substrings. +FILTER ?= +linux_integration_filter = $(if $(strip $(FILTER)),--filter '$(strip $(FILTER))') ifeq ($(UNAME_S),Darwin) SWIFT ?= /usr/bin/swift else @@ -239,7 +245,7 @@ ifeq (,$(wildcard bin/initfs.ext4)) @echo "missing bin/initfs.ext4; run 'make init' first (this also seeds the persistent imageStore at .local/integration-cache)" @exit 1 endif - $(call linux_run,CONTAINERIZATION_RELAXED_SANDBOX=1 ./bin/containerization-integration --kernel ./$(LINUX_INTEGRATION_KERNEL) --ch-binary ./bin/cloud-hypervisor --virtiofsd-binary ./bin/virtiofsd --max-concurrency 1,--kernel $(LINUX_INTEGRATION_KERNEL)) + $(call linux_run,CONTAINERIZATION_RELAXED_SANDBOX=1 ./bin/containerization-integration --kernel ./$(LINUX_INTEGRATION_KERNEL) --ch-binary ./bin/cloud-hypervisor --virtiofsd-binary ./bin/virtiofsd --max-concurrency 1 $(linux_integration_filter),--kernel $(LINUX_INTEGRATION_KERNEL)) # Builds the x86_64 deployment tarball. # diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 9036da502..870fb1ffb 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -93,39 +93,61 @@ final class CHHotplugProvider: HotplugProvider { // MARK: - HotplugProvider conformance - func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem { - guard case .virtioblk = block.runtimeOptions else { - throw ContainerizationError(.invalidArgument, message: "hotplug requires a virtio-blk mount") - } + func hotplug(_ rootfs: Mount, id: String) async throws -> AttachedFilesystem { + switch rootfs.runtimeOptions { + case .virtioblk: + let letter = try allocator.allocate() + let chId = "blk-\(id)-\(letter)" + let disk = CloudHypervisor.DiskConfig( + path: rootfs.source, + readonly: rootfs.options.contains("ro"), + id: chId, + imageType: .raw + ) - let letter = try allocator.allocate() - let chId = "blk-\(id)-\(letter)" - let disk = CloudHypervisor.DiskConfig( - path: block.source, - readonly: block.options.contains("ro"), - id: chId, - imageType: .raw - ) - - let pci: CloudHypervisor.PciDeviceInfo - do { - pci = try await chCall { try await self.client.vmAddDisk(disk) } - } catch { - try? allocator.release(letter) - throw error - } + let pci: CloudHypervisor.PciDeviceInfo + do { + pci = try await chCall { try await self.client.vmAddDisk(disk) } + } catch { + try? allocator.release(letter) + throw error + } + + let attached = AttachedFilesystem( + type: rootfs.type, + source: "/dev/vd\(letter)", + destination: rootfs.destination, + options: rootfs.options + ) - let attached = AttachedFilesystem( - type: block.type, - source: "/dev/vd\(letter)", - destination: block.destination, - options: block.options - ) + _records.withLock { + $0[id, default: []].append(HotplugRecord(chDeviceId: pci.id, kind: .block(letter: letter))) + } + return attached + + case .virtiofs: + // Compute the tag up front (throwing) so nothing can fail between + // committing the virtiofsd/device and recording the HotplugRecord — + // otherwise a thrown error would orphan a running virtiofsd. + let tag = try hashFilePath(path: rootfs.source) + let chDeviceId = try await ensureVirtiofsDevice( + tag: tag, + source: rootfs.source, + readonly: rootfs.options.contains("ro") + ) + _records.withLock { + $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) + } + return AttachedFilesystem( + type: rootfs.type, + source: tag, + destination: rootfs.destination, + options: rootfs.options + ) - _records.withLock { - $0[id, default: []].append(HotplugRecord(chDeviceId: pci.id, kind: .block(letter: letter))) + case .shared, .any: + throw ContainerizationError(.unsupported, message: "hotplug rootfs must be virtio-blk or virtiofs") } - return attached } func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { @@ -199,93 +221,75 @@ final class CHHotplugProvider: HotplugProvider { } for (tag, group) in byTag { - // Hold spawnLock across the existence check and the spawn / - // _tags write so two concurrent calls for the same tag can't - // both decide alreadyRunning=false and double-spawn virtiofsd - // (the second write would clobber the first in `_tags`, - // orphaning that process). - try await spawnLock.withLock { _ in - // Build per-container AttachedFilesystem entries up front. - // These depend only on Mount + allocator and don't need the - // chDeviceId, so surfacing any error here keeps the - // transactional shape: nothing irreversible has happened - // yet, no virtiofsd has spawned, no _tags entry written. - var attached: [AttachedFilesystem] = [] - for mount in group { - attached.append(try AttachedFilesystem(mount: mount, allocator: self.allocator)) - } + guard let source = group.first?.source else { continue } + let readonly = group.allSatisfy { $0.options.contains("ro") } + let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly) + // Record once per tag for this container. The AttachedFilesystem + // entries for these mounts are written by registerMounts (the sole + // _mounts writer), so we do NOT touch _mounts here. + _records.withLock { + $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) + } + } + } - let chDeviceId: String - - // Refcount-bump path. If a virtiofsd already serves this - // tag, increment refcount and use the cached deviceId. - let cachedDeviceId: String? = self._tags.withLock { tags in - if var state = tags[tag] { - state.refcount += 1 - tags[tag] = state - return state.chDeviceId - } - return nil + /// Ensure a virtio-fs device backed by `virtiofsd` exists for `tag`, + /// spawning one (and issuing `vm.add-fs`) on first use or bumping the + /// refcount of an existing one. Returns the cloud-hypervisor device id + /// (`vm.remove-device` keys on it). Serialized per-provider by `spawnLock` + /// so two concurrent callers for the same tag can't double-spawn. + private func ensureVirtiofsDevice(tag: String, source: String, readonly: Bool) async throws -> String { + try await spawnLock.withLock { _ in + // Refcount-bump path: a virtiofsd already serves this tag. + let cached: String? = self._tags.withLock { tags in + if var state = tags[tag] { + state.refcount += 1 + tags[tag] = state + return state.chDeviceId } + return nil + } + if let cached { + return cached + } - if let cached = cachedDeviceId { - chDeviceId = cached - } else { - // First-spawn path. Walk: spawn → vmAddFs → commit _tags, - // with rollback at every step so a partial failure can't - // leave a virtiofsd running unrecorded. - let socket = chVirtiofsSocketURL(workDir: self.workDir, tag: tag) - let readonly = group.allSatisfy { $0.options.contains("ro") } - guard let source = group.first?.source else { return } - let virtiofsdBinary = try CHVirtualMachineManager.resolveBinary( - self.virtiofsdBinaryOverride, - name: "virtiofsd" - ) - - let process = VirtiofsdProcess( - config: .init( - binary: virtiofsdBinary, - socketPath: socket, - sharedDir: URL(fileURLWithPath: source), - readonly: readonly - ), - logger: self.logger - ) - - try await process.start() - - let fsConfig = CloudHypervisor.FsConfig( - tag: tag, - socket: socket.path, - id: "fs-\(tag)" - ) - let pci: CloudHypervisor.PciDeviceInfo - do { - pci = try await chCall { try await self.client.vmAddFs(fsConfig) } - } catch { - await process.terminate(graceSeconds: 5) - try? FileManager.default.removeItem(at: socket) - throw error - } - - self._tags.withLock { - $0[tag] = VirtiofsdTagState(process: process, refcount: 1, chDeviceId: pci.id) - } - chDeviceId = pci.id - } + // First-spawn path: spawn → vm.add-fs → commit _tags, rolling back + // the process/socket if vm.add-fs fails. + let socket = chVirtiofsSocketURL(workDir: self.workDir, tag: tag) + let virtiofsdBinary = try CHVirtualMachineManager.resolveBinary( + self.virtiofsdBinaryOverride, + name: "virtiofsd" + ) + let process = VirtiofsdProcess( + config: .init( + binary: virtiofsdBinary, + socketPath: socket, + sharedDir: URL(fileURLWithPath: source), + readonly: readonly + ), + logger: self.logger + ) - // Bookkeeping. Both writes are non-throwing closures, and - // `attached` was built up front, so once we reach here - // nothing can fail between the refcount/spawn commit above - // and the per-container record below — the orphan window - // (tag committed, record missing) is closed. - self._records.withLock { - $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) - } - self._mounts.withLock { - $0[id, default: []].append(contentsOf: attached) - } + try await process.start() + + let fsConfig = CloudHypervisor.FsConfig( + tag: tag, + socket: socket.path, + id: "fs-\(tag)" + ) + let pci: CloudHypervisor.PciDeviceInfo + do { + pci = try await chCall { try await self.client.vmAddFs(fsConfig) } + } catch { + await process.terminate(graceSeconds: 5) + try? FileManager.default.removeItem(at: socket) + throw error + } + + self._tags.withLock { + $0[tag] = VirtiofsdTagState(process: process, refcount: 1, chDeviceId: pci.id) } + return pci.id } } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 30e87ba7d..f16cb8966 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -438,14 +438,18 @@ extension LinuxPod { additionalMounts: nonSharedMounts ) - // Mount the hotplugged container's virtiofs shares in the + // Mount this container's additional virtiofs shares in the // guest. create() does this for boot-time containers (the // /run/virtiofs loop); the hotplug path must do the same or // the container's bind mounts from /run/virtiofs/ fail // with ENOENT. - let newVirtiofsTags = (vm.mounts[id] ?? []) - .filter { $0.type == "virtiofs" } - .map { $0.source } + // + // Derive the tags from the additional mounts directly rather + // than from vm.mounts[id], so this is independent of the + // rootfs (which may be virtiofs or virtio-blk) and of mount + // ordering. The rootfs is mounted at /run/container//rootfs + // and is never consumed from /run/virtiofs. + let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) } if !newVirtiofsTags.isEmpty { // Tags already mounted in the guest at boot or by a // prior hotplug (i.e. present on another container). diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 949c6d2a8..ae1caec86 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -16,6 +16,7 @@ import ArgumentParser import Containerization +import ContainerizationArchive import ContainerizationEXT4 import ContainerizationError import ContainerizationExtras @@ -2221,4 +2222,145 @@ extension IntegrationSuite { throw error } } + + #if os(Linux) + /// Unpack an image's layers into a host directory to use as a virtiofs + /// (directory-share) rootfs. Assumes a single-layer image (the alpine + /// image used by the suite) so no OCI whiteout processing is required. + /// + /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it + /// here — virtiofsd shares it for the whole test. It is swept by + /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the + /// suite-end `removeItem(at: Self.testDir)`. + private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount { + let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir") + try? FileManager.default.removeItem(at: dir) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + + let platform = Platform(arch: "arm64", os: "linux", variant: "v8") + let manifest = try await image.manifest(for: platform) + for layer in manifest.layers { + let content = try await image.getContent(digest: layer.digest) + let filter: ContainerizationArchive.Filter + switch layer.mediaType { + case MediaTypes.imageLayer, MediaTypes.dockerImageLayer: + filter = .none + case MediaTypes.imageLayerGzip, MediaTypes.dockerImageLayerGzip: + filter = .gzip + case MediaTypes.imageLayerZstd, MediaTypes.dockerImageLayerZstd: + filter = .zstd + default: + throw IntegrationError.assert(msg: "unsupported layer media type \(layer.mediaType)") + } + let reader = try ArchiveReader(format: .paxRestricted, filter: filter, file: content.path) + _ = try reader.extractContents(to: dir) + } + + return .share(source: dir.absolutePath(), destination: "/") + } + + /// Hotplug a container with a virtiofs (directory-share) rootfs into a + /// running pod VM, plus an additional virtiofs file-mount. CH-only: VZ has + /// no runtime hotplug. + func testPodHotplugVirtiofsRootfs() async throws { + let id = "test-pod-hotplug-virtiofs-rootfs" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + // Boot-time seed container (block rootfs) so the target container is + // added strictly on the post-create (hotplug) path. + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + } + + try await pod.create() + + // Also hotplug a virtiofs file-mount onto the container. This exercises + // the /run/virtiofs holding-dir path for an additional virtiofs share on + // a container whose rootfs is itself virtiofs — the combination the + // rootfs-agnostic newVirtiofsTags derivation must handle. cat'ing the + // mounted file also confirms the virtiofs rootfs itself is usable. + let mountContent = "hello from hotplugged virtiofs file mount" + let hostFile = FileManager.default.uniqueTemporaryDirectory(create: true) + .appendingPathComponent("hotplug-mount.txt") + try mountContent.write(to: hostFile, atomically: true, encoding: .utf8) + + let virtiofsRootfs = try await unpackRootfsDirectory(bs.image, testID: id) + let buffer = BufferWriter() + try await pod.addContainer("hot", rootfs: virtiofsRootfs) { config in + config.process.arguments = ["/bin/cat", "/etc/hotplug-mount.txt"] + config.mounts.append(.share(source: hostFile.path, destination: "/etc/hotplug-mount.txt")) + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + guard String(data: buffer.data, encoding: .utf8) == mountContent else { + throw IntegrationError.assert( + msg: "expected '\(mountContent)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + + /// Hotplug a container with a block rootfs into a running pod VM. Guards + /// the existing block hotplug path against the registry-consolidation + /// change. CH-only. + func testPodHotplugBlockRootfs() async throws { + let id = "test-pod-hotplug-block-rootfs" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + } + + try await pod.create() + + let buffer = BufferWriter() + try await pod.addContainer("hot", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot")) { config in + config.process.arguments = ["/bin/echo", "hello from block rootfs"] + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + guard String(data: buffer.data, encoding: .utf8) == "hello from block rootfs\n" else { + throw IntegrationError.assert( + msg: "expected 'hello from block rootfs', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + #endif } diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index dfa274fcb..bf11e1cef 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -621,7 +621,13 @@ struct IntegrationSuite: AsyncParsableCommand { ] + macOS26Tests() let tests: [Test] = crossPlatformTests + macOSOnlyTests #else - let tests: [Test] = crossPlatformTests + // Hotplug into a running pod VM is CH-only (VZ has no runtime hotplug), + // and no pod test elsewhere exercises addContainer-after-create. + let linuxOnlyTests: [Test] = [ + Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), + Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs), + ] + let tests: [Test] = crossPlatformTests + linuxOnlyTests #endif let filteredTests: [Test] diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift index ff6d1c5f1..dd07ef54f 100644 --- a/vminitd/Sources/VminitdCore/Server+GRPC.swift +++ b/vminitd/Sources/VminitdCore/Server+GRPC.swift @@ -663,7 +663,47 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ ) #if os(Linux) - try mnt.mount(createWithPerms: 0o755) + do { + try mnt.mount(createWithPerms: 0o755) + } catch { + // A hot-plugged virtio device (virtio-blk / virtio-fs) may not + // be enumerated by the guest yet when the host issues this + // mount immediately after vm.add-disk / vm.add-fs: cloud- + // hypervisor places the device on the PCI bus but the guest + // does not always auto-probe it. Force a PCI rescan and retry + // with a bounded wait. Scoped to hot-plug-candidate sources so + // an ordinary mount failure isn't delayed. + let hotplugCandidate = request.type == "virtiofs" || request.source.hasPrefix("/dev/vd") + guard hotplugCandidate else { throw error } + + if let rescan = FileHandle(forWritingAtPath: "/sys/bus/pci/rescan") { + defer { try? rescan.close() } + do { + try rescan.write(contentsOf: Data("1".utf8)) + log.info("mount: triggered PCI bus rescan for hot-plugged device") + } catch { + log.error("mount: PCI rescan write failed", metadata: ["error": "\(error)"]) + } + } else { + log.error("mount: cannot open /sys/bus/pci/rescan") + } + + var mounted = false + for attempt in 1...20 { // up to ~2s for the device to enumerate + try? await Task.sleep(for: .milliseconds(100)) + do { + try mnt.mount(createWithPerms: 0o755) + log.info("mount: succeeded after PCI rescan", metadata: ["attempt": "\(attempt)"]) + mounted = true + break + } catch { + continue + } + } + if !mounted { + throw error + } + } return .init() #else fatalError("mount not supported on platform")