Skip to content

feat: virtiofs rootfs hotplug for LinuxPod on cloud-hypervisor #809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
crosbymichael wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: virtiofs rootfs hotplug for LinuxPod on cloud-hypervisor #809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ LINUX_INTEGRATION_KERNEL := kernel/vmlinuz-x86_64
else
LINUX_INTEGRATION_KERNEL := kernel/vmlinux-$(KERNEL_ARCH)
endif

# Optional test-name filter for `make linux-integration`, e.g.
# make linux-integration FILTER="pod hotplug"
# Comma-separated; a test is kept if its name contains ANY of the substrings.
FILTER ?=
linux_integration_filter = $(if $(strip $(FILTER)),--filter '$(strip $(FILTER))')
ifeq ($(UNAME_S),Darwin)
SWIFT ?= /usr/bin/swift
else
Expand Down Expand Up @@ -239,7 +245,7 @@ ifeq (,$(wildcard bin/initfs.ext4))
@echo "missing bin/initfs.ext4; run 'make init' first (this also seeds the persistent imageStore at .local/integration-cache)"
@exit 1
endif
$(call linux_run,CONTAINERIZATION_RELAXED_SANDBOX=1 ./bin/containerization-integration --kernel ./$(LINUX_INTEGRATION_KERNEL) --ch-binary ./bin/cloud-hypervisor --virtiofsd-binary ./bin/virtiofsd --max-concurrency 1,--kernel $(LINUX_INTEGRATION_KERNEL))
$(call linux_run,CONTAINERIZATION_RELAXED_SANDBOX=1 ./bin/containerization-integration --kernel ./$(LINUX_INTEGRATION_KERNEL) --ch-binary ./bin/cloud-hypervisor --virtiofsd-binary ./bin/virtiofsd --max-concurrency 1 $(linux_integration_filter),--kernel $(LINUX_INTEGRATION_KERNEL))

# Builds the x86_64 deployment tarball.
#
Expand Down
226 changes: 115 additions & 111 deletions Sources/Containerization/CHHotplugProvider.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,39 +93,61 @@ final class CHHotplugProvider: HotplugProvider {

// MARK: - HotplugProvider conformance

func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem {
guard case .virtioblk = block.runtimeOptions else {
throw ContainerizationError(.invalidArgument, message: "hotplug requires a virtio-blk mount")
}
func hotplug(_ rootfs: Mount, id: String) async throws -> AttachedFilesystem {
switch rootfs.runtimeOptions {
case .virtioblk:
let letter = try allocator.allocate()
let chId = "blk-\(id)-\(letter)"
let disk = CloudHypervisor.DiskConfig(
path: rootfs.source,
readonly: rootfs.options.contains("ro"),
id: chId,
imageType: .raw
)

let letter = try allocator.allocate()
let chId = "blk-\(id)-\(letter)"
let disk = CloudHypervisor.DiskConfig(
path: block.source,
readonly: block.options.contains("ro"),
id: chId,
imageType: .raw
)

let pci: CloudHypervisor.PciDeviceInfo
do {
pci = try await chCall { try await self.client.vmAddDisk(disk) }
} catch {
try? allocator.release(letter)
throw error
}
let pci: CloudHypervisor.PciDeviceInfo
do {
pci = try await chCall { try await self.client.vmAddDisk(disk) }
} catch {
try? allocator.release(letter)
throw error
}

let attached = AttachedFilesystem(
type: rootfs.type,
source: "/dev/vd\(letter)",
destination: rootfs.destination,
options: rootfs.options
)

let attached = AttachedFilesystem(
type: block.type,
source: "/dev/vd\(letter)",
destination: block.destination,
options: block.options
)
_records.withLock {
$0[id, default: []].append(HotplugRecord(chDeviceId: pci.id, kind: .block(letter: letter)))
}
return attached

case .virtiofs:
// Compute the tag up front (throwing) so nothing can fail between
// committing the virtiofsd/device and recording the HotplugRecord —
// otherwise a thrown error would orphan a running virtiofsd.
let tag = try hashFilePath(path: rootfs.source)
let chDeviceId = try await ensureVirtiofsDevice(
tag: tag,
source: rootfs.source,
readonly: rootfs.options.contains("ro")
)
_records.withLock {
$0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag)))
}
return AttachedFilesystem(
type: rootfs.type,
source: tag,
destination: rootfs.destination,
options: rootfs.options
)

_records.withLock {
$0[id, default: []].append(HotplugRecord(chDeviceId: pci.id, kind: .block(letter: letter)))
case .shared, .any:
throw ContainerizationError(.unsupported, message: "hotplug rootfs must be virtio-blk or virtiofs")
}
return attached
}

func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws {
Expand Down Expand Up @@ -199,93 +221,75 @@ final class CHHotplugProvider: HotplugProvider {
}

for (tag, group) in byTag {
// Hold spawnLock across the existence check and the spawn /
// _tags write so two concurrent calls for the same tag can't
// both decide alreadyRunning=false and double-spawn virtiofsd
// (the second write would clobber the first in `_tags`,
// orphaning that process).
try await spawnLock.withLock { _ in
// Build per-container AttachedFilesystem entries up front.
// These depend only on Mount + allocator and don't need the
// chDeviceId, so surfacing any error here keeps the
// transactional shape: nothing irreversible has happened
// yet, no virtiofsd has spawned, no _tags entry written.
var attached: [AttachedFilesystem] = []
for mount in group {
attached.append(try AttachedFilesystem(mount: mount, allocator: self.allocator))
}
guard let source = group.first?.source else { continue }
let readonly = group.allSatisfy { $0.options.contains("ro") }
let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly)
// Record once per tag for this container. The AttachedFilesystem
// entries for these mounts are written by registerMounts (the sole
// _mounts writer), so we do NOT touch _mounts here.
_records.withLock {
$0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag)))
}
}
}

let chDeviceId: String

// Refcount-bump path. If a virtiofsd already serves this
// tag, increment refcount and use the cached deviceId.
let cachedDeviceId: String? = self._tags.withLock { tags in
if var state = tags[tag] {
state.refcount += 1
tags[tag] = state
return state.chDeviceId
}
return nil
/// Ensure a virtio-fs device backed by `virtiofsd` exists for `tag`,
/// spawning one (and issuing `vm.add-fs`) on first use or bumping the
/// refcount of an existing one. Returns the cloud-hypervisor device id
/// (`vm.remove-device` keys on it). Serialized per-provider by `spawnLock`
/// so two concurrent callers for the same tag can't double-spawn.
private func ensureVirtiofsDevice(tag: String, source: String, readonly: Bool) async throws -> String {
try await spawnLock.withLock { _ in
// Refcount-bump path: a virtiofsd already serves this tag.
let cached: String? = self._tags.withLock { tags in
if var state = tags[tag] {
state.refcount += 1
tags[tag] = state
return state.chDeviceId
}
return nil
}
if let cached {
return cached
}

if let cached = cachedDeviceId {
chDeviceId = cached
} else {
// First-spawn path. Walk: spawn → vmAddFs → commit _tags,
// with rollback at every step so a partial failure can't
// leave a virtiofsd running unrecorded.
let socket = chVirtiofsSocketURL(workDir: self.workDir, tag: tag)
let readonly = group.allSatisfy { $0.options.contains("ro") }
guard let source = group.first?.source else { return }
let virtiofsdBinary = try CHVirtualMachineManager.resolveBinary(
self.virtiofsdBinaryOverride,
name: "virtiofsd"
)

let process = VirtiofsdProcess(
config: .init(
binary: virtiofsdBinary,
socketPath: socket,
sharedDir: URL(fileURLWithPath: source),
readonly: readonly
),
logger: self.logger
)

try await process.start()

let fsConfig = CloudHypervisor.FsConfig(
tag: tag,
socket: socket.path,
id: "fs-\(tag)"
)
let pci: CloudHypervisor.PciDeviceInfo
do {
pci = try await chCall { try await self.client.vmAddFs(fsConfig) }
} catch {
await process.terminate(graceSeconds: 5)
try? FileManager.default.removeItem(at: socket)
throw error
}

self._tags.withLock {
$0[tag] = VirtiofsdTagState(process: process, refcount: 1, chDeviceId: pci.id)
}
chDeviceId = pci.id
}
// First-spawn path: spawn → vm.add-fs → commit _tags, rolling back
// the process/socket if vm.add-fs fails.
let socket = chVirtiofsSocketURL(workDir: self.workDir, tag: tag)
let virtiofsdBinary = try CHVirtualMachineManager.resolveBinary(
self.virtiofsdBinaryOverride,
name: "virtiofsd"
)
let process = VirtiofsdProcess(
config: .init(
binary: virtiofsdBinary,
socketPath: socket,
sharedDir: URL(fileURLWithPath: source),
readonly: readonly
),
logger: self.logger
)

// Bookkeeping. Both writes are non-throwing closures, and
// `attached` was built up front, so once we reach here
// nothing can fail between the refcount/spawn commit above
// and the per-container record below — the orphan window
// (tag committed, record missing) is closed.
self._records.withLock {
$0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag)))
}
self._mounts.withLock {
$0[id, default: []].append(contentsOf: attached)
}
try await process.start()

let fsConfig = CloudHypervisor.FsConfig(
tag: tag,
socket: socket.path,
id: "fs-\(tag)"
)
let pci: CloudHypervisor.PciDeviceInfo
do {
pci = try await chCall { try await self.client.vmAddFs(fsConfig) }
} catch {
await process.terminate(graceSeconds: 5)
try? FileManager.default.removeItem(at: socket)
throw error
}

self._tags.withLock {
$0[tag] = VirtiofsdTagState(process: process, refcount: 1, chDeviceId: pci.id)
}
return pci.id
}
}

Expand Down
12 changes: 8 additions & 4 deletions Sources/Containerization/LinuxPod.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,14 +438,18 @@ extension LinuxPod {
additionalMounts: nonSharedMounts
)

// Mount the hotplugged container's virtiofs shares in the
// Mount this container's additional virtiofs shares in the
// guest. create() does this for boot-time containers (the
// /run/virtiofs loop); the hotplug path must do the same or
// the container's bind mounts from /run/virtiofs/<tag> fail
// with ENOENT.
let newVirtiofsTags = (vm.mounts[id] ?? [])
.filter { $0.type == "virtiofs" }
.map { $0.source }
//
// Derive the tags from the additional mounts directly rather
// than from vm.mounts[id], so this is independent of the
// rootfs (which may be virtiofs or virtio-blk) and of mount
// ordering. The rootfs is mounted at /run/container/<id>/rootfs
// and is never consumed from /run/virtiofs.
let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) }
if !newVirtiofsTags.isEmpty {
// Tags already mounted in the guest at boot or by a
// prior hotplug (i.e. present on another container).
Expand Down
Loading
Loading