From 8545a284f5f188cf6b7562e80814d20f0d74871c Mon Sep 17 00:00:00 2001 From: rain Date: Thu, 29 Oct 2026 03:05:21 +0800 Subject: [PATCH 1/3] feat(sandbox): run workspace.exec in the daemon's mount namespace via an ns-exec helper `workspace.exec` is an Engine exec in the container's initial mount namespace; the daemon is not involved. When the daemon serves from a private mount namespace (the #3/#499 prototype), a Workspace FUSE mount is absent from that namespace, so the Engine's `bash -c ` cannot see the Workspace. This adds a small setns helper so the command runs inside the daemon's namespace while the Engine keeps its Exec entry, streams, cancellation and exit code. - The hidden `layerfs-daemon ns-exec --pid-file [--directory ] [--uid --gid ] -- [args...]` entry reads the daemon pid, opens /proc//ns/mnt, joins it with setns(CLONE_NEWNS), re-enters the mount directory in the joined namespace, drops the root entry identity to the ordinary command identity, then execs the command. A missing pid file, an unreadable pid, a missing namespace, or a refused setns/chdir/drop prints a clear message and exits 125; the command is never run in the wrong namespace. - DaemonSetup gains an opt-in `private_mount_ns` flag (config record LFSD\x06). When set, the daemon entry takes unshare(CLONE_NEWNS) + MS_REC|MS_PRIVATE on `/` before the listener or any worker thread, then atomically writes `daemon.pid` into the local directory above the Overlay's directory (/layerfs-local/daemon.pid in a Sandbox). With the flag off no namespace is taken and no pid file is written. - layerfs-api builds the Engine argv: when the Sandbox's record carries the private-namespace pid file, the argv names the helper and the Engine starts it as root (`ExecRequest.root_launcher`), because the ordinary 501:20 command identity can neither open the daemon's namespace nor setns (verified: a non-root `nsenter` gets EPERM; root succeeds). The helper drops back to the command identity before exec. Without the pid file both the argv and the Engine user are unchanged. - Opt-in: `deploy.privateMountNs` or `LAYERFS_PRIVATE_MOUNT_NS=1` at Sandbox create; the pid file path is saved in and reopened from the sandbox record. Tests: - added engine/crates/layerfs-daemon/tests/ns_exec.rs: a real private namespace built with `unshare -m` and a tmpfs visible only there; ns-exec joins it (read and chdir), drops to 65534, refuses a missing/invalid pid file and a missing namespace without running the command, and the daemon writes its pid file only when the private namespace is asked for. - added unit tests: the wrapper argv decision (layerfs-api), the helper's argument parsing (layerfs-daemon) and the config flag/pid path (layerfs-bridge). - touched ExecRequest literals and the internal::exec call sites; the Engine protocol encoding tests still pass. - deleted: none. Not exercised: a real Sandbox end-to-end (a FUSE mount in the private namespace, then exec), and the daemon's full startup under the flag; the Sandbox container's /dev/fuse and Store are not part of this test set. Production LOC: 102763 -> 103008 (delta +245) core 38395 -> 38395 (0), engine 61709 -> 61954 (+245), api 2659 -> 2659 (0); reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 tools/production_loc.py --root on a `git archive` extract of 582bc786 and on this staged tree. Refs tokenaissance/layerfs#3, tokenaissance/layerfs#499. --- .../examples/sandbox_kernel_mount.rs | 2 + .../layerfs-api/examples/sandbox_lifecycle.rs | 2 + .../crates/layerfs-api/src/base/internal.rs | 13 +- engine/crates/layerfs-api/src/sandbox/exec.rs | 89 ++++- .../layerfs-api/tests/kernel_application.rs | 1 + .../layerfs-api/tests/observed_application.rs | 1 + .../tests/r5_1_actual_daemon_commit.rs | 1 + .../layerfs-api/tests/sandbox_operations.rs | 1 + .../crates/layerfs-bridge/src/daemon_setup.rs | 85 +++- engine/crates/layerfs-daemon/Cargo.toml | 2 +- .../layerfs-daemon/src/application/cli.rs | 72 +++- .../layerfs-daemon/src/application/mod.rs | 2 + .../layerfs-daemon/src/application/ns_exec.rs | 208 ++++++++++ engine/crates/layerfs-daemon/tests/ns_exec.rs | 378 ++++++++++++++++++ engine/crates/layerfs-fuse/Cargo.toml | 2 +- engine/crates/layerfs-fuse/src/mount/mod.rs | 3 +- .../crates/layerfs-fuse/src/mount/syscalls.rs | 21 + engine/crates/layerfs-node/src/discovery.rs | 6 + engine/crates/layerfs-node/src/exec.rs | 1 + engine/crates/layerfs-node/src/registry.rs | 3 + engine/crates/layerfs-node/src/sandbox.rs | 19 +- .../examples/engine_streams.rs | 1 + .../src/backend/docker/request.rs | 14 +- .../layerfs-sandbox/tests/engine_keepalive.rs | 1 + .../layerfs-sandbox/tests/engine_protocol.rs | 1 + 25 files changed, 911 insertions(+), 18 deletions(-) create mode 100644 engine/crates/layerfs-daemon/src/application/ns_exec.rs create mode 100644 engine/crates/layerfs-daemon/tests/ns_exec.rs diff --git a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs index c2e8f73671..56e7449df3 100644 --- a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs +++ b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs @@ -98,6 +98,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { idle_exit_s: 0, }, existing_store: None, + private_mount_ns: false, } } type Running = ( @@ -124,6 +125,7 @@ fn start( vec![], directory.into(), false, + None, ), } .map_err(|f| format!("create {f:?}"))?; diff --git a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs index f54c059ff8..f4427f45bc 100644 --- a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs +++ b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs @@ -37,6 +37,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { idle_exit_s: 0, }, existing_store: None, + private_mount_ns: false, } } fn ordinary(owner: &ManagedSandbox) -> Result<(), Box> { @@ -48,6 +49,7 @@ fn ordinary(owner: &ManagedSandbox) -> Result<(), Box> { vec![], "/".into(), false, + None, ) .map_err(|f| format!("create {f:?}"))?; let attached = created.start().map_err(|f| format!("start {f:?}"))?; diff --git a/engine/crates/layerfs-api/src/base/internal.rs b/engine/crates/layerfs-api/src/base/internal.rs index 7bffa705c0..1d26c26ae6 100644 --- a/engine/crates/layerfs-api/src/base/internal.rs +++ b/engine/crates/layerfs-api/src/base/internal.rs @@ -141,6 +141,8 @@ pub fn parts( /// The same ordinary launcher for external host commands and disjoint concurrent proof borrows. #[cfg(unix)] +/// `ns_pid_file` names a daemon's private-namespace pid file: the command then +/// runs through that daemon's `ns-exec` helper. `None` keeps the command as it is. pub fn exec( sandbox: &layerfs_sandbox::backend::docker::Sandbox, identity: layerfs_sandbox::CommandIdentity, @@ -148,9 +150,18 @@ pub fn exec( environment: Vec, directory: String, stdin: bool, + ns_pid_file: Option<&str>, ) -> Result< layerfs_sandbox::backend::docker::ExecCreated, Box, > { - crate::sandbox::exec::launch(sandbox, identity, arguments, environment, directory, stdin) + crate::sandbox::exec::launch( + sandbox, + identity, + arguments, + environment, + directory, + stdin, + ns_pid_file, + ) } diff --git a/engine/crates/layerfs-api/src/sandbox/exec.rs b/engine/crates/layerfs-api/src/sandbox/exec.rs index c0db8c8e5b..7a7ac39fc1 100644 --- a/engine/crates/layerfs-api/src/sandbox/exec.rs +++ b/engine/crates/layerfs-api/src/sandbox/exec.rs @@ -5,6 +5,9 @@ use layerfs_sandbox::{ CommandIdentity, }; +/// The daemon's entry inside its Sandbox container, as the deployment installs it. +const DAEMON_ENTRY: &str = "/usr/local/bin/layerfs-daemon"; + impl WorkspaceApi<'_> { /// Creates one ordinary command at this mount using the deployment's admitted identity. /// The returned Sandbox runtime owner owns Start, streams, exit status and cancellation. @@ -23,9 +26,15 @@ impl WorkspaceApi<'_> { environment, workspace.ready.directory.clone(), stdin, + None, ) } } +/// Creates the Engine Exec for one command. `ns_pid_file` is the pid file of a +/// daemon that serves from a private mount namespace: the command then runs +/// through that daemon's `ns-exec` helper, because the Workspace mounts are +/// absent from the namespace the Engine enters. `None` keeps the command as +/// it is. pub(crate) fn launch( sandbox: &Sandbox, identity: CommandIdentity, @@ -33,12 +42,90 @@ pub(crate) fn launch( environment: Vec, directory: String, stdin: bool, + ns_pid_file: Option<&str>, ) -> Result> { sandbox.create_exec(ExecRequest { identity, - arguments, + arguments: launch_arguments(ns_pid_file, identity, &directory, arguments), environment, directory, stdin, + root_launcher: ns_pid_file.is_some(), }) } +/// The Engine argv for one command. Without a private-namespace pid file it is +/// exactly the caller's argv. With one it names the daemon's helper, the pid +/// file, the mount directory the Engine starts in (only the directory under the +/// mount exists in its namespace) and the ordinary identity the helper drops +/// to, then the same argv after `--`: the helper execs it, so the Engine still +/// has one process tree, the same streams and the same exit code. +fn launch_arguments( + ns_pid_file: Option<&str>, + identity: CommandIdentity, + directory: &str, + arguments: Vec, +) -> Vec { + let Some(pid_file) = ns_pid_file else { + return arguments; + }; + let mut wrapped = vec![ + DAEMON_ENTRY.to_owned(), + "ns-exec".to_owned(), + "--pid-file".to_owned(), + pid_file.to_owned(), + "--directory".to_owned(), + directory.to_owned(), + "--uid".to_owned(), + identity.uid.to_string(), + "--gid".to_owned(), + identity.gid.to_string(), + "--".to_owned(), + ]; + wrapped.extend(arguments); + wrapped +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_sandbox_without_a_private_namespace_keeps_the_command_as_it_is() { + let identity = CommandIdentity { uid: 501, gid: 20 }; + let arguments = vec!["bash".into(), "-c".into(), "printf ordinary".into()]; + assert_eq!( + launch_arguments(None, identity, "/workspaces/1", arguments.clone()), + arguments + ); + } + + #[test] + fn a_private_namespace_sandbox_runs_the_daemon_helper() { + let identity = CommandIdentity { uid: 501, gid: 20 }; + let arguments = vec!["bash".into(), "-c".into(), "printf ordinary".into()]; + assert_eq!( + launch_arguments( + Some("/layerfs-local/daemon.pid"), + identity, + "/workspaces/1", + arguments, + ), + [ + "/usr/local/bin/layerfs-daemon", + "ns-exec", + "--pid-file", + "/layerfs-local/daemon.pid", + "--directory", + "/workspaces/1", + "--uid", + "501", + "--gid", + "20", + "--", + "bash", + "-c", + "printf ordinary", + ] + ); + } +} diff --git a/engine/crates/layerfs-api/tests/kernel_application.rs b/engine/crates/layerfs-api/tests/kernel_application.rs index 614e0d80c3..992a60410e 100644 --- a/engine/crates/layerfs-api/tests/kernel_application.rs +++ b/engine/crates/layerfs-api/tests/kernel_application.rs @@ -61,6 +61,7 @@ fn actual_daemon_install_hello_bind_status_no_constructor_and_session_end() { idle_exit_s: 0, }, existing_store: None, + private_mount_ns: false, }; let config = f.directory.join("daemon.config"); fs::write(&config, setup.encode().unwrap()).unwrap(); diff --git a/engine/crates/layerfs-api/tests/observed_application.rs b/engine/crates/layerfs-api/tests/observed_application.rs index 5fe9ff34a9..f9232d7f7b 100644 --- a/engine/crates/layerfs-api/tests/observed_application.rs +++ b/engine/crates/layerfs-api/tests/observed_application.rs @@ -85,6 +85,7 @@ fn configure(f: &support::Fixture, deployment: &Deployment) -> PathBuf { idle_exit_s: deployment.idle_exit_s, }, existing_store: deployment.existing_store.clone(), + private_mount_ns: false, }; let config = f.directory.join("daemon.config"); fs::write(&config, setup.encode().unwrap()).unwrap(); diff --git a/engine/crates/layerfs-api/tests/r5_1_actual_daemon_commit.rs b/engine/crates/layerfs-api/tests/r5_1_actual_daemon_commit.rs index 22108474eb..5158c049ec 100644 --- a/engine/crates/layerfs-api/tests/r5_1_actual_daemon_commit.rs +++ b/engine/crates/layerfs-api/tests/r5_1_actual_daemon_commit.rs @@ -123,6 +123,7 @@ fn start(f: &support::Fixture, mounts: &Path) -> (Process, SocketAddr) { idle_exit_s: 0, }, existing_store: None, + private_mount_ns: false, }; let config = f.directory.join("daemon.config"); fs::write(&config, setup.encode().unwrap()).unwrap(); diff --git a/engine/crates/layerfs-api/tests/sandbox_operations.rs b/engine/crates/layerfs-api/tests/sandbox_operations.rs index 64859816e2..7b04db1ebf 100644 --- a/engine/crates/layerfs-api/tests/sandbox_operations.rs +++ b/engine/crates/layerfs-api/tests/sandbox_operations.rs @@ -47,6 +47,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { idle_exit_s: 0, }, existing_store: None, + private_mount_ns: false, } } diff --git a/engine/crates/layerfs-bridge/src/daemon_setup.rs b/engine/crates/layerfs-bridge/src/daemon_setup.rs index 9ae2ee2406..2b838b6556 100644 --- a/engine/crates/layerfs-bridge/src/daemon_setup.rs +++ b/engine/crates/layerfs-bridge/src/daemon_setup.rs @@ -5,7 +5,8 @@ use crate::{ wire::{Reader, Writer}, }; use std::fmt; -const MAGIC: &[u8] = b"LFSD\x05"; +use std::path::{Path, PathBuf}; +const MAGIC: &[u8] = b"LFSD\x06"; /// Explicit daemon startup resources; these are windows/admission, not file caps. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct DaemonLimits { @@ -51,6 +52,11 @@ pub struct DaemonSetup { /// None selects the original one-time installer. Some selects explicit existing /// Store open using retained authority; never selected after an install error. pub existing_store: Option, + /// Start in a mount namespace of this process's own (prototype for #3/#499) + /// and write the pid file [`DaemonSetup::mount_ns_pid_file`] names, so a + /// command reaches the Workspace mounts only through the daemon's + /// `ns-exec` helper. Off by default; the flag is opt-in. + pub private_mount_ns: bool, } impl DaemonSetup { /// Encodes one bounded private config. No operation/flow byte limit is implied. @@ -77,6 +83,7 @@ impl DaemonSetup { if let Some(value) = &self.existing_store { out.blob(&value.encode()?)?; } + out.byte(u8::from(self.private_mount_ns))?; Ok(out.0) } /// Decodes exactly one config without fallback or profile selection. @@ -104,6 +111,11 @@ impl DaemonSetup { 1 => Some(StoreManifest::decode(input.blob(crate::wire::LIMIT)?)?), _ => return Err(ControlError("daemon config boolean")), }; + let private_mount_ns = match input.byte()? { + 0 => false, + 1 => true, + _ => return Err(ControlError("daemon config boolean")), + }; input.finish()?; let value = Self { listen, @@ -116,6 +128,7 @@ impl DaemonSetup { command_gid, limits, existing_store, + private_mount_ns, }; value.check()?; Ok(value) @@ -154,8 +167,22 @@ impl DaemonSetup { return Err(ControlError("explicit existing Disposable Store")); } } + if self.private_mount_ns && self.mount_ns_pid_file().is_none() { + return Err(ControlError("private mount namespace pid file path")); + } Ok(()) } + /// The pid file this daemon writes when [`Self::private_mount_ns`] is on: + /// `daemon.pid` in the local directory above the Overlay's own directory + /// (in a Sandbox that is `/layerfs-local`). `None` when the flag is off or + /// the Overlay path has no such local directory. + pub fn mount_ns_pid_file(&self) -> Option { + if !self.private_mount_ns { + return None; + } + let home = Path::new(&self.overlay).parent()?.parent()?; + Some(home.join("daemon.pid")) + } } impl fmt::Debug for DaemonSetup { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -170,6 +197,62 @@ impl fmt::Debug for DaemonSetup { .field("command_gid", &self.command_gid) .field("limits", &self.limits) .field("existing_store", &self.existing_store) + .field("private_mount_ns", &self.private_mount_ns) .finish() } } + +#[cfg(test)] +mod tests { + use super::*; + + fn setup(overlay: &str, private_mount_ns: bool) -> DaemonSetup { + DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: [1; 32], + control_peer: [2; 32], + store: "/layerfs-store/global/store.sqlite".into(), + overlay: overlay.into(), + mounts: "/workspaces".into(), + command_uid: 501, + command_gid: 20, + limits: DaemonLimits { + connections: 4, + handshake_ms: 3000, + cache_bytes: 64 * 1024, + namespaces: 8, + pager_kib: 1024, + serial_low_water: 0, + idle_exit_s: 0, + }, + existing_store: None, + private_mount_ns, + } + } + + #[test] + fn the_private_namespace_setting_survives_the_config_record() { + assert!(setup("/local/overlay/overlay.sqlite", false) + .mount_ns_pid_file() + .is_none()); + let encoded = setup("/local/overlay/overlay.sqlite", true) + .encode() + .unwrap(); + let decoded = DaemonSetup::decode(&encoded).unwrap(); + assert!(decoded.private_mount_ns); + assert_eq!( + decoded.mount_ns_pid_file(), + Some(PathBuf::from("/local/daemon.pid")) + ); + } + + #[test] + fn the_pid_file_is_beside_the_local_directory_above_the_overlay() { + assert_eq!( + setup("/layerfs-local/overlay/overlay.sqlite", true).mount_ns_pid_file(), + Some(PathBuf::from("/layerfs-local/daemon.pid")) + ); + // An Overlay path with no local directory cannot name a pid file. + assert!(setup("/overlay.sqlite", true).encode().is_err()); + } +} diff --git a/engine/crates/layerfs-daemon/Cargo.toml b/engine/crates/layerfs-daemon/Cargo.toml index ea15738653..2fb5fc162c 100644 --- a/engine/crates/layerfs-daemon/Cargo.toml +++ b/engine/crates/layerfs-daemon/Cargo.toml @@ -19,7 +19,7 @@ layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } # Concrete Store opening stays in application bootstrap, outside store/. layerfs-persistence = { path = "../layerfs-persistence" } -nix = { version = "=0.31.3", features = ["fs", "user", "process"] } +nix = { version = "=0.31.3", features = ["fs", "user", "process", "sched"] } [dev-dependencies] blake3 = { version = "=1.8.5", default-features = false } diff --git a/engine/crates/layerfs-daemon/src/application/cli.rs b/engine/crates/layerfs-daemon/src/application/cli.rs index e9d0285f1c..7863c5e589 100644 --- a/engine/crates/layerfs-daemon/src/application/cli.rs +++ b/engine/crates/layerfs-daemon/src/application/cli.rs @@ -1,6 +1,10 @@ -//! Thin actual deployment CLI; no launcher or shell command mode. +//! Thin actual deployment CLI; no launcher or shell command mode beyond the +//! hidden `ns-exec` helper that joins this daemon's mount namespace. use super::{config, Application, ApplicationError}; +#[cfg(target_os = "linux")] +use std::fs; use std::{ + ffi::{OsStr, OsString}, io::{self, Write}, net::TcpListener, path::Path, @@ -9,8 +13,19 @@ use std::{ /// Runs the Linux filesystem application from one protected configuration file. /// Args/env carry no key and no command; ordinary runtime execution is separate. +/// The one exception is `ns-exec`, a helper the Sandbox runtime starts through +/// the Engine so a command can run in this daemon's private mount namespace. pub fn main() -> ExitCode { - match run() { + let mut args = std::env::args_os(); + let _program = args.next(); + let args: Vec = args.collect(); + if args + .first() + .is_some_and(|flag| flag == OsStr::new("ns-exec")) + { + return ns_exec(&args[1..]); + } + match run(&args) { Ok(()) => ExitCode::SUCCESS, Err(cause) => { eprintln!("{cause}"); @@ -18,25 +33,50 @@ pub fn main() -> ExitCode { } } } -fn run() -> Result<(), ApplicationError> { - let mut args = std::env::args_os(); - let _program = args.next(); - if args.next().as_deref() != Some(std::ffi::OsStr::new("--config")) { + +#[cfg(target_os = "linux")] +fn ns_exec(args: &[OsString]) -> ExitCode { + super::ns_exec::main(args) +} + +#[cfg(not(target_os = "linux"))] +fn ns_exec(_args: &[OsString]) -> ExitCode { + eprintln!("layerfs-daemon ns-exec: requires Linux"); + ExitCode::FAILURE +} + +fn run(args: &[OsString]) -> Result<(), ApplicationError> { + if args.first().map(OsString::as_os_str) != Some(OsStr::new("--config")) { return Err(ApplicationError::Io(io::Error::new( io::ErrorKind::InvalidInput, "usage: layerfs-daemon --config ", ))); } - let path = args.next().ok_or_else(|| { + let path = args.get(1).ok_or_else(|| { ApplicationError::Io(io::Error::new(io::ErrorKind::InvalidInput, "config path")) })?; - if args.next().is_some() { + if args.len() != 2 { return Err(ApplicationError::Io(io::Error::new( io::ErrorKind::InvalidInput, "unexpected argument", ))); } - let setup = config::read(Path::new(&path)).map_err(ApplicationError::Io)?; + let setup = config::read(Path::new(path)).map_err(ApplicationError::Io)?; + // Before the listener and any worker thread: the daemon takes its own + // mount namespace so a Workspace FUSE mount is absent from the namespace + // an Engine exec or a `docker exec` enters (prototype for #3/#499). A + // failure refuses startup rather than serving with the old topology. + #[cfg(target_os = "linux")] + if setup.private_mount_ns { + layerfs_fuse::mount::private_namespace().map_err(ApplicationError::Io)?; + let pid_file = setup.mount_ns_pid_file().ok_or_else(|| { + ApplicationError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + "private mount namespace without a pid file path", + )) + })?; + write_pid_file(&pid_file).map_err(ApplicationError::Io)?; + } let listener = TcpListener::bind(&setup.listen).map_err(ApplicationError::Io)?; let application = Application::start(setup)?; println!( @@ -47,3 +87,17 @@ fn run() -> Result<(), ApplicationError> { application.watch_idle().map_err(ApplicationError::Io)?; application.serve(listener) } + +/// One atomic publish of this process's pid: a temporary file beside the +/// named one, then a rename. The directory exists because the protected +/// configuration already created the Overlay's own directory there. +#[cfg(target_os = "linux")] +fn write_pid_file(path: &Path) -> io::Result<()> { + let temporary = path.with_extension(format!("tmp{}", std::process::id())); + fs::write(&temporary, format!("{}\n", std::process::id()))?; + if let Err(error) = fs::rename(&temporary, path) { + let _ = fs::remove_file(&temporary); + return Err(error); + } + Ok(()) +} diff --git a/engine/crates/layerfs-daemon/src/application/mod.rs b/engine/crates/layerfs-daemon/src/application/mod.rs index e806120c7a..3bec6ee18f 100644 --- a/engine/crates/layerfs-daemon/src/application/mod.rs +++ b/engine/crates/layerfs-daemon/src/application/mod.rs @@ -7,6 +7,8 @@ mod diagnostics; mod failure; mod filesystem; mod idle; +#[cfg(target_os = "linux")] +mod ns_exec; mod owner; mod serve; pub use cli::main; diff --git a/engine/crates/layerfs-daemon/src/application/ns_exec.rs b/engine/crates/layerfs-daemon/src/application/ns_exec.rs new file mode 100644 index 0000000000..618f9f628a --- /dev/null +++ b/engine/crates/layerfs-daemon/src/application/ns_exec.rs @@ -0,0 +1,208 @@ +//! The hidden `ns-exec` entry: join the daemon's mount namespace, then run one +//! command in it (prototype for #3/#499). +//! +//! `workspace.exec` is an Engine exec in the container's initial mount +//! namespace. When the daemon serves from a private namespace, the Workspace +//! mounts are in that namespace only, so the Engine's entry command is this +//! helper: it reads the daemon's pid from the pid file the daemon wrote, joins +//! `/proc//ns/mnt`, drops the root identity the Engine starts it with back +//! to the ordinary command identity, and execs the command it was given. The +//! Engine still sees one process and the same streams and exit code. Nothing +//! is run when any step fails: a helper that cannot join must not silently run +//! the command in the wrong namespace. +use nix::{ + sched::{setns, CloneFlags}, + unistd::{setgid, setgroups, setuid, Gid, Uid}, +}; +use std::{ + ffi::{OsStr, OsString}, + fs, + os::unix::process::CommandExt, + process::Command, +}; + +/// Exit code of a helper that refused before running the command. +const REFUSED: u8 = 125; +const USAGE: &str = "usage: layerfs-daemon ns-exec --pid-file [--directory ] [--uid --gid ] -- [args...]"; + +pub(super) fn main(args: &[OsString]) -> std::process::ExitCode { + match run(args) { + // `run` only returns when it refused; a successful one execs. + Ok(()) => std::process::ExitCode::SUCCESS, + Err(message) => { + eprintln!("layerfs-daemon ns-exec: {message}"); + std::process::ExitCode::from(REFUSED) + } + } +} + +fn run(args: &[OsString]) -> Result<(), String> { + let (pid_file, directory, identity, command) = parse(args)?; + let pid = read_pid(&pid_file)?; + join_mount_namespace(pid)?; + // The Engine starts this process in the container's initial namespace, + // where the mount directory is only the empty directory the daemon + // mounted over. Enter the joined namespace's view of it before running. + if let Some(directory) = directory { + std::env::set_current_dir(&directory) + .map_err(|error| format!("enter directory {directory}: {error}"))?; + } + if let Some((uid, gid)) = identity { + drop_to(uid, gid)?; + } + let (program, rest) = command.split_first().ok_or_else(|| USAGE.to_owned())?; + let error = Command::new(program).args(rest).exec(); + Err(format!("exec {}: {error}", program.to_string_lossy())) +} + +/// `--pid-file [--directory ] [--uid --gid ] -- [args...]`. +fn parse( + args: &[OsString], +) -> Result<(String, Option, Option<(u32, u32)>, Vec), String> { + let mut rest = args.iter(); + let pid_file = match rest.next() { + Some(flag) if flag == OsStr::new("--pid-file") => path(rest.next())?, + _ => return Err(USAGE.to_owned()), + }; + let mut directory = None; + let mut uid = None; + let mut gid = None; + loop { + match rest.next() { + Some(flag) if flag == OsStr::new("--directory") && directory.is_none() => { + directory = Some(path(rest.next())?) + } + Some(flag) if flag == OsStr::new("--uid") && uid.is_none() => { + uid = Some(number(rest.next(), "uid")?) + } + Some(flag) if flag == OsStr::new("--gid") && gid.is_none() => { + gid = Some(number(rest.next(), "gid")?) + } + Some(flag) if flag == OsStr::new("--") => break, + _ => return Err(USAGE.to_owned()), + } + } + let command: Vec = rest.cloned().collect(); + if command.is_empty() { + return Err(USAGE.to_owned()); + } + match (uid, gid) { + (Some(0), _) | (_, Some(0)) => { + Err("--uid and --gid must name the non-root command identity".to_owned()) + } + (Some(uid), Some(gid)) => Ok((pid_file, directory, Some((uid, gid)), command)), + (None, None) => Ok((pid_file, directory, None, command)), + _ => Err("--uid and --gid must be given together".to_owned()), + } +} + +fn path(value: Option<&OsString>) -> Result { + value + .ok_or_else(|| USAGE.to_owned())? + .to_str() + .map(str::to_owned) + .ok_or_else(|| "a path is not UTF-8".to_owned()) +} + +fn number(value: Option<&OsString>, name: &str) -> Result { + value + .and_then(|value| value.to_str()) + .and_then(|value| value.parse().ok()) + .ok_or_else(|| format!("--{name} is not a number")) +} + +/// The daemon's pid from its pid file, or a clear refusal. +fn read_pid(pid_file: &str) -> Result { + let pid = fs::read_to_string(pid_file) + .map_err(|error| format!("read daemon pid file {pid_file}: {error}"))?; + match pid.trim().parse::() { + Ok(pid) if pid != 0 => Ok(pid), + _ => Err(format!("daemon pid file {pid_file} does not name a pid")), + } +} + +fn join_mount_namespace(pid: u32) -> Result<(), String> { + let namespace = format!("/proc/{pid}/ns/mnt"); + let file = fs::File::open(&namespace) + .map_err(|error| format!("open mount namespace {namespace}: {error}"))?; + setns(&file, CloneFlags::CLONE_NEWNS) + .map_err(|error| format!("join mount namespace {namespace}: {error}")) +} + +/// The Engine starts this helper as root, because only root holds the +/// capability the `setns` needs; the command must still run as the Sandbox's +/// ordinary command identity, with no supplementary groups and no capability. +fn drop_to(uid: u32, gid: u32) -> Result<(), String> { + setgroups(&[]).map_err(|error| format!("drop supplementary groups: {error}"))?; + setgid(Gid::from_raw(gid)).map_err(|error| format!("drop to gid {gid}: {error}"))?; + setuid(Uid::from_raw(uid)).map_err(|error| format!("drop to uid {uid}: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_command_identity_cannot_be_root() { + let args: Vec = ["--pid-file", "/p", "--uid", "0", "--gid", "0", "--", "true"] + .into_iter() + .map(OsString::from) + .collect(); + assert!(parse(&args).is_err()); + } + + #[test] + fn a_command_needs_a_separator_and_a_program() { + assert!(parse(&[OsString::from("--pid-file"), "/p".into()]).is_err()); + assert!(parse(&[OsString::from("--pid-file"), "/p".into(), "--".into()]).is_err()); + } + + #[test] + fn the_identity_is_all_or_nothing() { + let args = [ + "--pid-file".into(), + "/p".into(), + "--uid".into(), + "501".into(), + "--".into(), + "true".into(), + ]; + assert!(parse(&args).is_err()); + } + + #[test] + fn a_parsed_command_keeps_its_arguments() { + let args: Vec = ["--pid-file", "/p", "--uid", "501", "--gid", "20", "--"] + .into_iter() + .map(OsString::from) + .chain(["bash".into(), "-c".into(), "printf hello".into()]) + .collect(); + let (pid_file, directory, identity, command) = parse(&args).unwrap(); + assert_eq!(pid_file, "/p"); + assert_eq!(directory, None); + assert_eq!(identity, Some((501, 20))); + assert_eq!(command, ["bash", "-c", "printf hello"]); + } + + #[test] + fn a_missing_or_unreadable_pid_file_is_a_refusal() { + let directory = + std::env::temp_dir().join(format!("layerfs-ns-exec-{}", std::process::id())); + let _ = fs::remove_dir_all(&directory); + fs::create_dir(&directory).unwrap(); + let missing = directory.join("missing.pid"); + assert!(read_pid(missing.to_str().unwrap()).is_err()); + fs::write(&missing, b"not a pid\n").unwrap(); + assert!(read_pid(missing.to_str().unwrap()).is_err()); + fs::write(&missing, b"0\n").unwrap(); + assert!(read_pid(missing.to_str().unwrap()).is_err()); + fs::write(&missing, b"4321\n").unwrap(); + assert_eq!(read_pid(missing.to_str().unwrap()).unwrap(), 4321); + fs::remove_dir_all(&directory).unwrap(); + } + + #[test] + fn a_missing_namespace_is_a_refusal() { + assert!(join_mount_namespace(u32::MAX - 1).is_err()); + } +} diff --git a/engine/crates/layerfs-daemon/tests/ns_exec.rs b/engine/crates/layerfs-daemon/tests/ns_exec.rs new file mode 100644 index 0000000000..7dc68459f7 --- /dev/null +++ b/engine/crates/layerfs-daemon/tests/ns_exec.rs @@ -0,0 +1,378 @@ +//! The daemon's hidden `ns-exec` helper (prototype for #3/#499). +//! +//! A command started by the Engine is in the container's initial mount +//! namespace; when the daemon serves from a private one, the Workspace mounts +//! are absent there. The helper joins the namespace of the daemon whose pid the +//! pid file names, drops to the ordinary command identity, and execs the +//! command. These tests build that private namespace with `unshare -m`: a +//! tmpfs is mounted in it only, and a marker written there is invisible to the +//! test's own namespace, so a command that finds it proves it joined. +//! +//! Needs root with CAP_SYS_ADMIN (a Sandbox container has both) and +//! `unshare`/`mount`. A missing or invalid pid file must refuse, never run the +//! command in the wrong namespace. +#![cfg(target_os = "linux")] +use layerfs_bridge::daemon_setup::{DaemonLimits, DaemonSetup}; +use std::{ + fs, + io::{BufRead, BufReader, Write}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Child, ChildStdin, Command, Stdio}, + sync::mpsc::{self, Receiver}, + thread, + time::{Duration, Instant}, +}; + +/// Every bounded observation of a spawned process. +const WAIT: Duration = Duration::from_secs(10); +/// The marker only the private namespace's tmpfs holds. +const MARKER: &str = "only-in-the-private-namespace\n"; +/// The bin under test, built with this test. +const BINARY: &str = env!("CARGO_BIN_EXE_layerfs-daemon"); + +/// One process in a private copy of the mount namespace, with a tmpfs mounted +/// over the directory it was given. +struct Private { + child: Child, + stdin: ChildStdin, + lines: Receiver, + directory: PathBuf, + private: PathBuf, +} + +impl Private { + fn spawn() -> Self { + static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let label = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let directory = + std::env::temp_dir().join(format!("layerfs-ns-exec-{}-{label}", std::process::id())); + let _ = fs::remove_dir_all(&directory); + fs::create_dir(&directory).unwrap(); + let private = directory.join("private"); + fs::create_dir(&private).unwrap(); + // `unshare` keeps its pid: it execs the shell after the unshare. + let mut child = Command::new("unshare") + .args([ + "-m", + "--propagation", + "private", + "sh", + "-c", + PRIVATE, + "ns-exec-test", + ]) + .arg(&private) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .expect("unshare -m needs root with CAP_SYS_ADMIN"); + let stdout = child.stdout.take().unwrap(); + let stdin = child.stdin.take().unwrap(); + let (send, lines) = mpsc::channel(); + thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if send.send(line.unwrap_or_default()).is_err() { + break; + } + } + }); + let mut private = Self { + child, + stdin, + lines, + directory, + private, + }; + let pid = private.line("the private namespace reported its pid"); + assert_eq!( + pid.parse::().ok(), + Some(private.child.id()), + "the unshared process keeps its pid" + ); + private + } + fn line(&mut self, what: &str) -> String { + self.lines + .recv_timeout(WAIT) + .unwrap_or_else(|_| panic!("{what}")) + } + fn pid_file(&self) -> PathBuf { + self.directory.join("daemon.pid") + } + fn write_pid(&self) -> PathBuf { + let path = self.pid_file(); + fs::write(&path, format!("{}\n", self.child.id())).unwrap(); + path + } + fn stop(&mut self) { + let _ = writeln!(self.stdin, "stop"); + let deadline = Instant::now() + WAIT; + while Instant::now() < deadline { + if self.child.try_wait().unwrap().is_some() { + return; + } + thread::sleep(Duration::from_millis(10)); + } + } +} + +impl Drop for Private { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = fs::remove_dir_all(&self.directory); + } +} + +/// The shell in the new namespace: mount a tmpfs only there, write the marker, +/// report its pid, then hold until told to stop. +const PRIVATE: &str = r#"set -e +mount -t tmpfs tmpfs "$1" +printf 'only-in-the-private-namespace\n' > "$1/marker" +printf '%s\n' $$ +while read line; do [ "$line" = stop ] && exit 0; done"#; + +fn run(pid_file: &Path, command: &[&str], identity: Option<(u32, u32)>) -> (i32, String, String) { + run_in(pid_file, None, command, identity) +} + +fn run_in( + pid_file: &Path, + directory: Option<&Path>, + command: &[&str], + identity: Option<(u32, u32)>, +) -> (i32, String, String) { + let mut helper = Command::new(BINARY); + helper.args(["ns-exec", "--pid-file"]).arg(pid_file); + if let Some(directory) = directory { + helper.arg("--directory").arg(directory); + } + if let Some((uid, gid)) = identity { + helper + .arg("--uid") + .arg(uid.to_string()) + .arg("--gid") + .arg(gid.to_string()); + } + let output = helper + .arg("--") + .args(command) + .output() + .expect("the daemon binary runs"); + ( + output.status.code().unwrap_or(-1), + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) +} + +#[test] +fn a_command_joins_the_daemons_private_mount_namespace() { + let mut private = Private::spawn(); + let marker = private.private.join("marker"); + // The proof is meaningful only while the marker is absent here. + assert!(!marker.exists(), "the tmpfs is absent from this namespace"); + let pid_file = private.write_pid(); + let (code, out, err) = run( + &pid_file, + &["sh", "-c", &format!("cat {}", marker.display())], + None, + ); + assert_eq!(code, 0, "helper stderr: {err}"); + assert_eq!(out, MARKER, "the command read the private namespace's file"); + private.stop(); +} + +#[test] +fn the_command_starts_in_the_joined_namespaces_view_of_the_directory() { + let mut private = Private::spawn(); + let pid_file = private.write_pid(); + let (code, out, err) = run_in( + &pid_file, + Some(&private.private), + &["sh", "-c", "cat marker"], + None, + ); + assert_eq!(code, 0, "helper stderr: {err}"); + assert_eq!(out, MARKER); + private.stop(); +} + +#[test] +fn the_command_runs_as_the_identity_the_helper_drops_to() { + let mut private = Private::spawn(); + let pid_file = private.write_pid(); + let (code, out, err) = run( + &pid_file, + &["sh", "-c", "id -u; id -g"], + Some((65534, 65534)), + ); + assert_eq!(code, 0, "helper stderr: {err}"); + assert_eq!(out, "65534\n65534\n"); + private.stop(); +} + +#[test] +fn a_missing_pid_file_refuses_and_runs_nothing() { + let private = Private::spawn(); + let side_effect = private.directory.join("ran-anyway"); + let (code, _, err) = run( + &private.pid_file(), + &["sh", "-c", &format!("touch {}", side_effect.display())], + None, + ); + assert_eq!(code, 125); + assert!(err.contains("daemon pid file"), "stderr: {err}"); + assert!(!side_effect.exists(), "no command ran"); +} + +#[test] +fn a_pid_file_that_is_not_a_pid_refuses_and_runs_nothing() { + let private = Private::spawn(); + let pid_file = private.pid_file(); + fs::write(&pid_file, b"not a pid\n").unwrap(); + let side_effect = private.directory.join("ran-anyway"); + let (code, _, err) = run( + &pid_file, + &["sh", "-c", &format!("touch {}", side_effect.display())], + None, + ); + assert_eq!(code, 125); + assert!(err.contains("does not name a pid"), "stderr: {err}"); + assert!(!side_effect.exists(), "no command ran"); +} + +#[test] +fn a_pid_without_a_mount_namespace_refuses_and_runs_nothing() { + let private = Private::spawn(); + let pid_file = private.pid_file(); + fs::write(&pid_file, b"4294967294\n").unwrap(); + let side_effect = private.directory.join("ran-anyway"); + let (code, _, err) = run( + &pid_file, + &["sh", "-c", &format!("touch {}", side_effect.display())], + None, + ); + assert_eq!(code, 125); + assert!(err.contains("mount namespace"), "stderr: {err}"); + assert!(!side_effect.exists(), "no command ran"); +} + +/// The private configuration record of one daemon start, beside its own paths. +fn daemon_config(directory: &Path, private_mount_ns: bool) -> PathBuf { + for path in [directory.join("local"), directory.join("store")] { + fs::create_dir(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + } + fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).unwrap(); + let setup = DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: [1; 32], + control_peer: [2; 32], + store: directory + .join("store/sealed.sqlite") + .to_str() + .unwrap() + .into(), + overlay: directory + .join("local/overlay/overlay.sqlite") + .to_str() + .unwrap() + .into(), + mounts: directory.join("workspaces").to_str().unwrap().into(), + command_uid: 501, + command_gid: 20, + limits: DaemonLimits { + connections: 4, + handshake_ms: 3000, + cache_bytes: 64 * 1024, + namespaces: 2, + pager_kib: 1024, + serial_low_water: 0, + idle_exit_s: 0, + }, + existing_store: None, + private_mount_ns, + }; + let path = directory.join("daemon.setup"); + fs::write(&path, setup.encode().unwrap()).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + path +} + +fn unique_directory(label: &str) -> PathBuf { + static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let next = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let directory = std::env::temp_dir().join(format!( + "layerfs-ns-daemon-{}-{label}-{next}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + fs::create_dir(&directory).unwrap(); + directory +} + +/// The daemon writes the pid file only after it has taken its private mount +/// namespace, and taking it refuses startup on failure: a written pid file +/// therefore proves the unshare ran. Whether the daemon then serves or fails +/// (no `/dev/fuse` in this container) does not matter; the file stays. +#[test] +fn the_daemon_writes_its_pid_file_only_when_asked_for_a_private_namespace() { + let directory = unique_directory("on"); + let config = daemon_config(&directory, true); + let mut daemon = Command::new(BINARY) + .args(["--config"]) + .arg(&config) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid_file = directory.join("local/daemon.pid"); + let deadline = Instant::now() + WAIT; + while !pid_file.exists() { + assert!( + daemon.try_wait().unwrap().is_none() || pid_file.exists(), + "the daemon ended before writing its pid file" + ); + assert!(Instant::now() < deadline, "no pid file within {WAIT:?}"); + thread::sleep(Duration::from_millis(10)); + } + let pid: u32 = fs::read_to_string(&pid_file) + .unwrap() + .trim() + .parse() + .unwrap(); + assert_eq!(pid, daemon.id(), "the pid file names the daemon process"); + let _ = daemon.kill(); + let _ = daemon.wait(); + fs::remove_dir_all(&directory).unwrap(); + + let directory = unique_directory("off"); + let config = daemon_config(&directory, false); + let mut daemon = Command::new(BINARY) + .args(["--config"]) + .arg(&config) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid_file = directory.join("local/daemon.pid"); + let deadline = Instant::now() + WAIT; + loop { + assert!( + !pid_file.exists(), + "the flag is off; no pid file is written" + ); + if daemon.try_wait().unwrap().is_some() || Instant::now() >= deadline { + break; + } + thread::sleep(Duration::from_millis(10)); + } + let _ = daemon.kill(); + let _ = daemon.wait(); + fs::remove_dir_all(&directory).unwrap(); +} diff --git a/engine/crates/layerfs-fuse/Cargo.toml b/engine/crates/layerfs-fuse/Cargo.toml index 01eaa178ea..02a672f359 100644 --- a/engine/crates/layerfs-fuse/Cargo.toml +++ b/engine/crates/layerfs-fuse/Cargo.toml @@ -14,4 +14,4 @@ layerfs-content = { path = "../../../core/crates/layerfs-content" } [target.'cfg(target_os = "linux")'.dependencies] fuser = { version = "=0.18.0", default-features = false } libc = "=0.2.189" -nix = { version = "=0.31.3", features = ["fs", "mount", "feature"] } +nix = { version = "=0.31.3", features = ["fs", "mount", "feature", "sched"] } diff --git a/engine/crates/layerfs-fuse/src/mount/mod.rs b/engine/crates/layerfs-fuse/src/mount/mod.rs index 278d5031e6..36932d6d78 100644 --- a/engine/crates/layerfs-fuse/src/mount/mod.rs +++ b/engine/crates/layerfs-fuse/src/mount/mod.rs @@ -4,4 +4,5 @@ pub(crate) mod syscalls; pub(crate) use profile::negotiate; pub use profile::Negotiation; -pub use syscalls::{control_root, AbortWrite, MountEntry, CONTROL}; +// `private_namespace` is for the daemon entry only (prototype for #3/#499). +pub use syscalls::{control_root, private_namespace, AbortWrite, MountEntry, CONTROL}; diff --git a/engine/crates/layerfs-fuse/src/mount/syscalls.rs b/engine/crates/layerfs-fuse/src/mount/syscalls.rs index 0d16e8517e..74ccc4bddf 100644 --- a/engine/crates/layerfs-fuse/src/mount/syscalls.rs +++ b/engine/crates/layerfs-fuse/src/mount/syscalls.rs @@ -4,6 +4,7 @@ use nix::{ errno::Errno, mount::{mount, umount2, MntFlags, MsFlags}, + sched::{unshare, CloneFlags}, unistd::write, }; use std::{ @@ -77,6 +78,26 @@ pub(crate) fn attach(device: &OwnedFd, target: &Path, uid: u32, gid: u32) -> Res Some(data.as_str()), ) } +/// One `unshare` of the mount namespace and one recursive private propagation +/// of its root, so every FUSE mount this process makes is absent from the +/// namespace the process started in and nothing made here propagates back into +/// it. Called once, by the daemon entry, before any worker thread exists; the +/// daemon owns the namespace for its whole life and restores nothing on exit. +/// Prototype for #3/#499, gated by the daemon's private-namespace setting: the +/// Sandbox's original namespace (what a `docker exec` enters) no longer holds +/// a Workspace mount, so `workspace.exec` reaches it through the daemon's +/// `ns-exec` helper instead. +pub fn private_namespace() -> io::Result<()> { + unshare(CloneFlags::CLONE_NEWNS)?; + mount( + None::<&str>, + "/", + None::<&str>, + MsFlags::MS_REC | MsFlags::MS_PRIVATE, + None::<&str>, + )?; + Ok(()) +} /// Exactly one plain attempt. EBUSY is the kernel's reversible answer; no /// MNT_DETACH, MNT_FORCE or second call follows any outcome. pub(crate) fn detach(target: &Path) -> Result<(), Errno> { diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index c0e058d87c..e839e712e7 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -371,6 +371,10 @@ pub(crate) fn save_sandbox( ("secret", SdkValue::Bytes(secret.to_vec())), ("peer", SdkValue::Bytes(peer.to_vec())), ("instance", SdkValue::Bytes(instance.to_vec())), + ( + "nsExecPidFile", + SdkValue::opt_text(state.ns_pid_file.clone()), + ), ]), ) } @@ -471,6 +475,8 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { directory, runtime, identity: CommandIdentity { uid: 501, gid: 20 }, + // absent in a record written before the private-namespace prototype + ns_pid_file: value.get_text("nsExecPidFile").map(str::to_owned), controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), diff --git a/engine/crates/layerfs-node/src/exec.rs b/engine/crates/layerfs-node/src/exec.rs index 80a3120b2a..d0782e28d9 100644 --- a/engine/crates/layerfs-node/src/exec.rs +++ b/engine/crates/layerfs-node/src/exec.rs @@ -34,6 +34,7 @@ fn run( vec![HOME.to_owned()], directory.to_owned(), false, + state.ns_pid_file.as_deref(), ) .map_err(failed)?; let attached = created.start().map_err(failed)?; diff --git a/engine/crates/layerfs-node/src/registry.rs b/engine/crates/layerfs-node/src/registry.rs index aeeb25bc47..fea68a2283 100644 --- a/engine/crates/layerfs-node/src/registry.rs +++ b/engine/crates/layerfs-node/src/registry.rs @@ -47,6 +47,9 @@ pub(crate) struct SandboxState { pub(crate) directory: std::path::PathBuf, pub(crate) runtime: Sandbox, pub(crate) identity: CommandIdentity, + /// The pid file of a daemon that serves from a private mount namespace, or + /// `None` for one whose mounts are in the container's initial namespace. + pub(crate) ns_pid_file: Option, pub(crate) controls: crate::pool::Controls, pub(crate) mounts: Mutex>, pub(crate) processes: Mutex>, diff --git a/engine/crates/layerfs-node/src/sandbox.rs b/engine/crates/layerfs-node/src/sandbox.rs index edf23889a2..cc509dc181 100644 --- a/engine/crates/layerfs-node/src/sandbox.rs +++ b/engine/crates/layerfs-node/src/sandbox.rs @@ -54,7 +54,7 @@ pub(crate) const WORKSPACES: u32 = 16; /// for this process and the rest for others (#352 F-017). pub(crate) const DAEMON_CONNECTIONS: u16 = 8; -fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { +fn setup(private: [u8; 32], peer: [u8; 32], private_mount_ns: bool) -> DaemonSetup { DaemonSetup { listen: format!("0.0.0.0:{PORT}"), private_key: private, @@ -80,12 +80,23 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { idle_exit_s: IDLE_EXIT_S, }, existing_store: None, + private_mount_ns, } } /// A container name prefix when the deployment sets none. const PREFIX: &str = "layerfs-"; +/// Opt-in for the prototype private mount namespace (#3/#499): a truthy +/// `deploy.privateMountNs`, else `LAYERFS_PRIVATE_MOUNT_NS=1`. Off by default, +/// so an ordinary Sandbox keeps its mounts in the namespace exec enters. +fn private_mount_ns(args: &SdkValue) -> bool { + args.get("deploy") + .is_some_and(|deploy| deploy.get_bool("privateMountNs")) + || std::env::var_os("LAYERFS_PRIVATE_MOUNT_NS") + .is_some_and(|value| value.to_str() == Some("1")) +} + /// The container labels of Sandbox `id` of `project`: what it is, whose, /// and which. Saved in its record, so a reopen expects the same. pub(crate) fn labels(project: &str, id: &str) -> Vec<(String, String)> { @@ -187,7 +198,10 @@ fn start_claimed( let name = text(args, "project")?; let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; - let mut daemon_setup = setup(daemon.private, controller.public); + let mut daemon_setup = setup(daemon.private, controller.public, private_mount_ns(args)); + let ns_pid_file = daemon_setup + .mount_ns_pid_file() + .map(|path| path.to_string_lossy().into_owned()); let mut existing = project.manifest.clone(); existing.daemon_sqlite = installed.daemon_sqlite; daemon_setup.existing_store = Some(existing); @@ -245,6 +259,7 @@ fn start_claimed( directory: saved_directory.to_owned(), runtime, identity, + ns_pid_file, controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), diff --git a/engine/crates/layerfs-sandbox/examples/engine_streams.rs b/engine/crates/layerfs-sandbox/examples/engine_streams.rs index 2ef067fc16..16a269300b 100644 --- a/engine/crates/layerfs-sandbox/examples/engine_streams.rs +++ b/engine/crates/layerfs-sandbox/examples/engine_streams.rs @@ -56,6 +56,7 @@ fn main() { environment: vec![], directory: "/tmp".into(), stdin: true, + root_launcher: false, }; let created = docker.create_exec(container, request).unwrap(); let (cid, eid) = created.identity(); diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs index b7622d7efc..1537b2e2ed 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs @@ -12,11 +12,17 @@ use std::{ /// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. #[derive(Clone)] pub struct ExecRequest { + /// The ordinary command identity, which the command must end up as. pub identity: CommandIdentity, pub arguments: Vec, pub environment: Vec, pub directory: String, pub stdin: bool, + /// Start the entry process as root instead of `identity`. Only the daemon's + /// `ns-exec` helper needs this: joining the daemon's mount namespace needs + /// a capability only root holds, and the helper drops to `identity` before + /// it execs the command. Every other exec starts as `identity`. + pub root_launcher: bool, } impl fmt::Debug for ExecRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -26,6 +32,7 @@ impl fmt::Debug for ExecRequest { .field("environment_count", &self.environment.len()) .field("directory", &self.directory) .field("stdin", &self.stdin) + .field("root_launcher", &self.root_launcher) .finish() } } @@ -57,7 +64,12 @@ impl ExecRequest { out.write_all(b"{\"AttachStdout\":true,\"AttachStderr\":true,\"AttachStdin\":")?; out.write_all(if self.stdin { b"true" } else { b"false" })?; out.write_all(b",\"Tty\":false,\"Privileged\":false,\"User\":")?; - quoted(out, &format!("{}:{}", self.identity.uid, self.identity.gid))?; + let user = if self.root_launcher { + "0:0".to_owned() + } else { + format!("{}:{}", self.identity.uid, self.identity.gid) + }; + quoted(out, &user)?; out.write_all(b",\"WorkingDir\":")?; quoted(out, &self.directory)?; for (name, items) in [ diff --git a/engine/crates/layerfs-sandbox/tests/engine_keepalive.rs b/engine/crates/layerfs-sandbox/tests/engine_keepalive.rs index 7f8bc5ad60..a47e963f70 100644 --- a/engine/crates/layerfs-sandbox/tests/engine_keepalive.rs +++ b/engine/crates/layerfs-sandbox/tests/engine_keepalive.rs @@ -34,6 +34,7 @@ fn request() -> ExecRequest { environment: vec![], directory: "/tmp".into(), stdin: false, + root_launcher: false, } } fn response(status: u16, body: &str, close: bool) -> Vec { diff --git a/engine/crates/layerfs-sandbox/tests/engine_protocol.rs b/engine/crates/layerfs-sandbox/tests/engine_protocol.rs index c7c97e3284..0fdcbf61d2 100644 --- a/engine/crates/layerfs-sandbox/tests/engine_protocol.rs +++ b/engine/crates/layerfs-sandbox/tests/engine_protocol.rs @@ -20,6 +20,7 @@ fn request() -> ExecRequest { environment: vec!["X=a\"b\\c\n".into()], directory: "/tmp".into(), stdin: false, + root_launcher: false, } } fn sandbox_request() -> SandboxRequest { From 6554e7fdbc3aae40aa73a1f8007291f064c27cb1 Mon Sep 17 00:00:00 2001 From: rain Date: Thu, 29 Oct 2026 04:29:51 +0800 Subject: [PATCH 2/3] fix(sandbox): hand the daemon's mount namespace to ns-exec over SCM_RIGHTS The E2E showed the helper could not open the daemon's namespace: root in a Sandbox holds CAP_SYS_ADMIN but not CAP_SYS_PTRACE, and the daemon is not dumpable (`config.rs` prctl::set_dumpable(false)), so `/proc//ns/mnt` is EACCES for every Sandbox process. The same 0600 file opened fine from outside the container with CAP_SYS_PTRACE, so the kernel's dumpability check is the blocker, not the path. - The daemon, when `private_mount_ns` is on, binds `/ns-exec.sock` (mode 0600, stale path removed, unlinked on drop) and serves exactly one SCM_RIGHTS message carrying the descriptor of its own `/proc/self/ns/mnt` to an admitted peer: uid 0 and the same pid namespace, read from SO_PEERCRED and `/proc//ns/pid`. Nothing else is sent; a refused peer is closed without a message (`application/ns_fd.rs`). - `ns-exec` now requires `--socket`, receives that descriptor, joins the namespace, then keeps the existing `--directory` chdir, uid/gid drop and exec. The pid-file route is deleted: it can never work in a Sandbox, so the helper no longer names a path that only returns EACCES. - `DaemonSetup::check` requires the socket path for the private-namespace mode; the pid file stays as a diagnostic marker and is not written when absent. - The public `WorkspaceApi::exec` derives the namespace from the owner's `SandboxCreate.setup` instead of always passing none, so a private-ns Sandbox cannot run that path unwrapped in the initial namespace. - A live Sandbox whose record predates the socket (`nsExecPidFile` without `nsExecSocket`) refuses `workspace.exec` with `Unsupported` and an instruction to close and recreate it, rather than deriving a socket the old daemon never bound or repeating an opaque 125. - One `layerfs_sandbox::DAEMON_ENTRY` constant feeds the create request, the observed-topology check and the helper argv, so the entry path cannot drift. - `MountNamespace` carries a socket and has no empty state: `Some` always wraps and `None` never does. Tests: - ns-exec integration (11): join over a served fd (read and chdir), uid/gid drop, missing socket, a socket that sends nothing, a socket that sends a non-namespace file, no namespace source (integration 125), a peer in another pid namespace refused, socket mode 0600 with a stale path replaced and exactly one message (one byte, one fd, then EOF), a non-root peer refused, and the daemon serving socket and pid file only with the flag. - node unit (3): a saved socket wraps the command end to end (record -> mount_namespace() -> argv), a pid-only record requires recreation, and a plain Sandbox keeps its command. - bridge/api/daemon unit (2/2/7) and the 17 Engine protocol tests pass. - deleted: the pid-file exec tests and the pid-file helper route. Not exercised: a real Sandbox end-to-end; the host wiring and the same E2E rerun belong to the E2E agent. Production LOC: 103008 -> 103183 (delta +175) core 38395 -> 38395 (0), engine 61954 -> 62129 (+175), api 2659 -> 2659 (0); reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 tools/production_loc.py --root on a `git archive` extract of 8545a284 and on this staged tree. Refs tokenaissance/layerfs#3, tokenaissance/layerfs#499. --- .../crates/layerfs-api/src/base/internal.rs | 11 +- engine/crates/layerfs-api/src/sandbox/exec.rs | 94 ++++-- .../crates/layerfs-bridge/src/daemon_setup.rs | 39 ++- engine/crates/layerfs-daemon/Cargo.toml | 2 +- .../layerfs-daemon/src/application/cli.rs | 15 +- .../layerfs-daemon/src/application/mod.rs | 2 + .../layerfs-daemon/src/application/ns_exec.rs | 123 ++++--- .../layerfs-daemon/src/application/ns_fd.rs | 179 ++++++++++ engine/crates/layerfs-daemon/tests/ns_exec.rs | 316 ++++++++++++++---- engine/crates/layerfs-node/src/discovery.rs | 97 ++++++ engine/crates/layerfs-node/src/exec.rs | 11 +- engine/crates/layerfs-node/src/registry.rs | 19 +- engine/crates/layerfs-node/src/sandbox.rs | 5 + .../src/backend/docker/container.rs | 4 +- .../src/backend/docker/topology.rs | 3 +- engine/crates/layerfs-sandbox/src/lib.rs | 5 + 16 files changed, 737 insertions(+), 188 deletions(-) create mode 100644 engine/crates/layerfs-daemon/src/application/ns_fd.rs diff --git a/engine/crates/layerfs-api/src/base/internal.rs b/engine/crates/layerfs-api/src/base/internal.rs index 1d26c26ae6..dbd47cba13 100644 --- a/engine/crates/layerfs-api/src/base/internal.rs +++ b/engine/crates/layerfs-api/src/base/internal.rs @@ -141,8 +141,11 @@ pub fn parts( /// The same ordinary launcher for external host commands and disjoint concurrent proof borrows. #[cfg(unix)] -/// `ns_pid_file` names a daemon's private-namespace pid file: the command then -/// runs through that daemon's `ns-exec` helper. `None` keeps the command as it is. +pub use crate::sandbox::exec::{exec_arguments, MountNamespace}; + +/// `namespace` names a daemon that serves from a private mount namespace: the +/// command then runs through that daemon's `ns-exec` helper. `None` keeps the +/// command as it is. pub fn exec( sandbox: &layerfs_sandbox::backend::docker::Sandbox, identity: layerfs_sandbox::CommandIdentity, @@ -150,7 +153,7 @@ pub fn exec( environment: Vec, directory: String, stdin: bool, - ns_pid_file: Option<&str>, + namespace: Option>, ) -> Result< layerfs_sandbox::backend::docker::ExecCreated, Box, @@ -162,6 +165,6 @@ pub fn exec( environment, directory, stdin, - ns_pid_file, + namespace, ) } diff --git a/engine/crates/layerfs-api/src/sandbox/exec.rs b/engine/crates/layerfs-api/src/sandbox/exec.rs index 7a7ac39fc1..e31c734098 100644 --- a/engine/crates/layerfs-api/src/sandbox/exec.rs +++ b/engine/crates/layerfs-api/src/sandbox/exec.rs @@ -2,11 +2,19 @@ use crate::{ManagedSandbox, MountedWorkspace, WorkspaceApi}; use layerfs_sandbox::{ backend::docker::{ExecCreateFailure, ExecCreated, ExecRequest, Sandbox}, - CommandIdentity, + CommandIdentity, DAEMON_ENTRY, }; -/// The daemon's entry inside its Sandbox container, as the deployment installs it. -const DAEMON_ENTRY: &str = "/usr/local/bin/layerfs-daemon"; +/// The daemon socket a command gets its private mount namespace from. Only a +/// daemon that serves from a private namespace names one, so `Some` always +/// wraps the command and `None` never does: there is no empty middle state. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MountNamespace<'a> { + /// The daemon's socket. It sends the descriptor of its own mount + /// namespace; the daemon is not dumpable, so the helper cannot open + /// `/proc//ns/mnt` itself. + pub socket: &'a str, +} impl WorkspaceApi<'_> { /// Creates one ordinary command at this mount using the deployment's admitted identity. @@ -19,6 +27,15 @@ impl WorkspaceApi<'_> { environment: Vec, stdin: bool, ) -> Result> { + // The owner's own daemon setup says whether this Sandbox serves from a + // private namespace. When it does, the command must enter it through + // `ns-exec`; running it in the initial namespace would find only the + // empty directory under the mount. + let socket = sandbox + .request + .setup + .mount_ns_socket() + .map(|path| path.to_string_lossy().into_owned()); launch( &sandbox.runtime, sandbox.command_identity, @@ -26,15 +43,14 @@ impl WorkspaceApi<'_> { environment, workspace.ready.directory.clone(), stdin, - None, + socket.as_deref().map(|socket| MountNamespace { socket }), ) } } -/// Creates the Engine Exec for one command. `ns_pid_file` is the pid file of a -/// daemon that serves from a private mount namespace: the command then runs -/// through that daemon's `ns-exec` helper, because the Workspace mounts are -/// absent from the namespace the Engine enters. `None` keeps the command as -/// it is. +/// Creates the Engine Exec for one command. `namespace` names a daemon that +/// serves from a private mount namespace: the command then runs through that +/// daemon's `ns-exec` helper, because the Workspace mounts are absent from the +/// namespace the Engine enters. `None` keeps the command as it is. pub(crate) fn launch( sandbox: &Sandbox, identity: CommandIdentity, @@ -42,37 +58,37 @@ pub(crate) fn launch( environment: Vec, directory: String, stdin: bool, - ns_pid_file: Option<&str>, + namespace: Option>, ) -> Result> { sandbox.create_exec(ExecRequest { identity, - arguments: launch_arguments(ns_pid_file, identity, &directory, arguments), + arguments: exec_arguments(namespace, identity, &directory, arguments), environment, directory, stdin, - root_launcher: ns_pid_file.is_some(), + root_launcher: namespace.is_some(), }) } -/// The Engine argv for one command. Without a private-namespace pid file it is -/// exactly the caller's argv. With one it names the daemon's helper, the pid -/// file, the mount directory the Engine starts in (only the directory under the -/// mount exists in its namespace) and the ordinary identity the helper drops -/// to, then the same argv after `--`: the helper execs it, so the Engine still -/// has one process tree, the same streams and the same exit code. -fn launch_arguments( - ns_pid_file: Option<&str>, +/// The Engine argv for one command. Without a mount-namespace socket it is +/// exactly the caller's argv. With one it names the daemon's helper, the +/// socket, the mount directory the Engine starts in (only the directory under +/// the mount exists in its namespace) and the ordinary identity the helper +/// drops to, then the same argv after `--`: the helper execs it, so the Engine +/// still has one process tree, the same streams and the same exit code. +pub fn exec_arguments( + namespace: Option>, identity: CommandIdentity, directory: &str, arguments: Vec, ) -> Vec { - let Some(pid_file) = ns_pid_file else { + let Some(namespace) = namespace else { return arguments; }; let mut wrapped = vec![ DAEMON_ENTRY.to_owned(), "ns-exec".to_owned(), - "--pid-file".to_owned(), - pid_file.to_owned(), + "--socket".to_owned(), + namespace.socket.to_owned(), "--directory".to_owned(), directory.to_owned(), "--uid".to_owned(), @@ -89,32 +105,38 @@ fn launch_arguments( mod tests { use super::*; + fn identity() -> CommandIdentity { + CommandIdentity { uid: 501, gid: 20 } + } + + fn command() -> Vec { + vec!["bash".into(), "-c".into(), "printf ordinary".into()] + } + #[test] fn a_sandbox_without_a_private_namespace_keeps_the_command_as_it_is() { - let identity = CommandIdentity { uid: 501, gid: 20 }; - let arguments = vec!["bash".into(), "-c".into(), "printf ordinary".into()]; assert_eq!( - launch_arguments(None, identity, "/workspaces/1", arguments.clone()), - arguments + exec_arguments(None, identity(), "/workspaces/1", command()), + command() ); } #[test] - fn a_private_namespace_sandbox_runs_the_daemon_helper() { - let identity = CommandIdentity { uid: 501, gid: 20 }; - let arguments = vec!["bash".into(), "-c".into(), "printf ordinary".into()]; + fn a_private_namespace_sandbox_runs_the_daemon_helper_over_its_socket() { assert_eq!( - launch_arguments( - Some("/layerfs-local/daemon.pid"), - identity, + exec_arguments( + Some(MountNamespace { + socket: "/layerfs-local/ns-exec.sock", + }), + identity(), "/workspaces/1", - arguments, + command(), ), [ "/usr/local/bin/layerfs-daemon", "ns-exec", - "--pid-file", - "/layerfs-local/daemon.pid", + "--socket", + "/layerfs-local/ns-exec.sock", "--directory", "/workspaces/1", "--uid", diff --git a/engine/crates/layerfs-bridge/src/daemon_setup.rs b/engine/crates/layerfs-bridge/src/daemon_setup.rs index 2b838b6556..bb9e0886be 100644 --- a/engine/crates/layerfs-bridge/src/daemon_setup.rs +++ b/engine/crates/layerfs-bridge/src/daemon_setup.rs @@ -52,8 +52,9 @@ pub struct DaemonSetup { /// None selects the original one-time installer. Some selects explicit existing /// Store open using retained authority; never selected after an install error. pub existing_store: Option, - /// Start in a mount namespace of this process's own (prototype for #3/#499) - /// and write the pid file [`DaemonSetup::mount_ns_pid_file`] names, so a + /// Start in a mount namespace of this process's own (prototype for #3/#499), + /// serve its descriptor on [`DaemonSetup::mount_ns_socket`] and write the + /// diagnostic pid file [`DaemonSetup::mount_ns_pid_file`] names, so a /// command reaches the Workspace mounts only through the daemon's /// `ns-exec` helper. Off by default; the flag is opt-in. pub private_mount_ns: bool, @@ -167,21 +168,30 @@ impl DaemonSetup { return Err(ControlError("explicit existing Disposable Store")); } } - if self.private_mount_ns && self.mount_ns_pid_file().is_none() { - return Err(ControlError("private mount namespace pid file path")); + if self.private_mount_ns && self.mount_ns_socket().is_none() { + return Err(ControlError("private mount namespace socket path")); } Ok(()) } - /// The pid file this daemon writes when [`Self::private_mount_ns`] is on: - /// `daemon.pid` in the local directory above the Overlay's own directory - /// (in a Sandbox that is `/layerfs-local`). `None` when the flag is off or - /// the Overlay path has no such local directory. + /// The diagnostic pid file this daemon writes when + /// [`Self::private_mount_ns`] is on: `daemon.pid` in the local directory + /// above the Overlay's own directory (in a Sandbox that is + /// `/layerfs-local`). It is evidence, not an exec route. `None` when the + /// flag is off or the Overlay path has no such local directory. pub fn mount_ns_pid_file(&self) -> Option { + self.local_path("daemon.pid") + } + /// The socket this daemon serves its mount-namespace descriptor on when + /// [`Self::private_mount_ns`] is on, beside the pid file. + pub fn mount_ns_socket(&self) -> Option { + self.local_path("ns-exec.sock") + } + fn local_path(&self, name: &str) -> Option { if !self.private_mount_ns { return None; } let home = Path::new(&self.overlay).parent()?.parent()?; - Some(home.join("daemon.pid")) + Some(home.join(name)) } } impl fmt::Debug for DaemonSetup { @@ -235,6 +245,9 @@ mod tests { assert!(setup("/local/overlay/overlay.sqlite", false) .mount_ns_pid_file() .is_none()); + assert!(setup("/local/overlay/overlay.sqlite", false) + .mount_ns_socket() + .is_none()); let encoded = setup("/local/overlay/overlay.sqlite", true) .encode() .unwrap(); @@ -244,6 +257,10 @@ mod tests { decoded.mount_ns_pid_file(), Some(PathBuf::from("/local/daemon.pid")) ); + assert_eq!( + decoded.mount_ns_socket(), + Some(PathBuf::from("/local/ns-exec.sock")) + ); } #[test] @@ -252,6 +269,10 @@ mod tests { setup("/layerfs-local/overlay/overlay.sqlite", true).mount_ns_pid_file(), Some(PathBuf::from("/layerfs-local/daemon.pid")) ); + assert_eq!( + setup("/layerfs-local/overlay/overlay.sqlite", true).mount_ns_socket(), + Some(PathBuf::from("/layerfs-local/ns-exec.sock")) + ); // An Overlay path with no local directory cannot name a pid file. assert!(setup("/overlay.sqlite", true).encode().is_err()); } diff --git a/engine/crates/layerfs-daemon/Cargo.toml b/engine/crates/layerfs-daemon/Cargo.toml index 2fb5fc162c..d7759aac1e 100644 --- a/engine/crates/layerfs-daemon/Cargo.toml +++ b/engine/crates/layerfs-daemon/Cargo.toml @@ -19,7 +19,7 @@ layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } # Concrete Store opening stays in application bootstrap, outside store/. layerfs-persistence = { path = "../layerfs-persistence" } -nix = { version = "=0.31.3", features = ["fs", "user", "process", "sched"] } +nix = { version = "=0.31.3", features = ["fs", "user", "process", "sched", "socket"] } [dev-dependencies] blake3 = { version = "=1.8.5", default-features = false } diff --git a/engine/crates/layerfs-daemon/src/application/cli.rs b/engine/crates/layerfs-daemon/src/application/cli.rs index 7863c5e589..a672d9995a 100644 --- a/engine/crates/layerfs-daemon/src/application/cli.rs +++ b/engine/crates/layerfs-daemon/src/application/cli.rs @@ -67,16 +67,21 @@ fn run(args: &[OsString]) -> Result<(), ApplicationError> { // an Engine exec or a `docker exec` enters (prototype for #3/#499). A // failure refuses startup rather than serving with the old topology. #[cfg(target_os = "linux")] - if setup.private_mount_ns { + let _ns_fd = if setup.private_mount_ns { layerfs_fuse::mount::private_namespace().map_err(ApplicationError::Io)?; - let pid_file = setup.mount_ns_pid_file().ok_or_else(|| { + if let Some(pid_file) = setup.mount_ns_pid_file() { + write_pid_file(&pid_file).map_err(ApplicationError::Io)?; + } + let socket = setup.mount_ns_socket().ok_or_else(|| { ApplicationError::Io(io::Error::new( io::ErrorKind::InvalidInput, - "private mount namespace without a pid file path", + "private mount namespace without a socket path", )) })?; - write_pid_file(&pid_file).map_err(ApplicationError::Io)?; - } + Some(super::ns_fd::NsFd::start(&socket).map_err(ApplicationError::Io)?) + } else { + None + }; let listener = TcpListener::bind(&setup.listen).map_err(ApplicationError::Io)?; let application = Application::start(setup)?; println!( diff --git a/engine/crates/layerfs-daemon/src/application/mod.rs b/engine/crates/layerfs-daemon/src/application/mod.rs index 3bec6ee18f..5cf104d5f5 100644 --- a/engine/crates/layerfs-daemon/src/application/mod.rs +++ b/engine/crates/layerfs-daemon/src/application/mod.rs @@ -9,6 +9,8 @@ mod filesystem; mod idle; #[cfg(target_os = "linux")] mod ns_exec; +#[cfg(target_os = "linux")] +pub mod ns_fd; mod owner; mod serve; pub use cli::main; diff --git a/engine/crates/layerfs-daemon/src/application/ns_exec.rs b/engine/crates/layerfs-daemon/src/application/ns_exec.rs index 618f9f628a..f69b5d7187 100644 --- a/engine/crates/layerfs-daemon/src/application/ns_exec.rs +++ b/engine/crates/layerfs-daemon/src/application/ns_exec.rs @@ -4,26 +4,29 @@ //! `workspace.exec` is an Engine exec in the container's initial mount //! namespace. When the daemon serves from a private namespace, the Workspace //! mounts are in that namespace only, so the Engine's entry command is this -//! helper: it reads the daemon's pid from the pid file the daemon wrote, joins -//! `/proc//ns/mnt`, drops the root identity the Engine starts it with back -//! to the ordinary command identity, and execs the command it was given. The -//! Engine still sees one process and the same streams and exit code. Nothing -//! is run when any step fails: a helper that cannot join must not silently run -//! the command in the wrong namespace. +//! helper: it receives the daemon's mount-namespace descriptor over the +//! daemon's socket, joins it, drops the root identity the Engine starts it with +//! back to the ordinary command identity, and execs the command it was given. +//! The Engine still sees one process and the same streams and exit code. +//! Nothing is run when any step fails: a helper that cannot join must not +//! silently run the command in the wrong namespace. Every refusal exits 125 +//! with a `layerfs-daemon ns-exec:` reason on stderr, which the host sees as +//! the command's exit code and stderr (a mixed old/new pair therefore refuses +//! deliberately instead of running unwrapped). use nix::{ sched::{setns, CloneFlags}, unistd::{setgid, setgroups, setuid, Gid, Uid}, }; use std::{ ffi::{OsStr, OsString}, - fs, os::unix::process::CommandExt, + path::Path, process::Command, }; /// Exit code of a helper that refused before running the command. const REFUSED: u8 = 125; -const USAGE: &str = "usage: layerfs-daemon ns-exec --pid-file [--directory ] [--uid --gid ] -- [args...]"; +const USAGE: &str = "usage: layerfs-daemon ns-exec --socket [--directory ] [--uid --gid ] -- [args...]"; pub(super) fn main(args: &[OsString]) -> std::process::ExitCode { match run(args) { @@ -37,9 +40,14 @@ pub(super) fn main(args: &[OsString]) -> std::process::ExitCode { } fn run(args: &[OsString]) -> Result<(), String> { - let (pid_file, directory, identity, command) = parse(args)?; - let pid = read_pid(&pid_file)?; - join_mount_namespace(pid)?; + let (socket, directory, identity, command) = parse(args)?; + // The daemon is not dumpable, so a Sandbox root (CAP_SYS_ADMIN without + // CAP_SYS_PTRACE) cannot open its `/proc//ns/mnt`; the daemon sends + // the descriptor over this socket instead. + let namespace = super::ns_fd::receive(Path::new(&socket)) + .map_err(|error| format!("receive mount namespace from {socket}: {error}"))?; + setns(&namespace, CloneFlags::CLONE_NEWNS) + .map_err(|error| format!("join mount namespace from {socket}: {error}"))?; // The Engine starts this process in the container's initial namespace, // where the mount directory is only the empty directory the daemon // mounted over. Enter the joined namespace's view of it before running. @@ -55,20 +63,21 @@ fn run(args: &[OsString]) -> Result<(), String> { Err(format!("exec {}: {error}", program.to_string_lossy())) } -/// `--pid-file [--directory ] [--uid --gid ] -- [args...]`. +/// `--socket [--directory ] [--uid --gid ] -- +/// [args...]`. fn parse( args: &[OsString], ) -> Result<(String, Option, Option<(u32, u32)>, Vec), String> { let mut rest = args.iter(); - let pid_file = match rest.next() { - Some(flag) if flag == OsStr::new("--pid-file") => path(rest.next())?, - _ => return Err(USAGE.to_owned()), - }; + let mut socket = None; let mut directory = None; let mut uid = None; let mut gid = None; loop { match rest.next() { + Some(flag) if flag == OsStr::new("--socket") && socket.is_none() => { + socket = Some(path(rest.next())?) + } Some(flag) if flag == OsStr::new("--directory") && directory.is_none() => { directory = Some(path(rest.next())?) } @@ -83,6 +92,9 @@ fn parse( } } let command: Vec = rest.cloned().collect(); + let Some(socket) = socket else { + return Err(USAGE.to_owned()); + }; if command.is_empty() { return Err(USAGE.to_owned()); } @@ -90,8 +102,8 @@ fn parse( (Some(0), _) | (_, Some(0)) => { Err("--uid and --gid must name the non-root command identity".to_owned()) } - (Some(uid), Some(gid)) => Ok((pid_file, directory, Some((uid, gid)), command)), - (None, None) => Ok((pid_file, directory, None, command)), + (Some(uid), Some(gid)) => Ok((socket, directory, Some((uid, gid)), command)), + (None, None) => Ok((socket, directory, None, command)), _ => Err("--uid and --gid must be given together".to_owned()), } } @@ -111,24 +123,6 @@ fn number(value: Option<&OsString>, name: &str) -> Result { .ok_or_else(|| format!("--{name} is not a number")) } -/// The daemon's pid from its pid file, or a clear refusal. -fn read_pid(pid_file: &str) -> Result { - let pid = fs::read_to_string(pid_file) - .map_err(|error| format!("read daemon pid file {pid_file}: {error}"))?; - match pid.trim().parse::() { - Ok(pid) if pid != 0 => Ok(pid), - _ => Err(format!("daemon pid file {pid_file} does not name a pid")), - } -} - -fn join_mount_namespace(pid: u32) -> Result<(), String> { - let namespace = format!("/proc/{pid}/ns/mnt"); - let file = fs::File::open(&namespace) - .map_err(|error| format!("open mount namespace {namespace}: {error}"))?; - setns(&file, CloneFlags::CLONE_NEWNS) - .map_err(|error| format!("join mount namespace {namespace}: {error}")) -} - /// The Engine starts this helper as root, because only root holds the /// capability the `setns` needs; the command must still run as the Sandbox's /// ordinary command identity, with no supplementary groups and no capability. @@ -142,9 +136,16 @@ fn drop_to(uid: u32, gid: u32) -> Result<(), String> { mod tests { use super::*; + fn socket_command() -> Vec { + ["--socket", "/s", "--", "true"] + .into_iter() + .map(OsString::from) + .collect() + } + #[test] fn the_command_identity_cannot_be_root() { - let args: Vec = ["--pid-file", "/p", "--uid", "0", "--gid", "0", "--", "true"] + let args: Vec = ["--socket", "/s", "--uid", "0", "--gid", "0", "--", "true"] .into_iter() .map(OsString::from) .collect(); @@ -153,15 +154,27 @@ mod tests { #[test] fn a_command_needs_a_separator_and_a_program() { - assert!(parse(&[OsString::from("--pid-file"), "/p".into()]).is_err()); - assert!(parse(&[OsString::from("--pid-file"), "/p".into(), "--".into()]).is_err()); + assert!(parse(&[OsString::from("--socket"), "/s".into()]).is_err()); + assert!(parse(&[OsString::from("--socket"), "/s".into(), "--".into()]).is_err()); + } + + #[test] + fn a_socket_names_the_namespace() { + let (socket, directory, identity, command) = parse(&socket_command()).unwrap(); + assert_eq!(socket, "/s"); + assert_eq!(directory, None); + assert_eq!(identity, None); + assert_eq!(command, ["true"]); + // No socket, no namespace: the helper refuses. + let none: Vec = ["--", "true"].into_iter().map(OsString::from).collect(); + assert!(parse(&none).is_err()); } #[test] fn the_identity_is_all_or_nothing() { let args = [ - "--pid-file".into(), - "/p".into(), + "--socket".into(), + "/s".into(), "--uid".into(), "501".into(), "--".into(), @@ -172,37 +185,15 @@ mod tests { #[test] fn a_parsed_command_keeps_its_arguments() { - let args: Vec = ["--pid-file", "/p", "--uid", "501", "--gid", "20", "--"] + let args: Vec = ["--socket", "/s", "--uid", "501", "--gid", "20", "--"] .into_iter() .map(OsString::from) .chain(["bash".into(), "-c".into(), "printf hello".into()]) .collect(); - let (pid_file, directory, identity, command) = parse(&args).unwrap(); - assert_eq!(pid_file, "/p"); + let (socket, directory, identity, command) = parse(&args).unwrap(); + assert_eq!(socket, "/s"); assert_eq!(directory, None); assert_eq!(identity, Some((501, 20))); assert_eq!(command, ["bash", "-c", "printf hello"]); } - - #[test] - fn a_missing_or_unreadable_pid_file_is_a_refusal() { - let directory = - std::env::temp_dir().join(format!("layerfs-ns-exec-{}", std::process::id())); - let _ = fs::remove_dir_all(&directory); - fs::create_dir(&directory).unwrap(); - let missing = directory.join("missing.pid"); - assert!(read_pid(missing.to_str().unwrap()).is_err()); - fs::write(&missing, b"not a pid\n").unwrap(); - assert!(read_pid(missing.to_str().unwrap()).is_err()); - fs::write(&missing, b"0\n").unwrap(); - assert!(read_pid(missing.to_str().unwrap()).is_err()); - fs::write(&missing, b"4321\n").unwrap(); - assert_eq!(read_pid(missing.to_str().unwrap()).unwrap(), 4321); - fs::remove_dir_all(&directory).unwrap(); - } - - #[test] - fn a_missing_namespace_is_a_refusal() { - assert!(join_mount_namespace(u32::MAX - 1).is_err()); - } } diff --git a/engine/crates/layerfs-daemon/src/application/ns_fd.rs b/engine/crates/layerfs-daemon/src/application/ns_fd.rs new file mode 100644 index 0000000000..a3ef6c53b0 --- /dev/null +++ b/engine/crates/layerfs-daemon/src/application/ns_fd.rs @@ -0,0 +1,179 @@ +//! The daemon's own mount-namespace descriptor, served to the `ns-exec` helper. +//! +//! The daemon is not dumpable (`application/config.rs`) and a Sandbox has +//! CAP_SYS_ADMIN but not CAP_SYS_PTRACE, so even root inside the container +//! cannot open `/proc//ns/mnt` (`EACCES`). The daemon can open its own +//! namespace, so it hands the descriptor to a root helper over a private Unix +//! socket in one `SCM_RIGHTS` message. The peer must be root in this process's +//! pid namespace; nothing is read from it and nothing else is sent. +use nix::{ + sys::socket::{ + getsockopt, recvmsg, sendmsg, sockopt, ControlMessage, ControlMessageOwned, MsgFlags, + SockaddrStorage, UnixCredentials, + }, + unistd, +}; +use std::{ + fs::{self, File}, + io::{self, IoSlice, IoSliceMut}, + os::{ + fd::{AsRawFd, RawFd}, + unix::{ + fs::PermissionsExt, + net::{UnixListener, UnixStream}, + }, + }, + path::{Path, PathBuf}, + thread, +}; + +/// One private socket that serves this process's mount-namespace descriptor. +pub struct NsFd { + path: PathBuf, +} + +impl NsFd { + /// Bind the socket (mode 0600, any stale path removed) and serve it from + /// one background thread. Called once, after the private namespace exists + /// and before the daemon listens for control connections. + pub fn start(path: &Path) -> io::Result { + match fs::remove_file(path) { + Ok(()) => (), + Err(error) if error.kind() == io::ErrorKind::NotFound => (), + Err(error) => return Err(error), + } + let listener = UnixListener::bind(path)?; + if let Err(error) = fs::set_permissions(path, fs::Permissions::from_mode(0o600)) { + let _ = fs::remove_file(path); + return Err(error); + } + let path = path.to_owned(); + thread::Builder::new() + .name("layerfs-ns-fd".to_owned()) + .spawn(move || serve(listener))?; + Ok(Self { path }) + } +} + +impl Drop for NsFd { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +fn serve(listener: UnixListener) { + loop { + match listener.accept() { + Ok((stream, _)) => { + // One message and one close per connection; a refused peer + // gets nothing and cannot retry over the same connection. + let _ = send(stream); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(_) => return, + } + } +} + +/// Admit one accepted peer, then send exactly one descriptor for this process's +/// mount namespace. A refusal closes the connection without sending. +fn send(stream: UnixStream) -> io::Result<()> { + let credentials = getsockopt(&stream, sockopt::PeerCredentials)?; + if !admit(&credentials) { + return Err(io::Error::from(io::ErrorKind::PermissionDenied)); + } + let namespace = File::open("/proc/self/ns/mnt")?; + let descriptors = [namespace.as_raw_fd()]; + let control = [ControlMessage::ScmRights(&descriptors)]; + let payload = [0u8]; + sendmsg( + stream.as_raw_fd(), + &[IoSlice::new(&payload)], + &control, + MsgFlags::empty(), + None::<&SockaddrStorage>, + )?; + Ok(()) +} + +/// A root peer in this process's pid namespace. The socket itself is 0600 in +/// the daemon's private directory; both checks make a grant to a wrong process +/// impossible, and an unreadable pid namespace refuses. +fn admit(credentials: &UnixCredentials) -> bool { + admit_uid(credentials.uid()) && same_pid_namespace(credentials.pid()) +} + +fn admit_uid(uid: u32) -> bool { + uid == 0 +} + +fn same_pid_namespace(pid: i32) -> bool { + if pid <= 0 { + return false; + } + let peer = fs::read_link(format!("/proc/{pid}/ns/pid")); + let own = fs::read_link("/proc/self/ns/pid"); + matches!((peer, own), (Ok(peer), Ok(own)) if peer == own) +} + +/// Connect to a daemon's socket and receive the one mount-namespace descriptor. +/// The received raw descriptor is reopened through `/proc/self/fd` so the +/// returned file is owned: this crate forbids unsafe code and `SCM_RIGHTS` +/// yields a raw descriptor. The label is checked before anything is returned. +pub fn receive(path: &Path) -> io::Result { + let stream = UnixStream::connect(path)?; + let mut byte = [0u8; 1]; + let mut iov = [IoSliceMut::new(&mut byte)]; + let mut space = nix::cmsg_space!([RawFd; 1]); + let message = recvmsg::<()>( + stream.as_raw_fd(), + &mut iov, + Some(&mut space), + MsgFlags::empty(), + )?; + let mut descriptors = Vec::new(); + for control in message.cmsgs()? { + if let ControlMessageOwned::ScmRights(received) = control { + descriptors.extend(received); + } + } + let Some(descriptor) = descriptors.first().copied() else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the socket sent no mount namespace descriptor", + )); + }; + for extra in &descriptors[1..] { + let _ = unistd::close(*extra); + } + let label = fs::read_link(format!("/proc/self/fd/{descriptor}"))?; + let namespace = File::open(format!("/proc/self/fd/{descriptor}"))?; + let _ = unistd::close(descriptor); + if !label.to_string_lossy().starts_with("mnt:") { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("the socket sent {label:?}, not a mount namespace"), + )); + } + Ok(namespace) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_root_is_admitted() { + assert!(admit_uid(0)); + assert!(!admit_uid(501)); + assert!(!admit_uid(65534)); + } + + #[test] + fn a_peer_must_be_in_this_pid_namespace() { + assert!(same_pid_namespace(std::process::id() as i32)); + assert!(!same_pid_namespace(0)); + assert!(!same_pid_namespace(-1)); + assert!(!same_pid_namespace(i32::MAX)); + } +} diff --git a/engine/crates/layerfs-daemon/tests/ns_exec.rs b/engine/crates/layerfs-daemon/tests/ns_exec.rs index 7dc68459f7..e7189a41b9 100644 --- a/engine/crates/layerfs-daemon/tests/ns_exec.rs +++ b/engine/crates/layerfs-daemon/tests/ns_exec.rs @@ -2,21 +2,33 @@ //! //! A command started by the Engine is in the container's initial mount //! namespace; when the daemon serves from a private one, the Workspace mounts -//! are absent there. The helper joins the namespace of the daemon whose pid the -//! pid file names, drops to the ordinary command identity, and execs the -//! command. These tests build that private namespace with `unshare -m`: a -//! tmpfs is mounted in it only, and a marker written there is invisible to the -//! test's own namespace, so a command that finds it proves it joined. +//! are absent there. The helper receives the daemon's mount-namespace +//! descriptor over the daemon's socket, joins it, drops to the ordinary +//! command identity, and execs the command. These tests build that private +//! namespace with `unshare -m`: a tmpfs is mounted in it only, and a marker +//! written there is invisible to the test's own namespace, so a command that +//! finds it proves it joined. //! //! Needs root with CAP_SYS_ADMIN (a Sandbox container has both) and -//! `unshare`/`mount`. A missing or invalid pid file must refuse, never run the -//! command in the wrong namespace. +//! `unshare`/`mount`. A missing socket, a served non-namespace descriptor and a +//! refused peer must refuse, never run the command in the wrong namespace. #![cfg(target_os = "linux")] use layerfs_bridge::daemon_setup::{DaemonLimits, DaemonSetup}; +use layerfs_daemon::application::ns_fd::NsFd; +use nix::sys::socket::{ + recvmsg, sendmsg, ControlMessage, ControlMessageOwned, MsgFlags, SockaddrStorage, +}; use std::{ - fs, - io::{BufRead, BufReader, Write}, - os::unix::fs::PermissionsExt, + fs::{self, File}, + io::{BufRead, BufReader, IoSlice, IoSliceMut, Write}, + os::{ + fd::{AsRawFd, RawFd}, + unix::{ + fs::PermissionsExt, + net::{UnixListener, UnixStream}, + process::CommandExt, + }, + }, path::{Path, PathBuf}, process::{Child, ChildStdin, Command, Stdio}, sync::mpsc::{self, Receiver}, @@ -98,13 +110,10 @@ impl Private { .recv_timeout(WAIT) .unwrap_or_else(|_| panic!("{what}")) } - fn pid_file(&self) -> PathBuf { - self.directory.join("daemon.pid") - } - fn write_pid(&self) -> PathBuf { - let path = self.pid_file(); - fs::write(&path, format!("{}\n", self.child.id())).unwrap(); - path + /// This child's mount namespace, which root may open because the child is + /// dumpable; the daemon is not, which is why the daemon serves the fd. + fn namespace(&self) -> File { + File::open(format!("/proc/{}/ns/mnt", self.child.id())).unwrap() } fn stop(&mut self) { let _ = writeln!(self.stdin, "stop"); @@ -134,18 +143,24 @@ printf 'only-in-the-private-namespace\n' > "$1/marker" printf '%s\n' $$ while read line; do [ "$line" = stop ] && exit 0; done"#; -fn run(pid_file: &Path, command: &[&str], identity: Option<(u32, u32)>) -> (i32, String, String) { - run_in(pid_file, None, command, identity) +fn run_socket( + socket: &Path, + directory: Option<&Path>, + command: &[&str], + identity: Option<(u32, u32)>, +) -> (i32, String, String) { + run_command(socket, directory, command, identity, None) } -fn run_in( - pid_file: &Path, +fn run_command( + socket: &Path, directory: Option<&Path>, command: &[&str], identity: Option<(u32, u32)>, + as_uid: Option, ) -> (i32, String, String) { let mut helper = Command::new(BINARY); - helper.args(["ns-exec", "--pid-file"]).arg(pid_file); + helper.arg("ns-exec").arg("--socket").arg(socket); if let Some(directory) = directory { helper.arg("--directory").arg(directory); } @@ -156,6 +171,9 @@ fn run_in( .arg("--gid") .arg(gid.to_string()); } + if let Some(uid) = as_uid { + helper.uid(uid); + } let output = helper .arg("--") .args(command) @@ -168,96 +186,262 @@ fn run_in( ) } +/// A socket that serves one already-open descriptor to its first connection. +fn serve_once(path: &Path, namespace: File) -> thread::JoinHandle<()> { + let listener = listen_once(path); + thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let descriptors = [namespace.as_raw_fd()]; + let control = [ControlMessage::ScmRights(&descriptors)]; + sendmsg( + stream.as_raw_fd(), + &[IoSlice::new(&[0u8])], + &control, + MsgFlags::empty(), + None::<&SockaddrStorage>, + ) + .unwrap(); + }) +} + +/// A socket that accepts one connection and closes it without sending. +fn close_once(path: &Path) -> thread::JoinHandle<()> { + let listener = listen_once(path); + thread::spawn(move || { + let (_stream, _) = listener.accept().unwrap(); + }) +} + +fn listen_once(path: &Path) -> UnixListener { + if path.exists() { + fs::remove_file(path).unwrap(); + } + let listener = UnixListener::bind(path).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap(); + listener +} + +/// One received message: its payload bytes and every descriptor it carried. +fn receive_message(stream: &UnixStream) -> (usize, Vec) { + let mut byte = [0u8; 8]; + let mut iov = [IoSliceMut::new(&mut byte)]; + let mut space = nix::cmsg_space!([RawFd; 4]); + let message = recvmsg::<()>( + stream.as_raw_fd(), + &mut iov, + Some(&mut space), + MsgFlags::empty(), + ) + .unwrap(); + let mut descriptors = Vec::new(); + for control in message.cmsgs().unwrap() { + if let ControlMessageOwned::ScmRights(items) = control { + descriptors.extend(items); + } + } + (message.bytes, descriptors) +} + #[test] -fn a_command_joins_the_daemons_private_mount_namespace() { +fn a_command_joins_the_namespace_served_over_the_daemons_socket() { let mut private = Private::spawn(); let marker = private.private.join("marker"); - // The proof is meaningful only while the marker is absent here. assert!(!marker.exists(), "the tmpfs is absent from this namespace"); - let pid_file = private.write_pid(); - let (code, out, err) = run( - &pid_file, + let socket = private.directory.join("ns-exec.sock"); + let server = serve_once(&socket, private.namespace()); + let (code, out, err) = run_socket( + &socket, + None, &["sh", "-c", &format!("cat {}", marker.display())], None, ); + server.join().unwrap(); assert_eq!(code, 0, "helper stderr: {err}"); - assert_eq!(out, MARKER, "the command read the private namespace's file"); + assert_eq!(out, MARKER, "the command read the served namespace's file"); private.stop(); } #[test] -fn the_command_starts_in_the_joined_namespaces_view_of_the_directory() { +fn the_socket_command_starts_in_the_served_namespaces_directory() { let mut private = Private::spawn(); - let pid_file = private.write_pid(); - let (code, out, err) = run_in( - &pid_file, + let socket = private.directory.join("ns-exec.sock"); + let server = serve_once(&socket, private.namespace()); + let (code, out, err) = run_socket( + &socket, Some(&private.private), &["sh", "-c", "cat marker"], None, ); + server.join().unwrap(); assert_eq!(code, 0, "helper stderr: {err}"); assert_eq!(out, MARKER); private.stop(); } #[test] -fn the_command_runs_as_the_identity_the_helper_drops_to() { +fn the_socket_command_runs_as_the_identity_the_helper_drops_to() { let mut private = Private::spawn(); - let pid_file = private.write_pid(); - let (code, out, err) = run( - &pid_file, + let socket = private.directory.join("ns-exec.sock"); + let server = serve_once(&socket, private.namespace()); + let (code, out, err) = run_socket( + &socket, + None, &["sh", "-c", "id -u; id -g"], Some((65534, 65534)), ); + server.join().unwrap(); assert_eq!(code, 0, "helper stderr: {err}"); assert_eq!(out, "65534\n65534\n"); private.stop(); } #[test] -fn a_missing_pid_file_refuses_and_runs_nothing() { +fn a_missing_socket_refuses_and_runs_nothing() { let private = Private::spawn(); let side_effect = private.directory.join("ran-anyway"); - let (code, _, err) = run( - &private.pid_file(), + let (code, _, err) = run_socket( + &private.directory.join("absent.sock"), + None, &["sh", "-c", &format!("touch {}", side_effect.display())], None, ); assert_eq!(code, 125); - assert!(err.contains("daemon pid file"), "stderr: {err}"); + assert!(err.contains("receive mount namespace"), "stderr: {err}"); assert!(!side_effect.exists(), "no command ran"); } #[test] -fn a_pid_file_that_is_not_a_pid_refuses_and_runs_nothing() { +fn a_socket_that_sends_nothing_refuses_and_runs_nothing() { let private = Private::spawn(); - let pid_file = private.pid_file(); - fs::write(&pid_file, b"not a pid\n").unwrap(); + let socket = private.directory.join("silent.sock"); + let server = close_once(&socket); let side_effect = private.directory.join("ran-anyway"); - let (code, _, err) = run( - &pid_file, + let (code, _, err) = run_socket( + &socket, + None, &["sh", "-c", &format!("touch {}", side_effect.display())], None, ); + server.join().unwrap(); assert_eq!(code, 125); - assert!(err.contains("does not name a pid"), "stderr: {err}"); + assert!( + err.contains("sent no mount namespace descriptor"), + "stderr: {err}" + ); assert!(!side_effect.exists(), "no command ran"); } #[test] -fn a_pid_without_a_mount_namespace_refuses_and_runs_nothing() { +fn a_socket_that_sends_another_file_refuses_and_runs_nothing() { let private = Private::spawn(); - let pid_file = private.pid_file(); - fs::write(&pid_file, b"4294967294\n").unwrap(); + let socket = private.directory.join("wrong.sock"); + let server = serve_once(&socket, File::open("/dev/null").unwrap()); let side_effect = private.directory.join("ran-anyway"); - let (code, _, err) = run( - &pid_file, + let (code, _, err) = run_socket( + &socket, + None, + &["sh", "-c", &format!("touch {}", side_effect.display())], + None, + ); + server.join().unwrap(); + assert_eq!(code, 125); + assert!(err.contains("not a mount namespace"), "stderr: {err}"); + assert!(!side_effect.exists(), "no command ran"); +} + +#[test] +fn a_helper_without_a_namespace_source_refuses_and_runs_nothing() { + let directory = unique_directory("no-source"); + let side_effect = directory.join("ran-anyway"); + let output = Command::new(BINARY) + .args(["ns-exec", "--", "sh", "-c"]) + .arg(format!("touch {}", side_effect.display())) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(125)); + let err = String::from_utf8_lossy(&output.stderr); + assert!( + err.contains("usage: layerfs-daemon ns-exec"), + "stderr: {err}" + ); + assert!(!side_effect.exists(), "no command ran"); + fs::remove_dir_all(&directory).unwrap(); +} + +#[test] +fn a_helper_in_another_pid_namespace_is_refused_by_the_daemons_socket() { + let directory = unique_directory("foreign-pid-ns"); + let socket = directory.join("ns-exec.sock"); + let _server = NsFd::start(&socket).unwrap(); + // The child is root but in a pid namespace of its own; the daemon's + // admission requires this process's pid namespace. + let output = Command::new("unshare") + .args(["-p", "--fork", BINARY, "ns-exec", "--socket"]) + .arg(&socket) + .args(["--", "true"]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(125)); + let err = String::from_utf8_lossy(&output.stderr); + assert!( + err.contains("sent no mount namespace descriptor"), + "stderr: {err}" + ); + fs::remove_dir_all(&directory).unwrap(); +} + +#[test] +fn the_daemons_socket_is_private_replaces_a_stale_path_and_sends_one_message() { + let directory = unique_directory("socket"); + let socket = directory.join("ns-exec.sock"); + // A stale path from a dead daemon is replaced, not adopted. + fs::write(&socket, b"stale").unwrap(); + let server = NsFd::start(&socket).unwrap(); + let mode = fs::metadata(&socket).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "only root may reach the socket"); + let stream = UnixStream::connect(&socket).unwrap(); + let (bytes, descriptors) = receive_message(&stream); + assert_eq!(bytes, 1, "one message"); + assert_eq!(descriptors.len(), 1, "exactly one descriptor"); + assert_eq!( + fs::read_link(format!("/proc/self/fd/{}", descriptors[0])).unwrap(), + fs::read_link("/proc/self/ns/mnt").unwrap(), + "the descriptor is this process's mount namespace" + ); + nix::unistd::close(descriptors[0]).unwrap(); + // The server closes the connection after the one message. + let (bytes, descriptors) = receive_message(&stream); + assert_eq!(bytes, 0); + assert!(descriptors.is_empty()); + drop(server); + assert!(!socket.exists(), "the server removed its socket"); + fs::remove_dir_all(&directory).unwrap(); +} + +#[test] +fn the_daemons_socket_refuses_a_non_root_peer() { + let directory = unique_directory("non-root"); + let socket = directory.join("ns-exec.sock"); + let server = NsFd::start(&socket).unwrap(); + // The test widens the mode so a non-root peer can reach admission; the + // server then closes without sending. + fs::set_permissions(&socket, fs::Permissions::from_mode(0o666)).unwrap(); + let side_effect = directory.join("ran-anyway"); + let (code, _, err) = run_command( + &socket, + None, &["sh", "-c", &format!("touch {}", side_effect.display())], None, + Some(65534), ); assert_eq!(code, 125); - assert!(err.contains("mount namespace"), "stderr: {err}"); + assert!( + err.contains("sent no mount namespace descriptor"), + "stderr: {err}" + ); assert!(!side_effect.exists(), "no command ran"); + drop(server); + fs::remove_dir_all(&directory).unwrap(); } /// The private configuration record of one daemon start, beside its own paths. @@ -314,12 +498,13 @@ fn unique_directory(label: &str) -> PathBuf { directory } -/// The daemon writes the pid file only after it has taken its private mount -/// namespace, and taking it refuses startup on failure: a written pid file -/// therefore proves the unshare ran. Whether the daemon then serves or fails -/// (no `/dev/fuse` in this container) does not matter; the file stays. +/// The daemon writes the pid file and binds the socket only after it has taken +/// its private mount namespace, and taking it refuses startup on failure: a +/// served socket therefore proves the unshare ran. Whether the daemon then +/// serves or fails (no `/dev/fuse` in this container) does not matter; both +/// paths stay. #[test] -fn the_daemon_writes_its_pid_file_only_when_asked_for_a_private_namespace() { +fn the_daemon_serves_its_socket_and_pid_file_only_when_asked() { let directory = unique_directory("on"); let config = daemon_config(&directory, true); let mut daemon = Command::new(BINARY) @@ -331,13 +516,17 @@ fn the_daemon_writes_its_pid_file_only_when_asked_for_a_private_namespace() { .spawn() .unwrap(); let pid_file = directory.join("local/daemon.pid"); + let socket = directory.join("local/ns-exec.sock"); let deadline = Instant::now() + WAIT; - while !pid_file.exists() { + while !(pid_file.exists() && socket.exists()) { + assert!( + daemon.try_wait().unwrap().is_none() || socket.exists(), + "the daemon ended before serving its socket" + ); assert!( - daemon.try_wait().unwrap().is_none() || pid_file.exists(), - "the daemon ended before writing its pid file" + Instant::now() < deadline, + "no pid file and socket within {WAIT:?}" ); - assert!(Instant::now() < deadline, "no pid file within {WAIT:?}"); thread::sleep(Duration::from_millis(10)); } let pid: u32 = fs::read_to_string(&pid_file) @@ -361,11 +550,12 @@ fn the_daemon_writes_its_pid_file_only_when_asked_for_a_private_namespace() { .spawn() .unwrap(); let pid_file = directory.join("local/daemon.pid"); + let socket = directory.join("local/ns-exec.sock"); let deadline = Instant::now() + WAIT; loop { assert!( - !pid_file.exists(), - "the flag is off; no pid file is written" + !pid_file.exists() && !socket.exists(), + "the flag is off; neither a pid file nor a socket is written" ); if daemon.try_wait().unwrap().is_some() || Instant::now() >= deadline { break; diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index e839e712e7..c739be4ff9 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -375,9 +375,27 @@ pub(crate) fn save_sandbox( "nsExecPidFile", SdkValue::opt_text(state.ns_pid_file.clone()), ), + ("nsExecSocket", SdkValue::opt_text(state.ns_socket.clone())), ]), ) } +/// The socket a Sandbox's daemon serves its mount namespace on, when its +/// record names one. A record written before the socket existed names only the +/// diagnostic pid file; its daemon never bound a socket, so the Sandbox refuses +/// as an upgrade ([`ns_upgrade_required`]) instead of deriving a path that +/// would fail on every command. +fn saved_ns_socket(value: &SdkValue) -> Option { + value.get_text("nsExecSocket").map(str::to_owned) +} + +/// A live Sandbox whose record has the diagnostic pid file but no socket was +/// created by a daemon from before the socket existed. Its commands cannot +/// reach the private namespace, so they refuse with a recreate instruction +/// rather than silently running in the initial namespace. +fn ns_upgrade_required(value: &SdkValue) -> bool { + value.get_text("nsExecSocket").is_none() && value.get_text("nsExecPidFile").is_some() +} + /// A Sandbox whose container no longer runs it: the Engine no longer has the /// container, or has it stopped (it closed when idle, or Docker restarted), /// which lost its daemon and every mount in it. Only its records, and a @@ -476,7 +494,9 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { runtime, identity: CommandIdentity { uid: 501, gid: 20 }, // absent in a record written before the private-namespace prototype + ns_socket: saved_ns_socket(&value), ns_pid_file: value.get_text("nsExecPidFile").map(str::to_owned), + ns_upgrade_required: ns_upgrade_required(&value), controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), @@ -609,3 +629,80 @@ pub(crate) fn save_process(state: &SandboxState, id: &str, p: &Process) -> Resul ]), ) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::registry::mount_namespace; + use layerfs_api::internal::{exec_arguments, MountNamespace}; + use layerfs_sandbox::CommandIdentity; + + fn wrapped(record: &SdkValue, directory: &str) -> Vec { + let socket = saved_ns_socket(record); + let namespace: Option> = mount_namespace(socket.as_deref()); + exec_arguments( + namespace, + CommandIdentity { uid: 501, gid: 20 }, + directory, + vec!["bash".into(), "-c".into(), "printf ordinary".into()], + ) + } + + fn socket_record() -> SdkValue { + SdkValue::map([ + ( + "nsExecSocket", + SdkValue::text("/layerfs-local/ns-exec.sock"), + ), + ("nsExecPidFile", SdkValue::text("/layerfs-local/daemon.pid")), + ]) + } + + #[test] + fn a_saved_private_namespace_wraps_the_command() { + assert_eq!( + wrapped(&socket_record(), "/workspaces/1"), + [ + "/usr/local/bin/layerfs-daemon", + "ns-exec", + "--socket", + "/layerfs-local/ns-exec.sock", + "--directory", + "/workspaces/1", + "--uid", + "501", + "--gid", + "20", + "--", + "bash", + "-c", + "printf ordinary", + ] + ); + } + + #[test] + fn an_older_record_requires_a_recreated_sandbox() { + let record = + SdkValue::map([("nsExecPidFile", SdkValue::text("/layerfs-local/daemon.pid"))]); + // No socket is derived for a daemon that never bound one: the node + // refuses the exec as an upgrade instead of running unwrapped. + assert_eq!(saved_ns_socket(&record), None); + assert!(ns_upgrade_required(&record)); + assert_eq!( + wrapped(&record, "/workspaces/1"), + ["bash", "-c", "printf ordinary"] + ); + } + + #[test] + fn a_sandbox_without_a_private_namespace_keeps_the_command() { + let record = SdkValue::map([("id", SdkValue::text("1"))]); + assert_eq!(saved_ns_socket(&record), None); + assert!(!ns_upgrade_required(&record)); + assert_eq!( + wrapped(&record, "/workspaces/1"), + ["bash", "-c", "printf ordinary"] + ); + } +} diff --git a/engine/crates/layerfs-node/src/exec.rs b/engine/crates/layerfs-node/src/exec.rs index d0782e28d9..66797c2738 100644 --- a/engine/crates/layerfs-node/src/exec.rs +++ b/engine/crates/layerfs-node/src/exec.rs @@ -34,7 +34,7 @@ fn run( vec![HOME.to_owned()], directory.to_owned(), false, - state.ns_pid_file.as_deref(), + state.mount_namespace(), ) .map_err(failed)?; let attached = created.start().map_err(failed)?; @@ -59,6 +59,15 @@ fn run( /// `stdoutBytes` and `stderrBytes` sizes. pub(crate) fn exec(args: &SdkValue) -> Outcome { let state = sandbox(args)?; + if state.ns_upgrade_required { + return Err(refused( + "Unsupported", + format!( + "Sandbox {} was created by a daemon that does not serve its private mount namespace; close it and create a new Sandbox to run commands in its Workspaces", + state.id + ), + )); + } let path = text(args, "path")?; let command = text(args, "command")?; let directory = state.directory(path)?; diff --git a/engine/crates/layerfs-node/src/registry.rs b/engine/crates/layerfs-node/src/registry.rs index fea68a2283..d8480c295d 100644 --- a/engine/crates/layerfs-node/src/registry.rs +++ b/engine/crates/layerfs-node/src/registry.rs @@ -8,6 +8,7 @@ //! binding) also keeps the background processes it started; the daemon holds //! no process record. use crate::job::refused; +use layerfs_api::internal::MountNamespace; use layerfs_api::MountedWorkspace; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; use layerfs_bridge::provision::StoreManifest; @@ -47,14 +48,24 @@ pub(crate) struct SandboxState { pub(crate) directory: std::path::PathBuf, pub(crate) runtime: Sandbox, pub(crate) identity: CommandIdentity, - /// The pid file of a daemon that serves from a private mount namespace, or + /// The socket of a daemon that serves from a private mount namespace, or /// `None` for one whose mounts are in the container's initial namespace. + pub(crate) ns_socket: Option, + /// The daemon's diagnostic pid file; evidence, not an exec route. pub(crate) ns_pid_file: Option, + /// A record whose daemon predates the socket: this live Sandbox cannot run + /// a private-namespace command and must be recreated. + pub(crate) ns_upgrade_required: bool, pub(crate) controls: crate::pool::Controls, pub(crate) mounts: Mutex>, pub(crate) processes: Mutex>, } +/// The mount namespace a command must enter, from a daemon that serves one. +pub(crate) fn mount_namespace(socket: Option<&str>) -> Option> { + socket.map(|socket| MountNamespace { socket }) +} + /// The live counts of each Init still running, by project name. pub(crate) static INITS: LazyLock>>> = LazyLock::new(Default::default); @@ -141,6 +152,12 @@ pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, Sd } impl SandboxState { + /// Where this Sandbox's commands get the daemon's mount namespace, if it + /// serves from a private one. + pub(crate) fn mount_namespace(&self) -> Option> { + mount_namespace(self.ns_socket.as_deref()) + } + /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { self.mounted(path, |mount| mount.workspace.bound.token) diff --git a/engine/crates/layerfs-node/src/sandbox.rs b/engine/crates/layerfs-node/src/sandbox.rs index cc509dc181..c04fbf3561 100644 --- a/engine/crates/layerfs-node/src/sandbox.rs +++ b/engine/crates/layerfs-node/src/sandbox.rs @@ -199,6 +199,9 @@ fn start_claimed( let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; let mut daemon_setup = setup(daemon.private, controller.public, private_mount_ns(args)); + let ns_socket = daemon_setup + .mount_ns_socket() + .map(|path| path.to_string_lossy().into_owned()); let ns_pid_file = daemon_setup .mount_ns_pid_file() .map(|path| path.to_string_lossy().into_owned()); @@ -259,7 +262,9 @@ fn start_claimed( directory: saved_directory.to_owned(), runtime, identity, + ns_socket, ns_pid_file, + ns_upgrade_required: false, controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs index 0b1ceaa8a6..9143288f32 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs @@ -28,7 +28,9 @@ impl SandboxRequest { w.write_all(b":")?; quoted(w, value)?; } - w.write_all(b"},\"User\":\"0:0\",\"Entrypoint\":[\"/usr/local/bin/layerfs-daemon\"],\"Cmd\":[\"--config\",\"/layerfs-local/config/daemon.setup\"],\"Tty\":false,\"ExposedPorts\":{")?; + w.write_all(b"},\"User\":\"0:0\",\"Entrypoint\":[")?; + quoted(w, crate::DAEMON_ENTRY)?; + w.write_all(b"],\"Cmd\":[\"--config\",\"/layerfs-local/config/daemon.setup\"],\"Tty\":false,\"ExposedPorts\":{")?; quoted(w, &format!("{}/tcp", self.port))?; w.write_all(b":{}},\"HostConfig\":{\"Init\":true,\"RestartPolicy\":{\"Name\":\"no\"},\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"10m\",\"max-file\":\"2\"}},\"Privileged\":false,\"CapAdd\":[\"CAP_SYS_ADMIN\"],\"Devices\":[{\"PathOnHost\":\"/dev/fuse\",\"PathInContainer\":\"/dev/fuse\",\"CgroupPermissions\":\"rwm\"}],\"SecurityOpt\":[\"no-new-privileges=true\",\"apparmor=unconfined\",")?; quoted(w, SECCOMP)?; diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs index 9ee9ae628d..aac8261e77 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs @@ -4,6 +4,7 @@ use super::{ SandboxRequest, }; use crate::RuntimeError; +use crate::DAEMON_ENTRY; use std::io::Read; /// The container's whole system-call filter. It denies creation of a user /// namespace (`CLONE_NEWUSER` to `unshare` or `clone`; `clone3` answers @@ -52,7 +53,7 @@ pub(super) fn config( valid &= !j.boolean()?; } else if key.equals("Entrypoint") { once(&mut bits, 4)?; - valid &= strings(j, &["/usr/local/bin/layerfs-daemon"])?; + valid &= strings(j, &[DAEMON_ENTRY])?; } else if key.equals("Cmd") { once(&mut bits, 8)?; valid &= strings(j, &["--config", "/layerfs-local/config/daemon.setup"])?; diff --git a/engine/crates/layerfs-sandbox/src/lib.rs b/engine/crates/layerfs-sandbox/src/lib.rs index d1b8a35e44..4a40836727 100644 --- a/engine/crates/layerfs-sandbox/src/lib.rs +++ b/engine/crates/layerfs-sandbox/src/lib.rs @@ -1,5 +1,10 @@ //! Ordinary runtime operations, separate from filesystem admission and data ownership. #![forbid(unsafe_code)] + +/// The daemon's entry point inside its Sandbox container, as the deployment +/// installs it. The create request, the observed-topology check and the +/// `ns-exec` helper argv all name this one string. +pub const DAEMON_ENTRY: &str = "/usr/local/bin/layerfs-daemon"; #[cfg(unix)] pub mod backend; mod types; From 386f1b2272159de8c84f33129eb7c3c56377eaaa Mon Sep 17 00:00:00 2001 From: rain Date: Thu, 29 Oct 2026 05:57:38 +0800 Subject: [PATCH 3/3] style(sandbox): pass fmt, clippy and the product boundary guard CI failed in "fmt and guards" and in clippy. Formatting itself was already clean; the guard step and the clippy build were not. - clippy: the engine_streams example still built an ExecRequest without the new `root_launcher` field, so `--all-targets` could not compile; the example keeps its previous non-root launch. `ns_exec::parse` returned a four-tuple that tripped `clippy::type_complexity`, so it now returns a named `Parsed` value with the same fields, refusals and order. - guard: product source may not carry inline tests, and a `lib.rs` carries no definitions. The `daemon_setup` unit tests moved unchanged to `tests/private_mount_ns.rs`; the API argv-wrapping proof now runs through the public `WorkspaceApi::exec` against a private-namespace Sandbox; the helper's argv grammar is proven through the real `ns-exec` subprocess (`tests/ns_exec.rs`), where the socket-message and peer-admission proofs already covered the `ns_fd` predicates it used to unit-test. `DAEMON_ENTRY` moved from `lib.rs` to `types/deployment.rs`, re-exported at the same path. No behaviour changes: same argv, same refusals, same launch identity, same configuration bytes and same public paths. Verified: `cargo fmt --all -- --check`, the boundary and fuser guards and their unit tests, `cargo clippy --workspace --exclude layerfs-node --all-targets -- -D warnings`, `cargo check` on the four touched crates, and the daemon/sandbox tests (`ns_exec` 12/12 with FUSE, `engine_protocol` 17/17, `private_mount_ns` 2/2) in the pinned rust:1.85.1 container. --- engine/crates/layerfs-api/src/sandbox/exec.rs | 51 --------- .../layerfs-api/tests/sandbox_operations.rs | 54 +++++++++ .../crates/layerfs-bridge/src/daemon_setup.rs | 66 ----------- .../layerfs-bridge/tests/private_mount_ns.rs | 65 +++++++++++ .../layerfs-daemon/src/application/ns_exec.rs | 104 +++++------------- .../layerfs-daemon/src/application/ns_fd.rs | 20 ---- engine/crates/layerfs-daemon/tests/ns_exec.rs | 32 ++++++ .../examples/engine_streams.rs | 2 +- engine/crates/layerfs-sandbox/src/lib.rs | 6 +- .../layerfs-sandbox/src/types/deployment.rs | 6 + .../crates/layerfs-sandbox/src/types/mod.rs | 2 + 11 files changed, 189 insertions(+), 219 deletions(-) create mode 100644 engine/crates/layerfs-bridge/tests/private_mount_ns.rs create mode 100644 engine/crates/layerfs-sandbox/src/types/deployment.rs diff --git a/engine/crates/layerfs-api/src/sandbox/exec.rs b/engine/crates/layerfs-api/src/sandbox/exec.rs index e31c734098..fbfc08b895 100644 --- a/engine/crates/layerfs-api/src/sandbox/exec.rs +++ b/engine/crates/layerfs-api/src/sandbox/exec.rs @@ -100,54 +100,3 @@ pub fn exec_arguments( wrapped.extend(arguments); wrapped } - -#[cfg(test)] -mod tests { - use super::*; - - fn identity() -> CommandIdentity { - CommandIdentity { uid: 501, gid: 20 } - } - - fn command() -> Vec { - vec!["bash".into(), "-c".into(), "printf ordinary".into()] - } - - #[test] - fn a_sandbox_without_a_private_namespace_keeps_the_command_as_it_is() { - assert_eq!( - exec_arguments(None, identity(), "/workspaces/1", command()), - command() - ); - } - - #[test] - fn a_private_namespace_sandbox_runs_the_daemon_helper_over_its_socket() { - assert_eq!( - exec_arguments( - Some(MountNamespace { - socket: "/layerfs-local/ns-exec.sock", - }), - identity(), - "/workspaces/1", - command(), - ), - [ - "/usr/local/bin/layerfs-daemon", - "ns-exec", - "--socket", - "/layerfs-local/ns-exec.sock", - "--directory", - "/workspaces/1", - "--uid", - "501", - "--gid", - "20", - "--", - "bash", - "-c", - "printf ordinary", - ] - ); - } -} diff --git a/engine/crates/layerfs-api/tests/sandbox_operations.rs b/engine/crates/layerfs-api/tests/sandbox_operations.rs index 7b04db1ebf..71ab202910 100644 --- a/engine/crates/layerfs-api/tests/sandbox_operations.rs +++ b/engine/crates/layerfs-api/tests/sandbox_operations.rs @@ -62,7 +62,18 @@ impl Fixture { fn new(tail: Vec>) -> Self { Self::new_for(None, tail) } + /// A Sandbox whose daemon serves from a private mount namespace. + fn new_private(tail: Vec>) -> Self { + Self::build(None, tail, true) + } fn new_for(project: Option<&layerfs_api::SealedProject>, tail: Vec>) -> Self { + Self::build(project, tail, false) + } + fn build( + project: Option<&layerfs_api::SealedProject>, + tail: Vec>, + private_mount_ns: bool, + ) -> Self { let existing = project.is_some(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); @@ -133,6 +144,7 @@ impl Fixture { )); fs::write(&binary, b"original executable input").unwrap(); let mut deployment_setup = setup([32; 32], duplex::public_key(&[31; 32]).unwrap()); + deployment_setup.private_mount_ns = private_mount_ns; if let Some(project) = project { let mut manifest = project.manifest.clone(); manifest.daemon_sqlite = Some("3.51.0".into()); @@ -335,6 +347,48 @@ fn workspace_exec_returns_original_runtime_owner_at_the_mount_with_no_daemon_reg .any(|call| call.windows(12).any(|bytes| bytes == b"/exec/start "))); } #[test] +fn workspace_exec_wraps_the_command_for_a_private_mount_namespace_sandbox() { + const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; + let mut fixture = Fixture::new_private(vec![ + fixed(201, format!("{{\"Id\":\"{EID}\"}}").as_bytes()), + fixed(204, b""), + fixed(204, b""), + ]); + let created = WorkspaceApi::exec( + fixture.owner.as_ref().unwrap(), + &mounted(), + vec!["/bin/pwd".into()], + vec![], + false, + ) + .unwrap(); + assert_eq!(created.request().directory, "/workspaces/selected-mount"); + assert_eq!( + created.request().arguments, + [ + "/usr/local/bin/layerfs-daemon", + "ns-exec", + "--socket", + "/layerfs-local/ns-exec.sock", + "--directory", + "/workspaces/selected-mount", + "--uid", + "501", + "--gid", + "20", + "--", + "/bin/pwd", + ] + ); + // Only root holds the capability the helper's setns needs. + assert!(created.request().root_launcher); + fixture.api.close(fixture.owner.take().unwrap()).unwrap(); + let calls = fixture.calls(); + let request = std::str::from_utf8(&calls[6]).unwrap(); + assert!(request.starts_with(&format!("POST /v1.54/containers/{CID}/exec "))); + assert!(request.contains(r#""User":"0:0""#), "request: {request}"); +} +#[test] fn workspace_exec_failure_keeps_exact_original_request_and_container() { let mut fixture = Fixture::new(vec![ fixed(409, b"original exec refusal"), diff --git a/engine/crates/layerfs-bridge/src/daemon_setup.rs b/engine/crates/layerfs-bridge/src/daemon_setup.rs index bb9e0886be..64086c9abb 100644 --- a/engine/crates/layerfs-bridge/src/daemon_setup.rs +++ b/engine/crates/layerfs-bridge/src/daemon_setup.rs @@ -211,69 +211,3 @@ impl fmt::Debug for DaemonSetup { .finish() } } - -#[cfg(test)] -mod tests { - use super::*; - - fn setup(overlay: &str, private_mount_ns: bool) -> DaemonSetup { - DaemonSetup { - listen: "127.0.0.1:0".into(), - private_key: [1; 32], - control_peer: [2; 32], - store: "/layerfs-store/global/store.sqlite".into(), - overlay: overlay.into(), - mounts: "/workspaces".into(), - command_uid: 501, - command_gid: 20, - limits: DaemonLimits { - connections: 4, - handshake_ms: 3000, - cache_bytes: 64 * 1024, - namespaces: 8, - pager_kib: 1024, - serial_low_water: 0, - idle_exit_s: 0, - }, - existing_store: None, - private_mount_ns, - } - } - - #[test] - fn the_private_namespace_setting_survives_the_config_record() { - assert!(setup("/local/overlay/overlay.sqlite", false) - .mount_ns_pid_file() - .is_none()); - assert!(setup("/local/overlay/overlay.sqlite", false) - .mount_ns_socket() - .is_none()); - let encoded = setup("/local/overlay/overlay.sqlite", true) - .encode() - .unwrap(); - let decoded = DaemonSetup::decode(&encoded).unwrap(); - assert!(decoded.private_mount_ns); - assert_eq!( - decoded.mount_ns_pid_file(), - Some(PathBuf::from("/local/daemon.pid")) - ); - assert_eq!( - decoded.mount_ns_socket(), - Some(PathBuf::from("/local/ns-exec.sock")) - ); - } - - #[test] - fn the_pid_file_is_beside_the_local_directory_above_the_overlay() { - assert_eq!( - setup("/layerfs-local/overlay/overlay.sqlite", true).mount_ns_pid_file(), - Some(PathBuf::from("/layerfs-local/daemon.pid")) - ); - assert_eq!( - setup("/layerfs-local/overlay/overlay.sqlite", true).mount_ns_socket(), - Some(PathBuf::from("/layerfs-local/ns-exec.sock")) - ); - // An Overlay path with no local directory cannot name a pid file. - assert!(setup("/overlay.sqlite", true).encode().is_err()); - } -} diff --git a/engine/crates/layerfs-bridge/tests/private_mount_ns.rs b/engine/crates/layerfs-bridge/tests/private_mount_ns.rs new file mode 100644 index 0000000000..d148024c35 --- /dev/null +++ b/engine/crates/layerfs-bridge/tests/private_mount_ns.rs @@ -0,0 +1,65 @@ +//! The opt-in private mount namespace in the daemon's private config record. + +use layerfs_bridge::daemon_setup::{DaemonLimits, DaemonSetup}; +use std::path::PathBuf; + +fn setup(overlay: &str, private_mount_ns: bool) -> DaemonSetup { + DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: [1; 32], + control_peer: [2; 32], + store: "/layerfs-store/global/store.sqlite".into(), + overlay: overlay.into(), + mounts: "/workspaces".into(), + command_uid: 501, + command_gid: 20, + limits: DaemonLimits { + connections: 4, + handshake_ms: 3000, + cache_bytes: 64 * 1024, + namespaces: 8, + pager_kib: 1024, + serial_low_water: 0, + idle_exit_s: 0, + }, + existing_store: None, + private_mount_ns, + } +} + +#[test] +fn the_private_namespace_setting_survives_the_config_record() { + assert!(setup("/local/overlay/overlay.sqlite", false) + .mount_ns_pid_file() + .is_none()); + assert!(setup("/local/overlay/overlay.sqlite", false) + .mount_ns_socket() + .is_none()); + let encoded = setup("/local/overlay/overlay.sqlite", true) + .encode() + .unwrap(); + let decoded = DaemonSetup::decode(&encoded).unwrap(); + assert!(decoded.private_mount_ns); + assert_eq!( + decoded.mount_ns_pid_file(), + Some(PathBuf::from("/local/daemon.pid")) + ); + assert_eq!( + decoded.mount_ns_socket(), + Some(PathBuf::from("/local/ns-exec.sock")) + ); +} + +#[test] +fn the_pid_file_is_beside_the_local_directory_above_the_overlay() { + assert_eq!( + setup("/layerfs-local/overlay/overlay.sqlite", true).mount_ns_pid_file(), + Some(PathBuf::from("/layerfs-local/daemon.pid")) + ); + assert_eq!( + setup("/layerfs-local/overlay/overlay.sqlite", true).mount_ns_socket(), + Some(PathBuf::from("/layerfs-local/ns-exec.sock")) + ); + // An Overlay path with no local directory cannot name a pid file. + assert!(setup("/overlay.sqlite", true).encode().is_err()); +} diff --git a/engine/crates/layerfs-daemon/src/application/ns_exec.rs b/engine/crates/layerfs-daemon/src/application/ns_exec.rs index f69b5d7187..578ab83309 100644 --- a/engine/crates/layerfs-daemon/src/application/ns_exec.rs +++ b/engine/crates/layerfs-daemon/src/application/ns_exec.rs @@ -40,7 +40,12 @@ pub(super) fn main(args: &[OsString]) -> std::process::ExitCode { } fn run(args: &[OsString]) -> Result<(), String> { - let (socket, directory, identity, command) = parse(args)?; + let Parsed { + socket, + directory, + identity, + command, + } = parse(args)?; // The daemon is not dumpable, so a Sandbox root (CAP_SYS_ADMIN without // CAP_SYS_PTRACE) cannot open its `/proc//ns/mnt`; the daemon sends // the descriptor over this socket instead. @@ -63,11 +68,18 @@ fn run(args: &[OsString]) -> Result<(), String> { Err(format!("exec {}: {error}", program.to_string_lossy())) } +/// One parsed helper invocation: the daemon socket, the optional directory to +/// enter after joining, the ordinary identity to drop to and the command. +struct Parsed { + socket: String, + directory: Option, + identity: Option<(u32, u32)>, + command: Vec, +} + /// `--socket [--directory ] [--uid --gid ] -- /// [args...]`. -fn parse( - args: &[OsString], -) -> Result<(String, Option, Option<(u32, u32)>, Vec), String> { +fn parse(args: &[OsString]) -> Result { let mut rest = args.iter(); let mut socket = None; let mut directory = None; @@ -98,14 +110,20 @@ fn parse( if command.is_empty() { return Err(USAGE.to_owned()); } - match (uid, gid) { + let identity = match (uid, gid) { (Some(0), _) | (_, Some(0)) => { - Err("--uid and --gid must name the non-root command identity".to_owned()) + return Err("--uid and --gid must name the non-root command identity".to_owned()) } - (Some(uid), Some(gid)) => Ok((socket, directory, Some((uid, gid)), command)), - (None, None) => Ok((socket, directory, None, command)), - _ => Err("--uid and --gid must be given together".to_owned()), - } + (Some(uid), Some(gid)) => Some((uid, gid)), + (None, None) => None, + _ => return Err("--uid and --gid must be given together".to_owned()), + }; + Ok(Parsed { + socket, + directory, + identity, + command, + }) } fn path(value: Option<&OsString>) -> Result { @@ -131,69 +149,3 @@ fn drop_to(uid: u32, gid: u32) -> Result<(), String> { setgid(Gid::from_raw(gid)).map_err(|error| format!("drop to gid {gid}: {error}"))?; setuid(Uid::from_raw(uid)).map_err(|error| format!("drop to uid {uid}: {error}")) } - -#[cfg(test)] -mod tests { - use super::*; - - fn socket_command() -> Vec { - ["--socket", "/s", "--", "true"] - .into_iter() - .map(OsString::from) - .collect() - } - - #[test] - fn the_command_identity_cannot_be_root() { - let args: Vec = ["--socket", "/s", "--uid", "0", "--gid", "0", "--", "true"] - .into_iter() - .map(OsString::from) - .collect(); - assert!(parse(&args).is_err()); - } - - #[test] - fn a_command_needs_a_separator_and_a_program() { - assert!(parse(&[OsString::from("--socket"), "/s".into()]).is_err()); - assert!(parse(&[OsString::from("--socket"), "/s".into(), "--".into()]).is_err()); - } - - #[test] - fn a_socket_names_the_namespace() { - let (socket, directory, identity, command) = parse(&socket_command()).unwrap(); - assert_eq!(socket, "/s"); - assert_eq!(directory, None); - assert_eq!(identity, None); - assert_eq!(command, ["true"]); - // No socket, no namespace: the helper refuses. - let none: Vec = ["--", "true"].into_iter().map(OsString::from).collect(); - assert!(parse(&none).is_err()); - } - - #[test] - fn the_identity_is_all_or_nothing() { - let args = [ - "--socket".into(), - "/s".into(), - "--uid".into(), - "501".into(), - "--".into(), - "true".into(), - ]; - assert!(parse(&args).is_err()); - } - - #[test] - fn a_parsed_command_keeps_its_arguments() { - let args: Vec = ["--socket", "/s", "--uid", "501", "--gid", "20", "--"] - .into_iter() - .map(OsString::from) - .chain(["bash".into(), "-c".into(), "printf hello".into()]) - .collect(); - let (socket, directory, identity, command) = parse(&args).unwrap(); - assert_eq!(socket, "/s"); - assert_eq!(directory, None); - assert_eq!(identity, Some((501, 20))); - assert_eq!(command, ["bash", "-c", "printf hello"]); - } -} diff --git a/engine/crates/layerfs-daemon/src/application/ns_fd.rs b/engine/crates/layerfs-daemon/src/application/ns_fd.rs index a3ef6c53b0..fb77f972ae 100644 --- a/engine/crates/layerfs-daemon/src/application/ns_fd.rs +++ b/engine/crates/layerfs-daemon/src/application/ns_fd.rs @@ -157,23 +157,3 @@ pub fn receive(path: &Path) -> io::Result { } Ok(namespace) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn only_root_is_admitted() { - assert!(admit_uid(0)); - assert!(!admit_uid(501)); - assert!(!admit_uid(65534)); - } - - #[test] - fn a_peer_must_be_in_this_pid_namespace() { - assert!(same_pid_namespace(std::process::id() as i32)); - assert!(!same_pid_namespace(0)); - assert!(!same_pid_namespace(-1)); - assert!(!same_pid_namespace(i32::MAX)); - } -} diff --git a/engine/crates/layerfs-daemon/tests/ns_exec.rs b/engine/crates/layerfs-daemon/tests/ns_exec.rs index e7189a41b9..cfdb0eb662 100644 --- a/engine/crates/layerfs-daemon/tests/ns_exec.rs +++ b/engine/crates/layerfs-daemon/tests/ns_exec.rs @@ -368,6 +368,38 @@ fn a_helper_without_a_namespace_source_refuses_and_runs_nothing() { fs::remove_dir_all(&directory).unwrap(); } +#[test] +fn malformed_helper_argv_refuses_with_a_reason_and_runs_nothing() { + let cases: [(&str, &[&str]); 5] = [ + ("usage: layerfs-daemon ns-exec", &["--socket", "/s"]), + ("usage: layerfs-daemon ns-exec", &["--socket", "/s", "--"]), + ( + "--uid and --gid must be given together", + &["--socket", "/s", "--uid", "501", "--", "true"], + ), + ( + "--uid and --gid must name the non-root command identity", + &["--socket", "/s", "--uid", "0", "--gid", "0", "--", "true"], + ), + ( + "--uid is not a number", + &[ + "--socket", "/s", "--uid", "ordinary", "--gid", "20", "--", "true", + ], + ), + ]; + for (reason, argv) in cases { + let output = Command::new(BINARY) + .arg("ns-exec") + .args(argv) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(125), "argv: {argv:?}"); + let err = String::from_utf8_lossy(&output.stderr); + assert!(err.contains(reason), "argv: {argv:?}, stderr: {err}"); + } +} + #[test] fn a_helper_in_another_pid_namespace_is_refused_by_the_daemons_socket() { let directory = unique_directory("foreign-pid-ns"); diff --git a/engine/crates/layerfs-sandbox/examples/engine_streams.rs b/engine/crates/layerfs-sandbox/examples/engine_streams.rs index 16a269300b..f59575524f 100644 --- a/engine/crates/layerfs-sandbox/examples/engine_streams.rs +++ b/engine/crates/layerfs-sandbox/examples/engine_streams.rs @@ -34,7 +34,7 @@ fn main() { let container = ContainerId::parse(&args.next().expect("full owned container ID")).unwrap(); assert!(args.next().is_none()); let docker = Docker::new(socket, Duration::from_secs(3)).unwrap(); - let security=docker.create_exec(container,ExecRequest{identity:CommandIdentity{uid:501,gid:20},arguments:vec!["/bin/bash".into(),"-c".into(),"id -u; id -g; awk '/^(Groups|CapEff|CapAmb|NoNewPrivs):/{print}' /proc/self/status".into()],environment:vec![],directory:"/tmp".into(),stdin:false}).unwrap(); + let security=docker.create_exec(container,ExecRequest{identity:CommandIdentity{uid:501,gid:20},arguments:vec!["/bin/bash".into(),"-c".into(),"id -u; id -g; awk '/^(Groups|CapEff|CapAmb|NoNewPrivs):/{print}' /proc/self/status".into()],environment:vec![],directory:"/tmp".into(),stdin:false,root_launcher:false}).unwrap(); let (_, output, handle) = security.start().unwrap().into_parts().unwrap(); let mut identity = Vec::new(); output.copy_to(&mut identity, &mut io::sink()).unwrap(); diff --git a/engine/crates/layerfs-sandbox/src/lib.rs b/engine/crates/layerfs-sandbox/src/lib.rs index 4a40836727..9c560553c0 100644 --- a/engine/crates/layerfs-sandbox/src/lib.rs +++ b/engine/crates/layerfs-sandbox/src/lib.rs @@ -1,14 +1,10 @@ //! Ordinary runtime operations, separate from filesystem admission and data ownership. #![forbid(unsafe_code)] -/// The daemon's entry point inside its Sandbox container, as the deployment -/// installs it. The create request, the observed-topology check and the -/// `ns-exec` helper argv all name this one string. -pub const DAEMON_ENTRY: &str = "/usr/local/bin/layerfs-daemon"; #[cfg(unix)] pub mod backend; mod types; pub use types::{ CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, PendingRequest, - RuntimeError, WireFailure, + RuntimeError, WireFailure, DAEMON_ENTRY, }; diff --git a/engine/crates/layerfs-sandbox/src/types/deployment.rs b/engine/crates/layerfs-sandbox/src/types/deployment.rs new file mode 100644 index 0000000000..71859fda03 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/deployment.rs @@ -0,0 +1,6 @@ +//! The deployment paths the Engine and the daemon both name. + +/// The daemon's entry point inside its Sandbox container, as the deployment +/// installs it. The create request, the observed-topology check and the +/// `ns-exec` helper argv all name this one string. +pub const DAEMON_ENTRY: &str = "/usr/local/bin/layerfs-daemon"; diff --git a/engine/crates/layerfs-sandbox/src/types/mod.rs b/engine/crates/layerfs-sandbox/src/types/mod.rs index 69827343cb..f87d8340cb 100644 --- a/engine/crates/layerfs-sandbox/src/types/mod.rs +++ b/engine/crates/layerfs-sandbox/src/types/mod.rs @@ -1,8 +1,10 @@ //! Exact ordinary runtime identities, independent process/output knowledge and failures. +mod deployment; mod execution; mod failure; mod identity; mod transfer; +pub use deployment::DAEMON_ENTRY; pub use execution::{CommandIdentity, ExecInspection}; pub(crate) use failure::STOPPED; pub use failure::{ErrorDiagnostic, RuntimeError, WireFailure};