diff --git a/rs/moq-tokio/src/tls.rs b/rs/moq-tokio/src/tls.rs index a28dcfcd7cc..b90dbd6c8cd 100644 --- a/rs/moq-tokio/src/tls.rs +++ b/rs/moq-tokio/src/tls.rs @@ -2810,9 +2810,9 @@ mod tests { fn reloadable_client_roots_disable_server_resumption() { let (ca_a, server_cert, server_key, client_cert, client_key) = signed_certificates(); let (ca_b, _, _, _, _) = signed_certificates(); - // A dedicated directory, not `/tmp`: the watcher registers the parent, so - // unrelated churn there fires a reload that can read `ca_a` and store it - // after this test's own `reload()` installed `ca_b`. + // A dedicated directory, not `/tmp`: the watcher matches by file name within + // the parent, so another test's `root.pem` there fires a reload that can read + // `ca_a` and store it after this test's own `reload()` installed `ca_b`. let dir = tempfile::TempDir::new().unwrap(); let root_path = dir.path().join("root.pem"); std::fs::write(&root_path, ca_a).unwrap(); diff --git a/rs/moq-tokio/src/watch.rs b/rs/moq-tokio/src/watch.rs index 6ee6cc6d81f..d361bd7c936 100644 --- a/rs/moq-tokio/src/watch.rs +++ b/rs/moq-tokio/src/watch.rs @@ -5,8 +5,7 @@ use std::path::{Path, PathBuf}; use notify::Watcher; use tokio::sync::mpsc; -/// Watches a set of files and resolves whenever something changes in their -/// directories. +/// Watches a set of files and resolves whenever one of them changes. /// /// Reacting to the filesystem (rather than a SIGHUP/SIGUSR1) is what lets /// cert-manager, Kubernetes secret mounts, and `mv`-into-place rotate files with @@ -16,7 +15,7 @@ use tokio::sync::mpsc; /// cert-manager, and K8s secret mounts replace files by atomic rename or symlink /// swap, which changes the inode (and, for the K8s `..data` symlink, fires on the /// directory without ever naming the file), so a watch set directly on the path -/// would be missed. +/// would be missed. Events for anything else in those directories are ignored. pub struct Files { // Holds the OS watcher alive; dropping it stops events. _watcher: notify::RecommendedWatcher, @@ -31,8 +30,8 @@ impl Files { /// polling here. pub fn new(paths: &[PathBuf]) -> notify::Result { // A capacity-1 channel of unit wakeups coalesces the burst of raw events - // notify emits per change (and any unrelated churn in the directory): a - // full buffer already has a pending wakeup, so extra sends are dropped. + // notify emits per change: a full buffer already has a pending wakeup, so + // extra sends are dropped. let (tx, rx) = mpsc::channel(1); let watcher = callback(paths, move || { let _ = tx.try_send(()); @@ -44,9 +43,9 @@ impl Files { }) } - /// Resolve once the OS reports activity in a watched directory. The caller - /// reloads on return; reloads are idempotent, so the coarse "something - /// changed" granularity at worst costs an occasional redundant reload. + /// Resolve once the OS reports a change to a watched file. The caller reloads + /// on return; reloads are idempotent, so a redundant wakeup costs only a + /// redundant reload. pub async fn changed(&mut self) { // The sender lives inside `_watcher`, which we hold for `&mut self`, so the // channel can't be closed here. @@ -64,11 +63,11 @@ pub(crate) fn callback( paths: &[PathBuf], mut changed: impl FnMut() + Send + 'static, ) -> notify::Result { + let mut filter = Filter::new(paths); let mut watcher = notify::recommended_watcher(move |res: notify::Result| { let reload = match res { - Ok(event) => is_reload_trigger(&event.kind), - // A watcher error (e.g. inotify queue overflow) may mean we missed a - // real change, so reload to be safe. + Ok(event) => filter.matches(&event), + // A watcher error may mean we missed a real change, so reload to be safe. Err(_) => true, }; if reload { @@ -94,6 +93,78 @@ pub(crate) fn callback( Ok(watcher) } +/// Picks the events in the watched directories that concern the watched files. +/// +/// Anything else sharing a directory must be ignored: a relay logging into its +/// certificate directory otherwise reloads on every line, and each reload logs +/// another line, a self-sustaining storm. +struct Filter { + paths: Vec, + /// Each path with its directory canonicalized, as `locate` spells an event's path. + located: Vec, + /// Where each path currently resolves through its symlinks, `None` while it + /// doesn't resolve at all. + resolved: Vec>, +} + +impl Filter { + fn new(paths: &[PathBuf]) -> Self { + let paths = paths.to_vec(); + let located = paths.iter().filter_map(|p| locate(p)).collect(); + let resolved = resolve(&paths); + Self { + paths, + located, + resolved, + } + } + + fn matches(&mut self, event: ¬ify::Event) -> bool { + // The OS dropped events (e.g. an inotify queue overflow), so a real change + // may have gone unseen. + if event.need_rescan() { + return true; + } + if !is_reload_trigger(&event.kind) { + return false; + } + + let resolved = resolve(&self.paths); + + // An event names a watched file by its own path or, for a symlink, by the + // target it resolves to (an in-place write reports the target's name). + // Compared with the directory canonicalized, since backends spell it + // differently (FSEvents reports canonical paths). The file name check first + // skips that syscall for the unrelated churn that dominates these directories. + let watched = || self.located.iter().chain(resolved.iter().flatten()); + let named = event.paths.iter().any(|p| { + let name = p.file_name(); + name.is_some() + && watched().any(|f| f.file_name() == name) + && locate(p).is_some_and(|p| watched().any(|f| *f == p)) + }); + + // A Kubernetes secret mount links `cert.pem` to `..data/cert.pem` and rotates + // by swapping the `..data` symlink, so no event ever names the file. Catch + // that by where the paths resolve, refreshed on every event so a rotation + // that was named doesn't fire again on the next unrelated one. + let moved = resolved != self.resolved; + self.resolved = resolved; + + named || moved + } +} + +fn locate(path: &Path) -> Option { + let name = path.file_name()?; + let dir = path.parent().filter(|dir| !dir.as_os_str().is_empty()); + Some(std::fs::canonicalize(dir.unwrap_or(Path::new("."))).ok()?.join(name)) +} + +fn resolve(paths: &[PathBuf]) -> Vec> { + paths.iter().map(|p| std::fs::canonicalize(p).ok()).collect() +} + /// Whether a raw notify event reflects a real change that should trigger a reload. /// /// The reload path opens and reads the watched files, and notify's inotify backend @@ -166,4 +237,157 @@ mod tests { assert!(is_reload_trigger(&EventKind::Create(CreateKind::Any))); assert!(is_reload_trigger(&EventKind::Modify(ModifyKind::Any))); } + + fn event(kind: notify::EventKind, path: PathBuf) -> notify::Event { + notify::Event::new(kind).add_path(path) + } + + // A log written into the certificate directory must not reload: each reload + // logs a line, so reacting to it never stops. + #[test] + fn unrelated_files_do_not_trigger_reload() { + use notify::EventKind; + use notify::event::{AccessKind, AccessMode, CreateKind, DataChange, ModifyKind, RenameMode}; + + let dir = tempfile::TempDir::new().unwrap(); + let cert = dir.path().join("cert.pem"); + let key = dir.path().join("key.pem"); + let log = dir.path().join("relay.log"); + let mut filter = Filter::new(&[cert.clone(), key.clone()]); + + let write = EventKind::Modify(ModifyKind::Data(DataChange::Any)); + let closed = EventKind::Access(AccessKind::Close(AccessMode::Write)); + assert!(!filter.matches(&event(EventKind::Create(CreateKind::File), log.clone()))); + assert!(!filter.matches(&event(write, log.clone()))); + assert!(!filter.matches(&event(closed, log))); + + assert!(filter.matches(&event(closed, cert))); + let renamed = EventKind::Modify(ModifyKind::Name(RenameMode::To)); + assert!(filter.matches(&event(renamed, key))); + + // Dropped events may have hidden a rotation. + let rescan = notify::Event::new(EventKind::Other).set_flag(notify::event::Flag::Rescan); + assert!(filter.matches(&rescan)); + } + + // With the certificate and key in different directories, a file in one that + // shares the other's name is still unrelated. + #[test] + fn same_name_in_another_watched_directory_does_not_trigger_reload() { + use notify::EventKind; + use notify::event::{AccessKind, AccessMode}; + + let certs = tempfile::TempDir::new().unwrap(); + let keys = tempfile::TempDir::new().unwrap(); + let mut filter = Filter::new(&[certs.path().join("cert.pem"), keys.path().join("key.pem")]); + + let closed = EventKind::Access(AccessKind::Close(AccessMode::Write)); + assert!(!filter.matches(&event(closed, keys.path().join("cert.pem")))); + assert!(filter.matches(&event(closed, certs.path().join("cert.pem")))); + assert!(filter.matches(&event(closed, keys.path().join("key.pem")))); + } + + // A configured symlink whose target is rewritten in place reports the target's + // name, and nothing resolves anywhere new. + #[cfg(unix)] + #[test] + fn write_to_symlink_target_triggers_reload() { + use notify::EventKind; + use notify::event::{AccessKind, AccessMode}; + + let dir = tempfile::TempDir::new().unwrap(); + std::fs::write(dir.path().join("actual.pem"), "v1").unwrap(); + std::os::unix::fs::symlink("actual.pem", dir.path().join("cert.pem")).unwrap(); + let mut filter = Filter::new(&[dir.path().join("cert.pem")]); + + let closed = EventKind::Access(AccessKind::Close(AccessMode::Write)); + assert!(filter.matches(&event(closed, dir.path().join("actual.pem")))); + assert!(!filter.matches(&event(closed, dir.path().join("relay.log")))); + } + + // Drives a real inotify watch through a Kubernetes secret rotation while a log + // churns in the same directory. Each step blocks until its own events arrive, so + // the filter judges every event against the disk as it was at that step. + #[cfg(target_os = "linux")] + #[test] + fn kubernetes_rotation_reloads_and_log_writes_do_not() { + use std::os::unix::fs::symlink; + use std::sync::mpsc; + + // Lay out the mount like the kubelet: `tls.crt -> ..data/tls.crt` and + // `..data -> ..`. + let dir = tempfile::TempDir::new().unwrap(); + let path = |name: &str| dir.path().join(name); + std::fs::create_dir(path("..v1")).unwrap(); + std::fs::write(path("..v1/tls.crt"), "v1").unwrap(); + symlink("..v1", path("..data")).unwrap(); + symlink("..data/tls.crt", path("tls.crt")).unwrap(); + + let (tx, rx) = mpsc::channel(); + let mut watcher = match notify::recommended_watcher(tx) { + Ok(watcher) => watcher, + Err(err) if matches!(err.kind, notify::ErrorKind::MaxFilesWatch) => { + eprintln!("skipping kubernetes_rotation_reloads_and_log_writes_do_not: {err}"); + return; + } + Err(err) => panic!("failed to create watcher: {err}"), + }; + if let Err(err) = watcher.watch(dir.path(), notify::RecursiveMode::NonRecursive) { + if matches!(err.kind, notify::ErrorKind::MaxFilesWatch) { + eprintln!("skipping kubernetes_rotation_reloads_and_log_writes_do_not: {err}"); + return; + } + panic!("failed to watch: {err}"); + } + let mut filter = Filter::new(&[path("tls.crt")]); + + // Judge every event up to and including the first that `done` accepts. + let mut step = |done: &dyn Fn(¬ify::Event) -> bool| -> Vec { + let mut verdicts = Vec::new(); + loop { + // The awaited event is already on its way; a lost one hangs until the + // test runner's timeout. + let event = rx.recv().expect("watcher dropped").unwrap(); + verdicts.push(filter.matches(&event)); + if done(&event) { + return verdicts; + } + } + }; + let named = |name: &'static str| { + move |event: ¬ify::Event| event.paths.iter().any(|p| p.file_name() == Some(name.as_ref())) + }; + let closed = |name: &'static str| { + let named = named(name); + move |event: ¬ify::Event| { + use notify::event::{AccessKind, AccessMode}; + event.kind == notify::EventKind::Access(AccessKind::Close(AccessMode::Write)) && named(event) + } + }; + + std::fs::write(path("relay.log"), "reloading server certificates\n").unwrap(); + let verdicts = step(&closed("relay.log")); + assert!( + verdicts.iter().all(|reload| !reload), + "log write reloaded: {verdicts:?}" + ); + + // Rotate the way the kubelet does: write the new data, point `..data_tmp` at it, + // rename that over `..data`, then remove the old data. No event names `tls.crt`. + std::fs::create_dir(path("..v2")).unwrap(); + std::fs::write(path("..v2/tls.crt"), "v2").unwrap(); + symlink("..v2", path("..data_tmp")).unwrap(); + std::fs::rename(path("..data_tmp"), path("..data")).unwrap(); + std::fs::remove_dir_all(path("..v1")).unwrap(); + let verdicts = step(&named("..v1")); + let reloads = verdicts.iter().filter(|reload| **reload).count(); + assert_eq!(reloads, 1, "rotation should reload once: {verdicts:?}"); + + std::fs::write(path("relay.log"), "reloading server certificates\n").unwrap(); + let verdicts = step(&closed("relay.log")); + assert!( + verdicts.iter().all(|reload| !reload), + "log write reloaded: {verdicts:?}" + ); + } }