From a2096ff3e8e0cd4484bf9a1499c385b7f60e5a1c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:28:09 -0700 Subject: [PATCH 1/3] fix(tokio): reload TLS only when the watched files change The certificate watcher watches each file's parent directory and reloaded on any create or modify there. A relay logging into its certificate directory reloaded on every line, and each reload logged another line: about 29k reloads in 3s locally, and ~60k in a smoke run. Filter events to the watched file names, plus a change in where a watched path resolves through its symlinks, so the Kubernetes `..data` swap (which never names the file) still reloads. Also reload on notify's Rescan flag, which is how an inotify queue overflow arrives; it was being ignored. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/tls.rs | 6 +- rs/moq-tokio/src/watch.rs | 195 +++++++++++++++++++++++++++++++++++--- 2 files changed, 187 insertions(+), 14 deletions(-) diff --git a/rs/moq-tokio/src/tls.rs b/rs/moq-tokio/src/tls.rs index 79f56ecef66..78dbb6fca60 100644 --- a/rs/moq-tokio/src/tls.rs +++ b/rs/moq-tokio/src/tls.rs @@ -2810,9 +2810,9 @@ mod tests { fn reloadable_client_roots_disable_server_resumption() { let (ca_a, server_cert, server_key, client_cert, client_key) = signed_certificates(); let (ca_b, _, _, _, _) = signed_certificates(); - // A dedicated directory, not `/tmp`: the watcher registers the parent, so - // unrelated churn there fires a reload that can read `ca_a` and store it - // after this test's own `reload()` installed `ca_b`. + // A dedicated directory, not `/tmp`: the watcher matches by file name within + // the parent, so another test's `root.pem` there fires a reload that can read + // `ca_a` and store it after this test's own `reload()` installed `ca_b`. let dir = tempfile::TempDir::new().unwrap(); let root_path = dir.path().join("root.pem"); std::fs::write(&root_path, ca_a).unwrap(); diff --git a/rs/moq-tokio/src/watch.rs b/rs/moq-tokio/src/watch.rs index 6ee6cc6d81f..0b60f3f7e1f 100644 --- a/rs/moq-tokio/src/watch.rs +++ b/rs/moq-tokio/src/watch.rs @@ -5,8 +5,7 @@ use std::path::{Path, PathBuf}; use notify::Watcher; use tokio::sync::mpsc; -/// Watches a set of files and resolves whenever something changes in their -/// directories. +/// Watches a set of files and resolves whenever one of them changes. /// /// Reacting to the filesystem (rather than a SIGHUP/SIGUSR1) is what lets /// cert-manager, Kubernetes secret mounts, and `mv`-into-place rotate files with @@ -16,7 +15,7 @@ use tokio::sync::mpsc; /// cert-manager, and K8s secret mounts replace files by atomic rename or symlink /// swap, which changes the inode (and, for the K8s `..data` symlink, fires on the /// directory without ever naming the file), so a watch set directly on the path -/// would be missed. +/// would be missed. Events for anything else in those directories are ignored. pub struct Files { // Holds the OS watcher alive; dropping it stops events. _watcher: notify::RecommendedWatcher, @@ -31,8 +30,8 @@ impl Files { /// polling here. pub fn new(paths: &[PathBuf]) -> notify::Result { // A capacity-1 channel of unit wakeups coalesces the burst of raw events - // notify emits per change (and any unrelated churn in the directory): a - // full buffer already has a pending wakeup, so extra sends are dropped. + // notify emits per change: a full buffer already has a pending wakeup, so + // extra sends are dropped. let (tx, rx) = mpsc::channel(1); let watcher = callback(paths, move || { let _ = tx.try_send(()); @@ -44,9 +43,9 @@ impl Files { }) } - /// Resolve once the OS reports activity in a watched directory. The caller - /// reloads on return; reloads are idempotent, so the coarse "something - /// changed" granularity at worst costs an occasional redundant reload. + /// Resolve once the OS reports a change to a watched file. The caller reloads + /// on return; reloads are idempotent, so a redundant wakeup costs only a + /// redundant reload. pub async fn changed(&mut self) { // The sender lives inside `_watcher`, which we hold for `&mut self`, so the // channel can't be closed here. @@ -64,11 +63,11 @@ pub(crate) fn callback( paths: &[PathBuf], mut changed: impl FnMut() + Send + 'static, ) -> notify::Result { + let mut filter = Filter::new(paths); let mut watcher = notify::recommended_watcher(move |res: notify::Result| { let reload = match res { - Ok(event) => is_reload_trigger(&event.kind), - // A watcher error (e.g. inotify queue overflow) may mean we missed a - // real change, so reload to be safe. + Ok(event) => filter.matches(&event), + // A watcher error may mean we missed a real change, so reload to be safe. Err(_) => true, }; if reload { @@ -94,6 +93,59 @@ pub(crate) fn callback( Ok(watcher) } +/// Picks the events in the watched directories that concern the watched files. +/// +/// Anything else sharing a directory must be ignored: a relay logging into its +/// certificate directory otherwise reloads on every line, and each reload logs +/// another line, a self-sustaining storm. +struct Filter { + paths: Vec, + /// Where each path currently resolves through its symlinks, `None` while it + /// doesn't resolve at all. + resolved: Vec>, +} + +impl Filter { + fn new(paths: &[PathBuf]) -> Self { + let paths = paths.to_vec(); + let resolved = resolve(&paths); + Self { paths, resolved } + } + + fn matches(&mut self, event: ¬ify::Event) -> bool { + // The OS dropped events (e.g. an inotify queue overflow), so a real change + // may have gone unseen. + if event.need_rescan() { + return true; + } + if !is_reload_trigger(&event.kind) { + return false; + } + + // Compared by file name alone: the watch is non-recursive, and backends + // spell the directory differently (FSEvents reports canonical paths). + let named = event + .paths + .iter() + .filter_map(|p| p.file_name()) + .any(|name| self.paths.iter().any(|p| p.file_name() == Some(name))); + + // A Kubernetes secret mount links `cert.pem` to `..data/cert.pem` and rotates + // by swapping the `..data` symlink, so no event ever names the file. Catch + // that by where the paths resolve, refreshed on every event so a rotation + // that was named doesn't fire again on the next unrelated one. + let resolved = resolve(&self.paths); + let moved = resolved != self.resolved; + self.resolved = resolved; + + named || moved + } +} + +fn resolve(paths: &[PathBuf]) -> Vec> { + paths.iter().map(|p| std::fs::canonicalize(p).ok()).collect() +} + /// Whether a raw notify event reflects a real change that should trigger a reload. /// /// The reload path opens and reads the watched files, and notify's inotify backend @@ -166,4 +218,125 @@ mod tests { assert!(is_reload_trigger(&EventKind::Create(CreateKind::Any))); assert!(is_reload_trigger(&EventKind::Modify(ModifyKind::Any))); } + + fn event(kind: notify::EventKind, path: PathBuf) -> notify::Event { + notify::Event::new(kind).add_path(path) + } + + // A log written into the certificate directory must not reload: each reload + // logs a line, so reacting to it never stops. + #[test] + fn unrelated_files_do_not_trigger_reload() { + use notify::EventKind; + use notify::event::{AccessKind, AccessMode, CreateKind, DataChange, ModifyKind, RenameMode}; + + let dir = tempfile::TempDir::new().unwrap(); + let cert = dir.path().join("cert.pem"); + let key = dir.path().join("key.pem"); + let log = dir.path().join("relay.log"); + let mut filter = Filter::new(&[cert.clone(), key.clone()]); + + let write = EventKind::Modify(ModifyKind::Data(DataChange::Any)); + let closed = EventKind::Access(AccessKind::Close(AccessMode::Write)); + assert!(!filter.matches(&event(EventKind::Create(CreateKind::File), log.clone()))); + assert!(!filter.matches(&event(write, log.clone()))); + assert!(!filter.matches(&event(closed, log))); + + assert!(filter.matches(&event(closed, cert))); + let renamed = EventKind::Modify(ModifyKind::Name(RenameMode::To)); + assert!(filter.matches(&event(renamed, key))); + + // Dropped events may have hidden a rotation. + let rescan = notify::Event::new(EventKind::Other).set_flag(notify::event::Flag::Rescan); + assert!(filter.matches(&rescan)); + } + + // Drives a real inotify watch through a Kubernetes secret rotation while a log + // churns in the same directory. Each step blocks until its own events arrive, so + // the filter judges every event against the disk as it was at that step. + #[cfg(target_os = "linux")] + #[test] + fn kubernetes_rotation_reloads_and_log_writes_do_not() { + use std::os::unix::fs::symlink; + use std::sync::mpsc; + use std::time::Duration; + + // Lay out the mount like the kubelet: `tls.crt -> ..data/tls.crt` and + // `..data -> ..`. + let dir = tempfile::TempDir::new().unwrap(); + let path = |name: &str| dir.path().join(name); + std::fs::create_dir(path("..v1")).unwrap(); + std::fs::write(path("..v1/tls.crt"), "v1").unwrap(); + symlink("..v1", path("..data")).unwrap(); + symlink("..data/tls.crt", path("tls.crt")).unwrap(); + + let (tx, rx) = mpsc::channel(); + let mut watcher = match notify::recommended_watcher(tx) { + Ok(watcher) => watcher, + Err(err) if matches!(err.kind, notify::ErrorKind::MaxFilesWatch) => { + eprintln!("skipping kubernetes_rotation_reloads_and_log_writes_do_not: {err}"); + return; + } + Err(err) => panic!("failed to create watcher: {err}"), + }; + if let Err(err) = watcher.watch(dir.path(), notify::RecursiveMode::NonRecursive) { + if matches!(err.kind, notify::ErrorKind::MaxFilesWatch) { + eprintln!("skipping kubernetes_rotation_reloads_and_log_writes_do_not: {err}"); + return; + } + panic!("failed to watch: {err}"); + } + let mut filter = Filter::new(&[path("tls.crt")]); + + // Judge every event up to and including the first that `done` accepts. + let mut step = |done: &dyn Fn(¬ify::Event) -> bool| -> Vec { + let mut verdicts = Vec::new(); + loop { + // Only a hang guard: the awaited event is already on its way. + let event = rx + .recv_timeout(Duration::from_secs(10)) + .expect("missing event") + .unwrap(); + verdicts.push(filter.matches(&event)); + if done(&event) { + return verdicts; + } + } + }; + let named = |name: &'static str| { + move |event: ¬ify::Event| event.paths.iter().any(|p| p.file_name() == Some(name.as_ref())) + }; + let closed = |name: &'static str| { + let named = named(name); + move |event: ¬ify::Event| { + use notify::event::{AccessKind, AccessMode}; + event.kind == notify::EventKind::Access(AccessKind::Close(AccessMode::Write)) && named(event) + } + }; + + std::fs::write(path("relay.log"), "reloading server certificates\n").unwrap(); + let verdicts = step(&closed("relay.log")); + assert!( + verdicts.iter().all(|reload| !reload), + "log write reloaded: {verdicts:?}" + ); + + // Rotate the way the kubelet does: write the new data, point `..data_tmp` at it, + // rename that over `..data`, then remove the old data. No event names `tls.crt`. + std::fs::create_dir(path("..v2")).unwrap(); + std::fs::write(path("..v2/tls.crt"), "v2").unwrap(); + symlink("..v2", path("..data_tmp")).unwrap(); + std::fs::rename(path("..data_tmp"), path("..data")).unwrap(); + std::fs::remove_dir_all(path("..v1")).unwrap(); + let verdicts = step(&named("..v1")); + let reloads = verdicts.iter().filter(|reload| **reload).count(); + assert_eq!(reloads, 1, "rotation should reload once: {verdicts:?}"); + + std::fs::write(path("relay.log"), "reloading server certificates\n").unwrap(); + let verdicts = step(&closed("relay.log")); + assert!( + verdicts.iter().all(|reload| !reload), + "log write reloaded: {verdicts:?}" + ); + } } From 396dad070f3170f4ae141a30366d223685303129 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:49:44 -0700 Subject: [PATCH 2/3] fix(tokio): match watched files by directory too, and drop the test's wall-clock deadline Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/watch.rs | 57 +++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/rs/moq-tokio/src/watch.rs b/rs/moq-tokio/src/watch.rs index 0b60f3f7e1f..7c84394ef2f 100644 --- a/rs/moq-tokio/src/watch.rs +++ b/rs/moq-tokio/src/watch.rs @@ -100,6 +100,8 @@ pub(crate) fn callback( /// another line, a self-sustaining storm. struct Filter { paths: Vec, + /// Each path with its directory canonicalized, as `locate` spells an event's path. + located: Vec, /// Where each path currently resolves through its symlinks, `None` while it /// doesn't resolve at all. resolved: Vec>, @@ -108,8 +110,13 @@ struct Filter { impl Filter { fn new(paths: &[PathBuf]) -> Self { let paths = paths.to_vec(); + let located = paths.iter().filter_map(|p| locate(p)).collect(); let resolved = resolve(&paths); - Self { paths, resolved } + Self { + paths, + located, + resolved, + } } fn matches(&mut self, event: ¬ify::Event) -> bool { @@ -122,13 +129,15 @@ impl Filter { return false; } - // Compared by file name alone: the watch is non-recursive, and backends - // spell the directory differently (FSEvents reports canonical paths). - let named = event - .paths - .iter() - .filter_map(|p| p.file_name()) - .any(|name| self.paths.iter().any(|p| p.file_name() == Some(name))); + // Compared with the directory canonicalized, since backends spell it + // differently (FSEvents reports canonical paths). The file name check first + // skips that syscall for the unrelated churn that dominates these directories. + let named = event.paths.iter().any(|p| { + let name = p.file_name(); + name.is_some() + && self.located.iter().any(|f| f.file_name() == name) + && locate(p).is_some_and(|p| self.located.contains(&p)) + }); // A Kubernetes secret mount links `cert.pem` to `..data/cert.pem` and rotates // by swapping the `..data` symlink, so no event ever names the file. Catch @@ -142,6 +151,12 @@ impl Filter { } } +fn locate(path: &Path) -> Option { + let name = path.file_name()?; + let dir = path.parent().filter(|dir| !dir.as_os_str().is_empty()); + Some(std::fs::canonicalize(dir.unwrap_or(Path::new("."))).ok()?.join(name)) +} + fn resolve(paths: &[PathBuf]) -> Vec> { paths.iter().map(|p| std::fs::canonicalize(p).ok()).collect() } @@ -251,6 +266,23 @@ mod tests { assert!(filter.matches(&rescan)); } + // With the certificate and key in different directories, a file in one that + // shares the other's name is still unrelated. + #[test] + fn same_name_in_another_watched_directory_does_not_trigger_reload() { + use notify::EventKind; + use notify::event::{AccessKind, AccessMode}; + + let certs = tempfile::TempDir::new().unwrap(); + let keys = tempfile::TempDir::new().unwrap(); + let mut filter = Filter::new(&[certs.path().join("cert.pem"), keys.path().join("key.pem")]); + + let closed = EventKind::Access(AccessKind::Close(AccessMode::Write)); + assert!(!filter.matches(&event(closed, keys.path().join("cert.pem")))); + assert!(filter.matches(&event(closed, certs.path().join("cert.pem")))); + assert!(filter.matches(&event(closed, keys.path().join("key.pem")))); + } + // Drives a real inotify watch through a Kubernetes secret rotation while a log // churns in the same directory. Each step blocks until its own events arrive, so // the filter judges every event against the disk as it was at that step. @@ -259,7 +291,6 @@ mod tests { fn kubernetes_rotation_reloads_and_log_writes_do_not() { use std::os::unix::fs::symlink; use std::sync::mpsc; - use std::time::Duration; // Lay out the mount like the kubelet: `tls.crt -> ..data/tls.crt` and // `..data -> ..`. @@ -292,11 +323,9 @@ mod tests { let mut step = |done: &dyn Fn(¬ify::Event) -> bool| -> Vec { let mut verdicts = Vec::new(); loop { - // Only a hang guard: the awaited event is already on its way. - let event = rx - .recv_timeout(Duration::from_secs(10)) - .expect("missing event") - .unwrap(); + // The awaited event is already on its way; a lost one hangs until the + // test runner's timeout. + let event = rx.recv().expect("watcher dropped").unwrap(); verdicts.push(filter.matches(&event)); if done(&event) { return verdicts; From 9fbd5d6fad9563de185dbe58a9147eb18f21cf1b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:27:04 -0700 Subject: [PATCH 3/3] fix(tokio): reload on in-place writes to a watched symlink's target Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/watch.rs | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/rs/moq-tokio/src/watch.rs b/rs/moq-tokio/src/watch.rs index 7c84394ef2f..d361bd7c936 100644 --- a/rs/moq-tokio/src/watch.rs +++ b/rs/moq-tokio/src/watch.rs @@ -129,21 +129,25 @@ impl Filter { return false; } + let resolved = resolve(&self.paths); + + // An event names a watched file by its own path or, for a symlink, by the + // target it resolves to (an in-place write reports the target's name). // Compared with the directory canonicalized, since backends spell it // differently (FSEvents reports canonical paths). The file name check first // skips that syscall for the unrelated churn that dominates these directories. + let watched = || self.located.iter().chain(resolved.iter().flatten()); let named = event.paths.iter().any(|p| { let name = p.file_name(); name.is_some() - && self.located.iter().any(|f| f.file_name() == name) - && locate(p).is_some_and(|p| self.located.contains(&p)) + && watched().any(|f| f.file_name() == name) + && locate(p).is_some_and(|p| watched().any(|f| *f == p)) }); // A Kubernetes secret mount links `cert.pem` to `..data/cert.pem` and rotates // by swapping the `..data` symlink, so no event ever names the file. Catch // that by where the paths resolve, refreshed on every event so a rotation // that was named doesn't fire again on the next unrelated one. - let resolved = resolve(&self.paths); let moved = resolved != self.resolved; self.resolved = resolved; @@ -283,6 +287,24 @@ mod tests { assert!(filter.matches(&event(closed, keys.path().join("key.pem")))); } + // A configured symlink whose target is rewritten in place reports the target's + // name, and nothing resolves anywhere new. + #[cfg(unix)] + #[test] + fn write_to_symlink_target_triggers_reload() { + use notify::EventKind; + use notify::event::{AccessKind, AccessMode}; + + let dir = tempfile::TempDir::new().unwrap(); + std::fs::write(dir.path().join("actual.pem"), "v1").unwrap(); + std::os::unix::fs::symlink("actual.pem", dir.path().join("cert.pem")).unwrap(); + let mut filter = Filter::new(&[dir.path().join("cert.pem")]); + + let closed = EventKind::Access(AccessKind::Close(AccessMode::Write)); + assert!(filter.matches(&event(closed, dir.path().join("actual.pem")))); + assert!(!filter.matches(&event(closed, dir.path().join("relay.log")))); + } + // Drives a real inotify watch through a Kubernetes secret rotation while a log // churns in the same directory. Each step blocks until its own events arrive, so // the filter judges every event against the disk as it was at that step.