Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions rs/moq-tokio/src/tls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2810,9 +2810,9 @@ mod tests {
fn reloadable_client_roots_disable_server_resumption() {
let (ca_a, server_cert, server_key, client_cert, client_key) = signed_certificates();
let (ca_b, _, _, _, _) = signed_certificates();
// A dedicated directory, not `/tmp`: the watcher registers the parent, so
// unrelated churn there fires a reload that can read `ca_a` and store it
// after this test's own `reload()` installed `ca_b`.
// A dedicated directory, not `/tmp`: the watcher matches by file name within
// the parent, so another test's `root.pem` there fires a reload that can read
// `ca_a` and store it after this test's own `reload()` installed `ca_b`.
let dir = tempfile::TempDir::new().unwrap();
let root_path = dir.path().join("root.pem");
std::fs::write(&root_path, ca_a).unwrap();
Expand Down
246 changes: 235 additions & 11 deletions rs/moq-tokio/src/watch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ use std::path::{Path, PathBuf};
use notify::Watcher;
use tokio::sync::mpsc;

/// Watches a set of files and resolves whenever something changes in their
/// directories.
/// Watches a set of files and resolves whenever one of them changes.
///
/// Reacting to the filesystem (rather than a SIGHUP/SIGUSR1) is what lets
/// cert-manager, Kubernetes secret mounts, and `mv`-into-place rotate files with
Expand All @@ -16,7 +15,7 @@ use tokio::sync::mpsc;
/// cert-manager, and K8s secret mounts replace files by atomic rename or symlink
/// swap, which changes the inode (and, for the K8s `..data` symlink, fires on the
/// directory without ever naming the file), so a watch set directly on the path
/// would be missed.
/// would be missed. Events for anything else in those directories are ignored.
pub struct Files {
// Holds the OS watcher alive; dropping it stops events.
_watcher: notify::RecommendedWatcher,
Expand All @@ -31,8 +30,8 @@ impl Files {
/// polling here.
pub fn new(paths: &[PathBuf]) -> notify::Result<Self> {
// A capacity-1 channel of unit wakeups coalesces the burst of raw events
// notify emits per change (and any unrelated churn in the directory): a
// full buffer already has a pending wakeup, so extra sends are dropped.
// notify emits per change: a full buffer already has a pending wakeup, so
// extra sends are dropped.
let (tx, rx) = mpsc::channel(1);
let watcher = callback(paths, move || {
let _ = tx.try_send(());
Expand All @@ -44,9 +43,9 @@ impl Files {
})
}

/// Resolve once the OS reports activity in a watched directory. The caller
/// reloads on return; reloads are idempotent, so the coarse "something
/// changed" granularity at worst costs an occasional redundant reload.
/// Resolve once the OS reports a change to a watched file. The caller reloads
/// on return; reloads are idempotent, so a redundant wakeup costs only a
/// redundant reload.
pub async fn changed(&mut self) {
// The sender lives inside `_watcher`, which we hold for `&mut self`, so the
// channel can't be closed here.
Expand All @@ -64,11 +63,11 @@ pub(crate) fn callback(
paths: &[PathBuf],
mut changed: impl FnMut() + Send + 'static,
) -> notify::Result<notify::RecommendedWatcher> {
let mut filter = Filter::new(paths);
let mut watcher = notify::recommended_watcher(move |res: notify::Result<notify::Event>| {
let reload = match res {
Ok(event) => is_reload_trigger(&event.kind),
// A watcher error (e.g. inotify queue overflow) may mean we missed a
// real change, so reload to be safe.
Ok(event) => filter.matches(&event),
// A watcher error may mean we missed a real change, so reload to be safe.
Err(_) => true,
};
if reload {
Expand All @@ -94,6 +93,78 @@ pub(crate) fn callback(
Ok(watcher)
}

/// Picks the events in the watched directories that concern the watched files.
///
/// Anything else sharing a directory must be ignored: a relay logging into its
/// certificate directory otherwise reloads on every line, and each reload logs
/// another line, a self-sustaining storm.
struct Filter {
paths: Vec<PathBuf>,
/// Each path with its directory canonicalized, as `locate` spells an event's path.
located: Vec<PathBuf>,
/// Where each path currently resolves through its symlinks, `None` while it
/// doesn't resolve at all.
resolved: Vec<Option<PathBuf>>,
}

impl Filter {
fn new(paths: &[PathBuf]) -> Self {
let paths = paths.to_vec();
let located = paths.iter().filter_map(|p| locate(p)).collect();
let resolved = resolve(&paths);
Self {
paths,
located,
resolved,
}
}

fn matches(&mut self, event: &notify::Event) -> bool {
// The OS dropped events (e.g. an inotify queue overflow), so a real change
// may have gone unseen.
if event.need_rescan() {
return true;
}
if !is_reload_trigger(&event.kind) {
return false;
}

let resolved = resolve(&self.paths);

// An event names a watched file by its own path or, for a symlink, by the
// target it resolves to (an in-place write reports the target's name).
// Compared with the directory canonicalized, since backends spell it
// differently (FSEvents reports canonical paths). The file name check first
// skips that syscall for the unrelated churn that dominates these directories.
let watched = || self.located.iter().chain(resolved.iter().flatten());
let named = event.paths.iter().any(|p| {
let name = p.file_name();
name.is_some()
&& watched().any(|f| f.file_name() == name)
&& locate(p).is_some_and(|p| watched().any(|f| *f == p))
});

// A Kubernetes secret mount links `cert.pem` to `..data/cert.pem` and rotates
// by swapping the `..data` symlink, so no event ever names the file. Catch
// that by where the paths resolve, refreshed on every event so a rotation
// that was named doesn't fire again on the next unrelated one.
let moved = resolved != self.resolved;
self.resolved = resolved;

named || moved
}
}

fn locate(path: &Path) -> Option<PathBuf> {
let name = path.file_name()?;
let dir = path.parent().filter(|dir| !dir.as_os_str().is_empty());
Some(std::fs::canonicalize(dir.unwrap_or(Path::new("."))).ok()?.join(name))
}

fn resolve(paths: &[PathBuf]) -> Vec<Option<PathBuf>> {
paths.iter().map(|p| std::fs::canonicalize(p).ok()).collect()
}

/// Whether a raw notify event reflects a real change that should trigger a reload.
///
/// The reload path opens and reads the watched files, and notify's inotify backend
Expand Down Expand Up @@ -166,4 +237,157 @@ mod tests {
assert!(is_reload_trigger(&EventKind::Create(CreateKind::Any)));
assert!(is_reload_trigger(&EventKind::Modify(ModifyKind::Any)));
}

fn event(kind: notify::EventKind, path: PathBuf) -> notify::Event {
notify::Event::new(kind).add_path(path)
}

// A log written into the certificate directory must not reload: each reload
// logs a line, so reacting to it never stops.
#[test]
fn unrelated_files_do_not_trigger_reload() {
use notify::EventKind;
use notify::event::{AccessKind, AccessMode, CreateKind, DataChange, ModifyKind, RenameMode};

let dir = tempfile::TempDir::new().unwrap();
let cert = dir.path().join("cert.pem");
let key = dir.path().join("key.pem");
let log = dir.path().join("relay.log");
let mut filter = Filter::new(&[cert.clone(), key.clone()]);

let write = EventKind::Modify(ModifyKind::Data(DataChange::Any));
let closed = EventKind::Access(AccessKind::Close(AccessMode::Write));
assert!(!filter.matches(&event(EventKind::Create(CreateKind::File), log.clone())));
assert!(!filter.matches(&event(write, log.clone())));
assert!(!filter.matches(&event(closed, log)));

assert!(filter.matches(&event(closed, cert)));
let renamed = EventKind::Modify(ModifyKind::Name(RenameMode::To));
assert!(filter.matches(&event(renamed, key)));

// Dropped events may have hidden a rotation.
let rescan = notify::Event::new(EventKind::Other).set_flag(notify::event::Flag::Rescan);
assert!(filter.matches(&rescan));
}

// With the certificate and key in different directories, a file in one that
// shares the other's name is still unrelated.
#[test]
fn same_name_in_another_watched_directory_does_not_trigger_reload() {
use notify::EventKind;
use notify::event::{AccessKind, AccessMode};

let certs = tempfile::TempDir::new().unwrap();
let keys = tempfile::TempDir::new().unwrap();
let mut filter = Filter::new(&[certs.path().join("cert.pem"), keys.path().join("key.pem")]);

let closed = EventKind::Access(AccessKind::Close(AccessMode::Write));
assert!(!filter.matches(&event(closed, keys.path().join("cert.pem"))));
assert!(filter.matches(&event(closed, certs.path().join("cert.pem"))));
assert!(filter.matches(&event(closed, keys.path().join("key.pem"))));
}

// A configured symlink whose target is rewritten in place reports the target's
// name, and nothing resolves anywhere new.
#[cfg(unix)]
#[test]
fn write_to_symlink_target_triggers_reload() {
use notify::EventKind;
use notify::event::{AccessKind, AccessMode};

let dir = tempfile::TempDir::new().unwrap();
std::fs::write(dir.path().join("actual.pem"), "v1").unwrap();
std::os::unix::fs::symlink("actual.pem", dir.path().join("cert.pem")).unwrap();
let mut filter = Filter::new(&[dir.path().join("cert.pem")]);

let closed = EventKind::Access(AccessKind::Close(AccessMode::Write));
assert!(filter.matches(&event(closed, dir.path().join("actual.pem"))));
assert!(!filter.matches(&event(closed, dir.path().join("relay.log"))));
}

// Drives a real inotify watch through a Kubernetes secret rotation while a log
// churns in the same directory. Each step blocks until its own events arrive, so
// the filter judges every event against the disk as it was at that step.
#[cfg(target_os = "linux")]
#[test]
fn kubernetes_rotation_reloads_and_log_writes_do_not() {
use std::os::unix::fs::symlink;
use std::sync::mpsc;

// Lay out the mount like the kubelet: `tls.crt -> ..data/tls.crt` and
// `..data -> ..<timestamp>`.
let dir = tempfile::TempDir::new().unwrap();
let path = |name: &str| dir.path().join(name);
std::fs::create_dir(path("..v1")).unwrap();
std::fs::write(path("..v1/tls.crt"), "v1").unwrap();
symlink("..v1", path("..data")).unwrap();
symlink("..data/tls.crt", path("tls.crt")).unwrap();

let (tx, rx) = mpsc::channel();
let mut watcher = match notify::recommended_watcher(tx) {
Ok(watcher) => watcher,
Err(err) if matches!(err.kind, notify::ErrorKind::MaxFilesWatch) => {
eprintln!("skipping kubernetes_rotation_reloads_and_log_writes_do_not: {err}");
return;
}
Err(err) => panic!("failed to create watcher: {err}"),
};
if let Err(err) = watcher.watch(dir.path(), notify::RecursiveMode::NonRecursive) {
if matches!(err.kind, notify::ErrorKind::MaxFilesWatch) {
eprintln!("skipping kubernetes_rotation_reloads_and_log_writes_do_not: {err}");
return;
}
panic!("failed to watch: {err}");
}
let mut filter = Filter::new(&[path("tls.crt")]);

// Judge every event up to and including the first that `done` accepts.
let mut step = |done: &dyn Fn(&notify::Event) -> bool| -> Vec<bool> {
let mut verdicts = Vec::new();
loop {
// The awaited event is already on its way; a lost one hangs until the
// test runner's timeout.
let event = rx.recv().expect("watcher dropped").unwrap();
verdicts.push(filter.matches(&event));
if done(&event) {
return verdicts;
}
}
};
let named = |name: &'static str| {
move |event: &notify::Event| event.paths.iter().any(|p| p.file_name() == Some(name.as_ref()))
};
let closed = |name: &'static str| {
let named = named(name);
move |event: &notify::Event| {
use notify::event::{AccessKind, AccessMode};
event.kind == notify::EventKind::Access(AccessKind::Close(AccessMode::Write)) && named(event)
}
};

std::fs::write(path("relay.log"), "reloading server certificates\n").unwrap();
let verdicts = step(&closed("relay.log"));
assert!(
verdicts.iter().all(|reload| !reload),
"log write reloaded: {verdicts:?}"
);

// Rotate the way the kubelet does: write the new data, point `..data_tmp` at it,
// rename that over `..data`, then remove the old data. No event names `tls.crt`.
std::fs::create_dir(path("..v2")).unwrap();
std::fs::write(path("..v2/tls.crt"), "v2").unwrap();
symlink("..v2", path("..data_tmp")).unwrap();
std::fs::rename(path("..data_tmp"), path("..data")).unwrap();
std::fs::remove_dir_all(path("..v1")).unwrap();
let verdicts = step(&named("..v1"));
let reloads = verdicts.iter().filter(|reload| **reload).count();
assert_eq!(reloads, 1, "rotation should reload once: {verdicts:?}");

std::fs::write(path("relay.log"), "reloading server certificates\n").unwrap();
let verdicts = step(&closed("relay.log"));
assert!(
verdicts.iter().all(|reload| !reload),
"log write reloaded: {verdicts:?}"
);
}
}
Loading