devguard, runner, server, store: lifecycle of governed executions after launch (CSRG-U5)

crates/devguard/examples/fixture_authority.rs

@@ -9,8 +9,9 @@
 //! It prints one JSON line with the settings (none is a secret: the secret stays in its file)
 //! once the authority admits, then answers each `instances` line on standard input with a JSON
 //! line listing the consumer's registered instances, and each `charged` line with one listing
-//! the attempts that still hold resources and the budget they hold (CSRG-U3). It stops at the
-//! end of its input. Built only as an
+//! the attempts that still hold resources and the budget they hold (CSRG-U3). A `restart` line
+//! stops the authority and serves its journal again, as after a restart, and is answered once
+//! it admits again (CSRG-U5). It stops at the end of its input. Built only as an
 //! example, from this crate's development dependencies; no product links it.
 
 use std::io::{BufRead, Write};
@@ -33,14 +34,18 @@ fn main() {
         .map(|limit| limit.to_str().unwrap().parse().unwrap())
         .unwrap_or(2);
     let paths = AuthorityPaths::fixture(&base);
-    let fixture = Fixture::start(&base, &paths, max_instances);
+    let mut fixture = Fixture::start(&base, &paths, max_instances);
     let mut out = std::io::stdout().lock();
     writeln!(out, "{}", fixture.settings).unwrap();
     out.flush().unwrap();
     for line in std::io::stdin().lock().lines() {
         match line.unwrap().trim() {
             "instances" => writeln!(out, "{}", fixture.instances()).unwrap(),
             "charged" => writeln!(out, "{}", fixture.charged()).unwrap(),
+            "restart" => {
+                fixture = fixture.restart(&base);
+                writeln!(out, "{}", json!({"restarted": true})).unwrap()
+            }
             other => panic!("unknown command {other:?}"),
         }
         out.flush().unwrap();
@@ -148,6 +153,31 @@ impl Fixture {
         json!({"attempts": [], "committed": {"cpu_milli": 0, "memory_bytes": 0, "tasks": 0}})
     }
 
+    /// Stop serving and serve the same journal again, as after a restart: DevGuard turns
+    /// every committed attempt and registered instance Suspect.
+    #[cfg(target_os = "macos")]
+    fn restart(self, base: &Path) -> Self {
+        let Self {
+            settings,
+            authority,
+        } = self;
+        authority.stop().unwrap();
+        let authority = devguard_daemon::fixture::TestAuthority::restart(base).unwrap();
+        authority
+            .wait_until_admitting(std::time::Duration::from_secs(10))
+            .unwrap();
+        Self {
+            settings,
+            authority,
+        }
+    }
+
+    /// Without native evidence nothing is launched, so no test restarts this authority.
+    #[cfg(not(target_os = "macos"))]
+    fn restart(self, _base: &Path) -> Self {
+        panic!("a restart is only exercised with native evidence");
+    }
+
     fn stop(self) {
         #[cfg(target_os = "macos")]
         self.authority.stop().unwrap();

crates/devguard/src/admission.rs

@@ -259,6 +259,39 @@ impl Owner {
         )
     }
 
+    /// Ask DevGuard to observe the attempt's scope now and settle it by its usual rules
+    /// (CSRG-U5). Sent while the scope's exited root is still unreaped, it lets DevGuard keep
+    /// tracking the members left in the root's group; sent after the reap, it settles what
+    /// DevGuard can prove. It never releases a scope that cannot be proven ended. Answers the
+    /// attempt's record.
+    pub fn observe(&self, admission: &Admission) -> Answer {
+        self.attempt_request(admission, Request::Observe, effective_uid(), self.pid())
+    }
+
+    /// Ask DevGuard to send SIGKILL to every identity of the attempt's scope that it can
+    /// recheck (CSRG-U5): the group's members while the group provably is still the scope's, and
+    /// the escapes it knows. Delivery is not release evidence: DevGuard releases the scope only
+    /// once it has observed it ended.
+    pub fn terminate(&self, admission: &Admission) -> Termination {
+        let Some(wire) = self.wire(admission) else {
+            return Termination::NotSignalled;
+        };
+        let Ok((mut client, _)) = self.open_with(
+            effective_uid(),
+            crate::launch::launch_compatibility(),
+            self.pid(),
+        ) else {
+            return Termination::NotSignalled;
+        };
+        match client.terminate(wire.key, devguard_client::protocol::StopSignal::Kill) {
+            Ok(result) => Termination::Signalled {
+                signalled: result.signalled,
+                complete: result.complete,
+            },
+            Err(_) => Termination::NotSignalled,
+        }
+    }
+
     pub(crate) fn admit_with(&self, admission: &Admission, uid: u32, pid: u32) -> Answer {
         self.attempt_request(admission, Request::Admit, uid, pid)
     }
@@ -289,7 +322,7 @@ impl Owner {
             return Answer::Refused(ErrorCode::InvalidRequest);
         };
         let compatibility = match request {
-            Request::Abandon(_) => crate::launch::launch_compatibility(),
+            Request::Abandon(_) | Request::Observe => crate::launch::launch_compatibility(),
             Request::Admit | Request::Lookup | Request::Cancel => admission_compatibility(),
         };
         let (mut client, _) = match self.open_with(authority_uid, compatibility, owner_pid) {
@@ -303,6 +336,7 @@ impl Owner {
             Request::Lookup => client.lookup(key.clone()),
             Request::Cancel => client.cancel(key.clone()),
             Request::Abandon(reason) => client.abandon_launch(key.clone(), reason.wire()),
+            Request::Observe => client.observe(key.clone()),
         };
         match answered {
             Ok(record) => self.attempt(&record, &key, &fingerprint),
@@ -366,6 +400,16 @@ impl Owner {
     }
 }
 
+/// What DevGuard did for [`Owner::terminate`] (CSRG-U5).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Termination {
+    /// DevGuard signalled this many identities of the scope; `complete` when it could observe
+    /// and signal every target.
+    Signalled { signalled: u32, complete: bool },
+    /// DevGuard refused, or the request did not reach it: nothing is known to be signalled.
+    NotSignalled,
+}
+
 /// Why this owner holds no helper for a launch grant (CSRG-U4).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Abandon {
@@ -394,6 +438,7 @@ enum Request {
     Lookup,
     Cancel,
     Abandon(Abandon),
+    Observe,
 }
 
 /// An error answer is definite, except DevGuard's client's own code for a transport failure.

crates/devguard/src/launch.rs

@@ -985,6 +985,184 @@ pub(crate) mod tests {
             assert_eq!(charged(&provisioned), Budget::ZERO);
         }
 
+        /// Wait until the child `pid` has exited, without reaping it.
+        fn exited_unreaped(pid: u32) {
+            let deadline = Instant::now() + Duration::from_secs(10);
+            loop {
+                // SAFETY: an all-zero `siginfo_t` is valid; `WNOWAIT` leaves the child waitable.
+                let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() };
+                // SAFETY: `info` is a valid `siginfo_t` for `waitid` to fill.
+                let result = unsafe {
+                    libc::waitid(
+                        libc::P_PID,
+                        pid,
+                        &mut info,
+                        libc::WEXITED | libc::WNOHANG | libc::WNOWAIT,
+                    )
+                };
+                assert_eq!(result, 0, "{}", std::io::Error::last_os_error());
+                // SAFETY: `waitid` filled `info`, or left the zero it was given.
+                if unsafe { info.si_pid() } != 0 {
+                    return;
+                }
+                assert!(Instant::now() < deadline, "{pid} did not exit");
+                std::thread::sleep(Duration::from_millis(10));
+            }
+        }
+
+        fn observed(owner: &Owner, admission: &Admission) -> crate::admission::Attempt {
+            match owner.observe(admission) {
+                crate::admission::Answer::Attempt(attempt) => attempt,
+                other => panic!("not observed: {other:?}"),
+            }
+        }
+
+        /// A started helper whose executable exits at once, leaving a descendant in its group
+        /// for two seconds. DevGuard's background reconciler is paused, so only this owner's
+        /// observations settle the scope.
+        fn started_with_a_survivor(
+            provisioned: &Provisioned,
+            owner: &Owner,
+            id: &str,
+        ) -> Option<(Admission, Child)> {
+            let helper = helper_bin()?;
+            provisioned.authority.pause_reconciler(true);
+            let script = "sleep 2 </dev/null >/dev/null 2>&1 & exit 0";
+            let admission = admission(id, &["/bin/sh", "-c", script]);
+            admitted(owner, &admission);
+            let permit = granted(owner, &admission);
+            let invocation = owner
+                .helper(
+                    &helper,
+                    &admission,
+                    permit,
+                    Path::new("/bin/sh"),
+                    &["-c".into(), script.into()],
+                )
+                .unwrap();
+            let child = spawn(&invocation, Stdio::null());
+            assert_eq!(invocation.into_transcript().read(), TranscriptEnd::Started);
+            exited_unreaped(child.id());
+            Some((admission, child))
+        }
+
+        #[test]
+        fn an_observation_before_the_reap_keeps_a_surviving_descendant_tracked() {
+            let provisioned = admitting();
+            let owner = provisioned.owner();
+            let Some((admission, mut child)) =
+                started_with_a_survivor(&provisioned, &owner, "cs-observe-1")
+            else {
+                return;
+            };
+            // While the exited root still holds its PID, DevGuard adopts what is left in its
+            // group, and keeps tracking it after the reap.
+            let before = observed(&owner, &admission);
+            assert!(
+                before.phase != Phase::Released && !before.tracking_lost,
+                "{before:?}"
+            );
+            child.wait().unwrap();
+            let after = observed(&owner, &admission);
+            assert!(
+                after.phase != Phase::Released && !after.tracking_lost,
+                "{after:?}"
+            );
+            assert_ne!(charged(&provisioned), Budget::ZERO);
+            // Once the descendant has ended, an observation releases the scope.
+            let deadline = Instant::now() + Duration::from_secs(10);
+            let ended = loop {
+                let attempt = observed(&owner, &admission);
+                if attempt.phase == Phase::Released || Instant::now() > deadline {
+                    break attempt;
+                }
+                std::thread::sleep(Duration::from_millis(200));
+            };
+            assert_eq!(
+                (ended.phase, ended.release, ended.tracking_lost),
+                (Phase::Released, Some(Release::ScopeTerminated), false)
+            );
+            assert_eq!(charged(&provisioned), Budget::ZERO);
+        }
+
+        #[test]
+        fn a_reap_before_any_observation_loses_a_surviving_descendant() {
+            let provisioned = admitting();
+            let owner = provisioned.owner();
+            let Some((admission, mut child)) =
+                started_with_a_survivor(&provisioned, &owner, "cs-observe-2")
+            else {
+                return;
+            };
+            // Reaped first: what is left in the group can no longer be proven the scope's.
+            child.wait().unwrap();
+            let after = observed(&owner, &admission);
+            assert!(
+                after.tracking_lost && after.phase != Phase::Released,
+                "{after:?}"
+            );
+            // It stays charged even once the descendant has ended.
+            std::thread::sleep(Duration::from_secs(3));
+            let later = observed(&owner, &admission);
+            assert!(
+                later.tracking_lost && later.phase != Phase::Released,
+                "{later:?}"
+            );
+            assert_ne!(charged(&provisioned), Budget::ZERO);
+        }
+
+        #[test]
+        fn terminate_kills_the_whole_scope_and_it_is_released() {
+            let Some(helper) = helper_bin() else { return };
+            let provisioned = admitting();
+            let owner = provisioned.owner();
+            let script = "sleep 30 </dev/null >/dev/null 2>&1 & exec sleep 30";
+            let admission = admission("cs-terminate-1", &["/bin/sh", "-c", script]);
+            admitted(&owner, &admission);
+            let permit = granted(&owner, &admission);
+            let invocation = owner
+                .helper(
+                    &helper,
+                    &admission,
+                    permit,
+                    Path::new("/bin/sh"),
+                    &["-c".into(), script.into()],
+                )
+                .unwrap();
+            let mut child = spawn(&invocation, Stdio::null());
+            assert_eq!(invocation.into_transcript().read(), TranscriptEnd::Started);
+            // Until the root has exec'd its last command, its background child may not exist.
+            std::thread::sleep(Duration::from_millis(300));
+            // The root and its descendant both: a kill of the root alone would leave the other.
+            let termination = owner.terminate(&admission);
+            assert!(
+                matches!(
+                    termination,
+                    crate::admission::Termination::Signalled {
+                        signalled: 2..,
+                        complete: true
+                    }
+                ),
+                "{termination:?}"
+            );
+            exited_unreaped(child.id());
+            let _ = observed(&owner, &admission);
+            child.wait().unwrap();
+            let deadline = Instant::now() + Duration::from_secs(10);
+            let ended = loop {
+                let attempt = observed(&owner, &admission);
+                if attempt.phase == Phase::Released || Instant::now() > deadline {
+                    break attempt;
+                }
+                std::thread::sleep(Duration::from_millis(200));
+            };
+            assert_eq!(
+                (ended.phase, ended.release, ended.tracking_lost),
+                (Phase::Released, Some(Release::ScopeTerminated), false)
+            );
+            assert_eq!(charged(&provisioned), Budget::ZERO);
+        }
+
         #[test]
         fn a_lost_grant_is_found_committed_and_released_as_never_received() {
             let provisioned = admitting();