diff --git a/Cargo.lock b/Cargo.lock index 18c53c18ca0..0e869ab64d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -896,6 +896,7 @@ dependencies = [ "getrandom 0.4.3", "hex", "nix 0.31.3", + "nostr 0.44.7", "reqwest 0.13.4", "rmcp", "serde", diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 1352b31cad8..3ea2c856471 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -5,6 +5,8 @@ mod config; mod engram_fetch; mod filter; mod observer; +mod observer_gap; +mod observer_publisher; mod pool; mod pool_lifecycle; mod queue; @@ -36,6 +38,10 @@ use config::{ use filter::SubscriptionRule; use futures_util::FutureExt; use nostr::{PublicKey, ToBech32}; +use observer_gap::{ObserverGapCounts, ObserverGapReason, PendingObserverFrame}; +#[cfg(test)] +use observer_publisher::run_relay_observer_publisher; +use observer_publisher::spawn_relay_observer_publisher; use pool::{ AgentPool, ControlSignal, IdleSwitchResult, OwnedAgent, PromptContext, PromptOutcome, PromptResult, PromptSource, SessionState, TimeoutKind, @@ -385,64 +391,32 @@ const OBSERVER_PUBLISH_TICK: Duration = Duration::from_secs(1); /// one ~64KB frame, gathered queue-wide for the front channel, so a single /// channel drains at ~64KB/s and 4 MiB buys roughly **64 seconds** of /// sustained over-production before the oldest items are dropped WITH -/// accounting (a warn carrying the dropped-event count). With C channels -/// producing concurrently the slots round-robin between them, so the -/// per-channel drain is ~64KB/Cs and the budget shortens accordingly — -/// still bytes-per-slot, never events-per-slot (see -/// [`ObserverPublishQueue::next_frame`]). Beyond-budget floods therefore -/// degrade to designed, visible loss — strictly better than the -/// pre-batching pacer's silent 90/min drop. +/// accounting in source-event units. The signed gap frame makes any loss +/// visible to the Activity Ledger instead of permitting false completeness. const OBSERVER_PENDING_QUEUE_MAX_BYTES: usize = 4 * 1024 * 1024; /// Observer event kind for a batch envelope wrapping multiple events. -/// -/// The payload is `{"events": [, ...]}` with every inner event -/// carrying its own `seq`/`timestamp`, so consumers process inner events -/// exactly as they would unbatched ones. Single pending events are published -/// unwrapped, so the envelope only appears when there is something to batch. +/// Single pending events stay unwrapped for older consumers. const OBSERVER_BATCH_KIND: &str = "batch"; -/// Collects observer events awaiting a publish slot. -/// -/// Chunk-type events ride the [`ObserverChunkCoalescer`]; everything else is -/// appended in arrival order, force-flushing pending chunks first — the same -/// ordering rule the pre-batching publisher enforced, so merged chunk text can -/// never leapfrog a tool call that arrived mid-stream. -/// -/// Events wait here as EVENTS, not pre-sealed frames: each publish slot packs -/// one frame at publish time ([`Self::next_frame`]), so a backlog keeps -/// compacting into full frames instead of freezing into a frame queue. -/// -/// The queue is bounded by [`OBSERVER_PENDING_QUEUE_MAX_BYTES`]. When a -/// sustained flood outruns the one-frame-per-tick drain for longer than the -/// budget, the OLDEST events are dropped (the viewer wants recent state) with -/// accounting: a warning carrying the dropped-event count, and -/// `dropped_events` for tests. +/// Bounded event queue feeding the one-frame-per-tick relay publisher. +/// Chunks coalesce in order; overflow drops oldest work and records a gap. #[derive(Default)] struct ObserverPublishQueue { coalescer: ObserverChunkCoalescer, - /// `(serialized_len, source_events, event)`, oldest first. Length is - /// captured at enqueue (post-fit) so byte accounting never re-serializes - /// on eviction; `source_events` is how many GENERATED observer events the - /// entry represents (a merged chunk carries every chunk it absorbed), so - /// eviction accounting stays in source units after flush. + /// `(serialized_len, represented_source_events, event)`, oldest first. events: VecDeque<(usize, u64, observer::ObserverEvent)>, pending_bytes: usize, - /// SOURCE observer events lost to byte-budget eviction. Counted in - /// generated-event units, not retained entries: a coalesced entry that - /// merged N chunks accounts for N when evicted. A PUBLISHED merged entry - /// delivers all N sources' text in one event, so the invariant is - /// `ingested == dropped_events + Σ source_events over published events`. + /// Test counter in represented source-event units. dropped_events: u64, + /// Loss already observed before durable relay ingestion. It is removed + /// only while carried by an attempted signed frame and restored on error. + pending_gaps: ObserverGapCounts, } impl ObserverPublishQueue { fn ingest(&mut self, event: observer::ObserverEvent) { - // ObserverChunkCoalescer::ingest returns immediately-publishable events - // (force-flushed pending chunks + non-chunk passthrough, or a pending - // set displaced by the 60KB pre-flush); they join the queue in the - // order the coalescer emitted them, each carrying the count of source - // events it represents. + // Each ready item carries the number of source chunks it represents. for (source_events, ready) in self.coalescer.ingest(event) { self.enqueue(source_events, ready); } @@ -450,30 +424,19 @@ impl ObserverPublishQueue { } fn enqueue(&mut self, source_events: u64, mut event: observer::ObserverEvent) { - // Pre-trim at enqueue so (a) byte accounting reflects what will ship - // and (b) one oversized leaf cannot force every frame it touches into - // whole-envelope elision downstream. + // Pre-trim so accounting matches the bytes that can actually ship. fit_observer_event_to_budget(&mut event); let bytes = serialized_len(&event); self.pending_bytes += bytes; self.events.push_back((bytes, source_events, event)); } - /// Total bytes retained across BOTH stores — the event FIFO and the - /// coalescer's pending chunk buffer. The budget binds this sum; counting - /// only the FIFO would let a high-cardinality chunk flood (many distinct - /// coalescer keys, nothing ever flushing) grow unbounded outside the cap. + /// Bytes retained across the FIFO and pending chunk coalescer. fn total_pending_bytes(&self) -> usize { self.pending_bytes + self.coalescer.pending_bytes } - /// Enforce [`OBSERVER_PENDING_QUEUE_MAX_BYTES`] over the total, dropping - /// OLDEST items first with accounting in SOURCE-event units. Global age - /// order across the two stores is structural: every enqueue path flushes - /// the coalescer first, so every pending coalescer entry is strictly newer - /// than every queued event — eviction is queue front, then coalescer - /// front. The `> 1` guard never drops the sole remaining item (any single - /// fitted event or pre-flush-capped chunk entry is far under the budget). + /// Drop oldest items across both stores, preserving one fitted item. fn enforce_byte_budget(&mut self) { let mut dropped = 0u64; while self.total_pending_bytes() > OBSERVER_PENDING_QUEUE_MAX_BYTES @@ -488,6 +451,7 @@ impl ObserverPublishQueue { } if dropped > 0 { self.dropped_events += dropped; + self.note_gap(ObserverGapReason::PublishQueueEviction, dropped); tracing::warn!( dropped, total_dropped = self.dropped_events, @@ -497,46 +461,36 @@ impl ObserverPublishQueue { } } - /// True when nothing is waiting anywhere — the event queue AND the - /// coalescer's pending chunk buffer. fn is_empty(&self) -> bool { - self.events.is_empty() && self.coalescer.pending.is_empty() + self.events.is_empty() && self.coalescer.pending.is_empty() && self.pending_gaps.is_empty() } - /// Pack and remove AT MOST ONE publishable frame: the front event's - /// channel, gathered queue-wide in FIFO order (packed greedily until - /// adding the next event would push the envelope over - /// `OBSERVER_MAX_PLAINTEXT_LEN`). Singletons ship unwrapped. - /// - /// Two invariants bound the gather: - /// - A frame never mixes channels (the desktop archive indexes a frame - /// under its decrypted top-level `channelId`), and events keep their - /// FIFO order *within* each channel. Cross-channel frame order MAY - /// differ from arrival order — the desktop tolerates that everywhere: - /// the transcript store sorts + rebuilds on out-of-order arrival, the - /// archive is per-channel by construction, and the turn store's - /// watermark is keyed per (agent, channel). - /// - A NULL-channel event is a BARRIER nothing gathers across: null-scope - /// events (`agent_panic`-class) can causally couple to any channel, so - /// their relative order against every channel is preserved exactly. - /// Null-channel events themselves ship only as their contiguous front - /// run. - /// - /// Gathering queue-wide (not just the front run) is what keeps the drain - /// rate in BYTES per slot rather than front-run-length events per slot: - /// with round-robin producers (channel A, B, A, B, ...) a front-run - /// packer degrades to ~1 event per slot regardless of size, silently - /// growing latency without ever tripping the byte budget. - /// - /// Pending coalesced chunks are flushed into the queue first, so a - /// publish slot never leaves merged chunk text stranded behind the tick. - fn next_frame(&mut self) -> Option { + fn note_gap(&mut self, reason: ObserverGapReason, count: u64) { + self.pending_gaps.record(reason, count); + } + + fn restore_gaps(&mut self, gaps: ObserverGapCounts) { + self.pending_gaps.merge(gaps); + } + + /// Pack one same-channel frame. Null-channel events remain ordering + /// barriers; a pending gap rides first and is restored if publishing fails. + fn next_frame(&mut self) -> Option { for (source_events, ready) in self.coalescer.flush() { self.enqueue(source_events, ready); } - let channel = self.events.front()?.2.channel_id.clone(); + if self.events.is_empty() && self.pending_gaps.is_empty() { + return None; + } + let template = self.events.front().map(|(_, _, event)| event.clone()); + let channel = template.as_ref().and_then(|event| event.channel_id.clone()); + let reported_gaps = std::mem::take(&mut self.pending_gaps); let mut picked: Vec = Vec::new(); + if !reported_gaps.is_empty() { + picked.push(reported_gaps.clone().into_event(template.as_ref())); + } + let mut picked_source_events = 0u64; let mut kept: VecDeque<(usize, u64, observer::ObserverEvent)> = VecDeque::with_capacity(self.events.len()); let mut gathering = true; @@ -553,6 +507,7 @@ impl ObserverPublishQueue { gathering = false; } else { self.pending_bytes -= bytes; + picked_source_events = picked_source_events.saturating_add(source_events); } } else { if gathering && (channel.is_none() || event.channel_id.is_none()) { @@ -564,7 +519,11 @@ impl ObserverPublishQueue { } } self.events = kept; - Some(seal_batch(picked)) + Some(PendingObserverFrame { + event: seal_batch(picked), + source_events: picked_source_events, + reported_gaps, + }) } } @@ -600,99 +559,6 @@ fn batch_envelope(events: &[observer::ObserverEvent]) -> observer::ObserverEvent } } -fn spawn_relay_observer_publisher( - observer: observer::ObserverHandle, - publisher: RelayEventPublisher, - keys: nostr::Keys, - agent_pubkey_hex: String, - owner_pubkey_hex: String, - owner_pubkey: PublicKey, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - // Subscribe BEFORE snapshotting so an event emitted between the two - // calls is never lost: it lands in the snapshot, the live receiver, or - // both. The overlap is deduped in the run loop via the snapshot's - // high-water `seq` (monotonic, assigned at emit). - let rx = observer.subscribe(); - let snapshot = observer.snapshot(); - run_relay_observer_publisher( - snapshot, - rx, - publisher, - keys, - agent_pubkey_hex, - owner_pubkey_hex, - owner_pubkey, - ) - .await; - }) -} - -async fn run_relay_observer_publisher( - snapshot: Vec, - mut rx: tokio::sync::broadcast::Receiver, - publisher: RelayEventPublisher, - keys: nostr::Keys, - agent_pubkey_hex: String, - owner_pubkey_hex: String, - owner_pubkey: PublicKey, -) { - let mut queue = ObserverPublishQueue::default(); - let max_snapshot_seq = snapshot.iter().map(|event| event.seq).max().unwrap_or(0); - for event in snapshot { - queue.ingest(event); - } - - // Global pacer: AT MOST ONE relay frame per tick, no matter how many - // channels are active or how large the backlog is. `interval_at` starts - // the first tick a full period out, so a pre-loaded snapshot (up to the - // 1,000-event replay buffer on reconnect) cannot burst at t=0 — the old - // pacer's explicit "no initial burst" property, restored. - let mut publish_tick = tokio::time::interval_at( - tokio::time::Instant::now() + OBSERVER_PUBLISH_TICK, - OBSERVER_PUBLISH_TICK, - ); - publish_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - let mut closed = false; - loop { - tokio::select! { - result = rx.recv(), if !closed => { - match result { - Ok(event) => { - // Skip live events already delivered via the snapshot - // (the subscribe-before-snapshot overlap). - if event.seq <= max_snapshot_seq { - continue; - } - queue.ingest(event); - } - Err(tokio::sync::broadcast::error::RecvError::Lagged(count)) => { - tracing::warn!(dropped = count, "relay observer publisher lagged"); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - // Producer gone: stop selecting on the receiver and let - // the tick arm drain what remains — still one frame per - // tick. An unpaced final drain would be a burst bypass - // around everything the pacer exists to prevent. - closed = true; - } - } - } - _ = publish_tick.tick() => { - if let Some(frame) = queue.next_frame() { - publish_relay_observer_event( - &publisher, &keys, &agent_pubkey_hex, - &owner_pubkey_hex, &owner_pubkey, frame, - ).await; - } - if closed && queue.is_empty() { - break; - } - } - } - } -} - #[derive(Default)] struct ObserverChunkCoalescer { pending: Vec, @@ -1027,7 +893,7 @@ async fn publish_relay_observer_event( owner_pubkey_hex: &str, owner_pubkey: &PublicKey, mut event: observer::ObserverEvent, -) { +) -> Result<(), String> { // Trim oversized frames to fit the plaintext cap rather than letting // encrypt_observer_payload reject and drop them whole (silent telemetry loss). fit_observer_event_to_budget(&mut event); @@ -1035,7 +901,7 @@ async fn publish_relay_observer_event( Ok(encrypted) => encrypted, Err(error) => { tracing::warn!("failed to encrypt relay observer event: {error}"); - return; + return Err(format!("encrypt observer event: {error}")); } }; let builder = match buzz_sdk::build_agent_observer_frame( @@ -1047,19 +913,21 @@ async fn publish_relay_observer_event( Ok(builder) => builder, Err(error) => { tracing::warn!("failed to build relay observer event: {error}"); - return; + return Err(format!("build observer event: {error}")); } }; let signed = match builder.sign_with_keys(keys) { Ok(event) => event, Err(error) => { tracing::warn!("failed to sign relay observer event: {error}"); - return; + return Err(format!("sign observer event: {error}")); } }; if let Err(error) = publisher.publish_event(signed).await { tracing::warn!("relay observer event dropped: {error}"); + return Err(format!("publish observer event: {error}")); } + Ok(()) } /// Maximum age (seconds) for an observer control frame to be considered fresh. @@ -3508,8 +3376,9 @@ async fn tokio_main() -> Result<()> { } } - if let Some(handle) = relay_observer_publisher_task.take() { - handle.abort(); + drop(observer); + if let Some(task) = relay_observer_publisher_task.take() { + task.shutdown().await.map_err(anyhow::Error::msg)?; } // Graceful relay shutdown — sends WebSocket close frame and waits up to 5s @@ -5732,8 +5601,7 @@ mod observer_snapshot_race_tests { drop(observer); run_relay_observer_publisher( - snapshot, - rx, + (snapshot, rx, 0), publisher, agent_keys.clone(), agent_keys.public_key().to_hex(), @@ -5798,7 +5666,7 @@ mod observer_publish_queue_tests { fn drain_frames(queue: &mut ObserverPublishQueue) -> Vec { let mut frames = Vec::new(); while !queue.is_empty() { - frames.push(queue.next_frame().expect("queue not empty")); + frames.push(queue.next_frame().expect("queue not empty").event); } frames } @@ -5807,11 +5675,34 @@ mod observer_publish_queue_tests { /// singleton. fn frame_seqs(frame: &observer::ObserverEvent) -> Vec { match frame.payload.get("events").and_then(|v| v.as_array()) { - Some(inner) => inner.iter().map(|e| e["seq"].as_u64().unwrap()).collect(), + Some(inner) => inner + .iter() + .filter(|event| event["kind"] != observer_gap::OBSERVER_TELEMETRY_GAP_KIND) + .map(|event| event["seq"].as_u64().unwrap()) + .collect(), + None if frame.kind == observer_gap::OBSERVER_TELEMETRY_GAP_KIND => Vec::new(), None => vec![frame.seq], } } + fn gap_total(frame: &observer::ObserverEvent) -> u64 { + match frame + .payload + .get("events") + .and_then(|value| value.as_array()) + { + Some(inner) => inner + .iter() + .find(|event| event["kind"] == observer_gap::OBSERVER_TELEMETRY_GAP_KIND) + .and_then(|event| event["payload"]["droppedEvents"].as_u64()) + .unwrap_or(0), + None if frame.kind == observer_gap::OBSERVER_TELEMETRY_GAP_KIND => { + frame.payload["droppedEvents"].as_u64().unwrap_or(0) + } + None => 0, + } + } + /// Retained bytes computed by WALKING the entries, independently of the /// queue's own accumulator. Cap regressions must assert on this, not on /// `total_pending_bytes()` — asserting the counter against itself passed @@ -5898,7 +5789,7 @@ mod observer_publish_queue_tests { event(3, "acp_write", Some("chan-a")), ]); - let frame = queue.next_frame().expect("one frame"); + let frame = queue.next_frame().expect("one frame").event; assert!(queue.is_empty(), "one channel, one publish slot"); assert_eq!(frame.kind, OBSERVER_BATCH_KIND); assert_eq!(frame.seq, 3, "envelope mirrors the last inner event"); @@ -5912,7 +5803,7 @@ mod observer_publish_queue_tests { #[test] fn a_single_event_stays_unwrapped() { let mut queue = queue_of(vec![event(7, "turn_started", Some("chan-a"))]); - let frame = queue.next_frame().expect("one frame"); + let frame = queue.next_frame().expect("one frame").event; assert!(queue.is_empty()); assert_eq!(frame.kind, "turn_started"); assert_eq!(frame.seq, 7); @@ -6089,7 +5980,7 @@ mod observer_publish_queue_tests { queue.ingest(chunk(2, "world")); queue.ingest(event(3, "tool_call", Some("chan-a"))); - let frame = queue.next_frame().expect("one frame"); + let frame = queue.next_frame().expect("one frame").event; assert!(queue.is_empty()); let inner = frame.payload["events"].as_array().expect("batch of 2"); assert_eq!(inner.len(), 2, "two chunks coalesce into one event"); @@ -6117,7 +6008,7 @@ mod observer_publish_queue_tests { queue.ingest(e); assert!(!queue.is_empty(), "pending chunk counts as queued work"); - let frame = queue.next_frame().expect("chunk must ship"); + let frame = queue.next_frame().expect("chunk must ship").event; assert!(queue.is_empty()); assert_eq!( frame.payload["params"]["update"]["content"]["text"], @@ -6151,6 +6042,11 @@ mod observer_publish_queue_tests { ); let frames = drain_frames(&mut queue); + assert_eq!( + gap_total(&frames[0]), + queue.dropped_events, + "the first signed frame discloses every source event evicted" + ); let published: Vec = frames.iter().flat_map(frame_seqs).collect(); let expected: Vec = (queue.dropped_events + 1..=total as u64).collect(); assert_eq!( @@ -6456,8 +6352,7 @@ mod observer_publish_cadence_tests { assert_eq!(snapshot.len(), 3, "all three preloaded in the snapshot"); let task = tokio::spawn(run_relay_observer_publisher( - snapshot, - rx, + (snapshot, rx, 0), publisher, agent_keys.clone(), agent_keys.public_key().to_hex(), @@ -6535,8 +6430,7 @@ mod observer_publish_cadence_tests { drop(observer); let task = tokio::spawn(run_relay_observer_publisher( - snapshot, - rx, + (snapshot, rx, 0), publisher, agent_keys.clone(), agent_keys.public_key().to_hex(), @@ -6601,8 +6495,7 @@ mod observer_publish_cadence_tests { let snapshot = observer.snapshot(); let task = tokio::spawn(run_relay_observer_publisher( - snapshot, - rx, + (snapshot, rx, 0), publisher, agent_keys.clone(), agent_keys.public_key().to_hex(), diff --git a/crates/buzz-acp/src/observer.rs b/crates/buzz-acp/src/observer.rs index 7029e5af6d5..078b75631b8 100644 --- a/crates/buzz-acp/src/observer.rs +++ b/crates/buzz-acp/src/observer.rs @@ -40,6 +40,7 @@ struct ObserverInner { tx: broadcast::Sender, buffer: Mutex>, seq: AtomicU64, + replay_dropped: AtomicU64, } fn new_observer_handle() -> ObserverHandle { @@ -49,6 +50,7 @@ fn new_observer_handle() -> ObserverHandle { tx, buffer: Mutex::new(VecDeque::with_capacity(OBSERVER_BUFFER_CAP)), seq: AtomicU64::new(1), + replay_dropped: AtomicU64::new(0), }), } } @@ -85,11 +87,13 @@ impl ObserverHandle { } /// Subscribe to live observer events. + #[cfg(test)] pub fn subscribe(&self) -> broadcast::Receiver { self.inner.tx.subscribe() } /// Return the current replay buffer. + #[cfg(test)] pub fn snapshot(&self) -> Vec { match self.inner.buffer.lock() { Ok(buffer) => buffer.iter().cloned().collect(), @@ -100,6 +104,23 @@ impl ObserverHandle { } } + /// Atomically subscribe, snapshot, and take replay-buffer overflow count. + pub(crate) fn subscribe_with_snapshot( + &self, + ) -> (Vec, broadcast::Receiver, u64) { + let buffer = match self.inner.buffer.lock() { + Ok(buffer) => buffer, + Err(error) => { + tracing::warn!(target: "observer", "observer replay buffer lock poisoned: {error}"); + error.into_inner() + } + }; + let receiver = self.inner.tx.subscribe(); + let snapshot = buffer.iter().cloned().collect(); + let dropped = self.inner.replay_dropped.swap(0, Ordering::AcqRel); + (snapshot, receiver, dropped) + } + /// Emit a local observer event. pub fn emit( &self, @@ -124,6 +145,7 @@ impl ObserverHandle { Ok(mut buffer) => { if buffer.len() >= OBSERVER_BUFFER_CAP { buffer.pop_front(); + self.inner.replay_dropped.fetch_add(1, Ordering::Relaxed); } buffer.push_back(event.clone()); } @@ -164,3 +186,27 @@ pub fn context_for_turn( started_at: Some(started_at), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn subscribe_snapshot_reports_replay_overflow_in_source_units() { + let observer = ObserverHandle::in_process(); + for seq in 0..OBSERVER_BUFFER_CAP + 3 { + observer.emit( + "test", + None, + &ObserverContext::default(), + serde_json::json!({ "seq": seq }), + ); + } + + let (snapshot, _receiver, dropped) = observer.subscribe_with_snapshot(); + assert_eq!(snapshot.len(), OBSERVER_BUFFER_CAP); + assert_eq!(dropped, 3); + let (_, _, dropped_again) = observer.subscribe_with_snapshot(); + assert_eq!(dropped_again, 0, "reported gaps are not double counted"); + } +} diff --git a/crates/buzz-acp/src/observer_gap.rs b/crates/buzz-acp/src/observer_gap.rs new file mode 100644 index 00000000000..6220fe866f8 --- /dev/null +++ b/crates/buzz-acp/src/observer_gap.rs @@ -0,0 +1,295 @@ +//! Signed, in-band accounting for observer events lost before relay archival. + +use crate::observer; +use buzz_core::observer::{encrypt_observer_payload, OBSERVER_FRAME_TELEMETRY}; +use nostr::{Event, Keys, PublicKey}; + +pub(crate) const OBSERVER_TELEMETRY_GAP_KIND: &str = "observer_telemetry_gap"; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum ObserverGapReason { + ReplayBufferOverflow, + PublishQueueEviction, + BroadcastLag, + PublishFailure, + RelayQueueEviction, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct ObserverGapCounts { + replay_buffer_overflow: u64, + publish_queue_eviction: u64, + broadcast_lag: u64, + publish_failure: u64, + relay_queue_eviction: u64, + first_observed_at: Option, + last_observed_at: Option, +} + +pub(crate) struct PendingObserverFrame { + pub(crate) event: observer::ObserverEvent, + pub(crate) source_events: u64, + pub(crate) reported_gaps: ObserverGapCounts, +} + +impl ObserverGapCounts { + pub(crate) fn record(&mut self, reason: ObserverGapReason, count: u64) { + if count == 0 { + return; + } + let timestamp = chrono::Utc::now().to_rfc3339(); + self.first_observed_at + .get_or_insert_with(|| timestamp.clone()); + self.last_observed_at = Some(timestamp); + let bucket = match reason { + ObserverGapReason::ReplayBufferOverflow => &mut self.replay_buffer_overflow, + ObserverGapReason::PublishQueueEviction => &mut self.publish_queue_eviction, + ObserverGapReason::BroadcastLag => &mut self.broadcast_lag, + ObserverGapReason::PublishFailure => &mut self.publish_failure, + ObserverGapReason::RelayQueueEviction => &mut self.relay_queue_eviction, + }; + *bucket = bucket.saturating_add(count); + } + + pub(crate) fn merge(&mut self, other: Self) { + if other.is_empty() { + return; + } + if self.first_observed_at.is_none() { + self.first_observed_at = other.first_observed_at.clone(); + } + self.last_observed_at = other.last_observed_at.clone(); + self.replay_buffer_overflow = self + .replay_buffer_overflow + .saturating_add(other.replay_buffer_overflow); + self.publish_queue_eviction = self + .publish_queue_eviction + .saturating_add(other.publish_queue_eviction); + self.broadcast_lag = self.broadcast_lag.saturating_add(other.broadcast_lag); + self.publish_failure = self.publish_failure.saturating_add(other.publish_failure); + self.relay_queue_eviction = self + .relay_queue_eviction + .saturating_add(other.relay_queue_eviction); + } + + pub(crate) fn is_empty(&self) -> bool { + self.total() == 0 + } + + pub(crate) fn total(&self) -> u64 { + self.replay_buffer_overflow + .saturating_add(self.publish_queue_eviction) + .saturating_add(self.broadcast_lag) + .saturating_add(self.publish_failure) + .saturating_add(self.relay_queue_eviction) + } + + pub(crate) fn into_event( + self, + template: Option<&observer::ObserverEvent>, + ) -> observer::ObserverEvent { + let total = self.total(); + observer::ObserverEvent { + seq: template.map_or(0, |event| event.seq), + timestamp: chrono::Utc::now().to_rfc3339(), + kind: OBSERVER_TELEMETRY_GAP_KIND.to_string(), + agent_index: template.and_then(|event| event.agent_index), + channel_id: template.and_then(|event| event.channel_id.clone()), + session_id: None, + turn_id: None, + started_at: None, + payload: serde_json::json!({ + "droppedEvents": total, + "reasonCounts": { + "replayBufferOverflow": self.replay_buffer_overflow, + "publishQueueEviction": self.publish_queue_eviction, + "broadcastLag": self.broadcast_lag, + "publishFailure": self.publish_failure, + "relayQueueEviction": self.relay_queue_eviction, + }, + "firstObservedAt": self.first_observed_at, + "lastObservedAt": self.last_observed_at, + "scope": "publisher_global", + }), + } + } +} + +fn observer_owner(event: &Event) -> Result { + let owner = event + .tags + .iter() + .find_map(|tag| { + let parts = tag.as_slice(); + (parts.first().map(String::as_str) == Some("p")) + .then(|| parts.get(1)) + .flatten() + }) + .ok_or("observer frame is missing owner p tag")?; + PublicKey::from_hex(owner).map_err(|error| format!("invalid observer owner: {error}")) +} + +/// Count decrypted observer events represented by one signed relay frame. +pub(crate) fn represented_event_count(keys: Option<&Keys>, event: &Event) -> u64 { + let Some(keys) = keys else { return 1 }; + let Ok(owner) = observer_owner(event) else { + return 1; + }; + let Ok(plaintext) = + nostr::nips::nip44::decrypt(keys.secret_key(), &owner, event.content.as_str()) + else { + return 1; + }; + let Ok(value) = serde_json::from_str::(&plaintext) else { + return 1; + }; + represented_value_count(&value) +} + +fn represented_value_count(value: &serde_json::Value) -> u64 { + let Some(object) = value.as_object() else { + return 1; + }; + if object.get("kind").and_then(serde_json::Value::as_str) == Some(OBSERVER_TELEMETRY_GAP_KIND) { + return object + .get("payload") + .and_then(|payload| payload.get("droppedEvents")) + .and_then(serde_json::Value::as_u64) + .filter(|count| *count > 0) + .unwrap_or(1); + } + if let Some(events) = object + .get("payload") + .and_then(|payload| payload.get("events")) + .and_then(serde_json::Value::as_array) + { + return events + .iter() + .map(represented_value_count) + .sum::() + .max(1); + } + 1 +} + +/// Build a replacement gap frame using the dropped frame's owner scope. +pub(crate) fn signed_relay_gap( + keys: &Keys, + template: &Event, + dropped_events: u64, +) -> Result { + let owner = observer_owner(template)?; + let mut gaps = ObserverGapCounts::default(); + gaps.record(ObserverGapReason::RelayQueueEviction, dropped_events); + let payload = gaps.into_event(None); + let encrypted = encrypt_observer_payload(keys, &owner, &payload) + .map_err(|error| format!("encrypt relay gap: {error}"))?; + buzz_sdk::build_agent_observer_frame( + &owner.to_hex(), + &keys.public_key().to_hex(), + OBSERVER_FRAME_TELEMETRY, + &encrypted, + ) + .map_err(|error| format!("build relay gap: {error}"))? + .sign_with_keys(keys) + .map_err(|error| format!("sign relay gap: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gap_counts_merge_and_build_a_frame_payload() { + let mut gaps = ObserverGapCounts::default(); + gaps.record(ObserverGapReason::BroadcastLag, 2); + let mut later = ObserverGapCounts::default(); + later.record(ObserverGapReason::PublishFailure, 3); + gaps.merge(later); + + let event = gaps.into_event(None); + assert_eq!(event.kind, OBSERVER_TELEMETRY_GAP_KIND); + assert_eq!(event.payload["droppedEvents"], 5); + assert_eq!(event.payload["reasonCounts"]["broadcastLag"], 2); + assert_eq!(event.payload["reasonCounts"]["publishFailure"], 3); + assert_eq!(event.payload["scope"], "publisher_global"); + } + + #[test] + fn relay_gap_counts_batch_members_and_remains_owner_decryptable() { + let agent = Keys::generate(); + let owner = Keys::generate(); + let batch = serde_json::json!({ + "seq": 2, + "timestamp": "2026-08-22T12:00:00Z", + "kind": "batch", + "payload": {"events": [{"seq": 1}, {"seq": 2}]}, + }); + let encrypted = encrypt_observer_payload(&agent, &owner.public_key(), &batch).unwrap(); + let template = buzz_sdk::build_agent_observer_frame( + &owner.public_key().to_hex(), + &agent.public_key().to_hex(), + OBSERVER_FRAME_TELEMETRY, + &encrypted, + ) + .unwrap() + .sign_with_keys(&agent) + .unwrap(); + + assert_eq!(represented_event_count(Some(&agent), &template), 2); + let gap = signed_relay_gap(&agent, &template, 2).unwrap(); + let payload: serde_json::Value = + buzz_core::observer::decrypt_observer_payload(&owner, &gap).unwrap(); + assert_eq!(payload["kind"], OBSERVER_TELEMETRY_GAP_KIND); + assert_eq!(payload["payload"]["droppedEvents"], 2); + assert_eq!(payload["payload"]["reasonCounts"]["relayQueueEviction"], 2); + } + + #[test] + fn represented_event_count_recursively_preserves_nested_gap_totals() { + let agent = Keys::generate(); + let owner = Keys::generate(); + let nested = serde_json::json!({ + "seq": 9, + "timestamp": "2026-08-22T12:00:00Z", + "kind": "batch", + "payload": { + "events": [ + { + "seq": 7, + "timestamp": "2026-08-22T11:59:58Z", + "kind": OBSERVER_TELEMETRY_GAP_KIND, + "payload": { + "droppedEvents": 11, + "reasonCounts": { + "relayQueueEviction": 11 + } + } + }, + { + "seq": 8, + "timestamp": "2026-08-22T11:59:59Z", + "kind": "tool_call", + "payload": { "ok": true } + } + ] + } + }); + let encrypted = encrypt_observer_payload(&agent, &owner.public_key(), &nested).unwrap(); + let template = buzz_sdk::build_agent_observer_frame( + &owner.public_key().to_hex(), + &agent.public_key().to_hex(), + OBSERVER_FRAME_TELEMETRY, + &encrypted, + ) + .unwrap() + .sign_with_keys(&agent) + .unwrap(); + + assert_eq!( + represented_event_count(Some(&agent), &template), + 12, + "batch accounting must preserve signed gap droppedEvents recursively" + ); + } +} diff --git a/crates/buzz-acp/src/observer_publisher.rs b/crates/buzz-acp/src/observer_publisher.rs new file mode 100644 index 00000000000..8e2ecbc0223 --- /dev/null +++ b/crates/buzz-acp/src/observer_publisher.rs @@ -0,0 +1,205 @@ +use super::*; +use tokio::sync::{broadcast, oneshot}; + +pub(super) struct RelayObserverPublisherTask { + shutdown_tx: oneshot::Sender<()>, + handle: tokio::task::JoinHandle>, +} + +impl RelayObserverPublisherTask { + pub(super) async fn shutdown(self) -> Result<(), String> { + let _ = self.shutdown_tx.send(()); + self.handle + .await + .map_err(|error| format!("observer publisher join: {error}"))? + } +} + +pub(super) fn spawn_relay_observer_publisher( + observer: observer::ObserverHandle, + publisher: RelayEventPublisher, + keys: nostr::Keys, + agent_pubkey_hex: String, + owner_pubkey_hex: String, + owner_pubkey: PublicKey, +) -> RelayObserverPublisherTask { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let handle = tokio::spawn(async move { + let subscription = observer.subscribe_with_snapshot(); + run_relay_observer_publisher_until( + subscription, + publisher, + keys, + agent_pubkey_hex, + owner_pubkey_hex, + owner_pubkey, + Some(shutdown_rx), + ) + .await + }); + RelayObserverPublisherTask { + shutdown_tx, + handle, + } +} + +#[cfg(test)] +pub(super) async fn run_relay_observer_publisher( + subscription: ( + Vec, + broadcast::Receiver, + u64, + ), + publisher: RelayEventPublisher, + keys: nostr::Keys, + agent_pubkey_hex: String, + owner_pubkey_hex: String, + owner_pubkey: PublicKey, +) { + run_relay_observer_publisher_until( + subscription, + publisher, + keys, + agent_pubkey_hex, + owner_pubkey_hex, + owner_pubkey, + None, + ) + .await + .expect("observer publisher"); +} + +async fn run_relay_observer_publisher_until( + subscription: ( + Vec, + broadcast::Receiver, + u64, + ), + publisher: RelayEventPublisher, + keys: nostr::Keys, + agent_pubkey_hex: String, + owner_pubkey_hex: String, + owner_pubkey: PublicKey, + mut shutdown_rx: Option>, +) -> Result<(), String> { + let (snapshot, mut rx, replay_dropped) = subscription; + let mut queue = ObserverPublishQueue::default(); + queue.note_gap(ObserverGapReason::ReplayBufferOverflow, replay_dropped); + let max_snapshot_seq = snapshot.iter().map(|event| event.seq).max().unwrap_or(0); + for event in snapshot { + queue.ingest(event); + } + + let mut publish_tick = tokio::time::interval_at( + tokio::time::Instant::now() + OBSERVER_PUBLISH_TICK, + OBSERVER_PUBLISH_TICK, + ); + publish_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut closed = false; + loop { + tokio::select! { + _ = wait_for_shutdown(&mut shutdown_rx), if !closed => { + drain_receiver(&mut rx, &mut queue, max_snapshot_seq); + closed = true; + } + result = rx.recv(), if !closed => match result { + Ok(event) if event.seq > max_snapshot_seq => queue.ingest(event), + Ok(_) => {} + Err(broadcast::error::RecvError::Lagged(count)) => { + queue.note_gap(ObserverGapReason::BroadcastLag, count); + tracing::warn!(dropped = count, "relay observer publisher lagged"); + } + Err(broadcast::error::RecvError::Closed) => closed = true, + }, + _ = publish_tick.tick() => { + if let Some(frame) = queue.next_frame() { + let result = publish_relay_observer_event( + &publisher, &keys, &agent_pubkey_hex, + &owner_pubkey_hex, &owner_pubkey, frame.event, + ).await; + if result.is_err() { + queue.restore_gaps(frame.reported_gaps); + queue.note_gap(ObserverGapReason::PublishFailure, frame.source_events); + } + } + if closed && queue.is_empty() { + publisher.flush_observer().await.map_err(|error| { + format!("flush observer publisher: {error}") + })?; + break; + } + } + } + } + Ok(()) +} + +async fn wait_for_shutdown(shutdown_rx: &mut Option>) { + if let Some(receiver) = shutdown_rx { + let _ = receiver.await; + } else { + std::future::pending::<()>().await; + } +} + +fn drain_receiver( + rx: &mut broadcast::Receiver, + queue: &mut ObserverPublishQueue, + max_snapshot_seq: u64, +) { + loop { + match rx.try_recv() { + Ok(event) if event.seq > max_snapshot_seq => queue.ingest(event), + Ok(_) => {} + Err(broadcast::error::TryRecvError::Lagged(count)) => { + queue.note_gap(ObserverGapReason::BroadcastLag, count); + } + Err(broadcast::error::TryRecvError::Empty | broadcast::error::TryRecvError::Closed) => { + break; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn explicit_shutdown_drains_and_flushes_before_join() { + let observer = observer::ObserverHandle::in_process(); + let agent_keys = nostr::Keys::generate(); + let owner_keys = nostr::Keys::generate(); + let (publisher, mut published_rx) = RelayEventPublisher::test_pair(); + + for channel in [uuid::Uuid::new_v4(), uuid::Uuid::new_v4()] { + observer.emit( + "test_event", + None, + &observer::context_for(Some(channel), None, None), + serde_json::json!({ "channel": channel }), + ); + } + let task = spawn_relay_observer_publisher( + observer.clone(), + publisher, + agent_keys.clone(), + agent_keys.public_key().to_hex(), + owner_keys.public_key().to_hex(), + owner_keys.public_key(), + ); + let shutdown = tokio::spawn(task.shutdown()); + + tokio::task::yield_now().await; + assert!(published_rx.try_recv().is_err(), "shutdown must not burst"); + tokio::time::advance(Duration::from_secs(2)).await; + tokio::task::yield_now().await; + + shutdown.await.expect("shutdown join").expect("flush"); + let mut published = 0; + while published_rx.try_recv().is_ok() { + published += 1; + } + assert_eq!(published, 2, "every queued channel frame is flushed"); + } +} diff --git a/crates/buzz-acp/src/relay.rs b/crates/buzz-acp/src/relay.rs index 17a818867dd..ae1cb43dac5 100644 --- a/crates/buzz-acp/src/relay.rs +++ b/crates/buzz-acp/src/relay.rs @@ -21,7 +21,7 @@ //! `HarnessRelay` communicates with the background task via a `RelayCommand` //! channel. `next_event()` reads from the event receiver. -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::time::Duration; /// Default capacity of the event channel from background task to harness. @@ -105,13 +105,7 @@ const REQ_PACING_INTERVAL: Duration = Duration::from_millis(125); /// below the relay's 50-frames/5s budget, and ensures the select! loop is never /// blocked for more than one REQ's worth of I/O between drain ticks. const DRAIN_BUDGET_PER_ITER: usize = 1; -/// Maximum observer telemetry frames parked while the rate-limit gate is armed -/// (or the socket is down). The upstream publisher ships at most ONE batched -/// frame per second GLOBALLY (one publish slot per tick, regardless of how -/// many channels are active), so this covers ~4 minutes of gating; beyond that -/// the oldest frames are dropped with visible accounting -/// (`gated_observer_dropped`). Note each dropped frame may carry a whole batch -/// of events, so event-level loss is larger than the frame count. +/// About four minutes of one-frame-per-second observer backlog. const GATED_OBSERVER_QUEUE_CAP: usize = 256; use std::time::Instant; @@ -123,7 +117,7 @@ use buzz_core::kind::{ use futures_util::{SinkExt, StreamExt}; use nostr::{Event, EventBuilder, Keys, Kind, RelayUrl, Tag}; use serde_json::{json, Value}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use tracing::{debug, info, warn}; @@ -549,6 +543,8 @@ enum RelayCommand { SubscribeObserverControls, /// Publish a signed event to the relay (for typing indicators, etc.). PublishEvent { event: Box }, + /// Wait until every prior observer publish has been relay-accepted. + FlushObserver { tx: oneshot::Sender<()> }, /// Floor `since` for membership notification replay; events before startup are never re-delivered. SetStartupWatermark { ts: u64 }, } @@ -600,6 +596,16 @@ impl RelayEventPublisher { .map_err(|_| RelayError::ConnectionClosed) } + /// Wait until every prior observer publish has been relay-accepted. + pub async fn flush_observer(&self) -> Result<(), RelayError> { + let (tx, rx) = oneshot::channel(); + self.cmd_tx + .send(RelayCommand::FlushObserver { tx }) + .await + .map_err(|_| RelayError::ConnectionClosed)?; + rx.await.map_err(|_| RelayError::ConnectionClosed) + } + /// Test-only publisher pair: published events are forwarded to the /// returned receiver instead of a live relay socket. #[cfg(test)] @@ -608,10 +614,15 @@ impl RelayEventPublisher { let (event_tx, event_rx) = mpsc::channel(64); tokio::spawn(async move { while let Some(cmd) = cmd_rx.recv().await { - if let RelayCommand::PublishEvent { event } = cmd { - if event_tx.send(*event).await.is_err() { - break; + match cmd { + RelayCommand::PublishEvent { event } => match event_tx.send(*event).await { + Ok(()) => {} + Err(_) => break, + }, + RelayCommand::FlushObserver { tx } => { + let _ = tx.send(()); } + _ => {} } } }); @@ -619,6 +630,27 @@ impl RelayEventPublisher { } } +#[derive(Debug)] +struct TrackedObserverFrame { + event: Box, + represented_flush_seqs: Vec, +} + +impl TrackedObserverFrame { + fn new(event: Box, represented_flush_seqs: Vec) -> Self { + Self { + event, + represented_flush_seqs, + } + } +} + +#[derive(Debug)] +struct PendingObserverFlush { + target_seq: u64, + tx: oneshot::Sender<()>, +} + impl HarnessRelay { /// Connect to relay and authenticate via NIP-42. /// @@ -1064,19 +1096,18 @@ struct BgState { /// subscription. The main-loop drain re-sends the REQ once the gate clears, /// even when `rate_limited_pending` is empty. observer_resub_needed: bool, - /// Observer telemetry frames (kind 24200) parked while the rate-limit gate - /// is armed. Unlike typing indicators, these frames are durable telemetry: - /// dropping them silently loses turn history in the Desktop observer. - /// Bounded at `GATED_OBSERVER_QUEUE_CAP` (drop-oldest); drained by the - /// main loop one frame per pacing tick once the gate clears. - gated_observer_pending: VecDeque>, - /// Observer frames written to the socket but not yet acknowledged. The - /// relay's rate-limit NOTICE does not carry an event ID, so all unresolved - /// observer writes are moved back ahead of the parked FIFO when one arrives. - observer_in_flight: VecDeque>, - /// Frames evicted from the bounded pending/in-flight observer buffers since - /// summary log. Makes overflow loss visible instead of silent. + /// Observer telemetry parked during rate limiting or disconnects. + gated_observer_pending: VecDeque, + /// Written observer frames awaiting relay OK acknowledgment. + observer_in_flight: VecDeque, + /// Evicted represented events waiting for a signed gap frame. gated_observer_dropped: u64, + gated_observer_gap_template: Option>, + gated_observer_gap_flush_seqs: BTreeSet, + observer_next_flush_seq: u64, + observer_unsettled_flush_seqs: BTreeSet, + observer_flush_waiters: VecDeque, + observer_keys: Option, /// Channels whose REQ failed during `resubscribe_after_reconnect`. /// /// A single failed channel REQ is parked here instead of aborting the whole @@ -1112,6 +1143,12 @@ impl BgState { gated_observer_pending: VecDeque::new(), observer_in_flight: VecDeque::new(), gated_observer_dropped: 0, + gated_observer_gap_template: None, + gated_observer_gap_flush_seqs: BTreeSet::new(), + observer_next_flush_seq: 0, + observer_unsettled_flush_seqs: BTreeSet::new(), + observer_flush_waiters: VecDeque::new(), + observer_keys: None, resubscribe_retry: HashSet::new(), backoff_step: 0, } @@ -1211,14 +1248,47 @@ impl BgState { None } - /// Park an observer telemetry frame while the rate-limit gate is armed. - /// - /// Bounded drop-oldest queue: overflow evicts the oldest frame and counts - /// it in `gated_observer_dropped` so the loss is visible, never silent. - fn park_gated_observer_frame(&mut self, event: Box) { + fn issue_observer_frame(&mut self, event: Box) -> TrackedObserverFrame { + self.observer_next_flush_seq = self.observer_next_flush_seq.saturating_add(1); + let seq = self.observer_next_flush_seq; + self.observer_unsettled_flush_seqs.insert(seq); + TrackedObserverFrame::new(event, vec![seq]) + } + + fn queue_observer_flush(&mut self, tx: oneshot::Sender<()>) { + self.observer_flush_waiters.push_back(PendingObserverFlush { + target_seq: self.observer_next_flush_seq, + tx, + }); + self.maybe_complete_observer_flushes(); + } + + fn observer_flush_complete(&self, target_seq: u64) -> bool { + match self.observer_unsettled_flush_seqs.first().copied() { + Some(seq) => seq > target_seq, + None => true, + } + } + + fn maybe_complete_observer_flushes(&mut self) { + while let Some(waiter) = self.observer_flush_waiters.front() { + if !self.observer_flush_complete(waiter.target_seq) { + break; + } + let waiter = self + .observer_flush_waiters + .pop_front() + .expect("front waiter must exist"); + let _ = waiter.tx.send(()); + } + } + + /// Park telemetry, retaining bounded newest history plus signed gap state. + fn park_gated_observer_frame(&mut self, event: TrackedObserverFrame) { if self.gated_observer_pending.len() >= GATED_OBSERVER_QUEUE_CAP { - self.gated_observer_pending.pop_front(); - self.gated_observer_dropped += 1; + if let Some(dropped) = self.gated_observer_pending.pop_front() { + self.record_gated_observer_drop(dropped); + } warn!( dropped_total = self.gated_observer_dropped, "gated observer queue full — dropped oldest frame" @@ -1227,23 +1297,23 @@ impl BgState { self.gated_observer_pending.push_back(event); } - /// Restore unresolved observer writes ahead of frames parked after the - /// gate armed. NOTICE has no event ID, so conservatively retry every frame - /// without an OK; duplicate IDs are harmless at the relay. + /// Retry every unacknowledged frame after a rate-limit notice. fn requeue_observer_in_flight(&mut self) { while let Some(event) = self.observer_in_flight.pop_back() { self.gated_observer_pending.push_front(event); } while self.gated_observer_pending.len() > GATED_OBSERVER_QUEUE_CAP { - self.gated_observer_pending.pop_front(); - self.gated_observer_dropped += 1; + if let Some(dropped) = self.gated_observer_pending.pop_front() { + self.record_gated_observer_drop(dropped); + } } } - fn track_observer_in_flight(&mut self, event: Box) { + fn track_observer_in_flight(&mut self, event: TrackedObserverFrame) { if self.observer_in_flight.len() >= GATED_OBSERVER_QUEUE_CAP { - self.observer_in_flight.pop_front(); - self.gated_observer_dropped += 1; + if let Some(dropped) = self.observer_in_flight.pop_front() { + self.record_gated_observer_drop(dropped); + } warn!( dropped_total = self.gated_observer_dropped, "observer acknowledgment window full — dropped oldest frame" @@ -1252,13 +1322,75 @@ impl BgState { self.observer_in_flight.push_back(event); } + fn record_gated_observer_drop(&mut self, event: TrackedObserverFrame) { + let represented = + crate::observer_gap::represented_event_count(self.observer_keys.as_ref(), &event.event); + self.gated_observer_dropped = self.gated_observer_dropped.saturating_add(represented); + self.gated_observer_gap_flush_seqs + .extend(event.represented_flush_seqs.iter().copied()); + if self.gated_observer_gap_template.is_none() { + self.gated_observer_gap_template = Some(event.event); + } + } + + fn requeue_rejected_observer_frame(&mut self, event: TrackedObserverFrame) { + if self.gated_observer_pending.len() >= GATED_OBSERVER_QUEUE_CAP { + if let Some(dropped) = self.gated_observer_pending.pop_back() { + self.record_gated_observer_drop(dropped); + } + warn!( + dropped_total = self.gated_observer_dropped, + "gated observer queue full — dropped newest frame to preserve rejected oldest frame" + ); + } + self.gated_observer_pending.push_front(event); + } + + fn handle_observer_rejection(&mut self, event_id: &str, message: &str) -> bool { + let Some(index) = self + .observer_in_flight + .iter() + .position(|event| event.event.id.to_hex() == event_id) + else { + return false; + }; + let event = self + .observer_in_flight + .remove(index) + .expect("position yielded a valid observer frame"); + if message.starts_with("rate-limited:") { + let secs = parse_rate_limit_retry_secs(message).unwrap_or(0); + let deadline = self.set_rate_limit_gate(secs); + self.requeue_rejected_observer_frame(event); + warn!( + "observer frame {event_id} rejected by relay rate limit — requeued until ~{:.1}s from now", + deadline + .checked_duration_since(tokio::time::Instant::now()) + .unwrap_or_default() + .as_secs_f64() + ); + } else { + self.record_gated_observer_drop(event); + warn!( + dropped_total = self.gated_observer_dropped, + "observer frame {event_id} rejected by relay — converting to signed gap accounting" + ); + } + true + } + fn acknowledge_observer_frame(&mut self, event_id: &str) { if let Some(index) = self .observer_in_flight .iter() - .position(|event| event.id.to_hex() == event_id) + .position(|event| event.event.id.to_hex() == event_id) { - self.observer_in_flight.remove(index); + if let Some(frame) = self.observer_in_flight.remove(index) { + for seq in frame.represented_flush_seqs { + self.observer_unsettled_flush_seqs.remove(&seq); + } + self.maybe_complete_observer_flushes(); + } } } } @@ -1314,9 +1446,13 @@ fn apply_command_to_state(state: &mut BgState, cmd: RelayCommand) { // disconnected and are dropped. RelayCommand::PublishEvent { event } => { if event.kind.as_u16() as u32 == KIND_AGENT_OBSERVER_FRAME { - state.park_gated_observer_frame(event); + let tracked = state.issue_observer_frame(event); + state.park_gated_observer_frame(tracked); } } + RelayCommand::FlushObserver { tx } => { + state.queue_observer_flush(tx); + } // Already reconnecting — redundant. RelayCommand::Reconnect => {} // Callers MUST handle Shutdown before calling this function. @@ -1340,7 +1476,8 @@ fn retain_failed_command_intent(state: &mut BgState, cmd: RelayCommand) { RelayCommand::PublishEvent { event } if event.kind.as_u16() as u32 == KIND_AGENT_OBSERVER_FRAME => { - state.park_gated_observer_frame(event); + let tracked = state.issue_observer_frame(event); + state.park_gated_observer_frame(tracked); } RelayCommand::PublishEvent { .. } => {} cmd => apply_command_to_state(state, cmd), @@ -1509,7 +1646,8 @@ async fn execute_connected_command( pending = state.gated_observer_pending.len(), "rate-gated: parking observer frame for paced drain" ); - state.park_gated_observer_frame(event); + let tracked = state.issue_observer_frame(event); + state.park_gated_observer_frame(tracked); return true; } // Drop remaining ephemeral publishes while rate-gated. Stale typing @@ -1529,15 +1667,21 @@ async fn execute_connected_command( // next ping or read will detect the dead socket. A failed observer // frame is parked so the post-reconnect drain redelivers it. let is_observer = event.kind.as_u16() as u32 == KIND_AGENT_OBSERVER_FRAME; - if send_publish_event_frame(ws, &event).await { - if is_observer { - state.track_observer_in_flight(event); + if is_observer { + let tracked = state.issue_observer_frame(event); + if send_publish_event_frame(ws, &tracked.event).await { + state.track_observer_in_flight(tracked); + } else { + state.park_gated_observer_frame(tracked); } - } else if is_observer { - state.park_gated_observer_frame(event); + } else if send_publish_event_frame(ws, &event).await { } true } + RelayCommand::FlushObserver { tx } => { + state.queue_observer_flush(tx); + true + } RelayCommand::SetStartupWatermark { ts } => { state.startup_watermark = Some(ts); if state.membership_last_seen.is_none() { @@ -1574,6 +1718,7 @@ async fn run_background_task( auth_tag: Option, ) { let mut state = BgState::new(); + state.observer_keys = Some(keys.clone()); let handshake_ok = process_handshake_buffer( &mut ws, @@ -1793,8 +1938,10 @@ async fn run_background_task( } } - if budget > 0 && !state.gated_observer_pending.is_empty() { - let sent = drain_gated_observer_pending(&mut ws, &mut state, budget).await; + if budget > 0 + && (!state.gated_observer_pending.is_empty() || state.gated_observer_dropped > 0) + { + let sent = drain_gated_observer_pending(&mut ws, &mut state, &keys, budget).await; if sent > 0 { any_sent = true; } @@ -2392,7 +2539,11 @@ async fn handle_ws_message( warn!("mid-session AUTH rejected (event {event_id}): {message} — triggering reconnect"); return false; } - state.acknowledge_observer_frame(&event_id); + if accepted { + state.acknowledge_observer_frame(&event_id); + } else { + state.handle_observer_rejection(&event_id, &message); + } debug!("OK for event {event_id}: accepted={accepted} message={message}"); } } @@ -2651,15 +2802,11 @@ async fn send_publish_event_frame(ws: &mut WsStream, event: &Event) -> bool { true } -/// Drain parked observer telemetry frames once the rate-limit gate clears. -/// -/// Called by the main loop pacing timer. Sends at most `budget` frames without -/// sleeping — pacing is enforced by the caller via `drain_pacing_next`. Stops -/// immediately if the gate re-arms mid-drain. When the queue empties, any -/// overflow loss is summarized in one warning. Returns the number of frames sent. +/// Drain parked telemetry and then its signed loss marker within `budget`. async fn drain_gated_observer_pending( ws: &mut WsStream, state: &mut BgState, + keys: &Keys, budget: usize, ) -> usize { let mut sent = 0; @@ -2670,7 +2817,7 @@ async fn drain_gated_observer_pending( let Some(event) = state.gated_observer_pending.pop_front() else { break; }; - if !send_publish_event_frame(ws, &event).await { + if !send_publish_event_frame(ws, &event.event).await { // Socket may be dead — re-park at the front so the frame survives // reconnect (the post-reconnect drain will retry it in order). state.gated_observer_pending.push_front(event); @@ -2679,12 +2826,34 @@ async fn drain_gated_observer_pending( state.track_observer_in_flight(event); sent += 1; } - if state.gated_observer_pending.is_empty() && state.gated_observer_dropped > 0 { - warn!( - observer_frames_dropped = state.gated_observer_dropped, - "observer frames lost to gated-queue overflow" - ); - state.gated_observer_dropped = 0; + if sent < budget && state.gated_observer_pending.is_empty() && state.gated_observer_dropped > 0 + { + let result = state + .gated_observer_gap_template + .as_deref() + .ok_or_else(|| "observer gap template missing".to_string()) + .and_then(|template| { + crate::observer_gap::signed_relay_gap(keys, template, state.gated_observer_dropped) + }); + match result { + Ok(gap) if send_publish_event_frame(ws, &gap).await => { + let represented_flush_seqs = state + .gated_observer_gap_flush_seqs + .iter() + .copied() + .collect::>(); + state.gated_observer_dropped = 0; + state.gated_observer_gap_template = None; + state.gated_observer_gap_flush_seqs.clear(); + state.track_observer_in_flight(TrackedObserverFrame::new( + Box::new(gap), + represented_flush_seqs, + )); + sent += 1; + } + Ok(_) => {} + Err(error) => warn!("failed to build signed observer gap frame: {error}"), + } } sent } @@ -5855,7 +6024,6 @@ mod tests { ); } - /// Build a signed observer telemetry frame (kind 24200) for gate tests. fn make_observer_frame(keys: &Keys) -> Event { let recipient = Keys::generate(); let encrypted = buzz_core::observer::encrypt_observer_payload( @@ -5875,9 +6043,54 @@ mod tests { .expect("sign test observer frame") } - /// While the rate-limit gate is armed, an observer frame (kind 24200) is - /// parked — not silently dropped — and delivered by the drain once the - /// gate clears. A typing indicator in the same window stays dropped. + fn track_test_observer_frame(state: &mut BgState, event: Event) { + let tracked = state.issue_observer_frame(Box::new(event)); + state.track_observer_in_flight(tracked); + } + + fn park_test_observer_frame(state: &mut BgState, event: Event) { + let tracked = state.issue_observer_frame(Box::new(event)); + state.park_gated_observer_frame(tracked); + } + + #[tokio::test] + async fn relay_event_publisher_test_pair_flushes_prior_events() { + let (publisher, mut event_rx) = RelayEventPublisher::test_pair(); + let keys = Keys::generate(); + let first = make_observer_frame(&keys); + let second = make_observer_frame(&keys); + + publisher + .publish_event(first.clone()) + .await + .expect("publish first observer frame"); + publisher + .publish_event(second.clone()) + .await + .expect("publish second observer frame"); + publisher + .flush_observer() + .await + .expect("flush prior observer frames"); + + assert_eq!( + event_rx + .recv() + .await + .expect("receive first forwarded event") + .id, + first.id + ); + assert_eq!( + event_rx + .recv() + .await + .expect("receive second forwarded event") + .id, + second.id + ); + } + #[tokio::test] async fn gated_observer_frame_is_parked_then_drained_not_dropped() { let (mut client, mut server) = test_ws_pair().await; @@ -5885,7 +6098,6 @@ mod tests { let keys = Keys::generate(); state.rate_limit_gate = Some(tokio::time::Instant::now() + Duration::from_millis(150)); - // Observer frame while gated: parked, nothing on the wire. let observer_frame = make_observer_frame(&keys); let ok = execute_connected_command( &mut client, @@ -5903,7 +6115,6 @@ mod tests { "observer frame must be parked while gated" ); - // Typing indicator while gated: still dropped, not parked. let typing = EventBuilder::new(Kind::Custom(KIND_TYPING_INDICATOR as u16), "") .tags([Tag::parse(["h", &Uuid::new_v4().to_string()]).unwrap()]) .sign_with_keys(&keys) @@ -5930,10 +6141,9 @@ mod tests { "nothing may reach the wire while the gate is armed" ); - // Gate expires — the drain delivers the parked frame. tokio::time::sleep(Duration::from_millis(160)).await; assert_eq!( - drain_gated_observer_pending(&mut client, &mut state, 1).await, + drain_gated_observer_pending(&mut client, &mut state, &keys, 1).await, 1 ); assert!(state.gated_observer_pending.is_empty()); @@ -5947,9 +6157,6 @@ mod tests { ); } - /// Observer frames arriving while earlier parked frames are still queued - /// are appended behind them (order preserved), even if the gate has - /// already expired. #[tokio::test] async fn observer_frames_queue_behind_parked_backlog_in_order() { let (mut client, mut server) = test_ws_pair().await; @@ -5973,8 +6180,6 @@ mod tests { } assert_eq!(state.gated_observer_pending.len(), 2); - // Gate expires but the backlog is not drained yet — a third frame must - // queue behind it rather than jumping ahead on the wire. tokio::time::sleep(Duration::from_millis(60)).await; let third = make_observer_frame(&keys); let ok = execute_connected_command( @@ -5995,7 +6200,7 @@ mod tests { for expected in [&first, &second, &third] { assert_eq!( - drain_gated_observer_pending(&mut client, &mut state, 1).await, + drain_gated_observer_pending(&mut client, &mut state, &keys, 1).await, 1 ); let frame = next_test_frame(&mut server).await; @@ -6012,37 +6217,295 @@ mod tests { let rejected = make_observer_frame(&keys); let later = make_observer_frame(&keys); - state.track_observer_in_flight(Box::new(accepted.clone())); - state.track_observer_in_flight(Box::new(rejected.clone())); + track_test_observer_frame(&mut state, accepted.clone()); + track_test_observer_frame(&mut state, rejected.clone()); state.acknowledge_observer_frame(&accepted.id.to_hex()); - state.park_gated_observer_frame(Box::new(later.clone())); + park_test_observer_frame(&mut state, later.clone()); state.requeue_observer_in_flight(); let ids: Vec<_> = state .gated_observer_pending .iter() - .map(|event| event.id) + .map(|event| event.event.id) .collect(); assert_eq!(ids, [rejected.id, later.id]); assert!(state.observer_in_flight.is_empty()); } - /// The parked-frame queue is bounded: overflow evicts the oldest frame and - /// counts it; the drain resets the counter after logging the summary. + #[tokio::test] + async fn observer_ok_false_rate_limited_requeues_the_frame_under_the_gate() { + let (mut client, _server) = test_ws_pair().await; + let (event_tx, _event_rx) = mpsc::channel(1); + let (observer_control_tx, _observer_control_rx) = mpsc::channel(1); + let keys = Keys::generate(); + let mut state = BgState::new(); + let observer = make_observer_frame(&keys); + track_test_observer_frame(&mut state, observer.clone()); + + let ok = handle_ws_message( + Message::Text( + serde_json::to_string(&json!([ + "OK", + observer.id.to_hex(), + false, + "rate-limited: observer frame rate exceeded (100/sec per agent)", + ])) + .unwrap() + .into(), + ), + &mut client, + &event_tx, + &observer_control_tx, + &mut state, + &keys, + "wss://relay.example", + &keys.public_key().to_hex(), + None, + ) + .await; + + assert!(ok); + assert!( + state.rate_limit_gate.is_some(), + "rate-limited reject must arm the gate" + ); + assert!( + state.observer_in_flight.is_empty(), + "rejected frame must leave the in-flight window" + ); + assert_eq!(state.gated_observer_pending.len(), 1); + assert_eq!( + state + .gated_observer_pending + .front() + .map(|event| event.event.id), + Some(observer.id), + "rejected frame must be requeued for paced retry" + ); + assert_eq!( + state.gated_observer_dropped, 0, + "retryable rejection must not mint loss" + ); + } + + #[tokio::test] + async fn observer_ok_false_restricted_counts_gap_instead_of_requeueing() { + let (mut client, _server) = test_ws_pair().await; + let (event_tx, _event_rx) = mpsc::channel(1); + let (observer_control_tx, _observer_control_rx) = mpsc::channel(1); + let keys = Keys::generate(); + let mut state = BgState::new(); + state.observer_keys = Some(keys.clone()); + let observer = make_observer_frame(&keys); + track_test_observer_frame(&mut state, observer.clone()); + + let ok = handle_ws_message( + Message::Text( + serde_json::to_string(&json!([ + "OK", + observer.id.to_hex(), + false, + "restricted: observer frame is not authorized for this agent owner", + ])) + .unwrap() + .into(), + ), + &mut client, + &event_tx, + &observer_control_tx, + &mut state, + &keys, + "wss://relay.example", + &keys.public_key().to_hex(), + None, + ) + .await; + + assert!(ok); + assert!(state.observer_in_flight.is_empty()); + assert!( + state.gated_observer_pending.is_empty(), + "terminal rejection must not hot-loop the same frame" + ); + assert_eq!( + state.gated_observer_dropped, 1, + "terminal rejection must be converted into bounded loss" + ); + assert!(state.gated_observer_gap_template.is_some()); + } + + #[tokio::test] + async fn observer_flush_waits_for_ok_true_before_completing() { + let (mut client, _server) = test_ws_pair().await; + let (event_tx, _event_rx) = mpsc::channel(1); + let (observer_control_tx, _observer_control_rx) = mpsc::channel(1); + let keys = Keys::generate(); + let mut state = BgState::new(); + let observer = make_observer_frame(&keys); + track_test_observer_frame(&mut state, observer.clone()); + let (flush_tx, mut flush_rx) = oneshot::channel(); + state.queue_observer_flush(flush_tx); + + assert!(matches!( + flush_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + + let ok = handle_ws_message( + Message::Text( + serde_json::to_string(&json!(["OK", observer.id.to_hex(), true, "accepted"])) + .unwrap() + .into(), + ), + &mut client, + &event_tx, + &observer_control_tx, + &mut state, + &keys, + "wss://relay.example", + &keys.public_key().to_hex(), + None, + ) + .await; + + assert!(ok); + assert_eq!(flush_rx.await, Ok(())); + } + + #[tokio::test] + async fn observer_flush_waits_for_gap_acceptance_after_terminal_reject() { + let (mut client, _server) = test_ws_pair().await; + let (event_tx, _event_rx) = mpsc::channel(1); + let (observer_control_tx, _observer_control_rx) = mpsc::channel(1); + let keys = Keys::generate(); + let mut state = BgState::new(); + state.observer_keys = Some(keys.clone()); + let observer = make_observer_frame(&keys); + track_test_observer_frame(&mut state, observer.clone()); + let (flush_tx, mut flush_rx) = oneshot::channel(); + state.queue_observer_flush(flush_tx); + + let ok = handle_ws_message( + Message::Text( + serde_json::to_string(&json!([ + "OK", + observer.id.to_hex(), + false, + "restricted: observer frame is not authorized for this agent owner", + ])) + .unwrap() + .into(), + ), + &mut client, + &event_tx, + &observer_control_tx, + &mut state, + &keys, + "wss://relay.example", + &keys.public_key().to_hex(), + None, + ) + .await; + + assert!(ok); + assert!(matches!( + flush_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + assert_eq!( + drain_gated_observer_pending(&mut client, &mut state, &keys, 1).await, + 1, + "signed gap should be published for terminal rejection" + ); + let gap_id = state + .observer_in_flight + .front() + .map(|event| event.event.id.to_hex()) + .expect("gap must be in flight"); + + let ok = handle_ws_message( + Message::Text( + serde_json::to_string(&json!(["OK", gap_id, true, "accepted"])) + .unwrap() + .into(), + ), + &mut client, + &event_tx, + &observer_control_tx, + &mut state, + &keys, + "wss://relay.example", + &keys.public_key().to_hex(), + None, + ) + .await; + + assert!(ok); + assert_eq!(flush_rx.await, Ok(())); + } + + #[tokio::test] + async fn observer_flush_survives_disconnected_parking_until_reconnect_ack() { + let (mut client, _server) = test_ws_pair().await; + let (event_tx, _event_rx) = mpsc::channel(1); + let (observer_control_tx, _observer_control_rx) = mpsc::channel(1); + let keys = Keys::generate(); + let mut state = BgState::new(); + let observer = make_observer_frame(&keys); + apply_command_to_state( + &mut state, + RelayCommand::PublishEvent { + event: Box::new(observer.clone()), + }, + ); + let (flush_tx, mut flush_rx) = oneshot::channel(); + apply_command_to_state(&mut state, RelayCommand::FlushObserver { tx: flush_tx }); + + assert_eq!(state.gated_observer_pending.len(), 1); + assert!(matches!( + flush_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + assert_eq!( + drain_gated_observer_pending(&mut client, &mut state, &keys, 1).await, + 1 + ); + + let ok = handle_ws_message( + Message::Text( + serde_json::to_string(&json!(["OK", observer.id.to_hex(), true, "accepted"])) + .unwrap() + .into(), + ), + &mut client, + &event_tx, + &observer_control_tx, + &mut state, + &keys, + "wss://relay.example", + &keys.public_key().to_hex(), + None, + ) + .await; + + assert!(ok); + assert_eq!(flush_rx.await, Ok(())); + } + #[tokio::test] async fn gated_observer_queue_drops_oldest_on_overflow() { let mut state = BgState::new(); let keys = Keys::generate(); let first = make_observer_frame(&keys); - state.park_gated_observer_frame(Box::new(first.clone())); + park_test_observer_frame(&mut state, first.clone()); for _ in 1..GATED_OBSERVER_QUEUE_CAP { - state.park_gated_observer_frame(Box::new(make_observer_frame(&keys))); + park_test_observer_frame(&mut state, make_observer_frame(&keys)); } assert_eq!(state.gated_observer_pending.len(), GATED_OBSERVER_QUEUE_CAP); assert_eq!(state.gated_observer_dropped, 0); let overflow = make_observer_frame(&keys); - state.park_gated_observer_frame(Box::new(overflow.clone())); + park_test_observer_frame(&mut state, overflow.clone()); assert_eq!( state.gated_observer_pending.len(), GATED_OBSERVER_QUEUE_CAP, @@ -6053,14 +6516,23 @@ mod tests { !state .gated_observer_pending .iter() - .any(|e| e.id == first.id), + .any(|e| e.event.id == first.id), "oldest frame must be the one evicted" ); assert_eq!( - state.gated_observer_pending.back().map(|e| e.id), + state.gated_observer_pending.back().map(|e| e.event.id), Some(overflow.id), "newest frame must be retained" ); + state.gated_observer_pending.clear(); + let (mut client, mut server) = test_ws_pair().await; + assert_eq!( + drain_gated_observer_pending(&mut client, &mut state, &keys, 1).await, + 1 + ); + let wire = next_test_frame(&mut server).await; + assert_eq!(wire[1]["kind"], u64::from(KIND_AGENT_OBSERVER_FRAME)); + assert_eq!(state.gated_observer_dropped, 0); } /// is_dns_error correctly classifies platform resolver strings, including @@ -6069,22 +6541,16 @@ mod tests { fn is_dns_error_classification() { use tokio_tungstenite::tungstenite; - // macOS resolver (Http-wrapped, used in many existing tests) assert!(is_dns_error(&RelayError::Http( "nodename nor servname provided, or not known".into() ))); - // Linux resolver assert!(is_dns_error(&RelayError::Http( "Name or service not known".into() ))); - // BSD/Windows assert!(is_dns_error(&RelayError::Http("No such host".into()))); - // Another common variant assert!(is_dns_error(&RelayError::Http( "failed to lookup address information".into() ))); - // F15: production-shaped error — RelayError::WebSocket wrapping a - // tungstenite I/O error (the shape emitted by connect_async on macOS). let ws_io_err = RelayError::WebSocket(Box::new(tungstenite::Error::Io( std::io::Error::other("nodename nor servname provided, or not known"), ))); @@ -6092,7 +6558,6 @@ mod tests { is_dns_error(&ws_io_err), "WebSocket-wrapped I/O DNS error must be classified as DNS" ); - // Normal connection errors are NOT DNS errors. assert!(!is_dns_error(&RelayError::Timeout)); assert!(!is_dns_error(&RelayError::ConnectionClosed)); assert!(!is_dns_error(&RelayError::Http( diff --git a/crates/buzz-agent/Cargo.toml b/crates/buzz-agent/Cargo.toml index fabf75754e1..7433a6dcb24 100644 --- a/crates/buzz-agent/Cargo.toml +++ b/crates/buzz-agent/Cargo.toml @@ -41,17 +41,18 @@ axum = { workspace = true } base64 = "0.22" hex = { workspace = true } sha2 = { workspace = true } +nostr = { workspace = true } url = { workspace = true } urlencoding = "2" webbrowser = "1" dirs = "6" [target.'cfg(unix)'.dependencies] -nix = { version = "0.31", default-features = false, features = ["signal", "process"] } +nix = { version = "0.31", default-features = false, features = ["fs", "signal", "process"] } [dev-dependencies] tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "macros", "io-std", "io-util", "sync", "process", "time", "net"] } -nix = { version = "0.31", default-features = false, features = ["signal", "process"] } +nix = { version = "0.31", default-features = false, features = ["fs", "signal", "process"] } axum = { workspace = true } hex = { workspace = true } serde = { workspace = true } diff --git a/crates/buzz-agent/src/activity_ledger_today.rs b/crates/buzz-agent/src/activity_ledger_today.rs new file mode 100644 index 00000000000..005ad37a2ef --- /dev/null +++ b/crates/buzz-agent/src/activity_ledger_today.rs @@ -0,0 +1,834 @@ +use nostr::{Event, JsonUtil}; +use serde::Serialize; +use serde_json::{json, Map, Value}; +use sha2::Digest; + +use crate::types::{ToolDef, ToolResult, ToolResultContent}; + +pub const ACTIVITY_LEDGER_TODAY_TOOL: &str = "get_activity_ledger_today"; +const ACTIVITY_LEDGER_TODAY_SCHEMA: &str = "buzz.activity-ledger.today/v1"; +const ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE: &str = "buzz.activity-ledger.today.read/v1"; +const ACTIVITY_LEDGER_TODAY_PATH_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_PATH"; +const ACTIVITY_LEDGER_TODAY_CAPABILITY_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_CAPABILITY"; +const ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY"; +const ACTIVITY_LEDGER_TODAY_RELAY_URL_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_RELAY_URL"; +const ACTIVITY_LEDGER_MAX_LIFETIME_SECS: u64 = 24 * 60 * 60; +const ACTIVITY_LEDGER_MAX_SNAPSHOT_BYTES: u64 = 8 * 1024 * 1024; +const ACTIVITY_LEDGER_MAX_FUTURE_GENERATED_AT_SECS: u64 = 300; +const ACTIVITY_LEDGER_DEFAULT_LIMIT: usize = 25; +const ACTIVITY_LEDGER_TODAY_SIGNED_KIND: u16 = 24202; +const ACTIVITY_LEDGER_TODAY_SIGNED_TAG_MARKER: &str = "buzz-activity-ledger-today"; + +pub fn activity_ledger_today_enabled() -> bool { + env_non_empty(ACTIVITY_LEDGER_TODAY_PATH_ENV).is_some() + && env_non_empty(ACTIVITY_LEDGER_TODAY_CAPABILITY_ENV).is_some() + && env_non_empty(ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY_ENV).is_some() + && env_non_empty(ACTIVITY_LEDGER_TODAY_RELAY_URL_ENV).is_some() +} + +pub fn activity_ledger_today_def() -> ToolDef { + ToolDef { + name: ACTIVITY_LEDGER_TODAY_TOOL.to_owned(), + description: "Read the owner-authorized Buzz Activity Ledger Today snapshot from a local Desktop-produced file. Fails closed if the snapshot is missing, stale, misconfigured, or does not match the configured capability." + .to_owned(), + input_schema: json!({ + "type": "object", + "properties": { + "channelId": { + "type": "string", + "description": "Optional exact channel id filter." + }, + "agentPubkey": { + "type": "string", + "description": "Optional exact agent pubkey filter." + }, + "status": { + "type": "string", + "description": "Optional exact mission journal status filter." + }, + "proofState": { + "type": "string", + "description": "Optional exact proof state filter." + }, + "limit": { + "type": "integer", + "description": "Maximum journals to return, from 1 to 100." + }, + "before": { + "type": "object", + "description": "Optional continuation cursor from nextBefore for the next older page.", + "properties": { + "endedAt": { "type": "string" }, + "agentPubkey": { "type": "string" }, + "id": { "type": "string" } + }, + "required": ["endedAt", "agentPubkey", "id"], + "additionalProperties": false + }, + "includeEvents": { + "type": "boolean", + "description": "When true, include each journal's normalized events. Defaults to false." + } + } + }), + } +} + +pub async fn call_activity_ledger_today(arguments: &Value, max_text_bytes: usize) -> ToolResult { + let Some(path) = env_non_empty(ACTIVITY_LEDGER_TODAY_PATH_ENV) else { + return error_result(&format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: missing {ACTIVITY_LEDGER_TODAY_PATH_ENV}" + )); + }; + let Some(capability) = env_non_empty(ACTIVITY_LEDGER_TODAY_CAPABILITY_ENV) else { + return error_result(&format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: missing {ACTIVITY_LEDGER_TODAY_CAPABILITY_ENV}" + )); + }; + let Some(expected_owner_pubkey) = env_non_empty(ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY_ENV) else { + return error_result(&format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: missing {ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY_ENV}" + )); + }; + let Some(expected_relay_url) = env_non_empty(ACTIVITY_LEDGER_TODAY_RELAY_URL_ENV) else { + return error_result(&format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: missing {ACTIVITY_LEDGER_TODAY_RELAY_URL_ENV}" + )); + }; + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let arguments = arguments.clone(); + match tokio::task::spawn_blocking(move || { + read_activity_ledger_today( + &path, + &capability, + &expected_owner_pubkey, + &expected_relay_url, + &arguments, + now_secs, + ) + }) + .await + .unwrap_or_else(|e| Err(format!("{ACTIVITY_LEDGER_TODAY_TOOL}: task failed: {e}"))) + { + Ok(output) => success_result(output, max_text_bytes), + Err(msg) => error_result(&msg), + } +} + +fn success_result(output: String, max_text_bytes: usize) -> ToolResult { + if output.len() > max_text_bytes { + return error_result(&format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: query result is {} bytes and exceeds the {max_text_bytes}-byte model text budget; retry with includeEvents false or a smaller limit, then use nextBefore to retrieve older pages", + output.len() + )); + } + ToolResult { + provider_id: String::new(), + content: vec![ToolResultContent::Text(output)], + is_error: false, + } +} + +fn env_non_empty(name: &str) -> Option { + std::env::var(name).ok().and_then(|value| { + let trimmed = value.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_owned()) + }) +} + +#[derive(Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "camelCase")] +struct ActivityLedgerCursor { + ended_at: String, + agent_pubkey: String, + id: String, +} + +#[derive(Clone)] +struct ActivityLedgerQuery { + channel_id: Option, + agent_pubkey: Option, + status: Option, + proof_state: Option, + limit: usize, + before: Option, + include_events: bool, +} + +fn read_activity_ledger_today( + path: &str, + capability: &str, + expected_owner_pubkey: &str, + expected_relay_url: &str, + arguments: &Value, + now_secs: u64, +) -> Result { + let query = parse_activity_ledger_query(arguments)?; + let path_buf = std::path::PathBuf::from(path); + if !path_buf.is_absolute() { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot path must be absolute" + )); + } + let body = read_activity_ledger_snapshot_body(&path_buf)?; + let root: Value = serde_json::from_str(&body) + .map_err(|e| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: invalid snapshot JSON: {e}"))?; + let result = filter_activity_ledger_snapshot( + &root, + capability, + expected_owner_pubkey, + expected_relay_url, + &query, + now_secs, + )?; + serde_json::to_string_pretty(&result) + .map_err(|e| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: could not serialize query result: {e}")) +} + +#[cfg(unix)] +fn read_activity_ledger_snapshot_body(path: &std::path::Path) -> Result { + use nix::errno::Errno; + use nix::fcntl::{open, OFlag}; + use nix::sys::stat::{fstat, Mode, SFlag}; + use std::io::Read; + use std::os::fd::OwnedFd; + use std::os::unix::fs::PermissionsExt; + + let fd: OwnedFd = + open(path, OFlag::O_RDONLY | OFlag::O_NOFOLLOW, Mode::empty()).map_err(|e| { + if e == Errno::ELOOP { + format!("{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot path must not be a symlink") + } else { + format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: could not open snapshot {:?}: {e}", + path + ) + } + })?; + let stat = fstat(&fd).map_err(|e| { + format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: could not stat opened snapshot {:?}: {e}", + path + ) + })?; + if SFlag::from_bits_truncate(stat.st_mode) != SFlag::S_IFREG { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot path must be a regular file" + )); + } + let mode = stat.st_mode & 0o777; + if mode != 0o600 { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot mode must be 0600, got {:03o}", + mode + )); + } + + let mut file = std::fs::File::from(fd); + let metadata = file.metadata().map_err(|e| { + format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: could not read snapshot metadata {:?}: {e}", + path + ) + })?; + let file_mode = metadata.permissions().mode() & 0o777; + if file_mode != 0o600 { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot mode must be 0600, got {:03o}", + file_mode + )); + } + if metadata.len() > ACTIVITY_LEDGER_MAX_SNAPSHOT_BYTES { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot exceeds {ACTIVITY_LEDGER_MAX_SNAPSHOT_BYTES} bytes" + )); + } + + let mut body = String::new(); + file.read_to_string(&mut body).map_err(|e| { + format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: could not read snapshot {:?}: {e}", + path + ) + })?; + Ok(body) +} + +#[cfg(not(unix))] +fn read_activity_ledger_snapshot_body(path: &std::path::Path) -> Result { + let symlink_meta = std::fs::symlink_metadata(path).map_err(|e| { + format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: could not stat snapshot {:?}: {e}", + path + ) + })?; + if symlink_meta.file_type().is_symlink() { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot path must not be a symlink" + )); + } + if !symlink_meta.file_type().is_file() { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot path must be a regular file" + )); + } + if symlink_meta.len() > ACTIVITY_LEDGER_MAX_SNAPSHOT_BYTES { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot exceeds {ACTIVITY_LEDGER_MAX_SNAPSHOT_BYTES} bytes" + )); + } + std::fs::read_to_string(path).map_err(|e| { + format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: could not read snapshot {:?}: {e}", + path + ) + }) +} + +fn parse_activity_ledger_query(arguments: &Value) -> Result { + let object = arguments + .as_object() + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: arguments must be an object"))?; + Ok(ActivityLedgerQuery { + channel_id: optional_string_arg(object, "channelId")?, + agent_pubkey: optional_string_arg(object, "agentPubkey")?, + status: optional_string_arg(object, "status")?, + proof_state: optional_string_arg(object, "proofState")?, + limit: parse_limit_arg(object.get("limit"))?, + before: parse_cursor_arg(object.get("before"))?, + include_events: parse_bool_arg(object.get("includeEvents"))?, + }) +} + +fn parse_cursor_arg(value: Option<&Value>) -> Result, String> { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Ok(None); + }; + let object = value + .as_object() + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: before must be a cursor object"))?; + if object.len() != 3 { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: before must contain only endedAt, agentPubkey, and id" + )); + } + Ok(Some(ActivityLedgerCursor { + ended_at: required_string_field(object, "endedAt")?.to_owned(), + agent_pubkey: required_string_field(object, "agentPubkey")?.to_owned(), + id: required_string_field(object, "id")?.to_owned(), + })) +} + +fn optional_string_arg(object: &Map, key: &str) -> Result, String> { + match object.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(value)) => { + let trimmed = value.trim(); + if trimmed.is_empty() { + Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: {key} must not be blank" + )) + } else { + Ok(Some(trimmed.to_owned())) + } + } + Some(_) => Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: {key} must be a string" + )), + } +} + +fn parse_limit_arg(value: Option<&Value>) -> Result { + match value { + None | Some(Value::Null) => Ok(ACTIVITY_LEDGER_DEFAULT_LIMIT), + Some(Value::Number(number)) => { + let Some(limit) = number.as_u64() else { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: limit must be an integer from 1 to 100" + )); + }; + if !(1..=100).contains(&limit) { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: limit must be an integer from 1 to 100" + )); + } + Ok(limit as usize) + } + Some(_) => Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: limit must be an integer from 1 to 100" + )), + } +} + +fn parse_bool_arg(value: Option<&Value>) -> Result { + match value { + None | Some(Value::Null) => Ok(false), + Some(Value::Bool(flag)) => Ok(*flag), + Some(_) => Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: includeEvents must be a boolean" + )), + } +} + +fn filter_activity_ledger_snapshot( + root: &Value, + capability: &str, + expected_owner_pubkey: &str, + expected_relay_url: &str, + query: &ActivityLedgerQuery, + now_secs: u64, +) -> Result { + let object = root + .as_object() + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot root must be an object"))?; + require_string_field(object, "schema", ACTIVITY_LEDGER_TODAY_SCHEMA)?; + require_string_field(object, "capability", capability)?; + let owner_pubkey = required_string_field(object, "ownerPubkey")?; + if !is_hex_64(owner_pubkey) { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: ownerPubkey must be 64 lowercase hex chars" + )); + } + if owner_pubkey != expected_owner_pubkey { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: ownerPubkey mismatch: expected {expected_owner_pubkey:?}, got {owner_pubkey:?}" + )); + } + let relay_url = required_string_field(object, "relayUrl")?; + if relay_url != expected_relay_url { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: relayUrl mismatch: expected {expected_relay_url:?}, got {relay_url:?}" + )); + } + let generated_at = required_u64_field(object, "generatedAt")?; + let expires_at = required_u64_field(object, "expiresAt")?; + if generated_at > now_secs.saturating_add(ACTIVITY_LEDGER_MAX_FUTURE_GENERATED_AT_SECS) { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: generatedAt is more than {} seconds in the future", + ACTIVITY_LEDGER_MAX_FUTURE_GENERATED_AT_SECS + )); + } + if expires_at <= generated_at { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: expiresAt must be greater than generatedAt" + )); + } + if expires_at - generated_at > ACTIVITY_LEDGER_MAX_LIFETIME_SECS { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot lifetime exceeds {} seconds", + ACTIVITY_LEDGER_MAX_LIFETIME_SECS + )); + } + if now_secs >= expires_at { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot expired at {expires_at}" + )); + } + let snapshot_sha256 = required_string_field(object, "snapshotSha256")?; + require_lower_hex_len(snapshot_sha256, "snapshotSha256", 64)?; + let event_id = required_string_field(object, "eventId")?; + require_lower_hex_len(event_id, "eventId", 64)?; + let signature = required_string_field(object, "signature")?; + require_lower_hex_len(signature, "signature", 128)?; + verify_activity_ledger_snapshot_signature( + object, + &ActivityLedgerSnapshotSignatureFields { + owner_pubkey, + relay_url, + generated_at, + expires_at, + snapshot_sha256, + event_id, + signature, + }, + )?; + + let surface = object + .get("surface") + .and_then(Value::as_object) + .ok_or_else(|| { + format!("{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot surface must be an object") + })?; + let day = required_string_field(surface, "day")?.to_owned(); + let journals = surface + .get("journals") + .and_then(Value::as_array) + .ok_or_else(|| { + format!("{ACTIVITY_LEDGER_TODAY_TOOL}: surface.journals must be an array") + })?; + let source_projection = surface + .get("snapshotProjection") + .cloned() + .unwrap_or(Value::Null); + + let mut filtered = Vec::new(); + for journal in journals { + if journal_matches_query(journal, query)? { + let projected = strip_events_from_journal(journal, query.include_events)?; + journal_cursor(&projected)?; + filtered.push(projected); + } + } + let matching_journals = filtered.len(); + filtered.sort_by(|left, right| journal_sort_key(left).cmp(&journal_sort_key(right))); + if let Some(before) = &query.before { + filtered.retain(|journal| journal_cursor(journal).is_ok_and(|cursor| &cursor < before)); + } + let eligible_before_cursor = filtered.len(); + let truncated = eligible_before_cursor > query.limit; + if truncated { + let oldest_to_drop = filtered.len() - query.limit; + filtered.drain(..oldest_to_drop); + } + let next_before = if truncated { + filtered.first().map(journal_cursor).transpose()? + } else { + None + }; + + let channels = rebuild_filtered_channels(&filtered); + let failed = filtered + .iter() + .filter(|journal| string_field(journal, "status") == Some("failed")) + .count(); + let in_progress = filtered + .iter() + .filter(|journal| string_field(journal, "status") == Some("in_progress")) + .count(); + let claimed_without_evidence = filtered + .iter() + .filter(|journal| bool_field(journal, "claimedCompletionWithoutEvidence")) + .count(); + + Ok(json!({ + "schema": "buzz.activity-ledger.today.query-result/v1", + "sourceSchema": ACTIVITY_LEDGER_TODAY_SCHEMA, + "day": day, + "ownerPubkey": owner_pubkey, + "relayUrl": relay_url, + "generatedAt": generated_at, + "expiresAt": expires_at, + "capability": capability, + "sourceProjection": source_projection, + "filters": { + "channelId": query.channel_id, + "agentPubkey": query.agent_pubkey, + "status": query.status, + "proofState": query.proof_state, + "limit": query.limit, + "before": query.before.clone(), + "includeEvents": query.include_events, + }, + "counts": { + "matchingJournals": matching_journals, + "eligibleBeforeCursor": eligible_before_cursor, + "returnedJournals": filtered.len(), + "failed": failed, + "inProgress": in_progress, + "claimedWithoutEvidence": claimed_without_evidence, + }, + "truncated": truncated, + "nextBefore": next_before, + "journals": filtered, + "channels": channels, + })) +} + +fn journal_cursor(journal: &Value) -> Result { + let object = journal + .as_object() + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: each journal must be an object"))?; + Ok(ActivityLedgerCursor { + ended_at: required_string_field(object, "endedAt")?.to_owned(), + agent_pubkey: required_string_field(object, "agentPubkey")?.to_owned(), + id: required_string_field(object, "id")?.to_owned(), + }) +} + +fn journal_sort_key(journal: &Value) -> (&str, &str, &str) { + ( + string_field(journal, "endedAt").unwrap_or_default(), + string_field(journal, "agentPubkey").unwrap_or_default(), + string_field(journal, "id").unwrap_or_default(), + ) +} + +fn journal_matches_query(journal: &Value, query: &ActivityLedgerQuery) -> Result { + let object = journal + .as_object() + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: each journal must be an object"))?; + if let Some(channel_id) = &query.channel_id { + if object.get("channelId").and_then(Value::as_str) != Some(channel_id.as_str()) { + return Ok(false); + } + } + if let Some(agent_pubkey) = &query.agent_pubkey { + if object.get("agentPubkey").and_then(Value::as_str) != Some(agent_pubkey.as_str()) { + return Ok(false); + } + } + if let Some(status) = &query.status { + if object.get("status").and_then(Value::as_str) != Some(status.as_str()) { + return Ok(false); + } + } + if let Some(proof_state) = &query.proof_state { + if object.get("proofState").and_then(Value::as_str) != Some(proof_state.as_str()) { + return Ok(false); + } + } + Ok(true) +} + +fn strip_events_from_journal(journal: &Value, include_events: bool) -> Result { + if include_events { + return Ok(journal.clone()); + } + let mut object = journal + .as_object() + .cloned() + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: each journal must be an object"))?; + object.remove("events"); + Ok(Value::Object(object)) +} + +type FilteredChannelSummary = ( + Vec, + std::collections::BTreeSet, + std::collections::BTreeSet, + String, +); + +fn rebuild_filtered_channels(journals: &[Value]) -> Vec { + let mut channels: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for journal in journals { + let Some(channel_id) = string_field(journal, "channelId").map(str::to_owned) else { + continue; + }; + let journal_id = string_field(journal, "id").unwrap_or_default().to_owned(); + let agent_pubkey = string_field(journal, "agentPubkey") + .unwrap_or_default() + .to_owned(); + let agent_name = string_field(journal, "agentName") + .unwrap_or_default() + .to_owned(); + let ended_at = string_field(journal, "endedAt") + .unwrap_or_default() + .to_owned(); + + let entry = channels.entry(channel_id).or_insert_with(|| { + ( + Vec::new(), + std::collections::BTreeSet::new(), + std::collections::BTreeSet::new(), + ended_at.clone(), + ) + }); + entry.0.push(journal_id); + if !agent_pubkey.is_empty() { + entry.1.insert(agent_pubkey); + } + if !agent_name.is_empty() { + entry.2.insert(agent_name); + } + if ended_at > entry.3 { + entry.3 = ended_at; + } + } + + channels + .into_iter() + .map( + |(channel_id, (journal_ids, agent_pubkeys, agent_names, last_activity_at))| { + json!({ + "channelId": channel_id, + "journalIds": journal_ids, + "agentPubkeys": agent_pubkeys.into_iter().collect::>(), + "agentNames": agent_names.into_iter().collect::>(), + "lastActivityAt": last_activity_at, + }) + }, + ) + .collect() +} + +fn require_string_field( + object: &Map, + key: &str, + expected: &str, +) -> Result<(), String> { + let value = required_string_field(object, key)?; + if value != expected { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: {key} mismatch: expected {expected:?}, got {value:?}" + )); + } + Ok(()) +} + +fn require_lower_hex_len(value: &str, key: &str, len: usize) -> Result<(), String> { + if value.len() != len + || !value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: {key} must be {len} lowercase hex chars" + )); + } + Ok(()) +} + +fn required_string_field<'a>(object: &'a Map, key: &str) -> Result<&'a str, String> { + object + .get(key) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: missing string field {key}")) +} + +fn required_u64_field(object: &Map, key: &str) -> Result { + object + .get(key) + .and_then(Value::as_u64) + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: missing integer field {key}")) +} + +fn string_field<'a>(value: &'a Value, key: &str) -> Option<&'a str> { + value.as_object()?.get(key)?.as_str() +} + +fn bool_field(value: &Value, key: &str) -> bool { + value + .as_object() + .and_then(|object| object.get(key)) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +fn is_hex_64(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +struct ActivityLedgerSnapshotSignatureFields<'a> { + owner_pubkey: &'a str, + relay_url: &'a str, + generated_at: u64, + expires_at: u64, + snapshot_sha256: &'a str, + event_id: &'a str, + signature: &'a str, +} + +fn verify_activity_ledger_snapshot_signature( + object: &Map, + fields: &ActivityLedgerSnapshotSignatureFields<'_>, +) -> Result<(), String> { + let payload_json = canonical_activity_ledger_snapshot_payload_json( + object, + fields.owner_pubkey, + fields.relay_url, + fields.generated_at, + fields.expires_at, + )?; + let payload_sha256 = hex::encode(sha2::Sha256::digest(payload_json.as_bytes())); + if payload_sha256 != fields.snapshot_sha256 { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshotSha256 does not match the canonical payload" + )); + } + let event_json = serde_json::json!({ + "id": fields.event_id, + "pubkey": fields.owner_pubkey, + "created_at": fields.generated_at, + "kind": ACTIVITY_LEDGER_TODAY_SIGNED_KIND, + "tags": [ + ["t", ACTIVITY_LEDGER_TODAY_SIGNED_TAG_MARKER], + ["schema", ACTIVITY_LEDGER_TODAY_SCHEMA], + ["capability", ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE], + ["snapshot_sha256", fields.snapshot_sha256], + ["expires_at", fields.expires_at.to_string()] + ], + "content": payload_json, + "sig": fields.signature, + }) + .to_string(); + let event = Event::from_json(&event_json) + .map_err(|e| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: invalid signed snapshot event: {e}"))?; + event.verify().map_err(|e| { + format!("{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot signature verification failed: {e}") + })?; + if event.id.to_hex() != fields.event_id { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: eventId does not match the signed snapshot event" + )); + } + if event.pubkey.to_hex() != fields.owner_pubkey { + return Err(format!( + "{ACTIVITY_LEDGER_TODAY_TOOL}: snapshot signer is not the expected owner" + )); + } + Ok(()) +} + +fn canonical_activity_ledger_snapshot_payload_json( + object: &Map, + owner_pubkey: &str, + relay_url: &str, + generated_at: u64, + expires_at: u64, +) -> Result { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct CanonicalPayload<'a> { + schema: &'static str, + owner_pubkey: &'a str, + relay_url: &'a str, + generated_at: u64, + expires_at: u64, + capability: &'static str, + surface: &'a Value, + raw_events: &'a [Value], + } + + let surface = object + .get("surface") + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: missing surface field"))?; + let raw_events = object + .get("rawEvents") + .and_then(Value::as_array) + .ok_or_else(|| format!("{ACTIVITY_LEDGER_TODAY_TOOL}: missing rawEvents field"))?; + serde_json::to_string(&CanonicalPayload { + schema: ACTIVITY_LEDGER_TODAY_SCHEMA, + owner_pubkey, + relay_url, + generated_at, + expires_at, + capability: ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE, + surface, + raw_events, + }) + .map_err(|e| { + format!("{ACTIVITY_LEDGER_TODAY_TOOL}: could not canonicalize snapshot payload: {e}") + }) +} + +fn error_result(msg: &str) -> ToolResult { + ToolResult { + provider_id: String::new(), + content: vec![ToolResultContent::Text(msg.to_owned())], + is_error: true, + } +} + +#[cfg(test)] +#[path = "activity_ledger_today_tests.rs"] +mod tests; diff --git a/crates/buzz-agent/src/activity_ledger_today_tests.rs b/crates/buzz-agent/src/activity_ledger_today_tests.rs new file mode 100644 index 00000000000..cc5f38f279d --- /dev/null +++ b/crates/buzz-agent/src/activity_ledger_today_tests.rs @@ -0,0 +1,486 @@ +use super::*; +use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp}; +use std::path::PathBuf; +use tempfile::TempDir; + +const TEST_RELAY: &str = "wss://relay-a.test"; + +fn write_activity_snapshot( + dir: &TempDir, + capability: &str, + generated_at: u64, + expires_at: u64, +) -> PathBuf { + let path = dir.path().join("activity-ledger-today.json"); + let owner_secret = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let owner_keys = Keys::parse(owner_secret).unwrap(); + let owner_pubkey = owner_keys.public_key().to_hex(); + let unsigned_snapshot = json!({ + "schema": ACTIVITY_LEDGER_TODAY_SCHEMA, + "ownerPubkey": owner_pubkey, + "relayUrl": TEST_RELAY, + "generatedAt": generated_at, + "expiresAt": expires_at, + "capability": capability, + "surface": { + "day": "2026-08-21", + "journals": [ + { + "id": "journal-a", + "channelId": "chan-a", + "agentPubkey": "agent-a", + "agentName": "Honey", + "status": "completed", + "proofState": "RECEIPTED", + "endedAt": "2026-08-21T14:00:00.000Z", + "claimedCompletionWithoutEvidence": false, + "events": [ + { "id": "event-a", "detail": "receipted activity" } + ] + }, + { + "id": "journal-b", + "channelId": "chan-b", + "agentPubkey": "agent-b", + "agentName": "Fizz", + "status": "failed", + "proofState": "FAILED", + "endedAt": "2026-08-21T15:00:00.000Z", + "claimedCompletionWithoutEvidence": true, + "events": [ + { "id": "event-b", "detail": "failed activity" } + ] + } + ], + "snapshotProjection": { + "bounded": false, + "maxBytes": 6291456, + "originalJournals": 2, + "includedJournals": 2, + "omittedJournals": 0, + "omittedEvents": 0, + "textFieldsTruncated": 0 + } + }, + "rawEvents": [] + }); + let canonical_payload = canonical_activity_ledger_snapshot_payload_json( + unsigned_snapshot.as_object().unwrap(), + &owner_pubkey, + TEST_RELAY, + generated_at, + expires_at, + ) + .unwrap(); + let snapshot_sha256 = hex::encode(sha2::Sha256::digest(canonical_payload.as_bytes())); + let event = EventBuilder::new( + Kind::Custom(ACTIVITY_LEDGER_TODAY_SIGNED_KIND), + canonical_payload, + ) + .tags([ + Tag::parse(["t", ACTIVITY_LEDGER_TODAY_SIGNED_TAG_MARKER]).unwrap(), + Tag::parse(["schema", ACTIVITY_LEDGER_TODAY_SCHEMA]).unwrap(), + Tag::parse(["capability", capability]).unwrap(), + Tag::parse(["snapshot_sha256", &snapshot_sha256]).unwrap(), + Tag::parse(["expires_at", &expires_at.to_string()]).unwrap(), + ]) + .custom_created_at(Timestamp::from(generated_at)) + .sign_with_keys(&owner_keys) + .unwrap(); + let signed_snapshot = json!({ + "schema": ACTIVITY_LEDGER_TODAY_SCHEMA, + "ownerPubkey": owner_pubkey, + "relayUrl": TEST_RELAY, + "generatedAt": generated_at, + "expiresAt": expires_at, + "capability": capability, + "surface": unsigned_snapshot["surface"].clone(), + "rawEvents": [], + "snapshotSha256": snapshot_sha256, + "eventId": event.id.to_hex(), + "signature": event.sig.to_string(), + }); + std::fs::write(&path, serde_json::to_vec(&signed_snapshot).unwrap()).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + path +} + +fn write_custom_activity_snapshot(dir: &TempDir, snapshot: Value) -> PathBuf { + let path = dir.path().join("activity-ledger-today.json"); + std::fs::write(&path, serde_json::to_vec(&snapshot).unwrap()).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + path +} + +fn expected_owner_pubkey() -> String { + Keys::parse("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + .unwrap() + .public_key() + .to_hex() +} + +#[test] +fn desktop_canonical_payload_fixture_matches_honey_reconstruction() { + let fixture = include_str!("../../../test-fixtures/activity-ledger-today-desktop.json").trim(); + let root: Value = serde_json::from_str(fixture).unwrap(); + let object = root.as_object().unwrap(); + let reconstructed = canonical_activity_ledger_snapshot_payload_json( + object, + object["ownerPubkey"].as_str().unwrap(), + object["relayUrl"].as_str().unwrap(), + object["generatedAt"].as_u64().unwrap(), + object["expiresAt"].as_u64().unwrap(), + ) + .unwrap(); + assert_eq!(reconstructed, fixture); +} + +#[test] +fn activity_ledger_today_filters_and_strips_events_by_default() { + let tmp = TempDir::new().unwrap(); + let capability = ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE; + let path = write_activity_snapshot(&tmp, capability, 100, 160); + + let output = read_activity_ledger_today( + path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({"agentPubkey": "agent-a", "limit": 10}), + 120, + ) + .unwrap(); + let result: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(result["counts"]["matchingJournals"], 1); + assert_eq!(result["counts"]["returnedJournals"], 1); + assert_eq!(result["journals"][0]["id"], "journal-a"); + assert!(result["journals"][0].get("events").is_none()); + assert_eq!(result["channels"][0]["channelId"], "chan-a"); +} + +#[test] +fn activity_ledger_today_includes_events_when_requested() { + let tmp = TempDir::new().unwrap(); + let capability = ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE; + let path = write_activity_snapshot(&tmp, capability, 100, 160); + + let output = read_activity_ledger_today( + path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({"channelId": "chan-b", "includeEvents": true, "limit": 10}), + 120, + ) + .unwrap(); + let result: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(result["journals"][0]["id"], "journal-b"); + assert_eq!(result["journals"][0]["events"][0]["id"], "event-b"); + assert_eq!(result["counts"]["failed"], 1); + assert_eq!(result["counts"]["claimedWithoutEvidence"], 1); +} + +#[test] +fn activity_ledger_today_limit_keeps_newest_matching_journals() { + let tmp = TempDir::new().unwrap(); + let capability = ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE; + let path = write_activity_snapshot(&tmp, capability, 100, 160); + + let output = read_activity_ledger_today( + path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({"limit": 1}), + 120, + ) + .unwrap(); + let result: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(result["counts"]["matchingJournals"], 2); + assert_eq!(result["counts"]["returnedJournals"], 1); + assert_eq!(result["journals"][0]["id"], "journal-b"); + assert_eq!(result["channels"][0]["channelId"], "chan-b"); + assert_eq!(result["truncated"], true); + assert_eq!(result["nextBefore"]["id"], "journal-b"); + assert_eq!(result["sourceProjection"]["bounded"], false); + + let older_output = read_activity_ledger_today( + path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({"limit": 1, "before": result["nextBefore"].clone()}), + 120, + ) + .unwrap(); + let older: Value = serde_json::from_str(&older_output).unwrap(); + assert_eq!(older["journals"][0]["id"], "journal-a"); + assert_eq!(older["truncated"], false); + assert_eq!(older["nextBefore"], Value::Null); +} + +#[test] +fn activity_ledger_today_result_fails_instead_of_corrupting_json_over_budget() { + let output = serde_json::to_string(&json!({"journals": [{ + "id": "large", + "summary": "x".repeat(4_096) + }]})) + .unwrap(); + let result = success_result(output, 512); + let ToolResultContent::Text(text) = &result.content[0] else { + panic!("Today result must stay text-only"); + }; + assert!(result.is_error); + assert!(text.contains("exceeds the 512-byte model text budget")); + assert!(text.contains("smaller limit")); +} + +#[test] +fn activity_ledger_today_cursor_separates_same_time_and_id_across_agents() { + let left = json!({ + "endedAt": "2026-08-21T15:00:00Z", + "agentPubkey": "agent-a", + "id": "shared-journal", + }); + let right = json!({ + "endedAt": "2026-08-21T15:00:00Z", + "agentPubkey": "agent-b", + "id": "shared-journal", + }); + + assert!(journal_cursor(&left).unwrap() < journal_cursor(&right).unwrap()); +} + +#[test] +fn activity_ledger_today_rejects_relative_path() { + let error = read_activity_ledger_today( + "relative.json", + ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + error.contains("snapshot path must be absolute"), + "got: {error}" + ); +} + +#[test] +fn activity_ledger_today_rejects_oversized_snapshot_before_reading() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("oversized.json"); + let file = std::fs::File::create(&path).unwrap(); + file.set_len(ACTIVITY_LEDGER_MAX_SNAPSHOT_BYTES + 1) + .unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + let error = read_activity_ledger_today( + path.to_str().unwrap(), + ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!(error.contains("snapshot exceeds"), "got: {error}"); +} + +#[test] +fn activity_ledger_today_rejects_capability_mismatch_and_staleness() { + let tmp = TempDir::new().unwrap(); + let capability = ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE; + let path = write_activity_snapshot(&tmp, "wrong-capability", 100, 160); + + let capability_error = read_activity_ledger_today( + path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + capability_error.contains("capability mismatch"), + "got: {capability_error}" + ); + + let stale_path = write_activity_snapshot(&tmp, capability, 100, 110); + let stale_error = read_activity_ledger_today( + stale_path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + stale_error.contains("snapshot expired"), + "got: {stale_error}" + ); +} + +#[test] +fn activity_ledger_today_rejects_future_generated_at_and_uppercase_owner() { + let tmp = TempDir::new().unwrap(); + let capability = ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE; + + let future_path = write_activity_snapshot(&tmp, capability, 500, 560); + let future_error = read_activity_ledger_today( + future_path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + future_error.contains("generatedAt is more than 300 seconds in the future"), + "got: {future_error}" + ); + + let uppercase_owner_path = write_custom_activity_snapshot( + &tmp, + json!({ + "schema": ACTIVITY_LEDGER_TODAY_SCHEMA, + "ownerPubkey": "ABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCD", + "generatedAt": 100, + "expiresAt": 160, + "capability": capability, + "surface": { + "day": "2026-08-21", + "journals": [] + }, + "rawEvents": [], + "snapshotSha256": "a".repeat(64), + "eventId": "b".repeat(64), + "signature": "c".repeat(128), + }), + ); + let owner_error = read_activity_ledger_today( + uppercase_owner_path.to_str().unwrap(), + capability, + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + owner_error.contains("ownerPubkey must be 64 lowercase hex chars"), + "got: {owner_error}" + ); +} + +#[cfg(unix)] +#[test] +fn activity_ledger_today_rejects_symlink_and_non_0600_mode() { + use std::os::unix::fs::{symlink, PermissionsExt}; + + let tmp = TempDir::new().unwrap(); + let capability = ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE; + let path = write_activity_snapshot(&tmp, capability, 100, 160); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + let mode_error = read_activity_ledger_today( + path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + mode_error.contains("snapshot mode must be 0600"), + "got: {mode_error}" + ); + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + let link_path = tmp.path().join("linked.json"); + symlink(&path, &link_path).unwrap(); + let symlink_error = read_activity_ledger_today( + link_path.to_str().unwrap(), + capability, + &expected_owner_pubkey(), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + symlink_error.contains("must not be a symlink") + || symlink_error.contains("could not open snapshot"), + "got: {symlink_error}" + ); +} + +#[test] +fn activity_ledger_today_rejects_same_user_forged_rewrite_and_wrong_owner_env() { + let tmp = TempDir::new().unwrap(); + let capability = ACTIVITY_LEDGER_TODAY_CAPABILITY_VALUE; + let path = write_activity_snapshot(&tmp, capability, 100, 160); + let expected_owner = expected_owner_pubkey(); + let mut snapshot: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + snapshot["surface"]["journals"] = json!([{ "id": "forged" }]); + std::fs::write(&path, serde_json::to_vec(&snapshot).unwrap()).unwrap(); + let forged_error = read_activity_ledger_today( + path.to_str().unwrap(), + capability, + &expected_owner, + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + forged_error.contains("snapshotSha256 does not match") + || forged_error.contains("signature verification failed"), + "got: {forged_error}" + ); + + let fresh_path = write_activity_snapshot(&tmp, capability, 100, 160); + let wrong_owner_error = read_activity_ledger_today( + fresh_path.to_str().unwrap(), + capability, + &"f".repeat(64), + TEST_RELAY, + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + wrong_owner_error.contains("ownerPubkey mismatch"), + "got: {wrong_owner_error}" + ); + + let wrong_relay_error = read_activity_ledger_today( + fresh_path.to_str().unwrap(), + capability, + &expected_owner, + "wss://relay-b.test", + &json!({}), + 120, + ) + .unwrap_err(); + assert!( + wrong_relay_error.contains("relayUrl mismatch"), + "got: {wrong_relay_error}" + ); +} diff --git a/crates/buzz-agent/src/agent.rs b/crates/buzz-agent/src/agent.rs index 9258ce449f3..24bcfdb57b4 100644 --- a/crates/buzz-agent/src/agent.rs +++ b/crates/buzz-agent/src/agent.rs @@ -382,6 +382,9 @@ impl RunCtx<'_> { if !self.skills.is_empty() { tools.push(builtin::load_skill_def()); } + if builtin::activity_ledger_today_enabled() { + tools.push(builtin::activity_ledger_today_def()); + } round = round.saturating_add(1); let response_result = tokio::select! { biased; @@ -818,6 +821,18 @@ impl RunCtx<'_> { results[idx] = Some(result); continue; } + if call.name == builtin::ACTIVITY_LEDGER_TODAY_TOOL { + emit_in_progress(self.wire, self.session_id, call).await; + let mut result = builtin::call_activity_ledger_today( + &call.arguments, + self.cfg.max_tool_result_text_bytes, + ) + .await; + result.provider_id = call.provider_id.clone(); + emit_completed(self.wire, self.session_id, call, &result).await; + results[idx] = Some(result); + continue; + } // Hook tools (bare name starts with `_`) are invisible to the // LLM and only callable via `call_hooks`. Treat any direct diff --git a/crates/buzz-agent/src/builtin.rs b/crates/buzz-agent/src/builtin.rs index 9b604766d42..9c0b09f9596 100644 --- a/crates/buzz-agent/src/builtin.rs +++ b/crates/buzz-agent/src/builtin.rs @@ -11,6 +11,10 @@ use crate::mcp::truncate_at_boundary; use crate::types::{ToolDef, ToolResult, ToolResultContent}; pub const LOAD_SKILL_TOOL: &str = "load_skill"; +pub use crate::activity_ledger_today::{ + activity_ledger_today_def, activity_ledger_today_enabled, call_activity_ledger_today, + ACTIVITY_LEDGER_TODAY_TOOL, +}; /// Return the `ToolDef` for `load_skill` to include in the LLM tool list. pub fn load_skill_def() -> ToolDef { @@ -111,7 +115,6 @@ pub async fn call_load_skill(arguments: &Value, skills: &[SkillEntry]) -> ToolRe is_error: false, } } - /// Load a supporting file identified by `skill_name/rel_path`. /// Matches against the pre-enumerated `supporting_files` list and applies a /// canonicalize-based traversal guard before reading. @@ -272,7 +275,6 @@ mod tests { supporting_files, } } - #[tokio::test] async fn call_load_skill_missing_name_arg() { let result = call_load_skill(&serde_json::json!({}), &[]).await; diff --git a/crates/buzz-agent/src/lib.rs b/crates/buzz-agent/src/lib.rs index 98fa99ca5bf..912e1ac20b7 100644 --- a/crates/buzz-agent/src/lib.rs +++ b/crates/buzz-agent/src/lib.rs @@ -1,4 +1,5 @@ #![forbid(unsafe_code)] +mod activity_ledger_today; mod agent; pub mod auth; mod builtin; diff --git a/crates/buzz-agent/tests/hints_integration.rs b/crates/buzz-agent/tests/hints_integration.rs index 63a55514dbe..3739fd7c746 100644 --- a/crates/buzz-agent/tests/hints_integration.rs +++ b/crates/buzz-agent/tests/hints_integration.rs @@ -7,7 +7,10 @@ use std::process::Stdio; use std::sync::Arc; use std::time::Duration; +use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp}; +use serde::Serialize; use serde_json::{json, Value}; +use sha2::Digest; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::net::TcpListener; use tokio::sync::Mutex; @@ -572,3 +575,160 @@ async fn load_skill_tool_returns_body() { ); h.shutdown().await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn activity_ledger_today_tool_returns_filtered_snapshot() { + let tmp = tempfile::TempDir::new().unwrap(); + let cwd = tmp.path(); + let snapshot_path = cwd.join("activity-ledger-today.json"); + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let owner_keys = + Keys::parse("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef").unwrap(); + let owner_pubkey = owner_keys.public_key().to_hex(); + let relay_url = "wss://relay-a.test"; + let surface = json!({ + "day": "2026-08-21", + "journals": [ + { + "id": "journal-a", + "channelId": "chan-a", + "agentPubkey": "agent-a", + "agentName": "Honey", + "status": "completed", + "proofState": "RECEIPTED", + "endedAt": "2026-08-21T14:00:00.000Z", + "claimedCompletionWithoutEvidence": false, + "events": [{ "id": "event-a", "detail": "receipted activity" }] + } + ] + }); + let raw_events: Vec = Vec::new(); + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct CanonicalPayload<'a> { + schema: &'static str, + owner_pubkey: &'a str, + relay_url: &'a str, + generated_at: u64, + expires_at: u64, + capability: &'static str, + surface: &'a Value, + raw_events: &'a [Value], + } + let canonical_payload = serde_json::to_string(&CanonicalPayload { + schema: "buzz.activity-ledger.today/v1", + owner_pubkey: &owner_pubkey, + relay_url, + generated_at: now_secs.saturating_sub(60), + expires_at: now_secs + 300, + capability: "buzz.activity-ledger.today.read/v1", + surface: &surface, + raw_events: &raw_events, + }) + .unwrap(); + let snapshot_sha256 = hex::encode(sha2::Sha256::digest(canonical_payload.as_bytes())); + let signed_event = EventBuilder::new(Kind::Custom(24202), canonical_payload) + .tags([ + Tag::parse(["t", "buzz-activity-ledger-today"]).unwrap(), + Tag::parse(["schema", "buzz.activity-ledger.today/v1"]).unwrap(), + Tag::parse(["capability", "buzz.activity-ledger.today.read/v1"]).unwrap(), + Tag::parse(["snapshot_sha256", &snapshot_sha256]).unwrap(), + Tag::parse(["expires_at", &(now_secs + 300).to_string()]).unwrap(), + ]) + .custom_created_at(Timestamp::from(now_secs.saturating_sub(60))) + .sign_with_keys(&owner_keys) + .unwrap(); + let snapshot = json!({ + "schema": "buzz.activity-ledger.today/v1", + "ownerPubkey": owner_pubkey, + "relayUrl": relay_url, + "generatedAt": now_secs.saturating_sub(60), + "expiresAt": now_secs + 300, + "capability": "buzz.activity-ledger.today.read/v1", + "surface": surface, + "rawEvents": raw_events, + "snapshotSha256": snapshot_sha256, + "eventId": signed_event.id.to_hex(), + "signature": signed_event.sig.to_string(), + }); + std::fs::write(&snapshot_path, serde_json::to_vec(&snapshot).unwrap()).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&snapshot_path, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + + let tool_call = json!({ + "id": "cc-ledger", "object": "chat.completion", "model": "fake-model", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", "content": null, + "tool_calls": [{ + "id": "tc-ledger", "type": "function", + "function": { + "name": "get_activity_ledger_today", + "arguments": "{\"agentPubkey\":\"agent-a\"}" + } + }] + }, + "finish_reason": "tool_calls" + }] + }); + let end_turn = openai_text("done"); + + let llm = spawn_capturing_llm(vec![tool_call, end_turn]).await; + let mut h = Harness::spawn_with_env( + &llm.url, + &[ + ( + "BUZZ_ACTIVITY_LEDGER_TODAY_PATH", + snapshot_path.to_str().unwrap(), + ), + ( + "BUZZ_ACTIVITY_LEDGER_TODAY_CAPABILITY", + "buzz.activity-ledger.today.read/v1", + ), + ( + "BUZZ_ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY", + owner_pubkey.as_str(), + ), + ("BUZZ_ACTIVITY_LEDGER_TODAY_RELAY_URL", relay_url), + ], + ) + .await; + let sid = init_session(&mut h, cwd.to_str().unwrap()).await; + + let p = h + .send( + "session/prompt", + json!({"sessionId": sid, "prompt": [{"type":"text","text":"What did Honey do today?"}]}), + ) + .await; + let _ = h.recv_until(|v| v["id"] == json!(p)).await; + + let reqs = llm.captured.lock().await; + assert!( + reqs.len() >= 2, + "expected at least 2 LLM requests, got {}", + reqs.len() + ); + let round1 = serde_json::to_string(&reqs[0]).unwrap(); + assert!( + round1.contains("get_activity_ledger_today"), + "tool was not advertised in round 1: {round1}" + ); + let round2 = serde_json::to_string(&reqs[1]).unwrap(); + assert!( + round2.contains("journal-a"), + "tool result missing filtered journal: {round2}" + ); + assert!( + !round2.contains("\"events\""), + "events should be stripped by default: {round2}" + ); + h.shutdown().await; +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index fb60a351895..ccdf1ac675a 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1044,6 +1044,7 @@ dependencies = [ "getrandom 0.4.3", "hex", "nix 0.31.3", + "nostr 0.44.7", "reqwest 0.13.4", "rmcp", "serde", diff --git a/desktop/src-tauri/crates/buzz-terminal/src/env_fence_tests.rs b/desktop/src-tauri/crates/buzz-terminal/src/env_fence_tests.rs index 59e94a1f63b..7c4e2e261bc 100644 --- a/desktop/src-tauri/crates/buzz-terminal/src/env_fence_tests.rs +++ b/desktop/src-tauri/crates/buzz-terminal/src/env_fence_tests.rs @@ -22,6 +22,7 @@ const SECRET_KEYS: &[&str] = &[ "BUZZ_ACP_PRIVATE_KEY", "BUZZ_ACP_API_TOKEN", "BUZZ_RELAY_URL", + "BUZZ_ACTIVITY_LEDGER_TODAY_RELAY_URL", ]; const CANARY: &str = "SAMI_CANARY_MUST_NOT_LEAK"; diff --git a/desktop/src-tauri/src/archive/archive_db.rs b/desktop/src-tauri/src/archive/archive_db.rs index 7c69591b0f6..8390fc32099 100644 --- a/desktop/src-tauri/src/archive/archive_db.rs +++ b/desktop/src-tauri/src/archive/archive_db.rs @@ -154,6 +154,25 @@ impl ArchiveDb { .await .map_err(|e| format!("archive db task failed: {e}"))? } + + /// Run one task while excluding every ordinary in-process archive + /// connection. The closure must still take the appropriate SQLite lock + /// when its invariant must also hold against a second OS process. + pub async fn with_exclusive_conn(&self, task: F) -> Result + where + T: Send + 'static, + F: FnOnce(&Connection) -> Result + Send + 'static, + { + self.ensure_initialized().await?; + let path = self.db_path()?; + let _guard = self.maintenance.write().await; + tokio::task::spawn_blocking(move || { + let conn = store::open_archive_db(&path)?; + task(&conn) + }) + .await + .map_err(|e| format!("exclusive archive db task failed: {e}"))? + } } #[cfg(test)] diff --git a/desktop/src-tauri/src/archive/archive_db_tests.rs b/desktop/src-tauri/src/archive/archive_db_tests.rs index f8182f829ce..dc2a440d084 100644 --- a/desktop/src-tauri/src/archive/archive_db_tests.rs +++ b/desktop/src-tauri/src/archive/archive_db_tests.rs @@ -244,3 +244,56 @@ async fn test_with_conn_read_guard_blocks_writer_until_connection_drops() { "write guard is free again after the connection drops" ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_exclusive_connection_blocks_ordinary_archive_work() { + let (_dir, path) = temp_db(); + let db = Arc::new(ArchiveDb::with_test_path(path)); + db.warm_init().await.expect("init must succeed"); + + let exclusive_entered = Arc::new(AtomicUsize::new(0)); + let ordinary_entered = Arc::new(AtomicUsize::new(0)); + let hold = Latch::new(); + let exclusive = { + let db = Arc::clone(&db); + let exclusive_entered = Arc::clone(&exclusive_entered); + let hold = Arc::clone(&hold); + tokio::spawn(async move { + db.with_exclusive_conn(move |_conn| { + exclusive_entered.fetch_add(1, Ordering::SeqCst); + hold.wait(); + Ok(()) + }) + .await + }) + }; + await_until( + "exclusive connection to enter", + Duration::from_secs(10), + || exclusive_entered.load(Ordering::SeqCst) == 1, + ) + .await; + + let ordinary = { + let db = Arc::clone(&db); + let ordinary_entered = Arc::clone(&ordinary_entered); + tokio::spawn(async move { + db.with_conn(move |_conn| { + ordinary_entered.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!( + ordinary_entered.load(Ordering::SeqCst), + 0, + "ordinary work must wait for the exclusive archive connection" + ); + + hold.release(); + assert!(exclusive.await.unwrap().is_ok()); + assert!(ordinary.await.unwrap().is_ok()); + assert_eq!(ordinary_entered.load(Ordering::SeqCst), 1); +} diff --git a/desktop/src-tauri/src/archive/journal_authority.rs b/desktop/src-tauri/src/archive/journal_authority.rs new file mode 100644 index 00000000000..6c30a6d9ab3 --- /dev/null +++ b/desktop/src-tauri/src/archive/journal_authority.rs @@ -0,0 +1,885 @@ +//! Signed, durable authority records for Activity Ledger journals. +//! +//! Observer frames are evidence, not authority. These records let the active +//! owner explicitly override a journal summary or independently attest that a +//! receipt verifies a journal. The complete Nostr event is stored locally and +//! its id, signature, signer, schema, tags, and content bindings are validated +//! both before insertion and on every read. + +use nostr::{Event, EventBuilder, JsonUtil, Keys, Kind, PublicKey, Tag}; +use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; + +pub const KIND_JOURNAL_AUTHORITY: u16 = 24201; +const ARTIFACT_SCHEMA: &str = "buzz.activity-journal-authority/v3"; +const ARTIFACT_MARKER: &str = "buzz-activity-journal"; +const MAX_RELAY_URL_BYTES: usize = 2_048; +const MAX_JOURNAL_ID_CHARS: usize = 512; +const MAX_CORRELATION_ID_CHARS: usize = 512; +const MAX_TEXT_CHARS: usize = 20_000; +const MAX_RECEIPT_REF_CHARS: usize = 2_048; +const MAX_SOURCE_EVENTS: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JournalAuthorityArtifactType { + OwnerOverride, + Verification, +} + +impl JournalAuthorityArtifactType { + fn as_str(self) -> &'static str { + match self { + Self::OwnerOverride => "owner_override", + Self::Verification => "verification", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SignedArtifactContent { + schema: String, + relay_url: String, + agent_pubkey: String, + artifact_type: JournalAuthorityArtifactType, + journal_id: String, + correlation_id: String, + revision: i64, + summary: Option, + note: Option, + receipt_ref: Option, + source_event_ids: Vec, +} + +/// Validated wire response. Secret key material is never serialized. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JournalAuthorityArtifact { + pub owner_pubkey: String, + pub relay_url: String, + pub agent_pubkey: String, + pub event_id: String, + pub signature: String, + pub created_at: i64, + pub artifact_type: JournalAuthorityArtifactType, + pub journal_id: String, + pub correlation_id: String, + pub revision: i64, + pub summary: Option, + pub note: Option, + pub receipt_ref: Option, + pub source_event_ids: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OwnerJournalOverrideInput { + pub agent_pubkey: String, + pub journal_id: String, + pub correlation_id: String, + pub summary: String, + pub note: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct JournalVerificationInput { + pub agent_pubkey: String, + pub journal_id: String, + pub correlation_id: String, + pub receipt_ref: String, + pub source_event_ids: Vec, +} + +#[derive(Debug)] +struct StoredArtifactRow { + identity_pubkey: String, + relay_url: String, + agent_pubkey: String, + journal_id: String, + artifact_type: String, + event_id: String, + created_at: i64, + revision: i64, + raw_json: String, +} + +pub fn normalize_relay_scope(relay_url: &str) -> Result { + if relay_url.is_empty() || relay_url.len() > MAX_RELAY_URL_BYTES { + return Err(format!( + "journal authority relay URL must contain between 1 and {MAX_RELAY_URL_BYTES} bytes" + )); + } + buzz_core_pkg::relay::normalize_relay_url(relay_url) + .map_err(|error| format!("journal authority relay URL is invalid: {error}")) +} + +pub fn normalize_agent_scope(agent_pubkey: &str) -> Result { + PublicKey::from_hex(agent_pubkey.trim()) + .map(|pubkey| pubkey.to_hex()) + .map_err(|error| format!("journal authority managed agent pubkey is invalid: {error}")) +} + +fn checked_nonempty(value: &str, label: &str, max_chars: usize) -> Result { + let value = value.trim(); + let len = value.chars().count(); + if len == 0 || len > max_chars { + return Err(format!( + "{label} must contain between 1 and {max_chars} characters" + )); + } + Ok(value.to_owned()) +} + +fn checked_optional_text( + value: Option<&str>, + label: &str, + max_chars: usize, +) -> Result, String> { + value + .map(|text| checked_nonempty(text, label, max_chars)) + .transpose() +} + +fn normalize_source_event_ids(values: &[String]) -> Result, String> { + if values.is_empty() || values.len() > MAX_SOURCE_EVENTS { + return Err(format!( + "verification must bind between 1 and {MAX_SOURCE_EVENTS} source event IDs" + )); + } + let mut normalized = Vec::with_capacity(values.len()); + for value in values { + let value = value.trim().to_ascii_lowercase(); + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("source event IDs must be 64-character hexadecimal Nostr IDs".into()); + } + normalized.push(value); + } + normalized.sort(); + normalized.dedup(); + if normalized.len() != values.len() { + return Err("verification source event IDs must be unique".into()); + } + Ok(normalized) +} + +fn single_tag(event: &Event, name: &str) -> Result { + let values = event + .tags + .iter() + .filter_map(|tag| { + let parts = tag.as_slice(); + (parts.len() == 2 && parts[0] == name).then(|| parts[1].clone()) + }) + .collect::>(); + if values.len() != 1 { + return Err(format!( + "journal authority event must contain exactly one {name:?} tag" + )); + } + Ok(values[0].clone()) +} + +fn repeated_tags(event: &Event, name: &str) -> Vec { + event + .tags + .iter() + .filter_map(|tag| { + let parts = tag.as_slice(); + (parts.len() == 2 && parts[0] == name).then(|| parts[1].clone()) + }) + .collect() +} + +fn validate_content(content: &SignedArtifactContent) -> Result<(), String> { + if content.schema != ARTIFACT_SCHEMA { + return Err("unsupported journal authority artifact schema".into()); + } + if normalize_relay_scope(&content.relay_url)? != content.relay_url { + return Err("journal authority relay URL must be canonical".into()); + } + if normalize_agent_scope(&content.agent_pubkey)? != content.agent_pubkey { + return Err("journal authority managed agent pubkey must be canonical".into()); + } + checked_nonempty(&content.journal_id, "journalId", MAX_JOURNAL_ID_CHARS)?; + checked_nonempty( + &content.correlation_id, + "correlationId", + MAX_CORRELATION_ID_CHARS, + )?; + if content.correlation_id != content.journal_id { + return Err("journal authority correlationId must equal the stable journalId".into()); + } + if content.revision < 1 { + return Err("journal authority revision must be positive".into()); + } + match content.artifact_type { + JournalAuthorityArtifactType::OwnerOverride => { + let summary = content + .summary + .as_deref() + .ok_or_else(|| "owner override is missing summary".to_string())?; + checked_nonempty(summary, "summary", MAX_TEXT_CHARS)?; + checked_optional_text(content.note.as_deref(), "note", MAX_TEXT_CHARS)?; + if content.receipt_ref.is_some() || !content.source_event_ids.is_empty() { + return Err("owner override cannot contain verification evidence".into()); + } + } + JournalAuthorityArtifactType::Verification => { + if content.summary.is_some() || content.note.is_some() { + return Err("verification artifact cannot override owner text".into()); + } + let receipt_ref = content + .receipt_ref + .as_deref() + .ok_or_else(|| "verification artifact is missing receiptRef".to_string())?; + checked_nonempty(receipt_ref, "receiptRef", MAX_RECEIPT_REF_CHARS)?; + let normalized = normalize_source_event_ids(&content.source_event_ids)?; + if normalized != content.source_event_ids { + return Err("verification source event IDs must be sorted and normalized".into()); + } + } + } + Ok(()) +} + +fn artifact_from_event(event: &Event, content: SignedArtifactContent) -> JournalAuthorityArtifact { + JournalAuthorityArtifact { + owner_pubkey: event.pubkey.to_hex(), + relay_url: content.relay_url, + agent_pubkey: content.agent_pubkey, + event_id: event.id.to_hex(), + signature: event.sig.to_string(), + created_at: event.created_at.as_secs() as i64, + artifact_type: content.artifact_type, + journal_id: content.journal_id, + correlation_id: content.correlation_id, + revision: content.revision, + summary: content.summary, + note: content.note, + receipt_ref: content.receipt_ref, + source_event_ids: content.source_event_ids, + } +} + +/// Parse and verify every signed field. This is called on insert and read. +pub fn validate_signed_artifact( + raw_json: &str, + expected_owner_pubkey: &str, + expected_relay_url: &str, + expected_agent_pubkey: &str, +) -> Result { + let expected_relay_url = normalize_relay_scope(expected_relay_url)?; + let expected_agent_pubkey = normalize_agent_scope(expected_agent_pubkey)?; + let event = Event::from_json(raw_json) + .map_err(|error| format!("parse journal authority event: {error}"))?; + event + .verify() + .map_err(|error| format!("journal authority signature verification failed: {error}"))?; + if event.kind.as_u16() != KIND_JOURNAL_AUTHORITY { + return Err(format!( + "journal authority event must use kind {KIND_JOURNAL_AUTHORITY}" + )); + } + if event.pubkey.to_hex() != expected_owner_pubkey { + return Err("journal authority event signer is not the active owner identity".into()); + } + + let content: SignedArtifactContent = serde_json::from_str(&event.content) + .map_err(|error| format!("parse journal authority content: {error}"))?; + validate_content(&content)?; + + if content.relay_url != expected_relay_url { + return Err("journal authority event relay is not the active relay".into()); + } + if content.agent_pubkey != expected_agent_pubkey { + return Err("journal authority event managed agent is not the requested agent".into()); + } + + if single_tag(&event, "t")? != ARTIFACT_MARKER + || single_tag(&event, "relay_url")? != content.relay_url + || single_tag(&event, "agent_pubkey")? != content.agent_pubkey + || single_tag(&event, "artifact_type")? != content.artifact_type.as_str() + || single_tag(&event, "journal_id")? != content.journal_id + || single_tag(&event, "correlation_id")? != content.correlation_id + || single_tag(&event, "revision")? != content.revision.to_string() + { + return Err("journal authority tags do not match signed content".into()); + } + + match content.artifact_type { + JournalAuthorityArtifactType::OwnerOverride => { + if !repeated_tags(&event, "receipt_ref").is_empty() + || !repeated_tags(&event, "source_event").is_empty() + { + return Err("owner override contains verification-only tags".into()); + } + } + JournalAuthorityArtifactType::Verification => { + if single_tag(&event, "receipt_ref")? != content.receipt_ref.as_deref().unwrap_or("") { + return Err("verification receipt tag does not match signed content".into()); + } + let mut tagged_sources = repeated_tags(&event, "source_event"); + tagged_sources.sort(); + if tagged_sources != content.source_event_ids { + return Err("verification source-event tags do not match signed content".into()); + } + } + } + + Ok(artifact_from_event(&event, content)) +} + +fn tag(name: &str, value: &str) -> Result { + Tag::parse([name, value]).map_err(|error| format!("build {name} tag: {error}")) +} + +fn build_signed_artifact(keys: &Keys, content: SignedArtifactContent) -> Result { + validate_content(&content)?; + let mut tags = vec![ + tag("t", ARTIFACT_MARKER)?, + tag("relay_url", &content.relay_url)?, + tag("agent_pubkey", &content.agent_pubkey)?, + tag("artifact_type", content.artifact_type.as_str())?, + tag("journal_id", &content.journal_id)?, + tag("correlation_id", &content.correlation_id)?, + tag("revision", &content.revision.to_string())?, + ]; + if let Some(receipt_ref) = &content.receipt_ref { + tags.push(tag("receipt_ref", receipt_ref)?); + } + for source_event_id in &content.source_event_ids { + tags.push(tag("source_event", source_event_id)?); + } + let content_json = serde_json::to_string(&content) + .map_err(|error| format!("serialize journal authority content: {error}"))?; + EventBuilder::new(Kind::Custom(KIND_JOURNAL_AUTHORITY), content_json) + .tags(tags) + .sign_with_keys(keys) + .map(|event| event.as_json()) + .map_err(|error| format!("sign journal authority event: {error}")) +} + +pub fn build_owner_override_event( + keys: &Keys, + relay_url: &str, + input: &OwnerJournalOverrideInput, + revision: i64, +) -> Result { + let content = SignedArtifactContent { + schema: ARTIFACT_SCHEMA.to_string(), + relay_url: normalize_relay_scope(relay_url)?, + agent_pubkey: normalize_agent_scope(&input.agent_pubkey)?, + artifact_type: JournalAuthorityArtifactType::OwnerOverride, + journal_id: checked_nonempty(&input.journal_id, "journalId", MAX_JOURNAL_ID_CHARS)?, + correlation_id: checked_nonempty( + &input.correlation_id, + "correlationId", + MAX_CORRELATION_ID_CHARS, + )?, + revision, + summary: Some(checked_nonempty(&input.summary, "summary", MAX_TEXT_CHARS)?), + note: checked_optional_text(input.note.as_deref(), "note", MAX_TEXT_CHARS)?, + receipt_ref: None, + source_event_ids: Vec::new(), + }; + build_signed_artifact(keys, content) +} + +pub fn build_verification_event( + keys: &Keys, + relay_url: &str, + input: &JournalVerificationInput, + revision: i64, +) -> Result { + let content = SignedArtifactContent { + schema: ARTIFACT_SCHEMA.to_string(), + relay_url: normalize_relay_scope(relay_url)?, + agent_pubkey: normalize_agent_scope(&input.agent_pubkey)?, + artifact_type: JournalAuthorityArtifactType::Verification, + journal_id: checked_nonempty(&input.journal_id, "journalId", MAX_JOURNAL_ID_CHARS)?, + correlation_id: checked_nonempty( + &input.correlation_id, + "correlationId", + MAX_CORRELATION_ID_CHARS, + )?, + revision, + summary: None, + note: None, + receipt_ref: Some(checked_nonempty( + &input.receipt_ref, + "receiptRef", + MAX_RECEIPT_REF_CHARS, + )?), + source_event_ids: normalize_source_event_ids(&input.source_event_ids)?, + }; + build_signed_artifact(keys, content) +} + +pub fn next_revision( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + agent_pubkey: &str, + journal_id: &str, + artifact_type: JournalAuthorityArtifactType, +) -> Result { + let current = conn + .query_row( + "SELECT revision FROM journal_authority_artifacts + WHERE identity_pubkey = ?1 AND relay_url = ?2 + AND agent_pubkey = ?3 AND journal_id = ?4 AND artifact_type = ?5", + params![ + identity_pubkey, + relay_url, + agent_pubkey, + journal_id, + artifact_type.as_str() + ], + |row| row.get::<_, i64>(0), + ) + .optional() + .map_err(|error| format!("read journal authority revision: {error}"))?; + current + .unwrap_or(0) + .checked_add(1) + .ok_or_else(|| "journal authority revision overflow".to_string()) +} + +fn rollback(conn: &Connection) { + let _ = conn.execute_batch("ROLLBACK"); +} + +/// Insert a first revision or replace the current row with exactly revision+1. +/// Re-inserting the exact same signed event is an idempotent success; a stale +/// but valid event is rejected so it cannot replay over newer authority state. +pub fn upsert_signed_artifact( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + agent_pubkey: &str, + raw_json: &str, + stored_at: i64, +) -> Result { + let relay_url = normalize_relay_scope(relay_url)?; + let agent_pubkey = normalize_agent_scope(agent_pubkey)?; + let artifact = validate_signed_artifact(raw_json, identity_pubkey, &relay_url, &agent_pubkey)?; + conn.execute_batch("BEGIN IMMEDIATE") + .map_err(|error| format!("begin journal authority upsert: {error}"))?; + let result = (|| -> Result { + let current = conn + .query_row( + "SELECT event_id, revision FROM journal_authority_artifacts + WHERE identity_pubkey = ?1 AND relay_url = ?2 + AND agent_pubkey = ?3 AND journal_id = ?4 AND artifact_type = ?5", + params![ + identity_pubkey, + relay_url, + agent_pubkey, + artifact.journal_id, + artifact.artifact_type.as_str() + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + ) + .optional() + .map_err(|error| format!("read current journal authority artifact: {error}"))?; + + if let Some((current_event_id, current_revision)) = current { + if current_event_id == artifact.event_id { + conn.execute( + "UPDATE journal_authority_artifacts SET raw_json = ?1, stored_at = ?2 + WHERE identity_pubkey = ?3 AND relay_url = ?4 + AND agent_pubkey = ?5 AND journal_id = ?6 AND artifact_type = ?7", + params![ + raw_json, + stored_at, + identity_pubkey, + relay_url, + agent_pubkey, + artifact.journal_id, + artifact.artifact_type.as_str() + ], + ) + .map_err(|error| format!("refresh journal authority artifact: {error}"))?; + return Ok(artifact.clone()); + } + let expected = current_revision + .checked_add(1) + .ok_or_else(|| "journal authority revision overflow".to_string())?; + if artifact.revision != expected { + return Err(format!( + "stale journal authority replay: expected revision {expected}, got {}", + artifact.revision + )); + } + } else if artifact.revision != 1 { + return Err(format!( + "first journal authority revision must be 1, got {}", + artifact.revision + )); + } + + conn.execute( + "INSERT INTO journal_authority_artifacts + (identity_pubkey, relay_url, agent_pubkey, journal_id, artifact_type, + event_id, created_at, revision, raw_json, stored_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) + ON CONFLICT (identity_pubkey, relay_url, agent_pubkey, journal_id, artifact_type) DO UPDATE SET + event_id = excluded.event_id, + created_at = excluded.created_at, + revision = excluded.revision, + raw_json = excluded.raw_json, + stored_at = excluded.stored_at", + params![ + identity_pubkey, + relay_url, + agent_pubkey, + artifact.journal_id, + artifact.artifact_type.as_str(), + artifact.event_id, + artifact.created_at, + artifact.revision, + raw_json, + stored_at + ], + ) + .map_err(|error| format!("persist journal authority artifact: {error}"))?; + Ok(artifact.clone()) + })(); + + match result { + Ok(artifact) => { + if let Err(error) = conn.execute_batch("COMMIT") { + rollback(conn); + return Err(format!("commit journal authority artifact: {error}")); + } + Ok(artifact) + } + Err(error) => { + rollback(conn); + Err(error) + } + } +} + +fn validate_stored_row( + row: StoredArtifactRow, + expected_identity: &str, + expected_relay_url: &str, + expected_agent_pubkey: &str, +) -> Result { + let artifact = validate_signed_artifact( + &row.raw_json, + expected_identity, + expected_relay_url, + expected_agent_pubkey, + )?; + if row.identity_pubkey != expected_identity + || row.relay_url != expected_relay_url + || row.relay_url != artifact.relay_url + || row.agent_pubkey != expected_agent_pubkey + || row.agent_pubkey != artifact.agent_pubkey + || row.journal_id != artifact.journal_id + || row.artifact_type != artifact.artifact_type.as_str() + || row.event_id != artifact.event_id + || row.created_at != artifact.created_at + || row.revision != artifact.revision + { + return Err("stored journal authority columns do not match signed event".into()); + } + Ok(artifact) +} + +fn row_from_sql(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(StoredArtifactRow { + identity_pubkey: row.get(0)?, + relay_url: row.get(1)?, + agent_pubkey: row.get(2)?, + journal_id: row.get(3)?, + artifact_type: row.get(4)?, + event_id: row.get(5)?, + created_at: row.get(6)?, + revision: row.get(7)?, + raw_json: row.get(8)?, + }) +} + +pub fn get_journal_authority_artifacts( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + agent_pubkey: &str, + journal_id: &str, +) -> Result, String> { + let relay_url = normalize_relay_scope(relay_url)?; + let agent_pubkey = normalize_agent_scope(agent_pubkey)?; + let mut stmt = conn + .prepare( + "SELECT identity_pubkey, relay_url, agent_pubkey, journal_id, artifact_type, event_id, + created_at, revision, raw_json + FROM journal_authority_artifacts + WHERE identity_pubkey = ?1 AND relay_url = ?2 + AND agent_pubkey = ?3 AND journal_id = ?4 + ORDER BY artifact_type ASC", + ) + .map_err(|error| format!("prepare journal authority read: {error}"))?; + let rows = stmt + .query_map( + params![identity_pubkey, relay_url, agent_pubkey, journal_id], + row_from_sql, + ) + .map_err(|error| format!("query journal authority artifacts: {error}"))?; + rows.collect::, _>>() + .map_err(|error| format!("read journal authority artifact row: {error}"))? + .into_iter() + .map(|row| validate_stored_row(row, identity_pubkey, &relay_url, &agent_pubkey)) + .collect() +} + +/// Bounded range query suitable for the owner Today surface. It returns only +/// decoded public fields after revalidating every signed event; no secret keys +/// or raw key material cross the Tauri boundary. +pub fn query_journal_authority_artifacts( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + agent_pubkey: &str, + start_created_at: i64, + end_created_at: i64, + limit: i64, +) -> Result, String> { + let relay_url = normalize_relay_scope(relay_url)?; + let agent_pubkey = normalize_agent_scope(agent_pubkey)?; + if start_created_at >= end_created_at { + return Err("journal authority range must be half-open and non-empty".into()); + } + if !(1..=500).contains(&limit) { + return Err("journal authority query limit must be between 1 and 500".into()); + } + let mut stmt = conn + .prepare( + "SELECT identity_pubkey, relay_url, agent_pubkey, journal_id, artifact_type, event_id, + created_at, revision, raw_json + FROM journal_authority_artifacts + WHERE identity_pubkey = ?1 AND relay_url = ?2 + AND agent_pubkey = ?3 AND created_at >= ?4 AND created_at < ?5 + ORDER BY created_at DESC, event_id DESC + LIMIT ?6", + ) + .map_err(|error| format!("prepare journal authority range query: {error}"))?; + let rows = stmt + .query_map( + params![ + identity_pubkey, + relay_url, + agent_pubkey, + start_created_at, + end_created_at, + limit + ], + row_from_sql, + ) + .map_err(|error| format!("query journal authority range: {error}"))?; + rows.collect::, _>>() + .map_err(|error| format!("read journal authority range row: {error}"))? + .into_iter() + .map(|row| validate_stored_row(row, identity_pubkey, &relay_url, &agent_pubkey)) + .collect() +} + +fn validate_archived_observer_frame(event: &Event, identity_pubkey: &str) -> Result<(), String> { + if event.kind.as_u16() != super::KIND_AGENT_OBSERVER_FRAME { + return Err("archived observer event kind does not match".into()); + } + if !event.tags.iter().any(|tag| { + let values = tag.as_slice(); + values.len() >= 2 && values[0] == "p" && values[1] == identity_pubkey + }) { + return Err("observer frame #p does not match the archived owner".into()); + } + let tag_value = |name: &str| { + event.tags.iter().find_map(|tag| { + let values = tag.as_slice(); + (values.len() >= 2 && values[0] == name).then(|| values[1].clone()) + }) + }; + let agent_pubkey = + tag_value("agent").ok_or_else(|| "observer frame missing `agent` tag".to_string())?; + if event.pubkey.to_hex() != agent_pubkey { + return Err("observer frame author does not match agent tag".into()); + } + let frame = + tag_value("frame").ok_or_else(|| "observer frame missing `frame` tag".to_string())?; + if frame != super::OBSERVER_FRAME_TELEMETRY { + return Err(format!("expected frame=telemetry, got {frame:?}")); + } + Ok(()) +} + +/// Revalidate every source event referenced by a verification artifact against +/// the active owner's immutable archive. This prevents an otherwise well-signed owner +/// artifact from yielding VERIFIED when it cites absent, cross-identity, or +/// tampered observer evidence. Current collection preferences are deliberately +/// irrelevant after the event was accepted into an owner-scoped archive row. +pub fn validate_archived_verification_sources( + conn: &Connection, + owner_keys: &Keys, + artifact: &JournalAuthorityArtifact, +) -> Result<(), String> { + if artifact.artifact_type != JournalAuthorityArtifactType::Verification { + return Ok(()); + } + let identity_pubkey = owner_keys.public_key().to_hex(); + if artifact.owner_pubkey != identity_pubkey { + return Err("verification artifact owner does not match the active identity".into()); + } + let relay_url = normalize_relay_scope(&artifact.relay_url)?; + let mut correlation_bound = artifact.correlation_id == artifact.journal_id; + for source_event_id in &artifact.source_event_ids { + let mut stmt = conn + .prepare( + "SELECT ae.relay_url, ae.kind, ae.raw_json + FROM archived_events ae + INNER JOIN archived_event_scopes aes + ON aes.identity_pubkey = ae.identity_pubkey + AND aes.relay_url = ae.relay_url + AND aes.id = ae.id + WHERE ae.identity_pubkey = ?1 + AND ae.id = ?2 + AND aes.scope_type = 'owner_p' + AND aes.scope_value = ?1", + ) + .map_err(|error| format!("prepare verification source read: {error}"))?; + let rows = stmt + .query_map(params![identity_pubkey, source_event_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + )) + }) + .map_err(|error| format!("read verification source event: {error}"))? + .collect::, _>>() + .map_err(|error| format!("read verification source row: {error}"))?; + if rows.is_empty() { + return Err(format!( + "verification source event {source_event_id} is not archived" + )); + } + let mut validation_errors = Vec::new(); + let mut valid = false; + for (stored_relay_url, kind, raw_json) in rows { + match normalize_relay_scope(&stored_relay_url) { + Ok(stored_scope) if stored_scope == relay_url => {} + Ok(_) => { + validation_errors.push("source is archived under another relay".into()); + continue; + } + Err(error) => { + validation_errors.push(format!("stored relay is invalid: {error}")); + continue; + } + } + if kind != 24200 { + validation_errors.push("not an observer event".to_string()); + continue; + } + let event = match Event::from_json(&raw_json) { + Ok(event) if event.id.to_hex() == *source_event_id && event.verify().is_ok() => { + event + } + Ok(_) => { + validation_errors.push("signed ID or signature mismatch".to_string()); + continue; + } + Err(error) => { + validation_errors.push(format!("parse failed: {error}")); + continue; + } + }; + if event.pubkey.to_hex() != artifact.agent_pubkey { + validation_errors.push("observer source belongs to another managed agent".into()); + continue; + } + if let Err(error) = validate_archived_observer_frame(&event, &identity_pubkey) { + validation_errors.push(format!("observer authorization failed: {error}")); + continue; + } + let decoded = match buzz_core_pkg::observer::decrypt_observer_payload::( + owner_keys, &event, + ) { + Ok(decoded) => decoded, + Err(error) => { + validation_errors.push(format!("observer decrypt failed: {error}")); + continue; + } + }; + let leaves: Vec<&serde_json::Value> = + if decoded.get("kind").and_then(serde_json::Value::as_str) == Some("batch") { + decoded + .get("payload") + .and_then(|payload| payload.get("events")) + .and_then(serde_json::Value::as_array) + .map(|events| events.iter().collect()) + .unwrap_or_default() + } else { + vec![&decoded] + }; + let matching_leaves = leaves + .into_iter() + .filter(|leaf| { + ["journalKey", "turnId", "sessionId", "channelId"] + .into_iter() + .filter_map(|field| leaf.get(field).and_then(serde_json::Value::as_str)) + .any(|value| value == artifact.journal_id) + }) + .collect::>(); + if matching_leaves.is_empty() { + validation_errors.push("observer payload does not bind the journal".to_string()); + continue; + } + correlation_bound |= matching_leaves.iter().any(|leaf| { + let payload = leaf.get("payload"); + let triggering_matches = payload + .and_then(|value| value.get("triggeringEventIds")) + .and_then(serde_json::Value::as_array) + .is_some_and(|ids| { + ids.iter() + .any(|id| id.as_str() == Some(artifact.correlation_id.as_str())) + }); + let update = payload + .and_then(|value| value.get("params")) + .and_then(|value| value.get("update")); + let update_matches = ["toolCallId", "messageId"] + .into_iter() + .filter_map(|field| { + update + .and_then(|value| value.get(field)) + .and_then(serde_json::Value::as_str) + }) + .any(|value| value == artifact.correlation_id); + triggering_matches || update_matches + }); + valid = true; + break; + } + if !valid { + return Err(format!( + "verification source event {source_event_id} failed validation: {}", + validation_errors.join("; ") + )); + } + } + if !correlation_bound { + return Err(format!( + "verification sources do not bind correlation {:?}", + artifact.correlation_id + )); + } + Ok(()) +} + +#[cfg(test)] +#[path = "journal_authority_tests.rs"] +mod tests; diff --git a/desktop/src-tauri/src/archive/journal_authority_commands.rs b/desktop/src-tauri/src/archive/journal_authority_commands.rs new file mode 100644 index 00000000000..3ed8ba26f19 --- /dev/null +++ b/desktop/src-tauri/src/archive/journal_authority_commands.rs @@ -0,0 +1,414 @@ +//! Tauri command boundary for owner-authorized Activity Ledger artifacts. + +use tauri::State; + +use super::journal_authority::{ + self, JournalAuthorityArtifact, JournalVerificationInput, OwnerJournalOverrideInput, +}; +use super::today_snapshot::{self, TodaySnapshotReceipt}; +use super::{identity_pubkey, now_secs, observer_revision, observer_time, store}; +use crate::app_state::AppState; +use crate::managed_agents::nest_dir; +use crate::relay::relay_ws_url_with_override; + +// ── Activity Ledger owner authority ───────────────────────────────────────── + +fn active_relay_scope(state: &AppState, requested_relay_url: &str) -> Result { + let active = journal_authority::normalize_relay_scope(&relay_ws_url_with_override(state))?; + let requested = journal_authority::normalize_relay_scope(requested_relay_url)?; + if requested != active { + return Err("journal authority request relay is not the active relay".into()); + } + Ok(active) +} + +/// Persist an owner-authenticated journal summary override. The backend signs +/// the artifact with the active identity; callers never receive key material. +#[tauri::command] +pub async fn upsert_owner_journal_override( + state: State<'_, AppState>, + relay_url: String, + input: OwnerJournalOverrideInput, +) -> Result { + let keys = state.signing_keys()?; + let identity_pk = keys.public_key().to_hex(); + let relay_url = active_relay_scope(&state, &relay_url)?; + let agent_pubkey = journal_authority::normalize_agent_scope(&input.agent_pubkey)?; + let now = now_secs(); + state + .archive_db + .with_conn(move |conn| { + let revision = journal_authority::next_revision( + conn, + &identity_pk, + &relay_url, + &agent_pubkey, + input.journal_id.trim(), + journal_authority::JournalAuthorityArtifactType::OwnerOverride, + )?; + let raw = + journal_authority::build_owner_override_event(&keys, &relay_url, &input, revision)?; + journal_authority::upsert_signed_artifact( + conn, + &identity_pk, + &relay_url, + &agent_pubkey, + &raw, + now, + ) + }) + .await +} + +/// Persist an independent owner verification. It cannot be created without a +/// receipt reference and one or more source observer event IDs that are +/// currently present and signature-valid in this owner's archive. +#[tauri::command] +pub async fn upsert_journal_verification( + state: State<'_, AppState>, + relay_url: String, + input: JournalVerificationInput, +) -> Result { + let keys = state.signing_keys()?; + let identity_pk = keys.public_key().to_hex(); + let relay_url = active_relay_scope(&state, &relay_url)?; + let agent_pubkey = journal_authority::normalize_agent_scope(&input.agent_pubkey)?; + let now = now_secs(); + state + .archive_db + .with_conn(move |conn| { + let revision = journal_authority::next_revision( + conn, + &identity_pk, + &relay_url, + &agent_pubkey, + input.journal_id.trim(), + journal_authority::JournalAuthorityArtifactType::Verification, + )?; + let raw = + journal_authority::build_verification_event(&keys, &relay_url, &input, revision)?; + let artifact = journal_authority::validate_signed_artifact( + &raw, + &identity_pk, + &relay_url, + &agent_pubkey, + )?; + journal_authority::validate_archived_verification_sources(conn, &keys, &artifact)?; + journal_authority::upsert_signed_artifact( + conn, + &identity_pk, + &relay_url, + &agent_pubkey, + &raw, + now, + ) + }) + .await +} + +/// Read the current owner override and/or verification for one journal. Every +/// signed artifact and every verification source is revalidated fail-closed. +#[tauri::command] +pub async fn get_journal_authority_artifacts( + state: State<'_, AppState>, + relay_url: String, + agent_pubkey: String, + journal_id: String, +) -> Result, String> { + let keys = state.signing_keys()?; + let identity_pk = keys.public_key().to_hex(); + let relay_url = active_relay_scope(&state, &relay_url)?; + let agent_pubkey = journal_authority::normalize_agent_scope(&agent_pubkey)?; + let journal_id = journal_id.trim().to_owned(); + state + .archive_db + .with_conn(move |conn| { + let artifacts = journal_authority::get_journal_authority_artifacts( + conn, + &identity_pk, + &relay_url, + &agent_pubkey, + &journal_id, + )?; + for artifact in &artifacts { + journal_authority::validate_archived_verification_sources(conn, &keys, artifact)?; + } + Ok(artifacts) + }) + .await +} + +/// Bounded owner-only authority query used by Today surfaces and local +/// read-only consumers. No signing or secret key data is returned. +#[tauri::command] +pub async fn query_journal_authority_artifacts( + state: State<'_, AppState>, + relay_url: String, + agent_pubkey: String, + start_created_at: i64, + end_created_at: i64, + limit: Option, +) -> Result, String> { + let keys = state.signing_keys()?; + let identity_pk = keys.public_key().to_hex(); + let relay_url = active_relay_scope(&state, &relay_url)?; + let agent_pubkey = journal_authority::normalize_agent_scope(&agent_pubkey)?; + state + .archive_db + .with_conn(move |conn| { + let artifacts = journal_authority::query_journal_authority_artifacts( + conn, + &identity_pk, + &relay_url, + &agent_pubkey, + start_created_at, + end_created_at, + limit.unwrap_or(200), + )?; + for artifact in &artifacts { + journal_authority::validate_archived_verification_sources(conn, &keys, artifact)?; + } + Ok(artifacts) + }) + .await +} + +fn seal_snapshot_archive_fence( + snapshot_json: &str, + current_revision: i64, + current_unindexed: i64, +) -> Result { + let mut snapshot: serde_json::Value = serde_json::from_str(snapshot_json) + .map_err(|error| format!("invalid Today snapshot JSON: {error}"))?; + let projection = snapshot + .pointer("/surface/snapshotProjection") + .and_then(serde_json::Value::as_object) + .ok_or("Today snapshot must include snapshotProjection")?; + let declared_revision = projection + .get("archiveRevision") + .and_then(serde_json::Value::as_i64) + .filter(|value| *value >= 0) + .ok_or("Today snapshot must disclose archiveRevision")?; + let declared = projection + .get("unindexedObserverFrames") + .and_then(serde_json::Value::as_i64) + .filter(|value| *value >= 0) + .ok_or("Today snapshot must disclose unindexedObserverFrames")?; + let excluded = projection + .get("excludedObserverFrames") + .and_then(serde_json::Value::as_i64) + .filter(|value| *value >= declared) + .ok_or("Today snapshot exclusions must cover unindexed observer frames")?; + let source_dropped = projection + .get("sourceDroppedObserverEvents") + .and_then(serde_json::Value::as_i64) + .filter(|value| *value >= 0) + .ok_or("Today snapshot must disclose source-dropped observer events")?; + if (excluded > 0 || source_dropped > 0) + && projection + .get("bounded") + .and_then(serde_json::Value::as_bool) + != Some(true) + { + return Err("Today snapshot with observer evidence gaps must be bounded".into()); + } + if current_revision < declared_revision { + return Err("Today snapshot archive revision moved backwards".into()); + } + let revision_drift = current_revision - declared_revision; + if revision_drift > 0 { + invalidate_snapshot_journal_truth(&mut snapshot)?; + } + let current_unindexed = current_unindexed.max(declared); + let current_excluded = excluded + .checked_add(current_unindexed - declared) + .ok_or("Today snapshot exclusion count overflow")?; + let projection = snapshot + .pointer_mut("/surface/snapshotProjection") + .and_then(serde_json::Value::as_object_mut) + .ok_or("Today snapshot must include snapshotProjection")?; + projection.insert( + "archiveRevisionAtPublish".into(), + serde_json::Value::from(current_revision), + ); + projection.insert( + "archiveRevisionDrift".into(), + serde_json::Value::from(revision_drift), + ); + projection.insert( + "truthInvalidatedByArchiveDrift".into(), + serde_json::Value::Bool(revision_drift > 0), + ); + projection.insert( + "unindexedObserverFrames".into(), + serde_json::Value::from(current_unindexed), + ); + projection.insert( + "excludedObserverFrames".into(), + serde_json::Value::from(current_excluded), + ); + if revision_drift > 0 || current_excluded > 0 { + projection.insert("bounded".into(), serde_json::Value::Bool(true)); + } + serde_json::to_string(&snapshot) + .map_err(|error| format!("serialize fenced Today snapshot: {error}")) +} + +/// A forward archive revision means the reconstructed journal set is no +/// longer current. Preserve the bounded historical rows, but never sign stale +/// completion or verification as present truth. +fn invalidate_snapshot_journal_truth(snapshot: &mut serde_json::Value) -> Result<(), String> { + let surface = snapshot + .get_mut("surface") + .and_then(serde_json::Value::as_object_mut) + .ok_or("Today snapshot surface must be an object")?; + let Some(journals) = surface + .get_mut("journals") + .and_then(serde_json::Value::as_array_mut) + else { + return Ok(()); + }; + for journal in journals.iter_mut() { + let Some(journal) = journal.as_object_mut() else { + continue; + }; + journal.insert( + "status".into(), + serde_json::Value::String("incomplete".into()), + ); + journal.insert( + "proofState".into(), + serde_json::Value::String("UNKNOWN".into()), + ); + journal.insert( + "summary".into(), + serde_json::Value::String( + "Archive changed during publication; refresh before relying on this journal." + .into(), + ), + ); + journal.insert( + "summarySource".into(), + serde_json::Value::String("auto".into()), + ); + journal.insert( + "claimedCompletionWithoutEvidence".into(), + serde_json::Value::Bool(false), + ); + journal.insert("archiveRevisionStale".into(), serde_json::Value::Bool(true)); + if let Some(events) = journal + .get_mut("events") + .and_then(serde_json::Value::as_array_mut) + { + for event in events { + let Some(event) = event.as_object_mut() else { + continue; + }; + if event.get("proofState").and_then(serde_json::Value::as_str) == Some("VERIFIED") { + event.insert( + "proofState".into(), + serde_json::Value::String("UNKNOWN".into()), + ); + } + } + } + } + if let Some(counts) = surface + .get_mut("counts") + .and_then(serde_json::Value::as_object_mut) + { + counts.insert("failed".into(), serde_json::Value::from(0)); + counts.insert("inProgress".into(), serde_json::Value::from(0)); + counts.insert("claimedWithoutEvidence".into(), serde_json::Value::from(0)); + } + Ok(()) +} + +/// Publish under both the process-exclusive archive guard and a SQLite +/// immediate transaction so no in-process or second-process writer can +/// overtake the signed archive revision before atomic file replacement. +#[tauri::command] +pub async fn write_owner_today_snapshot( + state: State<'_, AppState>, + snapshot_json: String, +) -> Result { + let keys = state.signing_keys()?; + let identity_pk = keys.public_key().to_hex(); + let relay_url = relay_ws_url_with_override(&state); + let nest = nest_dir().ok_or("cannot resolve nest directory for Today snapshot")?; + state + .archive_db + .with_exclusive_conn(move |conn| { + if !observer_time::backfill_missing(conn, &identity_pk, &relay_url, &keys)? { + return Err("Today snapshot archive fence requires completed backfill".into()); + } + let tx = rusqlite::Transaction::new_unchecked( + conn, + rusqlite::TransactionBehavior::Immediate, + ) + .map_err(|error| format!("begin Today snapshot archive fence: {error}"))?; + let current_revision = observer_revision::current(&tx, &identity_pk, &relay_url)?; + let current_unindexed = + store::count_unindexed_observer_frames(&tx, &identity_pk, &relay_url)?; + let snapshot_json = + seal_snapshot_archive_fence(&snapshot_json, current_revision, current_unindexed)?; + let receipt = today_snapshot::write_owner_today_snapshot( + &nest, + &keys, + &identity_pk, + &relay_url, + &snapshot_json, + now_secs(), + )?; + tx.commit() + .map_err(|error| format!("finish Today snapshot archive fence: {error}"))?; + Ok(receipt) + }) + .await +} + +/// Read and revalidate the current owner's unexpired Today snapshot. +#[tauri::command] +pub fn read_owner_today_snapshot(state: State<'_, AppState>) -> Result { + let identity_pk = identity_pubkey(&state)?; + let relay_url = relay_ws_url_with_override(&state); + let nest = nest_dir().ok_or("cannot resolve nest directory for Today snapshot")?; + today_snapshot::read_owner_today_snapshot(&nest, &identity_pk, &relay_url, now_secs()) +} + +#[cfg(test)] +mod snapshot_fence_tests { + use super::seal_snapshot_archive_fence; + + #[test] + fn snapshot_fence_discloses_new_archive_activity() { + let snapshot = r#"{"surface":{"counts":{"journals":1,"failed":0,"inProgress":0,"claimedWithoutEvidence":0},"journals":[{"status":"completed","proofState":"VERIFIED","summary":"Verified complete","summarySource":"owner","claimedCompletionWithoutEvidence":false,"events":[{"proofState":"VERIFIED"}]}],"snapshotProjection":{"archiveRevision":7,"bounded":true,"excludedObserverFrames":2,"sourceDroppedObserverEvents":0,"unindexedObserverFrames":2}}}"#; + let sealed = seal_snapshot_archive_fence(snapshot, 8, 3).unwrap(); + let sealed: serde_json::Value = serde_json::from_str(&sealed).unwrap(); + let projection = &sealed["surface"]["snapshotProjection"]; + assert_eq!(projection["archiveRevision"], 7); + assert_eq!(projection["archiveRevisionAtPublish"], 8); + assert_eq!(projection["archiveRevisionDrift"], 1); + assert_eq!(projection["unindexedObserverFrames"], 3); + assert_eq!(projection["excludedObserverFrames"], 3); + assert_eq!(projection["bounded"], true); + assert_eq!(projection["truthInvalidatedByArchiveDrift"], true); + let journal = &sealed["surface"]["journals"][0]; + assert_eq!(journal["status"], "incomplete"); + assert_eq!(journal["proofState"], "UNKNOWN"); + assert_eq!(journal["summarySource"], "auto"); + assert_eq!(journal["events"][0]["proofState"], "UNKNOWN"); + assert_eq!(journal["archiveRevisionStale"], true); + assert!(seal_snapshot_archive_fence(snapshot, 6, 2) + .unwrap_err() + .contains("moved backwards")); + let false_complete = r#"{"surface":{"snapshotProjection":{"archiveRevision":7,"bounded":false,"excludedObserverFrames":2,"sourceDroppedObserverEvents":0,"unindexedObserverFrames":2}}}"#; + assert!(seal_snapshot_archive_fence(false_complete, 7, 2) + .unwrap_err() + .contains("must be bounded")); + let undisclosed_gap = r#"{"surface":{"snapshotProjection":{"archiveRevision":7,"bounded":false,"excludedObserverFrames":0,"sourceDroppedObserverEvents":1,"unindexedObserverFrames":0}}}"#; + assert!(seal_snapshot_archive_fence(undisclosed_gap, 7, 0) + .unwrap_err() + .contains("must be bounded")); + } +} diff --git a/desktop/src-tauri/src/archive/journal_authority_tests.rs b/desktop/src-tauri/src/archive/journal_authority_tests.rs new file mode 100644 index 00000000000..23054adce73 --- /dev/null +++ b/desktop/src-tauri/src/archive/journal_authority_tests.rs @@ -0,0 +1,638 @@ +use super::*; +use crate::archive::store::{self, open_archive_db, SCHEMA}; +use rusqlite::Connection; + +const RELAY_A: &str = "wss://relay-a.example"; +const RELAY_B: &str = "wss://relay-b.example"; +const AGENT_A: &str = "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798"; + +fn in_memory() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(SCHEMA).unwrap(); + conn +} + +fn override_input(summary: &str) -> OwnerJournalOverrideInput { + OwnerJournalOverrideInput { + agent_pubkey: AGENT_A.into(), + journal_id: "agent:channel:turn-1".into(), + correlation_id: "agent:channel:turn-1".into(), + summary: summary.into(), + note: Some("Owner corrected the narrative.".into()), + } +} + +fn verification_input() -> JournalVerificationInput { + JournalVerificationInput { + agent_pubkey: AGENT_A.into(), + journal_id: "agent:channel:turn-1".into(), + correlation_id: "agent:channel:turn-1".into(), + receipt_ref: "receipt://archive/tool-call-1".into(), + source_event_ids: vec!["a".repeat(64), "b".repeat(64)], + } +} + +fn observer_event(owner: &Keys, agent: &Keys, journal_id: &str, correlation_id: &str) -> Event { + let payload = serde_json::json!({ + "seq": 1, + "timestamp": "2026-08-21T14:00:00.000Z", + "kind": "turn_started", + "agentIndex": 0, + "channelId": "channel-1", + "sessionId": "session-1", + "turnId": journal_id, + "payload": { "triggeringEventIds": [correlation_id] } + }); + let ciphertext = + buzz_core_pkg::observer::encrypt_observer_payload(agent, &owner.public_key(), &payload) + .unwrap(); + EventBuilder::new(Kind::Custom(24200), ciphertext) + .tags([ + Tag::parse(["p", &owner.public_key().to_hex()]).unwrap(), + Tag::parse(["agent", &agent.public_key().to_hex()]).unwrap(), + Tag::parse(["frame", "telemetry"]).unwrap(), + ]) + .sign_with_keys(agent) + .unwrap() +} + +fn archive_observer(conn: &Connection, owner_pubkey: &str, relay_url: &str, event: &Event) { + store::upsert_archived_event( + conn, + owner_pubkey, + relay_url, + &event.id.to_hex(), + 24200, + &event.pubkey.to_hex(), + event.created_at.as_secs() as i64, + &event.as_json(), + 1, + ) + .unwrap(); +} + +fn scope_observer(conn: &Connection, owner_pubkey: &str, relay_url: &str, event: &Event) { + store::upsert_event_scope( + conn, + owner_pubkey, + relay_url, + &event.id.to_hex(), + "owner_p", + owner_pubkey, + 1, + ) + .unwrap(); +} + +#[test] +fn owner_override_is_signed_persisted_and_idempotent() { + let conn = in_memory(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let raw = build_owner_override_event( + &owner, + RELAY_A, + &override_input("Observed owner result."), + 1, + ) + .unwrap(); + + let first = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 10).unwrap(); + let replay = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 11).unwrap(); + assert_eq!(first, replay); + assert_eq!( + first.artifact_type, + JournalAuthorityArtifactType::OwnerOverride + ); + assert_eq!(first.summary.as_deref(), Some("Observed owner result.")); + assert_eq!( + get_journal_authority_artifacts(&conn, &owner_pk, RELAY_A, AGENT_A, &first.journal_id) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn stale_valid_revision_cannot_replay_over_current_state() { + let conn = in_memory(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let first = build_owner_override_event(&owner, RELAY_A, &override_input("First"), 1).unwrap(); + let second = build_owner_override_event(&owner, RELAY_A, &override_input("Second"), 2).unwrap(); + let stale = + build_owner_override_event(&owner, RELAY_A, &override_input("Stale rewrite"), 1).unwrap(); + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &first, 10).unwrap(); + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &second, 11).unwrap(); + let error = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &stale, 12).unwrap_err(); + assert!(error.contains("stale journal authority replay")); + + let rows = + get_journal_authority_artifacts(&conn, &owner_pk, RELAY_A, AGENT_A, "agent:channel:turn-1") + .unwrap(); + assert_eq!(rows[0].revision, 2); + assert_eq!(rows[0].summary.as_deref(), Some("Second")); +} + +#[test] +fn first_insert_must_start_at_revision_one() { + let conn = in_memory(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let raw = build_owner_override_event(&owner, RELAY_A, &override_input("Skipped"), 2).unwrap(); + let error = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 10).unwrap_err(); + assert!(error.contains("first journal authority revision must be 1")); +} + +#[test] +fn wrong_signer_fails_closed_and_identity_rows_are_isolated() { + let conn = in_memory(); + let owner = Keys::generate(); + let other = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let other_pk = other.public_key().to_hex(); + let raw = + build_owner_override_event(&owner, RELAY_A, &override_input("Owner only"), 1).unwrap(); + assert!( + upsert_signed_artifact(&conn, &other_pk, RELAY_A, AGENT_A, &raw, 10) + .unwrap_err() + .contains("signer is not the active owner") + ); + + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 10).unwrap(); + assert!(get_journal_authority_artifacts( + &conn, + &other_pk, + RELAY_A, + AGENT_A, + "agent:channel:turn-1" + ) + .unwrap() + .is_empty()); +} + +#[test] +fn tampered_signature_and_tampered_database_columns_fail_closed() { + let conn = in_memory(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let raw = + build_owner_override_event(&owner, RELAY_A, &override_input("Untampered"), 1).unwrap(); + let mut value: serde_json::Value = serde_json::from_str(&raw).unwrap(); + value["content"] = serde_json::Value::String("{}".into()); + let tampered = serde_json::to_string(&value).unwrap(); + assert!( + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &tampered, 10) + .unwrap_err() + .contains("signature verification failed") + ); + + let artifact = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 10).unwrap(); + conn.execute( + "UPDATE journal_authority_artifacts SET revision = 99 + WHERE identity_pubkey = ?1 AND journal_id = ?2", + params![owner_pk, artifact.journal_id], + ) + .unwrap(); + assert!(get_journal_authority_artifacts( + &conn, + &owner_pk, + RELAY_A, + AGENT_A, + "agent:channel:turn-1" + ) + .unwrap_err() + .contains("columns do not match signed event")); +} + +#[test] +fn verification_binds_receipt_correlation_and_source_events() { + let conn = in_memory(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let raw = build_verification_event(&owner, RELAY_A, &verification_input(), 1).unwrap(); + let artifact = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 10).unwrap(); + assert_eq!( + artifact.artifact_type, + JournalAuthorityArtifactType::Verification + ); + assert_eq!(artifact.correlation_id, "agent:channel:turn-1"); + assert_eq!( + artifact.receipt_ref.as_deref(), + Some("receipt://archive/tool-call-1") + ); + assert_eq!(artifact.source_event_ids, ["a".repeat(64), "b".repeat(64)]); +} + +#[test] +fn verification_missing_receipt_or_source_event_fails_closed() { + let owner = Keys::generate(); + let mut no_receipt = verification_input(); + no_receipt.receipt_ref = " ".into(); + assert!(build_verification_event(&owner, RELAY_A, &no_receipt, 1) + .unwrap_err() + .contains("receiptRef")); + + let mut no_source = verification_input(); + no_source.source_event_ids.clear(); + assert!(build_verification_event(&owner, RELAY_A, &no_source, 1) + .unwrap_err() + .contains("must bind between")); +} + +#[test] +fn verification_rejects_duplicate_and_malformed_source_ids() { + let owner = Keys::generate(); + let mut duplicate = verification_input(); + duplicate.source_event_ids = vec!["a".repeat(64), "a".repeat(64)]; + assert!(build_verification_event(&owner, RELAY_A, &duplicate, 1) + .unwrap_err() + .contains("must be unique")); + + let mut malformed = verification_input(); + malformed.source_event_ids = vec!["not-an-event".into()]; + assert!(build_verification_event(&owner, RELAY_A, &malformed, 1) + .unwrap_err() + .contains("64-character hexadecimal")); +} + +#[test] +fn verification_sources_must_exist_and_remain_valid_in_owner_archive() { + let conn = in_memory(); + let owner = Keys::generate(); + let agent = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let agent_pk = agent.public_key().to_hex(); + let relay_url = RELAY_A; + let event = observer_event(&owner, &agent, "agent:channel:turn-1", "tool-call-1"); + let event_id = event.id.to_hex(); + let input = JournalVerificationInput { + agent_pubkey: agent_pk.clone(), + source_event_ids: vec![event_id.clone()], + ..verification_input() + }; + let raw = build_verification_event(&owner, RELAY_A, &input, 1).unwrap(); + let artifact = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, &agent_pk, &raw, 10).unwrap(); + assert!( + validate_archived_verification_sources(&conn, &owner, &artifact) + .unwrap_err() + .contains("is not archived") + ); + + archive_observer(&conn, &owner_pk, RELAY_B, &event); + scope_observer(&conn, &owner_pk, RELAY_B, &event); + store::upsert_save_subscription( + &conn, &owner_pk, RELAY_B, "owner_p", &owner_pk, "[24200]", 1, + ) + .unwrap(); + assert!( + validate_archived_verification_sources(&conn, &owner, &artifact) + .unwrap_err() + .contains("another relay") + ); + + archive_observer(&conn, &owner_pk, relay_url, &event); + scope_observer(&conn, &owner_pk, relay_url, &event); + validate_archived_verification_sources(&conn, &owner, &artifact).unwrap(); + + let other_agent = Keys::generate().public_key().to_hex(); + let cross_agent_input = JournalVerificationInput { + agent_pubkey: other_agent.clone(), + source_event_ids: vec![event_id.clone()], + ..verification_input() + }; + let cross_agent_raw = build_verification_event(&owner, RELAY_A, &cross_agent_input, 1).unwrap(); + let cross_agent = + validate_signed_artifact(&cross_agent_raw, &owner_pk, RELAY_A, &other_agent).unwrap(); + assert!( + validate_archived_verification_sources(&conn, &owner, &cross_agent) + .unwrap_err() + .contains("another managed agent") + ); + + conn.execute( + "UPDATE archived_events SET raw_json = '{}' WHERE identity_pubkey = ?1 AND id = ?2", + params![owner_pk, event_id], + ) + .unwrap(); + assert!( + validate_archived_verification_sources(&conn, &owner, &artifact) + .unwrap_err() + .contains("failed validation") + ); +} + +#[test] +fn verification_finds_source_stored_under_equivalent_raw_relay_spelling() { + let conn = in_memory(); + let owner = Keys::generate(); + let agent = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let agent_pk = agent.public_key().to_hex(); + let raw_relay = "ws://localhost:3000"; + let event = observer_event(&owner, &agent, "agent:channel:turn-1", "tool-call-1"); + archive_observer(&conn, &owner_pk, raw_relay, &event); + scope_observer(&conn, &owner_pk, raw_relay, &event); + store::upsert_save_subscription( + &conn, &owner_pk, raw_relay, "owner_p", &owner_pk, "[24200]", 1, + ) + .unwrap(); + + let input = JournalVerificationInput { + agent_pubkey: agent_pk.clone(), + source_event_ids: vec![event.id.to_hex()], + ..verification_input() + }; + let raw = build_verification_event(&owner, raw_relay, &input, 1).unwrap(); + let artifact = + validate_signed_artifact(&raw, &owner_pk, "ws://127.0.0.1:3000", &agent_pk).unwrap(); + validate_archived_verification_sources(&conn, &owner, &artifact).unwrap(); +} + +#[test] +fn archived_verification_survives_owner_collection_disable() { + let conn = in_memory(); + let owner = Keys::generate(); + let agent = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let agent_pk = agent.public_key().to_hex(); + let event = observer_event(&owner, &agent, "agent:channel:turn-1", "tool-call-1"); + archive_observer(&conn, &owner_pk, RELAY_A, &event); + scope_observer(&conn, &owner_pk, RELAY_A, &event); + store::upsert_save_subscription( + &conn, &owner_pk, RELAY_A, "owner_p", &owner_pk, "[24200]", 1, + ) + .unwrap(); + + let input = JournalVerificationInput { + agent_pubkey: agent_pk.clone(), + source_event_ids: vec![event.id.to_hex()], + ..verification_input() + }; + let raw = build_verification_event(&owner, RELAY_A, &input, 1).unwrap(); + let artifact = validate_signed_artifact(&raw, &owner_pk, RELAY_A, &agent_pk).unwrap(); + validate_archived_verification_sources(&conn, &owner, &artifact).unwrap(); + + assert!( + store::delete_save_subscription(&conn, &owner_pk, RELAY_A, "owner_p", &owner_pk).unwrap() + ); + validate_archived_verification_sources(&conn, &owner, &artifact).unwrap(); + + conn.execute( + "DELETE FROM archived_event_scopes + WHERE identity_pubkey = ?1 AND relay_url = ?2 AND id = ?3 + AND scope_type = 'owner_p' AND scope_value = ?1", + params![owner_pk, RELAY_A, event.id.to_hex()], + ) + .unwrap(); + assert!( + validate_archived_verification_sources(&conn, &owner, &artifact) + .unwrap_err() + .contains("is not archived") + ); +} + +#[test] +fn verification_rejects_tagless_or_wrongly_bound_observer_sources() { + let conn = in_memory(); + let owner = Keys::generate(); + let agent = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let agent_pk = agent.public_key().to_hex(); + let relay_url = RELAY_A; + store::upsert_save_subscription( + &conn, &owner_pk, relay_url, "owner_p", &owner_pk, "[24200]", 1, + ) + .unwrap(); + + let authorized = observer_event(&owner, &agent, "agent:channel:turn-1", "tool-call-1"); + archive_observer(&conn, &owner_pk, relay_url, &authorized); + scope_observer(&conn, &owner_pk, relay_url, &authorized); + + let mut wrong_journal = verification_input(); + wrong_journal.agent_pubkey = agent_pk.clone(); + wrong_journal.journal_id = "different-turn".into(); + wrong_journal.correlation_id = "different-turn".into(); + wrong_journal.source_event_ids = vec![authorized.id.to_hex()]; + let raw = build_verification_event(&owner, RELAY_A, &wrong_journal, 1).unwrap(); + let artifact = validate_signed_artifact(&raw, &owner_pk, RELAY_A, &agent_pk).unwrap(); + assert!( + validate_archived_verification_sources(&conn, &owner, &artifact) + .unwrap_err() + .contains("does not bind the journal") + ); + + let mut wrong_correlation = verification_input(); + wrong_correlation.agent_pubkey = agent_pk.clone(); + wrong_correlation.correlation_id = "different-correlation".into(); + wrong_correlation.source_event_ids = vec![authorized.id.to_hex()]; + assert!( + build_verification_event(&owner, RELAY_A, &wrong_correlation, 1) + .unwrap_err() + .contains("must equal the stable journalId") + ); + + let ciphertext = buzz_core_pkg::observer::encrypt_observer_payload( + &agent, + &owner.public_key(), + &serde_json::json!({ + "seq": 2, + "timestamp": "2026-08-21T14:00:01.000Z", + "kind": "turn_started", + "turnId": "agent:channel:turn-1", + "payload": { "triggeringEventIds": ["tool-call-1"] } + }), + ) + .unwrap(); + let tagless = EventBuilder::new(Kind::Custom(24200), ciphertext) + .sign_with_keys(&agent) + .unwrap(); + archive_observer(&conn, &owner_pk, relay_url, &tagless); + scope_observer(&conn, &owner_pk, relay_url, &tagless); + let mut input = verification_input(); + input.agent_pubkey = agent_pk.clone(); + input.source_event_ids = vec![tagless.id.to_hex()]; + let raw = build_verification_event(&owner, RELAY_A, &input, 1).unwrap(); + let artifact = validate_signed_artifact(&raw, &owner_pk, RELAY_A, &agent_pk).unwrap(); + assert!( + validate_archived_verification_sources(&conn, &owner, &artifact) + .unwrap_err() + .contains("observer authorization failed") + ); +} + +#[test] +fn durable_artifact_survives_close_and_reopen() { + let db_file = tempfile::NamedTempFile::new().unwrap(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let raw = build_verification_event(&owner, RELAY_A, &verification_input(), 1).unwrap(); + { + let conn = open_archive_db(db_file.path()).unwrap(); + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 10).unwrap(); + } + let reopened = open_archive_db(db_file.path()).unwrap(); + let rows = get_journal_authority_artifacts( + &reopened, + &owner_pk, + RELAY_A, + AGENT_A, + "agent:channel:turn-1", + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0].receipt_ref.as_deref(), + Some("receipt://archive/tool-call-1") + ); +} + +#[test] +fn bounded_today_query_is_owner_scoped_and_returns_public_fields_only() { + let conn = in_memory(); + let owner = Keys::generate(); + let other = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let other_pk = other.public_key().to_hex(); + let raw = build_owner_override_event(&owner, RELAY_A, &override_input("Today"), 1).unwrap(); + let artifact = upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw, 10).unwrap(); + + let rows = query_journal_authority_artifacts( + &conn, + &owner_pk, + RELAY_A, + AGENT_A, + artifact.created_at - 1, + artifact.created_at + 1, + 10, + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert!(query_journal_authority_artifacts( + &conn, + &other_pk, + RELAY_A, + AGENT_A, + artifact.created_at - 1, + artifact.created_at + 1, + 10, + ) + .unwrap() + .is_empty()); + assert!( + query_journal_authority_artifacts(&conn, &owner_pk, RELAY_A, AGENT_A, 1, 2, 501).is_err() + ); +} + +#[test] +fn authority_is_signed_stored_and_read_within_one_canonical_relay() { + let conn = in_memory(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let raw_a = build_owner_override_event( + &owner, + "wss://Relay-A.Example:443/", + &override_input("Relay A"), + 1, + ) + .unwrap(); + + let artifact_a = + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, AGENT_A, &raw_a, 10).unwrap(); + assert_eq!(artifact_a.relay_url, RELAY_A); + assert!(get_journal_authority_artifacts( + &conn, + &owner_pk, + RELAY_B, + AGENT_A, + "agent:channel:turn-1", + ) + .unwrap() + .is_empty()); + + let wrong_scope = + upsert_signed_artifact(&conn, &owner_pk, RELAY_B, AGENT_A, &raw_a, 11).unwrap_err(); + assert!(wrong_scope.contains("relay")); + + let raw_b = build_owner_override_event(&owner, RELAY_B, &override_input("Relay B"), 1).unwrap(); + let artifact_b = + upsert_signed_artifact(&conn, &owner_pk, RELAY_B, AGENT_A, &raw_b, 12).unwrap(); + assert_eq!(artifact_b.revision, 1); + assert_eq!( + get_journal_authority_artifacts(&conn, &owner_pk, RELAY_A, AGENT_A, "agent:channel:turn-1",) + .unwrap()[0] + .summary + .as_deref(), + Some("Relay A") + ); + assert_eq!( + get_journal_authority_artifacts(&conn, &owner_pk, RELAY_B, AGENT_A, "agent:channel:turn-1",) + .unwrap()[0] + .summary + .as_deref(), + Some("Relay B") + ); +} + +#[test] +fn same_journal_authority_isolated_by_managed_agent() { + let conn = in_memory(); + let owner = Keys::generate(); + let owner_pk = owner.public_key().to_hex(); + let agent_a = Keys::generate().public_key().to_hex(); + let agent_b = Keys::generate().public_key().to_hex(); + let raw_a = build_owner_override_event( + &owner, + RELAY_A, + &OwnerJournalOverrideInput { + agent_pubkey: agent_a.clone(), + ..override_input("Agent A") + }, + 1, + ) + .unwrap(); + let raw_b = build_owner_override_event( + &owner, + RELAY_A, + &OwnerJournalOverrideInput { + agent_pubkey: agent_b.clone(), + ..override_input("Agent B") + }, + 1, + ) + .unwrap(); + + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, &agent_a, &raw_a, 10).unwrap(); + assert!( + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, &agent_b, &raw_a, 11) + .unwrap_err() + .contains("managed agent") + ); + upsert_signed_artifact(&conn, &owner_pk, RELAY_A, &agent_b, &raw_b, 12).unwrap(); + + let a = get_journal_authority_artifacts( + &conn, + &owner_pk, + RELAY_A, + &agent_a, + "agent:channel:turn-1", + ) + .unwrap(); + let b = get_journal_authority_artifacts( + &conn, + &owner_pk, + RELAY_A, + &agent_b, + "agent:channel:turn-1", + ) + .unwrap(); + assert_eq!(a[0].summary.as_deref(), Some("Agent A")); + assert_eq!(b[0].summary.as_deref(), Some("Agent B")); + assert_eq!(a[0].revision, 1); + assert_eq!(b[0].revision, 1); + let queried_a = + query_journal_authority_artifacts(&conn, &owner_pk, RELAY_A, &agent_a, 0, i64::MAX, 10) + .unwrap(); + assert_eq!(queried_a.len(), 1); + assert_eq!(queried_a[0].agent_pubkey, agent_a); +} diff --git a/desktop/src-tauri/src/archive/mod.rs b/desktop/src-tauri/src/archive/mod.rs index 1b246b3fa23..c8d01992aee 100644 --- a/desktop/src-tauri/src/archive/mod.rs +++ b/desktop/src-tauri/src/archive/mod.rs @@ -19,15 +19,19 @@ mod agent_usage; mod archive_db; +mod journal_authority; +pub mod journal_authority_commands; mod metric_store; +mod observer_revision; +mod observer_time; mod pipeline; pub mod retention; pub mod store; mod store_migrations; pub mod sync; +pub(crate) mod today_snapshot; pub use archive_db::ArchiveDb; - use pipeline::{commit_archive, plan_archive, query_buckets}; use nostr::Event; @@ -595,6 +599,132 @@ pub async fn read_archived_observer_events_for_channel( .await } +/// Read observer frames by inner time with stable signed-envelope pagination. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ArchivedObserverRangeInput { + start_created_at: i64, + end_created_at: i64, + agent_pubkey: Option, + channel_id: Option, + before_created_at: Option, + before_id: Option, + archive_revision: Option, + limit: Option, +} +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ArchivedObserverRangeOutput { + events: Vec, + backfill_complete: bool, + unindexed_observer_frames: i64, + archive_revision: i64, + restart_required: bool, + total_observer_frames: i64, + has_more: bool, + next_before_created_at: Option, + next_before_id: Option, +} +#[tauri::command] +pub async fn read_archived_observer_events_for_range( + state: State<'_, AppState>, + input: ArchivedObserverRangeInput, +) -> Result { + let ArchivedObserverRangeInput { + start_created_at, + end_created_at, + agent_pubkey, + channel_id, + before_created_at, + before_id, + archive_revision, + limit, + } = input; + if start_created_at >= end_created_at { + return Err("archive range must have start_created_at < end_created_at".into()); + } + if before_created_at.is_some() != before_id.is_some() { + return Err("archive range cursor requires both before_created_at and before_id".into()); + } + if archive_revision.is_some_and(|revision| revision < 0) { + return Err("archive revision must be non-negative".into()); + } + let limit = limit.unwrap_or(DEFAULT_READ_LIMIT); + if !(1..=500).contains(&limit) { + return Err("archive range limit must be between 1 and 500".into()); + } + let identity_pk = identity_pubkey(&state)?; + let relay_url = relay_ws_url_with_override(&state); + let owner_keys = state + .keys + .lock() + .map_err(|error| error.to_string())? + .clone(); + state + .archive_db + .with_conn(move |conn| { + let backfill_complete = + observer_time::backfill_missing(conn, &identity_pk, &relay_url, &owner_keys)?; + if !backfill_complete { + let current_revision = observer_revision::current(conn, &identity_pk, &relay_url)?; + return Ok(ArchivedObserverRangeOutput { + events: Vec::new(), + backfill_complete: false, + unindexed_observer_frames: 0, + archive_revision: current_revision, + restart_required: false, + total_observer_frames: 0, + has_more: false, + next_before_created_at: None, + next_before_id: None, + }); + } + let tx = conn + .unchecked_transaction() + .map_err(|error| format!("begin observer range snapshot: {error}"))?; + let current_revision = observer_revision::current(&tx, &identity_pk, &relay_url)?; + let restart_required = + archive_revision.is_some_and(|expected| expected != current_revision); + let (page_before_created_at, page_before_id) = if restart_required { + (None, None) + } else { + (before_created_at, before_id.as_deref()) + }; + let unindexed_observer_frames = + store::count_unindexed_observer_frames(&tx, &identity_pk, &relay_url)?; + let page = observer_time::read_archived_observer_event_page_for_range( + &tx, + &identity_pk, + &relay_url, + start_created_at, + end_created_at, + agent_pubkey.as_deref(), + channel_id.as_deref(), + page_before_created_at, + page_before_id, + limit, + )?; + let has_more = page.total_count > page.rows.len() as i64; + let next_before_created_at = page.rows.last().map(|row| row.created_at); + let next_before_id = page.rows.last().map(|row| row.id.clone()); + let events = page.rows.into_iter().map(|row| row.raw_json).collect(); + tx.commit() + .map_err(|error| format!("finish observer range snapshot: {error}"))?; + Ok(ArchivedObserverRangeOutput { + events, + backfill_complete: true, + unindexed_observer_frames, + archive_revision: current_revision, + restart_required, + total_observer_frames: page.total_count, + has_more, + next_before_created_at, + next_before_id, + }) + }) + .await +} + // ── index_observer_channel_id ───────────────────────────────────────────────── /// Index one or more archived observer frame ids with their decoded channelId. diff --git a/desktop/src-tauri/src/archive/observer_revision.rs b/desktop/src-tauri/src/archive/observer_revision.rs new file mode 100644 index 00000000000..da43ebccff4 --- /dev/null +++ b/desktop/src-tauri/src/archive/observer_revision.rs @@ -0,0 +1,259 @@ +//! Monotonic revision fence for owner-scoped observer archive projections. +//! +//! Every mutation that can change Today reconstruction advances a durable +//! `(identity, relay)` revision in the same SQLite transaction as the source +//! mutation. Multi-page readers can therefore restart instead of combining +//! pages from different archive states, and publishers can reject a projection +//! if accepted evidence changed after reconstruction. + +use rusqlite::{params, Connection, Transaction, TransactionBehavior}; + +const SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS observer_archive_revisions ( + identity_pubkey TEXT NOT NULL, + relay_url TEXT NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (identity_pubkey, relay_url), + CHECK (revision >= 0) +); + +CREATE TRIGGER IF NOT EXISTS observer_revision_event_insert +AFTER INSERT ON archived_events WHEN NEW.kind = 24200 BEGIN + INSERT INTO observer_archive_revisions VALUES (NEW.identity_pubkey, NEW.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; +END; +CREATE TRIGGER IF NOT EXISTS observer_revision_event_delete +AFTER DELETE ON archived_events WHEN OLD.kind = 24200 BEGIN + INSERT INTO observer_archive_revisions VALUES (OLD.identity_pubkey, OLD.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; +END; +DROP TRIGGER IF EXISTS observer_revision_scope_insert; +CREATE TRIGGER observer_revision_scope_insert +AFTER INSERT ON archived_event_scopes +WHEN NEW.scope_type = 'owner_p' + AND EXISTS ( + SELECT 1 FROM archived_events + WHERE identity_pubkey = NEW.identity_pubkey + AND relay_url = NEW.relay_url AND id = NEW.id AND kind = 24200 + ) BEGIN + INSERT INTO observer_archive_revisions VALUES (NEW.identity_pubkey, NEW.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; +END; +DROP TRIGGER IF EXISTS observer_revision_scope_delete; +CREATE TRIGGER observer_revision_scope_delete +AFTER DELETE ON archived_event_scopes +WHEN OLD.scope_type = 'owner_p' + AND EXISTS ( + SELECT 1 FROM archived_events + WHERE identity_pubkey = OLD.identity_pubkey + AND relay_url = OLD.relay_url AND id = OLD.id AND kind = 24200 + ) BEGIN + INSERT INTO observer_archive_revisions VALUES (OLD.identity_pubkey, OLD.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; +END; +CREATE TRIGGER IF NOT EXISTS observer_revision_time_insert +AFTER INSERT ON observer_time_index BEGIN + INSERT INTO observer_archive_revisions VALUES (NEW.identity_pubkey, NEW.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; +END; +CREATE TRIGGER IF NOT EXISTS observer_revision_time_update +AFTER UPDATE ON observer_time_index BEGIN + INSERT INTO observer_archive_revisions VALUES (NEW.identity_pubkey, NEW.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; +END; +CREATE TRIGGER IF NOT EXISTS observer_revision_time_delete +AFTER DELETE ON observer_time_index BEGIN + INSERT INTO observer_archive_revisions VALUES (OLD.identity_pubkey, OLD.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; +END; +"#; + +pub(super) fn ensure_schema(conn: &Connection) -> Result<(), String> { + if schema_is_current(conn)? { + return Ok(()); + } + + // Serialise the migration across direct/cross-process opens, then recheck + // after winning the write lock. Normal connections take the read-only fast + // path above; they must not drop/recreate triggers on every open. + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate) + .map_err(|error| format!("begin observer archive revision migration: {error}"))?; + if !schema_is_current(&tx)? { + tx.execute_batch(SCHEMA) + .map_err(|error| format!("initialize observer archive revision: {error}"))?; + } + tx.commit() + .map_err(|error| format!("commit observer archive revision migration: {error}")) +} + +fn schema_is_current(conn: &Connection) -> Result { + let (table_count, trigger_count, insert_sql, delete_sql): (i64, i64, String, String) = conn + .query_row( + "SELECT + (SELECT COUNT(*) FROM sqlite_master + WHERE type = 'table' AND name = 'observer_archive_revisions'), + (SELECT COUNT(*) FROM sqlite_master + WHERE type = 'trigger' AND name IN ( + 'observer_revision_event_insert', + 'observer_revision_event_delete', + 'observer_revision_scope_insert', + 'observer_revision_scope_delete', + 'observer_revision_time_insert', + 'observer_revision_time_update', + 'observer_revision_time_delete' + )), + COALESCE((SELECT sql FROM sqlite_master + WHERE type = 'trigger' AND name = 'observer_revision_scope_insert'), ''), + COALESCE((SELECT sql FROM sqlite_master + WHERE type = 'trigger' AND name = 'observer_revision_scope_delete'), '')", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .map_err(|error| format!("inspect observer archive revision schema: {error}"))?; + + Ok(table_count == 1 + && trigger_count == 7 + && insert_sql.contains("kind = 24200") + && delete_sql.contains("kind = 24200")) +} + +pub(super) fn current( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, +) -> Result { + conn.query_row( + "SELECT COALESCE((SELECT revision FROM observer_archive_revisions + WHERE identity_pubkey = ?1 AND relay_url = ?2), 0)", + params![identity_pubkey, relay_url], + |row| row.get(0), + ) + .map_err(|error| format!("read observer archive revision: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn observer_mutations_advance_revision_transactionally() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE archived_events ( + identity_pubkey TEXT, relay_url TEXT, id TEXT, kind INTEGER + ); + CREATE TABLE archived_event_scopes ( + identity_pubkey TEXT, relay_url TEXT, id TEXT, + scope_type TEXT, scope_value TEXT + ); + CREATE TABLE observer_time_index ( + identity_pubkey TEXT, relay_url TEXT, id TEXT, + observed_start_at INTEGER, observed_end_at INTEGER + ); + CREATE TABLE observer_archive_revisions ( + identity_pubkey TEXT, relay_url TEXT, revision INTEGER, + PRIMARY KEY (identity_pubkey, relay_url) + ); + CREATE TRIGGER observer_revision_scope_insert + AFTER INSERT ON archived_event_scopes WHEN NEW.scope_type = 'owner_p' BEGIN + INSERT INTO observer_archive_revisions VALUES (NEW.identity_pubkey, NEW.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; + END; + CREATE TRIGGER observer_revision_scope_delete + AFTER DELETE ON archived_event_scopes WHEN OLD.scope_type = 'owner_p' BEGIN + INSERT INTO observer_archive_revisions VALUES (OLD.identity_pubkey, OLD.relay_url, 1) + ON CONFLICT (identity_pubkey, relay_url) + DO UPDATE SET revision = revision + 1; + END;", + ) + .unwrap(); + ensure_schema(&conn).unwrap(); + ensure_schema(&conn).unwrap(); + assert_eq!(current(&conn, "owner", "relay").unwrap(), 0); + + conn.execute( + "INSERT INTO archived_events VALUES ('owner','relay','metric',44200)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO archived_event_scopes VALUES ('owner','relay','metric','owner_p','owner')", + [], + ) + .unwrap(); + assert_eq!(current(&conn, "owner", "relay").unwrap(), 0); + + conn.execute( + "INSERT INTO archived_events VALUES ('owner','relay','event',24200)", + [], + ) + .unwrap(); + assert_eq!(current(&conn, "owner", "relay").unwrap(), 1); + + let tx = conn.unchecked_transaction().unwrap(); + tx.execute( + "INSERT INTO observer_time_index VALUES ('owner','relay','event',1,1)", + [], + ) + .unwrap(); + assert_eq!(current(&tx, "owner", "relay").unwrap(), 2); + tx.rollback().unwrap(); + assert_eq!(current(&conn, "owner", "relay").unwrap(), 1); + } + + #[test] + fn owner_scope_for_agent_metric_does_not_advance_observer_revision() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE archived_events ( + identity_pubkey TEXT, relay_url TEXT, id TEXT, kind INTEGER + ); + CREATE TABLE archived_event_scopes ( + identity_pubkey TEXT, relay_url TEXT, id TEXT, + scope_type TEXT, scope_value TEXT + ); + CREATE TABLE observer_time_index ( + identity_pubkey TEXT, relay_url TEXT, id TEXT, + observed_start_at INTEGER, observed_end_at INTEGER + );", + ) + .unwrap(); + ensure_schema(&conn).unwrap(); + + conn.execute( + "INSERT INTO archived_events VALUES ('owner','relay','metric',44200)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO archived_event_scopes VALUES ('owner','relay','metric','owner_p','owner')", + [], + ) + .unwrap(); + assert_eq!(current(&conn, "owner", "relay").unwrap(), 0); + + conn.execute("DELETE FROM archived_event_scopes WHERE id = 'metric'", []) + .unwrap(); + assert_eq!(current(&conn, "owner", "relay").unwrap(), 0); + + conn.execute( + "INSERT INTO archived_events VALUES ('owner','relay','observer',24200)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO archived_event_scopes VALUES ('owner','relay','observer','owner_p','owner')", + [], + ) + .unwrap(); + assert_eq!(current(&conn, "owner", "relay").unwrap(), 2); + } +} diff --git a/desktop/src-tauri/src/archive/observer_time.rs b/desktop/src-tauri/src/archive/observer_time.rs new file mode 100644 index 00000000000..6ad2412c0cf --- /dev/null +++ b/desktop/src-tauri/src/archive/observer_time.rs @@ -0,0 +1,544 @@ +//! Rebuildable inner-time index for encrypted observer envelopes. + +use chrono::DateTime; +use nostr::{Event, JsonUtil, Keys}; +use rusqlite::{params, Connection}; + +#[cfg(test)] +use super::store; + +const BACKFILL_BATCH_LIMIT: i64 = 500; + +fn timestamp_seconds(value: &serde_json::Value) -> Option { + DateTime::parse_from_rfc3339(value.get("timestamp")?.as_str()?) + .ok() + .map(|timestamp| timestamp.timestamp()) +} + +fn collect_bounds(value: &serde_json::Value, start: &mut Option, end: &mut Option) { + if value.get("kind").and_then(serde_json::Value::as_str) == Some("batch") { + if let Some(events) = value + .get("payload") + .and_then(|payload| payload.get("events")) + .and_then(serde_json::Value::as_array) + { + for event in events { + collect_bounds(event, start, end); + } + } + return; + } + let Some(timestamp) = timestamp_seconds(value) else { + return; + }; + *start = Some(start.map_or(timestamp, |current| current.min(timestamp))); + *end = Some(end.map_or(timestamp, |current| current.max(timestamp))); +} + +pub(super) fn bounds(value: &serde_json::Value) -> (Option, Option) { + let mut start = None; + let mut end = None; + collect_bounds(value, &mut start, &mut end); + (start, end) +} + +/// Record inclusive inner timestamp bounds. NULL bounds are the durable +/// processed marker for malformed or undecryptable envelopes. +pub(super) fn upsert( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + event_id: &str, + observed_start_at: Option, + observed_end_at: Option, +) -> Result<(), String> { + conn.execute( + "INSERT OR IGNORE INTO observer_time_index + (identity_pubkey, relay_url, id, observed_start_at, observed_end_at) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + identity_pubkey, + relay_url, + event_id, + observed_start_at, + observed_end_at + ], + ) + .map_err(|error| format!("failed to upsert observer_time_index: {error}"))?; + Ok(()) +} + +fn read_missing( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, +) -> Result, String> { + let mut stmt = conn + .prepare( + "SELECT ae.id, ae.raw_json + FROM archived_events ae + INNER JOIN archived_event_scopes aes + ON aes.identity_pubkey = ae.identity_pubkey + AND aes.relay_url = ae.relay_url + AND aes.id = ae.id + WHERE ae.identity_pubkey = ?1 + AND ae.relay_url = ?2 + AND ae.kind = 24200 + AND aes.scope_type = 'owner_p' + AND aes.scope_value = ?1 + AND NOT EXISTS ( + SELECT 1 FROM observer_time_index oti + WHERE oti.identity_pubkey = ae.identity_pubkey + AND oti.relay_url = ae.relay_url + AND oti.id = ae.id + ) + ORDER BY ae.created_at DESC, ae.id DESC + LIMIT ?3", + ) + .map_err(|error| format!("prepare observer time backfill: {error}"))?; + let rows = stmt + .query_map( + params![identity_pubkey, relay_url, BACKFILL_BATCH_LIMIT + 1], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .map_err(|error| format!("query observer time backfill: {error}"))?; + rows.collect::, _>>() + .map_err(|error| format!("read observer time backfill: {error}")) +} + +/// Lazily migrate one bounded batch of historical signed envelopes. Every +/// examined row receives an index record, including failures, so repeated +/// reads make deterministic progress without monopolizing the archive actor. +pub(super) fn backfill_missing( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + owner_keys: &Keys, +) -> Result { + let mut rows = read_missing(conn, identity_pubkey, relay_url)?; + let complete = rows.len() <= BACKFILL_BATCH_LIMIT as usize; + rows.truncate(BACKFILL_BATCH_LIMIT as usize); + let tx = conn + .unchecked_transaction() + .map_err(|error| format!("begin observer time backfill: {error}"))?; + for (event_id, raw_json) in rows { + let observed_bounds = Event::from_json(&raw_json) + .ok() + .filter(|event| event.id.to_hex() == event_id && event.verify().is_ok()) + .and_then(|event| { + buzz_core_pkg::observer::decrypt_observer_payload::( + owner_keys, &event, + ) + .ok() + }) + .map(|value| bounds(&value)) + .unwrap_or((None, None)); + upsert( + &tx, + identity_pubkey, + relay_url, + &event_id, + observed_bounds.0, + observed_bounds.1, + )?; + } + tx.commit() + .map_err(|error| format!("commit observer time backfill: {error}"))?; + Ok(complete) +} + +/// Count owner-scoped observer envelopes whose inner timestamp cannot be +/// attributed to any day. These rows must be disclosed by every ranged Today +/// reconstruction: filtering them out silently could hide terminal or failure +/// evidence while advertising a complete surface. +pub(super) fn count_unindexed_observer_frames( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, +) -> Result { + conn.query_row( + "SELECT COUNT(*) + FROM archived_events ae + JOIN archived_event_scopes aes USING (identity_pubkey, relay_url, id) + JOIN observer_time_index oti USING (identity_pubkey, relay_url, id) + WHERE ae.identity_pubkey = ?1 AND ae.relay_url = ?2 + AND aes.scope_type = 'owner_p' AND aes.scope_value = ?1 + AND ae.kind = 24200 + AND (oti.observed_start_at IS NULL OR oti.observed_end_at IS NULL)", + params![identity_pubkey, relay_url], + |row| row.get(0), + ) + .map_err(|error| format!("count observer frames without inner time: {error}")) +} + +/// Read owner-scoped observer events whose decrypted inner timestamps overlap +/// a half-open time range. The compound outer cursor remains stable for paging. +pub(super) struct ObserverRangeRow { + pub raw_json: String, + pub created_at: i64, + pub id: String, +} + +pub(super) struct ObserverRangePage { + pub rows: Vec, + pub total_count: i64, +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn read_archived_observer_event_page_for_range( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + start_created_at: i64, + end_created_at: i64, + agent_pubkey: Option<&str>, + channel_id: Option<&str>, + before_created_at: Option, + before_id: Option<&str>, + limit: i64, +) -> Result { + let mut values: Vec> = vec![ + Box::new(identity_pubkey.to_owned()), + Box::new(relay_url.to_owned()), + Box::new(identity_pubkey.to_owned()), + Box::new(start_created_at), + Box::new(end_created_at), + ]; + let mut clauses = String::new(); + if let Some(agent) = agent_pubkey { + values.push(Box::new(agent.to_owned())); + clauses.push_str(&format!(" AND ae.pubkey = ?{}", values.len())); + } + if let Some(channel) = channel_id { + values.push(Box::new(channel.to_owned())); + clauses.push_str(&format!( + " AND EXISTS ( + SELECT 1 FROM observer_channel_index oci + WHERE oci.identity_pubkey = ae.identity_pubkey + AND oci.relay_url = ae.relay_url + AND oci.id = ae.id + AND oci.channel_id = ?{})", + values.len() + )); + } + if let (Some(created_at), Some(id)) = (before_created_at, before_id) { + values.push(Box::new(created_at)); + let created_at_slot = values.len(); + values.push(Box::new(id.to_owned())); + let id_slot = values.len(); + clauses.push_str(&format!( + " AND (ae.created_at < ?{created_at_slot} + OR (ae.created_at = ?{created_at_slot} AND ae.id < ?{id_slot}))" + )); + } + values.push(Box::new(limit)); + let limit_slot = values.len(); + let sql = format!( + "SELECT ae.raw_json, ae.created_at, ae.id, COUNT(*) OVER () + FROM archived_events ae + JOIN archived_event_scopes aes USING (identity_pubkey, relay_url, id) + JOIN observer_time_index oti USING (identity_pubkey, relay_url, id) + WHERE ae.identity_pubkey = ?1 AND ae.relay_url = ?2 + AND aes.scope_type = 'owner_p' AND aes.scope_value = ?3 + AND ae.kind = 24200 AND oti.observed_start_at IS NOT NULL + AND oti.observed_end_at >= ?4 AND oti.observed_start_at < ?5 + {clauses} + ORDER BY ae.created_at DESC, ae.id DESC LIMIT ?{limit_slot}" + ); + let refs: Vec<&dyn rusqlite::ToSql> = values.iter().map(|value| value.as_ref()).collect(); + let mut stmt = conn + .prepare(&sql) + .map_err(|error| format!("prepare observer inner-time range: {error}"))?; + let rows = stmt + .query_map(refs.as_slice(), |row| { + Ok(( + ObserverRangeRow { + raw_json: row.get(0)?, + created_at: row.get(1)?, + id: row.get(2)?, + }, + row.get::<_, i64>(3)?, + )) + }) + .map_err(|error| format!("query observer inner-time range: {error}"))?; + let collected = rows + .collect::, _>>() + .map_err(|error| format!("read observer inner-time range row: {error}"))?; + let total_count = collected.first().map_or(0, |(_, total)| *total); + Ok(ObserverRangePage { + rows: collected.into_iter().map(|(row, _)| row).collect(), + total_count, + }) +} + +#[allow(clippy::too_many_arguments)] +#[cfg(test)] +pub(super) fn read_archived_observer_events_for_range( + conn: &Connection, + identity_pubkey: &str, + relay_url: &str, + start_created_at: i64, + end_created_at: i64, + agent_pubkey: Option<&str>, + channel_id: Option<&str>, + before_created_at: Option, + before_id: Option<&str>, + limit: i64, +) -> Result, String> { + read_archived_observer_event_page_for_range( + conn, + identity_pubkey, + relay_url, + start_created_at, + end_created_at, + agent_pubkey, + channel_id, + before_created_at, + before_id, + limit, + ) + .map(|page| page.rows.into_iter().map(|row| row.raw_json).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use nostr::{EventBuilder, Kind, Tag}; + + const RELAY: &str = "wss://relay.example"; + + fn signed_observer(owner: &Keys, agent: &Keys, timestamp: &str) -> Event { + let payload = serde_json::json!({ + "seq": 1, + "timestamp": timestamp, + "kind": "turn_started", + "channelId": "channel-1", + "turnId": "turn-1", + "payload": {} + }); + let ciphertext = + buzz_core_pkg::observer::encrypt_observer_payload(agent, &owner.public_key(), &payload) + .unwrap(); + EventBuilder::new(Kind::Custom(24200), ciphertext) + .tags([ + Tag::parse(["p", &owner.public_key().to_hex()]).unwrap(), + Tag::parse(["agent", &agent.public_key().to_hex()]).unwrap(), + Tag::parse(["frame", "telemetry"]).unwrap(), + ]) + .sign_with_keys(agent) + .unwrap() + } + + fn archive(conn: &Connection, owner: &Keys, event: &Event) { + let owner_pubkey = owner.public_key().to_hex(); + store::upsert_archived_event( + conn, + &owner_pubkey, + RELAY, + &event.id.to_hex(), + 24200, + &event.pubkey.to_hex(), + 10_000, + &event.as_json(), + 10_000, + ) + .unwrap(); + store::upsert_event_scope( + conn, + &owner_pubkey, + RELAY, + &event.id.to_hex(), + "owner_p", + &owner_pubkey, + 10_000, + ) + .unwrap(); + } + + #[test] + fn batch_bounds_follow_inner_events_not_outer_publication_time() { + let value = serde_json::json!({ + "kind": "batch", + "timestamp": "2026-08-22T10:00:00Z", + "payload": { "events": [ + { "kind": "turn_started", "timestamp": "2026-08-21T23:59:58Z" }, + { "kind": "turn_completed", "timestamp": "2026-08-22T00:00:02Z" } + ] } + }); + assert_eq!(bounds(&value), (Some(1_787_356_798), Some(1_787_356_802))); + } + + #[test] + fn historical_backfill_is_durable_idempotent_and_uses_inner_time() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(store::SCHEMA).unwrap(); + let owner = Keys::generate(); + let agent = Keys::generate(); + let owner_pubkey = owner.public_key().to_hex(); + let event = signed_observer(&owner, &agent, "2026-08-21T23:59:59Z"); + archive(&conn, &owner, &event); + + assert!(backfill_missing(&conn, &owner_pubkey, RELAY, &owner).unwrap()); + assert!(backfill_missing(&conn, &owner_pubkey, RELAY, &owner).unwrap()); + let rows = read_archived_observer_events_for_range( + &conn, + &owner_pubkey, + RELAY, + 1_787_356_799, + 1_787_356_800, + None, + None, + None, + None, + 10, + ) + .unwrap(); + assert_eq!(rows, [event.as_json()]); + } + + #[test] + fn invalid_historical_row_gets_durable_null_marker() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(store::SCHEMA).unwrap(); + let owner = Keys::generate(); + let owner_pubkey = owner.public_key().to_hex(); + store::upsert_archived_event( + &conn, + &owner_pubkey, + RELAY, + "bad", + 24200, + "agent", + 10_000, + "{}", + 10_000, + ) + .unwrap(); + store::upsert_event_scope( + &conn, + &owner_pubkey, + RELAY, + "bad", + "owner_p", + &owner_pubkey, + 10_000, + ) + .unwrap(); + + assert!(backfill_missing(&conn, &owner_pubkey, RELAY, &owner).unwrap()); + assert!(backfill_missing(&conn, &owner_pubkey, RELAY, &owner).unwrap()); + let bounds: (Option, Option) = conn + .query_row( + "SELECT observed_start_at, observed_end_at FROM observer_time_index WHERE id='bad'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(bounds, (None, None)); + assert_eq!( + count_unindexed_observer_frames(&conn, &owner_pubkey, RELAY).unwrap(), + 1 + ); + assert!(read_archived_observer_events_for_range( + &conn, + &owner_pubkey, + RELAY, + 0, + i64::MAX, + None, + None, + None, + None, + 10, + ) + .unwrap() + .is_empty()); + } + + #[test] + fn range_page_counts_and_cursors_malformed_archived_json() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(store::SCHEMA).unwrap(); + let owner = Keys::generate(); + let owner_pubkey = owner.public_key().to_hex(); + store::upsert_archived_event( + &conn, + &owner_pubkey, + RELAY, + "malformed", + 24200, + "agent", + 10_000, + "{invalid-json", + 10_000, + ) + .unwrap(); + store::upsert_event_scope( + &conn, + &owner_pubkey, + RELAY, + "malformed", + "owner_p", + &owner_pubkey, + 10_000, + ) + .unwrap(); + upsert(&conn, &owner_pubkey, RELAY, "malformed", Some(20), Some(20)).unwrap(); + + let page = read_archived_observer_event_page_for_range( + &conn, + &owner_pubkey, + RELAY, + 10, + 30, + None, + None, + None, + None, + 10, + ) + .unwrap(); + assert_eq!(page.total_count, 1); + assert_eq!(page.rows.len(), 1); + assert_eq!(page.rows[0].raw_json, "{invalid-json"); + assert_eq!(page.rows[0].created_at, 10_000); + assert_eq!(page.rows[0].id, "malformed"); + } + + #[test] + fn historical_backfill_is_bounded_and_resumable() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(store::SCHEMA).unwrap(); + let owner = Keys::generate(); + let owner_pubkey = owner.public_key().to_hex(); + for index in 0..=BACKFILL_BATCH_LIMIT { + let id = format!("bad-{index:04}"); + store::upsert_archived_event( + &conn, + &owner_pubkey, + RELAY, + &id, + 24200, + "agent", + index, + "{}", + index, + ) + .unwrap(); + store::upsert_event_scope( + &conn, + &owner_pubkey, + RELAY, + &id, + "owner_p", + &owner_pubkey, + index, + ) + .unwrap(); + } + + assert!(!backfill_missing(&conn, &owner_pubkey, RELAY, &owner).unwrap()); + assert!(backfill_missing(&conn, &owner_pubkey, RELAY, &owner).unwrap()); + assert!(backfill_missing(&conn, &owner_pubkey, RELAY, &owner).unwrap()); + } +} diff --git a/desktop/src-tauri/src/archive/pipeline.rs b/desktop/src-tauri/src/archive/pipeline.rs index 98ff64dff48..69be06f7acd 100644 --- a/desktop/src-tauri/src/archive/pipeline.rs +++ b/desktop/src-tauri/src/archive/pipeline.rs @@ -15,7 +15,10 @@ use rusqlite::Connection; use crate::app_state::AppState; use crate::relay::query_relay; -use super::{store, validate_ephemeral_frame, ArchiveBatchResult, ArchiveCandidate, MatchedScope}; +use super::{ + observer_time, store, validate_ephemeral_frame, ArchiveBatchResult, ArchiveCandidate, + MatchedScope, +}; // ── Private helpers ─────────────────────────────────────────────────────────── @@ -459,12 +462,13 @@ pub(super) fn commit_archive( // Write a status row regardless of outcome so backfill never // re-processes this frame (INSERT OR IGNORE on PK is a no-op if // the row is already present from a prior run). - let channel_id_for_index: Option = - buzz_core_pkg::observer::decrypt_observer_payload::( - owner_keys, &p.event, - ) - .ok() - .and_then(|v| v.get("channelId")?.as_str().map(|s| s.to_owned())); + let decoded = buzz_core_pkg::observer::decrypt_observer_payload::( + owner_keys, &p.event, + ) + .ok(); + let channel_id_for_index: Option = decoded + .as_ref() + .and_then(|value| value.get("channelId")?.as_str().map(str::to_owned)); store::upsert_observer_channel_index( &tx, identity_pk, @@ -473,6 +477,18 @@ pub(super) fn commit_archive( channel_id_for_index.as_deref(), p.event.created_at.as_secs() as i64, )?; + let (observed_start_at, observed_end_at) = decoded + .as_ref() + .map(observer_time::bounds) + .unwrap_or((None, None)); + observer_time::upsert( + &tx, + identity_pk, + relay_url, + eid, + observed_start_at, + observed_end_at, + )?; persisted += 1; } diff --git a/desktop/src-tauri/src/archive/store.rs b/desktop/src-tauri/src/archive/store.rs index 54cb9a4193b..3a589b24d74 100644 --- a/desktop/src-tauri/src/archive/store.rs +++ b/desktop/src-tauri/src/archive/store.rs @@ -13,6 +13,9 @@ use std::path::Path; use rusqlite::{params, Connection, OptionalExtension}; use std::time::{Duration, Instant}; +pub(super) use super::observer_time::count_unindexed_observer_frames; +#[cfg(test)] +pub(super) use super::observer_time::read_archived_observer_events_for_range; use super::store_migrations::apply_schema_migrations; // ── Schema ───────────────────────────────────────────────────────────────── @@ -29,7 +32,7 @@ CREATE TABLE IF NOT EXISTS archived_events ( archived_at INTEGER NOT NULL, PRIMARY KEY (identity_pubkey, relay_url, id) ); - +CREATE INDEX IF NOT EXISTS idx_archived_events_identity_id ON archived_events (identity_pubkey, id); CREATE TABLE IF NOT EXISTS archived_event_scopes ( identity_pubkey TEXT NOT NULL, relay_url TEXT NOT NULL, @@ -72,6 +75,22 @@ CREATE TABLE IF NOT EXISTS observer_channel_index ( CREATE INDEX IF NOT EXISTS idx_observer_channel ON observer_channel_index (identity_pubkey, relay_url, channel_id, created_at DESC, id DESC); +-- Rebuildable index of the authoritative decrypted observer timestamps inside +-- each signed envelope. A row with NULL bounds is a durable processed marker +-- for malformed/undecryptable payloads. Range reads use these bounds instead +-- of the later outer signing time, so queued frames cannot miss a local day. +CREATE TABLE IF NOT EXISTS observer_time_index ( + identity_pubkey TEXT NOT NULL, + relay_url TEXT NOT NULL, + id TEXT NOT NULL, + observed_start_at INTEGER, + observed_end_at INTEGER, + PRIMARY KEY (identity_pubkey, relay_url, id) +); +CREATE INDEX IF NOT EXISTS idx_observer_time + ON observer_time_index + (identity_pubkey, relay_url, observed_start_at, observed_end_at, id); + -- One-row migration state table: tracks which idempotent migrations have run. CREATE TABLE IF NOT EXISTS archive_migrations ( name TEXT PRIMARY KEY, @@ -138,6 +157,28 @@ CREATE INDEX IF NOT EXISTS idx_agent_metric_reported -- reported_at, so their window membership is judged by event_created_at). CREATE INDEX IF NOT EXISTS idx_agent_metric_created ON agent_metric_index (identity_pubkey, relay_url, event_created_at, parse_status); + +-- Owner-authorized Activity Ledger artifacts. Each row contains the complete +-- signed Nostr event and is re-verified on every read. `revision` prevents an +-- older but otherwise valid owner event from replaying over newer journal +-- authority state. +CREATE TABLE IF NOT EXISTS journal_authority_artifacts ( + identity_pubkey TEXT NOT NULL, + relay_url TEXT NOT NULL, + agent_pubkey TEXT NOT NULL, + journal_id TEXT NOT NULL, + artifact_type TEXT NOT NULL, + event_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + revision INTEGER NOT NULL, + raw_json TEXT NOT NULL, + stored_at INTEGER NOT NULL, + PRIMARY KEY (identity_pubkey, relay_url, agent_pubkey, journal_id, artifact_type), + UNIQUE (identity_pubkey, relay_url, agent_pubkey, event_id), + CHECK (artifact_type IN ('owner_override', 'verification')), + CHECK (revision > 0) +); +CREATE INDEX IF NOT EXISTS idx_journal_authority_created ON journal_authority_artifacts (identity_pubkey, created_at DESC, event_id DESC); "; // ── Open / init ───────────────────────────────────────────────────────────── @@ -162,6 +203,7 @@ pub fn open_archive_db(path: &Path) -> Result { conn.execute_batch(SCHEMA) .map_err(|e| format!("failed to initialize archive schema: {e}"))?; + super::observer_revision::ensure_schema(&conn)?; apply_schema_migrations(&conn)?; diff --git a/desktop/src-tauri/src/archive/store_migration_tests.rs b/desktop/src-tauri/src/archive/store_migration_tests.rs index 6a40d7f4cd7..79fede8d5c9 100644 --- a/desktop/src-tauri/src/archive/store_migration_tests.rs +++ b/desktop/src-tauri/src/archive/store_migration_tests.rs @@ -865,3 +865,111 @@ fn migration_m3_reopen_twice_is_idempotent() { "M3: marker must still be present after idempotent second open" ); } + +#[test] +fn migration_m5_quarantines_unscoped_journal_authority() { + let db_file = tempfile::NamedTempFile::new().unwrap(); + { + let conn = Connection::open(db_file.path()).unwrap(); + conn.execute_batch( + "CREATE TABLE journal_authority_artifacts ( + identity_pubkey TEXT NOT NULL, + journal_id TEXT NOT NULL, + artifact_type TEXT NOT NULL, + event_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + revision INTEGER NOT NULL, + raw_json TEXT NOT NULL, + stored_at INTEGER NOT NULL, + PRIMARY KEY (identity_pubkey, journal_id, artifact_type), + UNIQUE (identity_pubkey, event_id) + ); + INSERT INTO journal_authority_artifacts VALUES + ('owner', 'journal', 'owner_override', 'event', 1, 1, '{}', 1);", + ) + .unwrap(); + } + + let conn = open_archive_db(db_file.path()).unwrap(); + let scoped_columns: Vec = conn + .prepare("PRAGMA table_info(journal_authority_artifacts)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(scoped_columns.iter().any(|name| name == "relay_url")); + let active_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM journal_authority_artifacts", + [], + |row| row.get(0), + ) + .unwrap(); + let quarantined_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM journal_authority_artifacts_unscoped_v1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(active_rows, 0); + assert_eq!(quarantined_rows, 1); +} + +#[test] +fn migration_m6_quarantines_relay_scoped_authority_and_is_idempotent() { + let db_file = tempfile::NamedTempFile::new().unwrap(); + { + let conn = Connection::open(db_file.path()).unwrap(); + conn.execute_batch( + "CREATE TABLE journal_authority_artifacts ( + identity_pubkey TEXT NOT NULL, relay_url TEXT NOT NULL, + journal_id TEXT NOT NULL, artifact_type TEXT NOT NULL, + event_id TEXT NOT NULL, created_at INTEGER NOT NULL, + revision INTEGER NOT NULL, raw_json TEXT NOT NULL, + stored_at INTEGER NOT NULL, + PRIMARY KEY (identity_pubkey, relay_url, journal_id, artifact_type), + UNIQUE (identity_pubkey, relay_url, event_id) + ); + INSERT INTO journal_authority_artifacts VALUES + ('owner', 'wss://relay.example', 'journal', 'owner_override', + 'event', 1, 1, '{}', 1);", + ) + .unwrap(); + } + + let conn = open_archive_db(db_file.path()).unwrap(); + let columns: Vec = conn + .prepare("PRAGMA table_info(journal_authority_artifacts)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(columns.iter().any(|name| name == "agent_pubkey")); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM journal_authority_artifacts", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 0 + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM journal_authority_artifacts_relay_scoped_v2", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); + drop(conn); + let reopened = open_archive_db(db_file.path()).unwrap(); + assert_eq!( + reopened.query_row("SELECT COUNT(*) FROM archive_migrations WHERE name = 'scope_journal_authority_to_agent'", [], |row| row.get::<_, i64>(0)).unwrap(), + 1 + ); +} diff --git a/desktop/src-tauri/src/archive/store_migrations.rs b/desktop/src-tauri/src/archive/store_migrations.rs index 35a21e25d45..d7f6e075446 100644 --- a/desktop/src-tauri/src/archive/store_migrations.rs +++ b/desktop/src-tauri/src/archive/store_migrations.rs @@ -17,13 +17,308 @@ use rusqlite::{params, Connection}; /// /// Ordering: M2 (column additions) runs before M1 (index rebuild) so that /// the M1 rebuild, which calls `insert_metric_index_row`, always operates -/// against a schema that includes the cache-read columns. M4 (`archive_meta` -/// + scope-age index) runs last; it is independent of M1–M3. +/// against a schema that includes the cache-read columns. M4 adds +/// `archive_meta` and the scope-age index independently of M1–M3. M5 then +/// scopes journal authority to the canonical relay, and M6 to the managed +/// agent identity. pub(super) fn apply_schema_migrations(conn: &Connection) -> Result<(), String> { migrate_add_cache_read_tokens(conn)?; migrate_add_cache_write_and_pricing(conn)?; migrate_add_harness_to_metric_index(conn)?; - migrate_add_archive_meta(conn) + migrate_add_archive_meta(conn)?; + migrate_scope_journal_authority_to_relay(conn)?; + migrate_scope_journal_authority_to_agent(conn) +} + +const RELAY_SCOPED_JOURNAL_AUTHORITY_SCHEMA: &str = r#" +CREATE TABLE journal_authority_artifacts ( + identity_pubkey TEXT NOT NULL, + relay_url TEXT NOT NULL, + journal_id TEXT NOT NULL, + artifact_type TEXT NOT NULL, + event_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + revision INTEGER NOT NULL, + raw_json TEXT NOT NULL, + stored_at INTEGER NOT NULL, + PRIMARY KEY (identity_pubkey, relay_url, journal_id, artifact_type), + UNIQUE (identity_pubkey, relay_url, event_id), + CHECK (artifact_type IN ('owner_override', 'verification')), + CHECK (revision > 0) +); +CREATE INDEX idx_journal_authority_created + ON journal_authority_artifacts + (identity_pubkey, relay_url, created_at DESC, event_id DESC); +"#; + +const AGENT_SCOPED_JOURNAL_AUTHORITY_SCHEMA: &str = r#" +CREATE TABLE journal_authority_artifacts ( + identity_pubkey TEXT NOT NULL, + relay_url TEXT NOT NULL, + agent_pubkey TEXT NOT NULL, + journal_id TEXT NOT NULL, + artifact_type TEXT NOT NULL, + event_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + revision INTEGER NOT NULL, + raw_json TEXT NOT NULL, + stored_at INTEGER NOT NULL, + PRIMARY KEY (identity_pubkey, relay_url, agent_pubkey, journal_id, artifact_type), + UNIQUE (identity_pubkey, relay_url, agent_pubkey, event_id), + CHECK (artifact_type IN ('owner_override', 'verification')), + CHECK (revision > 0) +); +CREATE INDEX idx_journal_authority_created + ON journal_authority_artifacts + (identity_pubkey, relay_url, agent_pubkey, created_at DESC, event_id DESC); +"#; + +/// M5: replace the unreleased owner+journal authority key with an +/// owner+relay+journal key. Legacy events did not sign a relay and therefore +/// cannot be safely inferred or copied into the authoritative table. Preserve +/// them under an explicitly unscoped quarantine name for audit/recovery, while +/// all current reads fail closed against the new table. +fn migrate_scope_journal_authority_to_relay(conn: &Connection) -> Result<(), String> { + const MARKER: &str = "scope_journal_authority_to_relay"; + let already_run: bool = conn + .query_row( + "SELECT COUNT(*) FROM archive_migrations WHERE name = ?1", + [MARKER], + |row| row.get::<_, i64>(0), + ) + .map_err(|error| format!("migration M5: guard check: {error}"))? + > 0; + if already_run { + return Ok(()); + } + + conn.execute_batch("BEGIN IMMEDIATE") + .map_err(|error| format!("migration M5: begin immediate: {error}"))?; + let result = (|| -> Result<(), String> { + let already_run: bool = conn + .query_row( + "SELECT COUNT(*) FROM archive_migrations WHERE name = ?1", + [MARKER], + |row| row.get::<_, i64>(0), + ) + .map_err(|error| format!("migration M5: in-lock guard check: {error}"))? + > 0; + if already_run { + return Ok(()); + } + + let columns: Vec<(String, i64)> = { + let mut stmt = conn + .prepare("PRAGMA table_info(journal_authority_artifacts)") + .map_err(|error| format!("migration M5: inspect authority table: {error}"))?; + let rows = stmt + .query_map([], |row| Ok((row.get(1)?, row.get(5)?))) + .map_err(|error| format!("migration M5: query authority columns: {error}"))? + .collect::, _>>() + .map_err(|error| format!("migration M5: read authority columns: {error}"))?; + rows + }; + let agent_scoped = columns.iter().any(|(name, _)| name == "agent_pubkey"); + let relay_scoped = columns.iter().any(|(name, _)| name == "relay_url"); + if agent_scoped { + // Fresh databases already use the final M6 shape. M6 validates + // its full key and creates the final index immediately below. + } else if relay_scoped { + let primary_key = columns + .iter() + .filter(|(_, position)| *position > 0) + .map(|(name, position)| (*position, name.as_str())) + .collect::>(); + if primary_key + != [ + (1, "identity_pubkey"), + (2, "relay_url"), + (3, "journal_id"), + (4, "artifact_type"), + ] + { + return Err("migration M5: relay-scoped authority primary key is invalid".into()); + } + conn.execute_batch( + "DROP INDEX IF EXISTS idx_journal_authority_created; + CREATE INDEX idx_journal_authority_created + ON journal_authority_artifacts + (identity_pubkey, relay_url, created_at DESC, event_id DESC);", + ) + .map_err(|error| format!("migration M5: rebuild scoped index: {error}"))?; + } else { + let quarantine_exists: bool = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master + WHERE type = 'table' + AND name = 'journal_authority_artifacts_unscoped_v1'", + [], + |row| row.get::<_, i64>(0), + ) + .map_err(|error| format!("migration M5: inspect quarantine table: {error}"))? + > 0; + if quarantine_exists { + return Err( + "migration M5: unscoped authority quarantine already exists without marker" + .into(), + ); + } + conn.execute_batch( + "ALTER TABLE journal_authority_artifacts + RENAME TO journal_authority_artifacts_unscoped_v1; + DROP INDEX IF EXISTS idx_journal_authority_created;", + ) + .map_err(|error| format!("migration M5: quarantine unscoped authority: {error}"))?; + conn.execute_batch(RELAY_SCOPED_JOURNAL_AUTHORITY_SCHEMA) + .map_err(|error| format!("migration M5: create scoped authority table: {error}"))?; + } + + conn.execute( + "INSERT INTO archive_migrations (name, applied_at) VALUES (?1, ?2)", + params![ + MARKER, + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 + ], + ) + .map_err(|error| format!("migration M5: record marker: {error}"))?; + Ok(()) + })(); + + match result { + Ok(()) => conn + .execute_batch("COMMIT") + .map_err(|error| format!("migration M5: commit: {error}")), + Err(error) => { + let _ = conn.execute_batch("ROLLBACK"); + Err(error) + } + } +} + +/// M6: bind every authority row to the managed agent whose journal it can +/// affect. Relay-scoped v2 events did not sign an agent identity, so they are +/// quarantined without inference rather than copied into authoritative state. +fn migrate_scope_journal_authority_to_agent(conn: &Connection) -> Result<(), String> { + const MARKER: &str = "scope_journal_authority_to_agent"; + let already_run: bool = conn + .query_row( + "SELECT COUNT(*) FROM archive_migrations WHERE name = ?1", + [MARKER], + |row| row.get::<_, i64>(0), + ) + .map_err(|error| format!("migration M6: guard check: {error}"))? + > 0; + if already_run { + return Ok(()); + } + + conn.execute_batch("BEGIN IMMEDIATE") + .map_err(|error| format!("migration M6: begin immediate: {error}"))?; + let result = (|| -> Result<(), String> { + let already_run: bool = conn + .query_row( + "SELECT COUNT(*) FROM archive_migrations WHERE name = ?1", + [MARKER], + |row| row.get::<_, i64>(0), + ) + .map_err(|error| format!("migration M6: in-lock guard check: {error}"))? + > 0; + if already_run { + return Ok(()); + } + + let columns: Vec<(String, i64)> = { + let mut stmt = conn + .prepare("PRAGMA table_info(journal_authority_artifacts)") + .map_err(|error| format!("migration M6: inspect authority table: {error}"))?; + let rows = stmt + .query_map([], |row| Ok((row.get(1)?, row.get(5)?))) + .map_err(|error| format!("migration M6: query authority columns: {error}"))? + .collect::, _>>() + .map_err(|error| format!("migration M6: read authority columns: {error}"))?; + rows + }; + let agent_scoped = columns.iter().any(|(name, _)| name == "agent_pubkey"); + if agent_scoped { + let primary_key = columns + .iter() + .filter(|(_, position)| *position > 0) + .map(|(name, position)| (*position, name.as_str())) + .collect::>(); + if primary_key + != [ + (1, "identity_pubkey"), + (2, "relay_url"), + (3, "agent_pubkey"), + (4, "journal_id"), + (5, "artifact_type"), + ] + { + return Err("migration M6: agent-scoped authority primary key is invalid".into()); + } + conn.execute_batch( + "DROP INDEX IF EXISTS idx_journal_authority_created; + CREATE INDEX idx_journal_authority_created + ON journal_authority_artifacts + (identity_pubkey, relay_url, agent_pubkey, + created_at DESC, event_id DESC);", + ) + .map_err(|error| format!("migration M6: rebuild agent-scoped index: {error}"))?; + } else { + if !columns.iter().any(|(name, _)| name == "relay_url") { + return Err("migration M6: authority table is not relay scoped".into()); + } + let quarantine_exists: bool = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master + WHERE type = 'table' + AND name = 'journal_authority_artifacts_relay_scoped_v2'", + [], + |row| row.get::<_, i64>(0), + ) + .map_err(|error| format!("migration M6: inspect quarantine table: {error}"))? + > 0; + if quarantine_exists { + return Err( + "migration M6: relay-scoped authority quarantine exists without marker".into(), + ); + } + conn.execute_batch( + "ALTER TABLE journal_authority_artifacts + RENAME TO journal_authority_artifacts_relay_scoped_v2; + DROP INDEX IF EXISTS idx_journal_authority_created;", + ) + .map_err(|error| format!("migration M6: quarantine relay authority: {error}"))?; + conn.execute_batch(AGENT_SCOPED_JOURNAL_AUTHORITY_SCHEMA) + .map_err(|error| format!("migration M6: create agent authority table: {error}"))?; + } + + conn.execute( + "INSERT INTO archive_migrations (name, applied_at) VALUES (?1, ?2)", + params![ + MARKER, + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 + ], + ) + .map_err(|error| format!("migration M6: record marker: {error}"))?; + Ok(()) + })(); + + match result { + Ok(()) => conn + .execute_batch("COMMIT") + .map_err(|error| format!("migration M6: commit: {error}")), + Err(error) => { + let _ = conn.execute_batch("ROLLBACK"); + Err(error) + } + } } /// M1: add `harness TEXT` column to `agent_metric_index` and rebuild index diff --git a/desktop/src-tauri/src/archive/store_tests.rs b/desktop/src-tauri/src/archive/store_tests.rs index c0f85430d4d..6ae48521616 100644 --- a/desktop/src-tauri/src/archive/store_tests.rs +++ b/desktop/src-tauri/src/archive/store_tests.rs @@ -167,25 +167,28 @@ fn test_merge_owner_p_kinds_two_conn_wal_both_kinds_survive() { let init_conn = open_archive_db(&db_path).unwrap(); drop(init_conn); + // Open both connections before spawning either worker. Schema setup is not + // part of this regression, and a setup failure in one worker would leave + // the other worker parked on the barrier forever instead of reporting the + // actual error. + let conn_a = open_archive_db(&db_path).unwrap(); + let conn_b = open_archive_db(&db_path).unwrap(); + // Barrier ensures both threads are inside `merge_owner_p_kinds` before // either one issues `BEGIN IMMEDIATE`, maximising the race window. let barrier = Arc::new(Barrier::new(2)); - let path_a = db_path.clone(); - let path_b = db_path.clone(); let bar_a = Arc::clone(&barrier); let bar_b = Arc::clone(&barrier); let handle_observer = thread::spawn(move || { - let conn = open_archive_db(&path_a).unwrap(); bar_a.wait(); // sync: both threads ready - merge_owner_p_kinds(&conn, "pk", "wss://r", "mypk", 24200, 1) + merge_owner_p_kinds(&conn_a, "pk", "wss://r", "mypk", 24200, 1) }); let handle_metric = thread::spawn(move || { - let conn = open_archive_db(&path_b).unwrap(); bar_b.wait(); // sync: both threads ready - merge_owner_p_kinds(&conn, "pk", "wss://r", "mypk", 44200, 2) + merge_owner_p_kinds(&conn_b, "pk", "wss://r", "mypk", 44200, 2) }); let res_observer = handle_observer.join().expect("observer thread panicked"); @@ -857,3 +860,124 @@ fn test_read_unindexed_observer_rows_excludes_processed_rows() { "ev-old must be excluded from unindexed rows after null-channel_id indexing" ); } + +fn insert_owner_observer( + conn: &Connection, + id: &str, + agent: &str, + created_at: i64, + channel: Option<&str>, +) { + upsert_archived_event( + conn, + "owner", + "wss://r", + id, + 24200, + agent, + created_at, + &format!(r#"{{"id":"{id}","created_at":{created_at}}}"#), + created_at, + ) + .unwrap(); + upsert_event_scope(conn, "owner", "wss://r", id, "owner_p", "owner", created_at).unwrap(); + if let Some(channel) = channel { + upsert_observer_channel_index(conn, "owner", "wss://r", id, Some(channel), created_at) + .unwrap(); + } + super::super::observer_time::upsert( + conn, + "owner", + "wss://r", + id, + Some(created_at), + Some(created_at), + ) + .unwrap(); +} + +#[test] +fn test_read_archived_observer_events_for_range_pages_without_gaps() { + let conn = in_memory(); + insert_owner_observer(&conn, "z", "agent-a", 1002, Some("ch-1")); + insert_owner_observer(&conn, "a", "agent-a", 1002, Some("ch-1")); + insert_owner_observer(&conn, "old", "agent-b", 1001, Some("ch-2")); + insert_owner_observer(&conn, "outside", "agent-a", 999, Some("ch-1")); + + let page_one = read_archived_observer_events_for_range( + &conn, "owner", "wss://r", 1000, 1003, None, None, None, None, 1, + ) + .unwrap(); + assert_eq!(page_one.len(), 1); + assert!(page_one[0].contains(r#""id":"z""#)); + + let page_two = read_archived_observer_events_for_range( + &conn, + "owner", + "wss://r", + 1000, + 1003, + None, + None, + Some(1002), + Some("z"), + 10, + ) + .unwrap(); + assert_eq!(page_two.len(), 2); + assert!(page_two[0].contains(r#""id":"a""#)); + assert!(page_two[1].contains(r#""id":"old""#)); + + let filtered = read_archived_observer_events_for_range( + &conn, + "owner", + "wss://r", + 1000, + 1003, + Some("agent-a"), + Some("ch-1"), + None, + None, + 10, + ) + .unwrap(); + assert_eq!(filtered.len(), 2); +} + +#[test] +fn test_archived_observer_range_survives_close_and_reopen() { + use tempfile::NamedTempFile; + + let db_file = NamedTempFile::new().unwrap(); + { + let conn = open_archive_db(db_file.path()).unwrap(); + insert_owner_observer(&conn, "persisted", "agent-a", 1001, Some("ch-1")); + } + let reopened = open_archive_db(db_file.path()).unwrap(); + let rows = read_archived_observer_events_for_range( + &reopened, "owner", "wss://r", 1000, 1002, None, None, None, None, 10, + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert!(rows[0].contains(r#""id":"persisted""#)); +} + +#[test] +fn test_observer_range_uses_inner_time_after_unbounded_publication_delay() { + let conn = in_memory(); + insert_owner_observer(&conn, "delayed", "agent-a", 10_000, Some("ch-1")); + conn.execute( + "UPDATE observer_time_index + SET observed_start_at = 1001, observed_end_at = 1001 + WHERE identity_pubkey = 'owner' AND relay_url = 'wss://r' AND id = 'delayed'", + [], + ) + .unwrap(); + + let rows = read_archived_observer_events_for_range( + &conn, "owner", "wss://r", 1000, 1002, None, None, None, None, 10, + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert!(rows[0].contains(r#""id":"delayed""#)); +} diff --git a/desktop/src-tauri/src/archive/sync.rs b/desktop/src-tauri/src/archive/sync.rs index 3730774e952..3ebdabf5dde 100644 --- a/desktop/src-tauri/src/archive/sync.rs +++ b/desktop/src-tauri/src/archive/sync.rs @@ -16,9 +16,17 @@ //! `useArchiveSync` gated on `observerReconciled`, and it survives the move as //! an explicit `start_archive_sync` command issued after the same gate. -use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc, time::Duration}; +use std::{ + collections::HashMap, + future::Future, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; -use nostr::JsonUtil; use serde_json::json; use tauri::{AppHandle, Emitter, Manager, State}; use tokio::{ @@ -33,6 +41,14 @@ use super::{ use crate::app_state::AppState; use crate::native_relay_client::{MatchedEvent, NativeRelayClient, RelaySession, Subscription}; +#[path = "sync_queue.rs"] +mod sync_queue; +use sync_queue::{ + durable_queue_path, retry_delay, AcknowledgedHead, DurableCandidate, DurableQueue, + EnqueueError, EnqueueResult, FlushOutcome, ARCHIVE_ATTEMPT_TIMEOUT, ARCHIVE_RETRY_ATTEMPTS, + ARCHIVE_RETRY_MAX_DELAY, +}; + /// Flush once this many events are buffered. Parity with the renderer manager. const FLUSH_BATCH_SIZE: usize = 25; /// Maximum time an event waits in the buffer before being flushed. @@ -43,6 +59,15 @@ const FLUSH_BATCH_SIZE: usize = 25; /// already pending, so a steady trickle still flushed every 2s rather than /// never. The behavior is preserved; the name is corrected. const FLUSH_DEADLINE: Duration = Duration::from_millis(2_000); +#[cfg(not(test))] +const SHUTDOWN_WAIT_TIMEOUT: Duration = Duration::from_secs(12); +const RETRYABLE_START_ERROR_PREFIX: &str = "archive sync start retryable:"; + +fn retryable_start_error(error: impl std::fmt::Display) -> String { + format!("{RETRYABLE_START_ERROR_PREFIX} {error}") +} +#[cfg(test)] +const SHUTDOWN_WAIT_TIMEOUT: Duration = Duration::from_secs(1); /// Emitted after a batch persists new agent-metric rows, so the renderer can /// invalidate its usage queries. Replaces the in-process `notifyAgentMetrics @@ -144,36 +169,6 @@ fn subscription_id(scope_type: &ScopeType, scope_value: &str, kinds: &[u64]) -> format!("archive:{}:{scope_value}:{kinds}", scope_type.as_str()) } -// ── Batching ───────────────────────────────────────────────────────────────── - -/// Buffered candidates plus the deadline of the oldest one. -#[derive(Default)] -struct PendingBatch { - candidates: Vec, - /// Set when the buffer goes from empty to non-empty, cleared on take. The - /// deadline belongs to the oldest buffered event, so a steady trickle of - /// arrivals cannot postpone its flush indefinitely. - deadline: Option, -} - -impl PendingBatch { - fn push(&mut self, candidate: ArchiveCandidate) { - if self.candidates.is_empty() { - self.deadline = Some(Instant::now() + FLUSH_DEADLINE); - } - self.candidates.push(candidate); - } - - fn is_full(&self) -> bool { - self.candidates.len() >= FLUSH_BATCH_SIZE - } - - fn take(&mut self) -> Vec { - self.deadline = None; - std::mem::take(&mut self.candidates) - } -} - // ── Sync loop ──────────────────────────────────────────────────────────────── /// Drives one archive sync session until `cancel` fires. @@ -187,50 +182,167 @@ async fn run_sync( reload: Arc, mut events: mpsc::Receiver, cancel: CancellationToken, -) { + mut queue: DurableQueue, +) -> Result<(), String> { let mut scopes: HashMap = HashMap::new(); - let mut pending = PendingBatch::default(); + let mut acknowledged_head = None; + // Restored work predates every live event this task can receive and is + // therefore eligible immediately. A fresh queue retains the existing 2s + // batching deadline from the first accepted event. + let mut flush_deadline = queue.has_work().then(Instant::now); + let mut backing_off = false; + // The durable inbox is the primary byte/storage-boundary handoff. This + // single live fallback exists only when both queue writes fail after the + // relay frame has already been consumed. Receive stays backpressured until + // the exact candidate becomes durable. + let mut pending_candidate = None; + let mut stopping = false; reconcile(io, &mut scopes).await; loop { // `Instant::far_future()` is not public; a long sleep stands in for // "no deadline" so the select arm can be unconditional. - let deadline = pending - .deadline - .unwrap_or_else(|| Instant::now() + Duration::from_secs(3600)); + let deadline = flush_deadline.unwrap_or_else(|| Instant::now() + Duration::from_secs(3600)); tokio::select! { - _ = cancel.cancelled() => break, - _ = reload.notified() => { + _ = cancel.cancelled(), if !stopping => { + stopping = true; + if pending_candidate.is_none() { + break; + } + flush_deadline = Some(Instant::now()); + backing_off = true; + } + _ = reload.notified(), if !stopping => { reconcile(io, &mut scopes).await; } - _ = tokio::time::sleep_until(deadline), if pending.deadline.is_some() => { - flush(io, pending.take()).await; + _ = tokio::time::sleep_until(deadline), if flush_deadline.is_some() => { + let cancel_flush = (!stopping).then_some(&cancel); + let flush_outcome = flush_head_with_retries( + io, + &mut queue, + &mut acknowledged_head, + cancel_flush, + ).await; + if flush_outcome == FlushOutcome::Cancelled { + stopping = true; + } + + if let Some(candidate) = pending_candidate.take() { + match accept_received_candidate(&mut queue, candidate) { + Ok(EnqueueResult::Accepted | EnqueueResult::Duplicate) => { + if stopping { + break; + } + flush_deadline = queue.has_work().then(Instant::now); + backing_off = false; + continue; + } + Err(EnqueueError::Rejected(error)) => { + eprintln!("buzz-desktop: archive sync: rejected invalid retained relay event: {error}"); + if stopping { + break; + } + } + Err(EnqueueError::Capacity { candidate, message }) + | Err(EnqueueError::Storage { candidate, message }) => { + eprintln!("buzz-desktop: archive sync: durable parking retry failed; retaining one event in live memory: {message}"); + pending_candidate = Some(candidate); + flush_deadline = Some(Instant::now() + ARCHIVE_RETRY_MAX_DELAY); + backing_off = true; + continue; + } + } + } + + if stopping { + break; + } + match flush_outcome { + FlushOutcome::Empty => { + flush_deadline = None; + backing_off = false; + } + FlushOutcome::Committed => { + flush_deadline = queue.has_work().then(Instant::now); + backing_off = false; + } + FlushOutcome::Retained | FlushOutcome::Cancelled => { + flush_deadline = Some(Instant::now() + ARCHIVE_RETRY_MAX_DELAY); + backing_off = true; + } + } } - received = events.recv() => { + // The count cap stops consumption before receive. The exact byte + // cap is known only after serialization; that one event is parked + // in a separate durable inbox before backpressure stops receive. + received = events.recv(), if !stopping && pending_candidate.is_none() && !queue.has_inbox() && queue.has_entry_capacity() => { let Some(event) = received else { break }; // A subscription we already closed can still have events in // flight; without its scope we cannot assert a match, and the // backend re-verifies scope claims anyway, so drop it. let Some(scope) = scopes.get(&event.subscription_id) else { continue }; - pending.push(ArchiveCandidate { - raw_event_json: event.event.as_json(), - matched_scope: MatchedScope { - scope_type: scope.scope_type.clone(), - scope_value: scope.scope_value.clone(), - }, - }); - if pending.is_full() { - flush(io, pending.take()).await; + let was_empty = queue.is_empty(); + let candidate = DurableCandidate::from_event(&event.event, scope); + match accept_received_candidate(&mut queue, candidate) { + Ok(EnqueueResult::Duplicate) => continue, + Ok(EnqueueResult::Accepted) => {} + Err(EnqueueError::Rejected(error)) => { + eprintln!("buzz-desktop: archive sync: rejected invalid relay event: {error}"); + continue; + } + Err(EnqueueError::Capacity { candidate, message }) => { + eprintln!("buzz-desktop: archive sync: durable capacity unavailable; retaining one event in live memory: {message}"); + pending_candidate = Some(candidate); + flush_deadline = Some(Instant::now() + ARCHIVE_RETRY_MAX_DELAY); + backing_off = true; + continue; + } + Err(EnqueueError::Storage { candidate, message }) => { + eprintln!("buzz-desktop: archive sync: durable storage unavailable; retaining one event in live memory: {message}"); + pending_candidate = Some(candidate); + flush_deadline = Some(Instant::now() + ARCHIVE_RETRY_MAX_DELAY); + backing_off = true; + continue; + } + } + if was_empty { + flush_deadline = Some(Instant::now() + FLUSH_DEADLINE); + backing_off = false; + } + if queue.len() >= FLUSH_BATCH_SIZE && !backing_off { + flush_deadline = Some(Instant::now()); } } } } - // Buffered events are already off the relay; dropping them on shutdown - // would lose them permanently for the ephemeral scope. - flush(io, pending.take()).await; + // Every accepted event is already durable. Teardown makes one bounded + // retry cycle per head and then returns; failures remain on disk for the + // next Tauri process instead of hanging shutdown or discarding the tail. + while queue.has_work() { + match flush_head_with_retries(io, &mut queue, &mut acknowledged_head, None).await { + FlushOutcome::Committed => {} + FlushOutcome::Empty => break, + FlushOutcome::Retained | FlushOutcome::Cancelled => break, + } + } + Ok(()) +} + +/// Accept a consumed relay event into the main queue or, when that queue hits +/// its serialized boundary, the separate durable inbox. The returned +/// candidate on error is still exact and can be retained for another attempt. +fn accept_received_candidate( + queue: &mut DurableQueue, + candidate: DurableCandidate, +) -> Result { + match queue.enqueue(candidate) { + Err(EnqueueError::Capacity { candidate, .. }) + | Err(EnqueueError::Storage { candidate, .. }) => queue.stash_inbox(candidate), + result => result, + } } /// Reloads the saved subscriptions and applies them to the session. @@ -250,21 +362,104 @@ async fn reconcile(io: &I, scopes: &mut HashMap(io: &I, candidates: Vec) { - if candidates.is_empty() { - return; +/// Deliver exactly the durable head. Archive calls and acknowledgment writes +/// are both single-flight. Once the backend acknowledges, `acknowledged_head` +/// prevents an acknowledgment-write retry from re-delivering the batch in the +/// same process. A process crash in that narrow interval can replay once; the +/// archive database's signed event-id + scope key makes that idempotent. +async fn flush_head_with_retries( + io: &I, + queue: &mut DurableQueue, + acknowledged_head: &mut Option, + cancel: Option<&CancellationToken>, +) -> FlushOutcome { + if queue.is_empty() { + *acknowledged_head = None; + return FlushOutcome::Empty; } - match io.archive(candidates).await { - // The backend is authoritative: a duplicate-only batch or one with no - // kind-44200 events must not invalidate usage queries. - Ok(result) if result.persisted_agent_metrics > 0 => io.notify_agent_metrics_changed(), - Ok(_) => {} - Err(error) => eprintln!("buzz-desktop: archive sync: archive_events failed: {error}"), + + if acknowledged_head.is_none() { + let mut archived = None; + for attempt in 0..ARCHIVE_RETRY_ATTEMPTS { + let archive = tokio::time::timeout(ARCHIVE_ATTEMPT_TIMEOUT, io.archive(queue.head())); + let result = if let Some(cancel) = cancel { + tokio::select! { + _ = cancel.cancelled() => return FlushOutcome::Cancelled, + result = archive => result, + } + } else { + archive.await + }; + match result { + Ok(Ok(result)) => { + archived = Some(AcknowledgedHead { + count: queue.head_len(), + persisted_agent_metrics: result.persisted_agent_metrics, + }); + break; + } + Ok(Err(error)) => eprintln!( + "buzz-desktop: archive sync: archive_events attempt {} failed: {error}", + attempt + 1 + ), + Err(_) => eprintln!( + "buzz-desktop: archive sync: archive_events attempt {} timed out", + attempt + 1 + ), + } + if attempt + 1 < ARCHIVE_RETRY_ATTEMPTS { + let sleep = tokio::time::sleep(retry_delay(attempt)); + if let Some(cancel) = cancel { + tokio::select! { + _ = cancel.cancelled() => return FlushOutcome::Cancelled, + _ = sleep => {} + } + } else { + sleep.await; + } + } + } + let Some(archived) = archived else { + return FlushOutcome::Retained; + }; + *acknowledged_head = Some(archived); } + + // Archive acknowledgment already happened. Retry only the atomic durable + // removal; do not call archive again while this process remembers the ack. + for attempt in 0..ARCHIVE_RETRY_ATTEMPTS { + let Some(acknowledged) = *acknowledged_head else { + eprintln!( + "buzz-desktop: archive sync: archive acknowledgment state missing; retaining durable head" + ); + return FlushOutcome::Retained; + }; + match queue.acknowledge_head(acknowledged.count) { + Ok(()) => { + if acknowledged.persisted_agent_metrics > 0 { + io.notify_agent_metrics_changed(); + } + *acknowledged_head = None; + return FlushOutcome::Committed; + } + Err(error) => eprintln!( + "buzz-desktop: archive sync: durable acknowledgment attempt {} failed: {error}", + attempt + 1 + ), + } + if attempt + 1 < ARCHIVE_RETRY_ATTEMPTS { + let sleep = tokio::time::sleep(retry_delay(attempt)); + if let Some(cancel) = cancel { + tokio::select! { + _ = cancel.cancelled() => return FlushOutcome::Cancelled, + _ = sleep => {} + } + } else { + sleep.await; + } + } + } + FlushOutcome::Retained } // ── Production wiring ──────────────────────────────────────────────────────── @@ -351,10 +546,38 @@ pub struct ArchiveSyncState { struct RunningSync { /// Identity + relay this task is bound to. A start request for the same - /// scope is a no-op, so a renderer remount does not churn the socket. + /// scope is a no-op only while this task is healthy, so a renderer remount + /// cannot mistake a timed-out teardown for success. scope: (String, String), cancel: CancellationToken, reload: Arc, + completion: Arc, +} + +#[derive(Default)] +struct SyncCompletion { + finished: AtomicBool, + notify: Notify, +} + +impl SyncCompletion { + fn finish(&self) { + self.finished.store(true, Ordering::Release); + self.notify.notify_waiters(); + } + + async fn wait(&self) { + loop { + if self.finished.load(Ordering::Acquire) { + return; + } + let notified = self.notify.notified(); + if self.finished.load(Ordering::Acquire) { + return; + } + notified.await; + } + } } /// Proof that the holder is the current archive-sync owner, and the lock that @@ -474,21 +697,30 @@ impl ArchiveSyncState { scope: (String, String), cancel: CancellationToken, reload: Arc, - ) -> Option> { + completion: Arc, + ) -> Result>, String> { let mut latest = self.latest.lock().await; if mark <= *latest { - return None; + return Ok(None); } *latest = mark; let mut running = self.running.lock().await; - // A same-scope remount keeps its socket: reinstalling would tear down a - // healthy relay session to replace it with an identical one. - if running - .as_ref() - .is_some_and(|current| current.scope == scope) - { - return None; + // A same-scope remount keeps a healthy socket. A finished task may be + // replaced before its cleanup wins this lock. A cancelled task that + // has not finished is still draining its durable queue, so neither + // race it nor report a healthy no-op. + if let Some(current) = running.as_ref().filter(|current| current.scope == scope) { + if current.completion.finished.load(Ordering::Acquire) { + running.take(); + } else if current.cancel.is_cancelled() { + return Err( + "archive sync for this owner and relay is still stopping; retry start after teardown" + .into(), + ); + } else { + return Ok(None); + } } if let Some(previous) = running.take() { previous.cancel.cancel(); @@ -497,11 +729,12 @@ impl ArchiveSyncState { scope, cancel, reload, + completion, }); - Some(ArchiveOwnership { + Ok(Some(ArchiveOwnership { _latest: latest, _running: running, - }) + })) } /// Releases ownership for a stop under `(epoch, lease)`, cancelling the @@ -518,15 +751,42 @@ impl ArchiveSyncState { /// newer start install its task, and then cancel that task on resume — /// stale cleanup stranding the newest owner. Both halves take `latest` then /// `running`, so the two can never interleave and the order is deadlock-free. - async fn end(&self, mark: (u64, u64)) { + async fn end(&self, mark: (u64, u64)) -> Result<(), String> { let mut latest = self.latest.lock().await; if mark < *latest { - return; + return Ok(()); } *latest = mark; - if let Some(running) = self.running.lock().await.take() { + let running = self.running.lock().await.take(); + if let Some(running) = running { running.cancel.cancel(); + if tokio::time::timeout(SHUTDOWN_WAIT_TIMEOUT, running.completion.wait()) + .await + .is_err() + { + // Keep the cancelled task visible so a later start cannot + // assume teardown completed and race the same durable queue. + *self.running.lock().await = Some(running); + return Err(format!( + "archive sync teardown did not finish within {}ms; durable queue retained", + SHUTDOWN_WAIT_TIMEOUT.as_millis() + )); + } + } + Ok(()) + } + + /// Clear a task that finished after a bounded stop timed out and reinserted + /// it. Pointer identity prevents an old task's late completion from + /// clearing a newer task installed for the same scope. + async fn clear_completed(&self, completion: &Arc) { + let mut running = self.running.lock().await; + if running.as_ref().is_some_and(|current| { + Arc::ptr_eq(¤t.completion, completion) + && current.completion.finished.load(Ordering::Acquire) + }) { + running.take(); } } } @@ -564,18 +824,36 @@ pub async fn start_archive_sync( let keys = state.signing_keys()?; let relay_url = crate::relay::relay_ws_url_with_override(&state); let scope = (keys.public_key().to_hex(), relay_url.clone()); + let queue_path = durable_queue_path(&app, &scope.0, &scope.1).map_err(retryable_start_error)?; // Only cheap handles before `begin`: a start that lost its mark, or a // same-scope remount, must not open a relay socket just to drop it again. let cancel = CancellationToken::new(); let reload = Arc::new(Notify::new()); + let completion = Arc::new(SyncCompletion::default()); let Some(ownership) = sync_state - .begin((epoch, lease), scope, cancel.clone(), Arc::clone(&reload)) - .await + .begin( + (epoch, lease), + scope.clone(), + cancel.clone(), + Arc::clone(&reload), + Arc::clone(&completion), + ) + .await? else { return Ok(()); }; + let queue = match DurableQueue::open(queue_path) { + Ok(queue) => queue, + Err(error) => { + completion.finish(); + drop(ownership); + sync_state.end((epoch, lease)).await?; + return Err(retryable_start_error(error)); + } + }; + // No NIP-OA auth tag: this is the owner's own session, authenticated as // the identity itself, exactly like the renderer's relay client. // @@ -591,8 +869,13 @@ pub async fn start_archive_sync( session: Arc::clone(&session), }; tauri::async_runtime::spawn(async move { - run_sync(&io, reload, events, cancel).await; + if let Err(error) = run_sync(&io, reload, events, cancel, queue).await { + eprintln!("buzz-desktop: archive sync stopped fail-closed: {error}"); + } session.set_subscriptions(Vec::new()).await; + completion.finish(); + let sync_state: State<'_, ArchiveSyncState> = io.app.state(); + sync_state.clear_completed(&completion).await; }); Ok(()) } @@ -608,10 +891,13 @@ pub async fn stop_archive_sync( epoch: u64, lease: u64, ) -> Result<(), String> { - sync_state.end((epoch, lease)).await; - Ok(()) + sync_state.end((epoch, lease)).await } +#[cfg(test)] +#[path = "sync_queue_tests.rs"] +mod sync_queue_tests; + #[cfg(test)] #[path = "sync_tests.rs"] mod sync_tests; diff --git a/desktop/src-tauri/src/archive/sync_queue.rs b/desktop/src-tauri/src/archive/sync_queue.rs new file mode 100644 index 00000000000..0912190686c --- /dev/null +++ b/desktop/src-tauri/src/archive/sync_queue.rs @@ -0,0 +1,682 @@ +//! Crash-resilient write-ahead queue for native archive sync. + +use std::{ + collections::HashSet, + fs, + path::{Path, PathBuf}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use nostr::{Event, JsonUtil}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use tauri::{AppHandle, Manager}; + +use super::FLUSH_BATCH_SIZE; +use crate::archive::{ArchiveCandidate, MatchedScope, ScopeType}; + +/// A queue is deliberately finite. Relay backpressure stops acceptance once +/// this file cannot grow safely; silently dropping the tail is never allowed. +pub(super) const MAX_DURABLE_QUEUE_ENTRIES: usize = 2_048; +const MAX_DURABLE_QUEUE_BYTES: usize = 8 * 1024 * 1024; +const MAX_DURABLE_INBOX_BYTES: usize = 1024 * 1024; +const MAX_ACKED_DEDUPE_KEYS: usize = 2_048; +const DURABLE_QUEUE_VERSION: u8 = 1; +const DURABLE_INBOX_VERSION: u8 = 1; + +pub(super) const ARCHIVE_RETRY_ATTEMPTS: usize = 4; +#[cfg(not(test))] +const ARCHIVE_RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +#[cfg(test)] +const ARCHIVE_RETRY_BASE_DELAY: Duration = Duration::from_millis(1); +#[cfg(not(test))] +pub(super) const ARCHIVE_RETRY_MAX_DELAY: Duration = Duration::from_millis(800); +#[cfg(test)] +pub(super) const ARCHIVE_RETRY_MAX_DELAY: Duration = Duration::from_millis(8); +#[cfg(not(test))] +pub(super) const ARCHIVE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2); +#[cfg(test)] +pub(super) const ARCHIVE_ATTEMPT_TIMEOUT: Duration = Duration::from_millis(50); +// ── Durable write-ahead queue ──────────────────────────────────────────────── + +/// Queue representation intentionally contains only the public signed Nostr +/// envelope and its asserted archive scope. Private keys, auth tokens, and +/// decrypted observer payloads never enter this file. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct DurableCandidate { + pub(super) event_id: String, + raw_event_json: String, + scope_type: ScopeType, + scope_value: String, +} + +impl DurableCandidate { + pub(super) fn from_event(event: &Event, scope: &MatchedScope) -> Self { + Self { + event_id: event.id.to_hex(), + raw_event_json: event.as_json(), + scope_type: scope.scope_type.clone(), + scope_value: scope.scope_value.clone(), + } + } + + fn dedupe_key(&self) -> String { + format!( + "{}:{}:{}", + self.event_id, + self.scope_type.as_str(), + self.scope_value + ) + } + + fn archive_candidate(&self) -> ArchiveCandidate { + ArchiveCandidate { + raw_event_json: self.raw_event_json.clone(), + matched_scope: MatchedScope { + scope_type: self.scope_type.clone(), + scope_value: self.scope_value.clone(), + }, + } + } + + fn validate(&self) -> Result<(), String> { + let event = Event::from_json(&self.raw_event_json) + .map_err(|error| format!("queued event is not valid Nostr JSON: {error}"))?; + event + .verify() + .map_err(|error| format!("queued event signature is invalid: {error}"))?; + if event.id.to_hex() != self.event_id { + return Err("queued event id does not match its signed envelope".into()); + } + if self.scope_value.is_empty() { + return Err("queued archive scope is empty".into()); + } + Ok(()) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct DurableQueueFile { + version: u8, + entries: Vec, + #[serde(default)] + acked_keys: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct DurableInboxFile { + version: u8, + candidate: DurableCandidate, +} + +#[derive(Debug)] +pub(super) struct DurableQueue { + path: PathBuf, + inbox_path: PathBuf, + entries: Vec, + /// One event may cross the serialized queue-byte boundary after socket + /// receive. It is durably parked here before any older queue head drains, + /// so cancellation or restart cannot lose the already-consumed frame. + inbox: Option, + /// A bounded durable tombstone window suppresses relay duplicates after an + /// acknowledged item has left the pending queue. A crash after archive + /// acknowledgment but before this tombstone commits can still replay once; + /// the archive database's signed-id/scope uniqueness is the final guard. + acked_keys: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum EnqueueResult { + Accepted, + Duplicate, +} + +#[derive(Debug)] +pub(super) enum EnqueueError { + Rejected(String), + Capacity { + candidate: DurableCandidate, + message: String, + }, + Storage { + candidate: DurableCandidate, + message: String, + }, +} + +impl std::fmt::Display for EnqueueError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Rejected(message) + | Self::Capacity { message, .. } + | Self::Storage { message, .. } => formatter.write_str(message), + } + } +} + +#[derive(Debug)] +enum QueuePersistError { + Capacity(String), + Storage(String), +} + +impl std::fmt::Display for QueuePersistError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Capacity(message) | Self::Storage(message) => formatter.write_str(message), + } + } +} + +impl DurableQueue { + pub(super) fn open(path: PathBuf) -> Result { + let parent = path + .parent() + .ok_or_else(|| "archive sync queue path has no parent".to_string())?; + ensure_owner_only_dir(parent)?; + + let inbox_path = durable_inbox_path(&path); + for blocked_path in [ + blocked_diagnostic_path(&path), + blocked_diagnostic_path(&inbox_path), + ] { + if blocked_path.exists() { + let diagnostic = fs::read_to_string(&blocked_path) + .unwrap_or_else(|_| "archive sync queue is blocked".to_string()); + return Err(format!( + "archive sync durable queue is fail-closed: {}", + diagnostic.trim() + )); + } + } + + let inbox = load_durable_inbox(&inbox_path)?; + + if !path.exists() { + return Ok(Self { + path, + inbox_path, + entries: Vec::new(), + inbox, + acked_keys: Vec::new(), + }); + } + + let bytes = fs::read(&path) + .map_err(|error| format!("read durable archive queue {}: {error}", path.display()))?; + if bytes.len() > MAX_DURABLE_QUEUE_BYTES { + return Err(quarantine_corrupt_queue( + &path, + format!( + "queue file is {} bytes; maximum is {MAX_DURABLE_QUEUE_BYTES}", + bytes.len() + ), + )); + } + let persisted: DurableQueueFile = serde_json::from_slice(&bytes).map_err(|error| { + quarantine_corrupt_queue(&path, format!("invalid queue JSON: {error}")) + })?; + if persisted.version != DURABLE_QUEUE_VERSION { + return Err(quarantine_corrupt_queue( + &path, + format!( + "unsupported queue version {}; expected {DURABLE_QUEUE_VERSION}", + persisted.version + ), + )); + } + if persisted.entries.len() > MAX_DURABLE_QUEUE_ENTRIES { + return Err(quarantine_corrupt_queue( + &path, + format!( + "queue contains {} entries; maximum is {MAX_DURABLE_QUEUE_ENTRIES}", + persisted.entries.len() + ), + )); + } + for entry in &persisted.entries { + if let Err(error) = entry.validate() { + return Err(quarantine_corrupt_queue(&path, error)); + } + } + + let mut changed = persisted.acked_keys.len() > MAX_ACKED_DEDUPE_KEYS; + let mut acked_seen = HashSet::new(); + let mut acked_keys = Vec::new(); + for key in persisted + .acked_keys + .into_iter() + .rev() + .take(MAX_ACKED_DEDUPE_KEYS) + .collect::>() + .into_iter() + .rev() + { + if acked_seen.insert(key.clone()) { + acked_keys.push(key); + } else { + changed = true; + } + } + let acked: HashSet<_> = acked_keys.iter().cloned().collect(); + let mut pending_seen = HashSet::new(); + let mut entries = Vec::new(); + for entry in persisted.entries { + let key = entry.dedupe_key(); + if acked.contains(&key) || !pending_seen.insert(key) { + changed = true; + continue; + } + entries.push(entry); + } + let mut inbox = inbox; + if let Some(candidate) = &inbox { + let key = candidate.dedupe_key(); + if acked.contains(&key) || pending_seen.contains(&key) { + remove_durable_inbox(&inbox_path)?; + inbox = None; + } + } + + let queue = Self { + path, + inbox_path, + entries, + inbox, + acked_keys, + }; + if changed { + queue.persist().map_err(|error| error.to_string())?; + } + Ok(queue) + } + + pub(super) fn len(&self) -> usize { + self.entries.len() + } + + pub(super) fn is_empty(&self) -> bool { + self.entries.is_empty() && self.inbox.is_none() + } + + pub(super) fn has_work(&self) -> bool { + !self.is_empty() + } + + pub(super) fn has_inbox(&self) -> bool { + self.inbox.is_some() + } + + pub(super) fn has_entry_capacity(&self) -> bool { + self.entries.len() < MAX_DURABLE_QUEUE_ENTRIES + } + + pub(super) fn enqueue( + &mut self, + candidate: DurableCandidate, + ) -> Result { + candidate.validate().map_err(EnqueueError::Rejected)?; + let key = candidate.dedupe_key(); + if self.acked_keys.iter().any(|existing| existing == &key) + || self + .entries + .iter() + .any(|existing| existing.dedupe_key() == key) + { + return Ok(EnqueueResult::Duplicate); + } + if self.entries.len() >= MAX_DURABLE_QUEUE_ENTRIES { + return Err(EnqueueError::Capacity { + candidate, + message: format!( + "archive sync durable queue limit reached ({MAX_DURABLE_QUEUE_ENTRIES} entries); event was not accepted" + ), + }); + } + + self.entries.push(candidate.clone()); + if let Err(error) = self.persist() { + self.entries.pop(); + let message = + format!("archive sync durable queue write failed; event was not accepted: {error}"); + return Err(match error { + QueuePersistError::Capacity(_) => EnqueueError::Capacity { candidate, message }, + QueuePersistError::Storage(_) => EnqueueError::Storage { candidate, message }, + }); + } + Ok(EnqueueResult::Accepted) + } + + pub(super) fn stash_inbox( + &mut self, + candidate: DurableCandidate, + ) -> Result { + candidate.validate().map_err(EnqueueError::Rejected)?; + if let Some(existing) = &self.inbox { + if existing.dedupe_key() == candidate.dedupe_key() { + return Ok(EnqueueResult::Duplicate); + } + return Err(EnqueueError::Capacity { + candidate, + message: "archive sync durable inbox already contains an event".into(), + }); + } + if self + .acked_keys + .iter() + .any(|key| key == &candidate.dedupe_key()) + || self + .entries + .iter() + .any(|entry| entry.dedupe_key() == candidate.dedupe_key()) + { + return Ok(EnqueueResult::Duplicate); + } + if let Err(error) = persist_inbox_file(&self.inbox_path, &candidate) { + let message = + format!("archive sync durable inbox write failed; event was not accepted: {error}"); + return Err(match error { + QueuePersistError::Capacity(_) => EnqueueError::Capacity { candidate, message }, + QueuePersistError::Storage(_) => EnqueueError::Storage { candidate, message }, + }); + } + self.inbox = Some(candidate); + Ok(EnqueueResult::Accepted) + } + + pub(super) fn head(&self) -> Vec { + if self.entries.is_empty() { + return self + .inbox + .iter() + .map(DurableCandidate::archive_candidate) + .collect(); + } + self.entries + .iter() + .take(FLUSH_BATCH_SIZE) + .map(DurableCandidate::archive_candidate) + .collect() + } + + pub(super) fn head_len(&self) -> usize { + if self.entries.is_empty() && self.inbox.is_some() { + 1 + } else { + self.entries.len().min(FLUSH_BATCH_SIZE) + } + } + + /// Atomically records both removal and the recent-ack tombstones. Memory is + /// changed only after the replacement file commits, so a failed write + /// leaves the durable head intact for another attempt or restart. + pub(super) fn acknowledge_head(&mut self, count: usize) -> Result<(), String> { + if self.entries.is_empty() { + let Some(inbox) = &self.inbox else { + return Err("invalid durable archive acknowledgment count".into()); + }; + if count != 1 { + return Err("invalid durable archive inbox acknowledgment count".into()); + } + let mut next_acked = self.acked_keys.clone(); + next_acked.push(inbox.dedupe_key()); + if next_acked.len() > MAX_ACKED_DEDUPE_KEYS { + next_acked.drain(..next_acked.len() - MAX_ACKED_DEDUPE_KEYS); + } + // Commit the tombstone before removing the separately durable + // inbox. A crash between these writes leaves a duplicate that open + // removes without redelivery, never an untracked accepted event. + persist_queue_file(&self.path, &[], &next_acked).map_err(|error| error.to_string())?; + remove_durable_inbox(&self.inbox_path)?; + self.inbox = None; + self.acked_keys = next_acked; + return Ok(()); + } + if count == 0 || count > self.entries.len() { + return Err("invalid durable archive acknowledgment count".into()); + } + let mut next_entries = self.entries[count..].to_vec(); + let mut next_acked = self.acked_keys.clone(); + next_acked.extend( + self.entries[..count] + .iter() + .map(DurableCandidate::dedupe_key), + ); + if next_acked.len() > MAX_ACKED_DEDUPE_KEYS { + next_acked.drain(..next_acked.len() - MAX_ACKED_DEDUPE_KEYS); + } + persist_queue_file(&self.path, &next_entries, &next_acked) + .map_err(|error| error.to_string())?; + self.entries = std::mem::take(&mut next_entries); + self.acked_keys = next_acked; + Ok(()) + } + + fn persist(&self) -> Result<(), QueuePersistError> { + persist_queue_file(&self.path, &self.entries, &self.acked_keys) + } + + #[cfg(test)] + pub(super) fn fill_to_capacity_for_test(&mut self, candidate: DurableCandidate) { + self.entries = vec![candidate; MAX_DURABLE_QUEUE_ENTRIES]; + } + + #[cfg(test)] + pub(super) fn fill_until_candidate_exceeds_byte_capacity_for_test( + &mut self, + filler: DurableCandidate, + incoming: &DurableCandidate, + ) { + loop { + let mut projected = self.entries.clone(); + projected.push(incoming.clone()); + let projected_len = serde_json::to_vec(&DurableQueueFile { + version: DURABLE_QUEUE_VERSION, + entries: projected, + acked_keys: self.acked_keys.clone(), + }) + .unwrap() + .len(); + if projected_len > MAX_DURABLE_QUEUE_BYTES { + break; + } + self.entries.push(filler.clone()); + assert!(self.entries.len() < MAX_DURABLE_QUEUE_ENTRIES); + } + self.persist().unwrap(); + } +} + +fn durable_inbox_path(queue_path: &Path) -> PathBuf { + queue_path.with_extension("inbox.json") +} + +fn load_durable_inbox(path: &Path) -> Result, String> { + if !path.exists() { + return Ok(None); + } + let bytes = fs::read(path) + .map_err(|error| format!("read durable archive inbox {}: {error}", path.display()))?; + if bytes.len() > MAX_DURABLE_INBOX_BYTES { + return Err(quarantine_corrupt_queue( + path, + format!( + "inbox file is {} bytes; maximum is {MAX_DURABLE_INBOX_BYTES}", + bytes.len() + ), + )); + } + let persisted: DurableInboxFile = serde_json::from_slice(&bytes) + .map_err(|error| quarantine_corrupt_queue(path, format!("invalid inbox JSON: {error}")))?; + if persisted.version != DURABLE_INBOX_VERSION { + return Err(quarantine_corrupt_queue( + path, + format!( + "unsupported inbox version {}; expected {DURABLE_INBOX_VERSION}", + persisted.version + ), + )); + } + persisted + .candidate + .validate() + .map_err(|error| quarantine_corrupt_queue(path, error))?; + Ok(Some(persisted.candidate)) +} + +fn persist_inbox_file(path: &Path, candidate: &DurableCandidate) -> Result<(), QueuePersistError> { + let payload = serde_json::to_vec(&DurableInboxFile { + version: DURABLE_INBOX_VERSION, + candidate: candidate.clone(), + }) + .map_err(|error| { + QueuePersistError::Storage(format!("serialize durable archive inbox: {error}")) + })?; + if payload.len() > MAX_DURABLE_INBOX_BYTES { + return Err(QueuePersistError::Capacity(format!( + "durable archive inbox would be {} bytes; maximum is {MAX_DURABLE_INBOX_BYTES}", + payload.len() + ))); + } + crate::managed_agents::storage::atomic_write_json_restricted(path, &payload) + .map_err(QueuePersistError::Storage) +} + +fn remove_durable_inbox(path: &Path) -> Result<(), String> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!( + "remove promoted durable archive inbox {}: {error}", + path.display() + )), + } +} + +fn persist_queue_file( + path: &Path, + entries: &[DurableCandidate], + acked_keys: &[String], +) -> Result<(), QueuePersistError> { + let payload = serde_json::to_vec(&DurableQueueFile { + version: DURABLE_QUEUE_VERSION, + entries: entries.to_vec(), + acked_keys: acked_keys.to_vec(), + }) + .map_err(|error| { + QueuePersistError::Storage(format!("serialize durable archive queue: {error}")) + })?; + if payload.len() > MAX_DURABLE_QUEUE_BYTES { + return Err(QueuePersistError::Capacity(format!( + "durable archive queue would be {} bytes; maximum is {MAX_DURABLE_QUEUE_BYTES}", + payload.len() + ))); + } + crate::managed_agents::storage::atomic_write_json_restricted(path, &payload) + .map_err(QueuePersistError::Storage) +} + +fn ensure_owner_only_dir(path: &Path) -> Result<(), String> { + fs::create_dir_all(path) + .map_err(|error| format!("create archive sync queue dir {}: {error}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(|error| { + format!( + "set archive sync queue dir {} permissions: {error}", + path.display() + ) + })?; + } + Ok(()) +} + +pub(super) fn blocked_diagnostic_path(path: &Path) -> PathBuf { + path.with_extension("blocked") +} + +fn quarantine_corrupt_queue(path: &Path, reason: String) -> String { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("archive-sync-pending.json"); + let quarantine = path.with_file_name(format!("{file_name}.corrupt-{stamp}")); + let move_result = fs::rename(path, &quarantine); + let diagnostic = match move_result { + Ok(()) => format!( + "corrupt pending state quarantined at {}: {reason}", + quarantine.display() + ), + Err(error) => format!( + "corrupt pending state could not be quarantined from {} ({error}): {reason}", + path.display() + ), + }; + if let Ok(payload) = serde_json::to_vec(&json!({ + "blocked": true, + "diagnostic": diagnostic, + })) { + let _ = crate::managed_agents::storage::atomic_write_json_restricted( + &blocked_diagnostic_path(path), + &payload, + ); + } + diagnostic +} + +pub(super) fn retry_delay(failed_attempt: usize) -> Duration { + let multiplier = 1u32 << failed_attempt.min(16); + ARCHIVE_RETRY_BASE_DELAY + .saturating_mul(multiplier) + .min(ARCHIVE_RETRY_MAX_DELAY) +} + +#[derive(Clone, Copy)] +pub(super) struct AcknowledgedHead { + pub(super) count: usize, + pub(super) persisted_agent_metrics: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum FlushOutcome { + Empty, + Committed, + Retained, + Cancelled, +} + +fn stable_scope_hash(value: &str) -> u64 { + // FNV-1a is intentionally fixed rather than DefaultHasher, whose output is + // not a persistence contract across Rust releases. + value + .as_bytes() + .iter() + .fold(0xcbf29ce484222325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3) + }) +} + +pub(super) fn durable_queue_path( + app: &AppHandle, + owner_pubkey: &str, + relay_url: &str, +) -> Result { + let base = app + .path() + .app_data_dir() + .map_err(|error| format!("resolve app data dir for archive sync queue: {error}"))? + .join("archive") + .join("sync-pending"); + Ok(base.join(format!( + "{owner_pubkey}-{:016x}.json", + stable_scope_hash(relay_url) + ))) +} diff --git a/desktop/src-tauri/src/archive/sync_queue_tests.rs b/desktop/src-tauri/src/archive/sync_queue_tests.rs new file mode 100644 index 00000000000..a125d3d32b4 --- /dev/null +++ b/desktop/src-tauri/src/archive/sync_queue_tests.rs @@ -0,0 +1,589 @@ +//! Focused crash/retry tests for native archive sync's durable queue. + +use super::sync_queue::*; +use super::*; +use nostr::{EventBuilder, JsonUtil, Keys, Kind}; +use std::{ + path::Path, + sync::{Arc, Mutex as StdMutex}, + time::Duration, +}; + +struct QueueIo { + attempts: StdMutex>>, + delivered: StdMutex>>, + failures_remaining: StdMutex, +} + +impl QueueIo { + fn new(failures: usize) -> Self { + Self { + attempts: StdMutex::new(Vec::new()), + delivered: StdMutex::new(Vec::new()), + failures_remaining: StdMutex::new(failures), + } + } + + fn attempt_count(&self) -> usize { + self.attempts.lock().unwrap().len() + } + + fn recover(&self) { + *self.failures_remaining.lock().unwrap() = 0; + } +} + +impl ArchiveSyncIo for QueueIo { + fn list_subscriptions(&self) -> BoxFuture<'_, Result, String>> { + Box::pin(async { + Ok(vec![SaveSubscription { + identity_pubkey: "owner".into(), + relay_url: "wss://relay.test".into(), + scope_type: "channel_h".into(), + scope_value: "channel-a".into(), + kinds: "[9]".into(), + created_at: 0, + }]) + }) + } + + fn set_subscriptions(&self, _subscriptions: Vec) -> BoxFuture<'_, ()> { + Box::pin(async {}) + } + + fn archive( + &self, + candidates: Vec, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let ids = candidates + .iter() + .map(|candidate| { + nostr::Event::from_json(&candidate.raw_event_json) + .unwrap() + .id + .to_hex() + }) + .collect::>(); + self.attempts.lock().unwrap().push(ids.clone()); + let mut failures = self.failures_remaining.lock().unwrap(); + if *failures > 0 { + *failures -= 1; + return Err("scripted archive failure".into()); + } + drop(failures); + self.delivered.lock().unwrap().push(ids); + Ok(ArchiveBatchResult { + persisted: 1, + persisted_agent_metrics: 0, + dropped: 0, + }) + }) + } + + fn notify_agent_metrics_changed(&self) {} +} + +fn signed_candidate(content: &str) -> DurableCandidate { + let event = EventBuilder::new(Kind::Custom(9), content) + .sign_with_keys(&Keys::generate()) + .unwrap(); + DurableCandidate::from_event( + &event, + &MatchedScope { + scope_type: ScopeType::ChannelH, + scope_value: "channel-a".into(), + }, + ) +} + +async fn run_restored_queue(io: Arc, path: &Path) -> Result<(), String> { + let (tx, rx) = mpsc::channel(1); + drop(tx); + // Drop the sender immediately: the loop reconciles, observes shutdown, and + // must drain the restored queue before returning. + run_sync( + io.as_ref(), + Arc::new(Notify::new()), + rx, + CancellationToken::new(), + DurableQueue::open(path.to_path_buf())?, + ) + .await +} + +#[tokio::test] +async fn restart_replays_pending_and_commits_ack_tombstone() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let candidate = signed_candidate("restart"); + { + let mut queue = DurableQueue::open(path.clone()).unwrap(); + assert_eq!( + queue.enqueue(candidate.clone()).unwrap(), + EnqueueResult::Accepted + ); + } + + let io = Arc::new(QueueIo::new(0)); + run_restored_queue(Arc::clone(&io), &path).await.unwrap(); + assert_eq!(io.delivered.lock().unwrap().len(), 1); + + let mut reopened = DurableQueue::open(path).unwrap(); + assert!(reopened.is_empty()); + assert_eq!( + reopened.enqueue(candidate).unwrap(), + EnqueueResult::Duplicate + ); +} + +#[tokio::test] +async fn fail_once_retries_same_head_without_reordering() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let first = signed_candidate("first"); + let second = signed_candidate("second"); + let mut queue = DurableQueue::open(path.clone()).unwrap(); + queue.enqueue(first.clone()).unwrap(); + queue.enqueue(second.clone()).unwrap(); + drop(queue); + + let io = Arc::new(QueueIo::new(1)); + run_restored_queue(Arc::clone(&io), &path).await.unwrap(); + assert_eq!(io.attempt_count(), 2); + let attempts = io.attempts.lock().unwrap(); + assert_eq!(attempts[0], attempts[1]); + assert_eq!(attempts[1], vec![first.event_id, second.event_id]); + assert!(DurableQueue::open(path).unwrap().is_empty()); +} + +#[tokio::test] +async fn repeated_failure_retains_head_for_restart() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let candidate = signed_candidate("retain"); + let mut queue = DurableQueue::open(path.clone()).unwrap(); + queue.enqueue(candidate).unwrap(); + drop(queue); + + let io = Arc::new(QueueIo::new(usize::MAX)); + run_restored_queue(Arc::clone(&io), &path).await.unwrap(); + assert_eq!(io.attempt_count(), ARCHIVE_RETRY_ATTEMPTS); + assert_eq!(DurableQueue::open(path).unwrap().len(), 1); +} + +#[test] +fn duplicate_pending_event_and_scope_is_written_once() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let candidate = signed_candidate("dedupe"); + let mut queue = DurableQueue::open(path.clone()).unwrap(); + assert_eq!( + queue.enqueue(candidate.clone()).unwrap(), + EnqueueResult::Accepted + ); + assert_eq!(queue.enqueue(candidate).unwrap(), EnqueueResult::Duplicate); + assert_eq!(DurableQueue::open(path).unwrap().len(), 1); +} + +#[test] +fn corrupt_pending_state_is_quarantined_and_stays_fail_closed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + std::fs::write(&path, b"{not-json").unwrap(); + + let error = DurableQueue::open(path.clone()).unwrap_err(); + assert!(error.contains("quarantined")); + assert!(blocked_diagnostic_path(&path).exists()); + assert!(DurableQueue::open(path.clone()) + .unwrap_err() + .contains("fail-closed")); + assert!(std::fs::read_dir(dir.path()).unwrap().any(|entry| { + entry + .unwrap() + .file_name() + .to_string_lossy() + .contains(".corrupt-") + })); +} + +#[test] +fn queue_bound_rejects_new_event_without_mutating_pending_head() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let candidate = signed_candidate("capacity"); + let mut queue = DurableQueue::open(path).unwrap(); + queue.fill_to_capacity_for_test(candidate); + let error = queue + .enqueue(signed_candidate("one-too-many")) + .unwrap_err() + .to_string(); + assert!(error.contains("limit reached")); + assert_eq!(queue.len(), MAX_DURABLE_QUEUE_ENTRIES); +} + +#[tokio::test] +async fn byte_capacity_inbox_survives_failed_teardown_and_restart() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let filler = signed_candidate(&"f".repeat(512 * 1024)); + let incoming = signed_candidate(&"i".repeat(512 * 1024)); + let incoming_id = incoming.event_id.clone(); + let mut queue = DurableQueue::open(path.clone()).unwrap(); + queue.fill_until_candidate_exceeds_byte_capacity_for_test(filler, &incoming); + + assert!(queue.len() < MAX_DURABLE_QUEUE_ENTRIES); + let retained = match queue.enqueue(incoming).unwrap_err() { + EnqueueError::Capacity { candidate, .. } => candidate, + other => panic!("expected byte-capacity backpressure, got {other}"), + }; + assert_eq!(retained.event_id, incoming_id); + assert_eq!( + queue.stash_inbox(retained).unwrap(), + EnqueueResult::Accepted + ); + drop(queue); + + let queue = DurableQueue::open(path.clone()).unwrap(); + assert!(queue.has_inbox()); + let failing_io = Arc::new(QueueIo::new(usize::MAX)); + let (_tx, rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + cancel.cancel(); + run_sync( + failing_io.as_ref(), + Arc::new(Notify::new()), + rx, + cancel, + queue, + ) + .await + .unwrap(); + assert_eq!(failing_io.attempt_count(), ARCHIVE_RETRY_ATTEMPTS); + assert!(DurableQueue::open(path.clone()).unwrap().has_inbox()); + + let recovered_io = Arc::new(QueueIo::new(0)); + run_restored_queue(Arc::clone(&recovered_io), &path) + .await + .unwrap(); + assert!( + recovered_io + .delivered + .lock() + .unwrap() + .iter() + .flatten() + .any(|id| id == &incoming_id), + "durable byte-cap inbox was not replayed after restart" + ); + assert!(DurableQueue::open(path).unwrap().is_empty()); +} + +#[tokio::test] +async fn inbox_write_failure_retries_without_consuming_another_event() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let scope = MatchedScope { + scope_type: ScopeType::ChannelH, + scope_value: "channel-a".into(), + }; + let incoming_event = EventBuilder::new(Kind::Custom(9), "i".repeat(512 * 1024)) + .sign_with_keys(&Keys::generate()) + .unwrap(); + let incoming_id = incoming_event.id.to_hex(); + let incoming_candidate = DurableCandidate::from_event(&incoming_event, &scope); + let filler = signed_candidate(&"f".repeat(512 * 1024)); + let mut queue = DurableQueue::open(path.clone()).unwrap(); + queue.fill_until_candidate_exceeds_byte_capacity_for_test(filler, &incoming_candidate); + + let inbox_path = path.with_extension("inbox.json"); + std::fs::create_dir(&inbox_path).unwrap(); + + let second_event = EventBuilder::new(Kind::Custom(9), "second") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let second_id = second_event.id.to_hex(); + let subscription_id = subscription_id(&ScopeType::ChannelH, "channel-a", &[9]); + let (tx, rx) = mpsc::channel(2); + tx.send(MatchedEvent { + subscription_id: subscription_id.clone(), + event: Box::new(incoming_event), + }) + .await + .unwrap(); + tx.send(MatchedEvent { + subscription_id, + event: Box::new(second_event), + }) + .await + .unwrap(); + + let io = Arc::new(QueueIo::new(usize::MAX)); + let task_io = Arc::clone(&io); + let handle = tokio::spawn(async move { + run_sync( + task_io.as_ref(), + Arc::new(Notify::new()), + rx, + CancellationToken::new(), + queue, + ) + .await + }); + + for _ in 0..10_000 { + if tx.capacity() == 1 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + tx.capacity(), + 1, + "the first event was not consumed into the live-retained retry slot" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + assert_eq!( + tx.capacity(), + 1, + "archive sync consumed a second event before the first became durable" + ); + assert!(!handle.is_finished(), "inbox write failure stopped sync"); + + std::fs::remove_dir(&inbox_path).unwrap(); + io.recover(); + drop(tx); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("inbox recovery timed out") + .expect("sync task panicked") + .expect("sync stopped instead of recovering inbox persistence"); + + let delivered = io.delivered.lock().unwrap(); + assert!(delivered.iter().flatten().any(|id| id == &incoming_id)); + assert!(delivered.iter().flatten().any(|id| id == &second_id)); + drop(delivered); + assert!(DurableQueue::open(path).unwrap().is_empty()); +} + +#[tokio::test] +async fn full_queue_backpressures_and_resumes_without_stopping_listener() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let mut queue = DurableQueue::open(path).unwrap(); + queue.fill_to_capacity_for_test(signed_candidate("capacity-head")); + + let event = EventBuilder::new(Kind::Custom(9), "arrived-while-full") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let event_id = event.id.to_hex(); + let (tx, rx) = mpsc::channel(1); + tx.send(MatchedEvent { + subscription_id: subscription_id(&ScopeType::ChannelH, "channel-a", &[9]), + event: Box::new(event), + }) + .await + .unwrap(); + drop(tx); + + let io = Arc::new(QueueIo::new(ARCHIVE_RETRY_ATTEMPTS)); + let task_io = Arc::clone(&io); + let handle = tokio::spawn(async move { + run_sync( + task_io.as_ref(), + Arc::new(Notify::new()), + rx, + CancellationToken::new(), + queue, + ) + .await + }); + + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("full-queue recovery timed out") + .expect("sync task panicked") + .expect("sync listener stopped at capacity"); + assert!( + io.delivered + .lock() + .unwrap() + .iter() + .flatten() + .any(|id| id == &event_id), + "event waiting behind the full queue was not archived" + ); +} + +#[tokio::test] +async fn teardown_is_bounded_and_leaves_failed_head_durable() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pending.json"); + let mut queue = DurableQueue::open(path.clone()).unwrap(); + queue.enqueue(signed_candidate("teardown")).unwrap(); + + let io = Arc::new(QueueIo::new(usize::MAX)); + let (_tx, rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + cancel.cancel(); + tokio::time::timeout( + SHUTDOWN_WAIT_TIMEOUT, + run_sync(io.as_ref(), Arc::new(Notify::new()), rx, cancel, queue), + ) + .await + .expect("teardown exceeded its bound") + .unwrap(); + + assert_eq!(io.attempt_count(), ARCHIVE_RETRY_ATTEMPTS); + assert_eq!(DurableQueue::open(path).unwrap().len(), 1); +} + +#[tokio::test] +async fn lifecycle_stop_waits_until_the_sync_task_finishes() { + let state = Arc::new(ArchiveSyncState::default()); + let completion = Arc::new(SyncCompletion::default()); + let ownership = state + .begin( + (1, 1), + ("owner".into(), "wss://relay.test".into()), + CancellationToken::new(), + Arc::new(Notify::new()), + Arc::clone(&completion), + ) + .await + .expect("lifecycle start is available") + .expect("start owns sync"); + drop(ownership); + + let stopper = tokio::spawn({ + let state = Arc::clone(&state); + async move { state.end((1, 1)).await } + }); + tokio::task::yield_now().await; + assert!( + !stopper.is_finished(), + "stop returned before sync completion" + ); + + completion.finish(); + stopper.await.unwrap().unwrap(); + assert!(state.running.lock().await.is_none()); +} + +#[tokio::test(start_paused = true)] +async fn lifecycle_stop_timeout_is_explicit_and_keeps_task_visible() { + let state = ArchiveSyncState::default(); + let completion = Arc::new(SyncCompletion::default()); + let ownership = state + .begin( + (1, 1), + ("owner".into(), "wss://relay.test".into()), + CancellationToken::new(), + Arc::new(Notify::new()), + Arc::clone(&completion), + ) + .await + .expect("lifecycle start is available") + .expect("start owns sync"); + drop(ownership); + + let error = state.end((1, 1)).await.unwrap_err(); + assert!(error.contains("teardown did not finish")); + assert!(state.running.lock().await.is_some()); + + completion.finish(); + state.clear_completed(&completion).await; + assert!(state.running.lock().await.is_none()); + + let restarted = Arc::new(SyncCompletion::default()); + restarted.finish(); + assert!(state + .begin( + (1, 2), + ("owner".into(), "wss://relay.test".into()), + CancellationToken::new(), + Arc::new(Notify::new()), + restarted, + ) + .await + .expect("lifecycle restart is available") + .is_some()); +} + +#[tokio::test(start_paused = true)] +async fn same_scope_start_during_timed_out_stop_is_explicitly_blocked() { + let state = ArchiveSyncState::default(); + let completion = Arc::new(SyncCompletion::default()); + let ownership = state + .begin( + (1, 1), + ("owner".into(), "wss://relay.test".into()), + CancellationToken::new(), + Arc::new(Notify::new()), + Arc::clone(&completion), + ) + .await + .unwrap() + .unwrap(); + drop(ownership); + + assert!(state.end((1, 1)).await.unwrap_err().contains("teardown")); + let blocked = state + .begin( + (1, 2), + ("owner".into(), "wss://relay.test".into()), + CancellationToken::new(), + Arc::new(Notify::new()), + Arc::new(SyncCompletion::default()), + ) + .await; + let Err(error) = blocked else { + panic!("same-scope start must be blocked while teardown is unfinished"); + }; + assert!(error.contains("still stopping")); + + completion.finish(); + state.clear_completed(&completion).await; + assert!(state + .begin( + (1, 3), + ("owner".into(), "wss://relay.test".into()), + CancellationToken::new(), + Arc::new(Notify::new()), + Arc::new(SyncCompletion::default()), + ) + .await + .unwrap() + .is_some()); +} + +#[tokio::test] +async fn healthy_same_scope_start_remains_an_idempotent_noop() { + let state = ArchiveSyncState::default(); + let first = CancellationToken::new(); + let ownership = state + .begin( + (1, 1), + ("owner".into(), "wss://relay.test".into()), + first.clone(), + Arc::new(Notify::new()), + Arc::new(SyncCompletion::default()), + ) + .await + .unwrap() + .unwrap(); + drop(ownership); + + let second = state + .begin( + (1, 2), + ("owner".into(), "wss://relay.test".into()), + CancellationToken::new(), + Arc::new(Notify::new()), + Arc::new(SyncCompletion::default()), + ) + .await + .unwrap(); + assert!(second.is_none()); + assert!(!first.is_cancelled()); +} diff --git a/desktop/src-tauri/src/archive/sync_tests.rs b/desktop/src-tauri/src/archive/sync_tests.rs index 3a39b5d5856..f3e0d9a05e2 100644 --- a/desktop/src-tauri/src/archive/sync_tests.rs +++ b/desktop/src-tauri/src/archive/sync_tests.rs @@ -7,12 +7,12 @@ use super::*; use nostr::{EventBuilder, Keys, Kind, Tag}; -use std::sync::Mutex as StdMutex; +use std::{path::PathBuf, sync::Mutex as StdMutex}; // ── Test doubles ───────────────────────────────────────────────────────────── -#[derive(Default)] struct FakeIo { + queue_dir: tempfile::TempDir, /// Successive results for `list_subscriptions`; the last one repeats so a /// reload that outruns the script does not panic. listings: StdMutex>>, @@ -21,9 +21,25 @@ struct FakeIo { /// What `archive` returns; drives the notify-on-metrics assertion. persisted_agent_metrics: StdMutex, archive_fails: StdMutex, + archive_failures_remaining: StdMutex, metrics_notifications: StdMutex, } +impl Default for FakeIo { + fn default() -> Self { + Self { + queue_dir: tempfile::tempdir().expect("queue tempdir"), + listings: StdMutex::new(Vec::new()), + applied: StdMutex::new(Vec::new()), + batches: StdMutex::new(Vec::new()), + persisted_agent_metrics: StdMutex::new(0), + archive_fails: StdMutex::new(false), + archive_failures_remaining: StdMutex::new(0), + metrics_notifications: StdMutex::new(0), + } + } +} + impl FakeIo { fn with_listings(listings: Vec>) -> Self { Self { @@ -50,6 +66,10 @@ impl FakeIo { }) .collect() } + + fn queue_path(&self) -> PathBuf { + self.queue_dir.path().join("pending.json") + } } impl ArchiveSyncIo for FakeIo { @@ -81,6 +101,12 @@ impl ArchiveSyncIo for FakeIo { if *self.archive_fails.lock().unwrap() { return Err("archive failed".to_string()); } + let mut failures = self.archive_failures_remaining.lock().unwrap(); + if *failures > 0 { + *failures -= 1; + return Err("archive failed once".to_string()); + } + drop(failures); Ok(ArchiveBatchResult { persisted: 0, persisted_agent_metrics: *self.persisted_agent_metrics.lock().unwrap(), @@ -133,29 +159,33 @@ fn matched(subscription_id: &str) -> MatchedEvent { /// Runs `run_sync` on a task, handing back the controls the tests drive it /// with. Every test cancels and joins, so a loop that fails to observe /// cancellation hangs the test rather than passing silently. -fn spawn_sync( - io: Arc, -) -> ( +type SpawnedSync = ( mpsc::Sender, Arc, CancellationToken, - tokio::task::JoinHandle<()>, -) { + tokio::task::JoinHandle>, +); + +fn spawn_sync(io: Arc) -> SpawnedSync { let (tx, rx) = mpsc::channel(64); let reload = Arc::new(Notify::new()); let cancel = CancellationToken::new(); + let queue = DurableQueue::open(io.queue_path()).expect("open test durable queue"); let handle = { let io = Arc::clone(&io); let reload = Arc::clone(&reload); let cancel = cancel.clone(); - tokio::spawn(async move { run_sync(io.as_ref(), reload, rx, cancel).await }) + tokio::spawn(async move { run_sync(io.as_ref(), reload, rx, cancel, queue).await }) }; (tx, reload, cancel, handle) } -async fn stop(cancel: CancellationToken, handle: tokio::task::JoinHandle<()>) { +async fn stop(cancel: CancellationToken, handle: tokio::task::JoinHandle>) { cancel.cancel(); - handle.await.expect("sync task panicked"); + handle + .await + .expect("sync task panicked") + .expect("sync task failed"); } // ── Filter construction ────────────────────────────────────────────────────── @@ -247,7 +277,6 @@ async fn subscribes_to_saved_configs_on_start() { )]])); let (_tx, _reload, cancel, handle) = spawn_sync(Arc::clone(&io)); - // The first reconcile races the cancel; wait for it to land. wait_for("initial subscribe", || !io.applied().is_empty()).await; assert_eq!(io.applied()[0].len(), 1); stop(cancel, handle).await; @@ -366,12 +395,11 @@ async fn events_for_an_unknown_subscription_are_dropped() { let (tx, _reload, cancel, handle) = spawn_sync(Arc::clone(&io)); wait_for("initial subscribe", || !io.applied().is_empty()).await; - // An event from a subscription we already closed: no scope, no archive. tx.send(matched("archive:channel_h:gone:[9]")) .await .unwrap(); cancel.cancel(); - handle.await.unwrap(); + handle.await.unwrap().unwrap(); assert!( io.archived().is_empty(), @@ -398,7 +426,7 @@ async fn buffered_events_flush_on_shutdown() { }) .await; cancel.cancel(); - handle.await.unwrap(); + handle.await.unwrap().unwrap(); let archived = io.archived(); assert_eq!(archived.len(), 1, "shutdown did not flush the buffer"); @@ -429,8 +457,6 @@ async fn does_not_notify_when_nothing_was_persisted() { let io = Arc::new(FakeIo::with_listings(vec![vec![saved( "owner_p", "owner-pk", "[44200]", )]])); - // persisted_agent_metrics stays 0: a duplicate-only batch must not - // invalidate the renderer's usage queries. let (tx, _reload, cancel, handle) = spawn_sync(Arc::clone(&io)); wait_for("initial subscribe", || !io.applied().is_empty()).await; let id = io.applied()[0][0].id.clone(); @@ -498,11 +524,12 @@ async fn a_failed_listing_leaves_the_previous_subscriptions_live() { let (tx, rx) = mpsc::channel(8); let reload = Arc::new(Notify::new()); let cancel = CancellationToken::new(); + let queue = DurableQueue::open(inner.queue_path()).expect("open test durable queue"); let handle = { let io = Arc::clone(&io); let reload = Arc::clone(&reload); let cancel = cancel.clone(); - tokio::spawn(async move { run_sync(io.as_ref(), reload, rx, cancel).await }) + tokio::spawn(async move { run_sync(io.as_ref(), reload, rx, cancel, queue).await }) }; wait_for("initial subscribe", || !inner.applied().is_empty()).await; @@ -521,47 +548,33 @@ async fn a_failed_listing_leaves_the_previous_subscriptions_live() { } // ── Lifecycle ownership ────────────────────────────────────────────────────── -// -// The renderer fires `start_archive_sync` and `stop_archive_sync` without -// awaiting them, and Tauri commands may complete in any order. These tests -// drive `ArchiveSyncState` through the same `claim_start`/`claim_stop`/ -// `install`/`stop` seam the commands use, applying the halves in a chosen -// order — which is the one thing the mounted-hook test cannot do, because its -// mock `invoke` resolves immediately and can only observe call order. - -/// Runs the ownership half of `start_archive_sync` under `lease`, returning the -/// installed task's cancel token. `None` means the start did not take -/// ownership. Mirrors the command minus the relay session and spawned loop, -/// which the ordering invariant does not involve. -/// -/// The ownership token is dropped before returning, so these tests apply the -/// halves sequentially as before. The test that needs it held across an -/// acquisition calls [`ArchiveSyncState::begin`] directly. -/// -/// The token is the task's identity: two starts produce distinct tokens, so a -/// test can name WHICH task survived an interleaving rather than only that one -/// did. "Something is running" is satisfiable by the stale start's task. + +/// Test half of `start_archive_sync`, without relay acquisition. async fn start_half( state: &ArchiveSyncState, mark: (u64, u64), scope: (&str, &str), ) -> Option { let cancel = CancellationToken::new(); + let completion = Arc::new(SyncCompletion::default()); + completion.finish(); state .begin( mark, (scope.0.to_string(), scope.1.to_string()), cancel.clone(), Arc::new(Notify::new()), + completion, ) .await + .expect("lifecycle start is available") .is_some() .then_some(cancel) } /// Runs `stop_archive_sync` under `mark`. async fn stop_half(state: &ArchiveSyncState, mark: (u64, u64)) { - state.end(mark).await; + state.end(mark).await.unwrap(); } async fn is_running(state: &ArchiveSyncState) -> bool { @@ -666,29 +679,6 @@ async fn ordered_start_and_stop_still_take_effect() { assert!(!is_running(&state).await, "the newer stop must take effect"); } -/// A same-scope remount that reaches the backend in order is still a no-op at -/// the socket, so the lease does not undo the idempotence the port relies on. -#[tokio::test] -async fn a_same_scope_restart_does_not_churn_the_running_task() { - let state = ArchiveSyncState::default(); - - let first = start_half(&state, (1, 1), SCOPE) - .await - .expect("first start"); - assert!( - start_half(&state, (1, 2), SCOPE).await.is_none(), - "a newer start for the same scope must not reinstall" - ); - assert!( - !first.is_cancelled(), - "the original task must not be torn down" - ); - assert!( - running_is(&state, &first).await, - "the original task must still be the installed one" - ); -} - /// An identity or relay change must replace the task rather than leaving the /// old scope's socket live. #[tokio::test] @@ -751,7 +741,7 @@ async fn a_stop_holds_its_lease_guard_across_the_cancellation() { let lease_held = state.latest.try_lock().is_err(); drop(running_guard); - stopper.await.expect("stop task"); + stopper.await.expect("stop task").expect("stop succeeds"); assert!( lease_held, @@ -885,19 +875,6 @@ async fn announced_epochs_strictly_increase() { } // ── Session acquisition ────────────────────────────────────────────────────── -// -// Ordering alone is not enough once a start has to acquire the shared relay -// session: `ensure_session` shuts down a different scope's socket and -// `attach_archive` replaces the archive sender, both destructively on entry. -// A start that checked its mark, yielded, and acquired afterwards would already -// have torn down the newer owner's session by the time it discovered it lost. -// -// Two things close that, and only one of them is testable here. That a -// superseded start cannot call `archive_session` at all is the token's job and -// is enforced by the compiler, not by a test — `ArchiveOwnership` is -// un-constructible outside this module, so the bypass does not compile. What -// this test pins is the property the token's usefulness rests on: while a -// winner holds it, no other start can claim. /// While a start holds its ownership token, a newer start cannot claim — so the /// window in which the shared session is acquired is exclusive. @@ -917,14 +894,18 @@ async fn a_newer_start_cannot_claim_while_the_owner_holds_its_token() { let state = Arc::new(ArchiveSyncState::default()); let first = CancellationToken::new(); + let first_completion = Arc::new(SyncCompletion::default()); + first_completion.finish(); let ownership = state .begin( (1, 1), (SCOPE.0.to_string(), SCOPE.1.to_string()), first.clone(), Arc::new(Notify::new()), + first_completion, ) .await + .expect("first lifecycle start is available") .expect("the first start claims ownership"); let second = CancellationToken::new(); @@ -934,14 +915,18 @@ async fn a_newer_start_cannot_claim_while_the_owner_holds_its_token() { let claimed = Arc::clone(&claimed); let second = second.clone(); async move { + let completion = Arc::new(SyncCompletion::default()); + completion.finish(); let won = state .begin( (1, 2), ("other-pubkey".to_string(), SCOPE.1.to_string()), second, Arc::new(Notify::new()), + completion, ) .await + .expect("competing lifecycle start is available") .is_some(); claimed.store(won, std::sync::atomic::Ordering::SeqCst); } diff --git a/desktop/src-tauri/src/archive/today_snapshot.rs b/desktop/src-tauri/src/archive/today_snapshot.rs new file mode 100644 index 00000000000..d522a5bd33a --- /dev/null +++ b/desktop/src-tauri/src/archive/today_snapshot.rs @@ -0,0 +1,982 @@ +use nostr::{Event, EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + io::Write, + path::{Path, PathBuf}, + sync::OnceLock, +}; + +#[path = "today_snapshot_authorization.rs"] +mod today_snapshot_authorization; +use today_snapshot_authorization::redact_embedded_authorization_headers; + +pub const TODAY_SNAPSHOT_SCHEMA: &str = "buzz.activity-ledger.today/v1"; +pub const TODAY_SNAPSHOT_CAPABILITY: &str = "buzz.activity-ledger.today.read/v1"; +pub const TODAY_SNAPSHOT_SIGNED_KIND: u16 = 24202; +const TODAY_SNAPSHOT_TAG_MARKER: &str = "buzz-activity-ledger-today"; +const MAX_SNAPSHOT_BYTES: usize = 8 * 1024 * 1024; +const MAX_RAW_EVENTS: usize = 10_000; +const MAX_LIFETIME_SECS: i64 = 24 * 60 * 60; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UnsignedOwnerTodaySnapshot { + pub schema: String, + pub owner_pubkey: String, + pub relay_url: String, + pub generated_at: i64, + pub expires_at: i64, + pub capability: String, + pub surface: serde_json::Value, + pub raw_events: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OwnerTodaySnapshot { + #[serde(flatten)] + pub payload: UnsignedOwnerTodaySnapshot, + pub snapshot_sha256: String, + pub event_id: String, + pub signature: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TodaySnapshotReceipt { + pub path: String, + pub owner_pubkey: String, + pub relay_url: String, + pub generated_at: i64, + pub expires_at: i64, + pub byte_length: usize, + pub sha256: String, +} + +pub(crate) fn snapshot_path(nest_dir: &Path, owner_pubkey: &str, relay_url: &str) -> PathBuf { + let relay_scope = hex::encode(Sha256::digest(relay_url.as_bytes())); + nest_dir.join("archive").join(format!( + "activity-ledger-today-{owner_pubkey}-{relay_scope}.json" + )) +} + +fn validate_owner_pubkey(owner_pubkey: &str) -> Result<(), String> { + if owner_pubkey.len() != 64 + || !owner_pubkey + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("Today snapshot ownerPubkey must be lowercase 64-character hex".into()); + } + Ok(()) +} + +fn normalize_relay_url(relay_url: &str) -> Result { + if relay_url.is_empty() || relay_url.len() > 2_048 { + return Err("Today snapshot relayUrl must contain between 1 and 2048 bytes".into()); + } + buzz_core_pkg::relay::normalize_relay_url(relay_url) + .map_err(|error| format!("Today snapshot relayUrl is invalid: {error}")) +} + +fn validate_hex(value: &str, label: &str, expected_len: usize) -> Result<(), String> { + if value.len() != expected_len || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(format!( + "Today snapshot {label} must be {expected_len}-character lowercase hex" + )); + } + if value.bytes().any(|byte| byte.is_ascii_uppercase()) { + return Err(format!( + "Today snapshot {label} must be {expected_len}-character lowercase hex" + )); + } + Ok(()) +} + +const REDACTED_IDENTITY_SECRET_TEXT: &str = "[REDACTED: identity secret material]"; +#[rustfmt::skip] +const EMBEDDED_SECRET_WORDS: &[&str] = &["secret", "password", "token", "key", "credential", "passphrase", "passwd", "auth", "authorization", "nsec", "apikey", "privatekey", "accesskey"]; + +fn split_secret_like_key(key: &str) -> Vec { + let mut words = Vec::new(); + let mut current = String::new(); + let chars: Vec = key.chars().collect(); + for (index, &ch) in chars.iter().enumerate() { + if matches!(ch, '_' | '-' | '.') { + if !current.is_empty() { + words.push(current.to_lowercase()); + current.clear(); + } + continue; + } + if ch.is_uppercase() { + let prev_lower = + !current.is_empty() && current.chars().last().is_some_and(|c| c.is_lowercase()); + let acronym_end = !current.is_empty() + && current.chars().last().is_some_and(|c| c.is_uppercase()) + && chars.get(index + 1).is_some_and(|c| c.is_lowercase()); + if prev_lower || acronym_end { + words.push(current.to_lowercase()); + current.clear(); + } + } + current.push(ch); + } + if !current.is_empty() { + words.push(current.to_lowercase()); + } + words +} + +fn looks_like_embedded_secret_key(key: &str) -> bool { + let split_words = split_secret_like_key(key); + if split_words + .iter() + .any(|word| EMBEDDED_SECRET_WORDS.contains(&word.as_str())) + { + return true; + } + + key.split(['_', '-', '.']) + .filter(|chunk| !chunk.is_empty()) + .map(|chunk| chunk.to_ascii_lowercase()) + .any(|chunk| EMBEDDED_SECRET_WORDS.contains(&chunk.as_str())) +} + +fn embedded_secret_assignment_regex() -> Result<&'static Regex, String> { + static REGEX: OnceLock> = OnceLock::new(); + match REGEX.get_or_init(|| { + Regex::new( + r#"(?i)(^|[^A-Za-z0-9_.-])([A-Za-z0-9_.-]+)(\s*=\s*)("[^"\r\n]*"|'[^'\r\n]*'|\[[^\]\r\n]*\]|[^\s,"';)\]}]+)"#, + ) + .map_err(|error| format!("embedded secret assignment regex is invalid: {error}")) + }) { + Ok(regex) => Ok(regex), + Err(error) => Err(error.clone()), + } +} + +fn embedded_nsec_regex() -> Result<&'static Regex, String> { + static REGEX: OnceLock> = OnceLock::new(); + match REGEX.get_or_init(|| { + Regex::new(r#"(?i)(^|[^A-Za-z0-9])(nsec1[^\s"'`,;)\]}]*)"#) + .map_err(|error| format!("embedded nsec regex is invalid: {error}")) + }) { + Ok(regex) => Ok(regex), + Err(error) => Err(error.clone()), + } +} + +fn embedded_secret_colon_regex() -> Result<&'static Regex, String> { + static REGEX: OnceLock> = OnceLock::new(); + match REGEX.get_or_init(|| { + Regex::new( + r#"(?i)(^|[,{]\s*)(["']?[A-Za-z0-9_.-]+["']?)(\s*:\s*)("(?:\\[^\r\n]|[^"\\\r\n])*"|'(?:\\[^\r\n]|[^'\\\r\n])*'|\[[^\]\r\n]*\]|[^\s,"';)\]}]+)"#, + ) + .map_err(|error| format!("embedded secret colon regex is invalid: {error}")) + }) { + Ok(regex) => Ok(regex), + Err(error) => Err(error.clone()), + } +} + +fn embedded_secret_flag_regex() -> Result<&'static Regex, String> { + static REGEX: OnceLock> = OnceLock::new(); + match REGEX.get_or_init(|| { + Regex::new( + r#"(?i)(^|\s)(--[A-Za-z0-9][A-Za-z0-9_.-]*)(\s+)("[^"\r\n]*"|'[^'\r\n]*'|\[[^\]\r\n]*\]|[^\s,"';)\]}]+)"#, + ) + .map_err(|error| format!("embedded secret flag regex is invalid: {error}")) + }) { + Ok(regex) => Ok(regex), + Err(error) => Err(error.clone()), + } +} + +fn is_existing_redaction_marker(value: &str) -> bool { + value == REDACTED_IDENTITY_SECRET_TEXT + || value == format!("\"{REDACTED_IDENTITY_SECRET_TEXT}\"") + || value == format!("'{REDACTED_IDENTITY_SECRET_TEXT}'") +} + +fn matched_text(captures: ®ex::Captures) -> String { + captures + .get(0) + .map_or_else(String::new, |matched| matched.as_str().to_string()) +} + +fn redacted_secret_value(value: &str) -> String { + if value.starts_with('"') && value.ends_with('"') { + format!("\"{REDACTED_IDENTITY_SECRET_TEXT}\"") + } else if value.starts_with('\'') && value.ends_with('\'') { + format!("'{REDACTED_IDENTITY_SECRET_TEXT}'") + } else { + REDACTED_IDENTITY_SECRET_TEXT.to_string() + } +} + +fn redact_embedded_secret_assignments(text: &str) -> Result<(String, usize), String> { + let mut redactions = 0; + let redacted = + embedded_secret_assignment_regex()?.replace_all(text, |captures: ®ex::Captures| { + let prefix = captures.get(1).map_or("", |value| value.as_str()); + let key = captures.get(2).map_or("", |value| value.as_str()); + let separator = captures.get(3).map_or("", |value| value.as_str()); + let value = captures.get(4).map_or("", |value| value.as_str()); + if !looks_like_embedded_secret_key(key) || is_existing_redaction_marker(value) { + return matched_text(captures); + } + redactions += 1; + format!("{prefix}{key}{separator}{}", redacted_secret_value(value)) + }); + Ok((redacted.into_owned(), redactions)) +} + +fn redact_embedded_secret_colons(text: &str) -> Result<(String, usize), String> { + let mut redactions = 0; + let redacted = + embedded_secret_colon_regex()?.replace_all(text, |captures: ®ex::Captures| { + let prefix = captures.get(1).map_or("", |value| value.as_str()); + let key_token = captures.get(2).map_or("", |value| value.as_str()); + let key = key_token.trim_matches(['"', '\'']); + let separator = captures.get(3).map_or("", |value| value.as_str()); + let value = captures.get(4).map_or("", |value| value.as_str()); + if !looks_like_embedded_secret_key(key) || is_existing_redaction_marker(value) { + return matched_text(captures); + } + redactions += 1; + format!( + "{prefix}{key_token}{separator}{}", + redacted_secret_value(value) + ) + }); + Ok((redacted.into_owned(), redactions)) +} + +fn redact_embedded_secret_flags(text: &str) -> Result<(String, usize), String> { + let mut redactions = 0; + let redacted = embedded_secret_flag_regex()?.replace_all(text, |captures: ®ex::Captures| { + let prefix = captures.get(1).map_or("", |value| value.as_str()); + let key = captures.get(2).map_or("", |value| value.as_str()); + let separator = captures.get(3).map_or("", |value| value.as_str()); + let value = captures.get(4).map_or("", |value| value.as_str()); + if !looks_like_embedded_secret_key(key) || is_existing_redaction_marker(value) { + return matched_text(captures); + } + redactions += 1; + format!("{prefix}{key}{separator}{}", redacted_secret_value(value)) + }); + Ok((redacted.into_owned(), redactions)) +} + +fn redact_embedded_nsec_tokens(text: &str) -> Result<(String, usize), String> { + let mut redactions = 0; + let redacted = embedded_nsec_regex()?.replace_all(text, |captures: ®ex::Captures| { + let prefix = captures.get(1).map_or("", |value| value.as_str()); + let secret = captures.get(2).map_or("", |value| value.as_str()); + if secret == REDACTED_IDENTITY_SECRET_TEXT { + return matched_text(captures); + } + redactions += 1; + format!("{prefix}{REDACTED_IDENTITY_SECRET_TEXT}") + }); + Ok((redacted.into_owned(), redactions)) +} + +fn redact_identity_secret_text(value: &mut serde_json::Value) -> Result { + match value { + serde_json::Value::Object(object) => { + let mut redactions = 0; + for (key, child) in object { + let normalized_key = key + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .flat_map(char::to_lowercase) + .collect::(); + if matches!( + normalized_key.as_str(), + "nsec" | "secretkey" | "privatekey" | "nostrsecretkey" + ) { + return Err("Today snapshot cannot contain identity secret fields".into()); + } + if matches!( + normalized_key.as_str(), + "authorization" | "proxyauthorization" + ) { + if child.as_str() == Some(REDACTED_IDENTITY_SECRET_TEXT) { + continue; + } + *child = serde_json::Value::String(REDACTED_IDENTITY_SECRET_TEXT.into()); + redactions += 1; + continue; + } + redactions += redact_identity_secret_text(child)?; + } + Ok(redactions) + } + serde_json::Value::Array(values) => { + let mut redactions = 0; + for child in values { + redactions += redact_identity_secret_text(child)?; + } + Ok(redactions) + } + serde_json::Value::String(text) => { + let (with_redacted_assignments, assignment_redactions) = + redact_embedded_secret_assignments(text)?; + let (with_redacted_colons, colon_redactions) = + redact_embedded_secret_colons(&with_redacted_assignments)?; + let (with_redacted_authorization_headers, authorization_header_redactions) = + redact_embedded_authorization_headers( + &with_redacted_colons, + REDACTED_IDENTITY_SECRET_TEXT, + )?; + let (with_redacted_flags, flag_redactions) = + redact_embedded_secret_flags(&with_redacted_authorization_headers)?; + let (redacted_text, nsec_redactions) = + redact_embedded_nsec_tokens(&with_redacted_flags)?; + if assignment_redactions + + colon_redactions + + authorization_header_redactions + + flag_redactions + + nsec_redactions + > 0 + { + *text = redacted_text; + } + Ok(assignment_redactions + + colon_redactions + + authorization_header_redactions + + flag_redactions + + nsec_redactions) + } + _ => Ok(0), + } +} + +fn clear_identity_secret_redaction_marker(surface: &mut serde_json::Value) -> Result<(), String> { + let surface = surface + .as_object_mut() + .ok_or_else(|| "Today snapshot surface must be a JSON object".to_string())?; + if let Some(projection) = surface.get_mut("snapshotProjection") { + let projection = projection.as_object_mut().ok_or_else(|| { + "Today snapshot surface.snapshotProjection must be a JSON object".to_string() + })?; + projection.remove("identitySecretsRedacted"); + } + Ok(()) +} + +fn record_identity_secret_redactions( + surface: &mut serde_json::Value, + redactions: usize, +) -> Result<(), String> { + if redactions == 0 { + return Ok(()); + } + let surface = surface + .as_object_mut() + .ok_or_else(|| "Today snapshot surface must be a JSON object".to_string())?; + let projection = surface + .entry("snapshotProjection") + .or_insert_with(|| serde_json::json!({})); + let projection = projection.as_object_mut().ok_or_else(|| { + "Today snapshot surface.snapshotProjection must be a JSON object".to_string() + })?; + projection.insert( + "identitySecretsRedacted".into(), + serde_json::Value::from(redactions), + ); + projection.insert("bounded".into(), serde_json::Value::Bool(true)); + Ok(()) +} + +fn parse_unsigned_snapshot( + snapshot_json: &str, + expected_owner_pubkey: &str, + expected_relay_url: &str, + now: i64, + require_unexpired: bool, + redact_secret_text: bool, +) -> Result { + if snapshot_json.is_empty() || snapshot_json.len() > MAX_SNAPSHOT_BYTES { + return Err(format!( + "Today snapshot must contain between 1 and {MAX_SNAPSHOT_BYTES} bytes" + )); + } + validate_owner_pubkey(expected_owner_pubkey)?; + let expected_relay_url = normalize_relay_url(expected_relay_url)?; + let mut snapshot: UnsignedOwnerTodaySnapshot = serde_json::from_str(snapshot_json) + .map_err(|error| format!("parse Today snapshot: {error}"))?; + if snapshot.schema != TODAY_SNAPSHOT_SCHEMA { + return Err("unsupported Today snapshot schema".into()); + } + if snapshot.capability != TODAY_SNAPSHOT_CAPABILITY { + return Err("unsupported Today snapshot capability".into()); + } + if snapshot.owner_pubkey != expected_owner_pubkey { + return Err("Today snapshot owner does not match the active identity".into()); + } + let snapshot_relay_url = normalize_relay_url(&snapshot.relay_url)?; + if snapshot_relay_url != expected_relay_url { + return Err("Today snapshot relay does not match the active workspace".into()); + } + snapshot.relay_url = expected_relay_url; + if snapshot.generated_at < 0 || snapshot.expires_at < 0 { + return Err("Today snapshot timestamps must be non-negative".into()); + } + if snapshot.generated_at > now + 300 { + return Err("Today snapshot generatedAt is too far in the future".into()); + } + if snapshot.expires_at <= snapshot.generated_at + || snapshot.expires_at - snapshot.generated_at > MAX_LIFETIME_SECS + { + return Err("Today snapshot lifetime must be positive and at most 24 hours".into()); + } + if require_unexpired && snapshot.expires_at <= now { + return Err("Today snapshot has expired".into()); + } + if !snapshot.surface.is_object() { + return Err("Today snapshot surface must be a JSON object".into()); + } + if snapshot.raw_events.len() > MAX_RAW_EVENTS { + return Err(format!( + "Today snapshot rawEvents exceeds {MAX_RAW_EVENTS} records" + )); + } + if redact_secret_text { + clear_identity_secret_redaction_marker(&mut snapshot.surface)?; + let mut redactions = redact_identity_secret_text(&mut snapshot.surface)?; + for event in &mut snapshot.raw_events { + redactions += redact_identity_secret_text(event)?; + } + record_identity_secret_redactions(&mut snapshot.surface, redactions)?; + } else { + let mut sanitized_surface = snapshot.surface.clone(); + let mut redactions = redact_identity_secret_text(&mut sanitized_surface)?; + for event in &snapshot.raw_events { + let mut sanitized_event = event.clone(); + redactions += redact_identity_secret_text(&mut sanitized_event)?; + } + if redactions > 0 { + return Err("Today snapshot contains unsanitized identity secret material".into()); + } + } + Ok(snapshot) +} + +fn canonical_payload_json(snapshot: &UnsignedOwnerTodaySnapshot) -> Result { + serde_json::to_string(snapshot) + .map_err(|error| format!("serialize canonical Today snapshot: {error}")) +} + +fn snapshot_sha256(snapshot: &UnsignedOwnerTodaySnapshot) -> Result { + let canonical_json = canonical_payload_json(snapshot)?; + Ok(hex::encode(Sha256::digest(canonical_json.as_bytes()))) +} + +fn tag(name: &str, value: &str) -> Result { + Tag::parse([name, value]).map_err(|error| format!("build Today snapshot {name} tag: {error}")) +} + +fn signed_snapshot_tags( + snapshot: &UnsignedOwnerTodaySnapshot, + snapshot_sha256: &str, +) -> Result, String> { + Ok(vec![ + tag("t", TODAY_SNAPSHOT_TAG_MARKER)?, + tag("schema", TODAY_SNAPSHOT_SCHEMA)?, + tag("capability", TODAY_SNAPSHOT_CAPABILITY)?, + tag("snapshot_sha256", snapshot_sha256)?, + tag("expires_at", &snapshot.expires_at.to_string())?, + ]) +} + +fn build_signed_snapshot( + owner_keys: &Keys, + snapshot: UnsignedOwnerTodaySnapshot, +) -> Result { + let owner_pubkey = owner_keys.public_key().to_hex(); + if owner_pubkey != snapshot.owner_pubkey { + return Err("Today snapshot signer does not match the active owner identity".into()); + } + let snapshot_sha256 = snapshot_sha256(&snapshot)?; + let content = canonical_payload_json(&snapshot)?; + let event = EventBuilder::new(Kind::Custom(TODAY_SNAPSHOT_SIGNED_KIND), content) + .tags(signed_snapshot_tags(&snapshot, &snapshot_sha256)?) + .custom_created_at(Timestamp::from(snapshot.generated_at as u64)) + .sign_with_keys(owner_keys) + .map_err(|error| format!("sign Today snapshot: {error}"))?; + if event.pubkey.to_hex() != snapshot.owner_pubkey { + return Err("Today snapshot signer does not match the active owner identity".into()); + } + Ok(OwnerTodaySnapshot { + payload: snapshot, + snapshot_sha256, + event_id: event.id.to_hex(), + signature: event.sig.to_string(), + }) +} + +fn signed_event_json(snapshot: &OwnerTodaySnapshot) -> Result { + let content = canonical_payload_json(&snapshot.payload)?; + let tags = signed_snapshot_tags(&snapshot.payload, &snapshot.snapshot_sha256)? + .into_iter() + .map(|tag| { + serde_json::Value::Array( + tag.as_slice() + .iter() + .map(|value| serde_json::Value::String(value.clone())) + .collect(), + ) + }) + .collect::>(); + Ok(serde_json::json!({ + "id": snapshot.event_id, + "pubkey": snapshot.payload.owner_pubkey, + "created_at": snapshot.payload.generated_at, + "kind": TODAY_SNAPSHOT_SIGNED_KIND, + "tags": tags, + "content": content, + "sig": snapshot.signature, + }) + .to_string()) +} + +fn parse_and_validate_signed_snapshot( + snapshot_json: &str, + expected_owner_pubkey: &str, + expected_relay_url: &str, + now: i64, + require_unexpired: bool, +) -> Result { + if snapshot_json.is_empty() || snapshot_json.len() > MAX_SNAPSHOT_BYTES { + return Err(format!( + "Today snapshot must contain between 1 and {MAX_SNAPSHOT_BYTES} bytes" + )); + } + validate_owner_pubkey(expected_owner_pubkey)?; + let snapshot: OwnerTodaySnapshot = serde_json::from_str(snapshot_json) + .map_err(|error| format!("parse Today snapshot: {error}"))?; + let payload_json = canonical_payload_json(&snapshot.payload)?; + parse_unsigned_snapshot( + &payload_json, + expected_owner_pubkey, + expected_relay_url, + now, + require_unexpired, + false, + )?; + validate_hex(&snapshot.snapshot_sha256, "snapshotSha256", 64)?; + validate_hex(&snapshot.event_id, "eventId", 64)?; + validate_hex(&snapshot.signature, "signature", 128)?; + let expected_snapshot_sha256 = hex::encode(Sha256::digest(payload_json.as_bytes())); + if snapshot.snapshot_sha256 != expected_snapshot_sha256 { + return Err("Today snapshot snapshotSha256 does not match the canonical payload".into()); + } + let event = Event::from_json(signed_event_json(&snapshot)?) + .map_err(|error| format!("parse signed Today snapshot event: {error}"))?; + event + .verify() + .map_err(|error| format!("Today snapshot signature verification failed: {error}"))?; + if event.kind.as_u16() != TODAY_SNAPSHOT_SIGNED_KIND { + return Err(format!( + "Today snapshot event must use kind {TODAY_SNAPSHOT_SIGNED_KIND}" + )); + } + if event.pubkey.to_hex() != expected_owner_pubkey { + return Err("Today snapshot signer is not the active owner identity".into()); + } + if event.id.to_hex() != snapshot.event_id { + return Err("Today snapshot eventId does not match the signed event".into()); + } + Ok(snapshot) +} + +#[cfg(unix)] +fn enforce_private_permissions(path: &Path, directory: bool) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + let mode = if directory { 0o700 } else { 0o600 }; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)) + .map_err(|error| format!("set private Today snapshot permissions: {error}")) +} + +#[cfg(not(unix))] +fn enforce_private_permissions(_path: &Path, _directory: bool) -> Result<(), String> { + Ok(()) +} + +pub fn write_owner_today_snapshot( + nest_dir: &Path, + owner_keys: &Keys, + expected_owner_pubkey: &str, + expected_relay_url: &str, + snapshot_json: &str, + now: i64, +) -> Result { + let snapshot = parse_unsigned_snapshot( + snapshot_json, + expected_owner_pubkey, + expected_relay_url, + now, + true, + true, + )?; + let signed_snapshot = build_signed_snapshot(owner_keys, snapshot)?; + let canonical_json = serde_json::to_string(&signed_snapshot) + .map_err(|error| format!("serialize signed Today snapshot: {error}"))?; + let archive_dir = nest_dir.join("archive"); + std::fs::create_dir_all(&archive_dir) + .map_err(|error| format!("create Today snapshot directory: {error}"))?; + enforce_private_permissions(&archive_dir, true)?; + + let destination = snapshot_path( + nest_dir, + expected_owner_pubkey, + &signed_snapshot.payload.relay_url, + ); + // Same-directory persistence keeps replacement atomic on every platform. + let mut temp_file = tempfile::Builder::new() + .prefix(".activity-ledger-today-") + .tempfile_in(&archive_dir) + .map_err(|error| format!("create Today snapshot temp file: {error}"))?; + enforce_private_permissions(temp_file.path(), false)?; + temp_file + .write_all(canonical_json.as_bytes()) + .map_err(|error| format!("write Today snapshot: {error}"))?; + temp_file + .as_file() + .sync_all() + .map_err(|error| format!("sync Today snapshot: {error}"))?; + temp_file + .persist(&destination) + .map_err(|error| format!("atomically publish Today snapshot: {}", error.error))?; + enforce_private_permissions(&destination, false)?; + #[cfg(unix)] + std::fs::File::open(&archive_dir) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync Today snapshot directory: {error}"))?; + + let sha256 = hex::encode(Sha256::digest(canonical_json.as_bytes())); + Ok(TodaySnapshotReceipt { + path: destination.to_string_lossy().into_owned(), + owner_pubkey: signed_snapshot.payload.owner_pubkey, + relay_url: signed_snapshot.payload.relay_url, + generated_at: signed_snapshot.payload.generated_at, + expires_at: signed_snapshot.payload.expires_at, + byte_length: canonical_json.len(), + sha256, + }) +} + +pub fn read_owner_today_snapshot( + nest_dir: &Path, + expected_owner_pubkey: &str, + expected_relay_url: &str, + now: i64, +) -> Result { + let expected_relay_url = normalize_relay_url(expected_relay_url)?; + let path = snapshot_path(nest_dir, expected_owner_pubkey, &expected_relay_url); + let raw = std::fs::read_to_string(&path) + .map_err(|error| format!("read owner Today snapshot: {error}"))?; + parse_and_validate_signed_snapshot( + &raw, + expected_owner_pubkey, + &expected_relay_url, + now, + true, + )?; + Ok(raw) +} + +#[cfg(test)] +mod tests { + use super::*; + use nostr::Keys; + + const TEST_RELAY: &str = "wss://relay-a.test"; + + fn snapshot_for_relay(owner: &str, relay_url: &str, generated_at: i64) -> String { + serde_json::json!({ + "schema": TODAY_SNAPSHOT_SCHEMA, + "ownerPubkey": owner, + "relayUrl": relay_url, + "generatedAt": generated_at, + "expiresAt": generated_at + 3600, + "capability": TODAY_SNAPSHOT_CAPABILITY, + "surface": {"date": "2026-08-21", "journals": []}, + "rawEvents": [{"journalId": "j-1", "proofState": "OBSERVED"}] + }) + .to_string() + } + + fn snapshot(owner: &str, generated_at: i64) -> String { + snapshot_for_relay(owner, TEST_RELAY, generated_at) + } + + #[test] + fn desktop_canonical_payload_matches_shared_consumer_fixture() { + let fixture = + include_str!("../../../../test-fixtures/activity-ledger-today-desktop.json").trim(); + let snapshot: UnsignedOwnerTodaySnapshot = serde_json::from_str(fixture).unwrap(); + assert_eq!(canonical_payload_json(&snapshot).unwrap(), fixture); + } + + #[test] + fn snapshot_is_atomic_private_and_owner_scoped() { + let dir = tempfile::tempdir().unwrap(); + let keys = Keys::parse(&"a".repeat(64)).unwrap(); + let owner = keys.public_key().to_hex(); + let receipt = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + TEST_RELAY, + &snapshot(&owner, 1000), + 1000, + ) + .unwrap(); + assert_eq!(receipt.owner_pubkey, owner); + assert_eq!(receipt.sha256.len(), 64); + let raw = read_owner_today_snapshot(dir.path(), &owner, TEST_RELAY, 1001).unwrap(); + assert!(raw.contains(TODAY_SNAPSHOT_CAPABILITY)); + assert!(raw.contains("\"snapshotSha256\"")); + assert!(raw.contains("\"eventId\"")); + assert!(raw.contains("\"signature\"")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&receipt.path) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + } + + #[test] + fn snapshot_wrong_owner_expiry_and_capability_fail_closed() { + let dir = tempfile::tempdir().unwrap(); + let owner_keys = Keys::parse(&"a".repeat(64)).unwrap(); + let owner = owner_keys.public_key().to_hex(); + let other = Keys::parse(&"b".repeat(64)).unwrap().public_key().to_hex(); + assert!(write_owner_today_snapshot( + dir.path(), + &owner_keys, + &other, + TEST_RELAY, + &snapshot(&owner, 1000), + 1000 + ) + .unwrap_err() + .contains("owner does not match")); + + let mut value: serde_json::Value = serde_json::from_str(&snapshot(&owner, 1000)).unwrap(); + value["capability"] = "write-anything".into(); + assert!(write_owner_today_snapshot( + dir.path(), + &owner_keys, + &owner, + TEST_RELAY, + &value.to_string(), + 1000 + ) + .unwrap_err() + .contains("capability")); + + assert!(write_owner_today_snapshot( + dir.path(), + &owner_keys, + &owner, + TEST_RELAY, + &snapshot(&owner, 1000), + 5000 + ) + .unwrap_err() + .contains("expired")); + } + + #[test] + fn snapshot_paths_and_validation_are_relay_scoped() { + let dir = tempfile::tempdir().unwrap(); + let keys = Keys::parse(&"a".repeat(64)).unwrap(); + let owner = keys.public_key().to_hex(); + let other_relay = "wss://relay-b.test"; + let first = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + TEST_RELAY, + &snapshot_for_relay(&owner, TEST_RELAY, 1000), + 1000, + ) + .unwrap(); + let second = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + other_relay, + &snapshot_for_relay(&owner, other_relay, 1000), + 1000, + ) + .unwrap(); + assert_ne!(first.path, second.path); + + let equivalent = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + "wss://RELAY-A.TEST:443/", + &snapshot_for_relay(&owner, TEST_RELAY, 1001), + 1001, + ) + .unwrap(); + assert_eq!(first.path, equivalent.path); + assert_eq!(equivalent.relay_url, TEST_RELAY); + + std::fs::copy(&second.path, &first.path).unwrap(); + let error = read_owner_today_snapshot(dir.path(), &owner, TEST_RELAY, 1001).unwrap_err(); + assert!(error.contains("relay does not match"), "got: {error}"); + } + + #[test] + fn snapshot_read_revalidates_tampering_and_replacement() { + let dir = tempfile::tempdir().unwrap(); + let keys = Keys::parse(&"a".repeat(64)).unwrap(); + let owner = keys.public_key().to_hex(); + let first = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + TEST_RELAY, + &snapshot(&owner, 1000), + 1000, + ) + .unwrap(); + let second = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + TEST_RELAY, + &snapshot(&owner, 1001), + 1001, + ) + .unwrap(); + assert_eq!(first.path, second.path); + assert_ne!(first.sha256, second.sha256); + let replaced: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&second.path).unwrap()).unwrap(); + assert_eq!(replaced["generatedAt"], 1001); + let mut value: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&second.path).unwrap()).unwrap(); + value["ownerPubkey"] = "b".repeat(64).into(); + std::fs::write(&second.path, value.to_string()).unwrap(); + assert!( + read_owner_today_snapshot(dir.path(), &owner, TEST_RELAY, 1002) + .unwrap_err() + .contains("owner does not match") + ); + } + + #[test] + fn snapshot_redacts_identity_secret_fields_and_values_without_suppressing_today() { + let dir = tempfile::tempdir().unwrap(); + let keys = Keys::parse(&"a".repeat(64)).unwrap(); + let owner = keys.public_key().to_hex(); + let mut field: serde_json::Value = serde_json::from_str(&snapshot(&owner, 1000)).unwrap(); + field["surface"]["secretKey"] = "do-not-export".into(); + assert!(write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + TEST_RELAY, + &field.to_string(), + 1000 + ) + .unwrap_err() + .contains("identity secret")); + + let mut value: serde_json::Value = serde_json::from_str(&snapshot(&owner, 1000)).unwrap(); + value["surface"]["journals"] = serde_json::json!([{ + "id": "journal-with-example", + "summary": "Documentation example: NoStR_SeCrEt_KeY=test-value-1 while MODE=debug stays visible", + "detail": "A generic secret-handling note is safe and keyboard=on stays visible" + }]); + let raw_event = &mut value["rawEvents"][0]; + raw_event["detail"] = + "Tool failed with ANTHROPIC_API_KEY=test-value-2, BUZZ_AUTH_TAG='test-value-3', and APIKEY=test-value-4, but normal prose remains".into(); + raw_event["message"] = + "Inline owner key nsec1mock000000000000000000000000000000000000000000000000000000 must not leave the owner boundary".into(); + raw_event["json"] = r#"{"OPENAI_API_KEY":"test-value-5","mode":"safe"}"#.into(); + raw_event["header"] = "curl -H 'Authorization: Digest username=\"u\", response=\"test-value-6\"' -H \"Proxy-Authorization: Basic test-value-10\"".into(); + raw_event["command"] = "tool --api-key test-value-7 --mode safe".into(); + raw_event["stringifiedHeader"] = + r#"{"Authorization":"Bearer test-value-8","mode":"safe"}"#.into(); + raw_event["stringifiedDigest"] = + r#"{"Authorization":"Digest username=\"u\", response=\"test-value-11\"","mode":"safe"}"#.into(); + raw_event["doubleQuotedDigest"] = + r#"curl -H "Authorization: Digest username="u", response="test-value-12"" --mode safe"# + .into(); + raw_event["Authorization"] = "Bearer test-value-9".into(); + let receipt = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + TEST_RELAY, + &value.to_string(), + 1000, + ) + .unwrap(); + let raw = read_owner_today_snapshot(dir.path(), &owner, TEST_RELAY, 1001).unwrap(); + #[rustfmt::skip] + let leaked = ["do-not-export", "test-value-1", "test-value-2", "test-value-3", "test-value-4", "test-value-5", "test-value-6", "test-value-7", "test-value-8", "test-value-9", "test-value-10", "test-value-11", "test-value-12"]; + for secret in leaked { + assert!(!raw.contains(secret), "snapshot leaked {secret}"); + } + assert!(!raw.to_ascii_lowercase().contains("nsec1mock")); + let stored: serde_json::Value = serde_json::from_str(&raw).unwrap(); + assert_eq!( + stored["surface"]["snapshotProjection"]["identitySecretsRedacted"], + 13 + ); + #[rustfmt::skip] + assert_eq!(stored["surface"]["journals"][0]["summary"], "Documentation example: NoStR_SeCrEt_KeY=[REDACTED: identity secret material] while MODE=debug stays visible"); + #[rustfmt::skip] + assert_eq!(stored["surface"]["journals"][0]["detail"], "A generic secret-handling note is safe and keyboard=on stays visible"); + #[rustfmt::skip] + let expected_fields = [("detail", "Tool failed with ANTHROPIC_API_KEY=[REDACTED: identity secret material], BUZZ_AUTH_TAG='[REDACTED: identity secret material]', and APIKEY=[REDACTED: identity secret material], but normal prose remains"), ("message", "Inline owner key [REDACTED: identity secret material] must not leave the owner boundary"), ("json", r#"{"OPENAI_API_KEY":"[REDACTED: identity secret material]","mode":"safe"}"#), ("header", "curl -H 'Authorization: [REDACTED: identity secret material]' -H \"Proxy-Authorization: [REDACTED: identity secret material]\""), ("command", "tool --api-key [REDACTED: identity secret material] --mode safe"), ("stringifiedHeader", r#"{"Authorization":"[REDACTED: identity secret material]","mode":"safe"}"#), ("stringifiedDigest", r#"{"Authorization":"[REDACTED: identity secret material]","mode":"safe"}"#), ("doubleQuotedDigest", "curl -H \"Authorization: [REDACTED: identity secret material]\" --mode safe"), ("Authorization", "[REDACTED: identity secret material]")]; + for (field, expected) in expected_fields { + assert_eq!(stored["rawEvents"][0][field], expected); + } + assert_eq!(receipt.owner_pubkey, owner); + } + + #[test] + fn snapshot_read_rejects_same_user_forged_rewrite() { + let dir = tempfile::tempdir().unwrap(); + let keys = Keys::parse(&"a".repeat(64)).unwrap(); + let owner = keys.public_key().to_hex(); + let receipt = write_owner_today_snapshot( + dir.path(), + &keys, + &owner, + TEST_RELAY, + &snapshot(&owner, 1000), + 1000, + ) + .unwrap(); + let mut value: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&receipt.path).unwrap()).unwrap(); + value["surface"]["journals"] = serde_json::json!([{ "id": "forged" }]); + std::fs::write(&receipt.path, value.to_string()).unwrap(); + let error = read_owner_today_snapshot(dir.path(), &owner, TEST_RELAY, 1001).unwrap_err(); + assert!( + error.contains("snapshotSha256 does not match") + || error.contains("signature verification failed"), + "got: {error}" + ); + } +} diff --git a/desktop/src-tauri/src/archive/today_snapshot_authorization.rs b/desktop/src-tauri/src/archive/today_snapshot_authorization.rs new file mode 100644 index 00000000000..3624339e978 --- /dev/null +++ b/desktop/src-tauri/src/archive/today_snapshot_authorization.rs @@ -0,0 +1,95 @@ +//! Scheme-independent embedded HTTP authorization redaction. + +use regex::Regex; +use std::sync::OnceLock; + +fn header_prefix_regex() -> Result<&'static Regex, String> { + static REGEX: OnceLock> = OnceLock::new(); + match REGEX.get_or_init(|| { + Regex::new(r#"(?i)(?:proxy-)?authorization\s*:\s*"#).map_err(|error| { + format!("embedded authorization header prefix regex is invalid: {error}") + }) + }) { + Ok(regex) => Ok(regex), + Err(error) => Err(error.clone()), + } +} + +fn is_outer_quote_boundary(text: &str, quote_index: usize) -> bool { + let trailing = &text[quote_index + 1..]; + let Some(next) = trailing.chars().next() else { + return true; + }; + if next.is_ascii_whitespace() || matches!(next, ';' | '|' | '&' | ')' | ']' | '}') { + return true; + } + if next != ',' { + return false; + } + trailing[1..] + .trim_start_matches(char::is_whitespace) + .chars() + .next() + .is_some_and(|ch| matches!(ch, '\'' | '"')) +} + +/// Redact through the header's outer closing quote, ignoring quoted Digest +/// parameters inside a single-quoted shell argument. Without an outer quote, +/// fail closed by consuming the rest of the line. +pub(super) fn redact_embedded_authorization_headers( + text: &str, + redaction_marker: &str, +) -> Result<(String, usize), String> { + let regex = header_prefix_regex()?; + let mut output = String::with_capacity(text.len()); + let mut redactions = 0; + let mut copied_until = 0; + let mut search_from = 0; + while let Some(prefix) = regex.find_at(text, search_from) { + let preceding = text[..prefix.start()].chars().next_back(); + if preceding.is_some_and(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-')) { + search_from = prefix.end(); + continue; + } + let outer_quote = preceding.filter(|ch| matches!(ch, '\'' | '"')); + let value_end = match outer_quote { + Some(quote) => text[prefix.end()..] + .char_indices() + .find_map(|(offset, ch)| { + if ch != quote { + return None; + } + let index = prefix.end() + offset; + let backslashes = text.as_bytes()[..index] + .iter() + .rev() + .take_while(|byte| **byte == b'\\') + .count(); + (backslashes % 2 == 0 && is_outer_quote_boundary(text, index)).then_some(index) + }) + .unwrap_or(text.len()), + None => text[prefix.end()..] + .find(['\r', '\n']) + .map_or(text.len(), |offset| prefix.end() + offset), + }; + let value = &text[prefix.end()..value_end]; + output.push_str(&text[copied_until..prefix.end()]); + if value.trim() == redaction_marker { + output.push_str(value); + } else { + output.push_str(redaction_marker); + redactions += 1; + } + copied_until = value_end; + search_from = value_end; + if value_end == text.len() { + break; + } + } + if redactions == 0 { + Ok((text.to_owned(), 0)) + } else { + output.push_str(&text[copied_until..]); + Ok((output, redactions)) + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 71a5eb3806e..c4dda496966 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -835,6 +835,7 @@ pub fn run() { archive::delete_save_subscription, archive::read_archived_events, archive::read_archived_observer_events_for_channel, + archive::read_archived_observer_events_for_range, archive::index_observer_channel_id, archive::read_unindexed_observer_rows, archive::get_agent_usage_series, @@ -844,6 +845,12 @@ pub fn run() { archive::sync::announce_archive_sync_epoch, archive::sync::start_archive_sync, archive::sync::stop_archive_sync, + archive::journal_authority_commands::upsert_owner_journal_override, + archive::journal_authority_commands::upsert_journal_verification, + archive::journal_authority_commands::get_journal_authority_artifacts, + archive::journal_authority_commands::query_journal_authority_artifacts, + archive::journal_authority_commands::write_owner_today_snapshot, + archive::journal_authority_commands::read_owner_today_snapshot, is_auto_update_supported, set_window_vibrancy, #[cfg(target_os = "macos")] diff --git a/desktop/src-tauri/src/managed_agents/activity_ledger_env.rs b/desktop/src-tauri/src/managed_agents/activity_ledger_env.rs new file mode 100644 index 00000000000..5ae3d2d877c --- /dev/null +++ b/desktop/src-tauri/src/managed_agents/activity_ledger_env.rs @@ -0,0 +1,116 @@ +use std::path::{Path, PathBuf}; + +const TODAY_CAPABILITY: &str = "buzz.activity-ledger.today.read/v1"; +const TODAY_PATH_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_PATH"; +const TODAY_CAPABILITY_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_CAPABILITY"; +const TODAY_OWNER_PUBKEY_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY"; +const TODAY_RELAY_URL_ENV: &str = "BUZZ_ACTIVITY_LEDGER_TODAY_RELAY_URL"; + +fn honey_today_env( + persona_id: Option<&str>, + owner_hex: Option<&str>, + relay_url: Option<&str>, + nest: Option<&Path>, +) -> Option<(PathBuf, &'static str, String, String)> { + if persona_id != Some("builtin:honey") { + return None; + } + let owner = owner_hex?; + if owner.len() != 64 + || !owner + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return None; + } + let relay_url = buzz_core_pkg::relay::normalize_relay_url(relay_url?).ok()?; + Some(( + crate::archive::today_snapshot::snapshot_path(nest?, owner, &relay_url), + TODAY_CAPABILITY, + owner.to_owned(), + relay_url, + )) +} + +pub fn configure( + command: &mut std::process::Command, + record: &super::ManagedAgentRecord, + owner_hex: Option<&str>, + relay_url: &str, +) { + command.env_remove(TODAY_PATH_ENV); + command.env_remove(TODAY_CAPABILITY_ENV); + command.env_remove(TODAY_OWNER_PUBKEY_ENV); + command.env_remove(TODAY_RELAY_URL_ENV); + let nest = super::nest_dir(); + if let Some((path, capability, owner_pubkey, expected_relay_url)) = honey_today_env( + record.persona_id.as_deref(), + owner_hex, + Some(relay_url), + nest.as_deref(), + ) { + command.env(TODAY_PATH_ENV, path); + command.env(TODAY_CAPABILITY_ENV, capability); + command.env(TODAY_OWNER_PUBKEY_ENV, owner_pubkey); + command.env(TODAY_RELAY_URL_ENV, expected_relay_url); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn today_env_is_owner_scoped_and_honey_only() { + let owner = "a".repeat(64); + let relay = "wss://relay-a.test"; + let nest = Path::new("/private/buzz-nest"); + let (path, capability, expected_owner, expected_relay) = + honey_today_env(Some("builtin:honey"), Some(&owner), Some(relay), Some(nest)).unwrap(); + assert_eq!( + path, + crate::archive::today_snapshot::snapshot_path(nest, &owner, relay) + ); + assert_eq!(capability, TODAY_CAPABILITY); + assert_eq!(expected_owner, owner); + assert_eq!(expected_relay, relay); + assert!(super::super::is_reserved_env_key(TODAY_OWNER_PUBKEY_ENV)); + assert!(super::super::is_reserved_env_key(TODAY_RELAY_URL_ENV)); + assert!( + honey_today_env(Some("builtin:fizz"), Some(&owner), Some(relay), Some(nest)).is_none() + ); + assert!( + honey_today_env(Some("builtin:honey"), Some("bad"), Some(relay), Some(nest)).is_none() + ); + assert!(honey_today_env( + Some("builtin:honey"), + Some(&owner), + Some("https://bad"), + Some(nest) + ) + .is_none()); + assert!(honey_today_env(Some("builtin:honey"), Some(&owner), Some(relay), None).is_none()); + } + + #[test] + fn today_env_canonicalizes_relay_before_path_and_contract() { + let owner = "a".repeat(64); + let nest = Path::new("/private/buzz-nest"); + let raw = honey_today_env( + Some("builtin:honey"), + Some(&owner), + Some("ws://localhost:3000/"), + Some(nest), + ) + .unwrap(); + let canonical = honey_today_env( + Some("builtin:honey"), + Some(&owner), + Some("ws://127.0.0.1:3000"), + Some(nest), + ) + .unwrap(); + assert_eq!(raw, canonical); + assert_eq!(raw.3, "ws://127.0.0.1:3000"); + } +} diff --git a/desktop/src-tauri/src/managed_agents/mod.rs b/desktop/src-tauri/src/managed_agents/mod.rs index 272c03348b9..9c90def2fac 100644 --- a/desktop/src-tauri/src/managed_agents/mod.rs +++ b/desktop/src-tauri/src/managed_agents/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod access_policy; +mod activity_ledger_env; mod agent_env; pub(crate) mod agent_events; pub(crate) mod agent_snapshot; diff --git a/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs b/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs index afaaa2b4eb3..099ced061ee 100644 --- a/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs +++ b/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs @@ -56,6 +56,13 @@ pub(crate) const RESERVED_ENV_KEYS: &[&str] = &[ // Stable agent identity used for git attribution and private-conversation // provenance must come from the managed-agent record, not user overrides. "BUZZ_ACP_DISPLAY_NAME", + // Owner Activity Ledger read capability. Desktop derives these from the + // active owner and private nest path for Honey; saved persona config may + // not redirect the reader to another file or weaken the contract marker. + "BUZZ_ACTIVITY_LEDGER_TODAY_PATH", + "BUZZ_ACTIVITY_LEDGER_TODAY_CAPABILITY", + "BUZZ_ACTIVITY_LEDGER_TODAY_OWNER_PUBKEY", + "BUZZ_ACTIVITY_LEDGER_TODAY_RELAY_URL", // Remote lifetime/presence policy: user env must not disable the // desktop/provider-owned bounds while the saved record still promises them. "BUZZ_ACP_EXIT_AFTER_INACTIVITY", diff --git a/desktop/src-tauri/src/managed_agents/runtime.rs b/desktop/src-tauri/src/managed_agents/runtime.rs index 0ce5ca7b219..c11f86f10f0 100644 --- a/desktop/src-tauri/src/managed_agents/runtime.rs +++ b/desktop/src-tauri/src/managed_agents/runtime.rs @@ -400,7 +400,6 @@ pub(crate) fn configure_runtime_cli( /// Spawn an agent process without holding any locks on records or runtimes. /// Returns the child process and log path on success. The caller is responsible /// for updating `ManagedAgentRecord` fields and inserting into the runtimes map. -/// /// `owner_hex`: the workspace owner's pubkey, used as a fallback for legacy /// records that have no NIP-OA `auth_tag`. See `build_respond_to_env`. pub fn spawn_agent_child( @@ -764,6 +763,7 @@ pub fn spawn_agent_child( command.env_remove("BUZZ_AUTH_TAG"); } + super::activity_ledger_env::configure(&mut command, record, owner_hex, &effective_relay_url); // Inbound author gate: who is this agent allowed to respond to? // Validation is strict here — a malformed allowlist on disk fails before // we spawn anything (the harness would also reject it, but we'd rather diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index e111f93ca0e..cb8098b8db6 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -49,6 +49,7 @@ import { useManagedAgentRuntimeReconciliation } from "@/features/agents/useManag import { useAutoRestartPolicy } from "@/features/agents/lib/useAutoRestartPolicy"; import { usePersonaSync } from "@/features/agents/lib/usePersonaSync"; import { useAgentObserverIngestion } from "@/features/agents/useAgentObserverIngestion"; +import { useActivityLedgerTodaySnapshot } from "@/features/agents/useActivityLedgerTodaySnapshot"; import { AgentManagementDialogs } from "@/features/agents/ui/AgentManagementDialogs"; import { RequestedAgentCreateDialogs } from "@/features/agents/ui/RequestedAgentCreateDialogs"; import { @@ -214,6 +215,7 @@ export function AppShell() { // The archive batch now persists in Rust, so the agent-metrics invalidation // signal arrives as a Tauri event rather than an in-process call. useArchiveAgentMetricsBridge(); + useActivityLedgerTodaySnapshot(); // Kind 44200 is relay-persisted (durable) and stays deferred: missed // startup frames can be replayed, so there's no ordering constraint. const deferredPubkey = startupReady ? identityQuery.data?.pubkey : undefined; diff --git a/desktop/src/features/agents/activityLedger.test.mjs b/desktop/src/features/agents/activityLedger.test.mjs new file mode 100644 index 00000000000..bda1912e3b0 --- /dev/null +++ b/desktop/src/features/agents/activityLedger.test.mjs @@ -0,0 +1,619 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + applyOwnerJournalOverride, + buildMissionJournal, + buildTodayActivitySurface, + normalizeActivityEvents, +} from "./activityLedger.ts"; + +const AGENT_A = "a".repeat(64); +const AGENT_B = "b".repeat(64); +const CHANNEL_A = "11111111-1111-1111-1111-111111111111"; +const CHANNEL_B = "22222222-2222-2222-2222-222222222222"; + +function observerEvent(overrides = {}) { + return { + seq: 1, + timestamp: "2026-08-21T14:00:00.000Z", + kind: "turn_started", + agentIndex: 0, + channelId: CHANNEL_A, + sessionId: "sess-1", + turnId: "turn-1", + payload: { source: "channel", triggeringEventIds: ["f".repeat(64)] }, + sourceEventId: (overrides.seq ?? 1).toString(16).padStart(64, "0"), + ...overrides, + }; +} + +function sessionUpdate(seq, sessionUpdate, update = {}, eventOverrides = {}) { + return observerEvent({ + ...eventOverrides, + seq, + sourceEventId: + eventOverrides.sourceEventId ?? seq.toString(16).padStart(64, "0"), + kind: "acp_read", + payload: { + method: "session/update", + params: { + sessionId: eventOverrides.sessionId ?? "sess-1", + update: { + sessionUpdate, + ...update, + }, + }, + }, + }); +} + +test("normalizeActivityEvents marks failed tool updates as FAILED and preserves tool correlation", () => { + const events = normalizeActivityEvents([ + observerEvent(), + sessionUpdate(2, "tool_call", { + toolCallId: "call-1", + status: "executing", + title: "shell", + kind: "shell", + rawInput: { command: "cargo test" }, + }), + sessionUpdate(3, "tool_call_update", { + toolCallId: "call-1", + status: "failed", + title: "shell", + kind: "shell", + rawInput: { command: "cargo test" }, + rawOutput: "boom", + }), + ]); + + const toolUpdate = events.find( + (event) => event.category === "tool" && event.status === "failed", + ); + assert.ok(toolUpdate); + assert.equal(toolUpdate.proofState, "FAILED"); + assert.equal(toolUpdate.correlationId, "call-1"); + assert.equal(toolUpdate.provenance.toolCallId, "call-1"); +}); + +test("buildMissionJournal flags turn completion without supporting evidence", () => { + const normalized = normalizeActivityEvents([ + observerEvent(), + observerEvent({ + seq: 2, + kind: "turn_completed", + payload: {}, + sourceEventId: "d".repeat(64), + }), + ]); + + const journal = buildMissionJournal(normalized); + assert.equal(journal.status, "ended_unverified"); + assert.equal(journal.proofState, "OBSERVED"); + assert.equal(journal.claimedCompletionWithoutEvidence, true); + assert.match(journal.summary, /without supporting evidence/i); +}); + +test("normalizeActivityEvents deduplicates duplicate observer frames", () => { + const duplicate = sessionUpdate(2, "tool_call", { + toolCallId: "call-dup", + status: "executing", + title: "read_file", + kind: "read_file", + rawInput: { path: "Cargo.toml" }, + }); + const events = normalizeActivityEvents([duplicate, duplicate]); + assert.equal(events.length, 1); +}); + +test("normalizeActivityEvents exposes producer telemetry gaps as UNKNOWN", () => { + const [gap] = normalizeActivityEvents([ + observerEvent({ + kind: "observer_telemetry_gap", + payload: { + droppedEvents: 4, + reasonCounts: { publishQueueEviction: 4 }, + }, + }), + ]); + + assert.equal(gap.title, "Observer telemetry gap"); + assert.equal(gap.status, "blocked"); + assert.equal(gap.proofState, "UNKNOWN"); + assert.match(gap.detail, /4 source events dropped/i); + assert.ok(gap.tags.includes("evidence_gap")); +}); + +test("signed provenance keeps same seq and timestamp with different ids distinct", () => { + const events = normalizeActivityEvents([ + observerEvent({ sourceEventId: "1".repeat(64) }), + observerEvent({ sourceEventId: "2".repeat(64) }), + ]); + assert.equal(events.length, 2); +}); + +test("signed batch siblings keep the same outer id without collapsing", () => { + const sourceEventId = "3".repeat(64); + const events = normalizeActivityEvents([ + observerEvent({ seq: 1, sourceEventId }), + observerEvent({ + seq: 2, + kind: "turn_completed", + sourceEventId, + }), + ]); + assert.equal(events.length, 2); + assert.equal(new Set(events.map((event) => event.id)).size, 2); + assert.deepEqual( + events.map((event) => event.provenance.sourceEventId), + [sourceEventId, sourceEventId], + ); +}); + +test("completed tool output is RECEIPTED but never implicitly VERIFIED", () => { + const events = normalizeActivityEvents([ + sessionUpdate(2, "tool_call_update", { + toolCallId: "call-receipt", + status: "completed", + title: "shell", + kind: "shell", + rawOutput: "ok", + }), + ]); + assert.equal(events[0].proofState, "RECEIPTED"); +}); + +test("assistant done claim does not verify an unconditional turn end", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent(), + sessionUpdate(2, "agent_message_chunk", { + messageId: "message-1", + content: [{ type: "text", text: "Done." }], + }), + observerEvent({ + seq: 3, + kind: "turn_completed", + payload: {}, + sourceEventId: "3".repeat(64), + }), + ]), + ); + assert.equal(journal.status, "ended_unverified"); + assert.equal(journal.proofState, "OBSERVED"); + assert.equal(journal.claimedCompletionWithoutEvidence, true); +}); + +test("agent-authored verifier fields cannot mint VERIFIED", () => { + const events = normalizeActivityEvents([ + observerEvent({ + kind: "proof_verified", + payload: { + verified: true, + verifierPubkey: "9".repeat(64), + receiptRef: "receipt:independent-1", + }, + }), + ]); + assert.equal(events[0].proofState, "CLAIMED"); +}); + +test("agent-authored journal override cannot rewrite the observed summary", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent(), + observerEvent({ + seq: 2, + kind: "journal_override", + payload: { + summary: "Owner says complete", + modifiedBy: "owner", + }, + }), + ]), + ); + assert.equal(journal.summarySource, "auto"); + assert.equal(journal.ownerModifiedBy, null); + assert.notEqual(journal.summary, "Owner says complete"); +}); + +test("stale started turn is incomplete and UNKNOWN", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([observerEvent()]), + { asOf: "2026-08-21T14:10:00.000Z", incompleteAfterMs: 60_000 }, + ); + assert.equal(journal.status, "incomplete"); + assert.equal(journal.proofState, "UNKNOWN"); +}); + +test("recent liveness keeps a long-running turn in progress", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent(), + observerEvent({ + seq: 2, + timestamp: "2026-08-21T14:09:00.000Z", + kind: "turn_liveness", + payload: {}, + }), + ]), + { asOf: "2026-08-21T14:10:00.000Z", incompleteAfterMs: 5 * 60_000 }, + ); + assert.equal(journal.status, "in_progress"); + assert.equal(journal.endedAt, "2026-08-21T14:09:00.000Z"); +}); + +test("real managed-agent runtime failure shape fails the journal", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent({ + kind: "managed_agent_runtime_lifecycle", + payload: { + pubkey: AGENT_A, + relayUrl: "wss://relay.example", + startNonce: "start-1", + lifecycle: "failed", + error: "pool wake task failed", + }, + }), + ]), + ); + assert.equal(journal.status, "failed"); + assert.equal(journal.proofState, "FAILED"); + assert.match(journal.summary, /pool wake task failed/); +}); + +test("a later runtime ready event clears an earlier runtime failure", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent({ + kind: "managed_agent_runtime_lifecycle", + payload: { + pubkey: AGENT_A, + startNonce: "start-1", + lifecycle: "failed", + error: "pool wake task failed", + }, + }), + observerEvent({ + seq: 2, + timestamp: "2026-08-21T14:01:00.000Z", + kind: "managed_agent_runtime_lifecycle", + payload: { + pubkey: AGENT_A, + startNonce: "start-2", + lifecycle: "ready", + }, + sourceEventId: "8".repeat(64), + }), + ]), + ); + assert.equal(journal.status, "observed"); + assert.equal(journal.proofState, "OBSERVED"); + assert.doesNotMatch(journal.summary, /pool wake task failed/); +}); + +test("the real listening to waking to ready lifecycle does not stay in progress", () => { + const events = ["listening", "waking", "ready"].map((lifecycle, index) => + observerEvent({ + seq: index + 1, + timestamp: `2026-08-21T14:0${index}:00.000Z`, + kind: "managed_agent_runtime_lifecycle", + payload: { lifecycle, startNonce: "start-1" }, + sourceEventId: `${index + 1}`.repeat(64), + }), + ); + const journal = buildMissionJournal(normalizeActivityEvents(events)); + assert.equal(journal.status, "observed"); + assert.equal(journal.proofState, "OBSERVED"); +}); + +test("a repeated runtime failure reports the latest failure reason", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent({ + kind: "managed_agent_runtime_lifecycle", + payload: { lifecycle: "failed", error: "old failure" }, + }), + observerEvent({ + seq: 2, + timestamp: "2026-08-21T14:01:00.000Z", + kind: "managed_agent_runtime_lifecycle", + payload: { lifecycle: "ready" }, + sourceEventId: "8".repeat(64), + }), + observerEvent({ + seq: 3, + timestamp: "2026-08-21T14:02:00.000Z", + kind: "managed_agent_runtime_lifecycle", + payload: { lifecycle: "failed", error: "new failure" }, + sourceEventId: "9".repeat(64), + }), + ]), + ); + assert.equal(journal.status, "failed"); + assert.equal(journal.proofState, "FAILED"); + assert.match(journal.summary, /new failure/); + assert.doesNotMatch(journal.summary, /old failure/); +}); + +test("successful retry prevents an earlier failed tool from failing the journal", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent(), + sessionUpdate(2, "tool_call_update", { + toolCallId: "call-retry", + status: "failed", + title: "shell", + rawOutput: "boom", + }), + sessionUpdate(3, "tool_call_update", { + toolCallId: "call-retry", + status: "completed", + title: "shell", + rawOutput: "ok", + }), + observerEvent({ + seq: 4, + kind: "turn_completed", + payload: {}, + sourceEventId: "6".repeat(64), + }), + ]), + ); + assert.equal(journal.status, "completed"); + assert.equal(journal.proofState, "RECEIPTED"); +}); + +test("unrecovered tool failure remains FAILED without claiming mission failure", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent(), + sessionUpdate(2, "tool_call_update", { + toolCallId: "call-failed", + status: "failed", + title: "shell", + rawOutput: "boom", + }), + observerEvent({ + seq: 3, + kind: "turn_completed", + payload: {}, + sourceEventId: "7".repeat(64), + }), + ]), + ); + assert.equal(journal.status, "ended_unverified"); + assert.equal(journal.proofState, "FAILED"); +}); + +test("buildMissionJournal reconstructs the same result from a restart replay", () => { + const raw = [ + observerEvent(), + sessionUpdate(2, "tool_call", { + toolCallId: "call-2", + status: "executing", + title: "shell", + kind: "shell", + rawInput: { command: "pnpm test" }, + }), + sessionUpdate(3, "tool_call_update", { + toolCallId: "call-2", + status: "completed", + title: "shell", + kind: "shell", + rawInput: { command: "pnpm test" }, + rawOutput: "3 passed", + }), + observerEvent({ + seq: 4, + kind: "turn_completed", + payload: {}, + sourceEventId: "c".repeat(64), + }), + ]; + + const first = buildMissionJournal(normalizeActivityEvents(raw)); + const replayed = buildMissionJournal(normalizeActivityEvents([...raw])); + + assert.deepEqual(replayed, first); +}); + +test("buildMissionJournal selects the most recently active overlapping turn", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent({ + turnId: "turn-a", + timestamp: "2026-08-21T10:00:00.000Z", + sourceEventId: "a".repeat(64), + }), + observerEvent({ + turnId: "turn-b", + timestamp: "2026-08-21T11:00:00.000Z", + sourceEventId: "b".repeat(64), + }), + observerEvent({ + turnId: "turn-a", + timestamp: "2026-08-21T12:00:00.000Z", + kind: "turn_completed", + payload: {}, + sourceEventId: "c".repeat(64), + }), + ]), + ); + assert.equal(journal.turnId, "turn-a"); + assert.equal(journal.endedAt, "2026-08-21T12:00:00.000Z"); +}); + +test("buildTodayActivitySurface keeps multi-agent handoffs distinct while aggregating one day", () => { + const feed = buildTodayActivitySurface( + [ + { + agentPubkey: AGENT_A, + agentName: "Fizz", + events: normalizeActivityEvents([ + observerEvent({ + sessionId: "sess-a", + turnId: "turn-a", + channelId: CHANNEL_A, + }), + sessionUpdate( + 2, + "tool_call_update", + { + toolCallId: "call-a", + status: "completed", + title: "shell", + kind: "shell", + rawInput: { command: "cargo test" }, + rawOutput: "ok", + }, + { sessionId: "sess-a", turnId: "turn-a", channelId: CHANNEL_A }, + ), + ]), + }, + { + agentPubkey: AGENT_B, + agentName: "Honey", + events: normalizeActivityEvents([ + observerEvent({ + sessionId: "sess-b", + turnId: "turn-b", + channelId: CHANNEL_A, + timestamp: "2026-08-21T14:05:00.000Z", + }), + sessionUpdate( + 2, + "plan", + { content: [{ type: "text", text: "Follow up with owner" }] }, + { + sessionId: "sess-b", + turnId: "turn-b", + channelId: CHANNEL_A, + timestamp: "2026-08-21T14:06:00.000Z", + }, + ), + ]), + }, + ], + { day: "2026-08-21" }, + ); + + assert.equal(feed.journals.length, 2); + assert.deepEqual( + feed.journals.map((journal) => journal.correlationId), + ["turn-a", "turn-b"], + ); + assert.equal(feed.channels[0].channelId, CHANNEL_A); + assert.deepEqual( + feed.channels[0].agentPubkeys.sort(), + [AGENT_A, AGENT_B].sort(), + ); +}); + +test("applyOwnerJournalOverride keeps owner edits separate from observed proof", () => { + const journal = buildMissionJournal( + normalizeActivityEvents([ + observerEvent(), + sessionUpdate(2, "tool_call_update", { + toolCallId: "call-3", + status: "completed", + title: "shell", + kind: "shell", + rawInput: { command: "npm test" }, + rawOutput: "ok", + }), + ]), + ); + + const overridden = applyOwnerJournalOverride(journal, { + summary: "Owner note: verify with Bumble before closing.", + modifiedAt: "2026-08-21T15:00:00.000Z", + modifiedBy: "owner", + }); + + assert.equal(overridden.summarySource, "owner"); + assert.equal( + overridden.summary, + "Owner note: verify with Bumble before closing.", + ); + assert.equal(overridden.proofState, journal.proofState); +}); + +test("buildTodayActivitySurface filters out work from other local days", () => { + const feed = buildTodayActivitySurface( + [ + { + agentPubkey: AGENT_A, + agentName: "Fizz", + events: normalizeActivityEvents([ + observerEvent({ + timestamp: "2026-08-20T23:58:00.000Z", + channelId: CHANNEL_B, + }), + observerEvent({ + seq: 2, + timestamp: "2026-08-21T15:00:00.000Z", + channelId: CHANNEL_A, + }), + ]), + }, + ], + { day: "2026-08-21" }, + ); + + assert.equal(feed.journals.length, 1); + assert.equal(feed.journals[0].channelId, CHANNEL_A); +}); + +test("buildTodayActivitySurface marks abandoned turns incomplete", () => { + const feed = buildTodayActivitySurface( + [ + { + agentPubkey: AGENT_A, + agentName: "Fizz", + events: normalizeActivityEvents([observerEvent()]), + }, + ], + { + day: "2026-08-21", + asOf: "2026-08-21T14:10:00.000Z", + incompleteAfterMs: 60_000, + }, + ); + + assert.equal(feed.journals[0].status, "incomplete"); + assert.equal(feed.journals[0].proofState, "UNKNOWN"); +}); + +test("Today marks a stale cross-midnight liveness-only turn incomplete", () => { + const feed = buildTodayActivitySurface( + [ + { + agentPubkey: AGENT_A, + agentName: "Fizz", + events: normalizeActivityEvents([ + observerEvent({ + timestamp: "2026-08-21T03:59:00.000Z", + }), + observerEvent({ + seq: 2, + timestamp: "2026-08-21T04:01:00.000Z", + kind: "turn_liveness", + sourceEventId: "e".repeat(64), + }), + ]), + }, + ], + { + day: "2026-08-21", + asOf: "2026-08-21T04:10:00.000Z", + incompleteAfterMs: 60_000, + }, + ); + + assert.equal(feed.journals.length, 1); + assert.equal(feed.journals[0].status, "incomplete"); + assert.equal(feed.journals[0].proofState, "UNKNOWN"); +}); diff --git a/desktop/src/features/agents/activityLedger.ts b/desktop/src/features/agents/activityLedger.ts new file mode 100644 index 00000000000..242b28b9a69 --- /dev/null +++ b/desktop/src/features/agents/activityLedger.ts @@ -0,0 +1,999 @@ +import { + observerEventIdentity, + type ObserverEvent, +} from "./ui/agentSessionTypes"; + +export type ActivityProofState = + | "OBSERVED" + | "CLAIMED" + | "RECEIPTED" + | "VERIFIED" + | "FAILED" + | "UNKNOWN"; + +export type ActivityStatus = + | "pending" + | "running" + | "completed" + | "failed" + | "blocked" + | "unknown"; + +export type MissionJournalStatus = + | "in_progress" + | "completed" + | "failed" + | "ended_unverified" + | "incomplete" + | "observed"; + +export type ActivityCategory = + | "turn" + | "tool" + | "message" + | "thought" + | "plan" + | "permission" + | "prompt" + | "status"; + +export type ActivityProvenance = { + sourceEventId: string | null; + sourcePubkey: string | null; + sourceKind: number | null; + sourceCreatedAt: number | null; + sourceSignature: string | null; + origin: "live_observer" | "historical_backfill" | "unknown"; + observerKind: string; + method: string | null; + sessionUpdate: string | null; + seq: number; + timestamp: string; + channelId: string | null; + sessionId: string | null; + turnId: string | null; + toolCallId: string | null; + messageId: string | null; + triggeringEventIds: string[]; +}; + +export type NormalizedActivityEvent = { + id: string; + journalKey: string; + correlationId: string; + category: ActivityCategory; + title: string; + detail: string | null; + status: ActivityStatus; + proofState: ActivityProofState; + timestamp: string; + channelId: string | null; + sessionId: string | null; + turnId: string | null; + toolCallId: string | null; + messageId: string | null; + provenance: ActivityProvenance; + tags: string[]; + ownerModifiedAt?: string | null; + ownerModifiedBy?: string | null; +}; + +export type MissionJournal = { + id: string; + journalKey: string; + correlationId: string; + channelId: string | null; + sessionId: string | null; + turnId: string | null; + startedAt: string; + endedAt: string; + status: MissionJournalStatus; + proofState: ActivityProofState; + summary: string; + summarySource: "auto" | "owner"; + ownerModifiedAt: string | null; + ownerModifiedBy: string | null; + claimedCompletionWithoutEvidence: boolean; + eventCount: number; + events: NormalizedActivityEvent[]; +}; + +export type MissionJournalOverride = { + summary: string; + modifiedAt: string; + modifiedBy: string; +}; + +export type MissionJournalBuildOptions = { + asOf?: string | Date; + incompleteAfterMs?: number; +}; + +export type TodayActivityFeedInput = { + agentPubkey: string; + agentName: string; + events: NormalizedActivityEvent[]; +}; + +export type TodayActivityChannel = { + channelId: string; + journalIds: string[]; + agentPubkeys: string[]; + agentNames: string[]; + lastActivityAt: string; +}; + +export type TodayActivityJournal = MissionJournal & { + agentPubkey: string; + agentName: string; +}; + +export type TodayActivitySurface = { + day: string; + journals: TodayActivityJournal[]; + channels: TodayActivityChannel[]; + counts: { + journals: number; + failed: number; + inProgress: number; + claimedWithoutEvidence: number; + }; +}; + +const PROOF_RANK: Record = { + UNKNOWN: 0, + CLAIMED: 1, + OBSERVED: 2, + RECEIPTED: 3, + VERIFIED: 4, + FAILED: 5, +}; + +function asRecord(value: unknown): Record { + return value && typeof value === "object" + ? (value as Record) + : {}; +} + +function asString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +function extractContentText(value: unknown): string | null { + if (typeof value === "string") { + return value.trim() || null; + } + const record = asRecord(value); + const text = asString(record.text); + if (text) return text; + if (Array.isArray(value)) { + const parts = value + .map((entry) => extractContentText(entry)) + .filter((entry): entry is string => Boolean(entry)); + return parts.length > 0 ? parts.join("\n") : null; + } + return null; +} + +function toolTitle(update: Record) { + return ( + asString(update.toolName) ?? + asString(update.kind) ?? + asString(update.title) ?? + "tool" + ); +} + +function toolCallId(update: Record) { + return asString(update.toolCallId) ?? asString(update.tool_call_id); +} + +function messageId(update: Record) { + return asString(update.messageId) ?? asString(update.message_id); +} + +function statusFromUpdate(status: string | null | undefined): ActivityStatus { + switch (status) { + case "pending": + return "pending"; + case "executing": + return "running"; + case "completed": + case "done": + return "completed"; + case "failed": + case "error": + return "failed"; + default: + return "unknown"; + } +} + +function proofStateForTool( + status: ActivityStatus, + output: unknown, +): ActivityProofState { + if (status === "failed") return "FAILED"; + if (status !== "completed") return "OBSERVED"; + return output === undefined || output === null ? "OBSERVED" : "RECEIPTED"; +} + +function triggeringEventIds(event: ObserverEvent): string[] { + const ids = asRecord(event.payload).triggeringEventIds; + return Array.isArray(ids) + ? ids.filter((id): id is string => typeof id === "string" && id.length > 0) + : []; +} + +function correlationId( + event: ObserverEvent, + update?: Record, + turnCorrelationId?: string | null, +) { + const toolId = update ? toolCallId(update) : null; + return ( + toolId ?? + triggeringEventIds(event)[0] ?? + turnCorrelationId ?? + event.journalKey ?? + event.turnId ?? + event.sessionId ?? + event.channelId ?? + `${event.kind}:${event.seq}` + ); +} + +function journalKey(event: ObserverEvent) { + return ( + event.journalKey ?? + event.turnId ?? + event.sessionId ?? + event.channelId ?? + "global" + ); +} + +function buildId( + event: ObserverEvent, + category: ActivityCategory, + suffix: string | null = null, +) { + return [category, observerEventIdentity(event), suffix] + .filter(Boolean) + .join(":"); +} + +function eventTagSet( + category: ActivityCategory, + updateType: string | null, + toolName: string | null, +): string[] { + const tags: string[] = [category]; + if (updateType) tags.push(updateType); + if (toolName) tags.push(`tool:${toolName}`); + return tags; +} + +function compareObserverEvents(left: ObserverEvent, right: ObserverEvent) { + const leftTime = Date.parse(left.timestamp); + const rightTime = Date.parse(right.timestamp); + if ( + Number.isFinite(leftTime) && + Number.isFinite(rightTime) && + leftTime !== rightTime + ) { + return leftTime - rightTime; + } + return left.seq - right.seq; +} + +function dedupeObserverEvents(events: readonly ObserverEvent[]) { + const seen = new Set(); + const deduped: ObserverEvent[] = []; + for (const event of [...events].sort(compareObserverEvents)) { + const key = observerEventIdentity(event); + if (seen.has(key)) continue; + seen.add(key); + deduped.push(event); + } + return deduped; +} + +function localDay(timestamp: string) { + const date = new Date(timestamp); + const year = date.getFullYear(); + const month = `${date.getMonth() + 1}`.padStart(2, "0"); + const day = `${date.getDate()}`.padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +function normalizeOne( + event: ObserverEvent, + turnCorrelationId: string | null, +): NormalizedActivityEvent | null { + const payload = asRecord(event.payload); + const method = asString(payload.method); + const update = + method === "session/update" + ? asRecord(asRecord(payload.params).update) + : null; + const updateType = update ? asString(update.sessionUpdate) : null; + const toolName = update ? toolTitle(update) : null; + const callId = update ? toolCallId(update) : null; + const msgId = update ? messageId(update) : null; + const base = { + journalKey: journalKey(event), + correlationId: correlationId(event, update ?? undefined, turnCorrelationId), + timestamp: event.timestamp, + channelId: event.channelId ?? null, + sessionId: event.sessionId ?? null, + turnId: event.turnId ?? null, + toolCallId: callId, + messageId: msgId, + provenance: { + sourceEventId: event.sourceEventId ?? null, + sourcePubkey: event.sourcePubkey ?? null, + sourceKind: event.sourceKind ?? null, + sourceCreatedAt: event.sourceCreatedAt ?? null, + sourceSignature: event.sourceSignature ?? null, + origin: + event.origin === "live_observer" || + event.origin === "historical_backfill" + ? event.origin + : "unknown", + observerKind: event.kind, + method: method ?? null, + sessionUpdate: updateType ?? null, + seq: event.seq, + timestamp: event.timestamp, + channelId: event.channelId ?? null, + sessionId: event.sessionId ?? null, + turnId: event.turnId ?? null, + toolCallId: callId, + messageId: msgId, + triggeringEventIds: triggeringEventIds(event), + } satisfies ActivityProvenance, + }; + + if (event.kind === "observer_telemetry_gap") { + const rawDropped = payload.droppedEvents; + const droppedEvents = + typeof rawDropped === "number" && + Number.isSafeInteger(rawDropped) && + rawDropped > 0 + ? rawDropped + : null; + return { + ...base, + id: buildId(event, "status", "telemetry-gap"), + category: "status", + title: "Observer telemetry gap", + detail: `${droppedEvents ?? "Unknown number of"} source event${ + droppedEvents === 1 ? "" : "s" + } dropped before archival.`, + status: "blocked", + proofState: "UNKNOWN", + tags: [ + ...eventTagSet("status", "observer_telemetry_gap", null), + "evidence_gap", + ], + }; + } + + if (event.kind === "turn_started") { + return { + ...base, + id: buildId(event, "turn"), + category: "turn", + title: "Turn started", + detail: null, + status: "running", + proofState: "OBSERVED", + tags: eventTagSet("turn", null, null), + }; + } + + if (event.kind === "turn_completed") { + return { + ...base, + id: buildId(event, "turn"), + category: "turn", + title: "Turn completed", + detail: null, + status: "completed", + proofState: "OBSERVED", + tags: eventTagSet("turn", null, null), + }; + } + + if (event.kind === "turn_error" || event.kind === "agent_panic") { + return { + ...base, + id: buildId(event, "turn"), + category: "turn", + title: event.kind === "agent_panic" ? "Agent crashed" : "Turn failed", + detail: asString(payload.error) ?? extractContentText(payload) ?? null, + status: "failed", + proofState: "FAILED", + tags: eventTagSet("turn", null, null), + }; + } + + if (event.kind === "managed_agent_runtime_lifecycle") { + const lifecycle = asRecord(event.payload); + const phase = asString(lifecycle.lifecycle); + const detail = asString(lifecycle.error) ?? phase; + const status = + phase === "failed" + ? "failed" + : phase === "ready" + ? "completed" + : "running"; + return { + ...base, + id: buildId(event, "status", "runtime-lifecycle"), + category: "status", + title: + phase === "failed" + ? "Runtime failed" + : phase === "ready" + ? "Runtime ready" + : "Runtime lifecycle observed", + detail, + status, + proofState: phase === "failed" ? "FAILED" : "OBSERVED", + tags: eventTagSet("status", "managed_agent_runtime_lifecycle", null), + }; + } + + if (event.kind === "turn_liveness") { + return { + ...base, + id: buildId(event, "status", "turn-liveness"), + category: "status", + title: "Turn active", + detail: null, + status: "running", + proofState: "OBSERVED", + tags: eventTagSet("status", "turn_liveness", null), + }; + } + + if (event.kind === "session_config_captured") { + const config = asRecord(event.payload); + const provider = asString(config.provider); + const model = asString(config.model); + const detail = + [ + provider ? `provider ${provider}` : null, + model ? `model ${model}` : null, + ] + .filter((entry): entry is string => Boolean(entry)) + .join(", ") || null; + return { + ...base, + id: buildId(event, "status", "session-config"), + category: "status", + title: "Session config captured", + detail, + status: "completed", + proofState: "OBSERVED", + tags: eventTagSet("status", "session_config_captured", null), + }; + } + + if (event.kind === "journal_override") { + return null; + } + + if (event.kind === "proof_verified" || event.kind === "proof_failed") { + const receiptRef = asString(payload.receiptRef); + const failed = event.kind === "proof_failed"; + return { + ...base, + id: buildId(event, "status", event.kind), + category: "status", + title: failed ? "Proof verification failed" : "Verification claimed", + detail: receiptRef, + status: failed ? "failed" : "completed", + // Agent-signed verifier fields are still self-reported claims. + proofState: failed ? "FAILED" : "CLAIMED", + tags: eventTagSet("status", event.kind, null), + }; + } + + if (event.kind === "session_resolved") { + return { + ...base, + id: buildId(event, "status"), + category: "status", + title: "Session ready", + detail: extractContentText(payload), + status: "running", + proofState: "OBSERVED", + tags: eventTagSet("status", null, null), + }; + } + + if (method === "session/request_permission") { + return { + ...base, + id: buildId(event, "permission"), + category: "permission", + title: "Permission requested", + detail: + asString(asRecord(asRecord(payload.params).title)) ?? + asString(asRecord(payload.params).message) ?? + null, + status: "blocked", + proofState: "OBSERVED", + tags: eventTagSet("permission", null, null), + }; + } + + if (event.kind === "acp_write" && !method) { + const result = asRecord(asRecord(payload.result).outcome); + const outcome = asString(result.outcome); + if (outcome) { + return { + ...base, + id: buildId(event, "permission"), + category: "permission", + title: "Permission resolved", + detail: outcome, + status: "completed", + proofState: "RECEIPTED", + tags: eventTagSet("permission", null, null), + }; + } + } + + if (event.kind === "acp_write" && method === "session/prompt") { + return { + ...base, + id: buildId(event, "prompt"), + category: "prompt", + title: "Prompt issued", + detail: extractContentText(asRecord(payload.params).prompt), + status: "completed", + proofState: "RECEIPTED", + tags: eventTagSet("prompt", null, null), + }; + } + + if (updateType === "tool_call" || updateType === "tool_call_update") { + const status = statusFromUpdate(asString(update?.status)); + const name = toolName ?? "tool"; + const output = update?.rawOutput ?? update?.content; + return { + ...base, + id: buildId(event, "tool", callId ?? name), + category: "tool", + title: name, + detail: + extractContentText(update?.rawOutput) ?? + extractContentText(update?.content) ?? + extractContentText(update?.rawInput) ?? + null, + status, + proofState: proofStateForTool(status, output), + tags: eventTagSet("tool", updateType, name), + }; + } + + if ( + updateType === "agent_message_chunk" || + updateType === "user_message_chunk" + ) { + return { + ...base, + id: buildId(event, "message", msgId ?? updateType), + category: "message", + title: + updateType === "agent_message_chunk" ? "Agent message" : "User message", + detail: extractContentText(update?.content), + status: "completed", + proofState: "CLAIMED", + tags: eventTagSet("message", updateType, null), + }; + } + + if (updateType === "agent_thought_chunk") { + return { + ...base, + id: buildId(event, "thought"), + category: "thought", + title: "Thought", + detail: extractContentText(update?.content), + status: "completed", + proofState: "CLAIMED", + tags: eventTagSet("thought", updateType, null), + }; + } + + if (updateType === "plan") { + return { + ...base, + id: buildId(event, "plan"), + category: "plan", + title: "Plan updated", + detail: extractContentText(update?.content), + status: "completed", + proofState: "CLAIMED", + tags: eventTagSet("plan", updateType, null), + }; + } + + const freeformText = asString(payload.text) ?? extractContentText(payload); + if (freeformText) { + return { + ...base, + id: buildId(event, "status"), + category: "status", + title: asString(payload.title) ?? event.kind, + detail: freeformText, + status: event.kind.includes("error") ? "failed" : "completed", + proofState: event.kind.includes("error") ? "FAILED" : "OBSERVED", + tags: eventTagSet("status", updateType, null), + }; + } + + return null; +} + +export function normalizeActivityEvents( + events: readonly ObserverEvent[], +): NormalizedActivityEvent[] { + const deduped = dedupeObserverEvents(events); + const latestLiveness = new Map(); + for (const event of deduped) { + if (event.kind !== "turn_liveness") continue; + const key = event.turnId ?? event.sessionId ?? event.channelId ?? "global"; + latestLiveness.set(key, event); + } + const compacted = deduped.filter((event) => { + if (event.kind !== "turn_liveness") return true; + const key = event.turnId ?? event.sessionId ?? event.channelId ?? "global"; + return latestLiveness.get(key) === event; + }); + const turnCorrelations = new Map(); + for (const event of compacted) { + const root = triggeringEventIds(event)[0]; + if (event.turnId && root) turnCorrelations.set(event.turnId, root); + } + return compacted + .map((event) => + normalizeOne( + event, + event.turnId ? (turnCorrelations.get(event.turnId) ?? null) : null, + ), + ) + .filter((event): event is NormalizedActivityEvent => Boolean(event)); +} + +function bestProofState(events: readonly NormalizedActivityEvent[]) { + return events.reduce((best, event) => { + return PROOF_RANK[event.proofState] > PROOF_RANK[best] + ? event.proofState + : best; + }, "UNKNOWN"); +} + +function buildSummary( + events: readonly NormalizedActivityEvent[], + status: MissionJournalStatus, + claimedCompletionWithoutEvidence: boolean, +) { + const ownerOverride = events.find((event) => event.ownerModifiedAt != null); + if (ownerOverride?.detail) { + return ownerOverride.detail; + } + + const toolNames = [ + ...new Set( + events + .filter((event) => event.category === "tool") + .map((event) => event.title), + ), + ]; + if (status === "failed") { + const failed = [...events] + .reverse() + .find( + (event) => + event.status === "failed" && + (event.category === "turn" || + event.provenance.observerKind === + "managed_agent_runtime_lifecycle"), + ); + return failed?.detail + ? `${failed.title}: ${failed.detail}` + : `${failed?.title ?? "Turn failed"} during observed execution.`; + } + if (claimedCompletionWithoutEvidence) { + return "Execution ended without supporting evidence for the requested outcome."; + } + if (status === "incomplete") { + return "Execution started but no terminal event was observed before the activity became stale."; + } + if (status === "completed" && toolNames.length > 0) { + return `Execution ended with receipted activity in ${toolNames.join(", ")}; outcome verification remains separate.`; + } + if (toolNames.length > 0) { + return `Observed work in ${toolNames.join(", ")}.`; + } + const claimed = events.find( + (event) => event.proofState === "CLAIMED" && event.detail, + ); + if (claimed?.detail) { + return claimed.detail; + } + return "Observed agent activity."; +} + +export function groupMissionJournals( + events: readonly NormalizedActivityEvent[], + options: MissionJournalBuildOptions = {}, +): MissionJournal[] { + const grouped = new Map(); + for (const event of [...events].sort((left, right) => { + const leftTime = Date.parse(left.timestamp); + const rightTime = Date.parse(right.timestamp); + return leftTime === rightTime + ? left.provenance.seq - right.provenance.seq + : leftTime - rightTime; + })) { + const bucket = grouped.get(event.journalKey) ?? []; + bucket.push(event); + grouped.set(event.journalKey, bucket); + } + + return [...grouped.entries()].map(([key, bucket]) => { + const startedAt = bucket[0]?.timestamp ?? new Date(0).toISOString(); + const endedAt = bucket[bucket.length - 1]?.timestamp ?? startedAt; + const latestRuntimeLifecycle = [...bucket] + .reverse() + .find( + (event) => + event.provenance.observerKind === "managed_agent_runtime_lifecycle", + ); + const effectiveProofEvents = latestRuntimeLifecycle + ? bucket.filter( + (event) => + event.provenance.observerKind !== + "managed_agent_runtime_lifecycle" || + event === latestRuntimeLifecycle, + ) + : bucket; + const hasTerminalFailure = + bucket.some( + (event) => event.status === "failed" && event.category === "turn", + ) || latestRuntimeLifecycle?.status === "failed"; + const hasCompletion = bucket.some( + (event) => event.category === "turn" && event.status === "completed", + ); + const successfulEvidence = bucket.filter( + (event) => + event.status === "completed" && + (event.proofState === "VERIFIED" || + (event.proofState === "RECEIPTED" && + (event.category === "tool" || event.category === "status"))), + ); + const claimedCompletionWithoutEvidence = + hasCompletion && successfulEvidence.length === 0; + const latestToolState = new Map(); + for (const event of bucket) { + if (event.category === "tool") { + latestToolState.set(event.correlationId, event.status); + } + } + const hasUnresolvedToolFailure = [...latestToolState.values()].some( + (status) => status === "failed", + ); + const hasProofFailure = bucket.some( + (event) => + event.provenance.observerKind === "proof_failed" || + (event.category === "status" && + event.proofState === "FAILED" && + event.provenance.observerKind !== "managed_agent_runtime_lifecycle"), + ); + const asOf = + options.asOf instanceof Date + ? options.asOf.getTime() + : options.asOf + ? Date.parse(options.asOf) + : Number.NaN; + const incompleteAfterMs = options.incompleteAfterMs ?? 5 * 60_000; + const isStaleIncomplete = + !hasCompletion && + !hasTerminalFailure && + Number.isFinite(asOf) && + asOf - Date.parse(endedAt) >= incompleteAfterMs && + bucket.some( + (event) => + (event.category === "turn" && event.status === "running") || + event.provenance.observerKind === "turn_liveness", + ); + + let status: MissionJournalStatus = "observed"; + if (hasTerminalFailure) { + status = "failed"; + } else if (claimedCompletionWithoutEvidence) { + status = "ended_unverified"; + } else if (hasCompletion) { + status = "completed"; + } else if (isStaleIncomplete) { + status = "incomplete"; + } else if ( + effectiveProofEvents.some((event) => event.status === "running") + ) { + status = "in_progress"; + } + + const ownerOverride = bucket.find((event) => event.ownerModifiedAt != null); + const proofState: ActivityProofState = + hasTerminalFailure || hasProofFailure || hasUnresolvedToolFailure + ? "FAILED" + : isStaleIncomplete + ? "UNKNOWN" + : successfulEvidence.some((event) => event.proofState === "VERIFIED") + ? "VERIFIED" + : successfulEvidence.some( + (event) => event.proofState === "RECEIPTED", + ) + ? "RECEIPTED" + : claimedCompletionWithoutEvidence + ? "OBSERVED" + : bestProofState(effectiveProofEvents); + + return { + id: key, + journalKey: key, + // The source-derived journal key survives paging, restarts, and midnight; + // per-event tool/message correlations remain intact. + correlationId: key, + channelId: bucket[0]?.channelId ?? null, + sessionId: bucket.find((event) => event.sessionId)?.sessionId ?? null, + turnId: bucket.find((event) => event.turnId)?.turnId ?? null, + startedAt, + endedAt, + status, + proofState, + summary: buildSummary(bucket, status, claimedCompletionWithoutEvidence), + summarySource: ownerOverride ? "owner" : "auto", + ownerModifiedAt: ownerOverride?.ownerModifiedAt ?? null, + ownerModifiedBy: ownerOverride?.ownerModifiedBy ?? null, + claimedCompletionWithoutEvidence, + eventCount: bucket.length, + events: bucket, + } satisfies MissionJournal; + }); +} + +export function buildMissionJournal( + events: readonly NormalizedActivityEvent[], + options: MissionJournalBuildOptions = {}, +): MissionJournal { + const journals = groupMissionJournals(events, options); + let latest: MissionJournal | undefined; + for (const journal of journals) { + if ( + !latest || + Date.parse(journal.endedAt) > Date.parse(latest.endedAt) || + (journal.endedAt === latest.endedAt && journal.id > latest.id) + ) { + latest = journal; + } + } + if (latest) return latest; + return { + id: "empty", + journalKey: "empty", + correlationId: "empty", + channelId: null, + sessionId: null, + turnId: null, + startedAt: new Date(0).toISOString(), + endedAt: new Date(0).toISOString(), + status: "observed", + proofState: "UNKNOWN", + summary: "No observed activity.", + summarySource: "auto", + ownerModifiedAt: null, + ownerModifiedBy: null, + claimedCompletionWithoutEvidence: false, + eventCount: 0, + events: [], + }; +} + +export function applyOwnerJournalOverride( + journal: MissionJournal, + override: MissionJournalOverride, +): MissionJournal { + return { + ...journal, + summary: override.summary, + summarySource: "owner", + ownerModifiedAt: override.modifiedAt, + ownerModifiedBy: override.modifiedBy, + }; +} + +export function buildTodayActivitySurface( + feeds: readonly TodayActivityFeedInput[], + options: { + day: string; + asOf?: string | Date; + incompleteAfterMs?: number; + }, +): TodayActivitySurface { + const journals: TodayActivityJournal[] = []; + const channels = new Map< + string, + { + journalIds: string[]; + agentPubkeys: Set; + agentNames: Set; + lastActivityAt: string; + } + >(); + + for (const feed of feeds) { + for (const journal of groupMissionJournals( + feed.events.filter((event) => localDay(event.timestamp) === options.day), + { + asOf: options.asOf ?? new Date(), + incompleteAfterMs: options.incompleteAfterMs, + }, + )) { + journals.push({ + ...journal, + agentPubkey: feed.agentPubkey, + agentName: feed.agentName, + }); + + if (!journal.channelId) continue; + const bucket = channels.get(journal.channelId) ?? { + journalIds: [], + agentPubkeys: new Set(), + agentNames: new Set(), + lastActivityAt: journal.endedAt, + }; + bucket.journalIds.push(journal.id); + bucket.agentPubkeys.add(feed.agentPubkey); + bucket.agentNames.add(feed.agentName); + if (Date.parse(journal.endedAt) > Date.parse(bucket.lastActivityAt)) { + bucket.lastActivityAt = journal.endedAt; + } + channels.set(journal.channelId, bucket); + } + } + + journals.sort( + (left, right) => Date.parse(left.startedAt) - Date.parse(right.startedAt), + ); + + return { + day: options.day, + journals, + channels: [...channels.entries()] + .map(([channelId, bucket]) => ({ + channelId, + journalIds: bucket.journalIds, + agentPubkeys: [...bucket.agentPubkeys], + agentNames: [...bucket.agentNames], + lastActivityAt: bucket.lastActivityAt, + })) + .sort((left, right) => left.channelId.localeCompare(right.channelId)), + counts: { + journals: journals.length, + failed: journals.filter((journal) => journal.status === "failed").length, + inProgress: journals.filter((journal) => journal.status === "in_progress") + .length, + claimedWithoutEvidence: journals.filter( + (journal) => journal.claimedCompletionWithoutEvidence, + ).length, + }, + }; +} diff --git a/desktop/src/features/agents/activityLedgerAuthority.test.mjs b/desktop/src/features/agents/activityLedgerAuthority.test.mjs new file mode 100644 index 00000000000..9df1b637c8e --- /dev/null +++ b/desktop/src/features/agents/activityLedgerAuthority.test.mjs @@ -0,0 +1,492 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildMissionJournal, + normalizeActivityEvents, +} from "./activityLedger.ts"; +import { + applyValidatedJournalAuthority, + journalAuthorityCorrelationId, + journalVerificationSources, +} from "./activityLedgerAuthority.ts"; + +const sourceId = "a".repeat(64); +const terminalSourceId = "b".repeat(64); +const relayUrl = "wss://relay.example"; +const agentPubkey = "a".repeat(64); + +function observedJournal() { + return buildMissionJournal( + normalizeActivityEvents([ + { + seq: 1, + timestamp: "2026-08-21T14:00:00.000Z", + kind: "turn_started", + sourceEventId: sourceId, + sourcePubkey: "agent-a", + sourceKind: 24200, + sourceCreatedAt: 1_787_319_600, + sourceSignature: "agent-signature", + origin: "historical_backfill", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: { triggeringEventIds: ["message-1"] }, + }, + { + seq: 2, + timestamp: "2026-08-21T14:01:00.000Z", + kind: "turn_completed", + sourceEventId: terminalSourceId, + sourcePubkey: "agent-a", + sourceKind: 24200, + sourceCreatedAt: 1_787_319_660, + sourceSignature: "agent-signature-2", + origin: "historical_backfill", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: {}, + }, + ]), + ); +} + +function artifact(overrides = {}) { + const journal = observedJournal(); + return { + ownerPubkey: "owner-a", + relayUrl, + agentPubkey, + eventId: "c".repeat(64), + signature: "owner-signature", + createdAt: 1_787_319_720, + artifactType: "verification", + journalId: journal.id, + correlationId: journal.correlationId, + revision: 1, + summary: null, + note: null, + receiptRef: "receipt:independent-check-1", + sourceEventIds: [sourceId, terminalSourceId].sort(), + ...overrides, + }; +} + +function applyAuthority( + journal, + artifacts, + scopedRelay = relayUrl, + scopedAgent = agentPubkey, +) { + return applyValidatedJournalAuthority( + journal, + artifacts, + scopedRelay, + scopedAgent, + ); +} + +test("stable journal correlation preserves pre-day authority across midnight", () => { + const currentSourceId = "d".repeat(64); + const journal = buildMissionJournal( + normalizeActivityEvents([ + { + seq: 2, + timestamp: "2026-08-22T00:00:05.000Z", + kind: "acp_read", + sourceEventId: currentSourceId, + sourcePubkey: "agent-a", + sourceKind: 24200, + sourceCreatedAt: 1_787_356_805, + sourceSignature: "agent-signature-current", + origin: "historical_backfill", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: { + method: "session/update", + params: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-after-midnight", + status: "completed", + output: { ok: true }, + }, + }, + }, + }, + ]), + ); + assert.equal(journal.correlationId, journal.id); + + const overridden = applyAuthority( + journal, + [ + artifact({ + artifactType: "owner_override", + journalId: journal.id, + correlationId: journal.id, + createdAt: 1_787_319_720, + summary: "Owner summary written before midnight", + receiptRef: null, + sourceEventIds: [], + }), + ], + relayUrl, + ); + assert.equal(overridden.summary, "Owner summary written before midnight"); + + const staleVerification = applyAuthority( + journal, + [ + artifact({ + journalId: journal.id, + correlationId: journal.id, + sourceEventIds: [sourceId], + }), + ], + relayUrl, + ); + assert.notEqual(staleVerification.proofState, "VERIFIED"); + + const currentVerification = applyAuthority( + journal, + [ + artifact({ + journalId: journal.id, + correlationId: journal.id, + sourceEventIds: [sourceId, currentSourceId].sort(), + }), + ], + relayUrl, + ); + assert.equal(currentVerification.proofState, "VERIFIED"); +}); + +test("owner-signed verification only promotes evidence bound to the journal", () => { + const journal = observedJournal(); + assert.notEqual(journal.proofState, "VERIFIED"); + + const verified = applyAuthority(journal, [artifact()], relayUrl); + assert.equal(verified.proofState, "VERIFIED"); + assert.equal(verified.status, "completed"); + assert.equal(verified.events.at(-1).title, "Owner verification"); + assert.deepEqual(verified.events.at(-1).provenance.triggeringEventIds, [ + sourceId, + terminalSourceId, + ]); + assert.equal(verified.events.at(-1).provenance.seq, 3); + assert.equal( + verified.events.at(-1).provenance.sourceSignature, + "owner-signature", + ); + + const crossJournal = applyAuthority( + journal, + [artifact({ sourceEventIds: ["f".repeat(64)] })], + relayUrl, + ); + assert.notEqual(crossJournal.proofState, "VERIFIED"); +}); + +test("later failure and incomplete states outrank an older verification", () => { + const journal = observedJournal(); + + for (const [status, proofState] of [ + ["failed", "FAILED"], + ["incomplete", "UNKNOWN"], + ]) { + const terminalJournal = { ...journal, status, proofState }; + const updated = applyAuthority(terminalJournal, [artifact()], relayUrl); + + assert.equal(updated.status, status); + assert.equal(updated.proofState, proofState); + assert.equal( + updated.events.some((event) => event.title === "Owner verification"), + false, + ); + } +}); + +test("later successful tool evidence invalidates an older verification", () => { + const journal = observedJournal(); + const laterSourceId = "d".repeat(64); + const laterTool = { + ...journal.events[0], + id: "later-tool", + category: "tool", + title: "Tool completed", + status: "completed", + proofState: "RECEIPTED", + timestamp: "2026-08-21T14:03:00.000Z", + provenance: { + ...journal.events[0].provenance, + sourceEventId: laterSourceId, + sourceCreatedAt: 1_787_319_780, + seq: 3, + timestamp: "2026-08-21T14:03:00.000Z", + }, + }; + const current = { + ...journal, + endedAt: laterTool.timestamp, + eventCount: journal.eventCount + 1, + events: [...journal.events, laterTool], + }; + + assert.notEqual( + applyAuthority(current, [artifact()], relayUrl).proofState, + "VERIFIED", + ); + assert.equal( + applyAuthority( + current, + [ + artifact({ + sourceEventIds: [sourceId, terminalSourceId, laterSourceId].sort(), + }), + ], + relayUrl, + ).proofState, + "VERIFIED", + ); +}); + +test("later turn completion invalidates an older verification", () => { + const journal = observedJournal(); + const laterSourceId = "e".repeat(64); + const laterTurn = { + ...journal.events.at(-1), + id: "later-turn", + timestamp: "2026-08-21T14:04:00.000Z", + provenance: { + ...journal.events.at(-1).provenance, + sourceEventId: laterSourceId, + sourceCreatedAt: 1_787_319_840, + seq: 4, + timestamp: "2026-08-21T14:04:00.000Z", + }, + }; + const current = { + ...journal, + endedAt: laterTurn.timestamp, + eventCount: journal.eventCount + 1, + events: [...journal.events, laterTurn], + }; + + assert.notEqual( + applyAuthority(current, [artifact()], relayUrl).proofState, + "VERIFIED", + ); +}); + +test("every later journal frame invalidates an older verification", () => { + const journal = observedJournal(); + for (const [index, category] of [ + "message", + "prompt", + "thought", + "plan", + ].entries()) { + const laterSourceId = (index + 10).toString(16).padStart(64, "0"); + const laterEvent = { + ...journal.events[0], + id: `later-${category}`, + category, + title: `Later ${category}`, + status: "completed", + proofState: category === "prompt" ? "RECEIPTED" : "CLAIMED", + timestamp: `2026-08-21T14:0${index + 3}:00.000Z`, + provenance: { + ...journal.events[0].provenance, + sourceEventId: laterSourceId, + sourceCreatedAt: 1_787_319_780 + index * 60, + observerKind: "acp_read", + seq: index + 3, + timestamp: `2026-08-21T14:0${index + 3}:00.000Z`, + }, + }; + const current = { + ...journal, + endedAt: laterEvent.timestamp, + eventCount: journal.eventCount + 1, + events: [...journal.events, laterEvent], + }; + + assert.notEqual( + applyAuthority(current, [artifact()], relayUrl).proofState, + "VERIFIED", + category, + ); + assert.equal( + applyAuthority( + current, + [ + artifact({ + sourceEventIds: [sourceId, terminalSourceId, laterSourceId].sort(), + }), + ], + relayUrl, + ).proofState, + "VERIFIED", + category, + ); + } +}); + +test("reapplying authority ignores its synthetic owner verification event", () => { + const journal = observedJournal(); + const verified = applyAuthority(journal, [artifact()], relayUrl); + const reapplied = applyAuthority(verified, [artifact()], relayUrl); + + assert.equal(reapplied.proofState, "VERIFIED"); + assert.equal(reapplied.events.at(-1).title, "Owner verification"); +}); + +test("large journal verification computes its sequence without argument spread", () => { + const journal = observedJournal(); + const template = journal.events.at(-1); + const events = [ + journal.events[0], + ...Array.from({ length: 149_999 }, (_, index) => ({ + ...template, + id: `large-${index}`, + provenance: { + ...template.provenance, + seq: index + 2, + }, + })), + ]; + const largeJournal = { + ...journal, + eventCount: events.length, + events, + }; + + const verified = applyAuthority(largeJournal, [artifact()], relayUrl); + assert.equal(verified.proofState, "VERIFIED"); + assert.equal(verified.events.at(-1).provenance.seq, 150_001); +}); + +test("verification writes use the journal correlation, not a tool-call correlation", () => { + const journal = observedJournal(); + const receiptSourceId = "e".repeat(64); + const toolEvidence = { + ...journal.events[0], + correlationId: "tool-call-1", + proofState: "RECEIPTED", + category: "tool", + toolCallId: "tool-call-1", + provenance: { + ...journal.events[0].provenance, + sourceEventId: receiptSourceId, + toolCallId: "tool-call-1", + triggeringEventIds: [], + }, + }; + const journalWithTool = { + ...journal, + events: [...journal.events, toolEvidence], + }; + + assert.equal(journalAuthorityCorrelationId(journalWithTool), journal.id); + assert.notEqual( + journalAuthorityCorrelationId(journalWithTool), + toolEvidence.correlationId, + ); + assert.deepEqual(journalVerificationSources(journalWithTool), { + sourceEventIds: [receiptSourceId], + hasReceiptedEvidence: true, + hasCorrelationEvidence: true, + hasSupportedSourceSet: true, + overflowCount: 0, + }); + + const missingCorrelationSource = { + ...journalWithTool, + correlationId: "message-without-an-observer-source", + }; + assert.equal( + journalVerificationSources(missingCorrelationSource).hasCorrelationEvidence, + false, + ); +}); + +test("verification source capacity fails closed before backend submission", () => { + const journal = observedJournal(); + const receipted = Array.from({ length: 257 }, (_, index) => ({ + ...journal.events[0], + id: `receipt-${index}`, + category: "tool", + proofState: "RECEIPTED", + provenance: { + ...journal.events[0].provenance, + sourceEventId: index.toString(16).padStart(64, "0"), + triggeringEventIds: [], + }, + })); + const atLimit = journalVerificationSources({ + ...journal, + events: [...journal.events, ...receipted.slice(0, 256)], + }); + assert.equal(atLimit.sourceEventIds.length, 256); + assert.equal(atLimit.hasSupportedSourceSet, true); + assert.equal(atLimit.overflowCount, 0); + + const overLimit = journalVerificationSources({ + ...journal, + events: [...journal.events, ...receipted.slice(0, 257)], + }); + assert.equal(overLimit.sourceEventIds.length, 257); + assert.equal(overLimit.hasSupportedSourceSet, false); + assert.equal(overLimit.overflowCount, 1); +}); + +test("latest owner override changes summary without changing proof", () => { + const journal = observedJournal(); + const base = artifact({ + artifactType: "owner_override", + eventId: "d".repeat(64), + summary: "First owner summary", + note: "clarified", + receiptRef: null, + sourceEventIds: [], + }); + const latest = { + ...base, + eventId: "e".repeat(64), + revision: 2, + createdAt: base.createdAt + 10, + summary: "Corrected owner summary", + }; + + const updated = applyAuthority(journal, [latest, base], relayUrl); + assert.equal(updated.summary, "Corrected owner summary"); + assert.equal(updated.summarySource, "owner"); + assert.equal(updated.ownerModifiedBy, "owner-a"); + assert.equal(updated.proofState, journal.proofState); + + const crossRelay = applyAuthority( + journal, + [{ ...latest, relayUrl: "wss://other-relay.example" }], + relayUrl, + ); + assert.equal(crossRelay.summarySource, "auto"); + assert.equal(crossRelay.summary, journal.summary); +}); + +test("same journal authority cannot cross managed-agent identity", () => { + const journal = observedJournal(); + const crossAgent = applyAuthority( + journal, + [artifact({ agentPubkey: "b".repeat(64) })], + relayUrl, + agentPubkey, + ); + assert.notEqual(crossAgent.proofState, "VERIFIED"); +}); diff --git a/desktop/src/features/agents/activityLedgerAuthority.ts b/desktop/src/features/agents/activityLedgerAuthority.ts new file mode 100644 index 00000000000..d26637c2de2 --- /dev/null +++ b/desktop/src/features/agents/activityLedgerAuthority.ts @@ -0,0 +1,256 @@ +import { + applyOwnerJournalOverride, + type MissionJournal, + type NormalizedActivityEvent, +} from "./activityLedger"; +import { canonicalRelayUrl } from "./managedAgentRuntimeStatus"; +import { normalizePubkey } from "@/shared/lib/pubkey"; + +/** A signature-verified owner artifact returned by the Tauri authority store. */ +export type ValidatedJournalAuthorityArtifact = { + ownerPubkey: string; + relayUrl: string; + agentPubkey: string; + eventId: string; + signature: string; + createdAt: number; + artifactType: "owner_override" | "verification"; + journalId: string; + correlationId: string; + revision: number; + summary: string | null; + note: string | null; + receiptRef: string | null; + sourceEventIds: string[]; +}; + +/** + * Authority artifacts are journal-scoped, even when their supporting evidence + * is a tool event with its own tool-call correlation. + */ +export function journalAuthorityCorrelationId( + journal: Pick, +): string { + return journal.correlationId; +} + +export type JournalVerificationSources = { + sourceEventIds: string[]; + hasReceiptedEvidence: boolean; + hasCorrelationEvidence: boolean; + hasSupportedSourceSet: boolean; + overflowCount: number; +}; + +export const MAX_JOURNAL_VERIFICATION_SOURCE_EVENTS = 256; + +function validSourceEventId(value: string | null | undefined): value is string { + return typeof value === "string" && /^[0-9a-f]{64}$/i.test(value); +} + +function isCurrentVerificationEvidence( + event: NormalizedActivityEvent, +): boolean { + if ( + event.provenance.sourceKind === 24201 || + event.provenance.observerKind === "owner_verification" + ) { + return false; + } + // Every retained observer frame changes the journal an owner is verifying. + // A later message, prompt, thought, or plan is still later activity, even + // when it is only CLAIMED. Requiring its signed source prevents an older + // receipt from promoting an expanded journal back to VERIFIED. + return validSourceEventId(event.provenance.sourceEventId); +} + +function currentVerificationSourceEventId( + journal: MissionJournal, +): string | null { + for (let index = journal.events.length - 1; index >= 0; index -= 1) { + const event = journal.events[index]; + if (event && isCurrentVerificationEvidence(event)) { + return event.provenance.sourceEventId; + } + } + return null; +} + +/** + * Bind verification to receipted work and the latest retained observer frame. + * The stable journal/turn correlation is bound separately by the backend; the + * source lookup below is a fail-closed fallback for nonstandard journals. + */ +export function journalVerificationSources( + journal: MissionJournal, +): JournalVerificationSources { + const correlationId = journalAuthorityCorrelationId(journal); + const receiptedSourceIds = journal.events + .filter((event) => event.proofState === "RECEIPTED") + .map((event) => event.provenance.sourceEventId) + .filter(validSourceEventId); + const correlationSourceId = journal.events.find( + (event) => + event.provenance.triggeringEventIds.includes(correlationId) || + event.toolCallId === correlationId || + event.messageId === correlationId, + )?.provenance.sourceEventId; + const currentSourceEventId = currentVerificationSourceEventId(journal); + const hasCorrelationEvidence = + correlationId === journal.id || validSourceEventId(correlationSourceId); + + const sourceEventIds = [ + ...new Set( + [correlationSourceId, ...receiptedSourceIds, currentSourceEventId].filter( + validSourceEventId, + ), + ), + ].sort(); + return { + sourceEventIds, + hasReceiptedEvidence: receiptedSourceIds.length > 0, + hasCorrelationEvidence, + hasSupportedSourceSet: + sourceEventIds.length <= MAX_JOURNAL_VERIFICATION_SOURCE_EVENTS, + overflowCount: Math.max( + 0, + sourceEventIds.length - MAX_JOURNAL_VERIFICATION_SOURCE_EVENTS, + ), + }; +} + +/** + * Overlay owner authority without rewriting the observed source journal. + * + * The backend verifies artifact ids, signatures, signer identity, relay scope, + * revision ordering, and every cited observer source before returning these + * values. The frontend additionally requires the latest retained evidence to + * be covered, so older verification cannot overwrite later activity. + */ +export function applyValidatedJournalAuthority( + journal: MissionJournal, + artifacts: readonly ValidatedJournalAuthorityArtifact[], + relayUrl: string, + agentPubkey: string, +): MissionJournal { + const relayScope = canonicalRelayUrl(relayUrl); + const agentScope = normalizePubkey(agentPubkey); + if (!relayScope || !/^[0-9a-f]{64}$/.test(agentScope)) return journal; + const matching = artifacts + .filter( + (artifact) => + artifact.relayUrl === relayScope && + artifact.agentPubkey === agentScope && + artifact.journalId === journal.id && + artifact.correlationId === journal.correlationId, + ) + .sort( + (left, right) => + left.revision - right.revision || + left.createdAt - right.createdAt || + left.eventId.localeCompare(right.eventId), + ); + + let result = journal; + const latestOverride = matching + .filter( + (artifact) => + artifact.artifactType === "owner_override" && + typeof artifact.summary === "string" && + artifact.summary.trim().length > 0, + ) + .at(-1); + if (latestOverride?.summary) { + result = applyOwnerJournalOverride(result, { + summary: latestOverride.summary, + modifiedAt: new Date(latestOverride.createdAt * 1_000).toISOString(), + modifiedBy: latestOverride.ownerPubkey, + }); + } + + const currentSourceEventId = currentVerificationSourceEventId(journal); + const latestVerification = matching + .filter( + (artifact) => + artifact.artifactType === "verification" && + Boolean(artifact.receiptRef?.trim()) && + artifact.sourceEventIds.length > 0 && + // The backend revalidates every cited source against this exact + // owner+relay+journal. A bounded Today window may omit the prior-day + // correlation root, so require the latest retained source here rather + // than incorrectly requiring all historical sources to be in memory. + validSourceEventId(currentSourceEventId) && + artifact.sourceEventIds.includes(currentSourceEventId), + ) + .at(-1); + if (!latestVerification) return result; + + // A receipt can verify work observed before it was issued, but it cannot + // erase later terminal evidence. Reapplying authority after a failed or + // stale-incomplete transition must preserve the journal's fail-closed proof. + if (result.status === "failed" || result.status === "incomplete") { + return result; + } + + const timestamp = new Date( + latestVerification.createdAt * 1_000, + ).toISOString(); + let maxSequence = 0; + for (const event of journal.events) { + maxSequence = Math.max(maxSequence, event.provenance.seq); + } + const verificationEvent: NormalizedActivityEvent = { + id: latestVerification.eventId, + journalKey: journal.journalKey, + correlationId: journal.correlationId, + category: "status", + title: "Owner verification", + detail: latestVerification.receiptRef, + status: "completed", + proofState: "VERIFIED", + timestamp, + channelId: journal.channelId, + sessionId: journal.sessionId, + turnId: journal.turnId, + toolCallId: null, + messageId: null, + provenance: { + sourceEventId: latestVerification.eventId, + sourcePubkey: latestVerification.ownerPubkey, + sourceKind: 24201, + sourceCreatedAt: latestVerification.createdAt, + sourceSignature: latestVerification.signature, + origin: "unknown", + observerKind: "owner_verification", + method: null, + sessionUpdate: null, + seq: maxSequence + 1, + timestamp, + channelId: journal.channelId, + sessionId: journal.sessionId, + turnId: journal.turnId, + toolCallId: null, + messageId: null, + triggeringEventIds: latestVerification.sourceEventIds, + }, + tags: ["owner-signed", "receipt-bound"], + ownerModifiedAt: timestamp, + ownerModifiedBy: latestVerification.ownerPubkey, + }; + + return { + ...result, + proofState: "VERIFIED", + status: + result.status === "ended_unverified" || result.status === "observed" + ? "completed" + : result.status, + claimedCompletionWithoutEvidence: false, + endedAt: + Date.parse(timestamp) > Date.parse(result.endedAt) + ? timestamp + : result.endedAt, + eventCount: result.eventCount + 1, + events: [...result.events, verificationEvent], + }; +} diff --git a/desktop/src/features/agents/activityLedgerToday.test.mjs b/desktop/src/features/agents/activityLedgerToday.test.mjs new file mode 100644 index 00000000000..8a4c2579af3 --- /dev/null +++ b/desktop/src/features/agents/activityLedgerToday.test.mjs @@ -0,0 +1,890 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + activityLedgerArchiveQueryRange, + activityLedgerDayRange, + applyAuthorityToTodayActivity, + buildBoundedTodayActivitySurface, + buildTodayActivityFromArchivedEvents, + buildTodayActivityFromArchivedPages, +} from "./activityLedgerToday.ts"; +import { journalVerificationSources } from "./activityLedgerAuthority.ts"; + +function relayEvent({ id, pubkey = "agent-a", agent = pubkey, decoded }) { + return { + id, + pubkey, + created_at: Math.floor(Date.parse(decoded.timestamp) / 1000), + kind: 24200, + tags: [["agent", agent]], + content: "encrypted", + sig: `sig-${id}`, + decoded, + }; +} + +test("Today reconstruction trusts only managed self-authored observer frames", async () => { + const timestamp = "2026-08-21T14:00:00.000Z"; + const valid = relayEvent({ + id: "valid", + decoded: { + seq: 1, + timestamp, + kind: "turn_started", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: { triggeringEventIds: ["message-1"] }, + }, + }); + const forged = relayEvent({ + id: "forged", + pubkey: "attacker", + agent: "agent-a", + decoded: { ...valid.decoded, seq: 2 }, + }); + const unknown = relayEvent({ + id: "unknown", + pubkey: "agent-z", + decoded: { ...valid.decoded, seq: 3 }, + }); + + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [valid, forged, unknown], + decrypt: async (event) => event.decoded, + }); + + assert.equal(surface.counts.journals, 1); + assert.equal(surface.journals[0].agentName, "Honey"); + assert.equal(surface.journals[0].events.length, 1); + assert.deepEqual(surface.journals[0].events[0].provenance, { + ...surface.journals[0].events[0].provenance, + sourceEventId: "valid", + sourcePubkey: "agent-a", + sourceKind: 24200, + sourceCreatedAt: valid.created_at, + sourceSignature: "sig-valid", + origin: "historical_backfill", + }); +}); + +test("Today reconstruction skips decrypt failures without admitting bad proof", async () => { + const event = relayEvent({ + id: "bad-ciphertext", + decoded: { + seq: 1, + timestamp: "2026-08-21T14:00:00.000Z", + kind: "turn_started", + payload: {}, + }, + }); + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [event], + decrypt: async () => { + throw new Error("decrypt failed"); + }, + }); + assert.equal(surface.counts.journals, 0); + assert.equal(surface.snapshotProjection.excludedObserverFrames, 1); + assert.equal(surface.snapshotProjection.bounded, true); +}); + +test("Today reconstruction discloses frames excluded before range paging", async () => { + const surface = await buildTodayActivityFromArchivedPages({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + pages: [{ events: [], unindexedObserverFrames: 2 }], + }); + + assert.equal(surface.counts.journals, 0); + assert.equal(surface.snapshotProjection.excludedObserverFrames, 2); + assert.equal(surface.snapshotProjection.unindexedObserverFrames, 2); + assert.equal(surface.snapshotProjection.bounded, true); +}); + +test("Today reconstruction discloses malformed rows and sustained-ingest fallback omissions", async () => { + const surface = await buildTodayActivityFromArchivedPages({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + pages: [ + { + events: [], + unindexedObserverFrames: 0, + rejectedArchiveRows: 1, + omittedObserverFrames: 8, + archiveRevision: 12, + }, + ], + }); + + assert.equal(surface.snapshotProjection.excludedObserverFrames, 9); + assert.equal(surface.snapshotProjection.malformedArchivedRows, 1); + assert.equal(surface.snapshotProjection.omittedObserverFrames, 8); + assert.equal(surface.snapshotProjection.archiveRevision, 12); + assert.equal(surface.snapshotProjection.bounded, true); +}); + +test("Today reconstruction is bounded by signed pre-archive source loss", async () => { + const gap = relayEvent({ + id: "telemetry-gap", + decoded: { + seq: 10, + timestamp: "2026-08-21T14:00:00.000Z", + kind: "observer_telemetry_gap", + agentIndex: 0, + channelId: "channel-1", + sessionId: null, + turnId: null, + payload: { + droppedEvents: 4, + reasonCounts: { publishQueueEviction: 4 }, + }, + }, + }); + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [gap], + decrypt: async (event) => event.decoded, + }); + + assert.equal(surface.counts.journals, 0); + assert.equal(surface.snapshotProjection.sourceDroppedObserverEvents, 4); + assert.equal(surface.snapshotProjection.bounded, true); +}); + +test("Today reconstruction reports malformed or empty observer batches", async () => { + const event = relayEvent({ + id: "empty-batch", + decoded: { + seq: 1, + timestamp: "2026-08-21T14:00:00.000Z", + kind: "batch", + payload: { events: [{ malformed: true }] }, + }, + }); + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [event], + decrypt: async (candidate) => candidate.decoded, + }); + assert.equal(surface.counts.journals, 0); + assert.equal(surface.snapshotProjection.excludedObserverFrames, 1); +}); + +test("Today reconstruction counts malformed members of a partially valid batch", async () => { + const timestamp = "2026-08-21T14:00:00.000Z"; + const event = relayEvent({ + id: "partial-batch", + decoded: { + seq: 2, + timestamp, + kind: "batch", + payload: { + events: [ + { + seq: 1, + timestamp, + kind: "turn_started", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: { triggeringEventIds: ["message-1"] }, + }, + { kind: "turn_error", malformed: true }, + ], + }, + }, + }); + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [event], + decrypt: async (candidate) => candidate.decoded, + }); + + assert.equal(surface.counts.journals, 1); + assert.equal(surface.snapshotProjection.excludedObserverFrames, 1); + assert.equal(surface.snapshotProjection.bounded, true); +}); + +test("Today reconstruction counts batch members with invalid timestamps", async () => { + const timestamp = "2026-08-21T14:00:00.000Z"; + const event = relayEvent({ + id: "invalid-time-batch", + decoded: { + seq: 3, + timestamp, + kind: "batch", + payload: { + events: [ + { + seq: 1, + timestamp, + kind: "turn_started", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-invalid-time", + payload: {}, + }, + { + seq: 2, + timestamp: "2026-02-30T14:00:00Z", + kind: "turn_error", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-invalid-time", + payload: {}, + }, + ], + }, + }, + }); + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [event], + decrypt: async (candidate) => candidate.decoded, + }); + + assert.equal(surface.counts.journals, 1); + assert.equal(surface.journals[0].status, "incomplete"); + assert.equal(surface.snapshotProjection.excludedObserverFrames, 1); + assert.equal(surface.snapshotProjection.bounded, true); +}); + +test("Today reconstruction expands every inner event from one signed batch", async () => { + const timestamp = "2026-08-21T14:00:00.000Z"; + const batch = relayEvent({ + id: "batch-frame", + decoded: { + seq: 99, + timestamp, + kind: "batch", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: { + events: [ + { + seq: 1, + timestamp, + kind: "turn_started", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: { triggeringEventIds: ["message-1"] }, + }, + { + seq: 2, + timestamp, + kind: "turn_completed", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: {}, + }, + ], + }, + }, + }); + + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [batch], + decrypt: async (event) => event.decoded, + }); + + assert.equal(surface.counts.journals, 1); + assert.equal(surface.journals[0].events.length, 2); + assert.deepEqual( + surface.journals[0].events.map((event) => event.provenance.sourceEventId), + ["batch-frame", "batch-frame"], + ); +}); + +test("Today reconstruction discards earlier pages after an archive restart", async () => { + const event = (id, turnId) => + relayEvent({ + id, + decoded: { + seq: 1, + timestamp: "2026-08-21T14:00:00.000Z", + kind: "turn_started", + turnId, + payload: {}, + }, + }); + const oldEvent = event("old-frame", "old-turn"); + const currentEvent = event("current-frame", "current-turn"); + const surface = await buildTodayActivityFromArchivedPages({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + pages: [ + { + events: [oldEvent], + unindexedObserverFrames: 1, + archiveRevision: 7, + }, + { + events: [], + unindexedObserverFrames: 0, + archiveRevision: 8, + reset: true, + }, + { + events: [currentEvent], + unindexedObserverFrames: 0, + archiveRevision: 8, + }, + ], + decrypt: async (candidate) => candidate.decoded, + }); + + assert.deepEqual( + surface.journals.map((journal) => journal.journalKey), + ["current-turn"], + ); + assert.equal(surface.snapshotProjection.unindexedObserverFrames, 0); + assert.equal(surface.snapshotProjection.archiveRevision, 8); +}); + +test("Today archive decryption is concurrency-bounded and page-incremental", async () => { + let active = 0; + let maxActive = 0; + let resumedAfterFirstPage = false; + const makePage = (offset) => + Array.from({ length: 20 }, (_, index) => + relayEvent({ + id: `frame-${offset + index}`, + decoded: { + seq: offset + index + 1, + timestamp: `2026-08-21T14:${String(offset + index).padStart(2, "0")}:00.000Z`, + kind: "turn_started", + turnId: `turn-${offset + index}`, + payload: {}, + }, + }), + ); + async function* pages() { + yield makePage(0); + assert.equal( + active, + 0, + "the next raw page was requested before decrypt drained", + ); + resumedAfterFirstPage = true; + yield makePage(20); + } + + const surface = await buildTodayActivityFromArchivedPages({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + pages: pages(), + decrypt: async (event) => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((resolve) => setTimeout(resolve, 1)); + active -= 1; + return event.decoded; + }, + }); + + assert.equal(resumedAfterFirstPage, true); + assert.equal(maxActive, 8); + assert.equal(surface.counts.journals, 40); +}); + +test("Today archive reconstruction bounds decoded history between pages", async () => { + const totalJournals = 1_000; + const pageSize = 100; + const largeClaim = "x".repeat(16 * 1024); + async function* pages() { + for (let offset = 0; offset < totalJournals; offset += pageSize) { + yield Array.from({ length: pageSize }, (_, pageIndex) => { + const index = offset + pageIndex; + const timestamp = new Date( + Date.parse("2026-08-21T14:00:00.000Z") + index * 1_000, + ).toISOString(); + return relayEvent({ + id: `large-frame-${index}`, + decoded: { + seq: index + 1, + timestamp, + kind: "acp_read", + turnId: `large-turn-${index}`, + payload: { + method: "session/update", + params: { + update: { + sessionUpdate: "agent_message_chunk", + content: { text: largeClaim }, + }, + }, + }, + }, + }); + }); + } + } + + const surface = await buildTodayActivityFromArchivedPages({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + pages: pages(), + decrypt: async (event) => event.decoded, + }); + + const encodedBytes = new TextEncoder().encode(JSON.stringify(surface)); + assert.ok(encodedBytes.byteLength <= 6 * 1024 * 1024); + assert.equal(surface.snapshotProjection.originalJournals, totalJournals); + assert.ok(surface.snapshotProjection.omittedJournals > 0); + assert.ok(surface.journals.length < totalJournals); + assert.equal(surface.journals.at(-1).id, "large-turn-999"); +}); + +test("Today archive checkpoints preserve long-journal verification sources", async () => { + const timestamp = (seq) => + new Date( + Date.parse("2026-08-21T14:00:00.000Z") + seq * 1_000, + ).toISOString(); + const decoded = [ + { + seq: 1, + timestamp: timestamp(1), + kind: "turn_started", + turnId: "long-turn", + payload: { triggeringEventIds: ["message-root"] }, + }, + { + seq: 2, + timestamp: timestamp(2), + kind: "acp_read", + turnId: "long-turn", + payload: { + method: "session/update", + params: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: "early-tool", + title: "write_file", + status: "completed", + rawOutput: "written", + }, + }, + }, + }, + ...Array.from({ length: 400 }, (_, index) => ({ + seq: index + 3, + timestamp: timestamp(index + 3), + kind: "acp_read", + turnId: "long-turn", + payload: { + method: "session/update", + params: { + update: { + sessionUpdate: "agent_message_chunk", + content: { text: `claim-${index}` }, + }, + }, + }, + })), + { + seq: 403, + timestamp: timestamp(403), + kind: "turn_completed", + turnId: "long-turn", + payload: {}, + }, + ]; + const archivedNewestFirst = decoded + .map((event) => + relayEvent({ + id: event.seq.toString(16).padStart(64, "0"), + decoded: event, + }), + ) + .reverse(); + async function* pages() { + for (let index = 0; index < archivedNewestFirst.length; index += 75) { + yield archivedNewestFirst.slice(index, index + 75); + } + } + + const surface = await buildTodayActivityFromArchivedPages({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + pages: pages(), + decrypt: async (event) => event.decoded, + }); + const journal = surface.journals[0]; + const sources = journalVerificationSources(journal); + + assert.equal(journal.eventCount, decoded.length); + assert.equal(journal.proofState, "RECEIPTED"); + assert.equal(sources.hasReceiptedEvidence, true); + assert.equal(sources.hasCorrelationEvidence, true); + assert.ok(sources.sourceEventIds.includes("2".padStart(64, "0"))); +}); + +test("Today archive checkpoints reserve the newest signed frame before receipts", async () => { + const timestamp = (seq) => + new Date( + Date.parse("2026-08-21T14:00:00.000Z") + seq * 1_000, + ).toISOString(); + const decoded = [ + { + seq: 1, + timestamp: timestamp(1), + kind: "turn_started", + turnId: "receipt-heavy-turn", + payload: {}, + }, + ...Array.from({ length: 300 }, (_, index) => ({ + seq: index + 2, + timestamp: timestamp(index + 2), + kind: "acp_read", + turnId: "receipt-heavy-turn", + payload: { + method: "session/update", + params: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: `tool-${index}`, + title: "write_file", + status: "completed", + rawOutput: "written", + }, + }, + }, + })), + { + seq: 302, + timestamp: timestamp(302), + kind: "acp_read", + turnId: "receipt-heavy-turn", + payload: { + method: "session/update", + params: { + update: { + sessionUpdate: "agent_message_chunk", + content: { text: "later journal activity" }, + }, + }, + }, + }, + ]; + const latestSourceId = decoded.at(-1).seq.toString(16).padStart(64, "0"); + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: decoded.map((event) => + relayEvent({ + id: event.seq.toString(16).padStart(64, "0"), + decoded: event, + }), + ), + decrypt: async (event) => event.decoded, + }); + const journal = surface.journals[0]; + + assert.equal(journal.eventCount, decoded.length); + assert.equal(journal.events.length, 300); + assert.equal(journal.events.at(-1).provenance.sourceEventId, latestSourceId); + assert.ok( + journalVerificationSources(journal).sourceEventIds.includes(latestSourceId), + ); +}); + +test("day range is half-open and rejects impossible dates", () => { + const range = activityLedgerDayRange("2026-08-21"); + assert.equal(range.endCreatedAt - range.startCreatedAt, 24 * 60 * 60); + assert.throws(() => activityLedgerDayRange("2026-02-30")); + assert.throws(() => activityLedgerDayRange("08/21/2026")); +}); + +test("Today archive query uses the authoritative inner-time day range", async () => { + const day = "2026-08-21"; + const exact = activityLedgerDayRange(day); + const query = activityLedgerArchiveQueryRange(day); + assert.deepEqual(query, exact); + + const innerTimestamp = new Date( + (exact.endCreatedAt - 1) * 1_000, + ).toISOString(); + const crossMidnightEnvelope = relayEvent({ + id: "cross-midnight", + decoded: { + seq: 1, + timestamp: innerTimestamp, + kind: "turn_started", + turnId: "midnight-turn", + payload: {}, + }, + }); + crossMidnightEnvelope.created_at = exact.endCreatedAt + 1; + + const previousDay = await buildTodayActivityFromArchivedEvents({ + day, + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [crossMidnightEnvelope], + decrypt: async (event) => event.decoded, + }); + assert.equal(previousDay.counts.journals, 1); + + const nextDay = new Date(exact.endCreatedAt * 1_000); + const nextDayKey = `${nextDay.getFullYear()}-${String(nextDay.getMonth() + 1).padStart(2, "0")}-${String(nextDay.getDate()).padStart(2, "0")}`; + const followingDay = await buildTodayActivityFromArchivedEvents({ + day: nextDayKey, + agents: [{ pubkey: "agent-a", name: "Honey" }], + events: [crossMidnightEnvelope], + decrypt: async (event) => event.decoded, + }); + assert.equal(followingDay.counts.journals, 0); +}); + +test("Today authority overlay recomputes evidence-gap counts", async () => { + const agentPubkey = "1".repeat(64); + const event = relayEvent({ + id: "a".repeat(64), + pubkey: agentPubkey, + decoded: { + seq: 1, + timestamp: "2026-08-21T14:00:00.000Z", + kind: "turn_started", + agentIndex: 0, + channelId: "channel-1", + sessionId: "session-1", + turnId: "turn-1", + payload: {}, + }, + }); + const ended = relayEvent({ + id: "b".repeat(64), + pubkey: agentPubkey, + decoded: { ...event.decoded, seq: 2, kind: "turn_completed" }, + }); + const surface = await buildTodayActivityFromArchivedEvents({ + day: "2026-08-21", + agents: [{ pubkey: agentPubkey, name: "Honey" }], + events: [event, ended], + decrypt: async (candidate) => candidate.decoded, + }); + const journal = surface.journals[0]; + const updated = applyAuthorityToTodayActivity( + surface, + [ + { + ownerPubkey: "owner-a", + relayUrl: "wss://relay.example", + agentPubkey, + eventId: "c".repeat(64), + signature: "owner-signature", + createdAt: ended.created_at + 1, + artifactType: "verification", + journalId: journal.id, + correlationId: journal.correlationId, + revision: 1, + summary: null, + note: null, + receiptRef: "receipt:owner-check", + sourceEventIds: [event.id, ended.id].sort(), + }, + ], + "wss://relay.example", + ); + + assert.equal(updated.journals[0].proofState, "VERIFIED"); + assert.equal(updated.counts.claimedWithoutEvidence, 0); + assert.equal(updated.channels[0].lastActivityAt, "2026-08-21T14:00:01.000Z"); +}); + +function snapshotJournal(id, minute, detail = null) { + const timestamp = `2026-08-21T14:${String(minute).padStart(2, "0")}:00.000Z`; + const event = { + id: `event-${id}`, + journalKey: id, + correlationId: `message-${id}`, + category: "tool", + title: "write_file", + detail, + status: "completed", + proofState: "RECEIPTED", + timestamp, + channelId: "channel-1", + sessionId: "session-1", + turnId: id, + toolCallId: `tool-${id}`, + messageId: null, + provenance: { + sourceEventId: `source-${id}`, + sourcePubkey: "agent-a", + sourceKind: 24200, + sourceCreatedAt: Math.floor(Date.parse(timestamp) / 1_000), + sourceSignature: "a".repeat(128), + origin: "historical_backfill", + observerKind: "acp_read", + method: "session/update", + sessionUpdate: "tool_call_update", + seq: minute, + timestamp, + channelId: "channel-1", + sessionId: "session-1", + turnId: id, + toolCallId: `tool-${id}`, + messageId: null, + triggeringEventIds: [`message-${id}`], + }, + tags: ["tool", "tool:write_file"], + }; + return { + id, + journalKey: id, + correlationId: `message-${id}`, + channelId: "channel-1", + sessionId: "session-1", + turnId: id, + startedAt: timestamp, + endedAt: timestamp, + status: "completed", + proofState: "RECEIPTED", + summary: `Completed ${id}`, + summarySource: "auto", + ownerModifiedAt: null, + ownerModifiedBy: null, + claimedCompletionWithoutEvidence: false, + eventCount: 1, + events: [event], + agentPubkey: "agent-a", + agentName: "Honey", + }; +} + +test("Today snapshot projection bounds oversized tool output", () => { + const journal = snapshotJournal( + "turn-large", + 1, + "x".repeat(10 * 1024 * 1024), + ); + const surface = { + day: "2026-08-21", + journals: [journal], + channels: [], + counts: { + journals: 1, + failed: 0, + inProgress: 0, + claimedWithoutEvidence: 0, + }, + }; + const maxBytes = 32 * 1024; + const bounded = buildBoundedTodayActivitySurface(surface, maxBytes); + + assert.ok( + new TextEncoder().encode(JSON.stringify(bounded)).byteLength <= maxBytes, + ); + assert.equal(bounded.journals.length, 1); + assert.equal(bounded.snapshotProjection.bounded, true); + assert.equal(bounded.snapshotProjection.textFieldsTruncated, 1); + assert.ok(bounded.journals[0].events[0].detail.length < 10 * 1024 * 1024); +}); + +test("Today snapshot projection drops oldest journals only as a final fallback", () => { + const journals = Array.from({ length: 10 }, (_, index) => + snapshotJournal(`turn-${index}`, index), + ); + const surface = { + day: "2026-08-21", + journals, + channels: [], + counts: { + journals: journals.length, + failed: 0, + inProgress: 0, + claimedWithoutEvidence: 0, + }, + }; + const maxBytes = 2_500; + const bounded = buildBoundedTodayActivitySurface(surface, maxBytes); + + assert.ok( + new TextEncoder().encode(JSON.stringify(bounded)).byteLength <= maxBytes, + ); + assert.ok(bounded.journals.length > 0); + assert.ok(bounded.journals.length < journals.length); + assert.equal(bounded.journals.at(-1).id, "turn-9"); + assert.deepEqual( + bounded.journals.map((journal) => journal.id), + journals.slice(-bounded.journals.length).map((journal) => journal.id), + ); + assert.equal( + bounded.snapshotProjection.omittedJournals, + journals.length - bounded.journals.length, + ); +}); + +test("Today snapshot compaction retains owner verification before receipt overflow", () => { + const base = snapshotJournal("verified-long", 1); + const receipts = Array.from({ length: 150 }, (_, index) => ({ + ...base.events[0], + id: `receipt-${index}`, + proofState: "VERIFIED", + provenance: { + ...base.events[0].provenance, + sourceEventId: index.toString(16).padStart(64, "0"), + seq: index + 1, + }, + })); + const verification = { + ...base.events[0], + id: "owner-verification", + title: "Owner verification", + proofState: "VERIFIED", + provenance: { + ...base.events[0].provenance, + sourceEventId: "f".repeat(64), + sourceKind: 24201, + observerKind: "owner_verification", + seq: 151, + }, + }; + const journal = { + ...base, + proofState: "VERIFIED", + eventCount: receipts.length + 1, + events: [...receipts, verification], + }; + const bounded = buildBoundedTodayActivitySurface({ + day: "2026-08-21", + journals: [journal], + channels: [], + counts: { + journals: 1, + failed: 0, + inProgress: 0, + claimedWithoutEvidence: 0, + }, + }); + + assert.equal(bounded.journals[0].events.length, 100); + assert.equal( + bounded.journals[0].events.some( + (event) => event.provenance.observerKind === "owner_verification", + ), + true, + ); +}); diff --git a/desktop/src/features/agents/activityLedgerToday.ts b/desktop/src/features/agents/activityLedgerToday.ts new file mode 100644 index 00000000000..4f095a1bbd1 --- /dev/null +++ b/desktop/src/features/agents/activityLedgerToday.ts @@ -0,0 +1,887 @@ +import type { RelayEvent } from "@/shared/api/types"; +import { decryptObserverEvent } from "@/shared/api/tauriObserver"; +import { applyValidatedJournalAuthority } from "./activityLedgerAuthority"; +import type { ValidatedJournalAuthorityArtifact } from "./activityLedgerAuthority"; +import { + buildTodayActivitySurface, + normalizeActivityEvents, + type NormalizedActivityEvent, + type TodayActivityJournal, + type TodayActivitySurface, +} from "./activityLedger"; +import type { ObserverEvent } from "./ui/agentSessionTypes"; + +export type ActivityLedgerAgentIdentity = { + pubkey: string; + name: string; +}; + +export type ArchivedObserverEventPage = { + events: readonly RelayEvent[]; + /** Frames already excluded because no trustworthy inner time was indexable. */ + unindexedObserverFrames: number; + rejectedArchiveRows?: number; + omittedObserverFrames?: number; + archiveRevision?: number; + reset?: boolean; +}; + +export const TODAY_SNAPSHOT_SURFACE_MAX_BYTES = 6 * 1024 * 1024; +const TODAY_SNAPSHOT_MAX_EVENTS_PER_JOURNAL = 100; +const TODAY_ARCHIVE_MAX_EVENTS_PER_JOURNAL = 300; +const TODAY_SNAPSHOT_MAX_SUMMARY_CHARS = 4_096; +const TODAY_SNAPSHOT_MAX_EVENT_DETAIL_CHARS = 8_192; + +export type TodaySnapshotProjection = { + bounded: boolean; + maxBytes: number; + originalJournals: number; + includedJournals: number; + omittedJournals: number; + omittedEvents: number; + excludedObserverFrames: number; + sourceDroppedObserverEvents: number; + unindexedObserverFrames: number; + malformedArchivedRows: number; + omittedObserverFrames: number; + archiveRevision: number; + archiveRevisionAtPublish: number; + archiveRevisionDrift: number; + truthInvalidatedByArchiveDrift: boolean; + textFieldsTruncated: number; +}; + +export type BoundedTodayActivitySurface = TodayActivitySurface & { + snapshotProjection: TodaySnapshotProjection; +}; + +function truncateSnapshotText(value: string, maxChars: number) { + if (value.length <= maxChars) return { value, truncated: false }; + return { value: `${value.slice(0, maxChars - 1)}…`, truncated: true }; +} + +function selectSnapshotEvents( + journal: TodayActivityJournal, + maxEvents: number, +) { + if (journal.events.length <= maxEvents) return journal.events; + const selected = new Set(); + const add = (index: number | undefined) => { + if ( + index !== undefined && + index >= 0 && + index < journal.events.length && + selected.size < maxEvents + ) { + selected.add(index); + } + }; + const addMatching = ( + predicate: (event: NormalizedActivityEvent) => boolean, + ) => { + for (let index = 0; index < journal.events.length; index += 1) { + if (predicate(journal.events[index])) add(index); + } + }; + + // Verification sources are capped at 256 in the owner-authority contract. + // Reserve the actual newest signed journal frame before any bulk priority + // class can fill the checkpoint. Every later frame changes verification + // freshness, including messages, prompts, thoughts, and plans. + add(0); + let latestVerificationEvidence = -1; + for (let index = journal.events.length - 1; index >= 0; index -= 1) { + const event = journal.events[index]; + if ( + event.provenance.sourceKind !== 24201 && + event.provenance.observerKind !== "owner_verification" && + typeof event.provenance.sourceEventId === "string" && + /^[0-9a-f]{64}$/i.test(event.provenance.sourceEventId) + ) { + latestVerificationEvidence = index; + break; + } + } + add(latestVerificationEvidence); + + // Preserve verification sources, their correlation root, and terminal + // truth before spending the remaining budget on transcript detail. + addMatching( + (event) => + event.provenance.observerKind === "owner_verification" || + event.provenance.sourceKind === 24201, + ); + addMatching((event) => event.proofState === "VERIFIED"); + add( + journal.events.findIndex( + (event) => + event.provenance.triggeringEventIds.includes(journal.correlationId) || + event.toolCallId === journal.correlationId || + event.messageId === journal.correlationId, + ), + ); + addMatching((event) => event.proofState === "FAILED"); + + addMatching((event) => event.category === "turn"); + addMatching((event) => event.proofState === "RECEIPTED"); + + const latestToolEvents = new Map(); + for (let index = 0; index < journal.events.length; index += 1) { + const event = journal.events[index]; + if (event.category === "tool") { + latestToolEvents.set(event.toolCallId ?? event.correlationId, index); + } + } + for (const index of latestToolEvents.values()) add(index); + + for ( + let index = journal.events.length - 1; + index >= 0 && selected.size < maxEvents; + index -= 1 + ) { + add(index); + } + return [...selected] + .sort((left, right) => left - right) + .map((index) => journal.events[index]); +} + +function compactSnapshotJournal( + journal: TodayActivityJournal, + maxEvents = TODAY_SNAPSHOT_MAX_EVENTS_PER_JOURNAL, +): { + journal: TodayActivityJournal; + textFieldsTruncated: number; +} { + const summary = truncateSnapshotText( + journal.summary, + TODAY_SNAPSHOT_MAX_SUMMARY_CHARS, + ); + const selectedEvents = selectSnapshotEvents(journal, maxEvents); + let textFieldsTruncated = summary.truncated ? 1 : 0; + const events = selectedEvents.map((event) => { + if (event.detail === null) return event; + const detail = truncateSnapshotText( + event.detail, + TODAY_SNAPSHOT_MAX_EVENT_DETAIL_CHARS, + ); + if (detail.truncated) textFieldsTruncated += 1; + return detail.truncated ? { ...event, detail: detail.value } : event; + }); + return { + journal: { + ...journal, + summary: summary.value, + events, + }, + textFieldsTruncated, + }; +} + +function snapshotSurfaceFromJournals(input: { + day: string; + journals: TodayActivityJournal[]; + maxBytes: number; + originalJournalCount: number; + originalEventCount: number; + excludedObserverFrames: number; + sourceDroppedObserverEvents: number; + unindexedObserverFrames: number; + malformedArchivedRows: number; + omittedObserverFrames: number; + archiveRevision: number; + textFieldsTruncated: number; +}): BoundedTodayActivitySurface { + const channels = new Map< + string, + { + journalIds: string[]; + agentPubkeys: Set; + agentNames: Set; + lastActivityAt: string; + } + >(); + for (const journal of input.journals) { + if (!journal.channelId) continue; + const bucket = channels.get(journal.channelId) ?? { + journalIds: [], + agentPubkeys: new Set(), + agentNames: new Set(), + lastActivityAt: journal.endedAt, + }; + bucket.journalIds.push(journal.id); + bucket.agentPubkeys.add(journal.agentPubkey); + bucket.agentNames.add(journal.agentName); + if (Date.parse(journal.endedAt) > Date.parse(bucket.lastActivityAt)) { + bucket.lastActivityAt = journal.endedAt; + } + channels.set(journal.channelId, bucket); + } + const includedEventCount = input.journals.reduce( + (count, journal) => count + journal.events.length, + 0, + ); + const omittedJournals = Math.max( + 0, + input.originalJournalCount - input.journals.length, + ); + const omittedEvents = Math.max( + 0, + input.originalEventCount - includedEventCount, + ); + return { + day: input.day, + journals: input.journals, + channels: [...channels.entries()] + .map(([channelId, bucket]) => ({ + channelId, + journalIds: bucket.journalIds, + agentPubkeys: [...bucket.agentPubkeys], + agentNames: [...bucket.agentNames], + lastActivityAt: bucket.lastActivityAt, + })) + .sort((left, right) => left.channelId.localeCompare(right.channelId)), + counts: { + journals: input.journals.length, + failed: input.journals.filter((journal) => journal.status === "failed") + .length, + inProgress: input.journals.filter( + (journal) => journal.status === "in_progress", + ).length, + claimedWithoutEvidence: input.journals.filter( + (journal) => journal.claimedCompletionWithoutEvidence, + ).length, + }, + snapshotProjection: { + bounded: + omittedJournals > 0 || + omittedEvents > 0 || + input.excludedObserverFrames > 0 || + input.sourceDroppedObserverEvents > 0 || + input.textFieldsTruncated > 0, + maxBytes: input.maxBytes, + originalJournals: input.originalJournalCount, + includedJournals: input.journals.length, + omittedJournals, + omittedEvents, + excludedObserverFrames: input.excludedObserverFrames, + sourceDroppedObserverEvents: input.sourceDroppedObserverEvents, + unindexedObserverFrames: input.unindexedObserverFrames, + malformedArchivedRows: input.malformedArchivedRows, + omittedObserverFrames: input.omittedObserverFrames, + archiveRevision: input.archiveRevision, + archiveRevisionAtPublish: input.archiveRevision, + archiveRevisionDrift: 0, + truthInvalidatedByArchiveDrift: false, + textFieldsTruncated: input.textFieldsTruncated, + }, + }; +} + +function snapshotSurfaceByteLength(surface: BoundedTodayActivitySurface) { + return new TextEncoder().encode(JSON.stringify(surface)).byteLength; +} + +/** + * Bound the signed Today projection without letting one large tool output + * suppress the entire feed. Summaries/details are capped first, then event + * bodies are removed, and only as a final fallback are the oldest journals + * omitted. The newest retained journals stay in chronological order. + */ +export function buildBoundedTodayActivitySurface( + surface: TodayActivitySurface, + maxBytes = TODAY_SNAPSHOT_SURFACE_MAX_BYTES, +): BoundedTodayActivitySurface { + const previousProjection = ( + surface as TodayActivitySurface & { + snapshotProjection?: TodaySnapshotProjection; + } + ).snapshotProjection; + const ordered = [...surface.journals].sort( + (left, right) => + Date.parse(left.startedAt) - Date.parse(right.startedAt) || + left.id.localeCompare(right.id), + ); + const compacted = ordered.map((journal) => compactSnapshotJournal(journal)); + const retainedOriginalEventCount = ordered.reduce( + (count, journal) => + count + Math.max(journal.eventCount, journal.events.length), + 0, + ); + const originalJournalCount = Math.max( + previousProjection?.originalJournals ?? 0, + ordered.length, + ); + const originalEventCount = Math.max( + previousProjection + ? previousProjection.omittedEvents + + ordered.reduce((count, journal) => count + journal.events.length, 0) + : 0, + retainedOriginalEventCount, + ); + const build = ( + journals: TodayActivityJournal[], + textFieldsTruncated: number, + ) => + snapshotSurfaceFromJournals({ + day: surface.day, + journals, + maxBytes, + originalJournalCount, + originalEventCount, + excludedObserverFrames: previousProjection?.excludedObserverFrames ?? 0, + sourceDroppedObserverEvents: + previousProjection?.sourceDroppedObserverEvents ?? 0, + unindexedObserverFrames: previousProjection?.unindexedObserverFrames ?? 0, + malformedArchivedRows: previousProjection?.malformedArchivedRows ?? 0, + omittedObserverFrames: previousProjection?.omittedObserverFrames ?? 0, + archiveRevision: previousProjection?.archiveRevision ?? 0, + textFieldsTruncated, + }); + const compactedTextCount = + (previousProjection?.textFieldsTruncated ?? 0) + + compacted.reduce((count, item) => count + item.textFieldsTruncated, 0); + let candidate = build( + compacted.map((item) => item.journal), + compactedTextCount, + ); + if (snapshotSurfaceByteLength(candidate) <= maxBytes) return candidate; + + const statusOnly = compacted.map((item) => ({ + ...item.journal, + events: [], + })); + candidate = build(statusOnly, compactedTextCount); + if (snapshotSurfaceByteLength(candidate) <= maxBytes) return candidate; + + let low = 0; + let high = statusOnly.length; + let best = build([], 0); + if (snapshotSurfaceByteLength(best) > maxBytes) { + throw new Error("Activity Ledger Today snapshot budget is too small."); + } + while (low <= high) { + const retained = Math.floor((low + high) / 2); + const retainedJournals = statusOnly.slice(statusOnly.length - retained); + const retainedTextCount = compacted + .slice(compacted.length - retained) + .reduce((count, item) => count + item.textFieldsTruncated, 0); + const attempt = build(retainedJournals, retainedTextCount); + if (snapshotSurfaceByteLength(attempt) <= maxBytes) { + best = attempt; + low = retained + 1; + } else { + high = retained - 1; + } + } + return best; +} + +type DecryptObserverEvent = (event: RelayEvent) => Promise; +const TODAY_ARCHIVE_DECRYPT_CONCURRENCY = 8; + +function localDayForTimestamp(timestamp: string) { + const date = new Date(timestamp); + const year = date.getFullYear(); + const month = `${date.getMonth() + 1}`.padStart(2, "0"); + const day = `${date.getDate()}`.padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +function journalProjectionKey(agentPubkey: string, journalKey: string) { + return `${agentPubkey}\u0000${journalKey}`; +} + +/** + * Keep only a bounded, reconstructable working set between archive pages. + * Unlike the final snapshot projection this never strips every event from a + * retained journal: older pages may still contain its turn start and + * correlation root. When the budget is exceeded, the oldest whole journals + * are omitted and the omission remains explicit in snapshotProjection. + */ +function buildArchiveReconstructionCheckpoint(input: { + surface: TodayActivitySurface; + originalJournalCount: number; + originalEventCount: number; + excludedObserverFrames: number; + sourceDroppedObserverEvents: number; + unindexedObserverFrames: number; + malformedArchivedRows: number; + omittedObserverFrames: number; + archiveRevision: number; + previousTextFieldsTruncated: number; +}): BoundedTodayActivitySurface { + const ordered = [...input.surface.journals].sort( + (left, right) => + Date.parse(left.startedAt) - Date.parse(right.startedAt) || + left.id.localeCompare(right.id), + ); + const compacted = ordered.map((journal) => + compactSnapshotJournal(journal, TODAY_ARCHIVE_MAX_EVENTS_PER_JOURNAL), + ); + const newlyTruncated = compacted.reduce( + (count, item) => count + item.textFieldsTruncated, + 0, + ); + const textFieldsTruncated = + input.previousTextFieldsTruncated + newlyTruncated; + const build = (journals: TodayActivityJournal[]) => + snapshotSurfaceFromJournals({ + day: input.surface.day, + journals, + maxBytes: TODAY_SNAPSHOT_SURFACE_MAX_BYTES, + originalJournalCount: input.originalJournalCount, + originalEventCount: input.originalEventCount, + excludedObserverFrames: input.excludedObserverFrames, + sourceDroppedObserverEvents: input.sourceDroppedObserverEvents, + unindexedObserverFrames: input.unindexedObserverFrames, + malformedArchivedRows: input.malformedArchivedRows, + omittedObserverFrames: input.omittedObserverFrames, + archiveRevision: input.archiveRevision, + textFieldsTruncated, + }); + const all = compacted.map((item) => item.journal); + const candidate = build(all); + if ( + snapshotSurfaceByteLength(candidate) <= TODAY_SNAPSHOT_SURFACE_MAX_BYTES + ) { + return candidate; + } + + let low = 0; + let high = all.length; + let best = build([]); + while (low <= high) { + const retained = Math.floor((low + high) / 2); + const attempt = build(all.slice(all.length - retained)); + if ( + snapshotSurfaceByteLength(attempt) <= TODAY_SNAPSHOT_SURFACE_MAX_BYTES + ) { + best = attempt; + low = retained + 1; + } else { + high = retained - 1; + } + } + return best; +} + +function observerAgentPubkey(event: RelayEvent): string | null { + const tag = event.tags.find( + (candidate) => candidate[0] === "agent" && candidate[1]?.length > 0, + ); + return tag?.[1] ?? null; +} + +function isRfc3339Timestamp(value: string): boolean { + const match = + /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-](\d{2}):(\d{2}))$/.exec( + value, + ); + if (!match || !Number.isFinite(Date.parse(value))) return false; + const [, yearText, monthText, dayText, hourText, minuteText, secondText] = + match; + const year = Number(yearText); + const month = Number(monthText); + const day = Number(dayText); + const offsetHour = Number(match[7] ?? 0); + const offsetMinute = Number(match[8] ?? 0); + const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const daysInMonth = [ + 31, + leapYear ? 29 : 28, + 31, + 30, + 31, + 30, + 31, + 31, + 30, + 31, + 30, + 31, + ]; + return ( + year >= 1 && + month >= 1 && + month <= 12 && + day >= 1 && + day <= (daysInMonth[month - 1] ?? 0) && + Number(hourText) <= 23 && + Number(minuteText) <= 59 && + Number(secondText) <= 59 && + offsetHour <= 23 && + offsetMinute <= 59 + ); +} + +function isObserverEvent(value: unknown): value is ObserverEvent { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + return ( + Number.isFinite(candidate.seq) && + typeof candidate.timestamp === "string" && + isRfc3339Timestamp(candidate.timestamp) && + typeof candidate.kind === "string" && + candidate.kind.length > 0 && + "payload" in candidate + ); +} + +function unwrapObserverEvents(value: unknown): { + events: ObserverEvent[]; + rejectedEvents: number; +} { + if (!isObserverEvent(value)) return { events: [], rejectedEvents: 1 }; + if (value.kind !== "batch") return { events: [value], rejectedEvents: 0 }; + if (!value.payload || typeof value.payload !== "object") { + return { events: [], rejectedEvents: 1 }; + } + const events = (value.payload as { events?: unknown }).events; + if (!Array.isArray(events)) return { events: [], rejectedEvents: 1 }; + const decodedEvents = events.filter(isObserverEvent); + return { + events: decodedEvents, + rejectedEvents: events.length - decodedEvents.length, + }; +} + +function observerTelemetryGapCount(event: ObserverEvent): number | null { + if (event.kind !== "observer_telemetry_gap") return null; + const payload = event.payload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + return 1; + } + const count = (payload as { droppedEvents?: unknown }).droppedEvents; + return typeof count === "number" && Number.isSafeInteger(count) && count > 0 + ? count + : 1; +} + +/** Return the local-time half-open Unix range used by the owner Today view. */ +export function activityLedgerDayRange(day: string): { + startCreatedAt: number; + endCreatedAt: number; +} { + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(day); + if (!match) throw new Error("Activity Ledger day must use YYYY-MM-DD."); + const year = Number(match[1]); + const monthIndex = Number(match[2]) - 1; + const date = Number(match[3]); + const start = new Date(year, monthIndex, date); + if ( + start.getFullYear() !== year || + start.getMonth() !== monthIndex || + start.getDate() !== date + ) { + throw new Error("Activity Ledger day is not a valid calendar date."); + } + const end = new Date(year, monthIndex, date + 1); + return { + startCreatedAt: Math.floor(start.getTime() / 1_000), + endCreatedAt: Math.floor(end.getTime() / 1_000), + }; +} + +/** Query the archive's authoritative decrypted observer-time index. */ +export function activityLedgerArchiveQueryRange(day: string): { + startCreatedAt: number; + endCreatedAt: number; +} { + return activityLedgerDayRange(day); +} + +/** + * Decrypt and normalize owner-archived observer frames into the Today surface. + * + * The signed outer pubkey and `agent` tag must agree with a managed agent. A + * frame that fails that authority check, fails decryption, or is malformed is + * excluded instead of being allowed to mint activity or proof. + */ +async function buildTodayActivityFromArchivedEventPages(input: { + day: string; + agents: readonly ActivityLedgerAgentIdentity[]; + pages: + | Iterable + | AsyncIterable; + decrypt?: DecryptObserverEvent; +}): Promise { + const decrypt = input.decrypt ?? decryptObserverEvent; + const trustedAgents = new Map( + input.agents.map((agent) => [agent.pubkey, agent] as const), + ); + let retainedEvents = new Map(); + const journalEventCounts = new Map(); + let originalEventCount = 0; + let excludedObserverFrames = 0; + let sourceDroppedObserverEvents = 0; + let unindexedObserverFrames = 0; + let malformedArchivedRows = 0; + let omittedObserverFrames = 0; + let archiveRevision = 0; + let textFieldsTruncated = 0; + let checkpoint: BoundedTodayActivitySurface | null = null; + + for await (const archivedPage of input.pages) { + const page = "events" in archivedPage ? archivedPage.events : archivedPage; + if ("events" in archivedPage) { + if (archivedPage.reset) { + retainedEvents = new Map(); + journalEventCounts.clear(); + originalEventCount = 0; + excludedObserverFrames = 0; + sourceDroppedObserverEvents = 0; + unindexedObserverFrames = 0; + malformedArchivedRows = 0; + omittedObserverFrames = 0; + textFieldsTruncated = 0; + checkpoint = null; + archiveRevision = archivedPage.archiveRevision ?? archiveRevision; + continue; + } + archiveRevision = archivedPage.archiveRevision ?? archiveRevision; + unindexedObserverFrames += archivedPage.unindexedObserverFrames; + malformedArchivedRows += archivedPage.rejectedArchiveRows ?? 0; + omittedObserverFrames += archivedPage.omittedObserverFrames ?? 0; + excludedObserverFrames += + archivedPage.unindexedObserverFrames + + (archivedPage.rejectedArchiveRows ?? 0) + + (archivedPage.omittedObserverFrames ?? 0); + } + const decodedPage: (ObserverEvent[] | null)[] = Array.from( + { length: page.length }, + () => null, + ); + let nextIndex = 0; + const decryptWorker = async () => { + for (;;) { + const index = nextIndex; + nextIndex += 1; + if (index >= page.length) return; + const relayEvent = page[index]; + if (!relayEvent) continue; + const agentPubkey = observerAgentPubkey(relayEvent); + if ( + !agentPubkey || + relayEvent.pubkey !== agentPubkey || + !trustedAgents.has(agentPubkey) + ) { + excludedObserverFrames += 1; + continue; + } + + try { + const decoded = await decrypt(relayEvent); + const { events: decodedEvents, rejectedEvents } = + unwrapObserverEvents(decoded); + excludedObserverFrames += rejectedEvents; + if (decodedEvents.length === 0) { + if (rejectedEvents === 0) excludedObserverFrames += 1; + continue; + } + decodedPage[index] = decodedEvents; + } catch { + excludedObserverFrames += 1; + // Archive reconciliation is fail-closed: one bad ciphertext cannot + // suppress the rest of the owner's durable activity surface. + } + } + }; + await Promise.all( + Array.from( + { length: Math.min(TODAY_ARCHIVE_DECRYPT_CONCURRENCY, page.length) }, + decryptWorker, + ), + ); + + // Normalize this page in archive order, not promise-completion order, so + // reconstruction remains deterministic even when decryption latency + // varies by frame. Decoded observer bodies live for this page only. + const pageObserverEvents = new Map(); + for (let index = 0; index < page.length; index += 1) { + const relayEvent = page[index]; + const decodedEvents = decodedPage[index]; + if (!relayEvent || !decodedEvents) continue; + const agentPubkey = observerAgentPubkey(relayEvent); + if (!agentPubkey) continue; + const bucket = pageObserverEvents.get(agentPubkey) ?? []; + for (const decodedEvent of decodedEvents) { + const gapCount = observerTelemetryGapCount(decodedEvent); + if (gapCount !== null) { + sourceDroppedObserverEvents = Math.min( + Number.MAX_SAFE_INTEGER, + sourceDroppedObserverEvents + gapCount, + ); + continue; + } + bucket.push({ + ...decodedEvent, + sourceEventId: relayEvent.id, + sourcePubkey: relayEvent.pubkey, + sourceKind: relayEvent.kind, + sourceCreatedAt: relayEvent.created_at, + sourceSignature: relayEvent.sig, + origin: "historical_backfill", + }); + } + pageObserverEvents.set(agentPubkey, bucket); + } + + const feeds = input.agents.map((agent) => { + const pageEvents = normalizeActivityEvents( + pageObserverEvents.get(agent.pubkey) ?? [], + ).filter((event) => localDayForTimestamp(event.timestamp) === input.day); + for (const event of pageEvents) { + originalEventCount += 1; + const key = journalProjectionKey(agent.pubkey, event.journalKey); + journalEventCounts.set(key, (journalEventCounts.get(key) ?? 0) + 1); + } + return { + agentPubkey: agent.pubkey, + agentName: agent.name, + events: [...(retainedEvents.get(agent.pubkey) ?? []), ...pageEvents], + }; + }); + + const surface = buildTodayActivitySurface(feeds, { day: input.day }); + for (const journal of surface.journals) { + journal.eventCount = + journalEventCounts.get( + journalProjectionKey(journal.agentPubkey, journal.journalKey), + ) ?? journal.eventCount; + } + checkpoint = buildArchiveReconstructionCheckpoint({ + surface, + originalJournalCount: journalEventCounts.size, + originalEventCount, + excludedObserverFrames, + sourceDroppedObserverEvents, + unindexedObserverFrames, + malformedArchivedRows, + omittedObserverFrames, + archiveRevision, + previousTextFieldsTruncated: textFieldsTruncated, + }); + textFieldsTruncated = checkpoint.snapshotProjection.textFieldsTruncated; + + // Retain only the bounded normalized projection for the next page. The + // raw ciphertext page, decoded ObserverEvents, and omitted journal bodies + // are now unreachable and can be reclaimed before the iterator advances. + retainedEvents = new Map(); + for (const journal of checkpoint.journals) { + const bucket = retainedEvents.get(journal.agentPubkey) ?? []; + bucket.push(...journal.events); + retainedEvents.set(journal.agentPubkey, bucket); + } + } + + return ( + checkpoint ?? + snapshotSurfaceFromJournals({ + day: input.day, + journals: [], + maxBytes: TODAY_SNAPSHOT_SURFACE_MAX_BYTES, + originalJournalCount: 0, + originalEventCount: 0, + excludedObserverFrames, + sourceDroppedObserverEvents, + unindexedObserverFrames, + malformedArchivedRows, + omittedObserverFrames, + archiveRevision, + textFieldsTruncated: 0, + }) + ); +} + +export async function buildTodayActivityFromArchivedEvents(input: { + day: string; + agents: readonly ActivityLedgerAgentIdentity[]; + events: readonly RelayEvent[]; + decrypt?: DecryptObserverEvent; +}): Promise { + return buildTodayActivityFromArchivedEventPages({ + day: input.day, + agents: input.agents, + pages: [input.events], + decrypt: input.decrypt, + }); +} + +/** Reconstruct Today while releasing each raw archive page after decryption. */ +export async function buildTodayActivityFromArchivedPages(input: { + day: string; + agents: readonly ActivityLedgerAgentIdentity[]; + pages: + | Iterable + | AsyncIterable; + decrypt?: DecryptObserverEvent; +}): Promise { + return buildTodayActivityFromArchivedEventPages(input); +} + +/** Apply backend-validated owner artifacts and recompute every derived count. */ +export function applyAuthorityToTodayActivity( + surface: TodayActivitySurface, + artifacts: readonly ValidatedJournalAuthorityArtifact[], + relayUrl: string, +): TodayActivitySurface { + const journals: TodayActivityJournal[] = surface.journals.map((journal) => ({ + ...applyValidatedJournalAuthority( + journal, + artifacts, + relayUrl, + journal.agentPubkey, + ), + agentPubkey: journal.agentPubkey, + agentName: journal.agentName, + })); + const channels = new Map< + string, + { + journalIds: string[]; + agentPubkeys: Set; + agentNames: Set; + lastActivityAt: string; + } + >(); + for (const journal of journals) { + if (!journal.channelId) continue; + const bucket = channels.get(journal.channelId) ?? { + journalIds: [], + agentPubkeys: new Set(), + agentNames: new Set(), + lastActivityAt: journal.endedAt, + }; + bucket.journalIds.push(journal.id); + bucket.agentPubkeys.add(journal.agentPubkey); + bucket.agentNames.add(journal.agentName); + if (Date.parse(journal.endedAt) > Date.parse(bucket.lastActivityAt)) { + bucket.lastActivityAt = journal.endedAt; + } + channels.set(journal.channelId, bucket); + } + + return { + ...surface, + journals, + channels: [...channels.entries()] + .map(([channelId, bucket]) => ({ + channelId, + journalIds: bucket.journalIds, + agentPubkeys: [...bucket.agentPubkeys], + agentNames: [...bucket.agentNames], + lastActivityAt: bucket.lastActivityAt, + })) + .sort((left, right) => left.channelId.localeCompare(right.channelId)), + counts: { + journals: journals.length, + failed: journals.filter((journal) => journal.status === "failed").length, + inProgress: journals.filter((journal) => journal.status === "in_progress") + .length, + claimedWithoutEvidence: journals.filter( + (journal) => journal.claimedCompletionWithoutEvidence, + ).length, + }, + }; +} diff --git a/desktop/src/features/agents/ingestArchivedObserverEvents.test.mjs b/desktop/src/features/agents/ingestArchivedObserverEvents.test.mjs index 343ce241335..d9a190f7171 100644 --- a/desktop/src/features/agents/ingestArchivedObserverEvents.test.mjs +++ b/desktop/src/features/agents/ingestArchivedObserverEvents.test.mjs @@ -154,6 +154,12 @@ describe("ingestArchivedObserverEvents", () => { ); assert.equal(archivedEvents.length, 1, "archive must contain 1 raw event"); assert.equal(archivedEvents[0].seq, 1); + assert.equal(archivedEvents[0].sourceEventId, "e".repeat(64)); + assert.equal(archivedEvents[0].sourcePubkey, AGENT_PUBKEY); + assert.equal(archivedEvents[0].sourceKind, 24200); + assert.equal(archivedEvents[0].sourceCreatedAt, 1000); + assert.equal(archivedEvents[0].sourceSignature, "s".repeat(128)); + assert.equal(archivedEvents[0].origin, "historical_backfill"); // Also verify the live snapshot is untouched — archive separation. const snap = getAgentObserverSnapshot(AGENT_PUBKEY, true); assert.equal( @@ -183,6 +189,28 @@ describe("ingestArchivedObserverEvents", () => { ); }); + it("test_signed_ids_keep_same_seq_timestamp_frames_distinct", async () => { + _testRegisterKnownAgents(SUB_ID, [AGENT_PUBKEY]); + const sameDecodedFrame = makeObserverEvent({ + seq: 5, + timestamp: "2026-01-01T00:00:05.000Z", + }); + await ingestArchivedObserverEvents( + [ + makeRawEvent({ id: "e".repeat(64) }), + makeRawEvent({ id: "f".repeat(64) }), + ], + () => Promise.resolve(sameDecodedFrame), + ); + + const archiveEvents = _testGetArchivedChannelEvents(AGENT_PUBKEY, "chan-1"); + assert.equal(archiveEvents.length, 2); + assert.deepEqual(archiveEvents.map((event) => event.sourceEventId).sort(), [ + "e".repeat(64), + "f".repeat(64), + ]); + }); + it("test_older_archived_event_sorts_before_live", async () => { // Pre-seed a newer live event (no channelId → goes to live path). const liveObs = makeObserverEvent({ @@ -235,7 +263,11 @@ describe("ingestArchivedObserverEvents", () => { const decryptFn = () => Promise.resolve(events[callIdx++]); // All three raw events pass the guards (same pubkey/agent tag). await ingestArchivedObserverEvents( - [makeRawEvent(), makeRawEvent(), makeRawEvent()], + [ + makeRawEvent({ id: "1".repeat(64) }), + makeRawEvent({ id: "2".repeat(64) }), + makeRawEvent({ id: "3".repeat(64) }), + ], decryptFn, ); // All have channelId "chan-1" — verify archive window, not live snapshot. @@ -1160,6 +1192,45 @@ describe("raw-event-level merge: stateful aggregates across live/archive boundar ); }); + it("test_batch_inner_events_share_signed_provenance_without_collapsing", async () => { + _testRegisterKnownAgents(SUB_ID, [AGENT_PUBKEY]); + const CHANNEL = "chan-batch-provenance"; + const raw = makeRawEvent({ id: "f".repeat(64), created_at: 4242 }); + const first = makeObserverEvent({ + seq: 41, + timestamp: "2026-01-01T00:41:00.000Z", + channelId: CHANNEL, + }); + const second = makeObserverEvent({ + seq: 42, + timestamp: "2026-01-01T00:41:01.000Z", + channelId: CHANNEL, + }); + const envelope = makeObserverEvent({ + seq: 42, + timestamp: second.timestamp, + kind: "batch", + channelId: CHANNEL, + payload: { events: [first, second] }, + }); + + await ingestArchivedObserverEvents([raw, raw], makeDecrypt(envelope)); + + const archived = _testGetArchivedChannelEvents(AGENT_PUBKEY, CHANNEL); + assert.deepEqual( + archived.map((event) => event.seq), + [41, 42], + "one signed batch must retain every distinct inner event exactly once", + ); + for (const event of archived) { + assert.equal(event.sourceEventId, raw.id); + assert.equal(event.sourcePubkey, raw.pubkey); + assert.equal(event.sourceCreatedAt, raw.created_at); + assert.equal(event.sourceSignature, raw.sig); + assert.equal(event.origin, "historical_backfill"); + } + }); + it("test_malformed_batch_envelope_degrades_to_itself", async () => { // A kind:"batch" envelope with no events array (harness bug shape) must // degrade to the envelope itself rather than being silently dropped, so a diff --git a/desktop/src/features/agents/observerRelayStore.ts b/desktop/src/features/agents/observerRelayStore.ts index 68fa290ad25..d1bc37bd8b6 100644 --- a/desktop/src/features/agents/observerRelayStore.ts +++ b/desktop/src/features/agents/observerRelayStore.ts @@ -14,10 +14,11 @@ import { import { normalizePubkey } from "@/shared/lib/pubkey"; import { useQueryClient } from "@tanstack/react-query"; import { agentConfigSurfaceQueryKey } from "@/features/agents/hooks"; -import type { - ConnectionState, - ObserverEvent, - TranscriptItem, +import { + observerEventIdentity, + type ConnectionState, + type ObserverEvent, + type TranscriptItem, } from "./ui/agentSessionTypes"; import { type TranscriptState, @@ -223,6 +224,22 @@ function observerTag(event: RelayEvent, tagName: string) { return event.tags.find((tag) => tag[0] === tagName)?.[1] ?? null; } +function withRelayProvenance( + parsed: ObserverEvent, + event: RelayEvent, + origin: "live_observer" | "historical_backfill", +): ObserverEvent { + return { + ...parsed, + sourceEventId: event.id, + sourcePubkey: event.pubkey, + sourceKind: event.kind, + sourceCreatedAt: event.created_at, + sourceSignature: event.sig, + origin, + }; +} + function appendAgentEvents( agentPubkey: string, events: readonly ObserverEvent[], @@ -257,15 +274,10 @@ function appendAgentEvents( const seen = allAtEnd ? new Set() - : new Set( - current.map( - (event) => - `${event.timestamp.length}:${event.timestamp}:${event.seq}`, - ), - ); + : new Set(current.map(observerEventIdentity)); const added: ObserverEvent[] = []; for (const event of admissible) { - const eventKey = `${event.timestamp.length}:${event.timestamp}:${event.seq}`; + const eventKey = observerEventIdentity(event); if (seen.has(eventKey)) continue; seen.add(eventKey); added.push(event); @@ -312,15 +324,9 @@ function appendAgentEvents( invalidateSnapshot(key); if (!trimmed) return sortedAdded; - const retainedKeys = new Set( - final.map( - (event) => `${event.timestamp.length}:${event.timestamp}:${event.seq}`, - ), - ); + const retainedKeys = new Set(final.map(observerEventIdentity)); return sortedAdded.filter((event) => - retainedKeys.has( - `${event.timestamp.length}:${event.timestamp}:${event.seq}`, - ), + retainedKeys.has(observerEventIdentity(event)), ); } @@ -346,7 +352,8 @@ function archiveChannelKey(agentPubkey: string, channelId: string): string { * the channel archive window grows only by explicit paged loads from SQLite, * so unbounded growth from live relay events is impossible. * - * Deduplicates on `(seq, timestamp)` — identical to `appendAgentEvent` — so + * Deduplicates on signed source id (with `(seq, timestamp)` for legacy/E2E + * events) — identical to `appendAgentEvent` — so * events that arrive on the live relay before the archive page is loaded are * silently skipped. The archive window and the live transcript are kept * strictly separate: live events never write here. @@ -362,11 +369,11 @@ function appendArchivedChannelEvent( const key = archiveChannelKey(agentPubkey, channelId); const current = archiveEventsByChannel.get(key) ?? []; - // Dedup: skip if (seq, timestamp) already present in the archive window. + // Dedup: prefer the signed envelope id; legacy/E2E events use seq+timestamp. if ( current.some( (existing) => - existing.seq === event.seq && existing.timestamp === event.timestamp, + observerEventIdentity(existing) === observerEventIdentity(event), ) ) { return false; @@ -563,7 +570,10 @@ async function handleRelayObserverEvent( if (activeGeneration !== generation) { return; } - processLiveObserverEvents(agentPubkey, unwrapObserverBatch(parsed)); + const events = unwrapObserverBatch(parsed).map((inner) => + withRelayProvenance(inner, event, "live_observer"), + ); + processLiveObserverEvents(agentPubkey, events); } catch (error) { if (activeGeneration !== generation) { return; @@ -810,7 +820,7 @@ export function useManagedAgentObserverBridge( * - The event sender (`pubkey`) must match the `agent` tag value. * - Event must decrypt successfully via `decryptObserverEvent`. * - * Routes through `appendAgentEvent` so dedup on `(seq, timestamp)` and + * Routes through the shared stores so signed-id deduplication and * sort are reused — archived events that are already present (live-delivered) * are silently skipped. Failed decryptions are silently dropped (same as * live path error handling). @@ -842,20 +852,25 @@ export async function ingestArchivedObserverEvents( try { const parsed = (await _decryptFn(event)) as ObserverEvent; for (const inner of unwrapObserverBatch(parsed)) { + const enriched = withRelayProvenance( + inner, + event, + "historical_backfill", + ); // Route archived events to the channel-scoped archive window (no cap) // rather than the per-agent live-relay store (MAX_OBSERVER_EVENTS cap). // Events without a channelId fall through to the live store so they // remain visible in the agent's general transcript. - if (inner.channelId) { + if (enriched.channelId) { const added = appendArchivedChannelEvent( agentPubkey, - inner.channelId, - inner, + enriched.channelId, + enriched, ); if (added) archiveChanged = true; } else { // Live path already calls notifyListeners() inside appendAgentEvent. - appendAgentEvent(agentPubkey, inner); + appendAgentEvent(agentPubkey, enriched); } } } catch { diff --git a/desktop/src/features/agents/ui/ManagedAgentSessionPanel.tsx b/desktop/src/features/agents/ui/ManagedAgentSessionPanel.tsx index 594fa1cbaa5..0856723006b 100644 --- a/desktop/src/features/agents/ui/ManagedAgentSessionPanel.tsx +++ b/desktop/src/features/agents/ui/ManagedAgentSessionPanel.tsx @@ -1,19 +1,42 @@ import * as React from "react"; import { + CheckCircle2, CircleAlert, CircleDot, Clock3, + Pencil, TerminalSquare, XCircle, } from "lucide-react"; import { isManagedAgentActive } from "@/features/agents/lib/managedAgentControlActions"; +import { useCommunities } from "@/features/communities/useCommunities"; +import { + buildMissionJournal, + normalizeActivityEvents, + type MissionJournal, +} from "@/features/agents/activityLedger"; +import { + applyValidatedJournalAuthority, + journalAuthorityCorrelationId, + journalVerificationSources, +} from "@/features/agents/activityLedgerAuthority"; +import { + getJournalAuthorityArtifacts, + upsertJournalVerification, + upsertOwnerJournalOverride, + type JournalAuthorityArtifact, +} from "@/shared/api/tauriArchive"; import type { UserProfileLookup } from "@/features/profile/lib/identity"; import type { ManagedAgent } from "@/shared/api/types"; import { cn } from "@/shared/lib/cn"; +import { useNow } from "@/shared/lib/useNow"; import { Badge } from "@/shared/ui/badge"; +import { Button } from "@/shared/ui/button"; +import { Input } from "@/shared/ui/input"; import { Skeleton } from "@/shared/ui/skeleton"; import { Spinner } from "@/shared/ui/spinner"; +import { Textarea } from "@/shared/ui/textarea"; import { AgentSessionTranscriptList, type AgentSessionTranscriptEmptyState, @@ -76,6 +99,7 @@ export function ManagedAgentSessionPanel({ rawEventsOverride, transcriptOverride, }: ManagedAgentSessionPanelProps) { + const relayUrl = useCommunities().activeCommunity?.relayUrl ?? null; const hasObserver = isManagedAgentActive(agent); // Always read from the store — archived frames are ingested regardless of // live status and must be renderable for idle agents with channel history. @@ -127,6 +151,14 @@ export function ManagedAgentSessionPanel({ () => deriveLatestSessionId(displayEvents), [displayEvents], ); + const journalAsOf = useNow(60_000); + const latestJournal = React.useMemo( + () => + buildMissionJournal(normalizeActivityEvents(combinedEvents), { + asOf: new Date(journalAsOf), + }), + [combinedEvents, journalAsOf], + ); return (
) : null} + {latestJournal.eventCount > 0 && relayUrl ? ( + + ) : null} + ( + [], + ); + const [mode, setMode] = React.useState<"summary" | "verify" | null>(null); + const [summary, setSummary] = React.useState(journal.summary); + const [receiptRef, setReceiptRef] = React.useState(""); + const [saving, setSaving] = React.useState(false); + const [authorityLoading, setAuthorityLoading] = React.useState(true); + const [error, setError] = React.useState(null); + + const reloadAuthority = React.useCallback(async () => { + const current = await getJournalAuthorityArtifacts( + relayUrl, + agentPubkey, + journal.id, + ); + setArtifacts(current); + }, [agentPubkey, journal.id, relayUrl]); + + React.useEffect(() => { + let cancelled = false; + setArtifacts([]); + setMode(null); + setSummary(journal.summary); + setReceiptRef(""); + setAuthorityLoading(true); + setError(null); + getJournalAuthorityArtifacts(relayUrl, agentPubkey, journal.id) + .then((current) => { + if (!cancelled) setArtifacts(current); + }) + .catch((loadError) => { + if (!cancelled) { + setError( + loadError instanceof Error + ? loadError.message + : "Owner journal proof could not be loaded.", + ); + } + }) + .finally(() => { + if (!cancelled) setAuthorityLoading(false); + }); + return () => { + cancelled = true; + }; + }, [agentPubkey, journal.id, journal.summary, relayUrl]); + + const authorizedJournal = React.useMemo( + () => + applyValidatedJournalAuthority(journal, artifacts, relayUrl, agentPubkey), + [agentPubkey, artifacts, journal, relayUrl], + ); + React.useEffect(() => { + if (mode !== "summary") setSummary(authorizedJournal.summary); + }, [authorizedJournal.summary, mode]); + const verificationSources = React.useMemo( + () => journalVerificationSources(journal), + [journal], + ); + const saveSummary = async () => { + if (!summary.trim() || saving) return; + setSaving(true); + setError(null); + try { + await upsertOwnerJournalOverride(relayUrl, { + agentPubkey, + journalId: journal.id, + correlationId: journal.correlationId, + summary: summary.trim(), + }); + await reloadAuthority(); + setMode(null); + } catch (saveError) { + setError( + saveError instanceof Error + ? saveError.message + : "The owner summary was not saved.", + ); + } finally { + setSaving(false); + } + }; + + const saveVerification = async () => { + if ( + !receiptRef.trim() || + !verificationSources.hasReceiptedEvidence || + !verificationSources.hasCorrelationEvidence || + !verificationSources.hasSupportedSourceSet || + saving + ) { + return; + } + setSaving(true); + setError(null); + try { + await upsertJournalVerification(relayUrl, { + agentPubkey, + journalId: journal.id, + correlationId: journalAuthorityCorrelationId(journal), + receiptRef: receiptRef.trim(), + sourceEventIds: verificationSources.sourceEventIds, + }); + await reloadAuthority(); + setMode(null); + setReceiptRef(""); + } catch (saveError) { + setError( + saveError instanceof Error + ? saveError.message + : "The verification was not saved.", + ); + } finally { + setSaving(false); + } + }; + + return ( +
+
+ + Mission journal + + + {journalStatusLabel(authorizedJournal.status)} + + {authorizedJournal.proofState} + {authorizedJournal.claimedCompletionWithoutEvidence ? ( + Evidence gap + ) : null} +
+

+ {authorizedJournal.summary} +

+ {authorizedJournal.summarySource === "owner" ? ( +

Owner edited

+ ) : null} + + {mode === "summary" ? ( +
+