diff --git a/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs b/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs index 88f4f018e..6a2c7252a 100644 --- a/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs +++ b/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs @@ -94,6 +94,9 @@ pub(super) async fn chat_room_presence_heartbeat( }) .into_response(); }; + // A person has Community open in front of them, so History asks present + // Homes for earlier posts while somebody reads them. + port.record_community_open(now_ts()); // Not shown is a successful heartbeat that announces nothing, not an // error: the person asked for this, and Chat has no failure to report. // Chat presents only a `/while-open` grant; "Always" is the timer's. diff --git a/elastos/crates/elastos-server/src/carrier.rs b/elastos/crates/elastos-server/src/carrier.rs index 3ee10b69a..7f73aac75 100644 --- a/elastos/crates/elastos-server/src/carrier.rs +++ b/elastos/crates/elastos-server/src/carrier.rs @@ -19509,6 +19509,75 @@ pub(crate) mod tests { shutdown_test_carrier_node(node).await; } + /// A consumer cursor number is not a frame identity. `gossip_leave` drops + /// the topic buffer and every cursor on it, and the next join starts a + /// buffer at position 0, so the same consumer peeks position 0 again and + /// finds a frame it never read. The Community driver binds its progress to + /// the frames it ingested because of this. + #[tokio::test] + async fn test_gossip_leave_and_join_give_the_same_cursor_a_new_frame() { + let dir = tempfile::tempdir().unwrap(); + let (signing_key, did) = elastos_identity::derive_did(&[66u8; 32]); + let node = start_carrier_node(&signing_key, &did, dir.path().to_path_buf()) + .await + .unwrap(); + let provider = CarrierGossipProvider::new(node.gossip_state.clone()); + let topic = "__test/gossip-cursor-reuse"; + let consumer = "community-consumer"; + let join = serde_json::json!({"op":"gossip_join_exact", "topic":topic, "peers":[]}); + let peek = serde_json::json!({ + "op":"gossip_peek", "topic":topic, "consumer_id":consumer, "limit":32 + }); + let send = |message: &str| serde_json::json!({"op":"gossip_send", "topic":topic, "message":message}); + let read_one = |response: &serde_json::Value| { + ( + response["data"]["cursor"].as_u64(), + response["data"]["next_cursor"].as_u64(), + response["data"]["messages"][0]["content"] + .as_str() + .map(str::to_string), + ) + }; + + assert_eq!(provider.send_raw(&join).await.unwrap()["status"], "ok"); + assert_eq!( + provider.send_raw(&send("first frame")).await.unwrap()["status"], + "ok" + ); + assert_eq!( + read_one(&provider.send_raw(&peek).await.unwrap()), + (Some(0), Some(1), Some("first frame".to_string())) + ); + assert_eq!( + provider + .send_raw(&serde_json::json!({ + "op":"gossip_ack", "topic":topic, "consumer_id":consumer, + "cursor":0, "next_cursor":1 + })) + .await + .unwrap()["status"], + "ok" + ); + assert_eq!( + provider + .send_raw(&serde_json::json!({"op":"gossip_leave", "topic":topic})) + .await + .unwrap()["status"], + "ok" + ); + assert_eq!(provider.send_raw(&join).await.unwrap()["status"], "ok"); + assert_eq!( + provider.send_raw(&send("second frame")).await.unwrap()["status"], + "ok" + ); + assert_eq!( + read_one(&provider.send_raw(&peek).await.unwrap()), + (Some(0), Some(1), Some("second frame".to_string())), + "the recreated topic hands the same consumer cursor a frame it never read" + ); + shutdown_test_carrier_node(node).await; + } + #[tokio::test] async fn test_gossip_empty_peek_requires_exact_empty_ack() { let dir = tempfile::tempdir().unwrap(); diff --git a/elastos/crates/elastos-server/src/collaboration_carrier.rs b/elastos/crates/elastos-server/src/collaboration_carrier.rs index ba263d858..4592f51a9 100644 --- a/elastos/crates/elastos-server/src/collaboration_carrier.rs +++ b/elastos/crates/elastos-server/src/collaboration_carrier.rs @@ -22,7 +22,7 @@ use crate::collaboration_protocol::{ const COLLABORATION_TOPIC_DOMAIN: &[u8] = b"elastos.collaboration.carrier.topic.v1"; const COLLABORATION_TOPIC_PREFIX: &str = "__elastos_internal/collaboration-v1/"; -const COLLABORATION_RECEIVE_BATCH_SIZE: usize = 32; +pub(crate) const COLLABORATION_RECEIVE_BATCH_SIZE: usize = 32; const MAX_RECEIVE_RESPONSE_BYTES: usize = MAX_COLLABORATION_TRANSPORT_FRAME_BYTES * COLLABORATION_RECEIVE_BATCH_SIZE + 64 * 1024; /// How long a joined Home may have no Community peer before it repairs the @@ -152,6 +152,12 @@ impl CollaborationCarrierReceiveBatch { pub fn rejected_frames(&self) -> usize { self.rejected_frames } + + /// The buffer position of the first frame in this batch. The Carrier + /// replays the batch from here until the Home acknowledges it. + pub fn cursor(&self) -> u64 { + self.cursor + } } #[derive(Deserialize)] diff --git a/elastos/crates/elastos-server/src/collaboration_core.rs b/elastos/crates/elastos-server/src/collaboration_core.rs index dcfa2839b..6f8f7e5af 100644 --- a/elastos/crates/elastos-server/src/collaboration_core.rs +++ b/elastos/crates/elastos-server/src/collaboration_core.rs @@ -49,7 +49,7 @@ pub(crate) const DEFAULT_CONVERSATION_SEND_METHOD: &str = "message.send"; const MAX_CORE_STATE_BYTES: usize = 24 * 1024 * 1024; const MAX_UNRESOLVED_OUTGOING: usize = 64; const MAX_OUTGOING_RECORDS: usize = 4_096; -const MAX_PENDING_INCOMING: usize = 32; +pub(crate) const MAX_PENDING_INCOMING: usize = 32; pub(crate) const MAX_PENDING_INCOMING_PER_SENDER: usize = 8; pub(crate) const MAX_ACCEPTANCE_RECEIPTS_PER_OUTGOING: usize = 32; const MAX_INCOMING_RECORDS_AND_TOMBSTONES: usize = 4_096; @@ -200,6 +200,8 @@ impl DurableIncomingAcceptance { pub(crate) struct CollaborationTransportIncomingAcceptance { acceptance_receipt_bytes: Vec, product_handoff_pending: bool, + message_hash: String, + message_expires_at: u64, } impl CollaborationTransportIncomingAcceptance { @@ -207,6 +209,18 @@ impl CollaborationTransportIncomingAcceptance { &self.acceptance_receipt_bytes } + /// The accepted post this receipt confirms. + pub(crate) fn message_hash(&self) -> &str { + &self.message_hash + } + + /// When the accepted post expires. A receipt for it is worth sending + /// until then and no longer: after that second the sender has dropped the + /// post and nobody waits for the confirmation. + pub(crate) fn message_expires_at(&self) -> u64 { + self.message_expires_at + } + #[cfg_attr( not(test), expect( @@ -3122,6 +3136,8 @@ fn transport_incoming_acceptance( accepted: DurableIncomingAcceptance, ) -> CollaborationTransportIncomingAcceptance { CollaborationTransportIncomingAcceptance { + message_hash: accepted.authorized.message().envelope_sha256().to_string(), + message_expires_at: accepted.authorized.message().envelope().payload.expires_at, acceptance_receipt_bytes: accepted.acceptance_receipt_bytes, product_handoff_pending: accepted.product_handoff_pending, } diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index 1f5c30697..ec6921d15 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -1,6 +1,7 @@ //! Bounded Community catch-up through the private Runtime/Carrier provider plane. //! Live gossip admission and its replay/receipt lifetime remain unchanged. +use std::collections::BTreeSet; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -31,6 +32,19 @@ pub(crate) const MAX_HISTORY_RESPONSE_BYTES: usize = MAX_CONVERSATION_HISTORY_BY pub(crate) const MAX_HISTORY_CARRIER_REPLY_BYTES: usize = MAX_HISTORY_RESPONSE_BYTES + 64; const MAX_PEERS_PER_PASS: usize = 4; const PEER_TIMEOUT: Duration = Duration::from_secs(5); +/// Wait before the first pass that nothing else asks for. +const HISTORY_BASE_WAIT_SECS: u64 = 15; +/// Longest wait after an answered pass while nobody is reading Community. +const HISTORY_MAX_WAIT_SECS: u64 = 900; +/// Longest wait while a person reads Community or a pass reaches nobody. +/// The Community link calls itself silent after 60 s (`COMMUNITY_LINK_SILENCE`), +/// so a Home that answers nothing for a minute is the longest silence this +/// Runtime already treats as normal. +const HISTORY_SHORT_MAX_WAIT_SECS: u64 = 60; +/// How long the Home page heartbeat keeps a person counted as reading +/// Community. The page sends it every 15 s, so three missed heartbeats end +/// the signal, the same tolerance a presence record carries. +const HISTORY_OPEN_SIGNAL_SECS: u64 = 45; // History requests carry this Home's own signed presence, which exists only // for people who granted Community presence. const NOT_SHOWN_DETAIL: &str = "Show yourself in Community to load earlier posts."; @@ -47,6 +61,7 @@ struct HistoryInner { registry: Arc, data_root: PathBuf, peers: Mutex, + cadence: Mutex, } #[derive(Default)] @@ -56,6 +71,69 @@ struct HistoryPeerRound { available: bool, } +/// When this Home next asks present Homes for earlier posts. Four things make +/// a pass due at once: a Community join, a Community connection that returns, +/// a Home that is present now and was absent at the check before, and a person +/// who opens Community. Any of them can leave this Home without posts the +/// others kept. +#[derive(Default)] +struct HistoryCadence { + due_at: u64, + wait: u64, + connected: bool, + watching: bool, + present: BTreeSet, +} + +impl HistoryCadence { + /// Does this check start a pass? Every check samples the Homes it sees + /// and the person reading Community, so a Home that lapses and returns + /// between two passes, and a person who opens Community between them, + /// both make the next check due. + fn due( + &mut self, + now: u64, + connected: bool, + present: BTreeSet, + watching: bool, + ) -> bool { + let returned = connected && !self.connected; + let arrived = present.difference(&self.present).next().is_some(); + let opened = watching && !self.watching; + self.connected = connected; + self.watching = watching; + self.present = present; + if returned || arrived || opened { + self.wait = 0; + self.due_at = now; + } + if now < self.due_at { + return false; + } + // The pass that follows owns the next wait. This provisional wait + // holds a pass that ends in an error to the short cadence. + self.due_at = now.saturating_add(HISTORY_BASE_WAIT_SECS); + true + } + + /// Answered passes grow the wait towards a quarter of an hour only while + /// nobody is reading Community. An open view or an unanswered pass keeps + /// the wait at a minute, so a quick reopening within the heartbeat's + /// lifetime still asks for the posts a returning Home holds. + fn pass_finished(&mut self, now: u64, answered: bool) { + let ceiling = if answered && !self.watching { + HISTORY_MAX_WAIT_SECS + } else { + HISTORY_SHORT_MAX_WAIT_SECS + }; + self.wait = self + .wait + .saturating_mul(2) + .clamp(HISTORY_BASE_WAIT_SECS, ceiling); + self.due_at = now.saturating_add(self.wait); + } +} + #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct HistoryRequest { @@ -98,6 +176,7 @@ impl CollaborationHistoryService { registry: registry.clone(), data_root, peers: Mutex::new(HistoryPeerRound::default()), + cadence: Mutex::new(HistoryCadence::default()), }), }; registry @@ -197,11 +276,65 @@ impl CollaborationHistoryService { .project_history(&self.inner.data_root, &messages, now) } + /// Runs a pass only when this Home has a reason to ask. The Runtime calls + /// this on its own cadence; `HistoryCadence` decides which of those checks + /// reaches other Homes. + pub(crate) async fn fetch_if_due(&self, now: u64) -> anyhow::Result<()> { + if !self.inner.product.community_membership().joined() { + self.reset_cadence()?; + return Ok(()); + } + let connected = self.inner.product.conversation_transport_view().available; + let present = self + .inner + .presence + .history_participants(now)? + .into_iter() + .map(|peer| peer.endpoint_did) + .collect(); + let watching = self + .inner + .presence + .community_open_at() + .is_some_and(|at| now.saturating_sub(at) <= HISTORY_OPEN_SIGNAL_SECS); + let due = self + .inner + .cadence + .lock() + .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? + .due(now, connected, present, watching); + if !due { + return Ok(()); + } + let answered = self.fetch_once(now).await; + self.inner + .cadence + .lock() + .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? + .pass_finished(now, matches!(answered, Ok(true))); + answered.map(|_| ()) + } + + /// Forgets the grown wait, so the next check starts a pass. A person who + /// leaves Community and joins it again owns this reset: the posts of the + /// time away are the reason the wait exists. + pub(crate) fn reset_cadence(&self) -> anyhow::Result<()> { + *self + .inner + .cadence + .lock() + .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? = + HistoryCadence::default(); + Ok(()) + } + /// Independently scheduled from live gossip. Four unavailable peers cannot /// monopolize a pass or starve later available participants across passes. - pub(crate) async fn fetch_once(&self, now: u64) -> anyhow::Result<()> { + /// Reports whether a Home answered this pass, which decides how long the + /// Runtime waits before the next one. + pub(crate) async fn fetch_once(&self, now: u64) -> anyhow::Result { if !self.inner.product.community_membership().joined() { - return Ok(()); + return Ok(false); } let (peers, own) = match async { Ok::<_, anyhow::Error>(( @@ -224,7 +357,7 @@ impl CollaborationHistoryService { OwnHistoryPresence::Usable(_) => {} OwnHistoryPresence::NotUsable => { self.inner.product.set_history_unavailable(NOT_SHOWN_DETAIL); - return Ok(()); + return Ok(false); } OwnHistoryPresence::Absent => { // Nobody chosen is a fact only the publisher knows; before its @@ -235,7 +368,7 @@ impl CollaborationHistoryService { } else { self.inner.product.set_history_searching(); } - return Ok(()); + return Ok(false); } } let local_endpoint = self.inner.product.local_device_did(); @@ -245,7 +378,7 @@ impl CollaborationHistoryService { .collect::>(); if remote.is_empty() { self.inner.product.set_history_unavailable(ALONE_DETAIL); - return Ok(()); + return Ok(false); } let round_complete = { let mut round = self @@ -282,7 +415,7 @@ impl CollaborationHistoryService { .await? else { self.inner.product.set_history_unavailable(NOT_SHOWN_DETAIL); - return Ok(()); + return Ok(false); }; let request = serde_json::to_value(self.request(own))?; // History reads have no remote effect. A lost response can be @@ -334,7 +467,7 @@ impl CollaborationHistoryService { if round_complete { round.available = false; } - Ok(()) + Ok(available) } } @@ -1289,6 +1422,331 @@ mod tests { } } + /// One fresh presence announcement from this Home's person, as its + /// Runtime publishes one while the person is shown. + async fn refresh_presence( + fixture: &Fixture, + now: u64, + ) -> crate::collaboration_presence::PreparedCollaborationPresence { + let root = crate::auth::principal_localhost_root(PRINCIPAL); + let profile = crate::collaboration_profile_authority::load_profile_authority( + fixture.root.path(), + PRINCIPAL, + &root, + ) + .unwrap() + .unwrap(); + let grant = crate::collaboration_presence_grant::presence_grant( + &fixture.authority, + fixture.root.path(), + PRINCIPAL, + &fixture.profile_did, + &fixture.presence.presence_scope(), + crate::collaboration_presence_grant::PresencePublisher::Runtime, + now, + ) + .await + .unwrap(); + fixture + .presence + .prepare_presence( + presence_request_binding( + &grant.bind_request_id(&format!("presence-at-{now}")), + PRINCIPAL, + &profile, + ) + .unwrap(), + &profile, + now, + ) + .unwrap() + } + + /// A presence record lives 45 s, so both Homes keep announcing their + /// person while a measurement runs for minutes. + async fn keep_both_present(a: &Fixture, b: &Fixture, now: u64) { + let own = refresh_presence(a, now).await; + a.presence + .project_authorized_presence(&own, now) + .await + .unwrap(); + let peer = refresh_presence(b, now).await; + // Both publishers complete their local projection so expired + // announcements can leave the outgoing store during long measurements. + b.presence + .project_authorized_presence(&peer, now) + .await + .unwrap(); + a.product + .test_core() + .accept_incoming_from_signed_source_for_test(peer.test_envelope_bytes(), now) + .unwrap(); + for presence in a.presence.pending_presences().unwrap() { + a.presence.project_handoff(&presence, now).unwrap(); + } + assert!( + a.presence + .history_participants(now) + .unwrap() + .iter() + .any(|participant| participant.endpoint_did == b.product.local_device_did()), + "the peer stays present through the measurement" + ); + } + + /// Counts every history request and either answers it with an empty + /// retained history or leaves the Home unreachable. + struct CadencePlane { + endpoint: String, + network_id: String, + conversation_id: String, + answers: bool, + calls: Mutex>, + } + + #[async_trait::async_trait] + impl ProviderCarrierInvoker for CadencePlane { + async fn invoke_carrier_provider( + &self, + route: &ProviderCarrierRoute, + _invocation: &ProviderInvocation, + _request: serde_json::Value, + ) -> Result { + let ProviderCarrierRoute::PeerDid { peer_did, .. } = route else { + panic!("history selects verified Profile endpoint"); + }; + self.calls.lock().unwrap().push(peer_did.clone()); + if !self.answers { + return Err(ProviderError::Unavailable("offline".to_string())); + } + Ok(serde_json::json!({ + "schema": RESPONSE_SCHEMA, + "network_id": self.network_id, + "conversation_id": self.conversation_id, + "sender_service": CHAT_SERVICE, + "endpoint_did": self.endpoint, + "envelopes": [], + })) + } + } + + /// Runtime checks, 15 s apart, measured as the seconds at which the + /// Runtime asked another Home for earlier posts while nobody was watching. + async fn seconds_of_history_asks(answers: bool, ticks: u64) -> Vec { + let now = now_secs(); + let a = fixture("history-measured", "community", now).await; + let b = fixture("history-measured", "community", now).await; + a.product.set_community_connected(true); + let (network_id, conversation_id) = a.product.history_scope(); + let plane = Arc::new(CadencePlane { + endpoint: b.product.local_device_did(), + network_id: network_id.to_string(), + conversation_id: conversation_id.to_string(), + answers, + calls: Mutex::new(Vec::new()), + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let mut asked_at = Vec::new(); + let mut asks = 0; + for tick in 0..ticks { + let at = now + tick * HISTORY_BASE_WAIT_SECS; + keep_both_present(&a, &b, at).await; + a.service.fetch_if_due(at).await.unwrap(); + let calls = plane.calls.lock().unwrap().len(); + if calls > asks { + asked_at.push(at - now); + asks = calls; + } + } + asked_at + } + + /// A Home whose passes bring answers backs off towards a quarter of an + /// hour, because the Homes it reached already gave it their posts. + #[tokio::test] + async fn answered_history_passes_back_off_to_the_long_wait() { + assert_eq!( + seconds_of_history_asks(true, 125).await, + [0, 15, 45, 105, 225, 465, 945, 1845] + ); + } + + /// A Home whose passes reach nobody keeps asking every minute. The Homes + /// it wants are present and can become reachable at any check, so a + /// person who opens Community waits one minute, not a quarter of an hour. + #[tokio::test] + async fn unanswered_history_passes_keep_the_short_wait() { + assert_eq!( + seconds_of_history_asks(false, 41).await, + [0, 15, 45, 105, 165, 225, 285, 345, 405, 465, 525, 585] + ); + } + + /// Ten minutes of Runtime checks, 15 s apart, as the cadence sees them. + /// The two measured tests above cover these seconds on the service path, + /// with the requests each pass made. + #[test] + fn the_history_wait_grows_only_on_an_answered_pass() { + let now = 1_800_000_000; + let peer = BTreeSet::from(["did:peer".to_string()]); + let passes = |answered: bool| { + let mut cadence = HistoryCadence::default(); + let mut due = Vec::new(); + for tick in 0..40 { + let at = now + tick * HISTORY_BASE_WAIT_SECS; + if cadence.due(at, true, peer.clone(), false) { + due.push(at - now); + cadence.pass_finished(at, answered); + } + } + due + }; + assert_eq!(passes(true), [0, 15, 45, 105, 225, 465]); + assert_eq!( + passes(false), + [0, 15, 45, 105, 165, 225, 285, 345, 405, 465, 525, 585] + ); + } + + /// Each reason to ask other Homes at once, one check at a time. + #[test] + fn a_new_home_a_returned_link_and_an_open_community_each_ask_at_once() { + let now = 1_800_000_000; + let peer = BTreeSet::from(["did:peer".to_string()]); + let joined = BTreeSet::from(["did:peer".to_string(), "did:newcomer".to_string()]); + let mut cadence = HistoryCadence::default(); + assert!(cadence.due(now, true, peer.clone(), false)); + cadence.pass_finished(now, true); + assert!(!cadence.due(now + 1, true, peer.clone(), false)); + // A Home this one did not see at the check before. + assert!(cadence.due(now + 2, true, joined.clone(), false)); + cadence.pass_finished(now + 2, true); + // A Home that lapses and returns between two passes. Every check + // samples the Homes it sees, so the return asks again. + assert!(!cadence.due(now + 3, true, peer.clone(), false)); + assert!(cadence.due(now + 4, true, joined.clone(), false)); + cadence.pass_finished(now + 4, true); + // A Community connection that returns. + assert!(!cadence.due(now + 5, false, joined.clone(), false)); + assert!(cadence.due(now + 6, true, joined.clone(), false)); + cadence.pass_finished(now + 6, true); + // A person who opens Community. + assert!(!cadence.due(now + 7, true, joined.clone(), false)); + assert!(cadence.due(now + 8, true, joined.clone(), true)); + cadence.pass_finished(now + 8, true); + // The wait holds while Community stays open. + assert!(!cadence.due(now + 9, true, joined, true)); + } + + #[tokio::test] + async fn a_history_check_inside_its_wait_asks_nobody() { + let now = now_secs(); + let a = fixture("history-cadence", "community", now).await; + let b = fixture("history-cadence", "community", now).await; + observe(&a, &b, now); + a.product.set_community_connected(true); + let plane = Arc::new(Plane { + source: a.product.local_device_did(), + peers: BTreeMap::from([(b.product.local_device_did(), b.registry.clone())]), + calls: Mutex::new(Vec::new()), + stall: false, + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let asks = || plane.calls.lock().unwrap().len(); + a.service.fetch_if_due(now).await.unwrap(); + assert_eq!(asks(), 1); + a.service.fetch_if_due(now + 14).await.unwrap(); + assert_eq!(asks(), 1); + a.service + .fetch_if_due(now + HISTORY_BASE_WAIT_SECS) + .await + .unwrap(); + assert_eq!(asks(), 2); + } + + /// A person opens Community inside a wait. The Runtime asks present Homes + /// at the next check, because the Home page heartbeat records the open + /// view on the presence port History reads. Nothing new travels between + /// Homes for this. + #[tokio::test] + async fn opening_community_asks_present_homes_at_the_next_check() { + let now = now_secs(); + let a = fixture("history-open", "community", now).await; + let b = fixture("history-open", "community", now).await; + observe(&a, &b, now); + a.product.set_community_connected(true); + let plane = Arc::new(Plane { + source: a.product.local_device_did(), + peers: BTreeMap::from([(b.product.local_device_did(), b.registry.clone())]), + calls: Mutex::new(Vec::new()), + stall: false, + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let asks = || plane.calls.lock().unwrap().len(); + a.service.fetch_if_due(now).await.unwrap(); + assert_eq!(asks(), 1); + a.service.fetch_if_due(now + 1).await.unwrap(); + assert_eq!(asks(), 1, "the wait holds while nothing changes"); + a.presence.record_community_open(now + 2); + a.service.fetch_if_due(now + 2).await.unwrap(); + assert_eq!(asks(), 2, "a person who opens Community waits for nothing"); + a.service.fetch_if_due(now + 3).await.unwrap(); + assert_eq!(asks(), 2, "an open view does not ask at every check"); + } + + /// Answered passes run long enough to reach the quiet Home's longest + /// wait. A ten-second switch away keeps the previous heartbeat fresh, + /// so reopening has to work even when every check sees an open view. + #[tokio::test] + async fn quick_reopening_community_keeps_history_within_the_short_wait() { + let now = now_secs(); + let a = fixture("history-quick-reopen", "community", now).await; + let b = fixture("history-quick-reopen", "community", now).await; + a.product.set_community_connected(true); + let (network_id, conversation_id) = a.product.history_scope(); + let plane = Arc::new(CadencePlane { + endpoint: b.product.local_device_did(), + network_id: network_id.to_string(), + conversation_id: conversation_id.to_string(), + answers: true, + calls: Mutex::new(Vec::new()), + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let asks = || plane.calls.lock().unwrap().len(); + let mut asked_at = Vec::new(); + for tick in 0..=63 { + let offset = tick * 15; + let at = now + offset; + keep_both_present(&a, &b, at).await; + a.presence.record_community_open(at); + let before = asks(); + a.service.fetch_if_due(at).await.unwrap(); + if asks() > before { + asked_at.push(offset); + } + } + assert_eq!(asked_at.last(), Some(&945)); + let before_reopening = asks(); + // The person switches away just after the last heartbeat and returns + // ten seconds later. There is no close signal or false watching sample. + a.presence.record_community_open(now + 955); + for offset in [960, 975, 990, 1005] { + let at = now + offset; + keep_both_present(&a, &b, at).await; + a.presence.record_community_open(at); + a.service.fetch_if_due(at).await.unwrap(); + } + assert_eq!( + asks(), + before_reopening + 1, + "a quick reopening gets another provider request within 60 seconds" + ); + assert!( + asked_at.windows(2).all(|pair| pair[1] - pair[0] <= 60), + "answered passes stay short while Community remains open: {asked_at:?}" + ); + } + #[tokio::test(start_paused = true)] async fn stalled_history_has_a_per_peer_deadline() { let now = now_secs(); diff --git a/elastos/crates/elastos-server/src/collaboration_presence.rs b/elastos/crates/elastos-server/src/collaboration_presence.rs index 1049f0d24..1bc51b247 100644 --- a/elastos/crates/elastos-server/src/collaboration_presence.rs +++ b/elastos/crates/elastos-server/src/collaboration_presence.rs @@ -42,6 +42,10 @@ pub struct CollaborationPresenceProductPort { /// The Runtime publisher's last pass: `Some(true)` it announced someone, /// `Some(false)` nobody here has a presence grant, `None` unknown. local_publication: Arc>>, + /// When a person on this Home last had Community open. The Home page + /// heartbeat sets it, and History reads it to ask for earlier posts while + /// somebody is reading Community. + community_open_at: Arc>>, /// Mints and validates Community presence grants with the Runtime's /// capability manager. `None` (standalone gateway, most tests): no grant /// can be made or honoured, so nobody is announced. @@ -184,6 +188,7 @@ impl CollaborationPresenceProductPort { core, read_model, local_publication: Arc::new(std::sync::Mutex::new(None)), + community_open_at: Arc::new(std::sync::Mutex::new(None)), presence_authority: None, own_principals: Arc::new(Mutex::new(std::collections::HashMap::new())), }) @@ -488,6 +493,20 @@ impl CollaborationPresenceProductPort { }) } + /// A person has Community in front of them right now. The Home page + /// heartbeat reports this every `HOME_PRESENCE_HEARTBEAT_MS`, so History + /// can ask for earlier posts while somebody is reading them. + pub(crate) fn record_community_open(&self, now: u64) { + if let Ok(mut current) = self.community_open_at.lock() { + *current = Some(now); + } + } + + /// When a person last had Community open on this Home. + pub(crate) fn community_open_at(&self) -> Option { + self.community_open_at.lock().ok().and_then(|at| *at) + } + pub(crate) fn record_local_publication(&self, state: Option) { if let Ok(mut current) = self.local_publication.lock() { *current = state; diff --git a/elastos/crates/elastos-server/src/collaboration_startup.rs b/elastos/crates/elastos-server/src/collaboration_startup.rs index 4f72740bb..0efbff394 100644 --- a/elastos/crates/elastos-server/src/collaboration_startup.rs +++ b/elastos/crates/elastos-server/src/collaboration_startup.rs @@ -812,11 +812,18 @@ async fn run_collaboration_history_worker( if changed.is_err() || *shutdown.borrow() { return; } } result = async { + // A person who joins Community again waits for the posts of + // the time away, not for the wait the passes before it grew. + // The change can arrive while this worker sleeps or asks, so + // the worker reads the change itself. + if joined.has_changed().unwrap_or(false) { + service.reset_cadence()?; + } if !*joined.borrow_and_update() { return Ok(()); } tokio::select! { biased; - _ = joined.changed() => Ok(()), - result = service.fetch_once(now_secs()) => result, + _ = joined.changed() => service.reset_cadence(), + result = service.fetch_if_due(now_secs()) => result, } } => { if result.is_err() { tracing::debug!("Community history refresh is unavailable"); } @@ -2821,6 +2828,164 @@ mod tests { assert!(service.history_task.is_none()); } + struct CountingHistoryInvoker { + asked: Arc, + calls: Arc, + } + + #[async_trait::async_trait] + impl elastos_runtime::provider::ProviderCarrierInvoker for CountingHistoryInvoker { + async fn invoke_carrier_provider( + &self, + _route: &elastos_runtime::provider::ProviderCarrierRoute, + invocation: &elastos_runtime::provider::ProviderInvocation, + _request: serde_json::Value, + ) -> Result { + assert_eq!( + invocation.target, + crate::collaboration_history::HISTORY_PROVIDER + ); + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.asked.notify_one(); + Err(ProviderError::Unavailable("offline".to_string())) + } + } + + /// A person leaves Community and joins it again. The history worker asks + /// for earlier posts at once, inside the wait its first pass set. The + /// test runs the real worker loop, so it covers the place where the + /// worker observes the membership change. + #[tokio::test(start_paused = true)] + async fn the_history_worker_asks_again_after_a_leave_and_a_join() { + for check_while_left in [false, true] { + let (temp, _) = configured_root(true); + let configuration = + load_and_accept_collaboration_startup_configuration(temp.path()).unwrap(); + let CollaborationNetworkConfiguration::Configured { + profile, + grant: Some(grant), + } = configuration.configuration + else { + panic!("expected configured conversation"); + }; + let now = now_secs(); + let (key, _) = generate_keypair(); + let core = Arc::new( + CollaborationCore::new( + temp.path(), + key.clone(), + (*profile).clone(), + grant.clone(), + CHAT_ROOM_CAPSULE, + ) + .unwrap(), + ); + let chat = CollaborationChatProductPort::new(core.clone()).unwrap(); + let presence = CollaborationPresenceProductPort::new(core.clone()) + .unwrap() + .with_presence_authority(crate::collaboration_presence_grant::test_authority()); + let (local_profile, presence_grant) = + grant_local_presence(temp.path(), &presence, "test-principal", now).await; + let prepared = presence + .prepare_presence( + presence_request_binding( + &presence_grant.bind_request_id("local-history-presence"), + "test-principal", + &local_profile, + ) + .unwrap(), + &local_profile, + now, + ) + .unwrap(); + presence.project_prepared_presence(&prepared, now).unwrap(); + let (remote_key, _) = generate_keypair(); + let remote_profile = profile_for_endpoint(&remote_key, "Remote"); + let remote = + DefaultConversationDeviceAuthority::new(remote_key, (*profile).clone(), grant) + .unwrap(); + let announcement = remote + .prepare_profile_outgoing( + &remote_profile, + SERVICE, + "elastos.chat.presence/v1", + serde_json::json!({}), + now, + // The clock this worker reads stands still in this test, + // so the record stays live and a presence lapse cannot + // explain a later pass. + 45, + ) + .unwrap(); + core.accept_incoming_from_signed_source_for_test(announcement.envelope_bytes(), now) + .unwrap(); + for handoff in presence.pending_presences().unwrap() { + presence.project_handoff(&handoff, now).unwrap(); + } + let registry = Arc::new(ProviderRegistry::new()); + let asked = Arc::new(tokio::sync::Notify::new()); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + registry + .set_carrier_invoker(Arc::new(CountingHistoryInvoker { + asked: asked.clone(), + calls: calls.clone(), + })) + .await; + let history = CollaborationHistoryService::new( + chat.clone(), + presence.clone(), + registry, + temp.path().to_path_buf(), + ) + .await + .unwrap(); + let membership = chat.community_membership().clone(); + membership.set_joined(true).unwrap(); + let (shutdown, shutdown_rx) = watch::channel(false); + let history_task = tokio::spawn(run_collaboration_history_worker(history, shutdown_rx)); + // The first pass reaches nobody and sets a wait. The clock the + // worker reads does not move in this test, so only a reset can + // make a second pass due. + tokio::time::timeout(Duration::from_secs(60), asked.notified()) + .await + .unwrap(); + if check_while_left { + // The worker runs one check at 15 s while the person is away. + membership.set_joined(false).unwrap(); + tokio::time::sleep(Duration::from_secs(20)).await; + membership.set_joined(true).unwrap(); + } else { + // The leave and the join both fall inside one worker wait. + membership.set_joined(false).unwrap(); + membership.set_joined(true).unwrap(); + } + tokio::time::timeout(Duration::from_secs(60), asked.notified()) + .await + .expect("the worker asks again after the join"); + let mut service = CollaborationRuntimeService { + shutdown, + task: None, + discovery_task: None, + presence_task: None, + history_task: Some(history_task), + product_port: Some(chat), + presence_port: Some(presence), + discovery_service: None, + worker_product_port: None, + worker_presence_port: None, + }; + tokio::time::timeout(Duration::from_millis(500), service.shutdown()) + .await + .unwrap() + .unwrap(); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 2, + "check_while_left: {check_while_left}" + ); + } + } + #[tokio::test(start_paused = true)] async fn worker_projects_received_chat_while_outgoing_or_receipt_send_is_blocked() { for blocked_outgoing in [true, false] { diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index 75d34ccbb..e57839317 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -1,6 +1,6 @@ //! Bounded Runtime driver for one durable collaboration core and Carrier subscription. -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -10,9 +10,12 @@ use elastos_runtime::provider::{ ProviderRegistry, ProviderTransfer, ResourceRequest, ResourceResponse, }; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use tokio::time::Instant; -use crate::collaboration_carrier::{CollaborationCarrierSendOutcome, JoinedCollaborationNetwork}; +use crate::collaboration_carrier::{ + CollaborationCarrierSendOutcome, JoinedCollaborationNetwork, COLLABORATION_RECEIVE_BATCH_SIZE, +}; use crate::collaboration_core::{CollaborationCore, CollaborationTransportIngestion}; use crate::collaboration_presence::CollaborationPresenceProductPort; use crate::collaboration_protocol::{ @@ -24,12 +27,39 @@ pub(crate) struct CollaborationTransportDriver { network: JoinedCollaborationNetwork, home_delivery: Option, gossip: Mutex, + ingested: Mutex, + receipts: Mutex, +} + +/// The frames this Home already ingested from the batch the Carrier holds at +/// `cursor`. The Carrier replays that exact range until the Home acknowledges +/// it (`carrier.rs:7140-7146`), so without this record a Home that cannot +/// acknowledge would ingest those frames again on every live cycle and gossip +/// each stored acceptance receipt again with it. +/// +/// Each entry names its own frame by digest, because a Carrier position +/// belongs to a topic buffer that a leave and a join recreate from position 0 +/// (`carrier.rs:6957-6988`, `carrier.rs:1344-1350`). A position alone would +/// let this record hide a frame it was never counted on. The record holds one +/// batch, so it stays at or under `COLLABORATION_RECEIVE_BATCH_SIZE` digests, +/// and the acknowledged range drops it. +#[derive(Default)] +struct IngestedBatch { + cursor: u64, + digests: Vec<[u8; 32]>, +} + +fn transport_frame_digest(frame: &[u8]) -> [u8; 32] { + Sha256::digest(frame).into() } const SHARED_FAST_RETRY_WINDOW_SECS: u64 = 3; const SHARED_CUSTODY_FAST_RETRY: Duration = Duration::from_millis(500); const SHARED_CUSTODY_RETRY: Duration = Duration::from_secs(5); const MAX_CUSTODY_RETRIES_PER_CYCLE: usize = 4; +/// One held receipt for each post the intake can hold. +const MAX_HELD_RECEIPTS: usize = crate::collaboration_core::MAX_PENDING_INCOMING; +const MAX_HELD_RECEIPT_BYTES: usize = 1024 * 1024; pub(crate) const SHARED_PROVIDER: &str = "collaboration.shared"; const SHARED_OP: &str = "deliver"; const MAX_SHARED_DELIVERY_BYTES: usize = MAX_COLLABORATION_TRANSPORT_FRAME_BYTES * 6 + 2048; @@ -58,6 +88,61 @@ struct GossipSchedule { sent: HashMap, } +/// The acceptance receipts this Home owes senders whose posts it holds +/// durably and whose receipt reached nobody. The post is delivered, so the +/// sender needs the confirmation and nothing else: this queue sends it again +/// without the sender re-sending the post to every Home on the topic. +/// +/// The queue holds at most `MAX_HELD_RECEIPTS` posts, as many as the intake +/// itself holds, and at most `MAX_HELD_RECEIPT_BYTES` of frames. The oldest +/// entry leaves first, because its sender has had the most time to re-send. +#[derive(Default)] +struct ReceiptQueue { + held: VecDeque, + bytes: usize, +} + +struct HeldReceipt { + message_hash: String, + receipt_frame: Vec, + /// The post this receipt confirms, as its sender signed it. The retry + /// presents it to the same gate the first send passed. + original_frame: Vec, + expires_at: u64, + last_send: Instant, + sends: u32, +} + +impl HeldReceipt { + fn bytes(&self) -> usize { + self.receipt_frame.len() + self.original_frame.len() + } +} + +impl ReceiptQueue { + fn retain(&mut self, keep: impl Fn(&HeldReceipt) -> bool) { + let mut dropped = 0; + self.held.retain(|held| { + if keep(held) { + return true; + } + dropped += held.bytes(); + false + }); + self.bytes = self.bytes.saturating_sub(dropped); + } + + fn drop_oldest(&mut self) -> bool { + match self.held.pop_front() { + Some(held) => { + self.bytes = self.bytes.saturating_sub(held.bytes()); + true + } + None => false, + } + } +} + /// Wait after `sends` gossip broadcasts of one original. Presence keeps the /// slow cadence; a Shared post backs off 5, 10, 20, 40, then 60 s until a /// Home receipt. @@ -320,6 +405,8 @@ impl CollaborationTransportDriver { network, home_delivery: None, gossip: Mutex::new(GossipSchedule::default()), + ingested: Mutex::new(IngestedBatch::default()), + receipts: Mutex::new(ReceiptQueue::default()), } } @@ -385,9 +472,123 @@ impl CollaborationTransportDriver { summary.attempted += homes.attempted; summary.remote_broadcasts += homes.remote_broadcasts; summary.send_failures += homes.send_failures; + // The owed receipts follow the originals on the same Carrier link, so + // one send waits for the one before it. + let receipts = self.retry_receipts_once(now).await?; + summary.attempted += receipts.attempted; + summary.remote_broadcasts += receipts.remote_broadcasts; + summary.local_only_buffered += receipts.local_only_buffered; + summary.send_failures += receipts.send_failures; Ok(summary) } + /// Sends the acceptance receipts that reached nobody again. Each one waits + /// 5, 10, 20, 40 and then 60 s between sends and leaves the queue when its + /// post expires, so a sender learns its post arrived without re-sending + /// the post itself. One live cycle sends at most + /// `MAX_CUSTODY_RETRIES_PER_CYCLE` of them, the bound a pending original + /// already uses. + async fn retry_receipts_once( + &self, + now: u64, + ) -> anyhow::Result { + if !self.core.community_membership().joined() { + return Ok(CollaborationOutgoingRetrySummary::default()); + } + let attempted_at = Instant::now(); + let due = { + let mut queue = self + .receipts + .lock() + .map_err(|_| anyhow::anyhow!("held receipt lock poisoned"))?; + queue.retain(|held| now < held.expires_at); + let mut due = Vec::new(); + for held in queue.held.iter_mut() { + if due.len() == MAX_CUSTODY_RETRIES_PER_CYCLE { + break; + } + if attempted_at.duration_since(held.last_send) + < gossip_resend_interval(true, held.sends) + { + continue; + } + held.last_send = attempted_at; + held.sends = held.sends.saturating_add(1); + due.push(( + held.message_hash.clone(), + held.receipt_frame.clone(), + held.original_frame.clone(), + )); + } + due + }; + let mut summary = CollaborationOutgoingRetrySummary::default(); + for (message_hash, receipt_frame, original_frame) in due { + self.core.community_membership().require_joined()?; + summary.attempted += 1; + match self + .network + .send_gossip(&self.core, &receipt_frame, Some(&original_frame)) + .await + { + Ok(CollaborationCarrierSendOutcome::RemoteBroadcast { .. }) => { + summary.remote_broadcasts += 1; + self.release_receipt(&message_hash)?; + } + Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) => { + summary.local_only_buffered += 1; + } + Err(_) => { + summary.send_failures += 1; + } + } + } + Ok(summary) + } + + /// Keeps the receipt for a post this Home holds durably and whose receipt + /// reached nobody. + fn hold_receipt( + &self, + message_hash: &str, + receipt_frame: &[u8], + original_frame: &[u8], + expires_at: u64, + ) -> anyhow::Result<()> { + let held = HeldReceipt { + message_hash: message_hash.to_string(), + receipt_frame: receipt_frame.to_vec(), + original_frame: original_frame.to_vec(), + expires_at, + last_send: Instant::now(), + sends: 1, + }; + let mut queue = self + .receipts + .lock() + .map_err(|_| anyhow::anyhow!("held receipt lock poisoned"))?; + queue.retain(|entry| entry.message_hash != held.message_hash); + while queue.held.len() >= MAX_HELD_RECEIPTS + || queue.bytes.saturating_add(held.bytes()) > MAX_HELD_RECEIPT_BYTES + { + if !queue.drop_oldest() { + break; + } + } + queue.bytes = queue.bytes.saturating_add(held.bytes()); + queue.held.push_back(held); + Ok(()) + } + + /// Forgets a receipt a neighbour took, so it never goes out twice. + fn release_receipt(&self, message_hash: &str) -> anyhow::Result<()> { + self.receipts + .lock() + .map_err(|_| anyhow::anyhow!("held receipt lock poisoned"))? + .retain(|held| held.message_hash != message_hash); + Ok(()) + } + /// Broadcasts each pending original at once, then resends it until a /// receipt settles it: Presence every 5 s, a Shared post with back-off /// (`gossip_resend_interval`). Gossip reaches Homes that no presence or @@ -515,10 +716,7 @@ impl CollaborationTransportDriver { // senders may never resend them, so this Home owns the retry. let mut held = self.core.take_held_frames(now).into_iter(); while let Some(frame) = held.next() { - if !self - .ingest_frame(frame.frame(), now, &mut summary, true) - .await - { + if !self.ingest_frame(frame.frame(), now, &mut summary).await { self.core .restore_held_frames(std::iter::once(frame).chain(held).collect()); return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); @@ -531,29 +729,89 @@ impl CollaborationTransportDriver { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } summary.carrier_rejected_frames = batch.rejected_frames(); - for envelope in batch.envelopes() { + let mut ingested = self.ingested_frames(batch.cursor(), batch.envelopes())?; + for envelope in batch.envelopes().iter().skip(ingested) { if !self.core.community_membership().joined() { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } - if !self.ingest_frame(envelope, now, &mut summary, false).await { + if !self.ingest_frame(envelope, now, &mut summary).await { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } + self.record_ingested_frame(batch.cursor(), ingested, envelope)?; + ingested += 1; } match self.network.ack(&batch).await { - Ok(()) => Ok(CollaborationIncomingOnceOutcome::Acknowledged(summary)), + Ok(()) => { + self.forget_ingested_frames()?; + Ok(CollaborationIncomingOnceOutcome::Acknowledged(summary)) + } Err(_) => Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)), } } - /// Ingests one frame and sends its receipt. Returns false when the cycle - /// must stop and retry this frame later. + /// How many frames at the front of this batch the record already names. + /// The count stops at the first frame the record does not name, so a + /// Carrier position that now carries another frame earns its own + /// ingestion, and a sender's genuine re-send at a new position does too. + fn ingested_frames(&self, cursor: u64, envelopes: &[Vec]) -> anyhow::Result { + let ingested = self + .ingested + .lock() + .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))?; + if ingested.cursor != cursor { + return Ok(0); + } + Ok(envelopes + .iter() + .zip(ingested.digests.iter()) + .take_while(|(envelope, digest)| transport_frame_digest(envelope) == **digest) + .count()) + } + + /// Names the frame at `position` of the batch at `cursor` as ingested. + fn record_ingested_frame( + &self, + cursor: u64, + position: usize, + frame: &[u8], + ) -> anyhow::Result<()> { + let mut ingested = self + .ingested + .lock() + .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))?; + if ingested.cursor != cursor { + *ingested = IngestedBatch { + cursor, + digests: Vec::new(), + }; + } + ingested.digests.truncate(position); + if ingested.digests.len() == position && position < COLLABORATION_RECEIVE_BATCH_SIZE { + ingested.digests.push(transport_frame_digest(frame)); + } + Ok(()) + } + + /// Drops the record once the Carrier accepts the range. The Carrier never + /// replays an acknowledged range, so the record has nothing left to say, + /// and a later range at the same position starts from nothing. + fn forget_ingested_frames(&self) -> anyhow::Result<()> { + *self + .ingested + .lock() + .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))? = + IngestedBatch::default(); + Ok(()) + } + + /// Ingests one frame and sends its receipt. Returns false when the frame + /// is not ingested and the cycle must retry it later. async fn ingest_frame( &self, frame: &[u8], now: u64, summary: &mut CollaborationIncomingOnceSummary, - held: bool, ) -> bool { match self.core.allows_gossip_frame(frame) { Ok(true) => {} @@ -588,12 +846,28 @@ impl CollaborationTransportDriver { { Ok(CollaborationCarrierSendOutcome::RemoteBroadcast { .. }) => { summary.acceptance_receipt_broadcasts += 1; - true + if self.release_receipt(accepted.message_hash()).is_err() { + return false; + } + } + // The acceptance is durable, so this frame is ingested. + // This Home owes the sender the confirmation and keeps the + // receipt until it goes out or the post expires. + Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) | Err(_) => { + if self + .hold_receipt( + accepted.message_hash(), + &receipt_frame, + frame, + accepted.message_expires_at(), + ) + .is_err() + { + return false; + } } - // A held frame's senders fetch its durable receipt again - // by retrying: direct by the Home call, gossip by re-broadcast. - Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) | Err(_) => held, } + true } } } @@ -627,8 +901,7 @@ mod tests { }; use elastos_runtime::provider::ProviderCarrierInvoker; use elastos_runtime::signature::{generate_keypair, SigningKey}; - use sha2::{Digest, Sha256}; - use std::collections::{BTreeMap, VecDeque}; + use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; const NETWORK: &str = "collaboration-transport-test"; const CONVERSATION: &str = "default-conversation"; @@ -669,6 +942,11 @@ mod tests { fn observe_durable_incoming_before_send(&self, core: Arc) { *self.durable_incoming_before_send.lock().unwrap() = Some(core); } + /// Drops the replies one cycle scripted but did not use, so the next + /// cycle starts from its own script. + fn drain_replies(&self) { + self.replies.lock().unwrap().clear(); + } fn requests(&self) -> Vec { self.requests.lock().unwrap().clone() } @@ -2762,11 +3040,12 @@ mod tests { .requests() .iter() .all(|request| request["op"] != "gossip_ack")); + // The retry resumes at the frame that failed: the three rejections the + // first cycle already consumed are not read again. assert_eq!( driver.process_incoming_once(NOW).await.unwrap(), CollaborationIncomingOnceOutcome::Acknowledged(CollaborationIncomingOnceSummary { carrier_rejected_frames: 2, - deterministic_rejections: 3, incoming_acceptances: 2, acceptance_receipt_broadcasts: 2, ..CollaborationIncomingOnceSummary::default() @@ -2837,7 +3116,7 @@ mod tests { } #[tokio::test] - async fn incoming_receipt_is_durable_before_send_and_replayed_until_exact_ack() { + async fn an_incoming_receipt_is_durable_before_its_single_send() { let fixture = Fixture::new(); let core = fixture.core(); let (sender, _) = generate_keypair(); @@ -2848,16 +3127,18 @@ mod tests { .to_vec(); let carrier = FakeCarrier::new(); let driver = fixture.driver(core.clone(), carrier.clone()).await; - // A separately opened Core reads the persisted acceptance before each send. + // A separately opened Core reads the persisted acceptance before the send. carrier.observe_durable_incoming_before_send(fixture.core()); - let remote = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + // A receipt that reaches no gossip neighbour leaves the acceptance + // durable, so the batch completes and the Home stops reading it. carrier.push([ peek(0, 1, vec![gossip(&frame)]), serde_json::json!({"status":"ok","broadcast":"local_only","data":{"remote_peer_count":0}}), + ack(0, 1), ]); assert!(matches!( driver.process_incoming_once(NOW).await.unwrap(), - CollaborationIncomingOnceOutcome::RetryRequired(_) + CollaborationIncomingOnceOutcome::Acknowledged(_) )); assert_eq!(core.summary().unwrap().pending_product_handoffs, 1); core.acknowledge_product_handoff(&collaboration_message_envelope_sha256(&envelope)) @@ -2867,19 +3148,10 @@ mod tests { assert_eq!(core.summary().unwrap().replay_tombstones, 0); *carrier.durable_incoming_before_send.lock().unwrap() = None; - carrier.push([peek(0, 1, vec![gossip(&frame)])]); + // A send that fails outright ingests the frame just the same. + carrier.push([peek(1, 2, vec![gossip(&frame)])]); carrier.push_failure("receipt send failed"); - assert!(matches!( - driver.process_incoming_once(NOW + 1).await.unwrap(), - CollaborationIncomingOnceOutcome::RetryRequired(_) - )); - carrier.push([peek(0, 1, vec![gossip(&frame)]), remote.clone()]); - carrier.push_failure("ack failed"); - assert!(matches!( - driver.process_incoming_once(NOW + 1).await.unwrap(), - CollaborationIncomingOnceOutcome::RetryRequired(_) - )); - carrier.push([peek(0, 1, vec![gossip(&frame)]), remote, ack(0, 1)]); + carrier.push([ack(1, 2)]); assert!(matches!( driver.process_incoming_once(NOW + 1).await.unwrap(), CollaborationIncomingOnceOutcome::Acknowledged(_) @@ -2890,8 +3162,7 @@ mod tests { .filter(|request| request["op"] == "gossip_send") .map(|request| request["message"].as_str().unwrap().as_bytes().to_vec()) .collect(); - assert_eq!(receipts.len(), 4); - assert!(receipts.windows(2).all(|pair| pair[0] == pair[1])); + assert_eq!(receipts.len(), 2); assert!(core.allows_gossip_receipt(&receipts[0], &frame).unwrap()); assert_eq!( request_ops(&carrier), @@ -2899,8 +3170,221 @@ mod tests { "gossip_join_exact", "gossip_peek", "gossip_send", + "gossip_ack", "gossip_peek", "gossip_send", + "gossip_ack", + ] + ); + } + + /// The message each `gossip_send` receipt accepts, in send order. + fn receipted_hashes(carrier: &FakeCarrier) -> Vec { + carrier + .requests() + .iter() + .filter(|request| request["op"] == "gossip_send") + .map(|request| receipt_message_hash(request["message"].as_str().unwrap())) + .collect() + } + + /// A post from its own sender, ready for the topic. + fn remote_post(fixture: &Fixture, body: &str) -> Vec { + let (key, message) = remote_message(fixture, generate_keypair().0, body); + sign_collaboration_transport_frame(&key, &message).unwrap() + } + + const PINNED_WINDOW: usize = 8; + const PINNED_CYCLES: usize = 5; + + /// Receipts the Home gossiped in each of `PINNED_CYCLES` live cycles that + /// all read the same Carrier range. `scripted` prepares one cycle's + /// replies after the pinned peek. + async fn receipts_per_pinned_cycle( + driver: &CollaborationTransportDriver, + carrier: &FakeCarrier, + window: &[serde_json::Value], + scripted: impl Fn(), + ) -> Vec { + let mut per_cycle = Vec::new(); + let mut seen = 0; + for _ in 0..PINNED_CYCLES { + carrier.push([peek(0, window.len() as u64, window.to_vec())]); + scripted(); + // No acknowledgement reply: the Carrier keeps replaying this range. + assert!(matches!( + driver.process_incoming_once(NOW).await.unwrap(), + CollaborationIncomingOnceOutcome::RetryRequired(_) + )); + carrier.drain_replies(); + let total = receipted_hashes(carrier).len(); + per_cycle.push(total - seen); + seen = total; + } + per_cycle + } + + /// The Carrier replays the same batch range until the Home acknowledges it + /// (`carrier.rs:7140-7146`). A Home that cannot acknowledge must receipt + /// those frames once, not once per 500 ms cycle. + #[tokio::test] + async fn a_pinned_batch_receipts_its_frames_once_when_the_acknowledgement_fails() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let window = (0..PINNED_WINDOW) + .map(|index| gossip(&remote_post(&fixture, &format!("post-{index}")))) + .collect::>(); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + let per_cycle = receipts_per_pinned_cycle(&driver, &carrier, &window, || { + carrier.push(std::iter::repeat_n(sent.clone(), PINNED_WINDOW)); + }) + .await; + assert_eq!(per_cycle, [PINNED_WINDOW, 0, 0, 0, 0]); + } + + /// A Home whose intake is full refuses every new frame (#409), so its + /// batch never completes. The frames it accepted before the refusal keep + /// their receipts; they do not earn a new one every cycle. + #[tokio::test] + async fn a_full_home_receipts_the_frames_it_accepted_once() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + for index in 0..crate::collaboration_core::MAX_PENDING_INCOMING - PINNED_WINDOW { + let frame = remote_post(&fixture, &format!("fill-{index}")); + assert!(matches!( + core.ingest_transport_frame(&frame, NOW).unwrap(), + CollaborationTransportIngestion::Incoming(_) + )); + } + // The last frame finds the intake full and keeps the batch unfinished. + let window = (0..=PINNED_WINDOW) + .map(|index| gossip(&remote_post(&fixture, &format!("post-{index}")))) + .collect::>(); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + let per_cycle = receipts_per_pinned_cycle(&driver, &carrier, &window, || { + carrier.push(std::iter::repeat_n(sent.clone(), PINNED_WINDOW)); + }) + .await; + assert_eq!(per_cycle, [PINNED_WINDOW, 0, 0, 0, 0]); + } + + /// A Home with no gossip neighbour reaches nobody with its receipts. It + /// keeps the durable acceptance and sends each receipt once in the live + /// cycle that ingested its frame, instead of once per cycle. The owed + /// receipts then wait in their own queue + /// (`receipts_that_reached_nobody_wait_in_a_bounded_queue`). + #[tokio::test] + async fn a_home_without_a_gossip_neighbour_receipts_its_frames_once() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let window = (0..PINNED_WINDOW) + .map(|index| gossip(&remote_post(&fixture, &format!("post-{index}")))) + .collect::>(); + let local_only = serde_json::json!({ + "status":"ok","broadcast":"local_only","data":{"remote_peer_count":0} + }); + let per_cycle = receipts_per_pinned_cycle(&driver, &carrier, &window, || { + carrier.push(std::iter::repeat_n(local_only.clone(), PINNED_WINDOW)); + }) + .await; + assert_eq!(per_cycle, [PINNED_WINDOW, 0, 0, 0, 0]); + + // Local-only delivery makes progress by itself: with the + // acknowledgement working, this Home completes the batch and moves on. + let receipts = receipted_hashes(&carrier).len(); + carrier.push([ + peek(0, window.len() as u64, window.clone()), + ack(0, window.len() as u64), + ]); + assert!(matches!( + driver.process_incoming_once(NOW).await.unwrap(), + CollaborationIncomingOnceOutcome::Acknowledged(_) + )); + assert_eq!(receipted_hashes(&carrier).len(), receipts); + } + + /// A sender re-sends its own post on its back-off until a Home receipts + /// it. That copy arrives at a new Carrier position, so it is receipted + /// again. + #[tokio::test] + async fn a_resent_post_is_receipted_again_at_its_new_carrier_position() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (key, message) = remote_message(&fixture, generate_keypair().0, "resent post"); + let frame = sign_collaboration_transport_frame(&key, &message).unwrap(); + let hash = collaboration_message_envelope_sha256(&message); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + for cursor in 0..2 { + carrier.push([ + peek(cursor, cursor + 1, vec![gossip(&frame)]), + sent.clone(), + ack(cursor, cursor + 1), + ]); + assert!(matches!( + driver.process_incoming_once(NOW + cursor).await.unwrap(), + CollaborationIncomingOnceOutcome::Acknowledged(_) + )); + } + assert_eq!(receipted_hashes(&carrier), [hash.clone(), hash]); + } + + /// A post this Home never read must be ingested and receipted, even when + /// it arrives at a Carrier position an earlier batch already used. A + /// `gossip_leave` drops the topic buffer and its cursors + /// (`carrier.rs:6957-6988`) and the next join starts a buffer at position + /// 0 (`carrier.rs:1344-1350`), which + /// `test_gossip_leave_and_join_give_the_same_cursor_a_new_frame` proves. + #[tokio::test] + async fn a_reused_cursor_ingests_its_frame_after_an_acknowledged_range() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (first_key, first_message) = + remote_message(&fixture, generate_keypair().0, "before the leave"); + let (next_key, next_message) = + remote_message(&fixture, generate_keypair().0, "after the join"); + let first = sign_collaboration_transport_frame(&first_key, &first_message).unwrap(); + let next = sign_collaboration_transport_frame(&next_key, &next_message).unwrap(); + let first_hash = collaboration_message_envelope_sha256(&first_message); + let next_hash = collaboration_message_envelope_sha256(&next_message); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + + let mut acknowledged = Vec::new(); + for (second, frame) in [(0, &first), (1, &next), (2, &first)] { + carrier.drain_replies(); + carrier.push([peek(0, 1, vec![gossip(frame)]), sent.clone(), ack(0, 1)]); + acknowledged.push(matches!( + driver.process_incoming_once(NOW + second).await, + Ok(CollaborationIncomingOnceOutcome::Acknowledged(_)) + )); + } + + assert_eq!( + receipted_hashes(&carrier), + [first_hash.clone(), next_hash, first_hash], + "each frame at the reused position earns its own receipt" + ); + assert_eq!( + acknowledged, + [true, true, true], + "every cycle completes its range" + ); + assert_eq!( + request_ops(&carrier), + [ + "gossip_join_exact", + "gossip_peek", + "gossip_send", + "gossip_ack", "gossip_peek", "gossip_send", "gossip_ack", @@ -2909,6 +3393,230 @@ mod tests { "gossip_ack", ] ); + assert_eq!( + core.summary().unwrap().pending_product_handoffs, + 2, + "both posts reached the product, and the replay added no third row" + ); + } + + /// The same reused position, with the earlier range still unacknowledged. + /// The held progress belongs to frames that are no longer there. + #[tokio::test] + async fn a_reused_cursor_ingests_its_frame_after_a_held_range() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (first_key, first_message) = + remote_message(&fixture, generate_keypair().0, "held before the leave"); + let (next_key, next_message) = + remote_message(&fixture, generate_keypair().0, "after the join"); + let first = sign_collaboration_transport_frame(&first_key, &first_message).unwrap(); + let next = sign_collaboration_transport_frame(&next_key, &next_message).unwrap(); + let first_hash = collaboration_message_envelope_sha256(&first_message); + let next_hash = collaboration_message_envelope_sha256(&next_message); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + + carrier.push([peek(0, 1, vec![gossip(&first)]), sent.clone()]); + carrier.push_failure("acknowledgement failed"); + assert!(matches!( + driver.process_incoming_once(NOW).await.unwrap(), + CollaborationIncomingOnceOutcome::RetryRequired(_) + )); + carrier.push([peek(0, 1, vec![gossip(&next)]), sent, ack(0, 1)]); + driver.process_incoming_once(NOW + 1).await.unwrap(); + + assert_eq!(receipted_hashes(&carrier), [first_hash, next_hash]); + assert_eq!(core.summary().unwrap().pending_product_handoffs, 2); + } + + /// A receipt that reached no neighbour stays this Home's to send. A + /// gossip-only post leaves its sender at 0, 5, 15, 35, 75, 135, 195 and + /// 255 s of its 300 s lifetime, so a receipt dropped at 255 s leaves a + /// post that this Home holds durably showing "Sending" until it expires. + #[tokio::test(start_paused = true)] + async fn a_receipt_that_reached_nobody_is_sent_again_until_the_post_expires() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (key, message) = remote_message(&fixture, generate_keypair().0, "a post received late"); + let frame = sign_collaboration_transport_frame(&key, &message).unwrap(); + let hash = collaboration_message_envelope_sha256(&message); + let local_only = serde_json::json!({ + "status":"ok","broadcast":"local_only","data":{"remote_peer_count":0} + }); + // The sender's last gossip attempt; the post expires 45 s later. + let received_at = NOW + 255; + carrier.push([ + peek(0, 1, vec![gossip(&frame)]), + local_only.clone(), + ack(0, 1), + ]); + assert!(matches!( + driver.process_incoming_once(received_at).await.unwrap(), + CollaborationIncomingOnceOutcome::Acknowledged(_) + )); + assert_eq!(receipted_hashes(&carrier), std::slice::from_ref(&hash)); + + let mut sent_at = Vec::new(); + let mut sends = 1; + for second in 0..=60 { + carrier.push([local_only.clone()]); + driver + .retry_outgoing_once(received_at + second) + .await + .unwrap(); + carrier.drain_replies(); + let total = receipted_hashes(&carrier).len(); + if total > sends { + sent_at.push(second); + sends = total; + } + tokio::time::advance(Duration::from_secs(1)).await; + } + assert_eq!( + sent_at, + [5, 15, 35], + "the receipt waits 5, 10 then 20 s, and the next attempt would \ + fall after the post expires" + ); + assert!(receipted_hashes(&carrier) + .iter() + .all(|receipted| *receipted == hash)); + } + + /// A receipt a neighbour took is never sent again. The limit this change + /// keeps is that a stuck batch costs one receipt per frame, not one per + /// 500 ms cycle. + #[tokio::test(start_paused = true)] + async fn a_receipt_that_reached_a_neighbour_is_not_sent_again() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let frame = remote_post(&fixture, "a post receipted at once"); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + carrier.push([peek(0, 1, vec![gossip(&frame)]), sent.clone(), ack(0, 1)]); + driver.process_incoming_once(NOW).await.unwrap(); + assert_eq!(receipted_hashes(&carrier).len(), 1); + for second in 0..=60 { + carrier.push([sent.clone()]); + driver.retry_outgoing_once(NOW + second).await.unwrap(); + carrier.drain_replies(); + tokio::time::advance(Duration::from_secs(1)).await; + } + assert_eq!( + receipted_hashes(&carrier).len(), + 1, + "a receipt that went out is not retried" + ); + } + + /// The receipts that reached nobody wait in a bounded queue: as many as + /// the intake holds accepted posts, and four sends in one live cycle. + #[tokio::test(start_paused = true)] + async fn receipts_that_reached_nobody_wait_in_a_bounded_queue() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let local_only = serde_json::json!({ + "status":"ok","broadcast":"local_only","data":{"remote_peer_count":0} + }); + let overflow = 8; + let posts = crate::collaboration_core::MAX_PENDING_INCOMING + overflow; + let mut hashes = Vec::new(); + for index in 0..posts { + let (key, message) = + remote_message(&fixture, generate_keypair().0, &format!("post-{index}")); + let frame = sign_collaboration_transport_frame(&key, &message).unwrap(); + let hash = collaboration_message_envelope_sha256(&message); + let cursor = index as u64; + carrier.push([ + peek(cursor, cursor + 1, vec![gossip(&frame)]), + local_only.clone(), + ack(cursor, cursor + 1), + ]); + driver.process_incoming_once(NOW).await.unwrap(); + // The product takes each post, so the intake never refuses one. + core.acknowledge_product_handoff(&hash).unwrap(); + hashes.push(hash); + } + let first_send = receipted_hashes(&carrier); + assert_eq!(first_send.len(), posts); + + tokio::time::advance(Duration::from_secs(5)).await; + let mut per_cycle = Vec::new(); + let mut sends = posts; + for _ in 0..20 { + carrier.push(std::iter::repeat_n(local_only.clone(), 8)); + driver.retry_outgoing_once(NOW + 5).await.unwrap(); + carrier.drain_replies(); + let total = receipted_hashes(&carrier).len(); + per_cycle.push(total - sends); + sends = total; + } + let retried = receipted_hashes(&carrier)[posts..].to_vec(); + assert!( + per_cycle + .iter() + .all(|count| *count <= MAX_CUSTODY_RETRIES_PER_CYCLE), + "a cycle sends at most four held receipts: {per_cycle:?}" + ); + assert_eq!( + retried.iter().collect::>().len(), + crate::collaboration_core::MAX_PENDING_INCOMING, + "the queue holds as many receipts as the intake holds posts" + ); + assert!( + hashes[..overflow] + .iter() + .all(|hash| !retried.contains(hash)), + "a full queue drops its oldest receipt for the newest one" + ); + } + + /// The reproduction instrument for the 14 Nov Community burst: what each + /// gossip message costs on the wire, as `gossip_send` encodes it + /// (`carrier.rs:7008-7027`). Run it with `--nocapture`. + #[tokio::test] + async fn community_resend_reproduction_table() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let post = remote_post(&fixture, "a post of ordinary length"); + let presence = presence_frame(&fixture, &generate_keypair().0, NOW); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + carrier.push([peek(0, 1, vec![gossip(&post)]), sent, ack(0, 1)]); + driver.process_incoming_once(NOW).await.unwrap(); + let receipt = carrier + .requests() + .iter() + .find(|request| request["op"] == "gossip_send") + .map(|request| request["message"].as_str().unwrap().as_bytes().to_vec()) + .unwrap(); + let wire = |frame: &[u8]| { + serde_json::to_vec(&crate::carrier::GossipMessage { + sender_id: core.local_device_did(), + sender_nick: "unknown".to_string(), + content: String::from_utf8(frame.to_vec()).unwrap(), + ts: NOW, + nonce: u64::MAX, + signature: None, + sender_session_id: None, + }) + .unwrap() + .len() + }; + println!( + "wire bytes: presence {} post {} receipt {}", + wire(&presence), + wire(&post), + wire(&receipt) + ); } #[tokio::test]