From 076df149c6885509c83b9aba38b4289cb9219df0 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 21:37:16 -0500 Subject: [PATCH 01/14] Community: reproduce a pinned gossip batch re-sending receipts Four tests drive the conditions that leave a live batch unacknowledged and record how many acceptance receipts leave the Home in each 500 ms cycle. The Carrier replays the same range until the Home acknowledges it, so every cycle re-ingests the frames the Home already accepted and gossips their stored receipts again. A fifth test measures what each gossip message costs on the wire. --- .../elastos-server/src/collaboration_core.rs | 2 +- .../src/collaboration_transport.rs | 189 ++++++++++++++++++ 2 files changed, 190 insertions(+), 1 deletion(-) diff --git a/elastos/crates/elastos-server/src/collaboration_core.rs b/elastos/crates/elastos-server/src/collaboration_core.rs index dcfa2839b..eb41e6e03 100644 --- a/elastos/crates/elastos-server/src/collaboration_core.rs +++ b/elastos/crates/elastos-server/src/collaboration_core.rs @@ -49,7 +49,7 @@ pub(crate) const DEFAULT_CONVERSATION_SEND_METHOD: &str = "message.send"; const MAX_CORE_STATE_BYTES: usize = 24 * 1024 * 1024; const MAX_UNRESOLVED_OUTGOING: usize = 64; const MAX_OUTGOING_RECORDS: usize = 4_096; -const MAX_PENDING_INCOMING: usize = 32; +pub(crate) const MAX_PENDING_INCOMING: usize = 32; pub(crate) const MAX_PENDING_INCOMING_PER_SENDER: usize = 8; pub(crate) const MAX_ACCEPTANCE_RECEIPTS_PER_OUTGOING: usize = 32; const MAX_INCOMING_RECORDS_AND_TOMBSTONES: usize = 4_096; diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index 75d34ccbb..9119ce3ab 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -669,6 +669,11 @@ mod tests { fn observe_durable_incoming_before_send(&self, core: Arc) { *self.durable_incoming_before_send.lock().unwrap() = Some(core); } + /// Drops the replies one cycle scripted but did not use, so the next + /// cycle starts from its own script. + fn drain_replies(&self) { + self.replies.lock().unwrap().clear(); + } fn requests(&self) -> Vec { self.requests.lock().unwrap().clone() } @@ -2911,6 +2916,190 @@ mod tests { ); } + /// The message each `gossip_send` receipt accepts, in send order. + fn receipted_hashes(carrier: &FakeCarrier) -> Vec { + carrier + .requests() + .iter() + .filter(|request| request["op"] == "gossip_send") + .map(|request| receipt_message_hash(request["message"].as_str().unwrap())) + .collect() + } + + /// A post from its own sender, ready for the topic. + fn remote_post(fixture: &Fixture, body: &str) -> Vec { + let (key, message) = remote_message(fixture, generate_keypair().0, body); + sign_collaboration_transport_frame(&key, &message).unwrap() + } + + const PINNED_WINDOW: usize = 8; + const PINNED_CYCLES: usize = 5; + + /// Receipts the Home gossiped in each of `PINNED_CYCLES` live cycles that + /// all read the same Carrier range. `scripted` prepares one cycle's + /// replies after the pinned peek. + async fn receipts_per_pinned_cycle( + driver: &CollaborationTransportDriver, + carrier: &FakeCarrier, + window: &[serde_json::Value], + scripted: impl Fn(), + ) -> Vec { + let mut per_cycle = Vec::new(); + let mut seen = 0; + for _ in 0..PINNED_CYCLES { + carrier.push([peek(0, window.len() as u64, window.to_vec())]); + scripted(); + // No acknowledgement reply: the Carrier keeps replaying this range. + assert!(matches!( + driver.process_incoming_once(NOW).await.unwrap(), + CollaborationIncomingOnceOutcome::RetryRequired(_) + )); + carrier.drain_replies(); + let total = receipted_hashes(carrier).len(); + per_cycle.push(total - seen); + seen = total; + } + per_cycle + } + + /// The Carrier replays the same batch range until the Home acknowledges it + /// (`carrier.rs:7140-7146`). A Home that cannot acknowledge must receipt + /// those frames once, not once per 500 ms cycle. + #[tokio::test] + async fn a_pinned_batch_receipts_its_frames_once_when_the_acknowledgement_fails() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let window = (0..PINNED_WINDOW) + .map(|index| gossip(&remote_post(&fixture, &format!("post-{index}")))) + .collect::>(); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + let per_cycle = receipts_per_pinned_cycle(&driver, &carrier, &window, || { + carrier.push(std::iter::repeat_n(sent.clone(), PINNED_WINDOW)); + }) + .await; + assert_eq!(per_cycle, [PINNED_WINDOW, 0, 0, 0, 0]); + } + + /// A Home whose intake is full refuses every new frame (#409), so its + /// batch never completes. The frames it accepted before the refusal keep + /// their receipts; they do not earn a new one every cycle. + #[tokio::test] + async fn a_full_home_receipts_the_frames_it_accepted_once() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + for index in 0..crate::collaboration_core::MAX_PENDING_INCOMING - PINNED_WINDOW { + let frame = remote_post(&fixture, &format!("fill-{index}")); + assert!(matches!( + core.ingest_transport_frame(&frame, NOW).unwrap(), + CollaborationTransportIngestion::Incoming(_) + )); + } + // The last frame finds the intake full and keeps the batch unfinished. + let window = (0..=PINNED_WINDOW) + .map(|index| gossip(&remote_post(&fixture, &format!("post-{index}")))) + .collect::>(); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + let per_cycle = receipts_per_pinned_cycle(&driver, &carrier, &window, || { + carrier.push(std::iter::repeat_n(sent.clone(), PINNED_WINDOW)); + }) + .await; + assert_eq!(per_cycle, [PINNED_WINDOW, 0, 0, 0, 0]); + } + + /// A Home with no gossip neighbour reaches nobody with its receipts. It + /// keeps the durable acceptance and lets each sender's own re-send ask for + /// the receipt again, instead of re-sending every receipt every cycle. + #[tokio::test] + async fn a_home_without_a_gossip_neighbour_receipts_its_frames_once() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let window = (0..PINNED_WINDOW) + .map(|index| gossip(&remote_post(&fixture, &format!("post-{index}")))) + .collect::>(); + let local_only = serde_json::json!({ + "status":"ok","broadcast":"local_only","data":{"remote_peer_count":0} + }); + let per_cycle = receipts_per_pinned_cycle(&driver, &carrier, &window, || { + carrier.push(std::iter::repeat_n(local_only.clone(), PINNED_WINDOW)); + }) + .await; + assert_eq!(per_cycle, [PINNED_WINDOW, 0, 0, 0, 0]); + } + + /// A sender re-sends its own post on its back-off until a Home receipts + /// it. That copy arrives at a new Carrier position, so it is receipted + /// again. + #[tokio::test] + async fn a_resent_post_is_receipted_again_at_its_new_carrier_position() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (key, message) = remote_message(&fixture, generate_keypair().0, "resent post"); + let frame = sign_collaboration_transport_frame(&key, &message).unwrap(); + let hash = collaboration_message_envelope_sha256(&message); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + for cursor in 0..2 { + carrier.push([ + peek(cursor, cursor + 1, vec![gossip(&frame)]), + sent.clone(), + ack(cursor, cursor + 1), + ]); + assert!(matches!( + driver.process_incoming_once(NOW + cursor).await.unwrap(), + CollaborationIncomingOnceOutcome::Acknowledged(_) + )); + } + assert_eq!(receipted_hashes(&carrier), [hash.clone(), hash]); + } + + /// The reproduction instrument for the 14 Nov Community burst: what each + /// gossip message costs on the wire, as `gossip_send` encodes it + /// (`carrier.rs:7008-7027`). Run it with `--nocapture`. + #[tokio::test] + async fn community_resend_reproduction_table() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let post = remote_post(&fixture, "a post of ordinary length"); + let presence = presence_frame(&fixture, &generate_keypair().0, NOW); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + carrier.push([peek(0, 1, vec![gossip(&post)]), sent, ack(0, 1)]); + driver.process_incoming_once(NOW).await.unwrap(); + let receipt = carrier + .requests() + .iter() + .find(|request| request["op"] == "gossip_send") + .map(|request| request["message"].as_str().unwrap().as_bytes().to_vec()) + .unwrap(); + let wire = |frame: &[u8]| { + serde_json::to_vec(&crate::carrier::GossipMessage { + sender_id: core.local_device_did(), + sender_nick: "unknown".to_string(), + content: String::from_utf8(frame.to_vec()).unwrap(), + ts: NOW, + nonce: u64::MAX, + signature: None, + sender_session_id: None, + }) + .unwrap() + .len() + }; + println!( + "wire bytes: presence {} post {} receipt {}", + wire(&presence), + wire(&post), + wire(&receipt) + ); + } + #[tokio::test] async fn outgoing_observations_never_replace_verified_remote_acceptance() { let fixture = Fixture::new(); From 9f35e767f16f28764e4962dab94013919e52bb1c Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 21:44:47 -0500 Subject: [PATCH 02/14] Community: a Home receipts each gossip frame once per Carrier position The Carrier replays a batch range until the Home acknowledges it. A Home that cannot acknowledge, because its intake is full or because the acknowledgement itself fails, read that range again every 500 ms and gossiped the stored acceptance receipt of every frame in it again. The driver now counts the frames it ingested from the range the Carrier holds and resumes at the first frame it has not ingested. A receipt that reaches no gossip neighbour also leaves the frame ingested: the acceptance is durable, and each sender re-sends its own frame until a receipt reaches it, which arrives at a new Carrier position and earns a fresh receipt. Nothing a Home sends changes shape, and replay protection is untouched: a frame skipped here is one this Home already accepted and recorded. --- .../src/collaboration_carrier.rs | 6 ++ .../src/collaboration_transport.rs | 97 ++++++++++++------- 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/elastos/crates/elastos-server/src/collaboration_carrier.rs b/elastos/crates/elastos-server/src/collaboration_carrier.rs index ba263d858..cb3f1540c 100644 --- a/elastos/crates/elastos-server/src/collaboration_carrier.rs +++ b/elastos/crates/elastos-server/src/collaboration_carrier.rs @@ -152,6 +152,12 @@ impl CollaborationCarrierReceiveBatch { pub fn rejected_frames(&self) -> usize { self.rejected_frames } + + /// The buffer position of the first frame in this batch. The Carrier + /// replays the batch from here until the Home acknowledges it. + pub fn cursor(&self) -> u64 { + self.cursor + } } #[derive(Deserialize)] diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index 9119ce3ab..f96be58c6 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -24,6 +24,18 @@ pub(crate) struct CollaborationTransportDriver { network: JoinedCollaborationNetwork, home_delivery: Option, gossip: Mutex, + ingested: Mutex, +} + +/// How far this Home got through the batch the Carrier holds at `cursor`. +/// The Carrier replays that exact range until the Home acknowledges it +/// (`carrier.rs:7140-7146`), so without this count a Home that cannot +/// acknowledge would ingest those frames again on every live cycle and gossip +/// each stored acceptance receipt again with it. +#[derive(Default)] +struct IngestedBatch { + cursor: u64, + frames: usize, } const SHARED_FAST_RETRY_WINDOW_SECS: u64 = 3; @@ -320,6 +332,7 @@ impl CollaborationTransportDriver { network, home_delivery: None, gossip: Mutex::new(GossipSchedule::default()), + ingested: Mutex::new(IngestedBatch::default()), } } @@ -515,10 +528,7 @@ impl CollaborationTransportDriver { // senders may never resend them, so this Home owns the retry. let mut held = self.core.take_held_frames(now).into_iter(); while let Some(frame) = held.next() { - if !self - .ingest_frame(frame.frame(), now, &mut summary, true) - .await - { + if !self.ingest_frame(frame.frame(), now, &mut summary).await { self.core .restore_held_frames(std::iter::once(frame).chain(held).collect()); return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); @@ -531,13 +541,16 @@ impl CollaborationTransportDriver { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } summary.carrier_rejected_frames = batch.rejected_frames(); - for envelope in batch.envelopes() { + let mut ingested = self.ingested_frames(batch.cursor())?; + for envelope in batch.envelopes().iter().skip(ingested) { if !self.core.community_membership().joined() { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } - if !self.ingest_frame(envelope, now, &mut summary, false).await { + if !self.ingest_frame(envelope, now, &mut summary).await { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } + ingested += 1; + self.record_ingested_frames(batch.cursor(), ingested)?; } match self.network.ack(&batch).await { @@ -546,14 +559,35 @@ impl CollaborationTransportDriver { } } - /// Ingests one frame and sends its receipt. Returns false when the cycle - /// must stop and retry this frame later. + /// Frames of the batch at `cursor` this Home has already ingested. A later + /// cursor starts its own count, and a Carrier range is always the same + /// buffer positions, so a sender's genuine re-send never lands in a count. + fn ingested_frames(&self, cursor: u64) -> anyhow::Result { + let ingested = self + .ingested + .lock() + .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))?; + Ok((ingested.cursor == cursor) + .then_some(ingested.frames) + .unwrap_or_default()) + } + + fn record_ingested_frames(&self, cursor: u64, frames: usize) -> anyhow::Result<()> { + *self + .ingested + .lock() + .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))? = + IngestedBatch { cursor, frames }; + Ok(()) + } + + /// Ingests one frame and sends its receipt. Returns false when the frame + /// is not ingested and the cycle must retry it later. async fn ingest_frame( &self, frame: &[u8], now: u64, summary: &mut CollaborationIncomingOnceSummary, - held: bool, ) -> bool { match self.core.allows_gossip_frame(frame) { Ok(true) => {} @@ -588,12 +622,14 @@ impl CollaborationTransportDriver { { Ok(CollaborationCarrierSendOutcome::RemoteBroadcast { .. }) => { summary.acceptance_receipt_broadcasts += 1; - true } - // A held frame's senders fetch its durable receipt again - // by retrying: direct by the Home call, gossip by re-broadcast. - Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) | Err(_) => held, + // The acceptance is durable, so this frame is ingested. + // Its sender keeps re-sending until a receipt reaches it, + // and each copy earns a fresh receipt of its own: direct + // by the Home call, on the topic by its new position. + Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) | Err(_) => {} } + true } } } @@ -2767,11 +2803,12 @@ mod tests { .requests() .iter() .all(|request| request["op"] != "gossip_ack")); + // The retry resumes at the frame that failed: the three rejections the + // first cycle already consumed are not read again. assert_eq!( driver.process_incoming_once(NOW).await.unwrap(), CollaborationIncomingOnceOutcome::Acknowledged(CollaborationIncomingOnceSummary { carrier_rejected_frames: 2, - deterministic_rejections: 3, incoming_acceptances: 2, acceptance_receipt_broadcasts: 2, ..CollaborationIncomingOnceSummary::default() @@ -2842,7 +2879,7 @@ mod tests { } #[tokio::test] - async fn incoming_receipt_is_durable_before_send_and_replayed_until_exact_ack() { + async fn an_incoming_receipt_is_durable_before_its_single_send() { let fixture = Fixture::new(); let core = fixture.core(); let (sender, _) = generate_keypair(); @@ -2853,16 +2890,18 @@ mod tests { .to_vec(); let carrier = FakeCarrier::new(); let driver = fixture.driver(core.clone(), carrier.clone()).await; - // A separately opened Core reads the persisted acceptance before each send. + // A separately opened Core reads the persisted acceptance before the send. carrier.observe_durable_incoming_before_send(fixture.core()); - let remote = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + // A receipt that reaches no gossip neighbour leaves the acceptance + // durable, so the batch completes and the Home stops reading it. carrier.push([ peek(0, 1, vec![gossip(&frame)]), serde_json::json!({"status":"ok","broadcast":"local_only","data":{"remote_peer_count":0}}), + ack(0, 1), ]); assert!(matches!( driver.process_incoming_once(NOW).await.unwrap(), - CollaborationIncomingOnceOutcome::RetryRequired(_) + CollaborationIncomingOnceOutcome::Acknowledged(_) )); assert_eq!(core.summary().unwrap().pending_product_handoffs, 1); core.acknowledge_product_handoff(&collaboration_message_envelope_sha256(&envelope)) @@ -2872,19 +2911,10 @@ mod tests { assert_eq!(core.summary().unwrap().replay_tombstones, 0); *carrier.durable_incoming_before_send.lock().unwrap() = None; - carrier.push([peek(0, 1, vec![gossip(&frame)])]); + // A send that fails outright ingests the frame just the same. + carrier.push([peek(1, 2, vec![gossip(&frame)])]); carrier.push_failure("receipt send failed"); - assert!(matches!( - driver.process_incoming_once(NOW + 1).await.unwrap(), - CollaborationIncomingOnceOutcome::RetryRequired(_) - )); - carrier.push([peek(0, 1, vec![gossip(&frame)]), remote.clone()]); - carrier.push_failure("ack failed"); - assert!(matches!( - driver.process_incoming_once(NOW + 1).await.unwrap(), - CollaborationIncomingOnceOutcome::RetryRequired(_) - )); - carrier.push([peek(0, 1, vec![gossip(&frame)]), remote, ack(0, 1)]); + carrier.push([ack(1, 2)]); assert!(matches!( driver.process_incoming_once(NOW + 1).await.unwrap(), CollaborationIncomingOnceOutcome::Acknowledged(_) @@ -2895,8 +2925,7 @@ mod tests { .filter(|request| request["op"] == "gossip_send") .map(|request| request["message"].as_str().unwrap().as_bytes().to_vec()) .collect(); - assert_eq!(receipts.len(), 4); - assert!(receipts.windows(2).all(|pair| pair[0] == pair[1])); + assert_eq!(receipts.len(), 2); assert!(core.allows_gossip_receipt(&receipts[0], &frame).unwrap()); assert_eq!( request_ops(&carrier), @@ -2904,10 +2933,6 @@ mod tests { "gossip_join_exact", "gossip_peek", "gossip_send", - "gossip_peek", - "gossip_send", - "gossip_peek", - "gossip_send", "gossip_ack", "gossip_peek", "gossip_send", From 847a9e0ea57eab5e2ce6cd611c6206568181e3c9 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 21:52:41 -0500 Subject: [PATCH 03/14] Community: ask for earlier posts when there is a reason to ask A shown Home asked up to four present Homes for their whole retained history every 15 s for as long as it ran. The request and reply formats stay as they are; what changes is when a Home sends one. HistoryCadence makes a pass due at once on a Community join, on a Community connection that returns, and on a Home that is present now and was absent at the last pass. A pass that none of those asked for waits twice as long as the one before it, up to 15 minutes. Ten minutes of Runtime checks now carry six passes instead of forty. --- .../src/collaboration_history.rs | 126 ++++++++++++++++++ .../src/collaboration_startup.rs | 2 +- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index 1f5c30697..e515b688d 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -1,6 +1,7 @@ //! Bounded Community catch-up through the private Runtime/Carrier provider plane. //! Live gossip admission and its replay/receipt lifetime remain unchanged. +use std::collections::BTreeSet; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -31,6 +32,10 @@ pub(crate) const MAX_HISTORY_RESPONSE_BYTES: usize = MAX_CONVERSATION_HISTORY_BY pub(crate) const MAX_HISTORY_CARRIER_REPLY_BYTES: usize = MAX_HISTORY_RESPONSE_BYTES + 64; const MAX_PEERS_PER_PASS: usize = 4; const PEER_TIMEOUT: Duration = Duration::from_secs(5); +/// Wait before the first pass that nothing else asks for. +const HISTORY_BASE_WAIT_SECS: u64 = 15; +/// Longest wait between two passes of a Home that keeps seeing the same Homes. +const HISTORY_MAX_WAIT_SECS: u64 = 900; // History requests carry this Home's own signed presence, which exists only // for people who granted Community presence. const NOT_SHOWN_DETAIL: &str = "Show yourself in Community to load earlier posts."; @@ -47,6 +52,7 @@ struct HistoryInner { registry: Arc, data_root: PathBuf, peers: Mutex, + cadence: Mutex, } #[derive(Default)] @@ -56,6 +62,41 @@ struct HistoryPeerRound { available: bool, } +/// When this Home next asks present Homes for earlier posts. Three things make +/// a pass due at once: a Community join, a Community connection that returns, +/// and a Home that is present now and was absent at the last pass, because any +/// of them can leave this Home without posts the others kept. A pass that none +/// of them asked for waits twice as long as the one before it. +#[derive(Default)] +struct HistoryCadence { + due_at: u64, + wait: u64, + connected: bool, + present: BTreeSet, +} + +impl HistoryCadence { + fn due(&mut self, now: u64, connected: bool, present: BTreeSet) -> bool { + let returned = connected && !self.connected; + let arrived = present.difference(&self.present).next().is_some(); + self.connected = connected; + if returned || arrived { + self.wait = 0; + self.due_at = now; + } + if now < self.due_at { + return false; + } + self.present = present; + self.wait = self + .wait + .saturating_mul(2) + .clamp(HISTORY_BASE_WAIT_SECS, HISTORY_MAX_WAIT_SECS); + self.due_at = now.saturating_add(self.wait); + true + } +} + #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct HistoryRequest { @@ -98,6 +139,7 @@ impl CollaborationHistoryService { registry: registry.clone(), data_root, peers: Mutex::new(HistoryPeerRound::default()), + cadence: Mutex::new(HistoryCadence::default()), }), }; registry @@ -197,6 +239,39 @@ impl CollaborationHistoryService { .project_history(&self.inner.data_root, &messages, now) } + /// Runs a pass only when this Home has a reason to ask. The Runtime calls + /// this on its own cadence; `HistoryCadence` decides which of those checks + /// reaches other Homes. + pub(crate) async fn fetch_if_due(&self, now: u64) -> anyhow::Result<()> { + if !self.inner.product.community_membership().joined() { + *self + .inner + .cadence + .lock() + .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? = + HistoryCadence::default(); + return Ok(()); + } + let connected = self.inner.product.conversation_transport_view().available; + let present = self + .inner + .presence + .history_participants(now)? + .into_iter() + .map(|peer| peer.endpoint_did) + .collect(); + let due = self + .inner + .cadence + .lock() + .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? + .due(now, connected, present); + if !due { + return Ok(()); + } + self.fetch_once(now).await + } + /// Independently scheduled from live gossip. Four unavailable peers cannot /// monopolize a pass or starve later available participants across passes. pub(crate) async fn fetch_once(&self, now: u64) -> anyhow::Result<()> { @@ -1289,6 +1364,57 @@ mod tests { } } + /// Ten minutes of Runtime checks, 15 s apart, for a Home that keeps + /// seeing the same Homes. + #[test] + fn earlier_posts_are_asked_for_on_a_growing_wait() { + let now = 1_800_000_000; + let present = BTreeSet::from(["did:peer".to_string()]); + let mut cadence = HistoryCadence::default(); + let checks = (0..40).map(|tick| now + tick * HISTORY_BASE_WAIT_SECS); + let due = checks + .clone() + .filter(|check| cadence.due(*check, true, present.clone())) + .map(|check| check - now) + .collect::>(); + assert_eq!(checks.count(), 40, "every check asked other Homes before"); + assert_eq!(due, [0, 15, 45, 105, 225, 465]); + + // A Home that was absent at the last pass is a reason to ask again. + let joined = BTreeSet::from(["did:peer".to_string(), "did:newcomer".to_string()]); + assert!(cadence.due(now + 600, true, joined.clone())); + assert!(!cadence.due(now + 601, true, joined.clone())); + // So is a Community connection that returns. + assert!(!cadence.due(now + 602, false, joined.clone())); + assert!(cadence.due(now + 603, true, joined)); + } + + #[tokio::test] + async fn a_history_check_inside_its_wait_asks_nobody() { + let now = now_secs(); + let a = fixture("history-cadence", "community", now).await; + let b = fixture("history-cadence", "community", now).await; + observe(&a, &b, now); + a.product.set_community_connected(true); + let plane = Arc::new(Plane { + source: a.product.local_device_did(), + peers: BTreeMap::from([(b.product.local_device_did(), b.registry.clone())]), + calls: Mutex::new(Vec::new()), + stall: false, + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let asks = || plane.calls.lock().unwrap().len(); + a.service.fetch_if_due(now).await.unwrap(); + assert_eq!(asks(), 1); + a.service.fetch_if_due(now + 14).await.unwrap(); + assert_eq!(asks(), 1); + a.service + .fetch_if_due(now + HISTORY_BASE_WAIT_SECS) + .await + .unwrap(); + assert_eq!(asks(), 2); + } + #[tokio::test(start_paused = true)] async fn stalled_history_has_a_per_peer_deadline() { let now = now_secs(); diff --git a/elastos/crates/elastos-server/src/collaboration_startup.rs b/elastos/crates/elastos-server/src/collaboration_startup.rs index 4f72740bb..c8cad0271 100644 --- a/elastos/crates/elastos-server/src/collaboration_startup.rs +++ b/elastos/crates/elastos-server/src/collaboration_startup.rs @@ -816,7 +816,7 @@ async fn run_collaboration_history_worker( tokio::select! { biased; _ = joined.changed() => Ok(()), - result = service.fetch_once(now_secs()) => result, + result = service.fetch_if_due(now_secs()) => result, } } => { if result.is_err() { tracing::debug!("Community history refresh is unavailable"); } From 9b9b283c08e6bb1aa8c2761bbb96f809dbf1cfe2 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 21:59:38 -0500 Subject: [PATCH 04/14] Community: write the ingested-frame lookup as Clippy asks --- .../crates/elastos-server/src/collaboration_transport.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index f96be58c6..a2da87d25 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -567,9 +567,11 @@ impl CollaborationTransportDriver { .ingested .lock() .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))?; - Ok((ingested.cursor == cursor) - .then_some(ingested.frames) - .unwrap_or_default()) + Ok(if ingested.cursor == cursor { + ingested.frames + } else { + 0 + }) } fn record_ingested_frames(&self, cursor: u64, frames: usize) -> anyhow::Result<()> { From 8bf06e92fb537284ae92651bcbf695858e7a1a71 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 01:24:47 -0500 Subject: [PATCH 05/14] test(community): show the four review findings as failing tests Each test states the behaviour a person needs. Four of them fail at 9b9b283c0 and the rest hold the limits the round-1 change keeps. - A reused consumer cursor skips its first frames, so a reconstructed topic loses a post and its receipt. - A receipt that reached nobody is never sent again, so a delivered post stays unconfirmed and its sender re-sends until expiry. - The history worker keeps its grown wait through a leave and a join. - The history cadence test counted its own iterator instead of the requests the Runtime made. --- elastos/crates/elastos-server/src/carrier.rs | 69 +++++ .../src/collaboration_history.rs | 144 ++++++++++ .../src/collaboration_startup.rs | 158 +++++++++++ .../src/collaboration_transport.rs | 245 ++++++++++++++++++ 4 files changed, 616 insertions(+) diff --git a/elastos/crates/elastos-server/src/carrier.rs b/elastos/crates/elastos-server/src/carrier.rs index 3ee10b69a..7f73aac75 100644 --- a/elastos/crates/elastos-server/src/carrier.rs +++ b/elastos/crates/elastos-server/src/carrier.rs @@ -19509,6 +19509,75 @@ pub(crate) mod tests { shutdown_test_carrier_node(node).await; } + /// A consumer cursor number is not a frame identity. `gossip_leave` drops + /// the topic buffer and every cursor on it, and the next join starts a + /// buffer at position 0, so the same consumer peeks position 0 again and + /// finds a frame it never read. The Community driver binds its progress to + /// the frames it ingested because of this. + #[tokio::test] + async fn test_gossip_leave_and_join_give_the_same_cursor_a_new_frame() { + let dir = tempfile::tempdir().unwrap(); + let (signing_key, did) = elastos_identity::derive_did(&[66u8; 32]); + let node = start_carrier_node(&signing_key, &did, dir.path().to_path_buf()) + .await + .unwrap(); + let provider = CarrierGossipProvider::new(node.gossip_state.clone()); + let topic = "__test/gossip-cursor-reuse"; + let consumer = "community-consumer"; + let join = serde_json::json!({"op":"gossip_join_exact", "topic":topic, "peers":[]}); + let peek = serde_json::json!({ + "op":"gossip_peek", "topic":topic, "consumer_id":consumer, "limit":32 + }); + let send = |message: &str| serde_json::json!({"op":"gossip_send", "topic":topic, "message":message}); + let read_one = |response: &serde_json::Value| { + ( + response["data"]["cursor"].as_u64(), + response["data"]["next_cursor"].as_u64(), + response["data"]["messages"][0]["content"] + .as_str() + .map(str::to_string), + ) + }; + + assert_eq!(provider.send_raw(&join).await.unwrap()["status"], "ok"); + assert_eq!( + provider.send_raw(&send("first frame")).await.unwrap()["status"], + "ok" + ); + assert_eq!( + read_one(&provider.send_raw(&peek).await.unwrap()), + (Some(0), Some(1), Some("first frame".to_string())) + ); + assert_eq!( + provider + .send_raw(&serde_json::json!({ + "op":"gossip_ack", "topic":topic, "consumer_id":consumer, + "cursor":0, "next_cursor":1 + })) + .await + .unwrap()["status"], + "ok" + ); + assert_eq!( + provider + .send_raw(&serde_json::json!({"op":"gossip_leave", "topic":topic})) + .await + .unwrap()["status"], + "ok" + ); + assert_eq!(provider.send_raw(&join).await.unwrap()["status"], "ok"); + assert_eq!( + provider.send_raw(&send("second frame")).await.unwrap()["status"], + "ok" + ); + assert_eq!( + read_one(&provider.send_raw(&peek).await.unwrap()), + (Some(0), Some(1), Some("second frame".to_string())), + "the recreated topic hands the same consumer cursor a frame it never read" + ); + shutdown_test_carrier_node(node).await; + } + #[tokio::test] async fn test_gossip_empty_peek_requires_exact_empty_ack() { let dir = tempfile::tempdir().unwrap(); diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index e515b688d..cb5ba7d60 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -1364,6 +1364,150 @@ mod tests { } } + /// One fresh presence announcement from this Home's person, as its + /// Runtime publishes one while the person is shown. + async fn refresh_presence( + fixture: &Fixture, + now: u64, + ) -> crate::collaboration_presence::PreparedCollaborationPresence { + let root = crate::auth::principal_localhost_root(PRINCIPAL); + let profile = crate::collaboration_profile_authority::load_profile_authority( + fixture.root.path(), + PRINCIPAL, + &root, + ) + .unwrap() + .unwrap(); + let grant = crate::collaboration_presence_grant::presence_grant( + &fixture.authority, + fixture.root.path(), + PRINCIPAL, + &fixture.profile_did, + &fixture.presence.presence_scope(), + crate::collaboration_presence_grant::PresencePublisher::Runtime, + now, + ) + .await + .unwrap(); + fixture + .presence + .prepare_presence( + presence_request_binding( + &grant.bind_request_id(&format!("presence-at-{now}")), + PRINCIPAL, + &profile, + ) + .unwrap(), + &profile, + now, + ) + .unwrap() + } + + /// A presence record lives 45 s, so both Homes keep announcing their + /// person while a measurement runs for minutes. + async fn keep_both_present(a: &Fixture, b: &Fixture, now: u64) { + let own = refresh_presence(a, now).await; + a.presence + .project_authorized_presence(&own, now) + .await + .unwrap(); + let peer = refresh_presence(b, now).await; + a.product + .test_core() + .accept_incoming_from_signed_source_for_test(peer.test_envelope_bytes(), now) + .unwrap(); + for presence in a.presence.pending_presences().unwrap() { + a.presence.project_handoff(&presence, now).unwrap(); + } + assert!( + a.presence + .history_participants(now) + .unwrap() + .iter() + .any(|participant| participant.endpoint_did == b.product.local_device_did()), + "the peer stays present through the measurement" + ); + } + + /// Counts every history request and either answers it with an empty + /// retained history or leaves the Home unreachable. + struct CadencePlane { + endpoint: String, + network_id: String, + conversation_id: String, + answers: bool, + calls: Mutex>, + } + + #[async_trait::async_trait] + impl ProviderCarrierInvoker for CadencePlane { + async fn invoke_carrier_provider( + &self, + route: &ProviderCarrierRoute, + _invocation: &ProviderInvocation, + _request: serde_json::Value, + ) -> Result { + let ProviderCarrierRoute::PeerDid { peer_did, .. } = route else { + panic!("history selects verified Profile endpoint"); + }; + self.calls.lock().unwrap().push(peer_did.clone()); + if !self.answers { + return Err(ProviderError::Unavailable("offline".to_string())); + } + Ok(serde_json::json!({ + "schema": RESPONSE_SCHEMA, + "network_id": self.network_id, + "conversation_id": self.conversation_id, + "sender_service": CHAT_SERVICE, + "endpoint_did": self.endpoint, + "envelopes": [], + })) + } + } + + /// Ten minutes of Runtime checks, measured as the Home-to-Home requests + /// they produce on the provider plane. A Home whose passes bring answers + /// backs off to a quarter of an hour. A Home whose passes reach nobody + /// keeps asking every minute, because the Homes it wants are present and + /// can become reachable at any check. + #[tokio::test] + async fn earlier_posts_are_asked_for_on_a_measured_cadence() { + for answers in [true, false] { + let now = now_secs(); + let a = fixture("history-measured", "community", now).await; + let b = fixture("history-measured", "community", now).await; + a.product.set_community_connected(true); + let (network_id, conversation_id) = a.product.history_scope(); + let plane = Arc::new(CadencePlane { + endpoint: b.product.local_device_did(), + network_id: network_id.to_string(), + conversation_id: conversation_id.to_string(), + answers, + calls: Mutex::new(Vec::new()), + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let mut asked_at = Vec::new(); + let mut asks = 0; + for tick in 0..41 { + let at = now + tick * HISTORY_BASE_WAIT_SECS; + keep_both_present(&a, &b, at).await; + a.service.fetch_if_due(at).await.unwrap(); + let calls = plane.calls.lock().unwrap().len(); + if calls > asks { + asked_at.push(at - now); + asks = calls; + } + } + let expected: Vec = if answers { + vec![0, 15, 45, 105, 225, 465] + } else { + vec![0, 15, 45, 105, 165, 225, 285, 345, 405, 465, 525, 585] + }; + assert_eq!(asked_at, expected, "answers: {answers}"); + } + } + /// Ten minutes of Runtime checks, 15 s apart, for a Home that keeps /// seeing the same Homes. #[test] diff --git a/elastos/crates/elastos-server/src/collaboration_startup.rs b/elastos/crates/elastos-server/src/collaboration_startup.rs index c8cad0271..77796b51b 100644 --- a/elastos/crates/elastos-server/src/collaboration_startup.rs +++ b/elastos/crates/elastos-server/src/collaboration_startup.rs @@ -2821,6 +2821,164 @@ mod tests { assert!(service.history_task.is_none()); } + struct CountingHistoryInvoker { + asked: Arc, + calls: Arc, + } + + #[async_trait::async_trait] + impl elastos_runtime::provider::ProviderCarrierInvoker for CountingHistoryInvoker { + async fn invoke_carrier_provider( + &self, + _route: &elastos_runtime::provider::ProviderCarrierRoute, + invocation: &elastos_runtime::provider::ProviderInvocation, + _request: serde_json::Value, + ) -> Result { + assert_eq!( + invocation.target, + crate::collaboration_history::HISTORY_PROVIDER + ); + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.asked.notify_one(); + Err(ProviderError::Unavailable("offline".to_string())) + } + } + + /// A person leaves Community and joins it again. The history worker asks + /// for earlier posts at once, inside the wait its first pass set. The + /// test runs the real worker loop, so it covers the place where the + /// worker observes the membership change. + #[tokio::test(start_paused = true)] + async fn the_history_worker_asks_again_after_a_leave_and_a_join() { + for check_while_left in [false, true] { + let (temp, _) = configured_root(true); + let configuration = + load_and_accept_collaboration_startup_configuration(temp.path()).unwrap(); + let CollaborationNetworkConfiguration::Configured { + profile, + grant: Some(grant), + } = configuration.configuration + else { + panic!("expected configured conversation"); + }; + let now = now_secs(); + let (key, _) = generate_keypair(); + let core = Arc::new( + CollaborationCore::new( + temp.path(), + key.clone(), + (*profile).clone(), + grant.clone(), + CHAT_ROOM_CAPSULE, + ) + .unwrap(), + ); + let chat = CollaborationChatProductPort::new(core.clone()).unwrap(); + let presence = CollaborationPresenceProductPort::new(core.clone()) + .unwrap() + .with_presence_authority(crate::collaboration_presence_grant::test_authority()); + let (local_profile, presence_grant) = + grant_local_presence(temp.path(), &presence, "test-principal", now).await; + let prepared = presence + .prepare_presence( + presence_request_binding( + &presence_grant.bind_request_id("local-history-presence"), + "test-principal", + &local_profile, + ) + .unwrap(), + &local_profile, + now, + ) + .unwrap(); + presence.project_prepared_presence(&prepared, now).unwrap(); + let (remote_key, _) = generate_keypair(); + let remote_profile = profile_for_endpoint(&remote_key, "Remote"); + let remote = + DefaultConversationDeviceAuthority::new(remote_key, (*profile).clone(), grant) + .unwrap(); + let announcement = remote + .prepare_profile_outgoing( + &remote_profile, + SERVICE, + "elastos.chat.presence/v1", + serde_json::json!({}), + now, + // The clock this worker reads stands still in this test, + // so the record stays live and a presence lapse cannot + // explain a later pass. + 45, + ) + .unwrap(); + core.accept_incoming_from_signed_source_for_test(announcement.envelope_bytes(), now) + .unwrap(); + for handoff in presence.pending_presences().unwrap() { + presence.project_handoff(&handoff, now).unwrap(); + } + let registry = Arc::new(ProviderRegistry::new()); + let asked = Arc::new(tokio::sync::Notify::new()); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + registry + .set_carrier_invoker(Arc::new(CountingHistoryInvoker { + asked: asked.clone(), + calls: calls.clone(), + })) + .await; + let history = CollaborationHistoryService::new( + chat.clone(), + presence.clone(), + registry, + temp.path().to_path_buf(), + ) + .await + .unwrap(); + let membership = chat.community_membership().clone(); + membership.set_joined(true).unwrap(); + let (shutdown, shutdown_rx) = watch::channel(false); + let history_task = tokio::spawn(run_collaboration_history_worker(history, shutdown_rx)); + // The first pass reaches nobody and sets a wait. The clock the + // worker reads does not move in this test, so only a reset can + // make a second pass due. + tokio::time::timeout(Duration::from_secs(60), asked.notified()) + .await + .unwrap(); + if check_while_left { + // The worker runs one check at 15 s while the person is away. + membership.set_joined(false).unwrap(); + tokio::time::sleep(Duration::from_secs(20)).await; + membership.set_joined(true).unwrap(); + } else { + // The leave and the join both fall inside one worker wait. + membership.set_joined(false).unwrap(); + membership.set_joined(true).unwrap(); + } + tokio::time::timeout(Duration::from_secs(60), asked.notified()) + .await + .expect("the worker asks again after the join"); + let mut service = CollaborationRuntimeService { + shutdown, + task: None, + discovery_task: None, + presence_task: None, + history_task: Some(history_task), + product_port: Some(chat), + presence_port: Some(presence), + discovery_service: None, + worker_product_port: None, + worker_presence_port: None, + }; + tokio::time::timeout(Duration::from_millis(500), service.shutdown()) + .await + .unwrap() + .unwrap(); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 2, + "check_while_left: {check_while_left}" + ); + } + } + #[tokio::test(start_paused = true)] async fn worker_projects_received_chat_while_outgoing_or_receipt_send_is_blocked() { for blocked_outgoing in [true, false] { diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index a2da87d25..b3e7d2db6 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -3057,6 +3057,19 @@ mod tests { }) .await; assert_eq!(per_cycle, [PINNED_WINDOW, 0, 0, 0, 0]); + + // Local-only delivery makes progress by itself: with the + // acknowledgement working, this Home completes the batch and moves on. + let receipts = receipted_hashes(&carrier).len(); + carrier.push([ + peek(0, window.len() as u64, window.clone()), + ack(0, window.len() as u64), + ]); + assert!(matches!( + driver.process_incoming_once(NOW).await.unwrap(), + CollaborationIncomingOnceOutcome::Acknowledged(_) + )); + assert_eq!(receipted_hashes(&carrier).len(), receipts); } /// A sender re-sends its own post on its back-off until a Home receipts @@ -3086,6 +3099,238 @@ mod tests { assert_eq!(receipted_hashes(&carrier), [hash.clone(), hash]); } + /// A post this Home never read must be ingested and receipted, even when + /// it arrives at a Carrier position an earlier batch already used. A + /// `gossip_leave` drops the topic buffer and its cursors + /// (`carrier.rs:6957-6988`) and the next join starts a buffer at position + /// 0 (`carrier.rs:1344-1350`), which + /// `test_gossip_leave_and_join_give_the_same_cursor_a_new_frame` proves. + #[tokio::test] + async fn a_reused_cursor_ingests_its_frame_after_an_acknowledged_range() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (first_key, first_message) = + remote_message(&fixture, generate_keypair().0, "before the leave"); + let (next_key, next_message) = + remote_message(&fixture, generate_keypair().0, "after the join"); + let first = sign_collaboration_transport_frame(&first_key, &first_message).unwrap(); + let next = sign_collaboration_transport_frame(&next_key, &next_message).unwrap(); + let first_hash = collaboration_message_envelope_sha256(&first_message); + let next_hash = collaboration_message_envelope_sha256(&next_message); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + + for (second, frame) in [(0, &first), (1, &next), (2, &first)] { + carrier.push([peek(0, 1, vec![gossip(frame)]), sent.clone(), ack(0, 1)]); + driver.process_incoming_once(NOW + second).await.unwrap(); + } + + assert_eq!( + receipted_hashes(&carrier), + [first_hash.clone(), next_hash, first_hash], + "each frame at the reused position earns its own receipt" + ); + assert_eq!( + request_ops(&carrier), + [ + "gossip_join_exact", + "gossip_peek", + "gossip_send", + "gossip_ack", + "gossip_peek", + "gossip_send", + "gossip_ack", + "gossip_peek", + "gossip_send", + "gossip_ack", + ] + ); + assert_eq!( + core.summary().unwrap().pending_product_handoffs, + 2, + "both posts reached the product, and the replay added no third row" + ); + } + + /// The same reused position, with the earlier range still unacknowledged. + /// The held progress belongs to frames that are no longer there. + #[tokio::test] + async fn a_reused_cursor_ingests_its_frame_after_a_held_range() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (first_key, first_message) = + remote_message(&fixture, generate_keypair().0, "held before the leave"); + let (next_key, next_message) = + remote_message(&fixture, generate_keypair().0, "after the join"); + let first = sign_collaboration_transport_frame(&first_key, &first_message).unwrap(); + let next = sign_collaboration_transport_frame(&next_key, &next_message).unwrap(); + let first_hash = collaboration_message_envelope_sha256(&first_message); + let next_hash = collaboration_message_envelope_sha256(&next_message); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + + carrier.push([peek(0, 1, vec![gossip(&first)]), sent.clone()]); + carrier.push_failure("acknowledgement failed"); + assert!(matches!( + driver.process_incoming_once(NOW).await.unwrap(), + CollaborationIncomingOnceOutcome::RetryRequired(_) + )); + carrier.push([peek(0, 1, vec![gossip(&next)]), sent, ack(0, 1)]); + driver.process_incoming_once(NOW + 1).await.unwrap(); + + assert_eq!(receipted_hashes(&carrier), [first_hash, next_hash]); + assert_eq!(core.summary().unwrap().pending_product_handoffs, 2); + } + + /// A receipt that reached no neighbour stays this Home's to send. A + /// gossip-only post leaves its sender at 0, 5, 15, 35, 75, 135, 195 and + /// 255 s of its 300 s lifetime, so a receipt dropped at 255 s leaves a + /// post that this Home holds durably showing "Sending" until it expires. + #[tokio::test(start_paused = true)] + async fn a_receipt_that_reached_nobody_is_sent_again_until_the_post_expires() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let (key, message) = remote_message(&fixture, generate_keypair().0, "a post received late"); + let frame = sign_collaboration_transport_frame(&key, &message).unwrap(); + let hash = collaboration_message_envelope_sha256(&message); + let local_only = serde_json::json!({ + "status":"ok","broadcast":"local_only","data":{"remote_peer_count":0} + }); + // The sender's last gossip attempt; the post expires 45 s later. + let received_at = NOW + 255; + carrier.push([ + peek(0, 1, vec![gossip(&frame)]), + local_only.clone(), + ack(0, 1), + ]); + assert!(matches!( + driver.process_incoming_once(received_at).await.unwrap(), + CollaborationIncomingOnceOutcome::Acknowledged(_) + )); + assert_eq!(receipted_hashes(&carrier), [hash.clone()]); + + let mut sent_at = Vec::new(); + let mut sends = 1; + for second in 0..=60 { + carrier.push([local_only.clone()]); + driver + .retry_outgoing_once(received_at + second) + .await + .unwrap(); + carrier.drain_replies(); + let total = receipted_hashes(&carrier).len(); + if total > sends { + sent_at.push(second); + sends = total; + } + tokio::time::advance(Duration::from_secs(1)).await; + } + assert_eq!( + sent_at, + [5, 15, 35], + "the receipt waits 5, 10 then 20 s, and the next attempt would \ + fall after the post expires" + ); + assert!(receipted_hashes(&carrier) + .iter() + .all(|receipted| *receipted == hash)); + } + + /// A receipt a neighbour took is never sent again. The limit this change + /// keeps is that a stuck batch costs one receipt per frame, not one per + /// 500 ms cycle. + #[tokio::test(start_paused = true)] + async fn a_receipt_that_reached_a_neighbour_is_not_sent_again() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let frame = remote_post(&fixture, "a post receipted at once"); + let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + carrier.push([peek(0, 1, vec![gossip(&frame)]), sent.clone(), ack(0, 1)]); + driver.process_incoming_once(NOW).await.unwrap(); + assert_eq!(receipted_hashes(&carrier).len(), 1); + for second in 0..=60 { + carrier.push([sent.clone()]); + driver.retry_outgoing_once(NOW + second).await.unwrap(); + carrier.drain_replies(); + tokio::time::advance(Duration::from_secs(1)).await; + } + assert_eq!( + receipted_hashes(&carrier).len(), + 1, + "a receipt that went out is not retried" + ); + } + + /// The receipts that reached nobody wait in a bounded queue: as many as + /// the intake holds accepted posts, and four sends in one live cycle. + #[tokio::test(start_paused = true)] + async fn receipts_that_reached_nobody_wait_in_a_bounded_queue() { + let fixture = Fixture::new(); + let core = fixture.core(); + let carrier = FakeCarrier::new(); + let driver = fixture.driver(core.clone(), carrier.clone()).await; + let local_only = serde_json::json!({ + "status":"ok","broadcast":"local_only","data":{"remote_peer_count":0} + }); + let overflow = 8; + let posts = crate::collaboration_core::MAX_PENDING_INCOMING + overflow; + let mut hashes = Vec::new(); + for index in 0..posts { + let (key, message) = + remote_message(&fixture, generate_keypair().0, &format!("post-{index}")); + let frame = sign_collaboration_transport_frame(&key, &message).unwrap(); + let hash = collaboration_message_envelope_sha256(&message); + let cursor = index as u64; + carrier.push([ + peek(cursor, cursor + 1, vec![gossip(&frame)]), + local_only.clone(), + ack(cursor, cursor + 1), + ]); + driver.process_incoming_once(NOW).await.unwrap(); + // The product takes each post, so the intake never refuses one. + core.acknowledge_product_handoff(&hash).unwrap(); + hashes.push(hash); + } + let first_send = receipted_hashes(&carrier); + assert_eq!(first_send.len(), posts); + + tokio::time::advance(Duration::from_secs(5)).await; + let mut per_cycle = Vec::new(); + let mut sends = posts; + for _ in 0..20 { + carrier.push(std::iter::repeat_n(local_only.clone(), 8)); + driver.retry_outgoing_once(NOW + 5).await.unwrap(); + carrier.drain_replies(); + let total = receipted_hashes(&carrier).len(); + per_cycle.push(total - sends); + sends = total; + } + let retried = receipted_hashes(&carrier)[posts..].to_vec(); + assert!( + per_cycle + .iter() + .all(|count| *count <= MAX_CUSTODY_RETRIES_PER_CYCLE), + "a cycle sends at most four held receipts: {per_cycle:?}" + ); + assert_eq!( + retried.iter().collect::>().len(), + crate::collaboration_core::MAX_PENDING_INCOMING, + "the queue holds as many receipts as the intake holds posts" + ); + assert!( + hashes[..overflow] + .iter() + .all(|hash| !retried.contains(hash)), + "a full queue drops its oldest receipt for the newest one" + ); + } + /// The reproduction instrument for the 14 Nov Community burst: what each /// gossip message costs on the wire, as `gossip_send` encodes it /// (`carrier.rs:7008-7027`). Run it with `--nocapture`. From 0f1f95cf7bf2f3d2d1ebf4e35691dc454f6cc889 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 01:34:51 -0500 Subject: [PATCH 06/14] test(community): state the reused-cursor and cadence failures directly The reused-cursor test now reports the missing receipt instead of the reply its skipped send left behind, and the two cadence measurements stand as their own tests. Co-authored-by: Cursor --- .../src/collaboration_history.rs | 86 +++++++++++-------- .../src/collaboration_transport.rs | 12 ++- 2 files changed, 59 insertions(+), 39 deletions(-) diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index cb5ba7d60..8423383b2 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -1466,46 +1466,56 @@ mod tests { } } - /// Ten minutes of Runtime checks, measured as the Home-to-Home requests - /// they produce on the provider plane. A Home whose passes bring answers - /// backs off to a quarter of an hour. A Home whose passes reach nobody - /// keeps asking every minute, because the Homes it wants are present and - /// can become reachable at any check. - #[tokio::test] - async fn earlier_posts_are_asked_for_on_a_measured_cadence() { - for answers in [true, false] { - let now = now_secs(); - let a = fixture("history-measured", "community", now).await; - let b = fixture("history-measured", "community", now).await; - a.product.set_community_connected(true); - let (network_id, conversation_id) = a.product.history_scope(); - let plane = Arc::new(CadencePlane { - endpoint: b.product.local_device_did(), - network_id: network_id.to_string(), - conversation_id: conversation_id.to_string(), - answers, - calls: Mutex::new(Vec::new()), - }); - a.registry.set_carrier_invoker(plane.clone()).await; - let mut asked_at = Vec::new(); - let mut asks = 0; - for tick in 0..41 { - let at = now + tick * HISTORY_BASE_WAIT_SECS; - keep_both_present(&a, &b, at).await; - a.service.fetch_if_due(at).await.unwrap(); - let calls = plane.calls.lock().unwrap().len(); - if calls > asks { - asked_at.push(at - now); - asks = calls; - } + /// Ten minutes of Runtime checks, 15 s apart, measured as the seconds at + /// which the Runtime asked another Home for earlier posts. + async fn seconds_of_history_asks(answers: bool) -> Vec { + let now = now_secs(); + let a = fixture("history-measured", "community", now).await; + let b = fixture("history-measured", "community", now).await; + a.product.set_community_connected(true); + let (network_id, conversation_id) = a.product.history_scope(); + let plane = Arc::new(CadencePlane { + endpoint: b.product.local_device_did(), + network_id: network_id.to_string(), + conversation_id: conversation_id.to_string(), + answers, + calls: Mutex::new(Vec::new()), + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let mut asked_at = Vec::new(); + let mut asks = 0; + for tick in 0..41 { + let at = now + tick * HISTORY_BASE_WAIT_SECS; + keep_both_present(&a, &b, at).await; + a.service.fetch_if_due(at).await.unwrap(); + let calls = plane.calls.lock().unwrap().len(); + if calls > asks { + asked_at.push(at - now); + asks = calls; } - let expected: Vec = if answers { - vec![0, 15, 45, 105, 225, 465] - } else { - vec![0, 15, 45, 105, 165, 225, 285, 345, 405, 465, 525, 585] - }; - assert_eq!(asked_at, expected, "answers: {answers}"); } + asked_at + } + + /// A Home whose passes bring answers backs off towards a quarter of an + /// hour, because the Homes it reached already gave it their posts. + #[tokio::test] + async fn answered_history_passes_back_off_to_the_long_wait() { + assert_eq!( + seconds_of_history_asks(true).await, + [0, 15, 45, 105, 225, 465] + ); + } + + /// A Home whose passes reach nobody keeps asking every minute. The Homes + /// it wants are present and can become reachable at any check, so a + /// person who opens Community waits one minute, not a quarter of an hour. + #[tokio::test] + async fn unanswered_history_passes_keep_the_short_wait() { + assert_eq!( + seconds_of_history_asks(false).await, + [0, 15, 45, 105, 165, 225, 285, 345, 405, 465, 525, 585] + ); } /// Ten minutes of Runtime checks, 15 s apart, for a Home that keeps diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index b3e7d2db6..3eb022b9e 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -3121,9 +3121,14 @@ mod tests { let next_hash = collaboration_message_envelope_sha256(&next_message); let sent = serde_json::json!({"status":"ok","data":{"remote_peer_count":1}}); + let mut acknowledged = Vec::new(); for (second, frame) in [(0, &first), (1, &next), (2, &first)] { + carrier.drain_replies(); carrier.push([peek(0, 1, vec![gossip(frame)]), sent.clone(), ack(0, 1)]); - driver.process_incoming_once(NOW + second).await.unwrap(); + acknowledged.push(matches!( + driver.process_incoming_once(NOW + second).await, + Ok(CollaborationIncomingOnceOutcome::Acknowledged(_)) + )); } assert_eq!( @@ -3131,6 +3136,11 @@ mod tests { [first_hash.clone(), next_hash, first_hash], "each frame at the reused position earns its own receipt" ); + assert_eq!( + acknowledged, + [true, true, true], + "every cycle completes its range" + ); assert_eq!( request_ops(&carrier), [ From 07d06b66e0a63cb17aa2b488e51416f31d2d2bd4 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 01:46:42 -0500 Subject: [PATCH 07/14] fix(community): bind the ingested-frame record to the frames themselves A Carrier position belongs to a topic buffer, and a leave and a join recreate that buffer from position 0. The record now names each ingested frame by its digest and holds one batch, so a frame at a position an earlier batch used is ingested and receipted. The acknowledged range drops the record, because the Carrier never replays it. Co-authored-by: Cursor --- .../src/collaboration_carrier.rs | 2 +- .../src/collaboration_transport.rs | 87 ++++++++++++++----- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/elastos/crates/elastos-server/src/collaboration_carrier.rs b/elastos/crates/elastos-server/src/collaboration_carrier.rs index cb3f1540c..4592f51a9 100644 --- a/elastos/crates/elastos-server/src/collaboration_carrier.rs +++ b/elastos/crates/elastos-server/src/collaboration_carrier.rs @@ -22,7 +22,7 @@ use crate::collaboration_protocol::{ const COLLABORATION_TOPIC_DOMAIN: &[u8] = b"elastos.collaboration.carrier.topic.v1"; const COLLABORATION_TOPIC_PREFIX: &str = "__elastos_internal/collaboration-v1/"; -const COLLABORATION_RECEIVE_BATCH_SIZE: usize = 32; +pub(crate) const COLLABORATION_RECEIVE_BATCH_SIZE: usize = 32; const MAX_RECEIVE_RESPONSE_BYTES: usize = MAX_COLLABORATION_TRANSPORT_FRAME_BYTES * COLLABORATION_RECEIVE_BATCH_SIZE + 64 * 1024; /// How long a joined Home may have no Community peer before it repairs the diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index 3eb022b9e..c0e19aa3f 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -10,9 +10,12 @@ use elastos_runtime::provider::{ ProviderRegistry, ProviderTransfer, ResourceRequest, ResourceResponse, }; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use tokio::time::Instant; -use crate::collaboration_carrier::{CollaborationCarrierSendOutcome, JoinedCollaborationNetwork}; +use crate::collaboration_carrier::{ + CollaborationCarrierSendOutcome, JoinedCollaborationNetwork, COLLABORATION_RECEIVE_BATCH_SIZE, +}; use crate::collaboration_core::{CollaborationCore, CollaborationTransportIngestion}; use crate::collaboration_presence::CollaborationPresenceProductPort; use crate::collaboration_protocol::{ @@ -27,15 +30,26 @@ pub(crate) struct CollaborationTransportDriver { ingested: Mutex, } -/// How far this Home got through the batch the Carrier holds at `cursor`. -/// The Carrier replays that exact range until the Home acknowledges it -/// (`carrier.rs:7140-7146`), so without this count a Home that cannot +/// The frames this Home already ingested from the batch the Carrier holds at +/// `cursor`. The Carrier replays that exact range until the Home acknowledges +/// it (`carrier.rs:7140-7146`), so without this record a Home that cannot /// acknowledge would ingest those frames again on every live cycle and gossip /// each stored acceptance receipt again with it. +/// +/// Each entry names its own frame by digest, because a Carrier position +/// belongs to a topic buffer that a leave and a join recreate from position 0 +/// (`carrier.rs:6957-6988`, `carrier.rs:1344-1350`). A position alone would +/// let this record hide a frame it was never counted on. The record holds one +/// batch, so it stays at or under `COLLABORATION_RECEIVE_BATCH_SIZE` digests, +/// and the acknowledged range drops it. #[derive(Default)] struct IngestedBatch { cursor: u64, - frames: usize, + digests: Vec<[u8; 32]>, +} + +fn transport_frame_digest(frame: &[u8]) -> [u8; 32] { + Sha256::digest(frame).into() } const SHARED_FAST_RETRY_WINDOW_SECS: u64 = 3; @@ -541,7 +555,7 @@ impl CollaborationTransportDriver { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } summary.carrier_rejected_frames = batch.rejected_frames(); - let mut ingested = self.ingested_frames(batch.cursor())?; + let mut ingested = self.ingested_frames(batch.cursor(), batch.envelopes())?; for envelope in batch.envelopes().iter().skip(ingested) { if !self.core.community_membership().joined() { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); @@ -549,37 +563,71 @@ impl CollaborationTransportDriver { if !self.ingest_frame(envelope, now, &mut summary).await { return Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)); } + self.record_ingested_frame(batch.cursor(), ingested, envelope)?; ingested += 1; - self.record_ingested_frames(batch.cursor(), ingested)?; } match self.network.ack(&batch).await { - Ok(()) => Ok(CollaborationIncomingOnceOutcome::Acknowledged(summary)), + Ok(()) => { + self.forget_ingested_frames()?; + Ok(CollaborationIncomingOnceOutcome::Acknowledged(summary)) + } Err(_) => Ok(CollaborationIncomingOnceOutcome::RetryRequired(summary)), } } - /// Frames of the batch at `cursor` this Home has already ingested. A later - /// cursor starts its own count, and a Carrier range is always the same - /// buffer positions, so a sender's genuine re-send never lands in a count. - fn ingested_frames(&self, cursor: u64) -> anyhow::Result { + /// How many frames at the front of this batch the record already names. + /// The count stops at the first frame the record does not name, so a + /// Carrier position that now carries another frame earns its own + /// ingestion, and a sender's genuine re-send at a new position does too. + fn ingested_frames(&self, cursor: u64, envelopes: &[Vec]) -> anyhow::Result { let ingested = self .ingested .lock() .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))?; - Ok(if ingested.cursor == cursor { - ingested.frames - } else { - 0 - }) + if ingested.cursor != cursor { + return Ok(0); + } + Ok(envelopes + .iter() + .zip(ingested.digests.iter()) + .take_while(|(envelope, digest)| transport_frame_digest(envelope) == **digest) + .count()) + } + + /// Names the frame at `position` of the batch at `cursor` as ingested. + fn record_ingested_frame( + &self, + cursor: u64, + position: usize, + frame: &[u8], + ) -> anyhow::Result<()> { + let mut ingested = self + .ingested + .lock() + .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))?; + if ingested.cursor != cursor { + *ingested = IngestedBatch { + cursor, + digests: Vec::new(), + }; + } + ingested.digests.truncate(position); + if ingested.digests.len() == position && position < COLLABORATION_RECEIVE_BATCH_SIZE { + ingested.digests.push(transport_frame_digest(frame)); + } + Ok(()) } - fn record_ingested_frames(&self, cursor: u64, frames: usize) -> anyhow::Result<()> { + /// Drops the record once the Carrier accepts the range. The Carrier never + /// replays an acknowledged range, so the record has nothing left to say, + /// and a later range at the same position starts from nothing. + fn forget_ingested_frames(&self) -> anyhow::Result<()> { *self .ingested .lock() .map_err(|_| anyhow::anyhow!("ingested batch lock poisoned"))? = - IngestedBatch { cursor, frames }; + IngestedBatch::default(); Ok(()) } @@ -665,7 +713,6 @@ mod tests { }; use elastos_runtime::provider::ProviderCarrierInvoker; use elastos_runtime::signature::{generate_keypair, SigningKey}; - use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; const NETWORK: &str = "collaboration-transport-test"; From 7bf5e631851072b6c196fd16bc09f7977dc6933c Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 01:51:18 -0500 Subject: [PATCH 08/14] fix(community): send an owed acceptance receipt again until the post expires A receipt that reached nobody left the sender with a delivered post it believed was unsent, so it re-sent the post to the whole topic until the post expired. The receipt now waits in its own queue: 5, 10, 20, 40 and then 60 s between sends, at most 32 posts and 1 MiB of frames, at most four sends in one live cycle, and no send after the post expires. A receipt a neighbour took leaves the queue at once. Co-authored-by: Cursor --- .../elastos-server/src/collaboration_core.rs | 16 ++ .../src/collaboration_transport.rs | 206 +++++++++++++++++- 2 files changed, 214 insertions(+), 8 deletions(-) diff --git a/elastos/crates/elastos-server/src/collaboration_core.rs b/elastos/crates/elastos-server/src/collaboration_core.rs index eb41e6e03..6f8f7e5af 100644 --- a/elastos/crates/elastos-server/src/collaboration_core.rs +++ b/elastos/crates/elastos-server/src/collaboration_core.rs @@ -200,6 +200,8 @@ impl DurableIncomingAcceptance { pub(crate) struct CollaborationTransportIncomingAcceptance { acceptance_receipt_bytes: Vec, product_handoff_pending: bool, + message_hash: String, + message_expires_at: u64, } impl CollaborationTransportIncomingAcceptance { @@ -207,6 +209,18 @@ impl CollaborationTransportIncomingAcceptance { &self.acceptance_receipt_bytes } + /// The accepted post this receipt confirms. + pub(crate) fn message_hash(&self) -> &str { + &self.message_hash + } + + /// When the accepted post expires. A receipt for it is worth sending + /// until then and no longer: after that second the sender has dropped the + /// post and nobody waits for the confirmation. + pub(crate) fn message_expires_at(&self) -> u64 { + self.message_expires_at + } + #[cfg_attr( not(test), expect( @@ -3122,6 +3136,8 @@ fn transport_incoming_acceptance( accepted: DurableIncomingAcceptance, ) -> CollaborationTransportIncomingAcceptance { CollaborationTransportIncomingAcceptance { + message_hash: accepted.authorized.message().envelope_sha256().to_string(), + message_expires_at: accepted.authorized.message().envelope().payload.expires_at, acceptance_receipt_bytes: accepted.acceptance_receipt_bytes, product_handoff_pending: accepted.product_handoff_pending, } diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index c0e19aa3f..1856d040a 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -1,6 +1,6 @@ //! Bounded Runtime driver for one durable collaboration core and Carrier subscription. -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -28,6 +28,7 @@ pub(crate) struct CollaborationTransportDriver { home_delivery: Option, gossip: Mutex, ingested: Mutex, + receipts: Mutex, } /// The frames this Home already ingested from the batch the Carrier holds at @@ -56,6 +57,9 @@ const SHARED_FAST_RETRY_WINDOW_SECS: u64 = 3; const SHARED_CUSTODY_FAST_RETRY: Duration = Duration::from_millis(500); const SHARED_CUSTODY_RETRY: Duration = Duration::from_secs(5); const MAX_CUSTODY_RETRIES_PER_CYCLE: usize = 4; +/// One held receipt for each post the intake can hold. +const MAX_HELD_RECEIPTS: usize = crate::collaboration_core::MAX_PENDING_INCOMING; +const MAX_HELD_RECEIPT_BYTES: usize = 1024 * 1024; pub(crate) const SHARED_PROVIDER: &str = "collaboration.shared"; const SHARED_OP: &str = "deliver"; const MAX_SHARED_DELIVERY_BYTES: usize = MAX_COLLABORATION_TRANSPORT_FRAME_BYTES * 6 + 2048; @@ -84,6 +88,61 @@ struct GossipSchedule { sent: HashMap, } +/// The acceptance receipts this Home owes senders whose posts it holds +/// durably and whose receipt reached nobody. The post is delivered, so the +/// sender needs the confirmation and nothing else: this queue sends it again +/// without the sender re-sending the post to every Home on the topic. +/// +/// The queue holds at most `MAX_HELD_RECEIPTS` posts, as many as the intake +/// itself holds, and at most `MAX_HELD_RECEIPT_BYTES` of frames. The oldest +/// entry leaves first, because its sender has had the most time to re-send. +#[derive(Default)] +struct ReceiptQueue { + held: VecDeque, + bytes: usize, +} + +struct HeldReceipt { + message_hash: String, + receipt_frame: Vec, + /// The post this receipt confirms, as its sender signed it. The retry + /// presents it to the same gate the first send passed. + original_frame: Vec, + expires_at: u64, + last_send: Instant, + sends: u32, +} + +impl HeldReceipt { + fn bytes(&self) -> usize { + self.receipt_frame.len() + self.original_frame.len() + } +} + +impl ReceiptQueue { + fn retain(&mut self, keep: impl Fn(&HeldReceipt) -> bool) { + let mut dropped = 0; + self.held.retain(|held| { + if keep(held) { + return true; + } + dropped += held.bytes(); + false + }); + self.bytes = self.bytes.saturating_sub(dropped); + } + + fn drop_oldest(&mut self) -> bool { + match self.held.pop_front() { + Some(held) => { + self.bytes = self.bytes.saturating_sub(held.bytes()); + true + } + None => false, + } + } +} + /// Wait after `sends` gossip broadcasts of one original. Presence keeps the /// slow cadence; a Shared post backs off 5, 10, 20, 40, then 60 s until a /// Home receipt. @@ -347,6 +406,7 @@ impl CollaborationTransportDriver { home_delivery: None, gossip: Mutex::new(GossipSchedule::default()), ingested: Mutex::new(IngestedBatch::default()), + receipts: Mutex::new(ReceiptQueue::default()), } } @@ -412,9 +472,123 @@ impl CollaborationTransportDriver { summary.attempted += homes.attempted; summary.remote_broadcasts += homes.remote_broadcasts; summary.send_failures += homes.send_failures; + // The owed receipts follow the originals on the same Carrier link, so + // one send waits for the one before it. + let receipts = self.retry_receipts_once(now).await?; + summary.attempted += receipts.attempted; + summary.remote_broadcasts += receipts.remote_broadcasts; + summary.local_only_buffered += receipts.local_only_buffered; + summary.send_failures += receipts.send_failures; + Ok(summary) + } + + /// Sends the acceptance receipts that reached nobody again. Each one waits + /// 5, 10, 20, 40 and then 60 s between sends and leaves the queue when its + /// post expires, so a sender learns its post arrived without re-sending + /// the post itself. One live cycle sends at most + /// `MAX_CUSTODY_RETRIES_PER_CYCLE` of them, the bound a pending original + /// already uses. + async fn retry_receipts_once( + &self, + now: u64, + ) -> anyhow::Result { + if !self.core.community_membership().joined() { + return Ok(CollaborationOutgoingRetrySummary::default()); + } + let attempted_at = Instant::now(); + let due = { + let mut queue = self + .receipts + .lock() + .map_err(|_| anyhow::anyhow!("held receipt lock poisoned"))?; + queue.retain(|held| now < held.expires_at); + let mut due = Vec::new(); + for held in queue.held.iter_mut() { + if due.len() == MAX_CUSTODY_RETRIES_PER_CYCLE { + break; + } + if attempted_at.duration_since(held.last_send) + < gossip_resend_interval(true, held.sends) + { + continue; + } + held.last_send = attempted_at; + held.sends = held.sends.saturating_add(1); + due.push(( + held.message_hash.clone(), + held.receipt_frame.clone(), + held.original_frame.clone(), + )); + } + due + }; + let mut summary = CollaborationOutgoingRetrySummary::default(); + for (message_hash, receipt_frame, original_frame) in due { + self.core.community_membership().require_joined()?; + summary.attempted += 1; + match self + .network + .send_gossip(&self.core, &receipt_frame, Some(&original_frame)) + .await + { + Ok(CollaborationCarrierSendOutcome::RemoteBroadcast { .. }) => { + summary.remote_broadcasts += 1; + self.release_receipt(&message_hash)?; + } + Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) => { + summary.local_only_buffered += 1; + } + Err(_) => { + summary.send_failures += 1; + } + } + } Ok(summary) } + /// Keeps the receipt for a post this Home holds durably and whose receipt + /// reached nobody. + fn hold_receipt( + &self, + message_hash: &str, + receipt_frame: &[u8], + original_frame: &[u8], + expires_at: u64, + ) -> anyhow::Result<()> { + let held = HeldReceipt { + message_hash: message_hash.to_string(), + receipt_frame: receipt_frame.to_vec(), + original_frame: original_frame.to_vec(), + expires_at, + last_send: Instant::now(), + sends: 1, + }; + let mut queue = self + .receipts + .lock() + .map_err(|_| anyhow::anyhow!("held receipt lock poisoned"))?; + queue.retain(|entry| entry.message_hash != held.message_hash); + while queue.held.len() >= MAX_HELD_RECEIPTS + || queue.bytes.saturating_add(held.bytes()) > MAX_HELD_RECEIPT_BYTES + { + if !queue.drop_oldest() { + break; + } + } + queue.bytes = queue.bytes.saturating_add(held.bytes()); + queue.held.push_back(held); + Ok(()) + } + + /// Forgets a receipt a neighbour took, so it never goes out twice. + fn release_receipt(&self, message_hash: &str) -> anyhow::Result<()> { + self.receipts + .lock() + .map_err(|_| anyhow::anyhow!("held receipt lock poisoned"))? + .retain(|held| held.message_hash != message_hash); + Ok(()) + } + /// Broadcasts each pending original at once, then resends it until a /// receipt settles it: Presence every 5 s, a Shared post with back-off /// (`gossip_resend_interval`). Gossip reaches Homes that no presence or @@ -672,12 +846,26 @@ impl CollaborationTransportDriver { { Ok(CollaborationCarrierSendOutcome::RemoteBroadcast { .. }) => { summary.acceptance_receipt_broadcasts += 1; + if self.release_receipt(accepted.message_hash()).is_err() { + return false; + } } // The acceptance is durable, so this frame is ingested. - // Its sender keeps re-sending until a receipt reaches it, - // and each copy earns a fresh receipt of its own: direct - // by the Home call, on the topic by its new position. - Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) | Err(_) => {} + // This Home owes the sender the confirmation and keeps the + // receipt until it goes out or the post expires. + Ok(CollaborationCarrierSendOutcome::LocalOnlyBuffered) | Err(_) => { + if self + .hold_receipt( + accepted.message_hash(), + &receipt_frame, + frame, + accepted.message_expires_at(), + ) + .is_err() + { + return false; + } + } } true } @@ -713,7 +901,7 @@ mod tests { }; use elastos_runtime::provider::ProviderCarrierInvoker; use elastos_runtime::signature::{generate_keypair, SigningKey}; - use std::collections::{BTreeMap, VecDeque}; + use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; const NETWORK: &str = "collaboration-transport-test"; const CONVERSATION: &str = "default-conversation"; @@ -3085,8 +3273,10 @@ mod tests { } /// A Home with no gossip neighbour reaches nobody with its receipts. It - /// keeps the durable acceptance and lets each sender's own re-send ask for - /// the receipt again, instead of re-sending every receipt every cycle. + /// keeps the durable acceptance and sends each receipt once in the live + /// cycle that ingested its frame, instead of once per cycle. The owed + /// receipts then wait in their own queue + /// (`receipts_that_reached_nobody_wait_in_a_bounded_queue`). #[tokio::test] async fn a_home_without_a_gossip_neighbour_receipts_its_frames_once() { let fixture = Fixture::new(); From c6da4a729f741fe67a280d55bf6794cc83edbbc7 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 01:52:08 -0500 Subject: [PATCH 09/14] fix(community): reset the history wait where the worker sees the join A person who left Community and joined it again kept the grown wait, so the Runtime could wait a quarter of an hour before it asked for the posts of the time away. The worker now reads the membership change itself, both before a check and while a check runs, and both paths use the one reset the service owns. Co-authored-by: Cursor --- .../src/collaboration_history.rs | 20 +++++++++++++------ .../src/collaboration_startup.rs | 9 ++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index 8423383b2..6a85ea5c9 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -244,12 +244,7 @@ impl CollaborationHistoryService { /// reaches other Homes. pub(crate) async fn fetch_if_due(&self, now: u64) -> anyhow::Result<()> { if !self.inner.product.community_membership().joined() { - *self - .inner - .cadence - .lock() - .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? = - HistoryCadence::default(); + self.reset_cadence()?; return Ok(()); } let connected = self.inner.product.conversation_transport_view().available; @@ -272,6 +267,19 @@ impl CollaborationHistoryService { self.fetch_once(now).await } + /// Forgets the grown wait, so the next check starts a pass. A person who + /// leaves Community and joins it again owns this reset: the posts of the + /// time away are the reason the wait exists. + pub(crate) fn reset_cadence(&self) -> anyhow::Result<()> { + *self + .inner + .cadence + .lock() + .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? = + HistoryCadence::default(); + Ok(()) + } + /// Independently scheduled from live gossip. Four unavailable peers cannot /// monopolize a pass or starve later available participants across passes. pub(crate) async fn fetch_once(&self, now: u64) -> anyhow::Result<()> { diff --git a/elastos/crates/elastos-server/src/collaboration_startup.rs b/elastos/crates/elastos-server/src/collaboration_startup.rs index 77796b51b..0efbff394 100644 --- a/elastos/crates/elastos-server/src/collaboration_startup.rs +++ b/elastos/crates/elastos-server/src/collaboration_startup.rs @@ -812,10 +812,17 @@ async fn run_collaboration_history_worker( if changed.is_err() || *shutdown.borrow() { return; } } result = async { + // A person who joins Community again waits for the posts of + // the time away, not for the wait the passes before it grew. + // The change can arrive while this worker sleeps or asks, so + // the worker reads the change itself. + if joined.has_changed().unwrap_or(false) { + service.reset_cadence()?; + } if !*joined.borrow_and_update() { return Ok(()); } tokio::select! { biased; - _ = joined.changed() => Ok(()), + _ = joined.changed() => service.reset_cadence(), result = service.fetch_if_due(now_secs()) => result, } } => { From b129432749730f6eee7c80b4b511937399dfb1b2 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 01:57:10 -0500 Subject: [PATCH 10/14] fix(community): keep a person off the history back-off A pass that reached nobody doubled the wait towards a quarter of an hour, so a person could wait that long for the posts a returning Home holds. Only a pass another Home answered grows the wait now; an unanswered pass stops at a minute, the silence the Community link already treats as normal. Every check samples the Homes it sees and the person reading Community, so a Home that lapses and returns between two passes asks again, and a person who opens Community asks at the next check. The Home page heartbeat carries that signal on the presence port History already reads, so nothing new travels between Homes. Co-authored-by: Cursor --- .../src/api/gateway_collaboration_presence.rs | 3 + .../src/collaboration_history.rs | 189 ++++++++++++++---- .../src/collaboration_presence.rs | 19 ++ 3 files changed, 173 insertions(+), 38 deletions(-) diff --git a/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs b/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs index 88f4f018e..6a2c7252a 100644 --- a/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs +++ b/elastos/crates/elastos-server/src/api/gateway_collaboration_presence.rs @@ -94,6 +94,9 @@ pub(super) async fn chat_room_presence_heartbeat( }) .into_response(); }; + // A person has Community open in front of them, so History asks present + // Homes for earlier posts while somebody reads them. + port.record_community_open(now_ts()); // Not shown is a successful heartbeat that announces nothing, not an // error: the person asked for this, and Chat has no failure to report. // Chat presents only a `/while-open` grant; "Always" is the timer's. diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index 6a85ea5c9..823b7f86b 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -34,8 +34,17 @@ const MAX_PEERS_PER_PASS: usize = 4; const PEER_TIMEOUT: Duration = Duration::from_secs(5); /// Wait before the first pass that nothing else asks for. const HISTORY_BASE_WAIT_SECS: u64 = 15; -/// Longest wait between two passes of a Home that keeps seeing the same Homes. +/// Longest wait between two passes of a Home whose passes bring answers. const HISTORY_MAX_WAIT_SECS: u64 = 900; +/// Longest wait between two passes of a Home whose passes reach nobody. The +/// Community link calls itself silent after 60 s (`COMMUNITY_LINK_SILENCE`), +/// so a Home that answers nothing for a minute is the longest silence this +/// Runtime already treats as normal. +const HISTORY_UNANSWERED_MAX_WAIT_SECS: u64 = 60; +/// How long the Home page heartbeat keeps a person counted as reading +/// Community. The page sends it every 15 s, so three missed heartbeats end +/// the signal, the same tolerance a presence record carries. +const HISTORY_OPEN_SIGNAL_SECS: u64 = 45; // History requests carry this Home's own signed presence, which exists only // for people who granted Community presence. const NOT_SHOWN_DETAIL: &str = "Show yourself in Community to load earlier posts."; @@ -62,38 +71,68 @@ struct HistoryPeerRound { available: bool, } -/// When this Home next asks present Homes for earlier posts. Three things make +/// When this Home next asks present Homes for earlier posts. Four things make /// a pass due at once: a Community join, a Community connection that returns, -/// and a Home that is present now and was absent at the last pass, because any -/// of them can leave this Home without posts the others kept. A pass that none -/// of them asked for waits twice as long as the one before it. +/// a Home that is present now and was absent at the check before, and a person +/// who opens Community. Any of them can leave this Home without posts the +/// others kept. #[derive(Default)] struct HistoryCadence { due_at: u64, wait: u64, connected: bool, + watching: bool, present: BTreeSet, } impl HistoryCadence { - fn due(&mut self, now: u64, connected: bool, present: BTreeSet) -> bool { + /// Does this check start a pass? Every check samples the Homes it sees + /// and the person reading Community, so a Home that lapses and returns + /// between two passes, and a person who opens Community between them, + /// both make the next check due. + fn due( + &mut self, + now: u64, + connected: bool, + present: BTreeSet, + watching: bool, + ) -> bool { let returned = connected && !self.connected; let arrived = present.difference(&self.present).next().is_some(); + let opened = watching && !self.watching; self.connected = connected; - if returned || arrived { + self.watching = watching; + self.present = present; + if returned || arrived || opened { self.wait = 0; self.due_at = now; } if now < self.due_at { return false; } - self.present = present; + // The pass that follows owns the next wait. This provisional wait + // holds a pass that ends in an error to the short cadence. + self.due_at = now.saturating_add(HISTORY_BASE_WAIT_SECS); + true + } + + /// The wait after a pass. A pass that brought posts from another Home + /// doubles the wait towards a quarter of an hour, because that Home gave + /// what it has. A pass that reached nobody keeps the wait at a minute, + /// because the Homes this one wants are present and can answer at any + /// check, and a person must never wait longer than that for the posts a + /// returning Home holds. + fn pass_finished(&mut self, now: u64, answered: bool) { + let ceiling = if answered { + HISTORY_MAX_WAIT_SECS + } else { + HISTORY_UNANSWERED_MAX_WAIT_SECS + }; self.wait = self .wait .saturating_mul(2) - .clamp(HISTORY_BASE_WAIT_SECS, HISTORY_MAX_WAIT_SECS); + .clamp(HISTORY_BASE_WAIT_SECS, ceiling); self.due_at = now.saturating_add(self.wait); - true } } @@ -255,16 +294,27 @@ impl CollaborationHistoryService { .into_iter() .map(|peer| peer.endpoint_did) .collect(); + let watching = self + .inner + .presence + .community_open_at() + .is_some_and(|at| now.saturating_sub(at) <= HISTORY_OPEN_SIGNAL_SECS); let due = self .inner .cadence .lock() .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? - .due(now, connected, present); + .due(now, connected, present, watching); if !due { return Ok(()); } - self.fetch_once(now).await + let answered = self.fetch_once(now).await; + self.inner + .cadence + .lock() + .map_err(|_| anyhow::anyhow!("history cadence is unavailable"))? + .pass_finished(now, matches!(answered, Ok(true))); + answered.map(|_| ()) } /// Forgets the grown wait, so the next check starts a pass. A person who @@ -282,9 +332,11 @@ impl CollaborationHistoryService { /// Independently scheduled from live gossip. Four unavailable peers cannot /// monopolize a pass or starve later available participants across passes. - pub(crate) async fn fetch_once(&self, now: u64) -> anyhow::Result<()> { + /// Reports whether a Home answered this pass, which decides how long the + /// Runtime waits before the next one. + pub(crate) async fn fetch_once(&self, now: u64) -> anyhow::Result { if !self.inner.product.community_membership().joined() { - return Ok(()); + return Ok(false); } let (peers, own) = match async { Ok::<_, anyhow::Error>(( @@ -307,7 +359,7 @@ impl CollaborationHistoryService { OwnHistoryPresence::Usable(_) => {} OwnHistoryPresence::NotUsable => { self.inner.product.set_history_unavailable(NOT_SHOWN_DETAIL); - return Ok(()); + return Ok(false); } OwnHistoryPresence::Absent => { // Nobody chosen is a fact only the publisher knows; before its @@ -318,7 +370,7 @@ impl CollaborationHistoryService { } else { self.inner.product.set_history_searching(); } - return Ok(()); + return Ok(false); } } let local_endpoint = self.inner.product.local_device_did(); @@ -328,7 +380,7 @@ impl CollaborationHistoryService { .collect::>(); if remote.is_empty() { self.inner.product.set_history_unavailable(ALONE_DETAIL); - return Ok(()); + return Ok(false); } let round_complete = { let mut round = self @@ -365,7 +417,7 @@ impl CollaborationHistoryService { .await? else { self.inner.product.set_history_unavailable(NOT_SHOWN_DETAIL); - return Ok(()); + return Ok(false); }; let request = serde_json::to_value(self.request(own))?; // History reads have no remote effect. A lost response can be @@ -417,7 +469,7 @@ impl CollaborationHistoryService { if round_complete { round.available = false; } - Ok(()) + Ok(available) } } @@ -1526,29 +1578,60 @@ mod tests { ); } - /// Ten minutes of Runtime checks, 15 s apart, for a Home that keeps - /// seeing the same Homes. + /// Ten minutes of Runtime checks, 15 s apart, as the cadence sees them. + /// The two measured tests above show the same seconds on the service + /// path, with the requests each pass made. #[test] - fn earlier_posts_are_asked_for_on_a_growing_wait() { + fn the_history_wait_grows_only_on_an_answered_pass() { let now = 1_800_000_000; - let present = BTreeSet::from(["did:peer".to_string()]); - let mut cadence = HistoryCadence::default(); - let checks = (0..40).map(|tick| now + tick * HISTORY_BASE_WAIT_SECS); - let due = checks - .clone() - .filter(|check| cadence.due(*check, true, present.clone())) - .map(|check| check - now) - .collect::>(); - assert_eq!(checks.count(), 40, "every check asked other Homes before"); - assert_eq!(due, [0, 15, 45, 105, 225, 465]); + let peer = BTreeSet::from(["did:peer".to_string()]); + let passes = |answered: bool| { + let mut cadence = HistoryCadence::default(); + let mut due = Vec::new(); + for tick in 0..40 { + let at = now + tick * HISTORY_BASE_WAIT_SECS; + if cadence.due(at, true, peer.clone(), false) { + due.push(at - now); + cadence.pass_finished(at, answered); + } + } + due + }; + assert_eq!(passes(true), [0, 15, 45, 105, 225, 465]); + assert_eq!( + passes(false), + [0, 15, 45, 105, 165, 225, 285, 345, 405, 465, 525, 585] + ); + } - // A Home that was absent at the last pass is a reason to ask again. + /// Each reason to ask other Homes at once, one check at a time. + #[test] + fn a_new_home_a_returned_link_and_an_open_community_each_ask_at_once() { + let now = 1_800_000_000; + let peer = BTreeSet::from(["did:peer".to_string()]); let joined = BTreeSet::from(["did:peer".to_string(), "did:newcomer".to_string()]); - assert!(cadence.due(now + 600, true, joined.clone())); - assert!(!cadence.due(now + 601, true, joined.clone())); - // So is a Community connection that returns. - assert!(!cadence.due(now + 602, false, joined.clone())); - assert!(cadence.due(now + 603, true, joined)); + let mut cadence = HistoryCadence::default(); + assert!(cadence.due(now, true, peer.clone(), false)); + cadence.pass_finished(now, true); + assert!(!cadence.due(now + 1, true, peer.clone(), false)); + // A Home this one did not see at the check before. + assert!(cadence.due(now + 2, true, joined.clone(), false)); + cadence.pass_finished(now + 2, true); + // A Home that lapses and returns between two passes. Every check + // samples the Homes it sees, so the return asks again. + assert!(!cadence.due(now + 3, true, peer.clone(), false)); + assert!(cadence.due(now + 4, true, joined.clone(), false)); + cadence.pass_finished(now + 4, true); + // A Community connection that returns. + assert!(!cadence.due(now + 5, false, joined.clone(), false)); + assert!(cadence.due(now + 6, true, joined.clone(), false)); + cadence.pass_finished(now + 6, true); + // A person who opens Community. + assert!(!cadence.due(now + 7, true, joined.clone(), false)); + assert!(cadence.due(now + 8, true, joined.clone(), true)); + cadence.pass_finished(now + 8, true); + // The wait holds while Community stays open. + assert!(!cadence.due(now + 9, true, joined, true)); } #[tokio::test] @@ -1577,6 +1660,36 @@ mod tests { assert_eq!(asks(), 2); } + /// A person opens Community inside a wait. The Runtime asks present Homes + /// at the next check, because the Home page heartbeat records the open + /// view on the presence port History reads. Nothing new travels between + /// Homes for this. + #[tokio::test] + async fn opening_community_asks_present_homes_at_the_next_check() { + let now = now_secs(); + let a = fixture("history-open", "community", now).await; + let b = fixture("history-open", "community", now).await; + observe(&a, &b, now); + a.product.set_community_connected(true); + let plane = Arc::new(Plane { + source: a.product.local_device_did(), + peers: BTreeMap::from([(b.product.local_device_did(), b.registry.clone())]), + calls: Mutex::new(Vec::new()), + stall: false, + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let asks = || plane.calls.lock().unwrap().len(); + a.service.fetch_if_due(now).await.unwrap(); + assert_eq!(asks(), 1); + a.service.fetch_if_due(now + 1).await.unwrap(); + assert_eq!(asks(), 1, "the wait holds while nothing changes"); + a.presence.record_community_open(now + 2); + a.service.fetch_if_due(now + 2).await.unwrap(); + assert_eq!(asks(), 2, "a person who opens Community waits for nothing"); + a.service.fetch_if_due(now + 3).await.unwrap(); + assert_eq!(asks(), 2, "an open view does not ask at every check"); + } + #[tokio::test(start_paused = true)] async fn stalled_history_has_a_per_peer_deadline() { let now = now_secs(); diff --git a/elastos/crates/elastos-server/src/collaboration_presence.rs b/elastos/crates/elastos-server/src/collaboration_presence.rs index 1049f0d24..1bc51b247 100644 --- a/elastos/crates/elastos-server/src/collaboration_presence.rs +++ b/elastos/crates/elastos-server/src/collaboration_presence.rs @@ -42,6 +42,10 @@ pub struct CollaborationPresenceProductPort { /// The Runtime publisher's last pass: `Some(true)` it announced someone, /// `Some(false)` nobody here has a presence grant, `None` unknown. local_publication: Arc>>, + /// When a person on this Home last had Community open. The Home page + /// heartbeat sets it, and History reads it to ask for earlier posts while + /// somebody is reading Community. + community_open_at: Arc>>, /// Mints and validates Community presence grants with the Runtime's /// capability manager. `None` (standalone gateway, most tests): no grant /// can be made or honoured, so nobody is announced. @@ -184,6 +188,7 @@ impl CollaborationPresenceProductPort { core, read_model, local_publication: Arc::new(std::sync::Mutex::new(None)), + community_open_at: Arc::new(std::sync::Mutex::new(None)), presence_authority: None, own_principals: Arc::new(Mutex::new(std::collections::HashMap::new())), }) @@ -488,6 +493,20 @@ impl CollaborationPresenceProductPort { }) } + /// A person has Community in front of them right now. The Home page + /// heartbeat reports this every `HOME_PRESENCE_HEARTBEAT_MS`, so History + /// can ask for earlier posts while somebody is reading them. + pub(crate) fn record_community_open(&self, now: u64) { + if let Ok(mut current) = self.community_open_at.lock() { + *current = Some(now); + } + } + + /// When a person last had Community open on this Home. + pub(crate) fn community_open_at(&self) -> Option { + self.community_open_at.lock().ok().and_then(|at| *at) + } + pub(crate) fn record_local_publication(&self, state: Option) { if let Ok(mut current) = self.local_publication.lock() { *current = state; From e34ced412cb5be91598cb12f9e3ec60ee336355a Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 02:21:03 -0500 Subject: [PATCH 11/14] test(collaboration): reproduce quick Community reopen history delay --- .../src/collaboration_history.rs | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index 823b7f86b..796f1cb3c 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -1690,6 +1690,59 @@ mod tests { assert_eq!(asks(), 2, "an open view does not ask at every check"); } + /// Answered passes run long enough to reach the quiet Home's longest + /// wait. A ten-second switch away keeps the previous heartbeat fresh, + /// so reopening has to work even when every check sees an open view. + #[tokio::test] + async fn quick_reopening_community_keeps_history_within_the_short_wait() { + let now = now_secs(); + let a = fixture("history-quick-reopen", "community", now).await; + let b = fixture("history-quick-reopen", "community", now).await; + a.product.set_community_connected(true); + let (network_id, conversation_id) = a.product.history_scope(); + let plane = Arc::new(CadencePlane { + endpoint: b.product.local_device_did(), + network_id: network_id.to_string(), + conversation_id: conversation_id.to_string(), + answers: true, + calls: Mutex::new(Vec::new()), + }); + a.registry.set_carrier_invoker(plane.clone()).await; + let asks = || plane.calls.lock().unwrap().len(); + let mut asked_at = Vec::new(); + for tick in 0..=63 { + let offset = tick * 15; + let at = now + offset; + keep_both_present(&a, &b, at).await; + a.presence.record_community_open(at); + let before = asks(); + a.service.fetch_if_due(at).await.unwrap(); + if asks() > before { + asked_at.push(offset); + } + } + assert_eq!(asked_at.last(), Some(&945)); + let before_reopening = asks(); + // The person switches away just after the last heartbeat and returns + // ten seconds later. There is no close signal or false watching sample. + a.presence.record_community_open(now + 955); + for offset in [960, 975, 990, 1005] { + let at = now + offset; + keep_both_present(&a, &b, at).await; + a.presence.record_community_open(at); + a.service.fetch_if_due(at).await.unwrap(); + } + assert_eq!( + asks(), + before_reopening + 1, + "a quick reopening gets another provider request within 60 seconds" + ); + assert!( + asked_at.windows(2).all(|pair| pair[1] - pair[0] <= 60), + "answered passes stay short while Community remains open: {asked_at:?}" + ); + } + #[tokio::test(start_paused = true)] async fn stalled_history_has_a_per_peer_deadline() { let now = now_secs(); From b6121e6c327bc8318cde520d75123f8ee718cacb Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 02:33:00 -0500 Subject: [PATCH 12/14] fix(collaboration): keep Community history short while watched --- .../src/collaboration_history.rs | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index 796f1cb3c..ff0468359 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -34,13 +34,13 @@ const MAX_PEERS_PER_PASS: usize = 4; const PEER_TIMEOUT: Duration = Duration::from_secs(5); /// Wait before the first pass that nothing else asks for. const HISTORY_BASE_WAIT_SECS: u64 = 15; -/// Longest wait between two passes of a Home whose passes bring answers. +/// Longest wait after an answered pass while nobody is reading Community. const HISTORY_MAX_WAIT_SECS: u64 = 900; -/// Longest wait between two passes of a Home whose passes reach nobody. The -/// Community link calls itself silent after 60 s (`COMMUNITY_LINK_SILENCE`), +/// Longest wait while a person reads Community or a pass reaches nobody. +/// The Community link calls itself silent after 60 s (`COMMUNITY_LINK_SILENCE`), /// so a Home that answers nothing for a minute is the longest silence this /// Runtime already treats as normal. -const HISTORY_UNANSWERED_MAX_WAIT_SECS: u64 = 60; +const HISTORY_SHORT_MAX_WAIT_SECS: u64 = 60; /// How long the Home page heartbeat keeps a person counted as reading /// Community. The page sends it every 15 s, so three missed heartbeats end /// the signal, the same tolerance a presence record carries. @@ -116,17 +116,15 @@ impl HistoryCadence { true } - /// The wait after a pass. A pass that brought posts from another Home - /// doubles the wait towards a quarter of an hour, because that Home gave - /// what it has. A pass that reached nobody keeps the wait at a minute, - /// because the Homes this one wants are present and can answer at any - /// check, and a person must never wait longer than that for the posts a - /// returning Home holds. + /// Answered passes grow the wait towards a quarter of an hour only while + /// nobody is reading Community. An open view or an unanswered pass keeps + /// the wait at a minute, so a quick reopening within the heartbeat's + /// lifetime still asks for the posts a returning Home holds. fn pass_finished(&mut self, now: u64, answered: bool) { - let ceiling = if answered { + let ceiling = if answered && !self.watching { HISTORY_MAX_WAIT_SECS } else { - HISTORY_UNANSWERED_MAX_WAIT_SECS + HISTORY_SHORT_MAX_WAIT_SECS }; self.wait = self .wait @@ -1526,9 +1524,9 @@ mod tests { } } - /// Ten minutes of Runtime checks, 15 s apart, measured as the seconds at - /// which the Runtime asked another Home for earlier posts. - async fn seconds_of_history_asks(answers: bool) -> Vec { + /// Runtime checks, 15 s apart, measured as the seconds at which the + /// Runtime asked another Home for earlier posts while nobody was watching. + async fn seconds_of_history_asks(answers: bool, ticks: u64) -> Vec { let now = now_secs(); let a = fixture("history-measured", "community", now).await; let b = fixture("history-measured", "community", now).await; @@ -1544,7 +1542,7 @@ mod tests { a.registry.set_carrier_invoker(plane.clone()).await; let mut asked_at = Vec::new(); let mut asks = 0; - for tick in 0..41 { + for tick in 0..ticks { let at = now + tick * HISTORY_BASE_WAIT_SECS; keep_both_present(&a, &b, at).await; a.service.fetch_if_due(at).await.unwrap(); @@ -1562,8 +1560,8 @@ mod tests { #[tokio::test] async fn answered_history_passes_back_off_to_the_long_wait() { assert_eq!( - seconds_of_history_asks(true).await, - [0, 15, 45, 105, 225, 465] + seconds_of_history_asks(true, 125).await, + [0, 15, 45, 105, 225, 465, 945, 1845] ); } @@ -1573,14 +1571,14 @@ mod tests { #[tokio::test] async fn unanswered_history_passes_keep_the_short_wait() { assert_eq!( - seconds_of_history_asks(false).await, + seconds_of_history_asks(false, 41).await, [0, 15, 45, 105, 165, 225, 285, 345, 405, 465, 525, 585] ); } /// Ten minutes of Runtime checks, 15 s apart, as the cadence sees them. - /// The two measured tests above show the same seconds on the service - /// path, with the requests each pass made. + /// The two measured tests above cover these seconds on the service path, + /// with the requests each pass made. #[test] fn the_history_wait_grows_only_on_an_answered_pass() { let now = 1_800_000_000; From e3ba213a6c38d97b115d2bff8ace572cca2cf2a4 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 02:33:56 -0500 Subject: [PATCH 13/14] test(collaboration): complete both cadence fixture presence projections --- elastos/crates/elastos-server/src/collaboration_history.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index ff0468359..ec6921d15 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -1471,6 +1471,12 @@ mod tests { .await .unwrap(); let peer = refresh_presence(b, now).await; + // Both publishers complete their local projection so expired + // announcements can leave the outgoing store during long measurements. + b.presence + .project_authorized_presence(&peer, now) + .await + .unwrap(); a.product .test_core() .accept_incoming_from_signed_source_for_test(peer.test_envelope_bytes(), now) From 54fb9a10964a819c32c20bd10dedea04c9bdc0f2 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sun, 15 Nov 2026 03:15:02 -0500 Subject: [PATCH 14/14] test(collaboration): borrow receipt hash in Clippy comparison --- elastos/crates/elastos-server/src/collaboration_transport.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index 1856d040a..e57839317 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -3458,7 +3458,7 @@ mod tests { driver.process_incoming_once(received_at).await.unwrap(), CollaborationIncomingOnceOutcome::Acknowledged(_) )); - assert_eq!(receipted_hashes(&carrier), [hash.clone()]); + assert_eq!(receipted_hashes(&carrier), std::slice::from_ref(&hash)); let mut sent_at = Vec::new(); let mut sends = 1;