From 55a2f8e1b012e05e35651b265b7af934acb7a7ac Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 12:46:35 -0500 Subject: [PATCH 01/15] test(carrier): a gossip neighbour that stops reading must not wedge the actor iroh-gossip 0.101.0 dispatches OutEvent::SendMessage with an awaiting send on a bounded per-peer channel (SEND_QUEUE_CAP = 64, net.rs:686), so one neighbour whose stream credit runs out parks the single actor loop and the node serves no topic, including new joins, while QUIC stays healthy. 0.102.0 replaces that channel with a byte-bounded queue that drops or aborts instead of awaiting. The test drives the real net actor over QUIC: a neighbour completes the membership handshake through the public protocol state, then holds its streams open and reads nothing. It asserts the two symptoms a Community node shows, that traffic to the other neighbours keeps flowing and that a fresh join still completes. --- elastos/Cargo.lock | 2 + elastos/crates/elastos-server/Cargo.toml | 4 + .../tests/gossip_actor_wedge.rs | 366 ++++++++++++++++++ 3 files changed, 372 insertions(+) create mode 100644 elastos/crates/elastos-server/tests/gossip_actor_wedge.rs diff --git a/elastos/Cargo.lock b/elastos/Cargo.lock index 4ad21f942..15ca8e10a 100644 --- a/elastos/Cargo.lock +++ b/elastos/Cargo.lock @@ -1664,6 +1664,7 @@ dependencies = [ "axum", "axum-server", "base64", + "bytes", "cid", "clap", "custody-provider", @@ -1697,6 +1698,7 @@ dependencies = [ "k256", "libc", "memchr", + "postcard", "qrcodegen", "rand 0.10.2", "rand 0.8.7", diff --git a/elastos/crates/elastos-server/Cargo.toml b/elastos/crates/elastos-server/Cargo.toml index 8cdf8f234..7f1f2c369 100644 --- a/elastos/crates/elastos-server/Cargo.toml +++ b/elastos/crates/elastos-server/Cargo.toml @@ -125,6 +125,10 @@ x-wing = { version = "=0.1.0", features = ["hazmat", "zeroize"] } # The rand generation iroh-gossip's membership state machine takes; the # Community hub test drives that state machine directly. gossip-rand = { package = "rand", version = "0.10", default-features = false, features = ["std_rng"] } +# The gossip wire encoding, for the neighbour that the actor-wedge test keeps +# silent after the membership handshake. +bytes = "1.7" +postcard = { version = "1", default-features = false, features = ["alloc", "use-std"] } [features] default = [] diff --git a/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs b/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs new file mode 100644 index 000000000..83bece517 --- /dev/null +++ b/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs @@ -0,0 +1,366 @@ +//! One neighbour that stops reading must not stop the gossip actor. +//! +//! iroh-gossip 0.101.0 dispatches `OutEvent::SendMessage` with +//! `active_send_tx.send(message).await` on a bounded per-peer channel +//! (`SEND_QUEUE_CAP = 64`, net.rs:686). A neighbour whose QUIC stream credit +//! runs out stalls that channel, and a bounded-mpsc `send().await` never +//! returns `Full`, so the single actor loop stops serving every topic. The +//! Community node then keeps its healthy QUIC connections and still admits no +//! new Home. iroh-gossip 0.102.0 replaces the channel with a byte-bounded +//! queue that drops or aborts instead of awaiting. +//! +//! The two assertions are the two symptoms a Community node shows: traffic to +//! the other neighbours keeps flowing, and a fresh join still completes. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{anyhow, Context, Result}; +use bytes::Bytes; +use futures_lite::StreamExt; +use gossip_rand::rngs::StdRng; +use gossip_rand::SeedableRng; +use iroh::address_lookup::memory::MemoryLookup; +use iroh::endpoint::{presets, Connection, RecvStream, SendStream}; +use iroh::protocol::Router; +use iroh::{Endpoint, EndpointAddr, EndpointId, RelayMode, SecretKey, TransportAddr}; +use iroh_gossip::api::{Event as ApiEvent, GossipReceiver}; +use iroh_gossip::net::Gossip; +use iroh_gossip::proto::{ + Command, Config as ProtoConfig, Event as ProtoEvent, InEvent, Message as WireMessage, OutEvent, + PeerData, State as ProtoState, TopicId, +}; +use tokio::time::timeout; + +/// Just under the 4096-byte default `max_message_size`, leaving header room. +const PAYLOAD_SIZE: usize = 3800; +/// Enough to exhaust the 1.25 MiB default stream receive window (about 330 +/// payloads) and the 64-message send queue, with room to spare. +const FLOOD_MESSAGES: usize = 500; +const PROBE_BUDGET: Duration = Duration::from_secs(15); +const JOIN_BUDGET: Duration = Duration::from_secs(30); + +#[tokio::test(flavor = "multi_thread")] +async fn a_neighbour_that_stops_reading_leaves_the_actor_serving_other_topics() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .ok(); + let flood_topic = TopicId::from_bytes([0xAB; 32]); + let probe_topic = TopicId::from_bytes([0xCD; 32]); + let lookup = MemoryLookup::new(); + + let silent_endpoint = Endpoint::builder(presets::Minimal) + .secret_key(SecretKey::generate()) + .relay_mode(RelayMode::Disabled) + .alpns(vec![iroh_gossip::ALPN.to_vec()]) + .bind() + .await + .context("bind the silent neighbour")?; + let silent_id = silent_endpoint.id(); + lookup.add_endpoint_info(loopback_addr(&silent_endpoint)); + let (went_silent, mut is_silent) = tokio::sync::mpsc::channel(1); + let silent_task = tokio::spawn(run_silent_neighbour( + silent_endpoint.clone(), + flood_topic, + went_silent, + )); + + let (hub_endpoint, hub, _hub_router) = gossip_node(&lookup).await?; + let (member_endpoint, member, _member_router) = gossip_node(&lookup).await?; + let hub_id = hub_endpoint.id(); + + let member_join = tokio::spawn({ + let member = member.clone(); + async move { member.subscribe_and_join(probe_topic, vec![hub_id]).await } + }); + let hub_probe = timeout(JOIN_BUDGET, hub.subscribe_and_join(probe_topic, vec![])) + .await + .context("the hub did not join the probe topic")??; + let member_probe = timeout(JOIN_BUDGET, member_join) + .await + .context("the member did not join the probe topic")???; + let (probe_sender, _hub_probe_receiver) = hub_probe.split(); + let (_member_sender, mut member_receiver) = member_probe.split(); + + probe_sender + .broadcast(Bytes::from_static(b"before the flood")) + .await + .context("broadcast before the flood")?; + wait_for(&mut member_receiver, b"before the flood", PROBE_BUDGET) + .await + .context("the probe path failed before the flood, so the fixture is wrong")?; + + let flood = timeout( + JOIN_BUDGET, + hub.subscribe_and_join(flood_topic, vec![silent_id]), + ) + .await + .context("the hub did not take the silent neighbour into its active view")??; + // Without this the neighbour is still reading, and the flood proves nothing. + timeout(JOIN_BUDGET, is_silent.recv()) + .await + .ok() + .flatten() + .context("the neighbour never reached the active view, so the fixture is wrong")?; + let (flood_sender, mut flood_receiver) = flood.split(); + tokio::spawn(async move { while flood_receiver.next().await.is_some() {} }); + + // Fire and forget: the flood may stall, and only its collateral matters. + let sent = Arc::new(AtomicUsize::new(0)); + let flood_task = tokio::spawn({ + let sent = sent.clone(); + async move { + let payload = Bytes::from(vec![0x5A; PAYLOAD_SIZE]); + for _ in 0..FLOOD_MESSAGES { + if flood_sender.broadcast(payload.clone()).await.is_err() { + break; + } + sent.fetch_add(1, Ordering::Relaxed); + } + } + }); + let flooded = wait_until_flood_settles(&flood_task, &sent).await; + + let probe_started = Instant::now(); + let delivery = async { + probe_sender + .broadcast(Bytes::from_static(b"during the flood")) + .await + .context("broadcast during the flood")?; + wait_for(&mut member_receiver, b"during the flood", PROBE_BUDGET).await + } + .await; + let probe_seconds = probe_started.elapsed().as_secs_f32(); + + let join_started = Instant::now(); + let (newcomer_endpoint, newcomer, _newcomer_router) = gossip_node(&lookup).await?; + let fresh_join = timeout( + JOIN_BUDGET, + newcomer.subscribe_and_join(probe_topic, vec![hub_id]), + ) + .await + .map_err(|_| anyhow!("a fresh join did not complete within {JOIN_BUDGET:?}")) + .and_then(|joined| joined.map_err(anyhow::Error::from)); + let join_seconds = join_started.elapsed().as_secs_f32(); + + silent_endpoint.close().await; + flood_task.abort(); + silent_task.abort(); + newcomer_endpoint.close().await; + member_endpoint.close().await; + hub_endpoint.close().await; + + let report = format!( + "{flooded} of {FLOOD_MESSAGES} broadcasts left the hub; \ + delivery to the other neighbour took {probe_seconds:.1} s; \ + the fresh join took {join_seconds:.1} s" + ); + delivery.with_context(|| format!("delivery to the other neighbour stopped: {report}"))?; + fresh_join.with_context(|| format!("a fresh join stopped: {report}"))?; + println!("{report}"); + Ok(()) +} + +/// Returns the broadcasts the hub accepted once the flood finishes or stops +/// making progress; a stalled flood is the state the assertions run against. +async fn wait_until_flood_settles( + flood: &tokio::task::JoinHandle<()>, + sent: &AtomicUsize, +) -> usize { + let mut last = 0; + loop { + tokio::time::sleep(Duration::from_secs(1)).await; + let now = sent.load(Ordering::Relaxed); + if flood.is_finished() || now == last { + return now; + } + last = now; + } +} + +async fn gossip_node(lookup: &MemoryLookup) -> Result<(Endpoint, Gossip, Router)> { + let endpoint = Endpoint::builder(presets::Minimal) + .secret_key(SecretKey::generate()) + .relay_mode(RelayMode::Disabled) + .bind() + .await + .context("bind a gossip endpoint")?; + endpoint + .address_lookup() + .map_err(|err| anyhow!("{err}"))? + .add(lookup.clone()); + lookup.add_endpoint_info(loopback_addr(&endpoint)); + let gossip = Gossip::builder().spawn(endpoint.clone()); + let router = Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .spawn(); + Ok((endpoint, gossip, router)) +} + +fn loopback_addr(endpoint: &Endpoint) -> EndpointAddr { + let sockets: Vec = endpoint + .bound_sockets() + .into_iter() + .map(|mut socket| { + if socket.ip().is_unspecified() { + socket.set_ip(if socket.is_ipv4() { + std::net::IpAddr::from([127, 0, 0, 1]) + } else { + std::net::IpAddr::from(std::net::Ipv6Addr::LOCALHOST) + }); + } + TransportAddr::Ip(socket) + }) + .collect(); + assert!(!sockets.is_empty(), "endpoint has no bound socket"); + EndpointAddr::from_parts(endpoint.id(), sockets) +} + +async fn wait_for(receiver: &mut GossipReceiver, expected: &[u8], budget: Duration) -> Result<()> { + timeout(budget, async { + while let Some(event) = receiver.next().await { + if let Ok(ApiEvent::Received(message)) = event { + if message.content.as_ref() == expected { + return Ok(()); + } + } + } + Err(anyhow!("the event stream ended")) + }) + .await + .map_err(|_| anyhow!("no delivery within {budget:?}"))? +} + +/// Speaks the real membership handshake through the public protocol state, so +/// the hub promotes it into the active view, then holds its streams open and +/// reads nothing. The endpoint driver keeps acknowledging at the transport +/// level, so QUIC stays healthy while the stream credit is never returned. +async fn run_silent_neighbour( + endpoint: Endpoint, + topic: TopicId, + went_silent: tokio::sync::mpsc::Sender<()>, +) -> Result<()> { + let mut state: ProtoState = ProtoState::new( + endpoint.id(), + PeerData::new(Vec::::new()), + ProtoConfig::default(), + StdRng::seed_from_u64(0xB0B), + ); + state + .handle( + InEvent::Command(topic, Command::Join(vec![])), + tokio::time::Instant::now(), + None, + ) + .for_each(drop); + let state = Arc::new(tokio::sync::Mutex::new(state)); + + while let Some(incoming) = endpoint.accept().await { + let Ok(connection) = incoming.await else { + continue; + }; + let state = state.clone(); + let went_silent = went_silent.clone(); + tokio::spawn(async move { + let remote = connection.remote_id(); + if let Err(error) = + go_silent_after_membership(connection, topic, remote, state, went_silent).await + { + eprintln!("the silent neighbour stopped: {error:#}"); + } + }); + } + Ok(()) +} + +async fn go_silent_after_membership( + connection: Connection, + topic: TopicId, + remote: EndpointId, + state: Arc>>, + went_silent: tokio::sync::mpsc::Sender<()>, +) -> Result<()> { + let mut recv = connection.accept_uni().await.context("accept a stream")?; + // The stream header starts with the bare topic id and has gained fields + // since 0.101, so match the prefix rather than the whole frame. + let header = read_frame(&mut recv) + .await + .context("read the topic header")?; + anyhow::ensure!( + header.starts_with(topic.as_bytes().as_slice()), + "unexpected topic in the stream header" + ); + + let mut send: Option = None; + loop { + let payload = read_frame(&mut recv).await.context("read a frame")?; + // postcard concatenates struct fields without framing, so the wire + // message is the 32 topic bytes followed by the frame. + let mut encoded = Vec::with_capacity(32 + payload.len()); + encoded.extend_from_slice(topic.as_bytes()); + encoded.extend_from_slice(&payload); + let message: WireMessage = + postcard::from_bytes(&encoded).context("decode a gossip message")?; + + let out: Vec> = state + .lock() + .await + .handle( + InEvent::RecvMessage(remote, message), + tokio::time::Instant::now(), + None, + ) + .collect(); + + let mut in_active_view = false; + for event in out { + match event { + OutEvent::SendMessage(_, message) => { + let bytes = postcard::to_stdvec(&message).context("encode a gossip message")?; + let stream = match send.as_mut() { + Some(stream) => stream, + None => { + let mut stream = + connection.open_uni().await.context("open a stream")?; + write_frame(&mut stream, topic.as_bytes()).await?; + send.insert(stream) + } + }; + write_frame(stream, &bytes[32..]).await?; + } + OutEvent::EmitEvent(_, ProtoEvent::NeighborUp(_)) => in_active_view = true, + _ => {} + } + } + if in_active_view { + break; + } + } + + went_silent.send(()).await.ok(); + std::future::pending::<()>().await; + unreachable!() +} + +async fn read_frame(recv: &mut RecvStream) -> Result> { + let mut length = [0u8; 4]; + recv.read_exact(&mut length).await.context("frame length")?; + let length = u32::from_be_bytes(length) as usize; + anyhow::ensure!( + length <= 1024 * 1024, + "frame of {length} bytes is too large" + ); + let mut frame = vec![0u8; length]; + recv.read_exact(&mut frame).await.context("frame body")?; + Ok(frame) +} + +async fn write_frame(send: &mut SendStream, payload: &[u8]) -> Result<()> { + send.write_all(&(payload.len() as u32).to_be_bytes()) + .await + .context("frame length")?; + send.write_all(payload).await.context("frame body")?; + Ok(()) +} From 4e6cdda26fe64a331400908f6a79f6ab9766dd75 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 13:03:06 -0500 Subject: [PATCH 02/15] test(carrier): Community link across Runtime versions Homes and the Community node update at different times, so the release has to hold for every pairing. The journey starts one Community node and two Homes from two named builds, joins them to one Community, then restarts the node and brings up a fresh Home against it, over the four arrangements. It stages only the Runtime and the localhost provider against the manifest checksum the Runtime verifies, so it runs on a machine that has no media tools for a full source Home. --- scripts/gossip-mixed-version-journey.py | 274 ++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 scripts/gossip-mixed-version-journey.py diff --git a/scripts/gossip-mixed-version-journey.py b/scripts/gossip-mixed-version-journey.py new file mode 100644 index 000000000..721f14b8c --- /dev/null +++ b/scripts/gossip-mixed-version-journey.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Community link across Runtime versions: a node and Homes built from different trees. + +Homes and the Community node update at different times, so a release must +hold for every pairing. Each arrangement starts one Community node and two +Homes from the named builds, joins them to one Community, then restarts the +node and brings up a fresh Home against it. + +The fixture needs only the Runtime and the localhost provider, so it runs on a +machine without the media tools that the full source Home installs. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import runpy +import shutil +import socket +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parent.parent +JOURNEY = runpy.run_path(str(ROOT / "scripts/ci-installed-journeys.py")) +free_port = JOURNEY["free_port"] +wait_healthy = JOURNEY["wait_healthy"] +cleanup_runtime = JOURNEY["cleanup_runtime"] +digest = JOURNEY["digest"] + +STAGED_PROVIDERS = ("localhost-provider",) +JOIN_LINE = "carrier: gossip_join" +DEAD_LINE = "Community link looks dead" +REBUILD_LINE = "rebuilding Carrier gossip" +#: A joined Home repairs after one silence window, so a pass must outlast it. +SETTLE_SECONDS = 90 + + +def platform_key(): + machine = {"x86_64": "amd64", "aarch64": "arm64"}.get(os.uname().machine) + if sys.platform != "linux" or machine is None: + raise RuntimeError("the mixed-version journey runs on Linux amd64 or arm64") + return f"linux-{machine}" + + +class Build: + """One Runtime tree: its source checkout and its own release target dir.""" + + def __init__(self, name, source, target): + self.name = name + self.source = Path(source).resolve() + self.target = Path(target).resolve() + self.runtime = self.target / "release/elastos" + self.manifest = self.source / "components.json" + for path in (self.runtime, self.manifest): + if not path.exists(): + raise RuntimeError(f"build {name} has no {path.name}: build it first") + self.runtime_sha256 = digest(self.runtime) + + def provider(self, provider): + return self.target / "release" / provider + + +def stage_home(build, root, carrier_address): + """One disposable data root that the Runtime's own checksum check accepts.""" + data = root / ".local/share/elastos" + (data / "bin").mkdir(mode=0o700, parents=True) + manifest = json.loads(build.manifest.read_text()) + for provider in STAGED_PROVIDERS: + staged = data / "bin" / provider + shutil.copy2(build.provider(provider), staged) + staged.chmod(0o700) + blob = staged.read_bytes() + entry = manifest["external"][provider]["platforms"][platform_key()] + entry["install_path"] = f"bin/{provider}" + entry["checksum"] = "sha256:" + hashlib.sha256(blob).hexdigest() + entry["size"] = len(blob) + (data / "components.json").write_text(json.dumps(manifest, indent=2)) + (data / "config.toml").write_text( + f'dev_mode = true\ntrusted_keys = []\ncarrier_bind_addr = "{carrier_address}"\n' + ) + return data + + +class Home: + """One Runtime process with its own Home, data root and log.""" + + def __init__(self, name, build, root, bind_host): + self.name = name + self.build = build + self.root = root + self.log = root / "runtime.private.log" + self.carrier_address = f"{bind_host}:{free_port(socket.SOCK_DGRAM)}" + self.api_address = f"{bind_host}:{free_port()}" + self.data = stage_home(build, root, self.carrier_address) + self.child = None + + def environment(self): + return { + "PATH": os.environ["PATH"], + "HOME": str(self.root), + "XDG_DATA_HOME": str(self.data.parent), + "ELASTOS_QUIET_RUNTIME_NOTICES": "1", + } + + def start(self): + assert self.child is None, "Home already has a Runtime" + with self.log.open("ab") as log: + self.child = subprocess.Popen( + [str(self.build.runtime), "gateway", "--addr", self.api_address], + env=self.environment(), + stdout=log, + stderr=log, + start_new_session=True, + ) + wait_healthy(self.child, self.api_address, f"mixed-version {self.name}") + + def stop(self): + if self.child is None: + return {"status": "passed", "after": {}} + row = cleanup_runtime(self.child, self.data) + self.child = None + return row + + def configure(self, *arguments): + with (self.root / "configuration.private.log").open("ab") as log: + subprocess.run( + [str(self.build.runtime), "collaboration-config", *map(str, arguments)], + env=self.environment(), + stdout=log, + stderr=log, + check=True, + timeout=120, + ) + + def lines(self, needle): + if not self.log.exists(): + return [] + text = self.log.read_text(errors="replace") + return [line for line in text.splitlines() if needle in line] + + +def observe(home, started): + """What an operator reads off one Home after the settle window.""" + return { + "home": home.name, + "build": home.build.name, + "joined": bool(home.lines(JOIN_LINE)), + "link_dead": len(home.lines(DEAD_LINE)), + "gossip_rebuilds": len(home.lines(REBUILD_LINE)), + "seconds": round(time.monotonic() - started), + } + + +def run_arrangement(node_build, home_build, root, bind_host): + """One node version against one Home version, with a fresh join after restart.""" + root.mkdir(mode=0o700, parents=True, exist_ok=False) + homes, record = {}, { + "node_build": node_build.name, + "home_build": home_build.name, + "node_runtime_sha256": node_build.runtime_sha256, + "home_runtime_sha256": home_build.runtime_sha256, + "observations": [], + "process_cleanup": [], + } + try: + for name, build in (("community", node_build), ("a", home_build), ("b", home_build)): + homes[name] = Home(name, build, root / name, bind_host) + node = homes["community"] + node.start() + node.configure("create-authority-key", "--key", root / "authority.key") + node.configure( + "export-local-bootstrap-receipt", + "--data-root", node.data, + "--runtime-kind", "gateway", + "--output", root / "bootstrap.json", + ) + node.configure( + "generate-initial", + "--authority-key", root / "authority.key", + "--network-id", "ci.gossip-mixed", + "--conversation-id", "community", + "--bootstrap-peer", root / "bootstrap.json", + "--output", root / "network.json", + ) + node.configure("verify", "--input", root / "network.json") + record["process_cleanup"].append(node.stop()) + + def admit(home): + target = home.data / "collaboration-network-v1.json" + shutil.copy2(root / "network.json", target) + target.chmod(0o600) + + for home in homes.values(): + admit(home) + started = time.monotonic() + for home in homes.values(): + home.start() + time.sleep(SETTLE_SECONDS) + record["observations"] += [observe(homes[name], started) for name in ("a", "b")] + + # The node restarts, as an updated node does; the Homes keep running. + record["process_cleanup"].append(homes["community"].stop()) + homes["community"].start() + fresh = Home("c", home_build, root / "c", bind_host) + homes["c"] = fresh + admit(fresh) + fresh_started = time.monotonic() + fresh.start() + time.sleep(SETTLE_SECONDS) + record["observations"].append(observe(fresh, fresh_started)) + record["observations"] += [observe(homes[name], started) for name in ("a", "b")] + record["status"] = ( + "passed" + if all(row["joined"] and row["link_dead"] == 0 for row in record["observations"]) + else "failed" + ) + except BaseException as error: + record["status"] = "failed" + record["failure"] = f"{type(error).__name__}: {error}"[:300] + raise + finally: + for home in homes.values(): + try: + record["process_cleanup"].append(home.stop()) + except (OSError, RuntimeError, AssertionError) as error: + record["process_cleanup"].append({"status": "failed", "error": str(error)[:200]}) + shutil.rmtree(root, ignore_errors=True) + return record + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--released-source", type=Path, required=True) + parser.add_argument("--released-target", type=Path, required=True) + parser.add_argument("--develop-source", type=Path, required=True) + parser.add_argument("--develop-target", type=Path, required=True) + parser.add_argument("--fixture-root", type=Path, required=True) + parser.add_argument("--bind-host", default="127.0.0.1") + parser.add_argument("--receipt", type=Path) + arguments = parser.parse_args() + builds = { + "released": Build("released", arguments.released_source, arguments.released_target), + "develop": Build("develop", arguments.develop_source, arguments.develop_target), + } + receipt = {"platform": platform_key(), "arrangements": []} + for node in ("released", "develop"): + for home in ("released", "develop"): + root = arguments.fixture_root / f"node-{node}-homes-{home}" + receipt["arrangements"].append( + run_arrangement(builds[node], builds[home], root, arguments.bind_host) + ) + receipt["status"] = ( + "passed" + if all(row["status"] == "passed" for row in receipt["arrangements"]) + else "failed" + ) + if arguments.receipt: + arguments.receipt.write_text(json.dumps(receipt, indent=2) + "\n") + for row in receipt["arrangements"]: + print(f"node={row['node_build']:<9} homes={row['home_build']:<9} {row['status']}") + for observation in row["observations"]: + print( + f" {observation['home']} joined={observation['joined']} " + f"link_dead={observation['link_dead']} " + f"rebuilds={observation['gossip_rebuilds']} " + f"after={observation['seconds']}s" + ) + print("mixed-version status:", receipt["status"]) + raise SystemExit(0 if receipt["status"] == "passed" else 1) + + +if __name__ == "__main__": + main() From 798bb088c5079e7f2c24ae45b325db987ddf05bb Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 13:06:03 -0500 Subject: [PATCH 03/15] test(carrier): keep every mixed-version arrangement observable One arrangement that fails now records its reason and the run continues to the others, and the settle window is a flag so a shake-out run is quick. --- scripts/gossip-mixed-version-journey.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/gossip-mixed-version-journey.py b/scripts/gossip-mixed-version-journey.py index 721f14b8c..d1ec90a0b 100644 --- a/scripts/gossip-mixed-version-journey.py +++ b/scripts/gossip-mixed-version-journey.py @@ -33,7 +33,7 @@ DEAD_LINE = "Community link looks dead" REBUILD_LINE = "rebuilding Carrier gossip" #: A joined Home repairs after one silence window, so a pass must outlast it. -SETTLE_SECONDS = 90 +DEFAULT_SETTLE_SECONDS = 90 def platform_key(): @@ -152,7 +152,7 @@ def observe(home, started): } -def run_arrangement(node_build, home_build, root, bind_host): +def run_arrangement(node_build, home_build, root, bind_host, settle): """One node version against one Home version, with a fresh join after restart.""" root.mkdir(mode=0o700, parents=True, exist_ok=False) homes, record = {}, { @@ -196,7 +196,7 @@ def admit(home): started = time.monotonic() for home in homes.values(): home.start() - time.sleep(SETTLE_SECONDS) + time.sleep(settle) record["observations"] += [observe(homes[name], started) for name in ("a", "b")] # The node restarts, as an updated node does; the Homes keep running. @@ -207,7 +207,7 @@ def admit(home): admit(fresh) fresh_started = time.monotonic() fresh.start() - time.sleep(SETTLE_SECONDS) + time.sleep(settle) record["observations"].append(observe(fresh, fresh_started)) record["observations"] += [observe(homes[name], started) for name in ("a", "b")] record["status"] = ( @@ -215,10 +215,9 @@ def admit(home): if all(row["joined"] and row["link_dead"] == 0 for row in record["observations"]) else "failed" ) - except BaseException as error: + except Exception as error: record["status"] = "failed" record["failure"] = f"{type(error).__name__}: {error}"[:300] - raise finally: for home in homes.values(): try: @@ -237,6 +236,7 @@ def main(): parser.add_argument("--develop-target", type=Path, required=True) parser.add_argument("--fixture-root", type=Path, required=True) parser.add_argument("--bind-host", default="127.0.0.1") + parser.add_argument("--settle-seconds", type=int, default=DEFAULT_SETTLE_SECONDS) parser.add_argument("--receipt", type=Path) arguments = parser.parse_args() builds = { @@ -248,7 +248,10 @@ def main(): for home in ("released", "develop"): root = arguments.fixture_root / f"node-{node}-homes-{home}" receipt["arrangements"].append( - run_arrangement(builds[node], builds[home], root, arguments.bind_host) + run_arrangement( + builds[node], builds[home], root, + arguments.bind_host, arguments.settle_seconds, + ) ) receipt["status"] = ( "passed" @@ -258,7 +261,8 @@ def main(): if arguments.receipt: arguments.receipt.write_text(json.dumps(receipt, indent=2) + "\n") for row in receipt["arrangements"]: - print(f"node={row['node_build']:<9} homes={row['home_build']:<9} {row['status']}") + print(f"node={row['node_build']:<9} homes={row['home_build']:<9} {row['status']}" + + (f" {row['failure']}" if "failure" in row else "")) for observation in row["observations"]: print( f" {observation['home']} joined={observation['joined']} " From 041b9efd575a3521ba8d6baaeb1a00b8ed013afd Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 13:17:56 -0500 Subject: [PATCH 04/15] test(carrier): run each mixed-version Home from its own data root The shared process census owns a Runtime by its data-root path, so a Home started from a build target escaped cleanup. Each Home now runs its own copy, a stuck cleanup still returns a result, and a failed arrangement keeps a log tail before its fixture tree goes away. --- scripts/gossip-mixed-version-journey.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/gossip-mixed-version-journey.py b/scripts/gossip-mixed-version-journey.py index d1ec90a0b..d72e3a62d 100644 --- a/scripts/gossip-mixed-version-journey.py +++ b/scripts/gossip-mixed-version-journey.py @@ -65,6 +65,11 @@ def stage_home(build, root, carrier_address): """One disposable data root that the Runtime's own checksum check accepts.""" data = root / ".local/share/elastos" (data / "bin").mkdir(mode=0o700, parents=True) + # The shared process census owns a Runtime by its data-root path, so each + # Home runs its own copy of the binary from its own bin directory. + runtime = data / "bin/elastos" + shutil.copy2(build.runtime, runtime) + runtime.chmod(0o700) manifest = json.loads(build.manifest.read_text()) for provider in STAGED_PROVIDERS: staged = data / "bin" / provider @@ -93,6 +98,7 @@ def __init__(self, name, build, root, bind_host): self.carrier_address = f"{bind_host}:{free_port(socket.SOCK_DGRAM)}" self.api_address = f"{bind_host}:{free_port()}" self.data = stage_home(build, root, self.carrier_address) + self.runtime = self.data / "bin/elastos" self.child = None def environment(self): @@ -107,7 +113,7 @@ def start(self): assert self.child is None, "Home already has a Runtime" with self.log.open("ab") as log: self.child = subprocess.Popen( - [str(self.build.runtime), "gateway", "--addr", self.api_address], + [str(self.runtime), "gateway", "--addr", self.api_address], env=self.environment(), stdout=log, stderr=log, @@ -125,7 +131,7 @@ def stop(self): def configure(self, *arguments): with (self.root / "configuration.private.log").open("ab") as log: subprocess.run( - [str(self.build.runtime), "collaboration-config", *map(str, arguments)], + [str(self.runtime), "collaboration-config", *map(str, arguments)], env=self.environment(), stdout=log, stderr=log, @@ -222,8 +228,15 @@ def admit(home): for home in homes.values(): try: record["process_cleanup"].append(home.stop()) - except (OSError, RuntimeError, AssertionError) as error: - record["process_cleanup"].append({"status": "failed", "error": str(error)[:200]}) + except Exception as error: # a stuck cleanup still leaves a result + record["process_cleanup"].append( + {"status": "failed", "error": f"{type(error).__name__}: {error}"[:200]} + ) + if record.get("status") != "passed": + # The fixture tree goes away, so a failure keeps its own last words. + record["log_tails"] = { + home.name: home.lines("")[-12:] for home in homes.values() + } shutil.rmtree(root, ignore_errors=True) return record @@ -263,6 +276,9 @@ def main(): for row in receipt["arrangements"]: print(f"node={row['node_build']:<9} homes={row['home_build']:<9} {row['status']}" + (f" {row['failure']}" if "failure" in row else "")) + for name, tail in row.get("log_tails", {}).items(): + for line in tail: + print(f" [{name}] {line[:200]}") for observation in row["observations"]: print( f" {observation['home']} joined={observation['joined']} " From 4adb7f72061b5b88a4e9dd6fab9613d4cf0a8616 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 13:29:17 -0500 Subject: [PATCH 05/15] test(carrier): probe a Home that stops reading its gossip stream The release gate asks what a Community node does while one joined Home holds its connection open and takes no more frames. The wedge arrangement stops that Home, asks a fresh Home to join, and reports the Community link lines it saw. --- scripts/gossip-mixed-version-journey.py | 76 +++++++++++++++++-------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/scripts/gossip-mixed-version-journey.py b/scripts/gossip-mixed-version-journey.py index d72e3a62d..27ece506f 100644 --- a/scripts/gossip-mixed-version-journey.py +++ b/scripts/gossip-mixed-version-journey.py @@ -16,6 +16,7 @@ from pathlib import Path import runpy import shutil +import signal import socket import subprocess import sys @@ -158,12 +159,19 @@ def observe(home, started): } -def run_arrangement(node_build, home_build, root, bind_host, settle): - """One node version against one Home version, with a fresh join after restart.""" +def run_arrangement(node_build, home_build, root, bind_host, settle, wedge=False): + """One node version against one Home version, with a fresh join after restart. + + With ``wedge`` the arrangement instead stops one joined Home from reading + its gossip stream while its connection stays up, and then asks a fresh + Home to join. That is the field symptom this release gate is about, so the + arrangement reports what it saw rather than a verdict. + """ root.mkdir(mode=0o700, parents=True, exist_ok=False) homes, record = {}, { "node_build": node_build.name, "home_build": home_build.name, + "shape": "wedge" if wedge else "restart", "node_runtime_sha256": node_build.runtime_sha256, "home_runtime_sha256": home_build.runtime_sha256, "observations": [], @@ -205,9 +213,16 @@ def admit(home): time.sleep(settle) record["observations"] += [observe(homes[name], started) for name in ("a", "b")] - # The node restarts, as an updated node does; the Homes keep running. - record["process_cleanup"].append(homes["community"].stop()) - homes["community"].start() + if wedge: + # SIGSTOP leaves the connection up and the peer unable to read, so + # the node keeps a neighbour that never takes another frame. + wedged = homes["b"].child.pid + os.kill(wedged, signal.SIGSTOP) + record["wedged_pid_stopped"] = True + else: + # The node restarts, as an updated node does; the Homes keep running. + record["process_cleanup"].append(homes["community"].stop()) + homes["community"].start() fresh = Home("c", home_build, root / "c", bind_host) homes["c"] = fresh admit(fresh) @@ -215,12 +230,17 @@ def admit(home): fresh.start() time.sleep(settle) record["observations"].append(observe(fresh, fresh_started)) - record["observations"] += [observe(homes[name], started) for name in ("a", "b")] - record["status"] = ( - "passed" - if all(row["joined"] and row["link_dead"] == 0 for row in record["observations"]) - else "failed" - ) + record["observations"] += [observe(homes[name], started) for name in ("a",)] + if wedge: + os.kill(wedged, signal.SIGCONT) + record["status"] = "observed" + else: + record["observations"] += [observe(homes["b"], started)] + record["status"] = ( + "passed" + if all(row["joined"] and row["link_dead"] == 0 for row in record["observations"]) + else "failed" + ) except Exception as error: record["status"] = "failed" record["failure"] = f"{type(error).__name__}: {error}"[:300] @@ -233,9 +253,13 @@ def admit(home): {"status": "failed", "error": f"{type(error).__name__}: {error}"[:200]} ) if record.get("status") != "passed": - # The fixture tree goes away, so a failure keeps its own last words. + # The fixture tree goes away, so a result keeps its own last words. record["log_tails"] = { - home.name: home.lines("")[-12:] for home in homes.values() + home.name: [ + line for line in home.lines("") + if any(needle in line for needle in (JOIN_LINE, DEAD_LINE, REBUILD_LINE)) + ][-6:] or home.lines("")[-6:] + for home in homes.values() } shutil.rmtree(root, ignore_errors=True) return record @@ -251,30 +275,36 @@ def main(): parser.add_argument("--bind-host", default="127.0.0.1") parser.add_argument("--settle-seconds", type=int, default=DEFAULT_SETTLE_SECONDS) parser.add_argument("--receipt", type=Path) + parser.add_argument("--wedge", action="store_true", + help="also probe a Home that stops reading its gossip stream") arguments = parser.parse_args() builds = { "released": Build("released", arguments.released_source, arguments.released_target), "develop": Build("develop", arguments.develop_source, arguments.develop_target), } receipt = {"platform": platform_key(), "arrangements": []} - for node in ("released", "develop"): - for home in ("released", "develop"): - root = arguments.fixture_root / f"node-{node}-homes-{home}" - receipt["arrangements"].append( - run_arrangement( - builds[node], builds[home], root, - arguments.bind_host, arguments.settle_seconds, - ) + shapes = [(node, home, False) for node in ("released", "develop") + for home in ("released", "develop")] + if arguments.wedge: + shapes += [(node, node, True) for node in ("released", "develop")] + for node, home, wedge in shapes: + name = f"node-{node}-homes-{home}" + ("-wedge" if wedge else "") + receipt["arrangements"].append( + run_arrangement( + builds[node], builds[home], arguments.fixture_root / name, + arguments.bind_host, arguments.settle_seconds, wedge, ) + ) receipt["status"] = ( "passed" - if all(row["status"] == "passed" for row in receipt["arrangements"]) + if all(row["status"] in ("passed", "observed") for row in receipt["arrangements"]) else "failed" ) if arguments.receipt: arguments.receipt.write_text(json.dumps(receipt, indent=2) + "\n") for row in receipt["arrangements"]: - print(f"node={row['node_build']:<9} homes={row['home_build']:<9} {row['status']}" + print(f"node={row['node_build']:<9} homes={row['home_build']:<9} " + f"{row['shape']:<7} {row['status']}" + (f" {row['failure']}" if "failure" in row else "")) for name, tail in row.get("log_tails", {}).items(): for line in tail: From 3b1cbd6ff77fa7874056378a08b5f717e13a846f Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 15:51:42 -0500 Subject: [PATCH 06/15] ci: name the test tools a rented Linux box installs scripts/setup-source-home.sh refuses to install a source Home without ffmpeg and ffprobe, so a rented box could not run the installed journeys. The box provisioner reads this list from the checked commit and installs a package only when its program is absent. Co-authored-by: Cursor --- scripts/ci-tools.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 scripts/ci-tools.txt diff --git a/scripts/ci-tools.txt b/scripts/ci-tools.txt new file mode 100644 index 000000000..4199f3616 --- /dev/null +++ b/scripts/ci-tools.txt @@ -0,0 +1,12 @@ +# Programs this repository's own checks start, beyond the base tool set that +# every box installs. One line per program: the program, then the Ubuntu +# package that provides it. A box installs the package only when the program is +# absent, so this list only adds. Write `a|b` when either program will do, and +# `macos` as the package when only a Mac supplies the program. +# +# scripts/setup-source-home.sh refuses to install a source Home without ffmpeg +# and ffprobe, and the Browser VM launcher looks for turnserver. The source-home +# CI job installs the same packages through CI_APT_PACKAGES. +ffmpeg ffmpeg +ffprobe ffmpeg +turnserver coturn From 8bd036acbe54cb62204496930725e29b650a17bd Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 16:09:03 -0500 Subject: [PATCH 07/15] test(carrier): bound the gossip wedge test and prove its pressure The wedge test could pass for the wrong reason. It took one second without progress as a settled flood, at any count. It dropped broadcast errors and task panics, so a silent neighbour that let go of its streams freed a wedged iroh-gossip 0.101.0 hub and the test passed. The wait for the flood and the endpoint shutdown had no time limit. The flood now ends in one of three ways: it completes, it stalls, or the fixture fails with its own error. A stall needs two consecutive samples without progress above a pressure floor of 392 broadcasts. The floor is the 64-message send queue of 0.101.0 plus the 1,250,000-byte default stream receive window in 3800-byte payloads. A flood that stays at or below the floor fails with "the fixture did not build enough pressure". A broadcast error, a panic in the flood, and an error or a panic in the silent neighbour fail the test with that cause. One deadline, the sum of the phase budgets plus a margin for bind and shutdown, covers every step. On iroh-gossip 0.102.0 the test passes in 1 s to 4 s. On 0.101.0 it fails as before: the hub stalls after 455 of 500 broadcasts, delivery stops, and a fresh join stops. cargo test --manifest-path elastos/Cargo.toml -p elastos-server \ --test gossip_actor_wedge -- --nocapture Co-authored-by: Cursor --- .../tests/gossip_actor_wedge.rs | 354 ++++++++++++------ 1 file changed, 237 insertions(+), 117 deletions(-) diff --git a/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs b/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs index 83bece517..b29e03e19 100644 --- a/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs +++ b/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs @@ -10,13 +10,18 @@ //! queue that drops or aborts instead of awaiting. //! //! The two assertions are the two symptoms a Community node shows: traffic to -//! the other neighbours keeps flowing, and a fresh join still completes. +//! the other neighbours keeps flowing, and a fresh join still completes. They +//! run only after the flood completed or stalled with more broadcasts accepted +//! than that stream credit and that channel can hold, which is the load that +//! parks the 0.101.0 actor. +use std::convert::Infallible; +use std::fmt; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, ensure, Context, Result}; use bytes::Bytes; use futures_lite::StreamExt; use gossip_rand::rngs::StdRng; @@ -25,21 +30,47 @@ use iroh::address_lookup::memory::MemoryLookup; use iroh::endpoint::{presets, Connection, RecvStream, SendStream}; use iroh::protocol::Router; use iroh::{Endpoint, EndpointAddr, EndpointId, RelayMode, SecretKey, TransportAddr}; -use iroh_gossip::api::{Event as ApiEvent, GossipReceiver}; +use iroh_gossip::api::{Event as ApiEvent, GossipReceiver, GossipSender}; use iroh_gossip::net::Gossip; use iroh_gossip::proto::{ Command, Config as ProtoConfig, Event as ProtoEvent, InEvent, Message as WireMessage, OutEvent, PeerData, State as ProtoState, TopicId, }; +use tokio::task::{JoinError, JoinHandle, JoinSet}; use tokio::time::timeout; /// Just under the 4096-byte default `max_message_size`, leaving header room. const PAYLOAD_SIZE: usize = 3800; -/// Enough to exhaust the 1.25 MiB default stream receive window (about 330 -/// payloads) and the 64-message send queue, with room to spare. +/// The per-peer channel that iroh-gossip 0.101.0 awaits on, in messages. +const SEND_QUEUE_CAP: usize = 64; +/// The default QUIC `stream_receive_window` in bytes, which is the credit a +/// neighbour grants on one stream before it reads. +const STREAM_RECEIVE_WINDOW: usize = 1_250_000; +/// The broadcasts that the neighbour's stream credit and the hub's send queue +/// absorb together. A flood that ends at or below this count may never have +/// met a full queue, so it says nothing about the hub. +const PRESSURE_FLOOR: usize = SEND_QUEUE_CAP + STREAM_RECEIVE_WINDOW / PAYLOAD_SIZE; const FLOOD_MESSAGES: usize = 500; + const PROBE_BUDGET: Duration = Duration::from_secs(15); const JOIN_BUDGET: Duration = Duration::from_secs(30); +/// iroh-gossip 0.102.0 holds each broadcast for up to 100 ms while the only +/// eager neighbour of its topic is busy (`BROADCAST_WAIT_LIMIT`), so a healthy +/// flood of `FLOOD_MESSAGES` can take up to 50 s. +const FLOOD_BUDGET: Duration = Duration::from_secs(60); +/// Ten of those holds, so a flood that the hub only slows still moves in every +/// sample. +const STALL_SAMPLE: Duration = Duration::from_secs(1); +const QUIET_SAMPLES_IN_A_STALL: u32 = 2; +/// Binding and closing the endpoints, which have no budget of their own. One +/// close takes up to 3 s here. +const UNBUDGETED_MARGIN: Duration = Duration::from_secs(30); +const TEST_DEADLINE: Duration = Duration::from_secs( + 5 * JOIN_BUDGET.as_secs() + + 2 * PROBE_BUDGET.as_secs() + + FLOOD_BUDGET.as_secs() + + UNBUDGETED_MARGIN.as_secs(), +); #[tokio::test(flavor = "multi_thread")] async fn a_neighbour_that_stops_reading_leaves_the_actor_serving_other_topics() -> Result<()> { @@ -47,6 +78,14 @@ async fn a_neighbour_that_stops_reading_leaves_the_actor_serving_other_topics() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .try_init() .ok(); + let report = timeout(TEST_DEADLINE, serve_others_beside_a_silent_neighbour()) + .await + .map_err(|_| anyhow!("the test did not finish within {TEST_DEADLINE:?}"))??; + println!("{report}"); + Ok(()) +} + +async fn serve_others_beside_a_silent_neighbour() -> Result { let flood_topic = TopicId::from_bytes([0xAB; 32]); let probe_topic = TopicId::from_bytes([0xCD; 32]); let lookup = MemoryLookup::new(); @@ -61,7 +100,7 @@ async fn a_neighbour_that_stops_reading_leaves_the_actor_serving_other_topics() let silent_id = silent_endpoint.id(); lookup.add_endpoint_info(loopback_addr(&silent_endpoint)); let (went_silent, mut is_silent) = tokio::sync::mpsc::channel(1); - let silent_task = tokio::spawn(run_silent_neighbour( + let mut silent_task = tokio::spawn(run_silent_neighbour( silent_endpoint.clone(), flood_topic, went_silent, @@ -71,113 +110,184 @@ async fn a_neighbour_that_stops_reading_leaves_the_actor_serving_other_topics() let (member_endpoint, member, _member_router) = gossip_node(&lookup).await?; let hub_id = hub_endpoint.id(); - let member_join = tokio::spawn({ - let member = member.clone(); - async move { member.subscribe_and_join(probe_topic, vec![hub_id]).await } - }); - let hub_probe = timeout(JOIN_BUDGET, hub.subscribe_and_join(probe_topic, vec![])) - .await - .context("the hub did not join the probe topic")??; - let member_probe = timeout(JOIN_BUDGET, member_join) - .await - .context("the member did not join the probe topic")???; - let (probe_sender, _hub_probe_receiver) = hub_probe.split(); - let (_member_sender, mut member_receiver) = member_probe.split(); + let observation = async { + let member_join = tokio::spawn({ + let member = member.clone(); + async move { member.subscribe_and_join(probe_topic, vec![hub_id]).await } + }); + let hub_probe = timeout(JOIN_BUDGET, hub.subscribe_and_join(probe_topic, vec![])) + .await + .context("the hub did not join the probe topic")??; + let member_probe = timeout(JOIN_BUDGET, member_join) + .await + .context("the member did not join the probe topic")???; + let (probe_sender, _hub_probe_receiver) = hub_probe.split(); + let (_member_sender, mut member_receiver) = member_probe.split(); - probe_sender - .broadcast(Bytes::from_static(b"before the flood")) - .await - .context("broadcast before the flood")?; - wait_for(&mut member_receiver, b"before the flood", PROBE_BUDGET) + deliver(&probe_sender, &mut member_receiver, b"before the flood") + .await + .context("the probe path failed before the flood, so the fixture is wrong")?; + + let (flood_sender, mut flood_receiver) = timeout( + JOIN_BUDGET, + hub.subscribe_and_join(flood_topic, vec![silent_id]), + ) .await - .context("the probe path failed before the flood, so the fixture is wrong")?; + .context("the hub did not take the silent neighbour into its active view")?? + .split(); + // Without this the neighbour is still reading, and the flood proves nothing. + timeout(JOIN_BUDGET, is_silent.recv()) + .await + .ok() + .flatten() + .context("the neighbour never reached the active view, so the fixture is wrong")?; + tokio::spawn(async move { while flood_receiver.next().await.is_some() {} }); + let flood = flood_until_settled(flood_sender).await?; - let flood = timeout( - JOIN_BUDGET, - hub.subscribe_and_join(flood_topic, vec![silent_id]), - ) - .await - .context("the hub did not take the silent neighbour into its active view")??; - // Without this the neighbour is still reading, and the flood proves nothing. - timeout(JOIN_BUDGET, is_silent.recv()) + let probe_started = Instant::now(); + let delivery = deliver(&probe_sender, &mut member_receiver, b"during the flood").await; + let probe_seconds = probe_started.elapsed().as_secs_f32(); + + let join_started = Instant::now(); + let (newcomer_endpoint, newcomer, _newcomer_router) = gossip_node(&lookup).await?; + let fresh_join = timeout( + JOIN_BUDGET, + newcomer.subscribe_and_join(probe_topic, vec![hub_id]), + ) .await - .ok() - .flatten() - .context("the neighbour never reached the active view, so the fixture is wrong")?; - let (flood_sender, mut flood_receiver) = flood.split(); - tokio::spawn(async move { while flood_receiver.next().await.is_some() {} }); - - // Fire and forget: the flood may stall, and only its collateral matters. - let sent = Arc::new(AtomicUsize::new(0)); - let flood_task = tokio::spawn({ - let sent = sent.clone(); - async move { - let payload = Bytes::from(vec![0x5A; PAYLOAD_SIZE]); - for _ in 0..FLOOD_MESSAGES { - if flood_sender.broadcast(payload.clone()).await.is_err() { - break; - } - sent.fetch_add(1, Ordering::Relaxed); - } - } - }); - let flooded = wait_until_flood_settles(&flood_task, &sent).await; + .map_err(|_| anyhow!("a fresh join did not complete within {JOIN_BUDGET:?}")) + .and_then(|joined| joined.map_err(anyhow::Error::from)); + let join_seconds = join_started.elapsed().as_secs_f32(); + newcomer_endpoint.close().await; - let probe_started = Instant::now(); - let delivery = async { - probe_sender - .broadcast(Bytes::from_static(b"during the flood")) - .await - .context("broadcast during the flood")?; - wait_for(&mut member_receiver, b"during the flood", PROBE_BUDGET).await - } - .await; - let probe_seconds = probe_started.elapsed().as_secs_f32(); - - let join_started = Instant::now(); - let (newcomer_endpoint, newcomer, _newcomer_router) = gossip_node(&lookup).await?; - let fresh_join = timeout( - JOIN_BUDGET, - newcomer.subscribe_and_join(probe_topic, vec![hub_id]), - ) - .await - .map_err(|_| anyhow!("a fresh join did not complete within {JOIN_BUDGET:?}")) - .and_then(|joined| joined.map_err(anyhow::Error::from)); - let join_seconds = join_started.elapsed().as_secs_f32(); + let report = format!( + "{flood}; delivery to the other neighbour took {probe_seconds:.1} s; \ + the fresh join took {join_seconds:.1} s" + ); + anyhow::Ok((flood, delivery, fresh_join, report)) + }; + // The neighbour ends only when it fails. A neighbour that lets go of its + // streams frees a wedged hub, and then both assertions pass. + let (flood, delivery, fresh_join, report) = tokio::select! { + biased; + ended = &mut silent_task => match ended + .context("the silent neighbour panicked")? + .context("the silent neighbour failed")? {}, + observed = observation => observed?, + }; - silent_endpoint.close().await; - flood_task.abort(); + // A hub that the neighbour frees resumes a parked flood, so stop the flood + // first. + let flood_stopped = flood.stop().await; silent_task.abort(); - newcomer_endpoint.close().await; + silent_endpoint.close().await; member_endpoint.close().await; hub_endpoint.close().await; - let report = format!( - "{flooded} of {FLOOD_MESSAGES} broadcasts left the hub; \ - delivery to the other neighbour took {probe_seconds:.1} s; \ - the fresh join took {join_seconds:.1} s" - ); + flood_stopped?; delivery.with_context(|| format!("delivery to the other neighbour stopped: {report}"))?; fresh_join.with_context(|| format!("a fresh join stopped: {report}"))?; - println!("{report}"); - Ok(()) + Ok(report) } -/// Returns the broadcasts the hub accepted once the flood finishes or stops -/// making progress; a stalled flood is the state the assertions run against. -async fn wait_until_flood_settles( - flood: &tokio::task::JoinHandle<()>, - sent: &AtomicUsize, -) -> usize { +/// A flood that put the hub under pressure. The hub is judged in either state. +enum SettledFlood { + /// The hub accepted every broadcast. + Completed, + /// The hub stopped accepting broadcasts above `PRESSURE_FLOOR`, and the + /// flood stays parked on the next one. + Stalled { + accepted: usize, + parked: JoinHandle>, + }, +} + +/// Broadcasts into a topic whose only neighbour reads nothing, until the flood +/// ends in one of three ways: it completed, it stalled, or the fixture failed. +/// A broadcast error, a panic, and a flood that stays at or below +/// `PRESSURE_FLOOR` are failures of the fixture, so they return the error. +async fn flood_until_settled(sender: GossipSender) -> Result { + let accepted = Arc::new(AtomicUsize::new(0)); + let mut task = tokio::spawn({ + let accepted = accepted.clone(); + async move { + let payload = Bytes::from(vec![0x5A; PAYLOAD_SIZE]); + for _ in 0..FLOOD_MESSAGES { + sender.broadcast(payload.clone()).await?; + accepted.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } + }); + + let started = Instant::now(); let mut last = 0; - loop { - tokio::time::sleep(Duration::from_secs(1)).await; - let now = sent.load(Ordering::Relaxed); - if flood.is_finished() || now == last { - return now; + let mut quiet_samples = 0; + while started.elapsed() < FLOOD_BUDGET { + if let Ok(ended) = timeout(STALL_SAMPLE, &mut task).await { + flood_result(ended)?; + ensure_pressure(accepted.load(Ordering::Relaxed))?; + return Ok(SettledFlood::Completed); } + let now = accepted.load(Ordering::Relaxed); + quiet_samples = if now == last { quiet_samples + 1 } else { 0 }; last = now; + if now > PRESSURE_FLOOR && quiet_samples >= QUIET_SAMPLES_IN_A_STALL { + return Ok(SettledFlood::Stalled { + accepted: now, + parked: task, + }); + } } + task.abort(); + ensure_pressure(last)?; + bail!( + "the flood neither completed nor stalled within {FLOOD_BUDGET:?}: \ + the hub accepted {last} of {FLOOD_MESSAGES} broadcasts and still accepts more" + ) +} + +impl SettledFlood { + /// Ends a parked flood, and reports what it did if it moved again after + /// the stall. + async fn stop(self) -> Result<()> { + let Self::Stalled { parked, .. } = self else { + return Ok(()); + }; + parked.abort(); + match parked.await { + Err(ended) if ended.is_cancelled() => Ok(()), + ended => flood_result(ended), + } + } +} + +impl fmt::Display for SettledFlood { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Completed => write!(f, "the hub accepted all {FLOOD_MESSAGES} flood broadcasts"), + Self::Stalled { accepted, .. } => write!( + f, + "the hub stalled after {accepted} of {FLOOD_MESSAGES} flood broadcasts" + ), + } + } +} + +fn flood_result(ended: Result, JoinError>) -> Result<()> { + ended + .context("the flood task panicked")? + .context("a flood broadcast failed") +} + +fn ensure_pressure(accepted: usize) -> Result<()> { + ensure!( + accepted > PRESSURE_FLOOR, + "the fixture did not build enough pressure: the flood ended after {accepted} \ + broadcasts, and the silent neighbour's stream credit plus the hub's send queue \ + absorb {PRESSURE_FLOOR}" + ); + Ok(()) } async fn gossip_node(lookup: &MemoryLookup) -> Result<(Endpoint, Gossip, Router)> { @@ -218,11 +328,19 @@ fn loopback_addr(endpoint: &Endpoint) -> EndpointAddr { EndpointAddr::from_parts(endpoint.id(), sockets) } -async fn wait_for(receiver: &mut GossipReceiver, expected: &[u8], budget: Duration) -> Result<()> { - timeout(budget, async { +async fn deliver( + sender: &GossipSender, + receiver: &mut GossipReceiver, + payload: &'static [u8], +) -> Result<()> { + timeout(PROBE_BUDGET, async { + sender + .broadcast(Bytes::from_static(payload)) + .await + .context("the hub refused the broadcast")?; while let Some(event) = receiver.next().await { if let Ok(ApiEvent::Received(message)) = event { - if message.content.as_ref() == expected { + if message.content.as_ref() == payload { return Ok(()); } } @@ -230,18 +348,19 @@ async fn wait_for(receiver: &mut GossipReceiver, expected: &[u8], budget: Durati Err(anyhow!("the event stream ended")) }) .await - .map_err(|_| anyhow!("no delivery within {budget:?}"))? + .map_err(|_| anyhow!("no delivery within {PROBE_BUDGET:?}"))? } /// Speaks the real membership handshake through the public protocol state, so /// the hub promotes it into the active view, then holds its streams open and /// reads nothing. The endpoint driver keeps acknowledging at the transport /// level, so QUIC stays healthy while the stream credit is never returned. +/// The first connection that fails ends the neighbour with that error. async fn run_silent_neighbour( endpoint: Endpoint, topic: TopicId, went_silent: tokio::sync::mpsc::Sender<()>, -) -> Result<()> { +) -> Result { let mut state: ProtoState = ProtoState::new( endpoint.id(), PeerData::new(Vec::::new()), @@ -257,22 +376,24 @@ async fn run_silent_neighbour( .for_each(drop); let state = Arc::new(tokio::sync::Mutex::new(state)); - while let Some(incoming) = endpoint.accept().await { - let Ok(connection) = incoming.await else { - continue; - }; - let state = state.clone(); - let went_silent = went_silent.clone(); - tokio::spawn(async move { - let remote = connection.remote_id(); - if let Err(error) = - go_silent_after_membership(connection, topic, remote, state, went_silent).await - { - eprintln!("the silent neighbour stopped: {error:#}"); + let mut connections = JoinSet::new(); + loop { + tokio::select! { + incoming = endpoint.accept() => { + let incoming = incoming.context("the silent endpoint closed")?; + let state = state.clone(); + let went_silent = went_silent.clone(); + connections.spawn(async move { + let connection = incoming.await.context("accept a connection")?; + let remote = connection.remote_id(); + go_silent_after_membership(connection, topic, remote, state, went_silent).await + }); } - }); + Some(ended) = connections.join_next() => { + match ended.context("a silent connection panicked")?? {} + } + } } - Ok(()) } async fn go_silent_after_membership( @@ -281,7 +402,7 @@ async fn go_silent_after_membership( remote: EndpointId, state: Arc>>, went_silent: tokio::sync::mpsc::Sender<()>, -) -> Result<()> { +) -> Result { let mut recv = connection.accept_uni().await.context("accept a stream")?; // The stream header starts with the bare topic id and has gained fields // since 0.101, so match the prefix rather than the whole frame. @@ -340,8 +461,7 @@ async fn go_silent_after_membership( } went_silent.send(()).await.ok(); - std::future::pending::<()>().await; - unreachable!() + std::future::pending().await } async fn read_frame(recv: &mut RecvStream) -> Result> { From f6fb42f807436c56f46502e892b34abf57fa96ba Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 16:48:16 -0500 Subject: [PATCH 08/15] test(carrier): name the builds and Homes of the two-Home journey A mixed-version run needs the Community node and the Homes to come from different installed builds. InstalledBuild binds an installed source Home to the checkout its installation receipt names, and Owner replaces the positional tuple for one running Home. The journey takes a Home build and an optional node build. The receipt records the node identity, and acceptance binds the node and each Home to the Runtime of its own build. One build still serves both by default, so the installed CI step is unchanged. Co-authored-by: Cursor --- scripts/ci-installed-journeys.py | 113 ++++++++++++++++++-------- scripts/ci-installed-two-home-test.py | 85 +++++++++++++------ 2 files changed, 140 insertions(+), 58 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index f85b89342..c3ee01852 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -20,6 +20,7 @@ import time import urllib.request from pathlib import Path +from typing import NamedTuple TIMING = runpy.run_path(str(Path(__file__).with_name("ci-installed-model-timing.py"))) UPSTREAM = runpy.run_path(str(Path(__file__).with_name("release-upstream-input.py"))) @@ -1713,11 +1714,11 @@ def ignore(directory, names): return target -def installed_message_identity(data): +def installed_message_identity(data, repo=None): """Use the same source/installed byte boundary as the other installed journeys.""" receipt = json.loads((data / "receipts/source-home-installation.json").read_text()) - candidate = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() - tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], text=True).strip() + candidate = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, cwd=repo).strip() + tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], text=True, cwd=repo).strip() runtime = digest(data / "bin/elastos") assert receipt["source"]["commit"] == candidate and receipt["source"]["tree"] == tree assert receipt["source"]["clean"] is True and receipt["runtime"]["parity"] is True @@ -1737,6 +1738,30 @@ def installed_message_identity(data): "installation_receipt_sha256": digest(data / "receipts/source-home-installation.json")} +class InstalledBuild(NamedTuple): + """One installed source Home that fixtures are copied from.""" + name: str + home: Path + data: Path + repo: Path + identity: dict + + +def installed_build(name, home, data, repo="."): + """Bind an installed Home to the checkout its installation receipt names.""" + home, data, repo = (Path(path).resolve() for path in (home, data, repo)) + return InstalledBuild(name, home, data, repo, installed_message_identity(data, repo)) + + +class Owner(NamedTuple): + """One running Home in an arrangement: a private copy of its build.""" + home: Path + data: Path + api: str + carrier: str + build: InstalledBuild + + def remove_message_fixture(root): """Remove only this fresh, process-free fixture; never follow a root/parent link.""" root = Path(os.path.abspath(root)) @@ -1833,7 +1858,7 @@ def attempt(label, action, seconds): # A failed Runtime census/stop still gets an independent PID-bound # attempt, then the next Home is stopped regardless of its result. attempt("fallback_" + name, lambda: stop_message_processes( - child, owned_processes(process_rows(), child.pid, fixtures[name][1])), 10) + child, owned_processes(process_rows(), child.pid, fixtures[name].data)), 10) def remove_and_measure(): attempt("disk_reserve", lambda: record.update(disk_after=require_disk_space(data)), 5) if attempt("fixture_removal", lambda: remove_message_fixture(root), 20): @@ -1849,6 +1874,7 @@ def remove_and_measure(): raise RuntimeError("two-Home cleanup failed; see installed journey receipt") +NODE = "community" # the bootstrap role; every other role is a member Home MESSAGE_NETWORKS = { "same-host": { "environment": {"ELASTOS_CARRIER_NETWORK": "direct", "ELASTOS_CARRIER_MDNS": "0"}, @@ -1879,17 +1905,20 @@ def message_endpoints_valid(homes, observations, arrangement): return counts == settings["launches"] -def two_home_receipt_valid(row, identity, network_arrangement="same-host"): +def two_home_receipt_valid(row, identity, network_arrangement="same-host", node_identity=None): + """``identity`` is the Home build; the node build is the same one unless named.""" + node_identity = node_identity or identity if row.get("network_arrangement") != network_arrangement or network_arrangement not in MESSAGE_NETWORKS: return False - if any(row.get(key) != value for key, value in identity.items()): + if any(row.get(key) != value for key, value in identity.items()) or row.get("node_identity") != node_identity: return False homes = row.get("homes", []) cleanup = row.get("process_cleanup", []) return (row.get("status") == "passed" and "failure" not in row and "browser_failure" not in row and row.get("fixture_removed") is True and len(homes) == 3 and {home.get("home") for home in homes} == {"a", "b", "community"} - and all(home.get("installed_runtime_sha256") == identity["installed_runtime_sha256"] for home in homes) + and all(home.get("installed_runtime_sha256") == (node_identity if home.get("home") == NODE else identity)[ + "installed_runtime_sha256"] for home in homes) and len(cleanup) == 5 and all(item.get("status") == "passed" and item.get("after") and sum(item["after"].values()) == 0 for item in cleanup) and row.get("browser_cross_home_http_requests") == 0 @@ -1902,38 +1931,43 @@ def two_home_receipt_valid(row, identity, network_arrangement="same-host"): and row["negative_controls"]["sender_delivered"].get("ui_detail") == "Sending") -def two_home_messages(home, data, evidence, network="same-host", fixture_root=None): - """Installed Community and Direct journey. Topology belongs to setup, not assertions.""" +def two_home_messages(home_build, evidence, network="same-host", fixture_root=None, node_build=None): + """Installed Community and Direct journey. Topology belongs to setup, not assertions. + + The Community node runs ``node_build`` and every other Home runs + ``home_build``; one build serves both unless a mixed-version run names two. + """ if network not in MESSAGE_NETWORKS: raise RuntimeError("unsupported two-Home network arrangement") settings = MESSAGE_NETWORKS[network] - identity = installed_message_identity(data) + node_build = node_build or home_build + data = home_build.data evidence.mkdir(parents=True, exist_ok=False) - root = fixture_root or home.parent / "two-home-message-fixture" + root = fixture_root or home_build.home.parent / "two-home-message-fixture" # Three fresh immutable copies; source fixtures never copy identity or user data. require_disk_space(data) root.mkdir(mode=0o700, parents=True, exist_ok=False) control = root / "control" control.mkdir(mode=0o700) - record = dict(identity, network_arrangement=network, results={}, homes=[], negative_controls={}, - direct_endpoint_observations=[]) + record = dict(home_build.identity, node_identity=node_build.identity, network_arrangement=network, results={}, + homes=[], negative_controls={}, direct_endpoint_observations=[]) children, fixtures, browser = {}, {}, None browser_processes = {} cleanup = [] started = time.monotonic() # Explicit environment excludes ambient relay, public-network and provider overrides. def environment(owner): - return {"PATH": os.environ["PATH"], "HOME": str(owner[0]), "XDG_DATA_HOME": str(owner[1].parent), + return {"PATH": os.environ["PATH"], "HOME": str(owner.home), "XDG_DATA_HOME": str(owner.data.parent), **settings["environment"], "ELASTOS_QUIET_RUNTIME_NOTICES": "1"} def launch(name): assert name not in children, "Home already has an owned Runtime" owner = fixtures[name] - with (owner[0] / "runtime.private.log").open("ab") as log: - child = subprocess.Popen([str(owner[1] / "bin/elastos"), "gateway", "--addr", owner[2]], + with (owner.home / "runtime.private.log").open("ab") as log: + child = subprocess.Popen([str(owner.data / "bin/elastos"), "gateway", "--addr", owner.api], env=environment(owner), stdout=log, stderr=log, start_new_session=True) children[name] = child - wait_healthy(child, owner[2], "two-Home " + name) - coordinates = json.loads(bounded_text(owner[1] / "gateway-runtime-coords.json")) + wait_healthy(child, owner.api, "two-Home " + name) + coordinates = json.loads(bounded_text(owner.data / "gateway-runtime-coords.json")) api = coordinates["api_url"].rstrip("/") token = post_json(api + "/api/auth/attach", {"secret": coordinates["attach_secret"], "scope": "shell"})["token"] ticket = post_json(api + "/api/provider/peer/get_ticket", {}, token)["data"]["ticket"] @@ -1942,49 +1976,50 @@ def launch(name): endpoints = decoded["endpoints"] assert len(endpoints) == 1, "fixture ticket must describe one installed Home" observed = endpoints[0]["addrs"] - assert observed == [{settings["ticket_transport"]: owner[3]}], "ticket differs from configured Carrier address" + assert observed == [{settings["ticket_transport"]: owner.carrier}], "ticket differs from configured Carrier address" record["direct_endpoint_observations"].append({"home": name, "ticket_addresses": observed}) def stop(name): child = children[name] with message_timeout(35): - row = cleanup_runtime(child, fixtures[name][1]) + row = cleanup_runtime(child, fixtures[name].data) cleanup.append(dict(row, home=name)) assert row["status"] == "passed", "two-Home process cleanup failed" del children[name] def configure(*arguments): - owner = fixtures["community"] + owner = fixtures[NODE] with (root / "configuration.private.log").open("ab") as log: - subprocess.run([str(owner[1] / "bin/elastos"), "collaboration-config", *map(str, arguments)], + subprocess.run([str(owner.data / "bin/elastos"), "collaboration-config", *map(str, arguments)], env=environment(owner), stdout=log, stderr=log, check=True, timeout=60) try: with message_timeout(360): - for name in ("community", "a", "b"): + for name in (NODE, "a", "b"): + build = node_build if name == NODE else home_build destination = root / name - copied = fresh_fixture(home, data, destination, include_engine=False) - assert installed_message_identity(copied) == identity + copied = fresh_fixture(build.home, build.data, destination, include_engine=False) + assert installed_message_identity(copied, build.repo) == build.identity carrier = f"{settings['bind_host']}:{free_port(socket.SOCK_DGRAM)}" with (copied / "config.toml").open("a") as config: config.write(f'carrier_bind_addr = "{carrier}"\n') - fixtures[name] = (destination, copied, f"{settings['bind_host']}:{free_port()}", carrier) + fixtures[name] = Owner(destination, copied, f"{settings['bind_host']}:{free_port()}", carrier, build) record["homes"].append({"home": name, "installed_runtime_sha256": digest(copied / "bin/elastos"), "carrier_bind_address": carrier}) - launch("community") + launch(NODE) configure("create-authority-key", "--key", root / "authority.key") - configure("export-local-bootstrap-receipt", "--data-root", fixtures["community"][1], + configure("export-local-bootstrap-receipt", "--data-root", fixtures[NODE].data, "--runtime-kind", "gateway", "--output", root / "bootstrap.json") configure("generate-initial", "--authority-key", root / "authority.key", "--network-id", "ci.two-home", "--conversation-id", "community", "--bootstrap-peer", root / "bootstrap.json", "--output", root / "network.json") configure("verify", "--input", root / "network.json") - stop("community") + stop(NODE) for owner in fixtures.values(): - target = owner[1] / "collaboration-network-v1.json" + target = owner.data / "collaboration-network-v1.json" shutil.copy2(root / "network.json", target) target.chmod(0o600) for name in fixtures: launch(name) - bases = [f"http://{settings['browser_host']}:" + fixtures[name][2].rsplit(":", 1)[1] for name in ("a", "b")] + bases = [f"http://{settings['browser_host']}:" + fixtures[name].api.rsplit(":", 1)[1] for name in ("a", "b")] with (root / "browser.private.log").open("wb") as log: - browser = subprocess.Popen(["node", "scripts/ci-installed-two-home-journey.mjs", *bases, + browser = subprocess.Popen(["node", str(Path(__file__).with_name("ci-installed-two-home-journey.mjs")), *bases, str(evidence), str(control), settings["browser_host"]], stdout=log, stderr=log, start_new_session=True) last, next_census = 0, 0 while browser.poll() is None: @@ -2010,7 +2045,7 @@ def configure(*arguments): assert all(record["negative_controls"].get(name, {}).get("assertion_rejected") is True for name in ("sender_delivered",)) except BaseException as error: - record["failure"] = observer_failure("two_home_messages", error, [str(root), str(home)]) + record["failure"] = observer_failure("two_home_messages", error, [str(root), str(home_build.home), str(node_build.home)]) raise finally: record["process_cleanup"] = cleanup @@ -2055,12 +2090,20 @@ def repeat(home, data, evidence, count): parser.add_argument("--two-home-messages", action="store_true") parser.add_argument("--network-arrangement", choices=MESSAGE_NETWORKS.keys(), default="same-host") parser.add_argument("--two-home-root", type=Path, help="fresh disposable fixture path") + # A mixed-version run names a second installed Home for the Community node. + parser.add_argument("--node-home", type=Path) + parser.add_argument("--node-data", type=Path) + parser.add_argument("--node-repo", type=Path, help="checkout the node Home was installed from") args = parser.parse_args() if args.two_home_messages: if args.repeat or args.home_only or args.carrier_get or args.finite_memory or args.model_repair: parser.error("two-Home messages is a separate installed journey") - two_home_messages(args.home.resolve(), args.data.resolve(), args.evidence.resolve(), - args.network_arrangement, args.two_home_root) + node = (args.node_home, args.node_data, args.node_repo) + if any(node) and not all(node): + parser.error("a node build needs --node-home, --node-data and --node-repo") + two_home_messages(installed_build("home", args.home, args.data), args.evidence.resolve(), + args.network_arrangement, args.two_home_root, + installed_build("node", *node) if all(node) else None) raise SystemExit(0) if args.model_repair and (sys.platform != "linux" or args.home_only or args.repeat or args.finite_memory): parser.error("--model-repair proves one production Linux Home with two signed models") diff --git a/scripts/ci-installed-two-home-test.py b/scripts/ci-installed-two-home-test.py index 218a5e48c..662ec54db 100644 --- a/scripts/ci-installed-two-home-test.py +++ b/scripts/ci-installed-two-home-test.py @@ -54,7 +54,8 @@ def identity(self): def receipt(self): identity = self.identity() - return {**identity, 'status':'passed', 'fixture_removed':True, 'network_arrangement':'same-host', + return {**identity, 'node_identity':identity, 'status':'passed', 'fixture_removed':True, + 'network_arrangement':'same-host', 'homes':[{'home':name, 'installed_runtime_sha256':identity['installed_runtime_sha256'], 'carrier_bind_address':f'127.0.0.1:{1025+index}'} for index,name in enumerate(('a','b','community'))], @@ -96,6 +97,24 @@ def test_acceptance_refuses_stale_identity_unchecked_failure_controls_or_survivi changed['homes'][1]['home'] = 'a' self.assertFalse(check(changed, self.identity())) + def test_acceptance_binds_the_node_and_each_home_to_its_own_build(self): + check = JOURNEY['two_home_receipt_valid'] + homes, node = self.identity(), {**self.identity(), 'candidate':'f'*40, 'installed_runtime_sha256':'9'*64} + mixed = self.receipt() + mixed['node_identity'] = node + next(home for home in mixed['homes'] if home['home'] == 'community')['installed_runtime_sha256'] = '9'*64 + self.assertTrue(check(mixed, homes, node_identity=node)) + self.assertFalse(check(mixed, homes), 'a same-version acceptance refuses another node build') + self.assertFalse(check(self.receipt(), homes, node_identity=node), 'the named node build did not run') + for name in ('community', 'a'): + changed = copy.deepcopy(mixed) + row = next(home for home in changed['homes'] if home['home'] == name) + row['installed_runtime_sha256'] = ('a' if name == 'community' else '9')*64 + self.assertFalse(check(changed, homes, node_identity=node), name+' ran the other build') + changed = self.receipt() + del changed['node_identity'] + self.assertFalse(check(changed, homes)) + def test_network_arrangement_and_each_observed_ticket_must_match_configuration(self): check = JOURNEY['two_home_receipt_valid'] valid = self.receipt() @@ -151,7 +170,8 @@ def disk(_): 'stop_message_processes':fallback, 'require_disk_space':disk}): with self.assertRaisesRegex(RuntimeError, 'cleanup failed'): finish(root, root, evidence, {'status':'passed'}, children, - {name:(root,root) for name in children}, stop, None, {}, time.monotonic()) + {name:JOURNEY['Owner'](root, root, '', '', None) for name in children}, + stop, None, {}, time.monotonic()) self.assertEqual(stopped, ['a','b','community']) fallback.assert_called_once() self.assertFalse(root.exists()) @@ -227,30 +247,49 @@ def test_sigkill_fallback_stops_a_detached_browser_after_node_exits(self): parent.kill() parent.wait() + def installed_data(self, data, commit, tree): + (data/'bin').mkdir(parents=True) + (data/'receipts').mkdir() + runtime = data/'bin/elastos' + runtime.write_bytes(b'fixture runtime') + manifest = {'capsules':{}} + for name in ('home','home-gui','people','inbox','chat-room'): + directory = data/'capsules'/name + directory.mkdir(parents=True) + (directory/'index.html').write_text('fixture page') + manifest['capsules'][name] = {'entrypoint':'index.html', + 'entrypoint_sha256':'sha256:'+JOURNEY['digest'](directory/'index.html'), + 'browser_assets':[{'path':'index.html','sha256':'sha256:'+JOURNEY['digest'](directory/'index.html')}]} + (data/'components.json').write_text(json.dumps(manifest)) + (data/'receipts/source-home-capsules.json').write_text('{}') + sha = 'sha256:'+JOURNEY['digest'](runtime) + receipt = {'source':{'commit':commit,'tree':tree,'clean':True}, + 'runtime':{'built_sha256':sha,'installed_sha256':sha,'parity':True}, + 'components_sha256':'sha256:'+JOURNEY['digest'](data/'components.json'), + 'source_home_capsule_metadata_receipt_sha256':'sha256:'+JOURNEY['digest'](data/'receipts/source-home-capsules.json')} + (data/'receipts/source-home-installation.json').write_text(json.dumps(receipt)) + return runtime + + def test_an_installed_build_is_bound_to_its_own_checkout(self): + with tempfile.TemporaryDirectory() as temporary: + repo, home = Path(temporary).resolve()/'released', Path(temporary).resolve()/'home' + repo.mkdir() + git = lambda *arguments: subprocess.check_output(['git', *arguments], cwd=repo, text=True).strip() + git('init', '--quiet') + git('-c', 'user.name=fixture', '-c', 'user.email=fixture@example.invalid', '-c', 'commit.gpgsign=false', + 'commit', '--quiet', '--allow-empty', '--message', 'released fixture') + self.installed_data(home/'data', git('rev-parse', 'HEAD'), git('rev-parse', 'HEAD^{tree}')) + build = JOURNEY['installed_build']('released', home, home/'data', repo) + self.assertEqual((build.name, build.home, build.data, build.repo), ('released', home, home/'data', repo)) + self.assertEqual(build.identity['candidate'], git('rev-parse', 'HEAD')) + with self.assertRaises(AssertionError): + JOURNEY['installed_build']('candidate', home, home/'data') + def test_source_identity_rejects_changed_runtime_and_browser_asset_before_launch(self): identify = JOURNEY['installed_message_identity'] with tempfile.TemporaryDirectory() as temporary: - data = Path(temporary).resolve() - (data/'bin').mkdir() - (data/'receipts').mkdir() - runtime = data/'bin/elastos' - runtime.write_bytes(b'fixture runtime') - manifest = {'capsules':{}} - for name in ('home','home-gui','people','inbox','chat-room'): - directory = data/'capsules'/name - directory.mkdir(parents=True) - (directory/'index.html').write_text('fixture page') - manifest['capsules'][name] = {'entrypoint':'index.html', - 'entrypoint_sha256':'sha256:'+JOURNEY['digest'](directory/'index.html'), - 'browser_assets':[{'path':'index.html','sha256':'sha256:'+JOURNEY['digest'](directory/'index.html')}]} - (data/'components.json').write_text(json.dumps(manifest)) - (data/'receipts/source-home-capsules.json').write_text('{}') - sha = 'sha256:'+JOURNEY['digest'](runtime) - receipt = {'source':{'commit':'c'*40,'tree':'d'*40,'clean':True}, - 'runtime':{'built_sha256':sha,'installed_sha256':sha,'parity':True}, - 'components_sha256':'sha256:'+JOURNEY['digest'](data/'components.json'), - 'source_home_capsule_metadata_receipt_sha256':'sha256:'+JOURNEY['digest'](data/'receipts/source-home-capsules.json')} - (data/'receipts/source-home-installation.json').write_text(json.dumps(receipt)) + data = Path(temporary).resolve()/'data' + runtime = self.installed_data(data, 'c'*40, 'd'*40) def observe(): with mock.patch.object(identify.__globals__['subprocess'], 'check_output', side_effect=['c'*40,'d'*40]): return identify(data) From 5af36f2d3e4b8eee9d17e35d5752d501d03a71ab Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 16:53:19 -0500 Subject: [PATCH 09/15] test(carrier): prove the Community link after every transition The journey passed when no dead-link warning appeared, and a remembered neighbour count hides a dead link from that warning. The journey now collects positive evidence instead. A table names four transitions: the first start, a Community node restart under running Homes, a fresh Home that joins the restarted node, and the restart of a stopped Home. After each one, every Home is a live process that reports a Carrier peer through its own status command. Then every member Home posts one fresh Community message, every other member reads that exact text, and one direct message arrives with its exact text. Acceptance refuses a receipt that lacks a transition, a Home, a live process, a peer count of 1 or more, an ordered Community pair or the direct message, and a text that an earlier transition already used. Co-authored-by: Cursor --- scripts/ci-installed-journeys.py | 170 ++++++++++++++++++---- scripts/ci-installed-two-home-journey.mjs | 70 +++++++-- scripts/ci-installed-two-home-test.py | 97 +++++++++++- 3 files changed, 289 insertions(+), 48 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index c3ee01852..01e2d7429 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -4,6 +4,7 @@ import base64 from contextlib import contextmanager import hashlib +import itertools import json import os import re @@ -1771,8 +1772,9 @@ def remove_message_fixture(root): shutil.rmtree(root) -# The 10-minute CI step reserves 360 s for work, 150 s for process cleanup, -# and 30 s for disk/removal/receipt operations: 540 s, with 60 s runner margin. +# The 10-minute CI step reserves 360 s for work, 195 s for process cleanup of +# four Homes, and 30 s for disk/removal/receipt operations: 585 s, with 15 s +# runner margin. class MessageDeadline(BaseException): """A phase deadline survives recoverable OS/network error handlers.""" @@ -1879,11 +1881,26 @@ def remove_and_measure(): "same-host": { "environment": {"ELASTOS_CARRIER_NETWORK": "direct", "ELASTOS_CARRIER_MDNS": "0"}, "bind_host": "127.0.0.1", "browser_host": "localhost", "ticket_transport": "Ip", - "launches": {"community": 2, "a": 1, "b": 2}, + "launches": {"community": 3, "a": 1, "b": 2, "c": 1}, }, } +class Transition(NamedTuple): + """A control action and the Homes that must prove themselves after it.""" + name: str + action: "str | None" # None: the journey's own first launch + homes: tuple + + +MESSAGE_TRANSITIONS = ( + Transition("start", None, (NODE, "a", "b")), + Transition("node_restart", "restart_community", (NODE, "a", "b")), + Transition("fresh_home", "start_c", (NODE, "a", "b", "c")), + Transition("restart_b", "start_b", (NODE, "a", "b", "c")), +) + + def message_endpoints_valid(homes, observations, arrangement): settings = MESSAGE_NETWORKS.get(arrangement) if settings is None: @@ -1905,6 +1922,56 @@ def message_endpoints_valid(homes, observations, arrangement): return counts == settings["launches"] +def home_peer_count(owner, environment): + """Carrier peers as the Home's own status command reports them; None when it reports no count.""" + status = subprocess.run([str(owner.data / "bin/elastos"), "home", "--status", "--json"], env=environment, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True, timeout=30).stdout + return json.loads(status)["runtime"].get("peer_count") + + +def transition_homes(transition, children, peer_count, budget=60): + """Liveness comes from each owned process itself. A peer count recovers + some time after a restart, so it is polled until every Home has one.""" + end = time.monotonic() + budget + counts = dict.fromkeys(transition.homes) + while True: + alive = {name: name in children and children[name].poll() is None for name in transition.homes} + for name in transition.homes: + if alive[name] and not counts[name]: + counts[name] = peer_count(name) + if not all(alive.values()) or all(counts.values()) or time.monotonic() >= end: + return {name: {"alive": alive[name], "peer_count": counts[name]} for name in transition.homes} + time.sleep(1) + + +def transition_evidence_valid(row): + """After every transition each Home is alive with a Carrier peer, every + member read every other member's fresh post, and one direct message + arrived, each with its exact text.""" + fresh = [] + for transition in MESSAGE_TRANSITIONS: + evidence = row.get("transitions", {}).get(transition.name, {}) + homes = evidence.get("homes", {}) + if set(homes) != set(transition.homes) or not all( + home.get("alive") is True and type(home.get("peer_count")) is int and home["peer_count"] >= 1 + for home in homes.values()): + return False + posts, direct = evidence.get("community", []), evidence.get("direct", {}) + pairs = {(post.get("from"), post.get("to")) for post in posts} + members = [name for name in transition.homes if name != NODE] + if (len(posts) != len(pairs) or pairs != set(itertools.permutations(members, 2)) + or (direct.get("from"), direct.get("to")) not in pairs): + return False + if not all(isinstance(item.get("sent"), str) and item["sent"] and item.get("received") == item["sent"] + for item in (*posts, direct)): + return False + texts = {post["from"]: post["sent"] for post in posts} + if any(post["sent"] != texts[post["from"]] for post in posts): + return False + fresh += [*texts.values(), direct["sent"]] + return len(set(fresh)) == len(fresh) + + def two_home_receipt_valid(row, identity, network_arrangement="same-host", node_identity=None): """``identity`` is the Home build; the node build is the same one unless named.""" node_identity = node_identity or identity @@ -1912,15 +1979,17 @@ def two_home_receipt_valid(row, identity, network_arrangement="same-host", node_ return False if any(row.get(key) != value for key, value in identity.items()) or row.get("node_identity") != node_identity: return False + launches = MESSAGE_NETWORKS[network_arrangement]["launches"] homes = row.get("homes", []) cleanup = row.get("process_cleanup", []) return (row.get("status") == "passed" and "failure" not in row and "browser_failure" not in row - and row.get("fixture_removed") is True and len(homes) == 3 - and {home.get("home") for home in homes} == {"a", "b", "community"} + and row.get("fixture_removed") is True and len(homes) == len(launches) + and {home.get("home") for home in homes} == set(launches) and all(home.get("installed_runtime_sha256") == (node_identity if home.get("home") == NODE else identity)[ "installed_runtime_sha256"] for home in homes) - and len(cleanup) == 5 and all(item.get("status") == "passed" and item.get("after") - and sum(item["after"].values()) == 0 for item in cleanup) + and len(cleanup) == sum(launches.values()) + and all(item.get("status") == "passed" and item.get("after") + and sum(item["after"].values()) == 0 for item in cleanup) and row.get("browser_cross_home_http_requests") == 0 and message_endpoints_valid(homes, row.get("direct_endpoint_observations", []), network_arrangement) and all(row.get("results", {}).get(name) == "passed" for name in ( @@ -1928,7 +1997,8 @@ def two_home_receipt_valid(row, identity, network_arrangement="same-host", node_ and all(row.get("negative_controls", {}).get(name, {}).get("assertion_rejected") is True for name in ("sender_delivered",)) and row["negative_controls"]["sender_delivered"].get("api_state") == "pending" - and row["negative_controls"]["sender_delivered"].get("ui_detail") == "Sending") + and row["negative_controls"]["sender_delivered"].get("ui_detail") == "Sending" + and transition_evidence_valid(row)) def two_home_messages(home_build, evidence, network="same-host", fixture_root=None, node_build=None): @@ -1944,13 +2014,13 @@ def two_home_messages(home_build, evidence, network="same-host", fixture_root=No data = home_build.data evidence.mkdir(parents=True, exist_ok=False) root = fixture_root or home_build.home.parent / "two-home-message-fixture" - # Three fresh immutable copies; source fixtures never copy identity or user data. + # Fresh immutable copies; source fixtures never copy identity or user data. require_disk_space(data) root.mkdir(mode=0o700, parents=True, exist_ok=False) control = root / "control" control.mkdir(mode=0o700) record = dict(home_build.identity, node_identity=node_build.identity, network_arrangement=network, results={}, - homes=[], negative_controls={}, direct_endpoint_observations=[]) + homes=[], negative_controls={}, direct_endpoint_observations=[], transitions={}) children, fixtures, browser = {}, {}, None browser_processes = {} cleanup = [] @@ -1990,19 +2060,50 @@ def configure(*arguments): with (root / "configuration.private.log").open("ab") as log: subprocess.run([str(owner.data / "bin/elastos"), "collaboration-config", *map(str, arguments)], env=environment(owner), stdout=log, stderr=log, check=True, timeout=60) + def stage(name): + build = node_build if name == NODE else home_build + destination = root / name + copied = fresh_fixture(build.home, build.data, destination, include_engine=False) + assert installed_message_identity(copied, build.repo) == build.identity + carrier = f"{settings['bind_host']}:{free_port(socket.SOCK_DGRAM)}" + with (copied / "config.toml").open("a") as config: + config.write(f'carrier_bind_addr = "{carrier}"\n') + fixtures[name] = Owner(destination, copied, f"{settings['bind_host']}:{free_port()}", carrier, build) + record["homes"].append({"home": name, "installed_runtime_sha256": digest(copied / "bin/elastos"), + "carrier_bind_address": carrier}) + def admit(name): + target = fixtures[name].data / "collaboration-network-v1.json" + shutil.copy2(root / "network.json", target) + target.chmod(0o600) + def base(name): + return f"http://{settings['browser_host']}:" + fixtures[name].api.rsplit(":", 1)[1] + def prove(transition): + homes = transition_homes(transition, children, + lambda name: home_peer_count(fixtures[name], environment(fixtures[name]))) + record["transitions"].setdefault(transition.name, {})["homes"] = homes + for name, home in homes.items(): + assert home["alive"], f"two-Home {name} is not running after {transition.name}" + assert home["peer_count"] is not None, f"two-Home {name} status reports no runtime.peer_count" + assert home["peer_count"] >= 1, f"two-Home {name} has no Carrier peer after {transition.name}" + def require_running(): + for name, child in children.items(): + assert child.poll() is None, f"two-Home {name} exited during the journey" + def restart_node(): + stop(NODE) + launch(NODE) + def start_fresh_home(): + stage("c") + admit("c") + launch("c") + return {"base": base("c")} + actions = {"stop_b": lambda: stop("b"), "start_b": lambda: launch("b"), + "restart_community": restart_node, "start_c": start_fresh_home} + start, *later = MESSAGE_TRANSITIONS + entered = {transition.action: transition for transition in later} try: with message_timeout(360): for name in (NODE, "a", "b"): - build = node_build if name == NODE else home_build - destination = root / name - copied = fresh_fixture(build.home, build.data, destination, include_engine=False) - assert installed_message_identity(copied, build.repo) == build.identity - carrier = f"{settings['bind_host']}:{free_port(socket.SOCK_DGRAM)}" - with (copied / "config.toml").open("a") as config: - config.write(f'carrier_bind_addr = "{carrier}"\n') - fixtures[name] = Owner(destination, copied, f"{settings['bind_host']}:{free_port()}", carrier, build) - record["homes"].append({"home": name, "installed_runtime_sha256": digest(copied / "bin/elastos"), - "carrier_bind_address": carrier}) + stage(name) launch(NODE) configure("create-authority-key", "--key", root / "authority.key") configure("export-local-bootstrap-receipt", "--data-root", fixtures[NODE].data, @@ -2011,18 +2112,20 @@ def configure(*arguments): "--conversation-id", "community", "--bootstrap-peer", root / "bootstrap.json", "--output", root / "network.json") configure("verify", "--input", root / "network.json") stop(NODE) - for owner in fixtures.values(): - target = owner.data / "collaboration-network-v1.json" - shutil.copy2(root / "network.json", target) - target.chmod(0o600) + for name in fixtures: + admit(name) for name in fixtures: launch(name) - bases = [f"http://{settings['browser_host']}:" + fixtures[name].api.rsplit(":", 1)[1] for name in ("a", "b")] + prove(start) with (root / "browser.private.log").open("wb") as log: - browser = subprocess.Popen(["node", str(Path(__file__).with_name("ci-installed-two-home-journey.mjs")), *bases, - str(evidence), str(control), settings["browser_host"]], stdout=log, stderr=log, start_new_session=True) + browser = subprocess.Popen(["node", str(Path(__file__).with_name("ci-installed-two-home-journey.mjs")), + base("a"), base("b"), str(evidence), str(control), settings["browser_host"]], + stdout=log, stderr=log, start_new_session=True) last, next_census = 0, 0 while browser.poll() is None: + # A Home that exits between transitions fails the journey at once, + # even when the other Homes could finish their exchange without it. + require_running() if time.monotonic() >= next_census: remember_message_processes(browser, browser_processes, "--two-home-fixture=" + str(control)) next_census = time.monotonic() + 1 @@ -2030,20 +2133,27 @@ def configure(*arguments): if request.exists(): value = private_json(request) if value["id"] > last: - assert value["id"] == last + 1 and value["action"] in {"stop_b", "start_b"} - stop("b") if value["action"] == "stop_b" else launch("b") + assert value["id"] == last + 1 and value["action"] in actions + reply = actions[value["action"]]() or {} + # The browser half exchanges messages only after the Homes have their peers back. + if value["action"] in entered: + prove(entered[value["action"]]) last = value["id"] - private_json(control / "response.json", {"id": last, "status": "passed"}) + private_json(control / "response.json", {"id": last, "status": "passed", **reply}) time.sleep(.1) observed = json.loads((evidence / "two-home-ui.json").read_text()) + for name, exchange in observed.pop("exchanges", {}).items(): + record["transitions"].setdefault(name, {}).update(exchange) record.update(observed) if browser.returncode and (control / "error.private.txt").exists(): record["browser_failure"] = redact((control / "error.private.txt").read_text(), [str(root)])[:1200] assert browser.returncode == 0, "installed two-Home browser journey failed" + require_running() assert all(record["results"].get(name) == "passed" for name in ( "community_post", "direct_received", "sender_delivered", "offline_pending", "restart_delivery")) assert all(record["negative_controls"].get(name, {}).get("assertion_rejected") is True for name in ("sender_delivered",)) + assert transition_evidence_valid(record), "two-Home transition evidence is incomplete" except BaseException as error: record["failure"] = observer_failure("two_home_messages", error, [str(root), str(home_build.home), str(node_build.home)]) raise diff --git a/scripts/ci-installed-two-home-journey.mjs b/scripts/ci-installed-two-home-journey.mjs index 402b8e1b4..1e2268d38 100644 --- a/scripts/ci-installed-two-home-journey.mjs +++ b/scripts/ci-installed-two-home-journey.mjs @@ -25,19 +25,18 @@ async function run() { const require = createRequire(import.meta.url); const {chromium} = require(process.env.BROWSER_OPERATOR_PLAYWRIGHT_CORE || '../elastos/tools/browser-playwright-engine/node_modules/playwright'); const [baseA, baseB, evidence, control, browserHost] = process.argv.slice(2); - const bases = [baseA, baseB]; - assertMessageOrigins(bases, browserHost); - const result = {stage:'sign_in', results:{}, negative_controls:{}, observations:[], browser_cross_home_http_requests:0}; + assertMessageOrigins([baseA, baseB], browserHost); + const result = {stage:'sign_in', results:{}, negative_controls:{}, observations:[], exchanges:{}, browser_cross_home_http_requests:0}; let sequence = 0; async function runtime(action) { const id = ++sequence; writeFileSync(join(control, 'request.json.next'), JSON.stringify({id, action}), {flag:'wx', mode:0o600}); renameSync(join(control, 'request.json.next'), join(control, 'request.json')); - await until('owned Runtime action', async () => { + return until('owned Runtime action', async () => { let reply; try {reply = JSON.parse(readFileSync(join(control, 'response.json'), 'utf8'));} catch (error) {if (error.code !== 'ENOENT') throw error;} - return reply?.id === id && reply.status === 'passed'; + return reply?.id === id && reply.status === 'passed' && reply; }, 120000); } const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms)); @@ -51,7 +50,8 @@ async function run() { result.stage='browser_launch'; browser = await chromium.launch({headless:true, args:['--two-home-fixture='+control]}); const homes=[]; - for(const [i,base] of bases.entries()) { + async function signIn(base) { + const i=homes.length, name='abc'[i]; const context=await browser.newContext({timezoneId:'UTC', serviceWorkers:'block', viewport:{width:1440,height:900}}); await context.route('**/*', route => { const url = new URL(route.request().url()); @@ -68,7 +68,7 @@ async function run() { result.stage='home_frontdoor_'+i; await page.goto(base+'/home/',{waitUntil:'domcontentloaded'}); result.stage='home_name_'+i; - await page.locator('#home-unlock-name').fill('Two Home '+(i?'B':'A'),{timeout:60000}); + await page.locator('#home-unlock-name').fill('Two Home '+name.toUpperCase(),{timeout:60000}); result.stage='passkey_registration_'+i; const registered=page.waitForResponse(r=>r.url().endsWith('/api/auth/passkey/register/complete')); await page.locator('#home-unlock-primary').click(); @@ -87,9 +87,11 @@ async function run() { return tab; } result.stage='people_launch_'+i; - const people=await app('people'); - homes.push({context,page,people,app,base}); + const home={name,context,page,people:await app('people'),app,base}; + homes.push(home); + return home; } + const a=await signIn(baseA), b=await signIn(baseB); async function api(page,path,body) { return page.evaluate(async ({path,body})=>{ const token=new URLSearchParams(location.hash.slice(1)).get('home_token'); @@ -98,12 +100,14 @@ async function run() { return r.json(); },{path,body}); } - const [a,b]=homes; - const communityA=await a.app('chat-room'), communityB=await b.app('chat-room'); + async function joinCommunity(home) { + await api(home.community, '/api/apps/chat-room/session/start', {}); + await home.community.locator('#message-input:enabled').waitFor({timeout:60000}); + } + const communityA=a.community=await a.app('chat-room'), communityB=b.community=await b.app('chat-room'); result.stage='community_post'; await communityA.locator('#message-input:enabled').waitFor({timeout:90000}); - await api(communityB, '/api/apps/chat-room/session/start', {}); - await communityB.locator('#message-input:enabled').waitFor({timeout:60000}); + await joinCommunity(b); const postText='Two Home Community '+crypto.randomUUID(); const posted=await api(communityA, '/api/apps/chat-room/objects/send', {request_id:'two-home:'+crypto.randomUUID(),body:postText}); let communityPoll; @@ -168,6 +172,43 @@ async function run() { assertSenderDelivery(settled,senderDetail); result.results.sender_delivered='passed'; result.direct={text,recipient_state:received.delivery_state,sender_state:settled.delivery_state,sender_detail:senderDetail}; + // One round after a transition: every Home posts, every other Home reads + // that exact text, and one direct message crosses the accepted contact. + async function exchange(transition) { + result.stage='exchange_'+transition; + const round={community:[]}; + for(const sender of homes) { + const sent=`Two Home ${transition} ${sender.name} ${crypto.randomUUID()}`; + const post=await api(sender.community,'/api/apps/chat-room/objects/send',{request_id:'two-home:'+crypto.randomUUID(),body:sent}); + for(const receiver of homes.filter(home=>home!==sender)) { + let poll; + await until(`Community ${transition} from ${sender.name} at ${receiver.name}`,async()=>{ + poll=await api(receiver.community,'/api/apps/chat-room/poll',{since:0,mark_read:false}); + return poll.objects.some(row=>row.message_id===post.message_id&&row.body===sent); + }); + round.community.push({from:sender.name,to:receiver.name,sent,received:assertCommunityMessage(poll,post.message_id,sent).body}); + } + } + const sent=`Two Home ${transition} direct ${crypto.randomUUID()}`; + await api(a.community,'/api/apps/chat-room/direct/messages/send',{request_id:'two-home:'+crypto.randomUUID(),conversation_id:conversation.conversation_id,text:sent}); + let messages; + const arrived=await until(`direct ${transition} at b`,async()=>{ + messages=(await api(b.community,path)).messages; + return messages.find(m=>m.text===sent&&m.direction==='incoming'&&m.delivery_state==='received'); + }); + round.direct={from:a.name,to:b.name,sent,received:assertReceivedDirect(messages,arrived.message_id,sent).text}; + return round; + } + result.exchanges.start=await exchange('start'); + await runtime('restart_community'); + result.exchanges.node_restart=await exchange('node_restart'); + const fresh=await runtime('start_c'); + assertMessageOrigins([fresh.base], browserHost); + const c=await signIn(fresh.base); + result.stage='fresh_home_community'; + c.community=await c.app('chat-room'); + await joinCommunity(c); + result.exchanges.fresh_home=await exchange('fresh_home'); result.stage='offline_pending'; // Close B's projections before stopping its owned Runtime; retain only its passkey context. for(const page of b.context.pages()) if(page!==b.page) await page.close(); @@ -212,6 +253,9 @@ async function run() { assertSenderDelivery(settledAfterRestart,recoveredDetail); result.recovery={text:offlineText,recipient_state:recovered.delivery_state,sender_state:settledAfterRestart.delivery_state,sender_detail:recoveredDetail}; result.results.restart_delivery='passed'; + b.community=await b.app('chat-room'); + await joinCommunity(b); + result.exchanges.restart_b=await exchange('restart_b'); await direct.screenshot({path:join(evidence,'direct-a-desktop.png'),fullPage:true}); await direct.setViewportSize({width:390,height:844}); await direct.screenshot({path:join(evidence,'direct-a-phone.png'),fullPage:true}); diff --git a/scripts/ci-installed-two-home-test.py b/scripts/ci-installed-two-home-test.py index 662ec54db..c95eec139 100644 --- a/scripts/ci-installed-two-home-test.py +++ b/scripts/ci-installed-two-home-test.py @@ -52,17 +52,33 @@ def identity(self): return {'candidate':'c'*40, 'source_tree':'d'*40, 'installed_runtime_sha256':'a'*64, 'source_components_sha256':'b'*64, 'installation_receipt_sha256':'e'*64} + # The journey's contract, spelled out here so a changed table cannot move the test with it. + TRANSITIONS = {'start':('community','a','b'), 'node_restart':('community','a','b'), + 'fresh_home':('community','a','b','c'), 'restart_b':('community','a','b','c')} + + def transitions(self): + rows = {} + for name, homes in self.TRANSITIONS.items(): + members = [home for home in homes if home != 'community'] + rows[name] = {'homes':{home:{'alive':True, 'peer_count':1} for home in homes}, + 'community':[{'from':sender, 'to':receiver, 'sent':f'{name} post from {sender}', + 'received':f'{name} post from {sender}'} + for sender in members for receiver in members if sender != receiver], + 'direct':{'from':'a', 'to':'b', 'sent':name+' direct', 'received':name+' direct'}} + return rows + def receipt(self): identity = self.identity() return {**identity, 'node_identity':identity, 'status':'passed', 'fixture_removed':True, 'network_arrangement':'same-host', 'homes':[{'home':name, 'installed_runtime_sha256':identity['installed_runtime_sha256'], 'carrier_bind_address':f'127.0.0.1:{1025+index}'} - for index,name in enumerate(('a','b','community'))], - 'process_cleanup':[{'status':'passed', 'after':{'runtime':0}} for _ in range(5)], + for index,name in enumerate(('a','b','c','community'))], + 'process_cleanup':[{'status':'passed', 'after':{'runtime':0}} for _ in range(7)], 'browser_cross_home_http_requests':0, 'direct_endpoint_observations':[{'home':name,'ticket_addresses':[{'Ip':f'127.0.0.1:{1025+index}'}]} - for index,name in ((2,'community'),(2,'community'),(0,'a'),(1,'b'),(1,'b'))], + for index,name in ((3,'community'),(3,'community'),(0,'a'),(1,'b'),(3,'community'),(2,'c'),(1,'b'))], + 'transitions':self.transitions(), 'results':{name:'passed' for name in ('community_post','direct_received','sender_delivered', 'offline_pending','restart_delivery')}, 'negative_controls':{name:{'assertion_rejected':True, 'api_state':'pending', 'ui_detail':'Sending'} @@ -78,7 +94,8 @@ def test_acceptance_refuses_stale_identity_unchecked_failure_controls_or_survivi self.assertFalse(check(changed, self.identity()), key) for key, value in [('status','failed'), ('failure',{'phase':'late failure'}), ('browser_failure','failed'), ('fixture_removed',False), ('homes',[]), ('process_cleanup',[]), - ('browser_cross_home_http_requests',1), ('direct_endpoint_observations',[])]: + ('browser_cross_home_http_requests',1), ('direct_endpoint_observations',[]), + ('transitions',{})]: changed = copy.deepcopy(valid) changed[key] = value self.assertFalse(check(changed, self.identity()), key) @@ -115,6 +132,76 @@ def test_acceptance_binds_the_node_and_each_home_to_its_own_build(self): del changed['node_identity'] self.assertFalse(check(changed, homes)) + def test_acceptance_requires_positive_evidence_after_every_transition(self): + check = JOURNEY['two_home_receipt_valid'] + valid = self.receipt() + def refused(label, change): + changed = copy.deepcopy(valid) + change(changed['transitions']) + self.assertFalse(check(changed, self.identity()), label) + for name, homes in self.TRANSITIONS.items(): + refused(name+' is missing', lambda rows: rows.pop(name)) + for home in homes: + refused(f'{home} is missing at {name}', lambda rows: rows[name]['homes'].pop(home)) + for alive in (False, None, 'true'): + refused(f'{home} alive={alive!r} at {name}', + lambda rows: rows[name]['homes'][home].update(alive=alive)) + for count in (0, None, True, '1'): + refused(f'{home} peer_count={count!r} at {name}', + lambda rows: rows[name]['homes'][home].update(peer_count=count)) + refused(f'{home} reports no peer count at {name}', lambda rows: rows[name]['homes'][home].pop('peer_count')) + for index, post in enumerate(valid['transitions'][name]['community']): + pair = f"{post['from']} to {post['to']} at {name}" + refused(pair+' is missing', lambda rows: rows[name]['community'].pop(index)) + refused(pair+' arrived as a prefix', + lambda rows: rows[name]['community'][index].update(received=post['sent'][:-1])) + refused(pair+' has no text', lambda rows: rows[name]['community'][index].update(sent='', received='')) + if len(homes) > 3: + refused(pair+' is a second post from its sender', lambda rows: rows[name]['community'][index].update( + sent='second post', received='second post')) + refused('a post is counted twice at '+name, + lambda rows: rows[name]['community'].__setitem__(1, rows[name]['community'][0])) + refused('direct is missing at '+name, lambda rows: rows[name].pop('direct')) + refused('direct arrived as a prefix at '+name, lambda rows: rows[name]['direct'].update(received=name+' direc')) + refused('direct has no recipient at '+name, lambda rows: rows[name]['direct'].update(to='a')) + def repost(rows): + for post in rows['node_restart']['community']: + post['sent'] = post['received'] = 'start post from '+post['from'] + refused('a post from before the node restart counts after it', repost) + refused('a direct message from before the node restart counts after it', + lambda rows: rows['node_restart']['direct'].update(sent='start direct', received='start direct')) + + def test_transition_homes_record_an_exited_process_and_wait_for_a_late_peer_count(self): + observe = JOURNEY['transition_homes'] + transition = JOURNEY['Transition']('node_restart', 'restart_community', ('community', 'a')) + running, exited = mock.Mock(**{'poll.return_value':None}), mock.Mock(**{'poll.return_value':1}) + late = {'community':iter([None, 0, 2]), 'a':iter([1])} + with mock.patch.object(observe.__globals__['time'], 'sleep'): + self.assertEqual(observe(transition, {'community':running, 'a':running}, lambda name: next(late[name])), + {'community':{'alive':True, 'peer_count':2}, 'a':{'alive':True, 'peer_count':1}}) + self.assertEqual(observe(transition, {'community':exited, 'a':running}, lambda name: 1)['community'], + {'alive':False, 'peer_count':None}) + self.assertEqual(observe(transition, {'a':running}, lambda name: 1)['community'], + {'alive':False, 'peer_count':None}) + for count in (None, 0): + self.assertEqual(observe(transition, {'community':running, 'a':running}, lambda name: count, budget=0)['a'], + {'alive':True, 'peer_count':count}) + + def test_peer_count_comes_from_the_home_status_command_and_an_absent_count_stays_absent(self): + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary).resolve() + (home/'bin').mkdir() + runtime = home/'bin/elastos' + runtime.write_text('#!/bin/sh\n[ "$*" = "home --status --json" ] || exit 2\ncat "$HOME/status.json"\n') + runtime.chmod(0o700) + owner = JOURNEY['Owner'](home, home, '', '', None) + environment = {'PATH':os.environ['PATH'], 'HOME':str(home)} + for status, expected in (({'runtime':{'running':True, 'peer_count':2}}, 2), + ({'runtime':{'running':True, 'peer_count':0}}, 0), + ({'runtime':{'running':True}}, None)): + (home/'status.json').write_text(json.dumps(status)) + self.assertEqual(JOURNEY['home_peer_count'](owner, environment), expected) + def test_network_arrangement_and_each_observed_ticket_must_match_configuration(self): check = JOURNEY['two_home_receipt_valid'] valid = self.receipt() @@ -122,7 +209,7 @@ def test_network_arrangement_and_each_observed_ticket_must_match_configuration(s changed['network_arrangement'] = 'two-nat' self.assertFalse(check(changed, self.identity())) self.assertFalse(check(valid, self.identity(), 'two-nat')) - for index in range(5): + for index in range(len(valid['direct_endpoint_observations'])): for address in ({'Relay':'https://relay.invalid'}, {'Ip':'192.0.2.1:1234'}, {'Ip':'127.0.0.1:1234'}): changed = copy.deepcopy(valid) changed['direct_endpoint_observations'][index]['ticket_addresses'] = [address] From 39cf35d2ee06e1adb6180ea4180f1b634791d932 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 16:54:03 -0500 Subject: [PATCH 10/15] test(carrier): SIGTERM enters the two-Home journey clean-up Ctrl-C reached the journey's clean-up through KeyboardInterrupt, but SIGTERM ended the process and left its Homes and fixture behind. The journey now installs a SIGTERM handler that raises, so the same clean-up stops every owned process, removes the fixture and writes the failure receipt. The handler stays in place during the clean-up and the journey restores the previous handler when it returns. Co-authored-by: Cursor --- scripts/ci-installed-journeys.py | 16 ++++++++++++++++ scripts/ci-installed-two-home-test.py | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index 01e2d7429..67736d438 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -1796,6 +1796,21 @@ def expired(_signum, _frame): signal.setitimer(signal.ITIMER_REAL, max(.001, timer[0] - (time.monotonic() - started)), timer[1]) +class MessageTermination(BaseException): + """SIGTERM enters the same clean-up as Ctrl-C.""" + + +@contextmanager +def message_termination(): + def terminated(_signum, _frame): + raise MessageTermination("two-Home journey received SIGTERM") + previous = signal.signal(signal.SIGTERM, terminated) + try: + yield + finally: + signal.signal(signal.SIGTERM, previous) + + def remember_message_processes(child, known, marker=None): rows = {pid: row for pid, row in process_rows().items() if not row["command"].endswith(" ")} @@ -2001,6 +2016,7 @@ def two_home_receipt_valid(row, identity, network_arrangement="same-host", node_ and transition_evidence_valid(row)) +@message_termination() def two_home_messages(home_build, evidence, network="same-host", fixture_root=None, node_build=None): """Installed Community and Direct journey. Topology belongs to setup, not assertions. diff --git a/scripts/ci-installed-two-home-test.py b/scripts/ci-installed-two-home-test.py index c95eec139..458bd3273 100644 --- a/scripts/ci-installed-two-home-test.py +++ b/scripts/ci-installed-two-home-test.py @@ -268,6 +268,33 @@ def disk(_): self.assertEqual(len(receipt['cleanup_failures']), 2 if disk_fails else 1) self.assertNotIn('disposable test key', json.dumps(receipt)) + def test_sigterm_during_the_journey_reaches_its_cleanup_and_restores_the_handler(self): + with tempfile.TemporaryDirectory() as temporary: + home, root, evidence = (Path(temporary).resolve()/name for name in ('home', 'fixture', 'evidence')) + (home/'data').mkdir(parents=True) + build = JOURNEY['InstalledBuild']('home', home, home/'data', home, self.identity()) + def terminated_while_staging(*_arguments, **_options): + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(1) + raise RuntimeError('SIGTERM did not interrupt the journey') + ignored = [] + operator = lambda *_: ignored.append(True) + previous = signal.signal(signal.SIGTERM, operator) + try: + with mock.patch.dict(JOURNEY['finish_message_fixture'].__globals__, { + 'fresh_fixture':terminated_while_staging, 'require_disk_space':lambda *_: {}}): + with self.assertRaises(JOURNEY['MessageTermination']): + JOURNEY['two_home_messages'](build, evidence, fixture_root=root) + self.assertIs(signal.getsignal(signal.SIGTERM), operator) + finally: + signal.signal(signal.SIGTERM, previous) + self.assertFalse(ignored) + self.assertFalse(root.exists()) + receipt = json.loads((evidence/'installed-journeys.json').read_text()) + self.assertTrue(receipt['fixture_removed']) + self.assertEqual(receipt['failure']['phase'], 'two_home_messages') + self.assertFalse(JOURNEY['two_home_receipt_valid'](receipt, self.identity())) + def test_work_deadline_interrupts_a_hung_phase_and_restores_the_timer(self): before = signal.getsignal(signal.SIGALRM) with self.assertRaises(JOURNEY['MessageDeadline']): From d70964c2fc9c90e5f7ddb4bb21e29ca605ce502a Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 16:57:58 -0500 Subject: [PATCH 11/15] test(carrier): drop a pair-count guard that no receipt can reach Acceptance compares the set of ordered Community pairs with the set the transition requires, and it checks the exact text of every recorded row. A repeated row therefore either leaves a pair out or passes the same checks twice, so the separate row count refused nothing on its own. Co-authored-by: Cursor --- scripts/ci-installed-journeys.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index 67736d438..f321e70be 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -1974,8 +1974,7 @@ def transition_evidence_valid(row): posts, direct = evidence.get("community", []), evidence.get("direct", {}) pairs = {(post.get("from"), post.get("to")) for post in posts} members = [name for name in transition.homes if name != NODE] - if (len(posts) != len(pairs) or pairs != set(itertools.permutations(members, 2)) - or (direct.get("from"), direct.get("to")) not in pairs): + if pairs != set(itertools.permutations(members, 2)) or (direct.get("from"), direct.get("to")) not in pairs: return False if not all(isinstance(item.get("sent"), str) and item["sent"] and item.get("received") == item["sent"] for item in (*posts, direct)): From 8b8643397cc054b7b5773fd1d1f528bc40d1c7f3 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 16:58:15 -0500 Subject: [PATCH 12/15] test(carrier): drive the mixed-version gate through the installed journey The driver staged its own Homes, read Runtime log lines and passed when no dead-link line appeared. An independent review refused that as proof. The driver now takes a released and a candidate installed source Home, and for each of the four node and Home pairings it runs the installed two-Home journey, which proves every transition with live processes, peer counts and exact message texts. One receipt names each arrangement, both build identities and the whole journey receipt. An arrangement passes only when the journey returns and its receipt passes acceptance for those two builds. A failed clean-up, a deadline, a leftover fixture or a missing arrangement fails the run, and SIGTERM stops it with a failed receipt. The wedge mode is removed. SIGSTOP also stops the QUIC acknowledgements of the stopped Runtime, so its connection times out and the mode did not hold the condition it named. The gossip actor wedge test covers a peer that keeps healthy transport and withholds stream credit. Co-authored-by: Cursor --- scripts/ci-installed-two-home-test.py | 109 ++++++- scripts/gossip-mixed-version-journey.py | 365 ++++-------------------- 2 files changed, 161 insertions(+), 313 deletions(-) diff --git a/scripts/ci-installed-two-home-test.py b/scripts/ci-installed-two-home-test.py index 458bd3273..f17d0037e 100644 --- a/scripts/ci-installed-two-home-test.py +++ b/scripts/ci-installed-two-home-test.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """Receipt guards for the installed two-Home journey; no Runtime or browser mocks.""" +import contextlib import copy +import io import json from pathlib import Path import runpy @@ -67,12 +69,13 @@ def transitions(self): 'direct':{'from':'a', 'to':'b', 'sent':name+' direct', 'received':name+' direct'}} return rows - def receipt(self): - identity = self.identity() - return {**identity, 'node_identity':identity, 'status':'passed', 'fixture_removed':True, + def receipt(self, identity=None, node=None): + identity = identity or self.identity() + node = node or identity + return {**identity, 'node_identity':node, 'status':'passed', 'fixture_removed':True, 'network_arrangement':'same-host', - 'homes':[{'home':name, 'installed_runtime_sha256':identity['installed_runtime_sha256'], - 'carrier_bind_address':f'127.0.0.1:{1025+index}'} + 'homes':[{'home':name, 'carrier_bind_address':f'127.0.0.1:{1025+index}', + 'installed_runtime_sha256':(node if name == 'community' else identity)['installed_runtime_sha256']} for index,name in enumerate(('a','b','c','community'))], 'process_cleanup':[{'status':'passed', 'after':{'runtime':0}} for _ in range(7)], 'browser_cross_home_http_requests':0, @@ -94,6 +97,7 @@ def test_acceptance_refuses_stale_identity_unchecked_failure_controls_or_survivi self.assertFalse(check(changed, self.identity()), key) for key, value in [('status','failed'), ('failure',{'phase':'late failure'}), ('browser_failure','failed'), ('fixture_removed',False), ('homes',[]), ('process_cleanup',[]), + ('process_cleanup',valid['process_cleanup'][:5]), ('browser_cross_home_http_requests',1), ('direct_endpoint_observations',[]), ('transitions',{})]: changed = copy.deepcopy(valid) @@ -117,9 +121,7 @@ def test_acceptance_refuses_stale_identity_unchecked_failure_controls_or_survivi def test_acceptance_binds_the_node_and_each_home_to_its_own_build(self): check = JOURNEY['two_home_receipt_valid'] homes, node = self.identity(), {**self.identity(), 'candidate':'f'*40, 'installed_runtime_sha256':'9'*64} - mixed = self.receipt() - mixed['node_identity'] = node - next(home for home in mixed['homes'] if home['home'] == 'community')['installed_runtime_sha256'] = '9'*64 + mixed = self.receipt(homes, node) self.assertTrue(check(mixed, homes, node_identity=node)) self.assertFalse(check(mixed, homes), 'a same-version acceptance refuses another node build') self.assertFalse(check(self.receipt(), homes, node_identity=node), 'the named node build did not run') @@ -159,7 +161,7 @@ def refused(label, change): if len(homes) > 3: refused(pair+' is a second post from its sender', lambda rows: rows[name]['community'][index].update( sent='second post', received='second post')) - refused('a post is counted twice at '+name, + refused('one post stands in for another at '+name, lambda rows: rows[name]['community'].__setitem__(1, rows[name]['community'][0])) refused('direct is missing at '+name, lambda rows: rows[name].pop('direct')) refused('direct arrived as a prefix at '+name, lambda rows: rows[name]['direct'].update(received=name+' direc')) @@ -202,6 +204,95 @@ def test_peer_count_comes_from_the_home_status_command_and_an_absent_count_stays (home/'status.json').write_text(json.dumps(status)) self.assertEqual(JOURNEY['home_peer_count'](owner, environment), expected) + def drive(self, outcome): + """Run the mixed-version driver over a stand-in journey that saves a valid receipt unless ``outcome`` interferes.""" + driver = runpy.run_path(str(ROOT/'scripts/gossip-mixed-version-journey.py')) + journey = driver['JOURNEY'] + identities = {'released':{**self.identity(), 'candidate':'1'*40, 'installed_runtime_sha256':'2'*64}, + 'candidate':self.identity()} + calls = [] + def stand_in(homes, evidence, fixture_root, node_build): + calls.append((node_build.name, homes.name, evidence.name, fixture_root.name)) + evidence.mkdir() + receipt = self.receipt(homes.identity, node_build.identity) + error = outcome(len(calls), receipt, fixture_root, journey) + (evidence/'installed-journeys.json').write_text(json.dumps(receipt)) + if error: + raise error + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + arguments = [argument for build in identities for part in ('home', 'data', 'repo') + for argument in (f'--{build}-{part}', str(root/build/part))] + raised = None + with mock.patch.dict(journey, {'two_home_messages':stand_in, 'installed_build':lambda name, *paths: + journey['InstalledBuild'](name, *paths, identities[name])}), contextlib.redirect_stdout(io.StringIO()): + try: + driver['main']([*arguments, '--evidence', str(root/'evidence'), '--fixture-root', str(root/'fixtures')]) + except BaseException as error: + raised = error + return raised, json.loads((root/'evidence/mixed-version-journey.json').read_text()), calls + + def test_mixed_version_driver_runs_every_pairing_into_its_own_evidence_and_fixture(self): + raised, receipt, calls = self.drive(lambda *_: None) + self.assertEqual(raised.code, 0) + names = ['node-released-homes-released', 'node-released-homes-candidate', + 'node-candidate-homes-released', 'node-candidate-homes-candidate'] + self.assertEqual(calls, [('released','released',names[0],names[0]), ('released','candidate',names[1],names[1]), + ('candidate','released',names[2],names[2]), ('candidate','candidate',names[3],names[3])]) + self.assertEqual((receipt['status'], receipt['fixture_root_removed']), ('passed', True)) + self.assertEqual([row['arrangement'] for row in receipt['arrangements']], names) + for row, (node, homes, *_) in zip(receipt['arrangements'], calls): + self.assertEqual((row['status'], row['node']['build'], row['homes']['build']), ('passed', node, homes)) + self.assertEqual(row['node']['installed_runtime_sha256'], ('2' if node == 'released' else 'a')*64) + self.assertEqual(row['homes']['installed_runtime_sha256'], ('2' if homes == 'released' else 'a')*64) + self.assertEqual(row['journey']['transitions'], self.transitions()) + + def test_mixed_version_driver_fails_on_a_failed_cleanup_an_unproven_receipt_or_a_leftover_fixture(self): + def deadline(index, receipt, _root, journey): + if index == 1: + receipt.update(failure={'phase':'two_home_messages'}) + return journey['MessageDeadline']('two-Home phase deadline') + def failed_cleanup(index, receipt, _root, _journey): + if index == 2: + receipt.update(status='failed', cleanup_failures=[{'phase':'stop_a'}]) + return RuntimeError('two-Home cleanup failed; see installed journey receipt') + def no_peer(index, receipt, _root, _journey): + if index == 3: + receipt['transitions']['node_restart']['homes']['a']['peer_count'] = 0 + def raised_after_its_receipt(index, _receipt, _root, _journey): + if index == 4: + return RuntimeError('late failure') + def leftover(index, _receipt, root, _journey): + if index == 4: + (root/'a').mkdir(parents=True) + for outcome, failed, removed in ((deadline, ['node-released-homes-released'], True), + (failed_cleanup, ['node-released-homes-candidate'], True), + (no_peer, ['node-candidate-homes-released'], True), + (raised_after_its_receipt, ['node-candidate-homes-candidate'], True), + (leftover, [], False)): + with self.subTest(outcome=outcome.__name__): + raised, receipt, calls = self.drive(outcome) + self.assertEqual(raised.code, 1) + self.assertEqual(len(calls), 4, 'the other arrangements stay observable') + self.assertEqual((receipt['status'], receipt['fixture_root_removed']), ('failed', removed)) + self.assertEqual([row['arrangement'] for row in receipt['arrangements'] if row['status'] != 'passed'], failed) + + def test_mixed_version_driver_stops_on_sigterm_and_keeps_a_failed_receipt(self): + def terminated(index, _receipt, _root, _journey): + if index == 2: + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(1) + ignored = [] + operator = lambda *_: ignored.append(True) + previous = signal.signal(signal.SIGTERM, operator) + try: + raised, receipt, calls = self.drive(terminated) + self.assertIs(signal.getsignal(signal.SIGTERM), operator) + finally: + signal.signal(signal.SIGTERM, previous) + self.assertEqual((type(raised).__name__, ignored, len(calls)), ('MessageTermination', [], 2)) + self.assertEqual((receipt['status'], len(receipt['arrangements'])), ('failed', 1)) + def test_network_arrangement_and_each_observed_ticket_must_match_configuration(self): check = JOURNEY['two_home_receipt_valid'] valid = self.receipt() diff --git a/scripts/gossip-mixed-version-journey.py b/scripts/gossip-mixed-version-journey.py index 27ece506f..16e8e68e2 100644 --- a/scripts/gossip-mixed-version-journey.py +++ b/scripts/gossip-mixed-version-journey.py @@ -1,321 +1,78 @@ #!/usr/bin/env python3 -"""Community link across Runtime versions: a node and Homes built from different trees. +"""Community link across Runtime versions: a node and Homes installed from different trees. Homes and the Community node update at different times, so a release must -hold for every pairing. Each arrangement starts one Community node and two -Homes from the named builds, joins them to one Community, then restarts the -node and brings up a fresh Home against it. - -The fixture needs only the Runtime and the localhost provider, so it runs on a -machine without the media tools that the full source Home installs. +hold for every pairing. Each arrangement runs the installed two-Home journey +with one build as the Community node and one build as the Homes. That journey +owns the transitions, their evidence and the clean-up; this driver chooses the +builds and keeps one receipt for the four arrangements. """ import argparse -import hashlib import json -import os from pathlib import Path import runpy -import shutil -import signal -import socket -import subprocess -import sys -import time - -ROOT = Path(__file__).resolve().parent.parent -JOURNEY = runpy.run_path(str(ROOT / "scripts/ci-installed-journeys.py")) -free_port = JOURNEY["free_port"] -wait_healthy = JOURNEY["wait_healthy"] -cleanup_runtime = JOURNEY["cleanup_runtime"] -digest = JOURNEY["digest"] - -STAGED_PROVIDERS = ("localhost-provider",) -JOIN_LINE = "carrier: gossip_join" -DEAD_LINE = "Community link looks dead" -REBUILD_LINE = "rebuilding Carrier gossip" -#: A joined Home repairs after one silence window, so a pass must outlast it. -DEFAULT_SETTLE_SECONDS = 90 - - -def platform_key(): - machine = {"x86_64": "amd64", "aarch64": "arm64"}.get(os.uname().machine) - if sys.platform != "linux" or machine is None: - raise RuntimeError("the mixed-version journey runs on Linux amd64 or arm64") - return f"linux-{machine}" - - -class Build: - """One Runtime tree: its source checkout and its own release target dir.""" - - def __init__(self, name, source, target): - self.name = name - self.source = Path(source).resolve() - self.target = Path(target).resolve() - self.runtime = self.target / "release/elastos" - self.manifest = self.source / "components.json" - for path in (self.runtime, self.manifest): - if not path.exists(): - raise RuntimeError(f"build {name} has no {path.name}: build it first") - self.runtime_sha256 = digest(self.runtime) - - def provider(self, provider): - return self.target / "release" / provider - - -def stage_home(build, root, carrier_address): - """One disposable data root that the Runtime's own checksum check accepts.""" - data = root / ".local/share/elastos" - (data / "bin").mkdir(mode=0o700, parents=True) - # The shared process census owns a Runtime by its data-root path, so each - # Home runs its own copy of the binary from its own bin directory. - runtime = data / "bin/elastos" - shutil.copy2(build.runtime, runtime) - runtime.chmod(0o700) - manifest = json.loads(build.manifest.read_text()) - for provider in STAGED_PROVIDERS: - staged = data / "bin" / provider - shutil.copy2(build.provider(provider), staged) - staged.chmod(0o700) - blob = staged.read_bytes() - entry = manifest["external"][provider]["platforms"][platform_key()] - entry["install_path"] = f"bin/{provider}" - entry["checksum"] = "sha256:" + hashlib.sha256(blob).hexdigest() - entry["size"] = len(blob) - (data / "components.json").write_text(json.dumps(manifest, indent=2)) - (data / "config.toml").write_text( - f'dev_mode = true\ntrusted_keys = []\ncarrier_bind_addr = "{carrier_address}"\n' - ) - return data - - -class Home: - """One Runtime process with its own Home, data root and log.""" - - def __init__(self, name, build, root, bind_host): - self.name = name - self.build = build - self.root = root - self.log = root / "runtime.private.log" - self.carrier_address = f"{bind_host}:{free_port(socket.SOCK_DGRAM)}" - self.api_address = f"{bind_host}:{free_port()}" - self.data = stage_home(build, root, self.carrier_address) - self.runtime = self.data / "bin/elastos" - self.child = None - def environment(self): - return { - "PATH": os.environ["PATH"], - "HOME": str(self.root), - "XDG_DATA_HOME": str(self.data.parent), - "ELASTOS_QUIET_RUNTIME_NOTICES": "1", - } +JOURNEY = runpy.run_path(str(Path(__file__).with_name("ci-installed-journeys.py"))) +BUILDS = ("released", "candidate") +ARRANGEMENTS = [(node, homes) for node in BUILDS for homes in BUILDS] - def start(self): - assert self.child is None, "Home already has a Runtime" - with self.log.open("ab") as log: - self.child = subprocess.Popen( - [str(self.runtime), "gateway", "--addr", self.api_address], - env=self.environment(), - stdout=log, - stderr=log, - start_new_session=True, - ) - wait_healthy(self.child, self.api_address, f"mixed-version {self.name}") - def stop(self): - if self.child is None: - return {"status": "passed", "after": {}} - row = cleanup_runtime(self.child, self.data) - self.child = None - return row - - def configure(self, *arguments): - with (self.root / "configuration.private.log").open("ab") as log: - subprocess.run( - [str(self.runtime), "collaboration-config", *map(str, arguments)], - env=self.environment(), - stdout=log, - stderr=log, - check=True, - timeout=120, - ) - - def lines(self, needle): - if not self.log.exists(): - return [] - text = self.log.read_text(errors="replace") - return [line for line in text.splitlines() if needle in line] - - -def observe(home, started): - """What an operator reads off one Home after the settle window.""" - return { - "home": home.name, - "build": home.build.name, - "joined": bool(home.lines(JOIN_LINE)), - "link_dead": len(home.lines(DEAD_LINE)), - "gossip_rebuilds": len(home.lines(REBUILD_LINE)), - "seconds": round(time.monotonic() - started), - } - - -def run_arrangement(node_build, home_build, root, bind_host, settle, wedge=False): - """One node version against one Home version, with a fresh join after restart. - - With ``wedge`` the arrangement instead stops one joined Home from reading - its gossip stream while its connection stays up, and then asks a fresh - Home to join. That is the field symptom this release gate is about, so the - arrangement reports what it saw rather than a verdict. - """ - root.mkdir(mode=0o700, parents=True, exist_ok=False) - homes, record = {}, { - "node_build": node_build.name, - "home_build": home_build.name, - "shape": "wedge" if wedge else "restart", - "node_runtime_sha256": node_build.runtime_sha256, - "home_runtime_sha256": home_build.runtime_sha256, - "observations": [], - "process_cleanup": [], - } +def run_arrangement(node, homes, evidence, fixture_root): + """One node build against one Home build; the journey's own receipt decides the verdict.""" + row = {"node": {"build": node.name, **node.identity}, "homes": {"build": homes.name, **homes.identity}} try: - for name, build in (("community", node_build), ("a", home_build), ("b", home_build)): - homes[name] = Home(name, build, root / name, bind_host) - node = homes["community"] - node.start() - node.configure("create-authority-key", "--key", root / "authority.key") - node.configure( - "export-local-bootstrap-receipt", - "--data-root", node.data, - "--runtime-kind", "gateway", - "--output", root / "bootstrap.json", - ) - node.configure( - "generate-initial", - "--authority-key", root / "authority.key", - "--network-id", "ci.gossip-mixed", - "--conversation-id", "community", - "--bootstrap-peer", root / "bootstrap.json", - "--output", root / "network.json", - ) - node.configure("verify", "--input", root / "network.json") - record["process_cleanup"].append(node.stop()) - - def admit(home): - target = home.data / "collaboration-network-v1.json" - shutil.copy2(root / "network.json", target) - target.chmod(0o600) - - for home in homes.values(): - admit(home) - started = time.monotonic() - for home in homes.values(): - home.start() - time.sleep(settle) - record["observations"] += [observe(homes[name], started) for name in ("a", "b")] - - if wedge: - # SIGSTOP leaves the connection up and the peer unable to read, so - # the node keeps a neighbour that never takes another frame. - wedged = homes["b"].child.pid - os.kill(wedged, signal.SIGSTOP) - record["wedged_pid_stopped"] = True - else: - # The node restarts, as an updated node does; the Homes keep running. - record["process_cleanup"].append(homes["community"].stop()) - homes["community"].start() - fresh = Home("c", home_build, root / "c", bind_host) - homes["c"] = fresh - admit(fresh) - fresh_started = time.monotonic() - fresh.start() - time.sleep(settle) - record["observations"].append(observe(fresh, fresh_started)) - record["observations"] += [observe(homes[name], started) for name in ("a",)] - if wedge: - os.kill(wedged, signal.SIGCONT) - record["status"] = "observed" - else: - record["observations"] += [observe(homes["b"], started)] - record["status"] = ( - "passed" - if all(row["joined"] and row["link_dead"] == 0 for row in record["observations"]) - else "failed" - ) - except Exception as error: - record["status"] = "failed" - record["failure"] = f"{type(error).__name__}: {error}"[:300] + JOURNEY["two_home_messages"](homes, evidence, fixture_root=fixture_root, node_build=node) + # A deadline ends one arrangement; SIGTERM and Ctrl-C end the run. + except (Exception, JOURNEY["MessageDeadline"]) as error: + row["failure"] = JOURNEY["observer_failure"]( + "two_home_messages", error, [str(fixture_root), str(node.home), str(homes.home)]) + receipt = evidence / "installed-journeys.json" + row["journey"] = json.loads(receipt.read_text()) if receipt.exists() else {} + row["status"] = "passed" if "failure" not in row and JOURNEY["two_home_receipt_valid"]( + row["journey"], homes.identity, node_identity=node.identity) else "failed" + return row + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + for build in BUILDS: + parser.add_argument(f"--{build}-home", type=Path, required=True, + help=f"isolated HOME of the installed {build} source Home") + parser.add_argument(f"--{build}-data", type=Path, required=True, help="its elastos data root") + parser.add_argument(f"--{build}-repo", type=Path, required=True, help="the checkout it was installed from") + parser.add_argument("--evidence", type=Path, required=True, help="fresh directory for receipts and screenshots") + parser.add_argument("--fixture-root", type=Path, required=True, help="fresh disposable fixture path") + arguments = parser.parse_args(argv) + builds = {build: JOURNEY["installed_build"](build, *(getattr(arguments, f"{build}_{part}") + for part in ("home", "data", "repo"))) for build in BUILDS} + evidence, fixture_root = arguments.evidence.resolve(), arguments.fixture_root.resolve() + evidence.mkdir(parents=True, exist_ok=False) + fixture_root.mkdir(mode=0o700, parents=True, exist_ok=False) + receipt = {"arrangements": []} + try: + with JOURNEY["message_termination"](): + for node, homes in ARRANGEMENTS: + name = f"node-{node}-homes-{homes}" + receipt["arrangements"].append({"arrangement": name, **run_arrangement( + builds[node], builds[homes], evidence / name, fixture_root / name)}) finally: - for home in homes.values(): - try: - record["process_cleanup"].append(home.stop()) - except Exception as error: # a stuck cleanup still leaves a result - record["process_cleanup"].append( - {"status": "failed", "error": f"{type(error).__name__}: {error}"[:200]} - ) - if record.get("status") != "passed": - # The fixture tree goes away, so a result keeps its own last words. - record["log_tails"] = { - home.name: [ - line for line in home.lines("") - if any(needle in line for needle in (JOIN_LINE, DEAD_LINE, REBUILD_LINE)) - ][-6:] or home.lines("")[-6:] - for home in homes.values() - } - shutil.rmtree(root, ignore_errors=True) - return record - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--released-source", type=Path, required=True) - parser.add_argument("--released-target", type=Path, required=True) - parser.add_argument("--develop-source", type=Path, required=True) - parser.add_argument("--develop-target", type=Path, required=True) - parser.add_argument("--fixture-root", type=Path, required=True) - parser.add_argument("--bind-host", default="127.0.0.1") - parser.add_argument("--settle-seconds", type=int, default=DEFAULT_SETTLE_SECONDS) - parser.add_argument("--receipt", type=Path) - parser.add_argument("--wedge", action="store_true", - help="also probe a Home that stops reading its gossip stream") - arguments = parser.parse_args() - builds = { - "released": Build("released", arguments.released_source, arguments.released_target), - "develop": Build("develop", arguments.develop_source, arguments.develop_target), - } - receipt = {"platform": platform_key(), "arrangements": []} - shapes = [(node, home, False) for node in ("released", "develop") - for home in ("released", "develop")] - if arguments.wedge: - shapes += [(node, node, True) for node in ("released", "develop")] - for node, home, wedge in shapes: - name = f"node-{node}-homes-{home}" + ("-wedge" if wedge else "") - receipt["arrangements"].append( - run_arrangement( - builds[node], builds[home], arguments.fixture_root / name, - arguments.bind_host, arguments.settle_seconds, wedge, - ) - ) - receipt["status"] = ( - "passed" - if all(row["status"] in ("passed", "observed") for row in receipt["arrangements"]) - else "failed" - ) - if arguments.receipt: - arguments.receipt.write_text(json.dumps(receipt, indent=2) + "\n") + # rmdir refuses a directory that still holds a fixture, so a leftover fails the run. + try: + fixture_root.rmdir() + receipt["fixture_root_removed"] = True + except OSError: + receipt["fixture_root_removed"] = False + receipt["status"] = "passed" if ( + receipt["fixture_root_removed"] and len(receipt["arrangements"]) == len(ARRANGEMENTS) + and all(row["status"] == "passed" for row in receipt["arrangements"])) else "failed" + (evidence / "mixed-version-journey.json").write_text(json.dumps(receipt, indent=2) + "\n") for row in receipt["arrangements"]: - print(f"node={row['node_build']:<9} homes={row['home_build']:<9} " - f"{row['shape']:<7} {row['status']}" - + (f" {row['failure']}" if "failure" in row else "")) - for name, tail in row.get("log_tails", {}).items(): - for line in tail: - print(f" [{name}] {line[:200]}") - for observation in row["observations"]: - print( - f" {observation['home']} joined={observation['joined']} " - f"link_dead={observation['link_dead']} " - f"rebuilds={observation['gossip_rebuilds']} " - f"after={observation['seconds']}s" - ) + failure = row.get("failure") or row["journey"].get("failure") or {} + print(f"{row['arrangement']}: {row['status']}" + (f" {failure['message']}" if failure else "")) + for name, transition in row["journey"].get("transitions", {}).items(): + peers = ", ".join(f"{home}={state['peer_count']}" for home, state in transition.get("homes", {}).items()) + print(f" {name}: peers {peers}; Community posts read {len(transition.get('community', []))}; " + f"direct {'read' if 'direct' in transition else 'missing'}") print("mixed-version status:", receipt["status"]) raise SystemExit(0 if receipt["status"] == "passed" else 1) From d2bcc717366ff1fa57abe961a83c05876f87e20f Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 17:11:48 -0500 Subject: [PATCH 13/15] ci: gate merges on the mixed-version Community journey Homes and the Community node update at different times, so a release must hold for every pairing. A Linux job installs the head and the released tag into two lean source Homes and runs the four arrangements against them. The tag is a workflow input with the current release as its default. Co-authored-by: Cursor --- .github/workflows/ci.yml | 119 ++++++++++++++++++++++++ scripts/ci-mixed-version-source-home.sh | 34 +++++++ 2 files changed, 153 insertions(+) create mode 100755 scripts/ci-mixed-version-source-home.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97dffd87f..ae747cc9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,10 @@ on: description: "Git ref to check out and test" required: false default: "" + mixed_version_tag: + description: "Released tag the mixed-version Community gate builds beside the head" + required: false + default: "" push: # develop pushes keep the shared build caches warm for every PR into develop. branches: [main, develop] @@ -29,6 +33,8 @@ env: # Reviewed ARM64 archive pin, shared by the producer and every CI consumer. CI_LLAMA_AMD64_SHA256: 0dee60d855f864aa9e8a7200014baf9419e3a4c9ccb3f3609532aad0ed34f2c0 CI_LLAMA_ARM64_SHA256: a92e404708feae91cbede43322e445971450de8a204f4ff51502a7f42f5df8b8 + # The released Runtime the mixed-version Community gate pairs the head with. + CI_MIXED_VERSION_TAG: ${{ inputs.mixed_version_tag || 'v2.0.1' }} # An explicit dispatch ref may select a tag independently of the workflow ref. CI_USE_CACHE: ${{ github.ref_type != 'tag' && !(github.event_name == 'workflow_dispatch' && inputs.ref != '') }} # Only develop pushes write shared caches (Rust, apt, Kubo, engine). PR, @@ -1047,6 +1053,119 @@ jobs: check_name: ubuntu-24.04-arm steps: *source_home_linux_steps + # Homes and the Community node update at different times, so a release must + # hold for every pairing of the head and the last released Runtime. The + # journey needs no engine and no model package, so this job installs two lean + # source Homes of its own instead of sharing the engine-bound jobs above. + # It gates merges and releases; a pull-request push runs it on request. + gossip-mixed-version-linux: + name: gossip-mixed-version-linux + runs-on: ubuntu-24.04 + # Two source-home builds have a 100-minute budget and the arrangements 45. + timeout-minutes: 160 + needs: source-gate + if: >- + github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' + || (github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref_type == 'tag')) + env: + CI_APT_PACKAGES: coturn e2fsprogs ffmpeg musl-tools nasm pkg-config + steps: + # Published tarballs keep the shipped Cargo release profile. + - name: use CI journey release profile + if: github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/develop') + run: | + echo "CARGO_PROFILE_RELEASE_LTO=false" >> "$GITHUB_ENV" + echo "CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16" >> "$GITHUB_ENV" + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} + fetch-depth: 0 + fetch-tags: true + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '26' + - uses: dtolnay/rust-toolchain@72504271e02fb719eb4eb3eaece3a8e3d06f57c0 # 1.91.0 + with: + targets: wasm32-unknown-unknown + # Each build keeps its own intermediates and is removed before the next. + - name: free hosted runner disk + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL + df -Pk . + - name: prepare apt archive cache permissions + run: | + sudo install -d -o "$(id -u)" -g "$(id -g)" -m 755 /var/cache/apt/archives + sudo chown -R "$(id -u):$(id -g)" /var/cache/apt/archives + - name: install host prerequisites + timeout-minutes: 13 + run: bash scripts/ci-apt-prerequisites.sh + - name: reviewed media tools + run: | + set -euo pipefail + MEDIA_TOOLS_DIR="$RUNNER_TEMP/mixed-version-media-tools" + install -d -m 700 "$MEDIA_TOOLS_DIR" + for tool in ffmpeg ffprobe; do + install -m 700 "$(command -v "$tool")" "$MEDIA_TOOLS_DIR/$tool" + done + { + echo "SETUP_SOURCE_HOME_MEDIA_TOOLS_DIR=$MEDIA_TOOLS_DIR" + echo "ELASTOS_NODE_BIN=$(command -v node)" + echo "ELASTOS_BROWSER_VM_TURNSERVER_BIN=$(command -v turnserver)" + } >> "$GITHUB_ENV" + - name: check out the released tag beside the head + run: | + set -euo pipefail + git worktree add --detach "$RUNNER_TEMP/released-tree" "refs/tags/$CI_MIXED_VERSION_TAG" + git -C "$RUNNER_TEMP/released-tree" rev-parse HEAD + - name: install the candidate source Home + timeout-minutes: 50 + env: + ELASTOS_COLLABORATION_STARTUP_MODE: isolated + run: bash scripts/ci-mixed-version-source-home.sh "$GITHUB_WORKSPACE" "$RUNNER_TEMP/candidate-home" + - name: install the released source Home + timeout-minutes: 50 + env: + ELASTOS_COLLABORATION_STARTUP_MODE: isolated + run: bash scripts/ci-mixed-version-source-home.sh "$RUNNER_TEMP/released-tree" "$RUNNER_TEMP/released-home" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '24' + # Playwright 1.59.1's installer hangs extracting Chromium under Node 26. + - name: prepare installed Home browser + timeout-minutes: 20 + run: | + npm ci --ignore-scripts --prefix elastos/tools/browser-playwright-engine + export PLAYWRIGHT_BROWSERS_PATH="$RUNNER_TEMP/installed-home-browser" + npx --prefix elastos/tools/browser-playwright-engine playwright install chromium --with-deps --only-shell + echo "PLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH" >> "$GITHUB_ENV" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '26' + - name: mixed-version Community and direct messages + timeout-minutes: 45 + run: | + set -euo pipefail + python3 scripts/gossip-mixed-version-journey.py \ + --candidate-home "$RUNNER_TEMP/candidate-home" \ + --candidate-data "$RUNNER_TEMP/candidate-home/.local/share/elastos" \ + --candidate-repo "$GITHUB_WORKSPACE" \ + --released-home "$RUNNER_TEMP/released-home" \ + --released-data "$RUNNER_TEMP/released-home/.local/share/elastos" \ + --released-repo "$RUNNER_TEMP/released-tree" \ + --evidence "$RUNNER_TEMP/mixed-version-journeys" \ + --fixture-root "$RUNNER_TEMP/mixed-version-fixture" \ + | tee -a "$GITHUB_STEP_SUMMARY" + - name: upload mixed-version journey + if: always() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: gossip-mixed-version-linux + path: | + ${{ runner.temp }}/mixed-version-journeys/**/*.json + ${{ runner.temp }}/mixed-version-journeys/**/*.png + retention-days: 7 + if-no-files-found: error + source-home-macos: name: source-home-macos runs-on: macos-15 diff --git a/scripts/ci-mixed-version-source-home.sh b/scripts/ci-mixed-version-source-home.sh new file mode 100755 index 000000000..3e90bf0a8 --- /dev/null +++ b/scripts/ci-mixed-version-source-home.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Install one checkout into its own isolated Home for the mixed-version gate. +# +# The gate pairs two Runtime versions, so one runner installs two source Homes. +# Each build keeps private intermediates and releases them before the next +# build starts; a hosted runner holds one set of release intermediates, not two. +set -euo pipefail +umask 022 + +SOURCE="$(cd "${1:?Set the checkout to install from}" && pwd)" +TARGET="${2:?Set the isolated HOME to install into}" +NAME="$(basename "$TARGET")" +SCRATCH="${RUNNER_TEMP:?Set RUNNER_TEMP}/${NAME}-cargo" + +test ! -e "$TARGET" +test ! -e "$SCRATCH" +install -d -m 700 "$TARGET" "$TARGET/.local/share" "$SCRATCH/target" "$SCRATCH/build" + +: "${SETUP_SOURCE_HOME_MEDIA_TOOLS_DIR:?Set the reviewed media tools directory}" +: "${ELASTOS_NODE_BIN:?Set the Node executable the Browser component uses}" +: "${ELASTOS_BROWSER_VM_TURNSERVER_BIN:?Set the TURN executable}" + +# Cargo and rustup stay in the runner's own home; only the product installs here. +cd "$SOURCE" +env HOME="$TARGET" XDG_DATA_HOME="$TARGET/.local/share" \ + CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}" \ + CARGO_TARGET_DIR="$SCRATCH/target" CARGO_BUILD_BUILD_DIR="$SCRATCH/build" \ + PATH="${CARGO_HOME:-$HOME/.cargo}/bin:$PATH" \ + SETUP_SOURCE_HOME_INSTALL_LLAMA_SERVER=0 SETUP_SOURCE_HOME_INSTALL_KUBO=0 \ + ELASTOS_QUIET_RUNTIME_NOTICES=1 \ + scripts/setup-source-home.sh + +rm -rf "$SCRATCH" +df -Pk . From 7f1f50290fcba8172944d5ef3cbc63c59ce645d2 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 17:38:34 -0500 Subject: [PATCH 14/15] test(carrier): read the Carrier Provider, not a count the gateway withholds gather_runtime_status returns early for a gateway Runtime and leaves runtime.peer_count unset, so every Home in this journey reported no count. The gateway control surface exposes one peer route on purpose, so each Home now proves its own Carrier Provider answers get_ticket through its own attach secret. The fresh Community and direct traffic after each transition carries the proof that the link works. Co-authored-by: Cursor --- scripts/ci-installed-journeys.py | 57 +++++++++----------- scripts/ci-installed-two-home-test.py | 72 +++++++++++++------------ scripts/gossip-mixed-version-journey.py | 5 +- 3 files changed, 68 insertions(+), 66 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index f321e70be..eac78fa22 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -1892,6 +1892,7 @@ def remove_and_measure(): NODE = "community" # the bootstrap role; every other role is a member Home +TICKET_MAX_LENGTH = 16384 MESSAGE_NETWORKS = { "same-host": { "environment": {"ELASTOS_CARRIER_NETWORK": "direct", "ELASTOS_CARRIER_MDNS": "0"}, @@ -1937,38 +1938,37 @@ def message_endpoints_valid(homes, observations, arrangement): return counts == settings["launches"] -def home_peer_count(owner, environment): - """Carrier peers as the Home's own status command reports them; None when it reports no count.""" - status = subprocess.run([str(owner.data / "bin/elastos"), "home", "--status", "--json"], env=environment, - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True, timeout=30).stdout - return json.loads(status)["runtime"].get("peer_count") +def carrier_ticket(owner): + """The Home's own Carrier Provider ticket, read through that Home's own + attach secret. The secret and the token stay inside this call.""" + coordinates = json.loads(bounded_text(owner.data / "gateway-runtime-coords.json")) + api = coordinates["api_url"].rstrip("/") + token = post_json(api + "/api/auth/attach", {"secret": coordinates["attach_secret"], "scope": "shell"})["token"] + ticket = post_json(api + "/api/provider/peer/get_ticket", {}, token)["data"]["ticket"] + assert 0 < len(ticket) <= TICKET_MAX_LENGTH + return ticket -def transition_homes(transition, children, peer_count, budget=60): - """Liveness comes from each owned process itself. A peer count recovers - some time after a restart, so it is polled until every Home has one.""" - end = time.monotonic() + budget - counts = dict.fromkeys(transition.homes) - while True: - alive = {name: name in children and children[name].poll() is None for name in transition.homes} - for name in transition.homes: - if alive[name] and not counts[name]: - counts[name] = peer_count(name) - if not all(alive.values()) or all(counts.values()) or time.monotonic() >= end: - return {name: {"alive": alive[name], "peer_count": counts[name]} for name in transition.homes} - time.sleep(1) +def transition_homes(transition, children, observe): + """Liveness comes from each owned process itself. The Carrier observation + comes from the Home's own Provider, which answers once the Home is healthy, + so only a live Home is asked and no poll is needed.""" + alive = {name: name in children and children[name].poll() is None for name in transition.homes} + return {name: {"alive": alive[name], "carrier_ticket_length": observe(name) if alive[name] else None} + for name in transition.homes} def transition_evidence_valid(row): - """After every transition each Home is alive with a Carrier peer, every - member read every other member's fresh post, and one direct message - arrived, each with its exact text.""" + """After every transition each Home runs and its own Carrier Provider + answers, every member read every other member's fresh post, and one direct + message arrived, each with its exact text.""" fresh = [] for transition in MESSAGE_TRANSITIONS: evidence = row.get("transitions", {}).get(transition.name, {}) homes = evidence.get("homes", {}) if set(homes) != set(transition.homes) or not all( - home.get("alive") is True and type(home.get("peer_count")) is int and home["peer_count"] >= 1 + home.get("alive") is True and type(home.get("carrier_ticket_length")) is int + and 0 < home["carrier_ticket_length"] <= TICKET_MAX_LENGTH for home in homes.values()): return False posts, direct = evidence.get("community", []), evidence.get("direct", {}) @@ -2052,11 +2052,7 @@ def launch(name): env=environment(owner), stdout=log, stderr=log, start_new_session=True) children[name] = child wait_healthy(child, owner.api, "two-Home " + name) - coordinates = json.loads(bounded_text(owner.data / "gateway-runtime-coords.json")) - api = coordinates["api_url"].rstrip("/") - token = post_json(api + "/api/auth/attach", {"secret": coordinates["attach_secret"], "scope": "shell"})["token"] - ticket = post_json(api + "/api/provider/peer/get_ticket", {}, token)["data"]["ticket"] - assert len(ticket) <= 16384 + ticket = carrier_ticket(owner) decoded = json.loads(base64.b32decode(ticket.upper() + "=" * (-len(ticket) % 8))) endpoints = decoded["endpoints"] assert len(endpoints) == 1, "fixture ticket must describe one installed Home" @@ -2093,13 +2089,12 @@ def admit(name): def base(name): return f"http://{settings['browser_host']}:" + fixtures[name].api.rsplit(":", 1)[1] def prove(transition): - homes = transition_homes(transition, children, - lambda name: home_peer_count(fixtures[name], environment(fixtures[name]))) + homes = transition_homes(transition, children, lambda name: len(carrier_ticket(fixtures[name]))) record["transitions"].setdefault(transition.name, {})["homes"] = homes for name, home in homes.items(): assert home["alive"], f"two-Home {name} is not running after {transition.name}" - assert home["peer_count"] is not None, f"two-Home {name} status reports no runtime.peer_count" - assert home["peer_count"] >= 1, f"two-Home {name} has no Carrier peer after {transition.name}" + assert home["carrier_ticket_length"], \ + f"two-Home {name} Carrier Provider answered no ticket after {transition.name}" def require_running(): for name, child in children.items(): assert child.poll() is None, f"two-Home {name} exited during the journey" diff --git a/scripts/ci-installed-two-home-test.py b/scripts/ci-installed-two-home-test.py index f17d0037e..26aaa47d2 100644 --- a/scripts/ci-installed-two-home-test.py +++ b/scripts/ci-installed-two-home-test.py @@ -62,7 +62,7 @@ def transitions(self): rows = {} for name, homes in self.TRANSITIONS.items(): members = [home for home in homes if home != 'community'] - rows[name] = {'homes':{home:{'alive':True, 'peer_count':1} for home in homes}, + rows[name] = {'homes':{home:{'alive':True, 'carrier_ticket_length':120} for home in homes}, 'community':[{'from':sender, 'to':receiver, 'sent':f'{name} post from {sender}', 'received':f'{name} post from {sender}'} for sender in members for receiver in members if sender != receiver], @@ -148,10 +148,11 @@ def refused(label, change): for alive in (False, None, 'true'): refused(f'{home} alive={alive!r} at {name}', lambda rows: rows[name]['homes'][home].update(alive=alive)) - for count in (0, None, True, '1'): - refused(f'{home} peer_count={count!r} at {name}', - lambda rows: rows[name]['homes'][home].update(peer_count=count)) - refused(f'{home} reports no peer count at {name}', lambda rows: rows[name]['homes'][home].pop('peer_count')) + for length in (0, None, True, '120', 16385): + refused(f'{home} carrier_ticket_length={length!r} at {name}', + lambda rows: rows[name]['homes'][home].update(carrier_ticket_length=length)) + refused(f'{home} reports no Carrier observation at {name}', + lambda rows: rows[name]['homes'][home].pop('carrier_ticket_length')) for index, post in enumerate(valid['transitions'][name]['community']): pair = f"{post['from']} to {post['to']} at {name}" refused(pair+' is missing', lambda rows: rows[name]['community'].pop(index)) @@ -173,36 +174,41 @@ def repost(rows): refused('a direct message from before the node restart counts after it', lambda rows: rows['node_restart']['direct'].update(sent='start direct', received='start direct')) - def test_transition_homes_record_an_exited_process_and_wait_for_a_late_peer_count(self): + def test_transition_homes_observe_only_a_live_home_and_record_an_exited_one(self): observe = JOURNEY['transition_homes'] transition = JOURNEY['Transition']('node_restart', 'restart_community', ('community', 'a')) running, exited = mock.Mock(**{'poll.return_value':None}), mock.Mock(**{'poll.return_value':1}) - late = {'community':iter([None, 0, 2]), 'a':iter([1])} - with mock.patch.object(observe.__globals__['time'], 'sleep'): - self.assertEqual(observe(transition, {'community':running, 'a':running}, lambda name: next(late[name])), - {'community':{'alive':True, 'peer_count':2}, 'a':{'alive':True, 'peer_count':1}}) - self.assertEqual(observe(transition, {'community':exited, 'a':running}, lambda name: 1)['community'], - {'alive':False, 'peer_count':None}) - self.assertEqual(observe(transition, {'a':running}, lambda name: 1)['community'], - {'alive':False, 'peer_count':None}) - for count in (None, 0): - self.assertEqual(observe(transition, {'community':running, 'a':running}, lambda name: count, budget=0)['a'], - {'alive':True, 'peer_count':count}) + self.assertEqual(observe(transition, {'community':running, 'a':running}, lambda name: len(name)), + {'community':{'alive':True, 'carrier_ticket_length':9}, + 'a':{'alive':True, 'carrier_ticket_length':1}}) + def refuse(name): + if name == 'community': + raise AssertionError('an exited Home was asked for a ticket') + return 1 + self.assertEqual(observe(transition, {'community':exited, 'a':running}, refuse)['community'], + {'alive':False, 'carrier_ticket_length':None}) + self.assertEqual(observe(transition, {'a':running}, refuse)['community'], + {'alive':False, 'carrier_ticket_length':None}) - def test_peer_count_comes_from_the_home_status_command_and_an_absent_count_stays_absent(self): + def test_the_carrier_observation_uses_the_homes_own_secret_at_shell_scope_and_bounds_the_ticket(self): with tempfile.TemporaryDirectory() as temporary: - home = Path(temporary).resolve() - (home/'bin').mkdir() - runtime = home/'bin/elastos' - runtime.write_text('#!/bin/sh\n[ "$*" = "home --status --json" ] || exit 2\ncat "$HOME/status.json"\n') - runtime.chmod(0o700) - owner = JOURNEY['Owner'](home, home, '', '', None) - environment = {'PATH':os.environ['PATH'], 'HOME':str(home)} - for status, expected in (({'runtime':{'running':True, 'peer_count':2}}, 2), - ({'runtime':{'running':True, 'peer_count':0}}, 0), - ({'runtime':{'running':True}}, None)): - (home/'status.json').write_text(json.dumps(status)) - self.assertEqual(JOURNEY['home_peer_count'](owner, environment), expected) + data = Path(temporary).resolve() + (data/'gateway-runtime-coords.json').write_text(json.dumps( + {'api_url':'http://127.0.0.1:1/', 'attach_secret':'fixture-secret'})) + owner = JOURNEY['Owner'](data, data, '', '', None) + posts = [] + def post_json(url, body, token=None): + posts.append((url, body, token)) + return {'token':'fixture-token'} if url.endswith('/attach') else {'data':{'ticket':ticket}} + with mock.patch.dict(JOURNEY['carrier_ticket'].__globals__, {'post_json':post_json}): + ticket = 'T'*120 + self.assertEqual(JOURNEY['carrier_ticket'](owner), ticket) + self.assertEqual(posts, [('http://127.0.0.1:1/api/auth/attach', + {'secret':'fixture-secret', 'scope':'shell'}, None), + ('http://127.0.0.1:1/api/provider/peer/get_ticket', {}, 'fixture-token')]) + for ticket in ('', 'T'*(JOURNEY['TICKET_MAX_LENGTH']+1)): + with self.assertRaises(AssertionError): + JOURNEY['carrier_ticket'](owner) def drive(self, outcome): """Run the mixed-version driver over a stand-in journey that saves a valid receipt unless ``outcome`` interferes.""" @@ -256,9 +262,9 @@ def failed_cleanup(index, receipt, _root, _journey): if index == 2: receipt.update(status='failed', cleanup_failures=[{'phase':'stop_a'}]) return RuntimeError('two-Home cleanup failed; see installed journey receipt') - def no_peer(index, receipt, _root, _journey): + def no_carrier(index, receipt, _root, _journey): if index == 3: - receipt['transitions']['node_restart']['homes']['a']['peer_count'] = 0 + receipt['transitions']['node_restart']['homes']['a']['carrier_ticket_length'] = 0 def raised_after_its_receipt(index, _receipt, _root, _journey): if index == 4: return RuntimeError('late failure') @@ -267,7 +273,7 @@ def leftover(index, _receipt, root, _journey): (root/'a').mkdir(parents=True) for outcome, failed, removed in ((deadline, ['node-released-homes-released'], True), (failed_cleanup, ['node-released-homes-candidate'], True), - (no_peer, ['node-candidate-homes-released'], True), + (no_carrier, ['node-candidate-homes-released'], True), (raised_after_its_receipt, ['node-candidate-homes-candidate'], True), (leftover, [], False)): with self.subTest(outcome=outcome.__name__): diff --git a/scripts/gossip-mixed-version-journey.py b/scripts/gossip-mixed-version-journey.py index 16e8e68e2..05ee7c706 100644 --- a/scripts/gossip-mixed-version-journey.py +++ b/scripts/gossip-mixed-version-journey.py @@ -70,8 +70,9 @@ def main(argv=None): failure = row.get("failure") or row["journey"].get("failure") or {} print(f"{row['arrangement']}: {row['status']}" + (f" {failure['message']}" if failure else "")) for name, transition in row["journey"].get("transitions", {}).items(): - peers = ", ".join(f"{home}={state['peer_count']}" for home, state in transition.get("homes", {}).items()) - print(f" {name}: peers {peers}; Community posts read {len(transition.get('community', []))}; " + homes = ", ".join(f"{home}={'carrier' if state['carrier_ticket_length'] else 'silent'}" + for home, state in transition.get("homes", {}).items()) + print(f" {name}: {homes}; Community posts read {len(transition.get('community', []))}; " f"direct {'read' if 'direct' in transition else 'missing'}") print("mixed-version status:", receipt["status"]) raise SystemExit(0 if receipt["status"] == "passed" else 1) From 3438813a5db25a86ef800e3e4b6cac241a10362d Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Sat, 14 Nov 2026 17:38:38 -0500 Subject: [PATCH 15/15] ci: run the mixed-version gate inside the installed source-home job A separate job needs its own source Home, and the release policy keeps one source Home per job. Three gated steps in source-home-linux build the released tag beside the candidate over the same build directories and run the four arrangements, so the second build reuses what the first one compiled. Co-authored-by: Cursor --- .github/workflows/ci.yml | 149 ++++++------------------ scripts/ci-mixed-version-source-home.sh | 27 +++-- 2 files changed, 48 insertions(+), 128 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae747cc9b..c85389be2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -745,7 +745,8 @@ jobs: source-home-linux: name: source-home-linux (${{ matrix.os }}) runs-on: ${{ matrix.os }} - timeout-minutes: 120 + # The mixed-version steps add a second Runtime build and four arrangements. + timeout-minutes: 180 needs: [source-gate, engine-llama-amd64] env: CI_APT_PACKAGES: coturn e2fsprogs ffmpeg musl-tools nasm pkg-config @@ -958,6 +959,39 @@ jobs: env: CI_SOURCE_HOME: ${{ runner.temp }}/elastos-source-home run: scripts/ci-installed-journeys.sh two-home-messages + # Homes and the Community node update at different times, so a release + # must hold for every pairing of this candidate and the last released + # Runtime. Merges and releases pay for the second build; a pull-request + # push runs the same-version journey above and skips these three steps. + - name: check out the released tag beside the candidate + id: released-tag + if: matrix.os == 'ubuntu-24.04' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref_type == 'tag'))) + run: | + set -euo pipefail + git fetch --no-tags --depth 1 origin "refs/tags/$CI_MIXED_VERSION_TAG:refs/tags/$CI_MIXED_VERSION_TAG" + git worktree add --detach "$RUNNER_TEMP/released-tree" "refs/tags/$CI_MIXED_VERSION_TAG" + git -C "$RUNNER_TEMP/released-tree" rev-parse HEAD + - name: install the released Runtime beside the candidate + if: steps.released-tag.outcome == 'success' + timeout-minutes: 45 + env: + ELASTOS_COLLABORATION_STARTUP_MODE: isolated + run: bash scripts/ci-mixed-version-source-home.sh "$RUNNER_TEMP/released-tree" "$RUNNER_TEMP/elastos-released-home" + - name: mixed-version Community and direct messages + if: steps.released-tag.outcome == 'success' + timeout-minutes: 45 + run: | + set -euo pipefail + python3 scripts/gossip-mixed-version-journey.py \ + --candidate-home "$RUNNER_TEMP/elastos-source-home" \ + --candidate-data "$RUNNER_TEMP/elastos-source-home/.local/share/elastos" \ + --candidate-repo "$GITHUB_WORKSPACE" \ + --released-home "$RUNNER_TEMP/elastos-released-home" \ + --released-data "$RUNNER_TEMP/elastos-released-home/.local/share/elastos" \ + --released-repo "$RUNNER_TEMP/released-tree" \ + --evidence "$RUNNER_TEMP/source-home-journeys/mixed-version" \ + --fixture-root "$RUNNER_TEMP/mixed-version-fixture" \ + | tee -a "$GITHUB_STEP_SUMMARY" - name: summarize installed model journey if: always() env: @@ -1053,119 +1087,6 @@ jobs: check_name: ubuntu-24.04-arm steps: *source_home_linux_steps - # Homes and the Community node update at different times, so a release must - # hold for every pairing of the head and the last released Runtime. The - # journey needs no engine and no model package, so this job installs two lean - # source Homes of its own instead of sharing the engine-bound jobs above. - # It gates merges and releases; a pull-request push runs it on request. - gossip-mixed-version-linux: - name: gossip-mixed-version-linux - runs-on: ubuntu-24.04 - # Two source-home builds have a 100-minute budget and the arrangements 45. - timeout-minutes: 160 - needs: source-gate - if: >- - github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' - || (github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref_type == 'tag')) - env: - CI_APT_PACKAGES: coturn e2fsprogs ffmpeg musl-tools nasm pkg-config - steps: - # Published tarballs keep the shipped Cargo release profile. - - name: use CI journey release profile - if: github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/develop') - run: | - echo "CARGO_PROFILE_RELEASE_LTO=false" >> "$GITHUB_ENV" - echo "CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16" >> "$GITHUB_ENV" - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - with: - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} - fetch-depth: 0 - fetch-tags: true - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: '26' - - uses: dtolnay/rust-toolchain@72504271e02fb719eb4eb3eaece3a8e3d06f57c0 # 1.91.0 - with: - targets: wasm32-unknown-unknown - # Each build keeps its own intermediates and is removed before the next. - - name: free hosted runner disk - run: | - sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL - df -Pk . - - name: prepare apt archive cache permissions - run: | - sudo install -d -o "$(id -u)" -g "$(id -g)" -m 755 /var/cache/apt/archives - sudo chown -R "$(id -u):$(id -g)" /var/cache/apt/archives - - name: install host prerequisites - timeout-minutes: 13 - run: bash scripts/ci-apt-prerequisites.sh - - name: reviewed media tools - run: | - set -euo pipefail - MEDIA_TOOLS_DIR="$RUNNER_TEMP/mixed-version-media-tools" - install -d -m 700 "$MEDIA_TOOLS_DIR" - for tool in ffmpeg ffprobe; do - install -m 700 "$(command -v "$tool")" "$MEDIA_TOOLS_DIR/$tool" - done - { - echo "SETUP_SOURCE_HOME_MEDIA_TOOLS_DIR=$MEDIA_TOOLS_DIR" - echo "ELASTOS_NODE_BIN=$(command -v node)" - echo "ELASTOS_BROWSER_VM_TURNSERVER_BIN=$(command -v turnserver)" - } >> "$GITHUB_ENV" - - name: check out the released tag beside the head - run: | - set -euo pipefail - git worktree add --detach "$RUNNER_TEMP/released-tree" "refs/tags/$CI_MIXED_VERSION_TAG" - git -C "$RUNNER_TEMP/released-tree" rev-parse HEAD - - name: install the candidate source Home - timeout-minutes: 50 - env: - ELASTOS_COLLABORATION_STARTUP_MODE: isolated - run: bash scripts/ci-mixed-version-source-home.sh "$GITHUB_WORKSPACE" "$RUNNER_TEMP/candidate-home" - - name: install the released source Home - timeout-minutes: 50 - env: - ELASTOS_COLLABORATION_STARTUP_MODE: isolated - run: bash scripts/ci-mixed-version-source-home.sh "$RUNNER_TEMP/released-tree" "$RUNNER_TEMP/released-home" - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: '24' - # Playwright 1.59.1's installer hangs extracting Chromium under Node 26. - - name: prepare installed Home browser - timeout-minutes: 20 - run: | - npm ci --ignore-scripts --prefix elastos/tools/browser-playwright-engine - export PLAYWRIGHT_BROWSERS_PATH="$RUNNER_TEMP/installed-home-browser" - npx --prefix elastos/tools/browser-playwright-engine playwright install chromium --with-deps --only-shell - echo "PLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH" >> "$GITHUB_ENV" - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: '26' - - name: mixed-version Community and direct messages - timeout-minutes: 45 - run: | - set -euo pipefail - python3 scripts/gossip-mixed-version-journey.py \ - --candidate-home "$RUNNER_TEMP/candidate-home" \ - --candidate-data "$RUNNER_TEMP/candidate-home/.local/share/elastos" \ - --candidate-repo "$GITHUB_WORKSPACE" \ - --released-home "$RUNNER_TEMP/released-home" \ - --released-data "$RUNNER_TEMP/released-home/.local/share/elastos" \ - --released-repo "$RUNNER_TEMP/released-tree" \ - --evidence "$RUNNER_TEMP/mixed-version-journeys" \ - --fixture-root "$RUNNER_TEMP/mixed-version-fixture" \ - | tee -a "$GITHUB_STEP_SUMMARY" - - name: upload mixed-version journey - if: always() - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 - with: - name: gossip-mixed-version-linux - path: | - ${{ runner.temp }}/mixed-version-journeys/**/*.json - ${{ runner.temp }}/mixed-version-journeys/**/*.png - retention-days: 7 - if-no-files-found: error - source-home-macos: name: source-home-macos runs-on: macos-15 diff --git a/scripts/ci-mixed-version-source-home.sh b/scripts/ci-mixed-version-source-home.sh index 3e90bf0a8..ef5ea5d20 100755 --- a/scripts/ci-mixed-version-source-home.sh +++ b/scripts/ci-mixed-version-source-home.sh @@ -1,34 +1,33 @@ #!/usr/bin/env bash -# Install one checkout into its own isolated Home for the mixed-version gate. +# Install a second checkout into its own isolated Home, beside the candidate. # -# The gate pairs two Runtime versions, so one runner installs two source Homes. -# Each build keeps private intermediates and releases them before the next -# build starts; a hosted runner holds one set of release intermediates, not two. +# The mixed-version Community gate pairs two Runtime versions, so one job +# installs two source Homes. Both share the job's build directories, so the +# second build reuses every dependency the first one compiled. set -euo pipefail umask 022 SOURCE="$(cd "${1:?Set the checkout to install from}" && pwd)" TARGET="${2:?Set the isolated HOME to install into}" -NAME="$(basename "$TARGET")" -SCRATCH="${RUNNER_TEMP:?Set RUNNER_TEMP}/${NAME}-cargo" +MEDIA_TOOLS_DIR="${TARGET}-media-tools" test ! -e "$TARGET" -test ! -e "$SCRATCH" -install -d -m 700 "$TARGET" "$TARGET/.local/share" "$SCRATCH/target" "$SCRATCH/build" +install -d -m 700 "$TARGET" "$TARGET/.local/share" "$MEDIA_TOOLS_DIR" +for tool in ffmpeg ffprobe; do + install -m 700 "$(command -v "$tool")" "$MEDIA_TOOLS_DIR/$tool" +done -: "${SETUP_SOURCE_HOME_MEDIA_TOOLS_DIR:?Set the reviewed media tools directory}" -: "${ELASTOS_NODE_BIN:?Set the Node executable the Browser component uses}" -: "${ELASTOS_BROWSER_VM_TURNSERVER_BIN:?Set the TURN executable}" +NODE_BIN="$(command -v node)" +TURNSERVER_BIN="$(command -v turnserver)" # Cargo and rustup stay in the runner's own home; only the product installs here. cd "$SOURCE" env HOME="$TARGET" XDG_DATA_HOME="$TARGET/.local/share" \ CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}" \ - CARGO_TARGET_DIR="$SCRATCH/target" CARGO_BUILD_BUILD_DIR="$SCRATCH/build" \ PATH="${CARGO_HOME:-$HOME/.cargo}/bin:$PATH" \ + SETUP_SOURCE_HOME_MEDIA_TOOLS_DIR="$MEDIA_TOOLS_DIR" \ + ELASTOS_NODE_BIN="$NODE_BIN" ELASTOS_BROWSER_VM_TURNSERVER_BIN="$TURNSERVER_BIN" \ SETUP_SOURCE_HOME_INSTALL_LLAMA_SERVER=0 SETUP_SOURCE_HOME_INSTALL_KUBO=0 \ ELASTOS_QUIET_RUNTIME_NOTICES=1 \ scripts/setup-source-home.sh - -rm -rf "$SCRATCH" df -Pk .