diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97dffd87f..c85389be2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,10 @@ on: description: "Git ref to check out and test" required: false default: "" + mixed_version_tag: + description: "Released tag the mixed-version Community gate builds beside the head" + required: false + default: "" push: # develop pushes keep the shared build caches warm for every PR into develop. branches: [main, develop] @@ -29,6 +33,8 @@ env: # Reviewed ARM64 archive pin, shared by the producer and every CI consumer. CI_LLAMA_AMD64_SHA256: 0dee60d855f864aa9e8a7200014baf9419e3a4c9ccb3f3609532aad0ed34f2c0 CI_LLAMA_ARM64_SHA256: a92e404708feae91cbede43322e445971450de8a204f4ff51502a7f42f5df8b8 + # The released Runtime the mixed-version Community gate pairs the head with. + CI_MIXED_VERSION_TAG: ${{ inputs.mixed_version_tag || 'v2.0.1' }} # An explicit dispatch ref may select a tag independently of the workflow ref. CI_USE_CACHE: ${{ github.ref_type != 'tag' && !(github.event_name == 'workflow_dispatch' && inputs.ref != '') }} # Only develop pushes write shared caches (Rust, apt, Kubo, engine). PR, @@ -739,7 +745,8 @@ jobs: source-home-linux: name: source-home-linux (${{ matrix.os }}) runs-on: ${{ matrix.os }} - timeout-minutes: 120 + # The mixed-version steps add a second Runtime build and four arrangements. + timeout-minutes: 180 needs: [source-gate, engine-llama-amd64] env: CI_APT_PACKAGES: coturn e2fsprogs ffmpeg musl-tools nasm pkg-config @@ -952,6 +959,39 @@ jobs: env: CI_SOURCE_HOME: ${{ runner.temp }}/elastos-source-home run: scripts/ci-installed-journeys.sh two-home-messages + # Homes and the Community node update at different times, so a release + # must hold for every pairing of this candidate and the last released + # Runtime. Merges and releases pay for the second build; a pull-request + # push runs the same-version journey above and skips these three steps. + - name: check out the released tag beside the candidate + id: released-tag + if: matrix.os == 'ubuntu-24.04' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref_type == 'tag'))) + run: | + set -euo pipefail + git fetch --no-tags --depth 1 origin "refs/tags/$CI_MIXED_VERSION_TAG:refs/tags/$CI_MIXED_VERSION_TAG" + git worktree add --detach "$RUNNER_TEMP/released-tree" "refs/tags/$CI_MIXED_VERSION_TAG" + git -C "$RUNNER_TEMP/released-tree" rev-parse HEAD + - name: install the released Runtime beside the candidate + if: steps.released-tag.outcome == 'success' + timeout-minutes: 45 + env: + ELASTOS_COLLABORATION_STARTUP_MODE: isolated + run: bash scripts/ci-mixed-version-source-home.sh "$RUNNER_TEMP/released-tree" "$RUNNER_TEMP/elastos-released-home" + - name: mixed-version Community and direct messages + if: steps.released-tag.outcome == 'success' + timeout-minutes: 45 + run: | + set -euo pipefail + python3 scripts/gossip-mixed-version-journey.py \ + --candidate-home "$RUNNER_TEMP/elastos-source-home" \ + --candidate-data "$RUNNER_TEMP/elastos-source-home/.local/share/elastos" \ + --candidate-repo "$GITHUB_WORKSPACE" \ + --released-home "$RUNNER_TEMP/elastos-released-home" \ + --released-data "$RUNNER_TEMP/elastos-released-home/.local/share/elastos" \ + --released-repo "$RUNNER_TEMP/released-tree" \ + --evidence "$RUNNER_TEMP/source-home-journeys/mixed-version" \ + --fixture-root "$RUNNER_TEMP/mixed-version-fixture" \ + | tee -a "$GITHUB_STEP_SUMMARY" - name: summarize installed model journey if: always() env: diff --git a/elastos/Cargo.lock b/elastos/Cargo.lock index 4ad21f942..15ca8e10a 100644 --- a/elastos/Cargo.lock +++ b/elastos/Cargo.lock @@ -1664,6 +1664,7 @@ dependencies = [ "axum", "axum-server", "base64", + "bytes", "cid", "clap", "custody-provider", @@ -1697,6 +1698,7 @@ dependencies = [ "k256", "libc", "memchr", + "postcard", "qrcodegen", "rand 0.10.2", "rand 0.8.7", diff --git a/elastos/crates/elastos-server/Cargo.toml b/elastos/crates/elastos-server/Cargo.toml index 8cdf8f234..7f1f2c369 100644 --- a/elastos/crates/elastos-server/Cargo.toml +++ b/elastos/crates/elastos-server/Cargo.toml @@ -125,6 +125,10 @@ x-wing = { version = "=0.1.0", features = ["hazmat", "zeroize"] } # The rand generation iroh-gossip's membership state machine takes; the # Community hub test drives that state machine directly. gossip-rand = { package = "rand", version = "0.10", default-features = false, features = ["std_rng"] } +# The gossip wire encoding, for the neighbour that the actor-wedge test keeps +# silent after the membership handshake. +bytes = "1.7" +postcard = { version = "1", default-features = false, features = ["alloc", "use-std"] } [features] default = [] diff --git a/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs b/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs new file mode 100644 index 000000000..b29e03e19 --- /dev/null +++ b/elastos/crates/elastos-server/tests/gossip_actor_wedge.rs @@ -0,0 +1,486 @@ +//! One neighbour that stops reading must not stop the gossip actor. +//! +//! iroh-gossip 0.101.0 dispatches `OutEvent::SendMessage` with +//! `active_send_tx.send(message).await` on a bounded per-peer channel +//! (`SEND_QUEUE_CAP = 64`, net.rs:686). A neighbour whose QUIC stream credit +//! runs out stalls that channel, and a bounded-mpsc `send().await` never +//! returns `Full`, so the single actor loop stops serving every topic. The +//! Community node then keeps its healthy QUIC connections and still admits no +//! new Home. iroh-gossip 0.102.0 replaces the channel with a byte-bounded +//! queue that drops or aborts instead of awaiting. +//! +//! The two assertions are the two symptoms a Community node shows: traffic to +//! the other neighbours keeps flowing, and a fresh join still completes. They +//! run only after the flood completed or stalled with more broadcasts accepted +//! than that stream credit and that channel can hold, which is the load that +//! parks the 0.101.0 actor. + +use std::convert::Infallible; +use std::fmt; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{anyhow, bail, ensure, Context, Result}; +use bytes::Bytes; +use futures_lite::StreamExt; +use gossip_rand::rngs::StdRng; +use gossip_rand::SeedableRng; +use iroh::address_lookup::memory::MemoryLookup; +use iroh::endpoint::{presets, Connection, RecvStream, SendStream}; +use iroh::protocol::Router; +use iroh::{Endpoint, EndpointAddr, EndpointId, RelayMode, SecretKey, TransportAddr}; +use iroh_gossip::api::{Event as ApiEvent, GossipReceiver, GossipSender}; +use iroh_gossip::net::Gossip; +use iroh_gossip::proto::{ + Command, Config as ProtoConfig, Event as ProtoEvent, InEvent, Message as WireMessage, OutEvent, + PeerData, State as ProtoState, TopicId, +}; +use tokio::task::{JoinError, JoinHandle, JoinSet}; +use tokio::time::timeout; + +/// Just under the 4096-byte default `max_message_size`, leaving header room. +const PAYLOAD_SIZE: usize = 3800; +/// The per-peer channel that iroh-gossip 0.101.0 awaits on, in messages. +const SEND_QUEUE_CAP: usize = 64; +/// The default QUIC `stream_receive_window` in bytes, which is the credit a +/// neighbour grants on one stream before it reads. +const STREAM_RECEIVE_WINDOW: usize = 1_250_000; +/// The broadcasts that the neighbour's stream credit and the hub's send queue +/// absorb together. A flood that ends at or below this count may never have +/// met a full queue, so it says nothing about the hub. +const PRESSURE_FLOOR: usize = SEND_QUEUE_CAP + STREAM_RECEIVE_WINDOW / PAYLOAD_SIZE; +const FLOOD_MESSAGES: usize = 500; + +const PROBE_BUDGET: Duration = Duration::from_secs(15); +const JOIN_BUDGET: Duration = Duration::from_secs(30); +/// iroh-gossip 0.102.0 holds each broadcast for up to 100 ms while the only +/// eager neighbour of its topic is busy (`BROADCAST_WAIT_LIMIT`), so a healthy +/// flood of `FLOOD_MESSAGES` can take up to 50 s. +const FLOOD_BUDGET: Duration = Duration::from_secs(60); +/// Ten of those holds, so a flood that the hub only slows still moves in every +/// sample. +const STALL_SAMPLE: Duration = Duration::from_secs(1); +const QUIET_SAMPLES_IN_A_STALL: u32 = 2; +/// Binding and closing the endpoints, which have no budget of their own. One +/// close takes up to 3 s here. +const UNBUDGETED_MARGIN: Duration = Duration::from_secs(30); +const TEST_DEADLINE: Duration = Duration::from_secs( + 5 * JOIN_BUDGET.as_secs() + + 2 * PROBE_BUDGET.as_secs() + + FLOOD_BUDGET.as_secs() + + UNBUDGETED_MARGIN.as_secs(), +); + +#[tokio::test(flavor = "multi_thread")] +async fn a_neighbour_that_stops_reading_leaves_the_actor_serving_other_topics() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init() + .ok(); + let report = timeout(TEST_DEADLINE, serve_others_beside_a_silent_neighbour()) + .await + .map_err(|_| anyhow!("the test did not finish within {TEST_DEADLINE:?}"))??; + println!("{report}"); + Ok(()) +} + +async fn serve_others_beside_a_silent_neighbour() -> Result { + let flood_topic = TopicId::from_bytes([0xAB; 32]); + let probe_topic = TopicId::from_bytes([0xCD; 32]); + let lookup = MemoryLookup::new(); + + let silent_endpoint = Endpoint::builder(presets::Minimal) + .secret_key(SecretKey::generate()) + .relay_mode(RelayMode::Disabled) + .alpns(vec![iroh_gossip::ALPN.to_vec()]) + .bind() + .await + .context("bind the silent neighbour")?; + let silent_id = silent_endpoint.id(); + lookup.add_endpoint_info(loopback_addr(&silent_endpoint)); + let (went_silent, mut is_silent) = tokio::sync::mpsc::channel(1); + let mut silent_task = tokio::spawn(run_silent_neighbour( + silent_endpoint.clone(), + flood_topic, + went_silent, + )); + + let (hub_endpoint, hub, _hub_router) = gossip_node(&lookup).await?; + let (member_endpoint, member, _member_router) = gossip_node(&lookup).await?; + let hub_id = hub_endpoint.id(); + + let observation = async { + let member_join = tokio::spawn({ + let member = member.clone(); + async move { member.subscribe_and_join(probe_topic, vec![hub_id]).await } + }); + let hub_probe = timeout(JOIN_BUDGET, hub.subscribe_and_join(probe_topic, vec![])) + .await + .context("the hub did not join the probe topic")??; + let member_probe = timeout(JOIN_BUDGET, member_join) + .await + .context("the member did not join the probe topic")???; + let (probe_sender, _hub_probe_receiver) = hub_probe.split(); + let (_member_sender, mut member_receiver) = member_probe.split(); + + deliver(&probe_sender, &mut member_receiver, b"before the flood") + .await + .context("the probe path failed before the flood, so the fixture is wrong")?; + + let (flood_sender, mut flood_receiver) = timeout( + JOIN_BUDGET, + hub.subscribe_and_join(flood_topic, vec![silent_id]), + ) + .await + .context("the hub did not take the silent neighbour into its active view")?? + .split(); + // Without this the neighbour is still reading, and the flood proves nothing. + timeout(JOIN_BUDGET, is_silent.recv()) + .await + .ok() + .flatten() + .context("the neighbour never reached the active view, so the fixture is wrong")?; + tokio::spawn(async move { while flood_receiver.next().await.is_some() {} }); + let flood = flood_until_settled(flood_sender).await?; + + let probe_started = Instant::now(); + let delivery = deliver(&probe_sender, &mut member_receiver, b"during the flood").await; + let probe_seconds = probe_started.elapsed().as_secs_f32(); + + let join_started = Instant::now(); + let (newcomer_endpoint, newcomer, _newcomer_router) = gossip_node(&lookup).await?; + let fresh_join = timeout( + JOIN_BUDGET, + newcomer.subscribe_and_join(probe_topic, vec![hub_id]), + ) + .await + .map_err(|_| anyhow!("a fresh join did not complete within {JOIN_BUDGET:?}")) + .and_then(|joined| joined.map_err(anyhow::Error::from)); + let join_seconds = join_started.elapsed().as_secs_f32(); + newcomer_endpoint.close().await; + + let report = format!( + "{flood}; delivery to the other neighbour took {probe_seconds:.1} s; \ + the fresh join took {join_seconds:.1} s" + ); + anyhow::Ok((flood, delivery, fresh_join, report)) + }; + // The neighbour ends only when it fails. A neighbour that lets go of its + // streams frees a wedged hub, and then both assertions pass. + let (flood, delivery, fresh_join, report) = tokio::select! { + biased; + ended = &mut silent_task => match ended + .context("the silent neighbour panicked")? + .context("the silent neighbour failed")? {}, + observed = observation => observed?, + }; + + // A hub that the neighbour frees resumes a parked flood, so stop the flood + // first. + let flood_stopped = flood.stop().await; + silent_task.abort(); + silent_endpoint.close().await; + member_endpoint.close().await; + hub_endpoint.close().await; + + flood_stopped?; + delivery.with_context(|| format!("delivery to the other neighbour stopped: {report}"))?; + fresh_join.with_context(|| format!("a fresh join stopped: {report}"))?; + Ok(report) +} + +/// A flood that put the hub under pressure. The hub is judged in either state. +enum SettledFlood { + /// The hub accepted every broadcast. + Completed, + /// The hub stopped accepting broadcasts above `PRESSURE_FLOOR`, and the + /// flood stays parked on the next one. + Stalled { + accepted: usize, + parked: JoinHandle>, + }, +} + +/// Broadcasts into a topic whose only neighbour reads nothing, until the flood +/// ends in one of three ways: it completed, it stalled, or the fixture failed. +/// A broadcast error, a panic, and a flood that stays at or below +/// `PRESSURE_FLOOR` are failures of the fixture, so they return the error. +async fn flood_until_settled(sender: GossipSender) -> Result { + let accepted = Arc::new(AtomicUsize::new(0)); + let mut task = tokio::spawn({ + let accepted = accepted.clone(); + async move { + let payload = Bytes::from(vec![0x5A; PAYLOAD_SIZE]); + for _ in 0..FLOOD_MESSAGES { + sender.broadcast(payload.clone()).await?; + accepted.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } + }); + + let started = Instant::now(); + let mut last = 0; + let mut quiet_samples = 0; + while started.elapsed() < FLOOD_BUDGET { + if let Ok(ended) = timeout(STALL_SAMPLE, &mut task).await { + flood_result(ended)?; + ensure_pressure(accepted.load(Ordering::Relaxed))?; + return Ok(SettledFlood::Completed); + } + let now = accepted.load(Ordering::Relaxed); + quiet_samples = if now == last { quiet_samples + 1 } else { 0 }; + last = now; + if now > PRESSURE_FLOOR && quiet_samples >= QUIET_SAMPLES_IN_A_STALL { + return Ok(SettledFlood::Stalled { + accepted: now, + parked: task, + }); + } + } + task.abort(); + ensure_pressure(last)?; + bail!( + "the flood neither completed nor stalled within {FLOOD_BUDGET:?}: \ + the hub accepted {last} of {FLOOD_MESSAGES} broadcasts and still accepts more" + ) +} + +impl SettledFlood { + /// Ends a parked flood, and reports what it did if it moved again after + /// the stall. + async fn stop(self) -> Result<()> { + let Self::Stalled { parked, .. } = self else { + return Ok(()); + }; + parked.abort(); + match parked.await { + Err(ended) if ended.is_cancelled() => Ok(()), + ended => flood_result(ended), + } + } +} + +impl fmt::Display for SettledFlood { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Completed => write!(f, "the hub accepted all {FLOOD_MESSAGES} flood broadcasts"), + Self::Stalled { accepted, .. } => write!( + f, + "the hub stalled after {accepted} of {FLOOD_MESSAGES} flood broadcasts" + ), + } + } +} + +fn flood_result(ended: Result, JoinError>) -> Result<()> { + ended + .context("the flood task panicked")? + .context("a flood broadcast failed") +} + +fn ensure_pressure(accepted: usize) -> Result<()> { + ensure!( + accepted > PRESSURE_FLOOR, + "the fixture did not build enough pressure: the flood ended after {accepted} \ + broadcasts, and the silent neighbour's stream credit plus the hub's send queue \ + absorb {PRESSURE_FLOOR}" + ); + Ok(()) +} + +async fn gossip_node(lookup: &MemoryLookup) -> Result<(Endpoint, Gossip, Router)> { + let endpoint = Endpoint::builder(presets::Minimal) + .secret_key(SecretKey::generate()) + .relay_mode(RelayMode::Disabled) + .bind() + .await + .context("bind a gossip endpoint")?; + endpoint + .address_lookup() + .map_err(|err| anyhow!("{err}"))? + .add(lookup.clone()); + lookup.add_endpoint_info(loopback_addr(&endpoint)); + let gossip = Gossip::builder().spawn(endpoint.clone()); + let router = Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .spawn(); + Ok((endpoint, gossip, router)) +} + +fn loopback_addr(endpoint: &Endpoint) -> EndpointAddr { + let sockets: Vec = endpoint + .bound_sockets() + .into_iter() + .map(|mut socket| { + if socket.ip().is_unspecified() { + socket.set_ip(if socket.is_ipv4() { + std::net::IpAddr::from([127, 0, 0, 1]) + } else { + std::net::IpAddr::from(std::net::Ipv6Addr::LOCALHOST) + }); + } + TransportAddr::Ip(socket) + }) + .collect(); + assert!(!sockets.is_empty(), "endpoint has no bound socket"); + EndpointAddr::from_parts(endpoint.id(), sockets) +} + +async fn deliver( + sender: &GossipSender, + receiver: &mut GossipReceiver, + payload: &'static [u8], +) -> Result<()> { + timeout(PROBE_BUDGET, async { + sender + .broadcast(Bytes::from_static(payload)) + .await + .context("the hub refused the broadcast")?; + while let Some(event) = receiver.next().await { + if let Ok(ApiEvent::Received(message)) = event { + if message.content.as_ref() == payload { + return Ok(()); + } + } + } + Err(anyhow!("the event stream ended")) + }) + .await + .map_err(|_| anyhow!("no delivery within {PROBE_BUDGET:?}"))? +} + +/// Speaks the real membership handshake through the public protocol state, so +/// the hub promotes it into the active view, then holds its streams open and +/// reads nothing. The endpoint driver keeps acknowledging at the transport +/// level, so QUIC stays healthy while the stream credit is never returned. +/// The first connection that fails ends the neighbour with that error. +async fn run_silent_neighbour( + endpoint: Endpoint, + topic: TopicId, + went_silent: tokio::sync::mpsc::Sender<()>, +) -> Result { + let mut state: ProtoState = ProtoState::new( + endpoint.id(), + PeerData::new(Vec::::new()), + ProtoConfig::default(), + StdRng::seed_from_u64(0xB0B), + ); + state + .handle( + InEvent::Command(topic, Command::Join(vec![])), + tokio::time::Instant::now(), + None, + ) + .for_each(drop); + let state = Arc::new(tokio::sync::Mutex::new(state)); + + let mut connections = JoinSet::new(); + loop { + tokio::select! { + incoming = endpoint.accept() => { + let incoming = incoming.context("the silent endpoint closed")?; + let state = state.clone(); + let went_silent = went_silent.clone(); + connections.spawn(async move { + let connection = incoming.await.context("accept a connection")?; + let remote = connection.remote_id(); + go_silent_after_membership(connection, topic, remote, state, went_silent).await + }); + } + Some(ended) = connections.join_next() => { + match ended.context("a silent connection panicked")?? {} + } + } + } +} + +async fn go_silent_after_membership( + connection: Connection, + topic: TopicId, + remote: EndpointId, + state: Arc>>, + went_silent: tokio::sync::mpsc::Sender<()>, +) -> Result { + let mut recv = connection.accept_uni().await.context("accept a stream")?; + // The stream header starts with the bare topic id and has gained fields + // since 0.101, so match the prefix rather than the whole frame. + let header = read_frame(&mut recv) + .await + .context("read the topic header")?; + anyhow::ensure!( + header.starts_with(topic.as_bytes().as_slice()), + "unexpected topic in the stream header" + ); + + let mut send: Option = None; + loop { + let payload = read_frame(&mut recv).await.context("read a frame")?; + // postcard concatenates struct fields without framing, so the wire + // message is the 32 topic bytes followed by the frame. + let mut encoded = Vec::with_capacity(32 + payload.len()); + encoded.extend_from_slice(topic.as_bytes()); + encoded.extend_from_slice(&payload); + let message: WireMessage = + postcard::from_bytes(&encoded).context("decode a gossip message")?; + + let out: Vec> = state + .lock() + .await + .handle( + InEvent::RecvMessage(remote, message), + tokio::time::Instant::now(), + None, + ) + .collect(); + + let mut in_active_view = false; + for event in out { + match event { + OutEvent::SendMessage(_, message) => { + let bytes = postcard::to_stdvec(&message).context("encode a gossip message")?; + let stream = match send.as_mut() { + Some(stream) => stream, + None => { + let mut stream = + connection.open_uni().await.context("open a stream")?; + write_frame(&mut stream, topic.as_bytes()).await?; + send.insert(stream) + } + }; + write_frame(stream, &bytes[32..]).await?; + } + OutEvent::EmitEvent(_, ProtoEvent::NeighborUp(_)) => in_active_view = true, + _ => {} + } + } + if in_active_view { + break; + } + } + + went_silent.send(()).await.ok(); + std::future::pending().await +} + +async fn read_frame(recv: &mut RecvStream) -> Result> { + let mut length = [0u8; 4]; + recv.read_exact(&mut length).await.context("frame length")?; + let length = u32::from_be_bytes(length) as usize; + anyhow::ensure!( + length <= 1024 * 1024, + "frame of {length} bytes is too large" + ); + let mut frame = vec![0u8; length]; + recv.read_exact(&mut frame).await.context("frame body")?; + Ok(frame) +} + +async fn write_frame(send: &mut SendStream, payload: &[u8]) -> Result<()> { + send.write_all(&(payload.len() as u32).to_be_bytes()) + .await + .context("frame length")?; + send.write_all(payload).await.context("frame body")?; + Ok(()) +} diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index f85b89342..eac78fa22 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -4,6 +4,7 @@ import base64 from contextlib import contextmanager import hashlib +import itertools import json import os import re @@ -20,6 +21,7 @@ import time import urllib.request from pathlib import Path +from typing import NamedTuple TIMING = runpy.run_path(str(Path(__file__).with_name("ci-installed-model-timing.py"))) UPSTREAM = runpy.run_path(str(Path(__file__).with_name("release-upstream-input.py"))) @@ -1713,11 +1715,11 @@ def ignore(directory, names): return target -def installed_message_identity(data): +def installed_message_identity(data, repo=None): """Use the same source/installed byte boundary as the other installed journeys.""" receipt = json.loads((data / "receipts/source-home-installation.json").read_text()) - candidate = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() - tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], text=True).strip() + candidate = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, cwd=repo).strip() + tree = subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], text=True, cwd=repo).strip() runtime = digest(data / "bin/elastos") assert receipt["source"]["commit"] == candidate and receipt["source"]["tree"] == tree assert receipt["source"]["clean"] is True and receipt["runtime"]["parity"] is True @@ -1737,6 +1739,30 @@ def installed_message_identity(data): "installation_receipt_sha256": digest(data / "receipts/source-home-installation.json")} +class InstalledBuild(NamedTuple): + """One installed source Home that fixtures are copied from.""" + name: str + home: Path + data: Path + repo: Path + identity: dict + + +def installed_build(name, home, data, repo="."): + """Bind an installed Home to the checkout its installation receipt names.""" + home, data, repo = (Path(path).resolve() for path in (home, data, repo)) + return InstalledBuild(name, home, data, repo, installed_message_identity(data, repo)) + + +class Owner(NamedTuple): + """One running Home in an arrangement: a private copy of its build.""" + home: Path + data: Path + api: str + carrier: str + build: InstalledBuild + + def remove_message_fixture(root): """Remove only this fresh, process-free fixture; never follow a root/parent link.""" root = Path(os.path.abspath(root)) @@ -1746,8 +1772,9 @@ def remove_message_fixture(root): shutil.rmtree(root) -# The 10-minute CI step reserves 360 s for work, 150 s for process cleanup, -# and 30 s for disk/removal/receipt operations: 540 s, with 60 s runner margin. +# The 10-minute CI step reserves 360 s for work, 195 s for process cleanup of +# four Homes, and 30 s for disk/removal/receipt operations: 585 s, with 15 s +# runner margin. class MessageDeadline(BaseException): """A phase deadline survives recoverable OS/network error handlers.""" @@ -1769,6 +1796,21 @@ def expired(_signum, _frame): signal.setitimer(signal.ITIMER_REAL, max(.001, timer[0] - (time.monotonic() - started)), timer[1]) +class MessageTermination(BaseException): + """SIGTERM enters the same clean-up as Ctrl-C.""" + + +@contextmanager +def message_termination(): + def terminated(_signum, _frame): + raise MessageTermination("two-Home journey received SIGTERM") + previous = signal.signal(signal.SIGTERM, terminated) + try: + yield + finally: + signal.signal(signal.SIGTERM, previous) + + def remember_message_processes(child, known, marker=None): rows = {pid: row for pid, row in process_rows().items() if not row["command"].endswith(" ")} @@ -1833,7 +1875,7 @@ def attempt(label, action, seconds): # A failed Runtime census/stop still gets an independent PID-bound # attempt, then the next Home is stopped regardless of its result. attempt("fallback_" + name, lambda: stop_message_processes( - child, owned_processes(process_rows(), child.pid, fixtures[name][1])), 10) + child, owned_processes(process_rows(), child.pid, fixtures[name].data)), 10) def remove_and_measure(): attempt("disk_reserve", lambda: record.update(disk_after=require_disk_space(data)), 5) if attempt("fixture_removal", lambda: remove_message_fixture(root), 20): @@ -1849,15 +1891,32 @@ def remove_and_measure(): raise RuntimeError("two-Home cleanup failed; see installed journey receipt") +NODE = "community" # the bootstrap role; every other role is a member Home +TICKET_MAX_LENGTH = 16384 MESSAGE_NETWORKS = { "same-host": { "environment": {"ELASTOS_CARRIER_NETWORK": "direct", "ELASTOS_CARRIER_MDNS": "0"}, "bind_host": "127.0.0.1", "browser_host": "localhost", "ticket_transport": "Ip", - "launches": {"community": 2, "a": 1, "b": 2}, + "launches": {"community": 3, "a": 1, "b": 2, "c": 1}, }, } +class Transition(NamedTuple): + """A control action and the Homes that must prove themselves after it.""" + name: str + action: "str | None" # None: the journey's own first launch + homes: tuple + + +MESSAGE_TRANSITIONS = ( + Transition("start", None, (NODE, "a", "b")), + Transition("node_restart", "restart_community", (NODE, "a", "b")), + Transition("fresh_home", "start_c", (NODE, "a", "b", "c")), + Transition("restart_b", "start_b", (NODE, "a", "b", "c")), +) + + def message_endpoints_valid(homes, observations, arrangement): settings = MESSAGE_NETWORKS.get(arrangement) if settings is None: @@ -1879,19 +1938,72 @@ def message_endpoints_valid(homes, observations, arrangement): return counts == settings["launches"] -def two_home_receipt_valid(row, identity, network_arrangement="same-host"): +def carrier_ticket(owner): + """The Home's own Carrier Provider ticket, read through that Home's own + attach secret. The secret and the token stay inside this call.""" + coordinates = json.loads(bounded_text(owner.data / "gateway-runtime-coords.json")) + api = coordinates["api_url"].rstrip("/") + token = post_json(api + "/api/auth/attach", {"secret": coordinates["attach_secret"], "scope": "shell"})["token"] + ticket = post_json(api + "/api/provider/peer/get_ticket", {}, token)["data"]["ticket"] + assert 0 < len(ticket) <= TICKET_MAX_LENGTH + return ticket + + +def transition_homes(transition, children, observe): + """Liveness comes from each owned process itself. The Carrier observation + comes from the Home's own Provider, which answers once the Home is healthy, + so only a live Home is asked and no poll is needed.""" + alive = {name: name in children and children[name].poll() is None for name in transition.homes} + return {name: {"alive": alive[name], "carrier_ticket_length": observe(name) if alive[name] else None} + for name in transition.homes} + + +def transition_evidence_valid(row): + """After every transition each Home runs and its own Carrier Provider + answers, every member read every other member's fresh post, and one direct + message arrived, each with its exact text.""" + fresh = [] + for transition in MESSAGE_TRANSITIONS: + evidence = row.get("transitions", {}).get(transition.name, {}) + homes = evidence.get("homes", {}) + if set(homes) != set(transition.homes) or not all( + home.get("alive") is True and type(home.get("carrier_ticket_length")) is int + and 0 < home["carrier_ticket_length"] <= TICKET_MAX_LENGTH + for home in homes.values()): + return False + posts, direct = evidence.get("community", []), evidence.get("direct", {}) + pairs = {(post.get("from"), post.get("to")) for post in posts} + members = [name for name in transition.homes if name != NODE] + if pairs != set(itertools.permutations(members, 2)) or (direct.get("from"), direct.get("to")) not in pairs: + return False + if not all(isinstance(item.get("sent"), str) and item["sent"] and item.get("received") == item["sent"] + for item in (*posts, direct)): + return False + texts = {post["from"]: post["sent"] for post in posts} + if any(post["sent"] != texts[post["from"]] for post in posts): + return False + fresh += [*texts.values(), direct["sent"]] + return len(set(fresh)) == len(fresh) + + +def two_home_receipt_valid(row, identity, network_arrangement="same-host", node_identity=None): + """``identity`` is the Home build; the node build is the same one unless named.""" + node_identity = node_identity or identity if row.get("network_arrangement") != network_arrangement or network_arrangement not in MESSAGE_NETWORKS: return False - if any(row.get(key) != value for key, value in identity.items()): + if any(row.get(key) != value for key, value in identity.items()) or row.get("node_identity") != node_identity: return False + launches = MESSAGE_NETWORKS[network_arrangement]["launches"] homes = row.get("homes", []) cleanup = row.get("process_cleanup", []) return (row.get("status") == "passed" and "failure" not in row and "browser_failure" not in row - and row.get("fixture_removed") is True and len(homes) == 3 - and {home.get("home") for home in homes} == {"a", "b", "community"} - and all(home.get("installed_runtime_sha256") == identity["installed_runtime_sha256"] for home in homes) - and len(cleanup) == 5 and all(item.get("status") == "passed" and item.get("after") - and sum(item["after"].values()) == 0 for item in cleanup) + and row.get("fixture_removed") is True and len(homes) == len(launches) + and {home.get("home") for home in homes} == set(launches) + and all(home.get("installed_runtime_sha256") == (node_identity if home.get("home") == NODE else identity)[ + "installed_runtime_sha256"] for home in homes) + and len(cleanup) == sum(launches.values()) + and all(item.get("status") == "passed" and item.get("after") + and sum(item["after"].values()) == 0 for item in cleanup) and row.get("browser_cross_home_http_requests") == 0 and message_endpoints_valid(homes, row.get("direct_endpoint_observations", []), network_arrangement) and all(row.get("results", {}).get(name) == "passed" for name in ( @@ -1899,95 +2011,131 @@ def two_home_receipt_valid(row, identity, network_arrangement="same-host"): and all(row.get("negative_controls", {}).get(name, {}).get("assertion_rejected") is True for name in ("sender_delivered",)) and row["negative_controls"]["sender_delivered"].get("api_state") == "pending" - and row["negative_controls"]["sender_delivered"].get("ui_detail") == "Sending") + and row["negative_controls"]["sender_delivered"].get("ui_detail") == "Sending" + and transition_evidence_valid(row)) + +@message_termination() +def two_home_messages(home_build, evidence, network="same-host", fixture_root=None, node_build=None): + """Installed Community and Direct journey. Topology belongs to setup, not assertions. -def two_home_messages(home, data, evidence, network="same-host", fixture_root=None): - """Installed Community and Direct journey. Topology belongs to setup, not assertions.""" + The Community node runs ``node_build`` and every other Home runs + ``home_build``; one build serves both unless a mixed-version run names two. + """ if network not in MESSAGE_NETWORKS: raise RuntimeError("unsupported two-Home network arrangement") settings = MESSAGE_NETWORKS[network] - identity = installed_message_identity(data) + node_build = node_build or home_build + data = home_build.data evidence.mkdir(parents=True, exist_ok=False) - root = fixture_root or home.parent / "two-home-message-fixture" - # Three fresh immutable copies; source fixtures never copy identity or user data. + root = fixture_root or home_build.home.parent / "two-home-message-fixture" + # Fresh immutable copies; source fixtures never copy identity or user data. require_disk_space(data) root.mkdir(mode=0o700, parents=True, exist_ok=False) control = root / "control" control.mkdir(mode=0o700) - record = dict(identity, network_arrangement=network, results={}, homes=[], negative_controls={}, - direct_endpoint_observations=[]) + record = dict(home_build.identity, node_identity=node_build.identity, network_arrangement=network, results={}, + homes=[], negative_controls={}, direct_endpoint_observations=[], transitions={}) children, fixtures, browser = {}, {}, None browser_processes = {} cleanup = [] started = time.monotonic() # Explicit environment excludes ambient relay, public-network and provider overrides. def environment(owner): - return {"PATH": os.environ["PATH"], "HOME": str(owner[0]), "XDG_DATA_HOME": str(owner[1].parent), + return {"PATH": os.environ["PATH"], "HOME": str(owner.home), "XDG_DATA_HOME": str(owner.data.parent), **settings["environment"], "ELASTOS_QUIET_RUNTIME_NOTICES": "1"} def launch(name): assert name not in children, "Home already has an owned Runtime" owner = fixtures[name] - with (owner[0] / "runtime.private.log").open("ab") as log: - child = subprocess.Popen([str(owner[1] / "bin/elastos"), "gateway", "--addr", owner[2]], + with (owner.home / "runtime.private.log").open("ab") as log: + child = subprocess.Popen([str(owner.data / "bin/elastos"), "gateway", "--addr", owner.api], env=environment(owner), stdout=log, stderr=log, start_new_session=True) children[name] = child - wait_healthy(child, owner[2], "two-Home " + name) - coordinates = json.loads(bounded_text(owner[1] / "gateway-runtime-coords.json")) - api = coordinates["api_url"].rstrip("/") - token = post_json(api + "/api/auth/attach", {"secret": coordinates["attach_secret"], "scope": "shell"})["token"] - ticket = post_json(api + "/api/provider/peer/get_ticket", {}, token)["data"]["ticket"] - assert len(ticket) <= 16384 + wait_healthy(child, owner.api, "two-Home " + name) + ticket = carrier_ticket(owner) decoded = json.loads(base64.b32decode(ticket.upper() + "=" * (-len(ticket) % 8))) endpoints = decoded["endpoints"] assert len(endpoints) == 1, "fixture ticket must describe one installed Home" observed = endpoints[0]["addrs"] - assert observed == [{settings["ticket_transport"]: owner[3]}], "ticket differs from configured Carrier address" + assert observed == [{settings["ticket_transport"]: owner.carrier}], "ticket differs from configured Carrier address" record["direct_endpoint_observations"].append({"home": name, "ticket_addresses": observed}) def stop(name): child = children[name] with message_timeout(35): - row = cleanup_runtime(child, fixtures[name][1]) + row = cleanup_runtime(child, fixtures[name].data) cleanup.append(dict(row, home=name)) assert row["status"] == "passed", "two-Home process cleanup failed" del children[name] def configure(*arguments): - owner = fixtures["community"] + owner = fixtures[NODE] with (root / "configuration.private.log").open("ab") as log: - subprocess.run([str(owner[1] / "bin/elastos"), "collaboration-config", *map(str, arguments)], + subprocess.run([str(owner.data / "bin/elastos"), "collaboration-config", *map(str, arguments)], env=environment(owner), stdout=log, stderr=log, check=True, timeout=60) + def stage(name): + build = node_build if name == NODE else home_build + destination = root / name + copied = fresh_fixture(build.home, build.data, destination, include_engine=False) + assert installed_message_identity(copied, build.repo) == build.identity + carrier = f"{settings['bind_host']}:{free_port(socket.SOCK_DGRAM)}" + with (copied / "config.toml").open("a") as config: + config.write(f'carrier_bind_addr = "{carrier}"\n') + fixtures[name] = Owner(destination, copied, f"{settings['bind_host']}:{free_port()}", carrier, build) + record["homes"].append({"home": name, "installed_runtime_sha256": digest(copied / "bin/elastos"), + "carrier_bind_address": carrier}) + def admit(name): + target = fixtures[name].data / "collaboration-network-v1.json" + shutil.copy2(root / "network.json", target) + target.chmod(0o600) + def base(name): + return f"http://{settings['browser_host']}:" + fixtures[name].api.rsplit(":", 1)[1] + def prove(transition): + homes = transition_homes(transition, children, lambda name: len(carrier_ticket(fixtures[name]))) + record["transitions"].setdefault(transition.name, {})["homes"] = homes + for name, home in homes.items(): + assert home["alive"], f"two-Home {name} is not running after {transition.name}" + assert home["carrier_ticket_length"], \ + f"two-Home {name} Carrier Provider answered no ticket after {transition.name}" + def require_running(): + for name, child in children.items(): + assert child.poll() is None, f"two-Home {name} exited during the journey" + def restart_node(): + stop(NODE) + launch(NODE) + def start_fresh_home(): + stage("c") + admit("c") + launch("c") + return {"base": base("c")} + actions = {"stop_b": lambda: stop("b"), "start_b": lambda: launch("b"), + "restart_community": restart_node, "start_c": start_fresh_home} + start, *later = MESSAGE_TRANSITIONS + entered = {transition.action: transition for transition in later} try: with message_timeout(360): - for name in ("community", "a", "b"): - destination = root / name - copied = fresh_fixture(home, data, destination, include_engine=False) - assert installed_message_identity(copied) == identity - carrier = f"{settings['bind_host']}:{free_port(socket.SOCK_DGRAM)}" - with (copied / "config.toml").open("a") as config: - config.write(f'carrier_bind_addr = "{carrier}"\n') - fixtures[name] = (destination, copied, f"{settings['bind_host']}:{free_port()}", carrier) - record["homes"].append({"home": name, "installed_runtime_sha256": digest(copied / "bin/elastos"), - "carrier_bind_address": carrier}) - launch("community") + for name in (NODE, "a", "b"): + stage(name) + launch(NODE) configure("create-authority-key", "--key", root / "authority.key") - configure("export-local-bootstrap-receipt", "--data-root", fixtures["community"][1], + configure("export-local-bootstrap-receipt", "--data-root", fixtures[NODE].data, "--runtime-kind", "gateway", "--output", root / "bootstrap.json") configure("generate-initial", "--authority-key", root / "authority.key", "--network-id", "ci.two-home", "--conversation-id", "community", "--bootstrap-peer", root / "bootstrap.json", "--output", root / "network.json") configure("verify", "--input", root / "network.json") - stop("community") - for owner in fixtures.values(): - target = owner[1] / "collaboration-network-v1.json" - shutil.copy2(root / "network.json", target) - target.chmod(0o600) + stop(NODE) + for name in fixtures: + admit(name) for name in fixtures: launch(name) - bases = [f"http://{settings['browser_host']}:" + fixtures[name][2].rsplit(":", 1)[1] for name in ("a", "b")] + prove(start) with (root / "browser.private.log").open("wb") as log: - browser = subprocess.Popen(["node", "scripts/ci-installed-two-home-journey.mjs", *bases, - str(evidence), str(control), settings["browser_host"]], stdout=log, stderr=log, start_new_session=True) + browser = subprocess.Popen(["node", str(Path(__file__).with_name("ci-installed-two-home-journey.mjs")), + base("a"), base("b"), str(evidence), str(control), settings["browser_host"]], + stdout=log, stderr=log, start_new_session=True) last, next_census = 0, 0 while browser.poll() is None: + # A Home that exits between transitions fails the journey at once, + # even when the other Homes could finish their exchange without it. + require_running() if time.monotonic() >= next_census: remember_message_processes(browser, browser_processes, "--two-home-fixture=" + str(control)) next_census = time.monotonic() + 1 @@ -1995,22 +2143,29 @@ def configure(*arguments): if request.exists(): value = private_json(request) if value["id"] > last: - assert value["id"] == last + 1 and value["action"] in {"stop_b", "start_b"} - stop("b") if value["action"] == "stop_b" else launch("b") + assert value["id"] == last + 1 and value["action"] in actions + reply = actions[value["action"]]() or {} + # The browser half exchanges messages only after the Homes have their peers back. + if value["action"] in entered: + prove(entered[value["action"]]) last = value["id"] - private_json(control / "response.json", {"id": last, "status": "passed"}) + private_json(control / "response.json", {"id": last, "status": "passed", **reply}) time.sleep(.1) observed = json.loads((evidence / "two-home-ui.json").read_text()) + for name, exchange in observed.pop("exchanges", {}).items(): + record["transitions"].setdefault(name, {}).update(exchange) record.update(observed) if browser.returncode and (control / "error.private.txt").exists(): record["browser_failure"] = redact((control / "error.private.txt").read_text(), [str(root)])[:1200] assert browser.returncode == 0, "installed two-Home browser journey failed" + require_running() assert all(record["results"].get(name) == "passed" for name in ( "community_post", "direct_received", "sender_delivered", "offline_pending", "restart_delivery")) assert all(record["negative_controls"].get(name, {}).get("assertion_rejected") is True for name in ("sender_delivered",)) + assert transition_evidence_valid(record), "two-Home transition evidence is incomplete" except BaseException as error: - record["failure"] = observer_failure("two_home_messages", error, [str(root), str(home)]) + record["failure"] = observer_failure("two_home_messages", error, [str(root), str(home_build.home), str(node_build.home)]) raise finally: record["process_cleanup"] = cleanup @@ -2055,12 +2210,20 @@ def repeat(home, data, evidence, count): parser.add_argument("--two-home-messages", action="store_true") parser.add_argument("--network-arrangement", choices=MESSAGE_NETWORKS.keys(), default="same-host") parser.add_argument("--two-home-root", type=Path, help="fresh disposable fixture path") + # A mixed-version run names a second installed Home for the Community node. + parser.add_argument("--node-home", type=Path) + parser.add_argument("--node-data", type=Path) + parser.add_argument("--node-repo", type=Path, help="checkout the node Home was installed from") args = parser.parse_args() if args.two_home_messages: if args.repeat or args.home_only or args.carrier_get or args.finite_memory or args.model_repair: parser.error("two-Home messages is a separate installed journey") - two_home_messages(args.home.resolve(), args.data.resolve(), args.evidence.resolve(), - args.network_arrangement, args.two_home_root) + node = (args.node_home, args.node_data, args.node_repo) + if any(node) and not all(node): + parser.error("a node build needs --node-home, --node-data and --node-repo") + two_home_messages(installed_build("home", args.home, args.data), args.evidence.resolve(), + args.network_arrangement, args.two_home_root, + installed_build("node", *node) if all(node) else None) raise SystemExit(0) if args.model_repair and (sys.platform != "linux" or args.home_only or args.repeat or args.finite_memory): parser.error("--model-repair proves one production Linux Home with two signed models") diff --git a/scripts/ci-installed-two-home-journey.mjs b/scripts/ci-installed-two-home-journey.mjs index 402b8e1b4..1e2268d38 100644 --- a/scripts/ci-installed-two-home-journey.mjs +++ b/scripts/ci-installed-two-home-journey.mjs @@ -25,19 +25,18 @@ async function run() { const require = createRequire(import.meta.url); const {chromium} = require(process.env.BROWSER_OPERATOR_PLAYWRIGHT_CORE || '../elastos/tools/browser-playwright-engine/node_modules/playwright'); const [baseA, baseB, evidence, control, browserHost] = process.argv.slice(2); - const bases = [baseA, baseB]; - assertMessageOrigins(bases, browserHost); - const result = {stage:'sign_in', results:{}, negative_controls:{}, observations:[], browser_cross_home_http_requests:0}; + assertMessageOrigins([baseA, baseB], browserHost); + const result = {stage:'sign_in', results:{}, negative_controls:{}, observations:[], exchanges:{}, browser_cross_home_http_requests:0}; let sequence = 0; async function runtime(action) { const id = ++sequence; writeFileSync(join(control, 'request.json.next'), JSON.stringify({id, action}), {flag:'wx', mode:0o600}); renameSync(join(control, 'request.json.next'), join(control, 'request.json')); - await until('owned Runtime action', async () => { + return until('owned Runtime action', async () => { let reply; try {reply = JSON.parse(readFileSync(join(control, 'response.json'), 'utf8'));} catch (error) {if (error.code !== 'ENOENT') throw error;} - return reply?.id === id && reply.status === 'passed'; + return reply?.id === id && reply.status === 'passed' && reply; }, 120000); } const sleep = ms => new Promise(resolve=>setTimeout(resolve,ms)); @@ -51,7 +50,8 @@ async function run() { result.stage='browser_launch'; browser = await chromium.launch({headless:true, args:['--two-home-fixture='+control]}); const homes=[]; - for(const [i,base] of bases.entries()) { + async function signIn(base) { + const i=homes.length, name='abc'[i]; const context=await browser.newContext({timezoneId:'UTC', serviceWorkers:'block', viewport:{width:1440,height:900}}); await context.route('**/*', route => { const url = new URL(route.request().url()); @@ -68,7 +68,7 @@ async function run() { result.stage='home_frontdoor_'+i; await page.goto(base+'/home/',{waitUntil:'domcontentloaded'}); result.stage='home_name_'+i; - await page.locator('#home-unlock-name').fill('Two Home '+(i?'B':'A'),{timeout:60000}); + await page.locator('#home-unlock-name').fill('Two Home '+name.toUpperCase(),{timeout:60000}); result.stage='passkey_registration_'+i; const registered=page.waitForResponse(r=>r.url().endsWith('/api/auth/passkey/register/complete')); await page.locator('#home-unlock-primary').click(); @@ -87,9 +87,11 @@ async function run() { return tab; } result.stage='people_launch_'+i; - const people=await app('people'); - homes.push({context,page,people,app,base}); + const home={name,context,page,people:await app('people'),app,base}; + homes.push(home); + return home; } + const a=await signIn(baseA), b=await signIn(baseB); async function api(page,path,body) { return page.evaluate(async ({path,body})=>{ const token=new URLSearchParams(location.hash.slice(1)).get('home_token'); @@ -98,12 +100,14 @@ async function run() { return r.json(); },{path,body}); } - const [a,b]=homes; - const communityA=await a.app('chat-room'), communityB=await b.app('chat-room'); + async function joinCommunity(home) { + await api(home.community, '/api/apps/chat-room/session/start', {}); + await home.community.locator('#message-input:enabled').waitFor({timeout:60000}); + } + const communityA=a.community=await a.app('chat-room'), communityB=b.community=await b.app('chat-room'); result.stage='community_post'; await communityA.locator('#message-input:enabled').waitFor({timeout:90000}); - await api(communityB, '/api/apps/chat-room/session/start', {}); - await communityB.locator('#message-input:enabled').waitFor({timeout:60000}); + await joinCommunity(b); const postText='Two Home Community '+crypto.randomUUID(); const posted=await api(communityA, '/api/apps/chat-room/objects/send', {request_id:'two-home:'+crypto.randomUUID(),body:postText}); let communityPoll; @@ -168,6 +172,43 @@ async function run() { assertSenderDelivery(settled,senderDetail); result.results.sender_delivered='passed'; result.direct={text,recipient_state:received.delivery_state,sender_state:settled.delivery_state,sender_detail:senderDetail}; + // One round after a transition: every Home posts, every other Home reads + // that exact text, and one direct message crosses the accepted contact. + async function exchange(transition) { + result.stage='exchange_'+transition; + const round={community:[]}; + for(const sender of homes) { + const sent=`Two Home ${transition} ${sender.name} ${crypto.randomUUID()}`; + const post=await api(sender.community,'/api/apps/chat-room/objects/send',{request_id:'two-home:'+crypto.randomUUID(),body:sent}); + for(const receiver of homes.filter(home=>home!==sender)) { + let poll; + await until(`Community ${transition} from ${sender.name} at ${receiver.name}`,async()=>{ + poll=await api(receiver.community,'/api/apps/chat-room/poll',{since:0,mark_read:false}); + return poll.objects.some(row=>row.message_id===post.message_id&&row.body===sent); + }); + round.community.push({from:sender.name,to:receiver.name,sent,received:assertCommunityMessage(poll,post.message_id,sent).body}); + } + } + const sent=`Two Home ${transition} direct ${crypto.randomUUID()}`; + await api(a.community,'/api/apps/chat-room/direct/messages/send',{request_id:'two-home:'+crypto.randomUUID(),conversation_id:conversation.conversation_id,text:sent}); + let messages; + const arrived=await until(`direct ${transition} at b`,async()=>{ + messages=(await api(b.community,path)).messages; + return messages.find(m=>m.text===sent&&m.direction==='incoming'&&m.delivery_state==='received'); + }); + round.direct={from:a.name,to:b.name,sent,received:assertReceivedDirect(messages,arrived.message_id,sent).text}; + return round; + } + result.exchanges.start=await exchange('start'); + await runtime('restart_community'); + result.exchanges.node_restart=await exchange('node_restart'); + const fresh=await runtime('start_c'); + assertMessageOrigins([fresh.base], browserHost); + const c=await signIn(fresh.base); + result.stage='fresh_home_community'; + c.community=await c.app('chat-room'); + await joinCommunity(c); + result.exchanges.fresh_home=await exchange('fresh_home'); result.stage='offline_pending'; // Close B's projections before stopping its owned Runtime; retain only its passkey context. for(const page of b.context.pages()) if(page!==b.page) await page.close(); @@ -212,6 +253,9 @@ async function run() { assertSenderDelivery(settledAfterRestart,recoveredDetail); result.recovery={text:offlineText,recipient_state:recovered.delivery_state,sender_state:settledAfterRestart.delivery_state,sender_detail:recoveredDetail}; result.results.restart_delivery='passed'; + b.community=await b.app('chat-room'); + await joinCommunity(b); + result.exchanges.restart_b=await exchange('restart_b'); await direct.screenshot({path:join(evidence,'direct-a-desktop.png'),fullPage:true}); await direct.setViewportSize({width:390,height:844}); await direct.screenshot({path:join(evidence,'direct-a-phone.png'),fullPage:true}); diff --git a/scripts/ci-installed-two-home-test.py b/scripts/ci-installed-two-home-test.py index 218a5e48c..26aaa47d2 100644 --- a/scripts/ci-installed-two-home-test.py +++ b/scripts/ci-installed-two-home-test.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """Receipt guards for the installed two-Home journey; no Runtime or browser mocks.""" +import contextlib import copy +import io import json from pathlib import Path import runpy @@ -52,16 +54,34 @@ def identity(self): return {'candidate':'c'*40, 'source_tree':'d'*40, 'installed_runtime_sha256':'a'*64, 'source_components_sha256':'b'*64, 'installation_receipt_sha256':'e'*64} - def receipt(self): - identity = self.identity() - return {**identity, 'status':'passed', 'fixture_removed':True, 'network_arrangement':'same-host', - 'homes':[{'home':name, 'installed_runtime_sha256':identity['installed_runtime_sha256'], - 'carrier_bind_address':f'127.0.0.1:{1025+index}'} - for index,name in enumerate(('a','b','community'))], - 'process_cleanup':[{'status':'passed', 'after':{'runtime':0}} for _ in range(5)], + # The journey's contract, spelled out here so a changed table cannot move the test with it. + TRANSITIONS = {'start':('community','a','b'), 'node_restart':('community','a','b'), + 'fresh_home':('community','a','b','c'), 'restart_b':('community','a','b','c')} + + def transitions(self): + rows = {} + for name, homes in self.TRANSITIONS.items(): + members = [home for home in homes if home != 'community'] + rows[name] = {'homes':{home:{'alive':True, 'carrier_ticket_length':120} for home in homes}, + 'community':[{'from':sender, 'to':receiver, 'sent':f'{name} post from {sender}', + 'received':f'{name} post from {sender}'} + for sender in members for receiver in members if sender != receiver], + 'direct':{'from':'a', 'to':'b', 'sent':name+' direct', 'received':name+' direct'}} + return rows + + def receipt(self, identity=None, node=None): + identity = identity or self.identity() + node = node or identity + return {**identity, 'node_identity':node, 'status':'passed', 'fixture_removed':True, + 'network_arrangement':'same-host', + 'homes':[{'home':name, 'carrier_bind_address':f'127.0.0.1:{1025+index}', + 'installed_runtime_sha256':(node if name == 'community' else identity)['installed_runtime_sha256']} + for index,name in enumerate(('a','b','c','community'))], + 'process_cleanup':[{'status':'passed', 'after':{'runtime':0}} for _ in range(7)], 'browser_cross_home_http_requests':0, 'direct_endpoint_observations':[{'home':name,'ticket_addresses':[{'Ip':f'127.0.0.1:{1025+index}'}]} - for index,name in ((2,'community'),(2,'community'),(0,'a'),(1,'b'),(1,'b'))], + for index,name in ((3,'community'),(3,'community'),(0,'a'),(1,'b'),(3,'community'),(2,'c'),(1,'b'))], + 'transitions':self.transitions(), 'results':{name:'passed' for name in ('community_post','direct_received','sender_delivered', 'offline_pending','restart_delivery')}, 'negative_controls':{name:{'assertion_rejected':True, 'api_state':'pending', 'ui_detail':'Sending'} @@ -77,7 +97,9 @@ def test_acceptance_refuses_stale_identity_unchecked_failure_controls_or_survivi self.assertFalse(check(changed, self.identity()), key) for key, value in [('status','failed'), ('failure',{'phase':'late failure'}), ('browser_failure','failed'), ('fixture_removed',False), ('homes',[]), ('process_cleanup',[]), - ('browser_cross_home_http_requests',1), ('direct_endpoint_observations',[])]: + ('process_cleanup',valid['process_cleanup'][:5]), + ('browser_cross_home_http_requests',1), ('direct_endpoint_observations',[]), + ('transitions',{})]: changed = copy.deepcopy(valid) changed[key] = value self.assertFalse(check(changed, self.identity()), key) @@ -96,6 +118,187 @@ def test_acceptance_refuses_stale_identity_unchecked_failure_controls_or_survivi changed['homes'][1]['home'] = 'a' self.assertFalse(check(changed, self.identity())) + def test_acceptance_binds_the_node_and_each_home_to_its_own_build(self): + check = JOURNEY['two_home_receipt_valid'] + homes, node = self.identity(), {**self.identity(), 'candidate':'f'*40, 'installed_runtime_sha256':'9'*64} + mixed = self.receipt(homes, node) + self.assertTrue(check(mixed, homes, node_identity=node)) + self.assertFalse(check(mixed, homes), 'a same-version acceptance refuses another node build') + self.assertFalse(check(self.receipt(), homes, node_identity=node), 'the named node build did not run') + for name in ('community', 'a'): + changed = copy.deepcopy(mixed) + row = next(home for home in changed['homes'] if home['home'] == name) + row['installed_runtime_sha256'] = ('a' if name == 'community' else '9')*64 + self.assertFalse(check(changed, homes, node_identity=node), name+' ran the other build') + changed = self.receipt() + del changed['node_identity'] + self.assertFalse(check(changed, homes)) + + def test_acceptance_requires_positive_evidence_after_every_transition(self): + check = JOURNEY['two_home_receipt_valid'] + valid = self.receipt() + def refused(label, change): + changed = copy.deepcopy(valid) + change(changed['transitions']) + self.assertFalse(check(changed, self.identity()), label) + for name, homes in self.TRANSITIONS.items(): + refused(name+' is missing', lambda rows: rows.pop(name)) + for home in homes: + refused(f'{home} is missing at {name}', lambda rows: rows[name]['homes'].pop(home)) + for alive in (False, None, 'true'): + refused(f'{home} alive={alive!r} at {name}', + lambda rows: rows[name]['homes'][home].update(alive=alive)) + for length in (0, None, True, '120', 16385): + refused(f'{home} carrier_ticket_length={length!r} at {name}', + lambda rows: rows[name]['homes'][home].update(carrier_ticket_length=length)) + refused(f'{home} reports no Carrier observation at {name}', + lambda rows: rows[name]['homes'][home].pop('carrier_ticket_length')) + for index, post in enumerate(valid['transitions'][name]['community']): + pair = f"{post['from']} to {post['to']} at {name}" + refused(pair+' is missing', lambda rows: rows[name]['community'].pop(index)) + refused(pair+' arrived as a prefix', + lambda rows: rows[name]['community'][index].update(received=post['sent'][:-1])) + refused(pair+' has no text', lambda rows: rows[name]['community'][index].update(sent='', received='')) + if len(homes) > 3: + refused(pair+' is a second post from its sender', lambda rows: rows[name]['community'][index].update( + sent='second post', received='second post')) + refused('one post stands in for another at '+name, + lambda rows: rows[name]['community'].__setitem__(1, rows[name]['community'][0])) + refused('direct is missing at '+name, lambda rows: rows[name].pop('direct')) + refused('direct arrived as a prefix at '+name, lambda rows: rows[name]['direct'].update(received=name+' direc')) + refused('direct has no recipient at '+name, lambda rows: rows[name]['direct'].update(to='a')) + def repost(rows): + for post in rows['node_restart']['community']: + post['sent'] = post['received'] = 'start post from '+post['from'] + refused('a post from before the node restart counts after it', repost) + refused('a direct message from before the node restart counts after it', + lambda rows: rows['node_restart']['direct'].update(sent='start direct', received='start direct')) + + def test_transition_homes_observe_only_a_live_home_and_record_an_exited_one(self): + observe = JOURNEY['transition_homes'] + transition = JOURNEY['Transition']('node_restart', 'restart_community', ('community', 'a')) + running, exited = mock.Mock(**{'poll.return_value':None}), mock.Mock(**{'poll.return_value':1}) + self.assertEqual(observe(transition, {'community':running, 'a':running}, lambda name: len(name)), + {'community':{'alive':True, 'carrier_ticket_length':9}, + 'a':{'alive':True, 'carrier_ticket_length':1}}) + def refuse(name): + if name == 'community': + raise AssertionError('an exited Home was asked for a ticket') + return 1 + self.assertEqual(observe(transition, {'community':exited, 'a':running}, refuse)['community'], + {'alive':False, 'carrier_ticket_length':None}) + self.assertEqual(observe(transition, {'a':running}, refuse)['community'], + {'alive':False, 'carrier_ticket_length':None}) + + def test_the_carrier_observation_uses_the_homes_own_secret_at_shell_scope_and_bounds_the_ticket(self): + with tempfile.TemporaryDirectory() as temporary: + data = Path(temporary).resolve() + (data/'gateway-runtime-coords.json').write_text(json.dumps( + {'api_url':'http://127.0.0.1:1/', 'attach_secret':'fixture-secret'})) + owner = JOURNEY['Owner'](data, data, '', '', None) + posts = [] + def post_json(url, body, token=None): + posts.append((url, body, token)) + return {'token':'fixture-token'} if url.endswith('/attach') else {'data':{'ticket':ticket}} + with mock.patch.dict(JOURNEY['carrier_ticket'].__globals__, {'post_json':post_json}): + ticket = 'T'*120 + self.assertEqual(JOURNEY['carrier_ticket'](owner), ticket) + self.assertEqual(posts, [('http://127.0.0.1:1/api/auth/attach', + {'secret':'fixture-secret', 'scope':'shell'}, None), + ('http://127.0.0.1:1/api/provider/peer/get_ticket', {}, 'fixture-token')]) + for ticket in ('', 'T'*(JOURNEY['TICKET_MAX_LENGTH']+1)): + with self.assertRaises(AssertionError): + JOURNEY['carrier_ticket'](owner) + + def drive(self, outcome): + """Run the mixed-version driver over a stand-in journey that saves a valid receipt unless ``outcome`` interferes.""" + driver = runpy.run_path(str(ROOT/'scripts/gossip-mixed-version-journey.py')) + journey = driver['JOURNEY'] + identities = {'released':{**self.identity(), 'candidate':'1'*40, 'installed_runtime_sha256':'2'*64}, + 'candidate':self.identity()} + calls = [] + def stand_in(homes, evidence, fixture_root, node_build): + calls.append((node_build.name, homes.name, evidence.name, fixture_root.name)) + evidence.mkdir() + receipt = self.receipt(homes.identity, node_build.identity) + error = outcome(len(calls), receipt, fixture_root, journey) + (evidence/'installed-journeys.json').write_text(json.dumps(receipt)) + if error: + raise error + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + arguments = [argument for build in identities for part in ('home', 'data', 'repo') + for argument in (f'--{build}-{part}', str(root/build/part))] + raised = None + with mock.patch.dict(journey, {'two_home_messages':stand_in, 'installed_build':lambda name, *paths: + journey['InstalledBuild'](name, *paths, identities[name])}), contextlib.redirect_stdout(io.StringIO()): + try: + driver['main']([*arguments, '--evidence', str(root/'evidence'), '--fixture-root', str(root/'fixtures')]) + except BaseException as error: + raised = error + return raised, json.loads((root/'evidence/mixed-version-journey.json').read_text()), calls + + def test_mixed_version_driver_runs_every_pairing_into_its_own_evidence_and_fixture(self): + raised, receipt, calls = self.drive(lambda *_: None) + self.assertEqual(raised.code, 0) + names = ['node-released-homes-released', 'node-released-homes-candidate', + 'node-candidate-homes-released', 'node-candidate-homes-candidate'] + self.assertEqual(calls, [('released','released',names[0],names[0]), ('released','candidate',names[1],names[1]), + ('candidate','released',names[2],names[2]), ('candidate','candidate',names[3],names[3])]) + self.assertEqual((receipt['status'], receipt['fixture_root_removed']), ('passed', True)) + self.assertEqual([row['arrangement'] for row in receipt['arrangements']], names) + for row, (node, homes, *_) in zip(receipt['arrangements'], calls): + self.assertEqual((row['status'], row['node']['build'], row['homes']['build']), ('passed', node, homes)) + self.assertEqual(row['node']['installed_runtime_sha256'], ('2' if node == 'released' else 'a')*64) + self.assertEqual(row['homes']['installed_runtime_sha256'], ('2' if homes == 'released' else 'a')*64) + self.assertEqual(row['journey']['transitions'], self.transitions()) + + def test_mixed_version_driver_fails_on_a_failed_cleanup_an_unproven_receipt_or_a_leftover_fixture(self): + def deadline(index, receipt, _root, journey): + if index == 1: + receipt.update(failure={'phase':'two_home_messages'}) + return journey['MessageDeadline']('two-Home phase deadline') + def failed_cleanup(index, receipt, _root, _journey): + if index == 2: + receipt.update(status='failed', cleanup_failures=[{'phase':'stop_a'}]) + return RuntimeError('two-Home cleanup failed; see installed journey receipt') + def no_carrier(index, receipt, _root, _journey): + if index == 3: + receipt['transitions']['node_restart']['homes']['a']['carrier_ticket_length'] = 0 + def raised_after_its_receipt(index, _receipt, _root, _journey): + if index == 4: + return RuntimeError('late failure') + def leftover(index, _receipt, root, _journey): + if index == 4: + (root/'a').mkdir(parents=True) + for outcome, failed, removed in ((deadline, ['node-released-homes-released'], True), + (failed_cleanup, ['node-released-homes-candidate'], True), + (no_carrier, ['node-candidate-homes-released'], True), + (raised_after_its_receipt, ['node-candidate-homes-candidate'], True), + (leftover, [], False)): + with self.subTest(outcome=outcome.__name__): + raised, receipt, calls = self.drive(outcome) + self.assertEqual(raised.code, 1) + self.assertEqual(len(calls), 4, 'the other arrangements stay observable') + self.assertEqual((receipt['status'], receipt['fixture_root_removed']), ('failed', removed)) + self.assertEqual([row['arrangement'] for row in receipt['arrangements'] if row['status'] != 'passed'], failed) + + def test_mixed_version_driver_stops_on_sigterm_and_keeps_a_failed_receipt(self): + def terminated(index, _receipt, _root, _journey): + if index == 2: + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(1) + ignored = [] + operator = lambda *_: ignored.append(True) + previous = signal.signal(signal.SIGTERM, operator) + try: + raised, receipt, calls = self.drive(terminated) + self.assertIs(signal.getsignal(signal.SIGTERM), operator) + finally: + signal.signal(signal.SIGTERM, previous) + self.assertEqual((type(raised).__name__, ignored, len(calls)), ('MessageTermination', [], 2)) + self.assertEqual((receipt['status'], len(receipt['arrangements'])), ('failed', 1)) + def test_network_arrangement_and_each_observed_ticket_must_match_configuration(self): check = JOURNEY['two_home_receipt_valid'] valid = self.receipt() @@ -103,7 +306,7 @@ def test_network_arrangement_and_each_observed_ticket_must_match_configuration(s changed['network_arrangement'] = 'two-nat' self.assertFalse(check(changed, self.identity())) self.assertFalse(check(valid, self.identity(), 'two-nat')) - for index in range(5): + for index in range(len(valid['direct_endpoint_observations'])): for address in ({'Relay':'https://relay.invalid'}, {'Ip':'192.0.2.1:1234'}, {'Ip':'127.0.0.1:1234'}): changed = copy.deepcopy(valid) changed['direct_endpoint_observations'][index]['ticket_addresses'] = [address] @@ -151,7 +354,8 @@ def disk(_): 'stop_message_processes':fallback, 'require_disk_space':disk}): with self.assertRaisesRegex(RuntimeError, 'cleanup failed'): finish(root, root, evidence, {'status':'passed'}, children, - {name:(root,root) for name in children}, stop, None, {}, time.monotonic()) + {name:JOURNEY['Owner'](root, root, '', '', None) for name in children}, + stop, None, {}, time.monotonic()) self.assertEqual(stopped, ['a','b','community']) fallback.assert_called_once() self.assertFalse(root.exists()) @@ -161,6 +365,33 @@ def disk(_): self.assertEqual(len(receipt['cleanup_failures']), 2 if disk_fails else 1) self.assertNotIn('disposable test key', json.dumps(receipt)) + def test_sigterm_during_the_journey_reaches_its_cleanup_and_restores_the_handler(self): + with tempfile.TemporaryDirectory() as temporary: + home, root, evidence = (Path(temporary).resolve()/name for name in ('home', 'fixture', 'evidence')) + (home/'data').mkdir(parents=True) + build = JOURNEY['InstalledBuild']('home', home, home/'data', home, self.identity()) + def terminated_while_staging(*_arguments, **_options): + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(1) + raise RuntimeError('SIGTERM did not interrupt the journey') + ignored = [] + operator = lambda *_: ignored.append(True) + previous = signal.signal(signal.SIGTERM, operator) + try: + with mock.patch.dict(JOURNEY['finish_message_fixture'].__globals__, { + 'fresh_fixture':terminated_while_staging, 'require_disk_space':lambda *_: {}}): + with self.assertRaises(JOURNEY['MessageTermination']): + JOURNEY['two_home_messages'](build, evidence, fixture_root=root) + self.assertIs(signal.getsignal(signal.SIGTERM), operator) + finally: + signal.signal(signal.SIGTERM, previous) + self.assertFalse(ignored) + self.assertFalse(root.exists()) + receipt = json.loads((evidence/'installed-journeys.json').read_text()) + self.assertTrue(receipt['fixture_removed']) + self.assertEqual(receipt['failure']['phase'], 'two_home_messages') + self.assertFalse(JOURNEY['two_home_receipt_valid'](receipt, self.identity())) + def test_work_deadline_interrupts_a_hung_phase_and_restores_the_timer(self): before = signal.getsignal(signal.SIGALRM) with self.assertRaises(JOURNEY['MessageDeadline']): @@ -227,30 +458,49 @@ def test_sigkill_fallback_stops_a_detached_browser_after_node_exits(self): parent.kill() parent.wait() + def installed_data(self, data, commit, tree): + (data/'bin').mkdir(parents=True) + (data/'receipts').mkdir() + runtime = data/'bin/elastos' + runtime.write_bytes(b'fixture runtime') + manifest = {'capsules':{}} + for name in ('home','home-gui','people','inbox','chat-room'): + directory = data/'capsules'/name + directory.mkdir(parents=True) + (directory/'index.html').write_text('fixture page') + manifest['capsules'][name] = {'entrypoint':'index.html', + 'entrypoint_sha256':'sha256:'+JOURNEY['digest'](directory/'index.html'), + 'browser_assets':[{'path':'index.html','sha256':'sha256:'+JOURNEY['digest'](directory/'index.html')}]} + (data/'components.json').write_text(json.dumps(manifest)) + (data/'receipts/source-home-capsules.json').write_text('{}') + sha = 'sha256:'+JOURNEY['digest'](runtime) + receipt = {'source':{'commit':commit,'tree':tree,'clean':True}, + 'runtime':{'built_sha256':sha,'installed_sha256':sha,'parity':True}, + 'components_sha256':'sha256:'+JOURNEY['digest'](data/'components.json'), + 'source_home_capsule_metadata_receipt_sha256':'sha256:'+JOURNEY['digest'](data/'receipts/source-home-capsules.json')} + (data/'receipts/source-home-installation.json').write_text(json.dumps(receipt)) + return runtime + + def test_an_installed_build_is_bound_to_its_own_checkout(self): + with tempfile.TemporaryDirectory() as temporary: + repo, home = Path(temporary).resolve()/'released', Path(temporary).resolve()/'home' + repo.mkdir() + git = lambda *arguments: subprocess.check_output(['git', *arguments], cwd=repo, text=True).strip() + git('init', '--quiet') + git('-c', 'user.name=fixture', '-c', 'user.email=fixture@example.invalid', '-c', 'commit.gpgsign=false', + 'commit', '--quiet', '--allow-empty', '--message', 'released fixture') + self.installed_data(home/'data', git('rev-parse', 'HEAD'), git('rev-parse', 'HEAD^{tree}')) + build = JOURNEY['installed_build']('released', home, home/'data', repo) + self.assertEqual((build.name, build.home, build.data, build.repo), ('released', home, home/'data', repo)) + self.assertEqual(build.identity['candidate'], git('rev-parse', 'HEAD')) + with self.assertRaises(AssertionError): + JOURNEY['installed_build']('candidate', home, home/'data') + def test_source_identity_rejects_changed_runtime_and_browser_asset_before_launch(self): identify = JOURNEY['installed_message_identity'] with tempfile.TemporaryDirectory() as temporary: - data = Path(temporary).resolve() - (data/'bin').mkdir() - (data/'receipts').mkdir() - runtime = data/'bin/elastos' - runtime.write_bytes(b'fixture runtime') - manifest = {'capsules':{}} - for name in ('home','home-gui','people','inbox','chat-room'): - directory = data/'capsules'/name - directory.mkdir(parents=True) - (directory/'index.html').write_text('fixture page') - manifest['capsules'][name] = {'entrypoint':'index.html', - 'entrypoint_sha256':'sha256:'+JOURNEY['digest'](directory/'index.html'), - 'browser_assets':[{'path':'index.html','sha256':'sha256:'+JOURNEY['digest'](directory/'index.html')}]} - (data/'components.json').write_text(json.dumps(manifest)) - (data/'receipts/source-home-capsules.json').write_text('{}') - sha = 'sha256:'+JOURNEY['digest'](runtime) - receipt = {'source':{'commit':'c'*40,'tree':'d'*40,'clean':True}, - 'runtime':{'built_sha256':sha,'installed_sha256':sha,'parity':True}, - 'components_sha256':'sha256:'+JOURNEY['digest'](data/'components.json'), - 'source_home_capsule_metadata_receipt_sha256':'sha256:'+JOURNEY['digest'](data/'receipts/source-home-capsules.json')} - (data/'receipts/source-home-installation.json').write_text(json.dumps(receipt)) + data = Path(temporary).resolve()/'data' + runtime = self.installed_data(data, 'c'*40, 'd'*40) def observe(): with mock.patch.object(identify.__globals__['subprocess'], 'check_output', side_effect=['c'*40,'d'*40]): return identify(data) diff --git a/scripts/ci-mixed-version-source-home.sh b/scripts/ci-mixed-version-source-home.sh new file mode 100755 index 000000000..ef5ea5d20 --- /dev/null +++ b/scripts/ci-mixed-version-source-home.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Install a second checkout into its own isolated Home, beside the candidate. +# +# The mixed-version Community gate pairs two Runtime versions, so one job +# installs two source Homes. Both share the job's build directories, so the +# second build reuses every dependency the first one compiled. +set -euo pipefail +umask 022 + +SOURCE="$(cd "${1:?Set the checkout to install from}" && pwd)" +TARGET="${2:?Set the isolated HOME to install into}" +MEDIA_TOOLS_DIR="${TARGET}-media-tools" + +test ! -e "$TARGET" +install -d -m 700 "$TARGET" "$TARGET/.local/share" "$MEDIA_TOOLS_DIR" +for tool in ffmpeg ffprobe; do + install -m 700 "$(command -v "$tool")" "$MEDIA_TOOLS_DIR/$tool" +done + +NODE_BIN="$(command -v node)" +TURNSERVER_BIN="$(command -v turnserver)" + +# Cargo and rustup stay in the runner's own home; only the product installs here. +cd "$SOURCE" +env HOME="$TARGET" XDG_DATA_HOME="$TARGET/.local/share" \ + CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}" \ + PATH="${CARGO_HOME:-$HOME/.cargo}/bin:$PATH" \ + SETUP_SOURCE_HOME_MEDIA_TOOLS_DIR="$MEDIA_TOOLS_DIR" \ + ELASTOS_NODE_BIN="$NODE_BIN" ELASTOS_BROWSER_VM_TURNSERVER_BIN="$TURNSERVER_BIN" \ + SETUP_SOURCE_HOME_INSTALL_LLAMA_SERVER=0 SETUP_SOURCE_HOME_INSTALL_KUBO=0 \ + ELASTOS_QUIET_RUNTIME_NOTICES=1 \ + scripts/setup-source-home.sh +df -Pk . diff --git a/scripts/ci-tools.txt b/scripts/ci-tools.txt new file mode 100644 index 000000000..4199f3616 --- /dev/null +++ b/scripts/ci-tools.txt @@ -0,0 +1,12 @@ +# Programs this repository's own checks start, beyond the base tool set that +# every box installs. One line per program: the program, then the Ubuntu +# package that provides it. A box installs the package only when the program is +# absent, so this list only adds. Write `a|b` when either program will do, and +# `macos` as the package when only a Mac supplies the program. +# +# scripts/setup-source-home.sh refuses to install a source Home without ffmpeg +# and ffprobe, and the Browser VM launcher looks for turnserver. The source-home +# CI job installs the same packages through CI_APT_PACKAGES. +ffmpeg ffmpeg +ffprobe ffmpeg +turnserver coturn diff --git a/scripts/gossip-mixed-version-journey.py b/scripts/gossip-mixed-version-journey.py new file mode 100644 index 000000000..05ee7c706 --- /dev/null +++ b/scripts/gossip-mixed-version-journey.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Community link across Runtime versions: a node and Homes installed from different trees. + +Homes and the Community node update at different times, so a release must +hold for every pairing. Each arrangement runs the installed two-Home journey +with one build as the Community node and one build as the Homes. That journey +owns the transitions, their evidence and the clean-up; this driver chooses the +builds and keeps one receipt for the four arrangements. +""" +import argparse +import json +from pathlib import Path +import runpy + +JOURNEY = runpy.run_path(str(Path(__file__).with_name("ci-installed-journeys.py"))) +BUILDS = ("released", "candidate") +ARRANGEMENTS = [(node, homes) for node in BUILDS for homes in BUILDS] + + +def run_arrangement(node, homes, evidence, fixture_root): + """One node build against one Home build; the journey's own receipt decides the verdict.""" + row = {"node": {"build": node.name, **node.identity}, "homes": {"build": homes.name, **homes.identity}} + try: + JOURNEY["two_home_messages"](homes, evidence, fixture_root=fixture_root, node_build=node) + # A deadline ends one arrangement; SIGTERM and Ctrl-C end the run. + except (Exception, JOURNEY["MessageDeadline"]) as error: + row["failure"] = JOURNEY["observer_failure"]( + "two_home_messages", error, [str(fixture_root), str(node.home), str(homes.home)]) + receipt = evidence / "installed-journeys.json" + row["journey"] = json.loads(receipt.read_text()) if receipt.exists() else {} + row["status"] = "passed" if "failure" not in row and JOURNEY["two_home_receipt_valid"]( + row["journey"], homes.identity, node_identity=node.identity) else "failed" + return row + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + for build in BUILDS: + parser.add_argument(f"--{build}-home", type=Path, required=True, + help=f"isolated HOME of the installed {build} source Home") + parser.add_argument(f"--{build}-data", type=Path, required=True, help="its elastos data root") + parser.add_argument(f"--{build}-repo", type=Path, required=True, help="the checkout it was installed from") + parser.add_argument("--evidence", type=Path, required=True, help="fresh directory for receipts and screenshots") + parser.add_argument("--fixture-root", type=Path, required=True, help="fresh disposable fixture path") + arguments = parser.parse_args(argv) + builds = {build: JOURNEY["installed_build"](build, *(getattr(arguments, f"{build}_{part}") + for part in ("home", "data", "repo"))) for build in BUILDS} + evidence, fixture_root = arguments.evidence.resolve(), arguments.fixture_root.resolve() + evidence.mkdir(parents=True, exist_ok=False) + fixture_root.mkdir(mode=0o700, parents=True, exist_ok=False) + receipt = {"arrangements": []} + try: + with JOURNEY["message_termination"](): + for node, homes in ARRANGEMENTS: + name = f"node-{node}-homes-{homes}" + receipt["arrangements"].append({"arrangement": name, **run_arrangement( + builds[node], builds[homes], evidence / name, fixture_root / name)}) + finally: + # rmdir refuses a directory that still holds a fixture, so a leftover fails the run. + try: + fixture_root.rmdir() + receipt["fixture_root_removed"] = True + except OSError: + receipt["fixture_root_removed"] = False + receipt["status"] = "passed" if ( + receipt["fixture_root_removed"] and len(receipt["arrangements"]) == len(ARRANGEMENTS) + and all(row["status"] == "passed" for row in receipt["arrangements"])) else "failed" + (evidence / "mixed-version-journey.json").write_text(json.dumps(receipt, indent=2) + "\n") + for row in receipt["arrangements"]: + failure = row.get("failure") or row["journey"].get("failure") or {} + print(f"{row['arrangement']}: {row['status']}" + (f" {failure['message']}" if failure else "")) + for name, transition in row["journey"].get("transitions", {}).items(): + homes = ", ".join(f"{home}={'carrier' if state['carrier_ticket_length'] else 'silent'}" + for home, state in transition.get("homes", {}).items()) + print(f" {name}: {homes}; Community posts read {len(transition.get('community', []))}; " + f"direct {'read' if 'direct' in transition else 'missing'}") + print("mixed-version status:", receipt["status"]) + raise SystemExit(0 if receipt["status"] == "passed" else 1) + + +if __name__ == "__main__": + main()