diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9ccfa2974d..1aa11716ab 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -246,15 +246,24 @@ jobs: recipe: rs c-tests # Reports what each fault costs a mesh of real relays, swept over relays and # broadcasts per relay; fails only on an unfed reader, a broadcast shown after its - # source ended, or a mesh that never goes quiet. About 10 minutes a cell. + # source ended, or a mesh that never goes quiet. About 10 minutes a cell. A failed + # run keeps its directory under the hidden .tmp/, so its relay and reader logs ride along. - name: Mesh replay (4 relays) recipe: test mesh-replay --mesh test/mesh-replay/ring4.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay artifact: mesh-replay-ring4 - artifact-path: target/bench/nightly/mesh-replay + hidden-files: true + artifact-path: | + target/bench/nightly/mesh-replay + .tmp/harness/mesh-replay-*/*/*.log + .tmp/harness/mesh-replay-*/*/*.jsonl - name: Mesh replay (6 relays) recipe: test mesh-replay --mesh test/mesh-replay/ring6.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay artifact: mesh-replay-ring6 - artifact-path: target/bench/nightly/mesh-replay + hidden-files: true + artifact-path: | + target/bench/nightly/mesh-replay + .tmp/harness/mesh-replay-*/*/*.log + .tmp/harness/mesh-replay-*/*/*.jsonl steps: - name: Checkout @@ -298,6 +307,7 @@ jobs: with: name: ${{ matrix.artifact }} path: ${{ matrix.artifact-path }} + include-hidden-files: ${{ matrix.hidden-files || false }} retention-days: 45 # Only scheduled runs serve as a later run's base. diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index f0a3d81372..eb9b96798f 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -127,3 +127,11 @@ the base moves past it). Dropped: the aac `step` `av_lag_ms_p50` of 5.1 against 5, a one-night move near the threshold, and the Kotlin wrapper's Gradle plugin resolution failure, a one-off after two passing nights (re-check either if it repeats). + +The mesh replay's violations with no known entry on 2026-11-19 and 11-20 +came from debug-built relays starved for CPU on the 4-core runner, and from +a strand counted again at every later event; both fixed in +[moq-dev/moq#9236](https://github.com/moq-dev/moq/pull/9236). Its `shown` +rows (a relay still showing a broadcast after its source ended) appeared only +in those starved runs and did not reproduce locally. A failed replay job now +uploads its relay and reader logs, so read those if `shown` recurs. diff --git a/quest/m0/nightly/mesh-replay-violations.md b/quest/m0/nightly/mesh-replay-violations.md deleted file mode 100644 index 73da9a9acc..0000000000 --- a/quest/m0/nightly/mesh-replay-violations.md +++ /dev/null @@ -1,24 +0,0 @@ -# [M] The mesh replay's unknown violations are explained - -## Goal - -Nightly's 4- and 6-relay mesh replay jobs report no violation without a known -entry: each new one is fixed at its cause, or listed as known with the quest -that owns it. - -## Plan - -The replay jobs were added by -[moq-dev/moq#9153](https://github.com/moq-dev/moq/pull/9153) and first ran on -2026-11-19 (run [44562804083](https://github.com/moq-dev/moq/actions/runs/44562804083)). About 40 violations are -already listed as known and point at existing quests. These have no entry: - -- ring of 4: `end a's broadcasts: shown x64` -- ring of 6: `reopen link a-b: unfed (stuck) x74 ... ended: internal error` - -The 6-relay rolls that hung (`roll f to new`, `roll b to old`) belong to -[drain-exit](/quest/m0/nightly/drain-exit.md); re-check them once it lands. -Reproduce each with the replay's seed and fix in moq-net, one cause per -commit; if one turns out to be a known quest's case, add the entry instead. - -Public API: none expected. Wire: none expected. diff --git a/quest/m4/route-forward-states.md b/quest/m4/route-forward-states.md index f381eda2d5..37d2af3959 100644 --- a/quest/m4/route-forward-states.md +++ b/quest/m4/route-forward-states.md @@ -15,8 +15,8 @@ Decided 2026-11-20 (maintainer, /quest-plan of the main → release audit): ✅ (rejected: m2, which puts an L routing refactor right before the cut; dropping it and fixing races one replay at a time). [moq-dev/moq#9231](https://github.com/moq-dev/moq/pull/9231) fixed the race the audit found, and -[mesh replay violations](/quest/m0/nightly/mesh-replay-violations.md) owns the replay's open -strands. If that quest traces one to forwarding, move this quest up. +[the nightly record](/quest/m0/nightly/README.md) records the mesh replay's resolved +violations. If a replay traces a new strand to forwarding, move this quest up. The shape today: `Node::forward` (`rs/moq-net/src/route/node.rs`) returns `Option>`, where the outer `None` waits and `Some(None)` refuses, and announce diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index e78e9ebea9..104f681050 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -23,6 +23,10 @@ just test mesh-replay --new ./moq-relay-from-a-branch # roll onto another b ## What runs +This checkout's relay, `moq-bench`, and `moq-shaper` build optimized, like the release +they roll against: debug relays on a 4-core runner fall behind at 32 broadcasts, and the +run then measures the build instead of the protocol. + Every relay listens on loopback with a QUIC idle timeout of 10s and a 10s drain, as the defaults are, and publishes its stats so its `/metrics` counts routes and control messages. A relay dials each link it owns through its own `moq-shaper`, which holds @@ -91,7 +95,9 @@ where a new relay sits between a new one and an old one. - **unfed**: once an event settles, a reader whose relay shows its live broadcast read nothing since the fault ended or for `stall`. `stuck` means its follow holds a broadcast whose track ended and never handed out another; `starved` means it holds - one whose groups stopped. + one whose groups stopped. A reader still unfed from an earlier event, having read + nothing since, is reported as unfed since that event and matched against its known + entries, so one strand fails the run once, not again at every later event. - **unrouted**: at the start, before any fault, a reader's relay does not show a live broadcast, so nothing behind it would be tested. A mesh starting on the new build with a link marked `upstream` may hide a route, so there it is only reported, as after every other diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 5e2ebf8efb..47c0efe0fb 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -669,6 +669,8 @@ def __init__(self, args, mesh: Mesh, script: Script) -> None: self.auth = Auth(self.auth_port, self.dir / "auth.jsonl") self.events: list[dict] = [] self.violations: list[dict] = [] + # Each reader path left unfed, by the event that left it so and its last read then. + self.strands: dict[tuple[str, str], dict] = {} self.stop = threading.Event() self.lock = threading.Lock() self.poller = threading.Thread(target=self.poll, daemon=True) @@ -926,32 +928,53 @@ def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: # A baseline with a reader left unrouted tests nothing behind it. if self.baseline(report["kind"]): self.violation(report, "unrouted", unrouted) - for kind in ("stuck", "starved"): - items = [item for item in unfed if item["kind"] == kind] - if items: - self.violation(report, "unfed", items, kind) + + # A reader still unfed from an earlier event, having read nothing since, is that event's + # strand: it keeps that event's known entry rather than failing every later event again. + # A strand ends only with a read, so one whose source stopped meanwhile is kept for later. + def last(reader: str, path: str) -> float | None: + follow = self.relays[reader].readout.paths.get(path) + return follow.last if follow else None + + strands = {key: began for key, began in self.strands.items() if began["last"] == last(*key)} + groups = {} + for item in unfed: + key = (item["reader"], item["path"]) + began = strands.get(key) or {"name": report["name"], "kind": report["kind"], "last": last(*key)} + strands[key] = began + groups.setdefault((began["name"], item["kind"]), (began, []))[1].append(item) + self.strands = strands + for (name, kind), (began, items) in groups.items(): + self.violation(report, "unfed", items, kind, None if name == report["name"] else began) if shown: self.violation(report, "shown", shown) died = [proc.name for proc in self.procs() if proc.died] if died: self.violation(report, "died", died) - def violation(self, report: dict, check: str, items: list, kind: str | None = None) -> None: + def violation( + self, report: dict, check: str, items: list, kind: str | None = None, began: dict | None = None + ) -> None: + """Record a violation of `report`'s event, or one carried from the earlier event `began`, + which names the event whose known entries it is matched against.""" + event = (began or report)["kind"] # The start is the baseline before any fault, so no known flaw excuses it. known = next( ( k for k in self.script.known - if report["kind"] != "start" + if event != "start" and k["check"] == check and k.get("kind", kind) == kind - and k.get("event", report["kind"]) == report["kind"] + and k.get("event", event) == event ), None, ) entry = {"event": report["name"], "check": check, "count": len(items), "items": items[:20]} if kind: entry["kind"] = kind + if began: + entry["began"] = began["name"] if known: entry["known"] = known["quest"] report.setdefault("violations", []).append(entry) @@ -1275,6 +1298,7 @@ def markdown(report: dict) -> str: for entry in report["violations"] + report["known"]: tag = f" (known: {entry['known']})" if "known" in entry else "" check = entry["check"] + (f" ({entry['kind']})" if entry.get("kind") else "") + check += f" since {entry['began']}" if "began" in entry else "" lines.append(f"- {entry['event']}: {check} x{entry['count']}{tag}: `{json.dumps(entry['items'][:3])}`") lines += ["", NOTE, ""] return "\n".join(lines) diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 149010457f..20480c3a6a 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -215,6 +215,44 @@ def test_a_known_flaw_never_excuses_the_start(self): self.assertNotIn("known", run.violations[0]) self.assertEqual(run.violations[1]["known"], "/q.md") + def test_a_strand_keeps_the_event_it_began_in(self): + follow = replay.Follow(last=5.0) + run = SimpleNamespace( + script=SimpleNamespace(known=[{"check": "unfed", "event": "kill", "quest": "/q.md"}]), + relays={ + "a": SimpleNamespace( + name="a", + readout=SimpleNamespace(paths={"replay/b/0": follow}), + publisher=SimpleNamespace(alive=True), + # Not alive, so the check asks it nothing over HTTP. + proc=SimpleNamespace(alive=False), + ) + }, + procs=lambda: [], + violations=[], + strands={}, + ) + run.violation = lambda *args: replay.Run.violation(run, *args) + unfed = [{"reader": "a", "path": "replay/b/0", "kind": "starved"}] + + def check(kind): + report = {"name": kind, "kind": kind} + replay.Run.check(run, report, unfed, []) + return report["violations"][0] + + self.assertEqual(check("kill")["known"], "/q.md") + # Unread since the kill, so the next event carries the kill's strand and its known entry. + carried = check("end-source") + self.assertEqual((carried["began"], carried["known"]), ("kill", "/q.md")) + # Its source stopped, so it is not unfed; unread still, it carries on once the source is back. + replay.Run.check(run, {"name": "end", "kind": "end-source"}, [], []) + self.assertEqual(check("end-source")["began"], "kill") + # Read once since: a new strand, which no entry excuses. + follow.last = 9.0 + fresh = check("roll") + self.assertNotIn("began", fresh) + self.assertNotIn("known", fresh) + class ProcTest(unittest.TestCase): def test_only_an_exit_the_run_did_not_ask_for_is_a_death(self): diff --git a/test/mesh-replay/run.sh b/test/mesh-replay/run.sh index 15c36546e4..618fd9ba2e 100755 --- a/test/mesh-replay/run.sh +++ b/test/mesh-replay/run.sh @@ -75,11 +75,12 @@ harness_begin mesh-replay "$RERUN" REPORT="${REPORT:-$HARNESS_RUN/report}" # ── binaries ──────────────────────────────────────────────────────────────── +# Optimized, like the release it rolls against; see README.md. cd "$WORKSPACE" -cargo build --locked -p moq-relay -p moq-bench -p moq-shaper +cargo build --locked --release -p moq-relay -p moq-bench -p moq-shaper TARGET=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys;print(json.load(sys.stdin)["target_directory"])') for binary in moq-relay moq-bench moq-shaper; do - harness_stage "new/$binary" "$TARGET/debug/$binary" + harness_stage "new/$binary" "$TARGET/release/$binary" done if [[ -n "$NEW" ]]; then harness_stage new/relay "$NEW"