From 1edb02285c96aa985293996176d598b66a280dff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 10:18:56 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m0/nightly/mesh-replay-violations.md Quest-Claim: 61864fdcd5bcdb46 From 17a932a51c077694ea570bd4ef6150492345903d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 11:50:55 -0800 Subject: [PATCH 2/8] fix(test): the mesh replay builds optimized and reports a strand once Debug relays on the 4-core nightly runner fall behind at 32 broadcasts once rolled in, so every reader starved. A reader still unfed from an earlier event now keeps that event's known entry instead of failing every later one. Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/README.md | 8 +++++++- test/mesh-replay/replay.py | 35 ++++++++++++++++++++++++++------- test/mesh-replay/replay.test.py | 35 +++++++++++++++++++++++++++++++++ test/mesh-replay/run.sh | 5 +++-- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index e78e9ebea9..104f681050 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -23,6 +23,10 @@ just test mesh-replay --new ./moq-relay-from-a-branch # roll onto another b ## What runs +This checkout's relay, `moq-bench`, and `moq-shaper` build optimized, like the release +they roll against: debug relays on a 4-core runner fall behind at 32 broadcasts, and the +run then measures the build instead of the protocol. + Every relay listens on loopback with a QUIC idle timeout of 10s and a 10s drain, as the defaults are, and publishes its stats so its `/metrics` counts routes and control messages. A relay dials each link it owns through its own `moq-shaper`, which holds @@ -91,7 +95,9 @@ where a new relay sits between a new one and an old one. - **unfed**: once an event settles, a reader whose relay shows its live broadcast read nothing since the fault ended or for `stall`. `stuck` means its follow holds a broadcast whose track ended and never handed out another; `starved` means it holds - one whose groups stopped. + one whose groups stopped. A reader still unfed from an earlier event, having read + nothing since, is reported as unfed since that event and matched against its known + entries, so one strand fails the run once, not again at every later event. - **unrouted**: at the start, before any fault, a reader's relay does not show a live broadcast, so nothing behind it would be tested. A mesh starting on the new build with a link marked `upstream` may hide a route, so there it is only reported, as after every other diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 5e2ebf8efb..a75a15cc2b 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -669,6 +669,8 @@ def __init__(self, args, mesh: Mesh, script: Script) -> None: self.auth = Auth(self.auth_port, self.dir / "auth.jsonl") self.events: list[dict] = [] self.violations: list[dict] = [] + # Each reader path left unfed, by the event that left it so and its last read then. + self.strands: dict[tuple[str, str], dict] = {} self.stop = threading.Event() self.lock = threading.Lock() self.poller = threading.Thread(target=self.poll, daemon=True) @@ -926,32 +928,50 @@ def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: # A baseline with a reader left unrouted tests nothing behind it. if self.baseline(report["kind"]): self.violation(report, "unrouted", unrouted) - for kind in ("stuck", "starved"): - items = [item for item in unfed if item["kind"] == kind] - if items: - self.violation(report, "unfed", items, kind) + # A reader still unfed from an earlier event, having read nothing since, is that event's + # strand: it keeps that event's known entry rather than failing every later event again. + strands, groups = {}, {} + for item in unfed: + key = (item["reader"], item["path"]) + follow = self.relays[item["reader"]].readout.paths.get(item["path"]) + last = follow.last if follow else None + began = self.strands.get(key) + if began is None or began["last"] != last: + began = {"name": report["name"], "kind": report["kind"], "last": last} + strands[key] = began + groups.setdefault((began["name"], item["kind"]), (began, []))[1].append(item) + self.strands = strands + for (name, kind), (began, items) in groups.items(): + self.violation(report, "unfed", items, kind, None if name == report["name"] else began) if shown: self.violation(report, "shown", shown) died = [proc.name for proc in self.procs() if proc.died] if died: self.violation(report, "died", died) - def violation(self, report: dict, check: str, items: list, kind: str | None = None) -> None: + def violation( + self, report: dict, check: str, items: list, kind: str | None = None, began: dict | None = None + ) -> None: + """Record a violation of `report`'s event, or one carried from the earlier event `began`, + which names the event whose known entries it is matched against.""" + event = (began or report)["kind"] # The start is the baseline before any fault, so no known flaw excuses it. known = next( ( k for k in self.script.known - if report["kind"] != "start" + if event != "start" and k["check"] == check and k.get("kind", kind) == kind - and k.get("event", report["kind"]) == report["kind"] + and k.get("event", event) == event ), None, ) entry = {"event": report["name"], "check": check, "count": len(items), "items": items[:20]} if kind: entry["kind"] = kind + if began: + entry["began"] = began["name"] if known: entry["known"] = known["quest"] report.setdefault("violations", []).append(entry) @@ -1275,6 +1295,7 @@ def markdown(report: dict) -> str: for entry in report["violations"] + report["known"]: tag = f" (known: {entry['known']})" if "known" in entry else "" check = entry["check"] + (f" ({entry['kind']})" if entry.get("kind") else "") + check += f" since {entry['began']}" if "began" in entry else "" lines.append(f"- {entry['event']}: {check} x{entry['count']}{tag}: `{json.dumps(entry['items'][:3])}`") lines += ["", NOTE, ""] return "\n".join(lines) diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 149010457f..395e8171e6 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -215,6 +215,41 @@ def test_a_known_flaw_never_excuses_the_start(self): self.assertNotIn("known", run.violations[0]) self.assertEqual(run.violations[1]["known"], "/q.md") + def test_a_strand_keeps_the_event_it_began_in(self): + follow = replay.Follow(last=5.0) + run = SimpleNamespace( + script=SimpleNamespace(known=[{"check": "unfed", "event": "kill", "quest": "/q.md"}]), + relays={ + "a": SimpleNamespace( + name="a", + readout=SimpleNamespace(paths={"replay/b/0": follow}), + publisher=SimpleNamespace(alive=True), + # Not alive, so the check asks it nothing over HTTP. + proc=SimpleNamespace(alive=False), + ) + }, + procs=lambda: [], + violations=[], + strands={}, + ) + run.violation = lambda *args: replay.Run.violation(run, *args) + unfed = [{"reader": "a", "path": "replay/b/0", "kind": "starved"}] + + def check(kind): + report = {"name": kind, "kind": kind} + replay.Run.check(run, report, unfed, []) + return report["violations"][0] + + self.assertEqual(check("kill")["known"], "/q.md") + # Unread since the kill, so the next event carries the kill's strand and its known entry. + carried = check("end-source") + self.assertEqual((carried["began"], carried["known"]), ("kill", "/q.md")) + # Read once since: a new strand, which no entry excuses. + follow.last = 9.0 + fresh = check("roll") + self.assertNotIn("began", fresh) + self.assertNotIn("known", fresh) + class ProcTest(unittest.TestCase): def test_only_an_exit_the_run_did_not_ask_for_is_a_death(self): diff --git a/test/mesh-replay/run.sh b/test/mesh-replay/run.sh index 15c36546e4..618fd9ba2e 100755 --- a/test/mesh-replay/run.sh +++ b/test/mesh-replay/run.sh @@ -75,11 +75,12 @@ harness_begin mesh-replay "$RERUN" REPORT="${REPORT:-$HARNESS_RUN/report}" # ── binaries ──────────────────────────────────────────────────────────────── +# Optimized, like the release it rolls against; see README.md. cd "$WORKSPACE" -cargo build --locked -p moq-relay -p moq-bench -p moq-shaper +cargo build --locked --release -p moq-relay -p moq-bench -p moq-shaper TARGET=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys;print(json.load(sys.stdin)["target_directory"])') for binary in moq-relay moq-bench moq-shaper; do - harness_stage "new/$binary" "$TARGET/debug/$binary" + harness_stage "new/$binary" "$TARGET/release/$binary" done if [[ -n "$NEW" ]]; then harness_stage new/relay "$NEW" From 68bf33ba475c45086a8415135c39c50193b386ad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 12:10:09 -0800 Subject: [PATCH 3/8] quest: the mesh replay's unknown violations are explained Nightly uploads a failed replay's relay and reader logs, since the shown rows did not reproduce locally. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 13 +++++++++--- quest/m0/nightly/README.md | 8 ++++++++ quest/m0/nightly/mesh-replay-violations.md | 24 ---------------------- 3 files changed, 18 insertions(+), 27 deletions(-) delete mode 100644 quest/m0/nightly/mesh-replay-violations.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9ccfa2974d..639eb21532 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -246,15 +246,22 @@ jobs: recipe: rs c-tests # Reports what each fault costs a mesh of real relays, swept over relays and # broadcasts per relay; fails only on an unfed reader, a broadcast shown after its - # source ended, or a mesh that never goes quiet. About 10 minutes a cell. + # source ended, or a mesh that never goes quiet. About 10 minutes a cell. A failed + # run keeps its directory, so its relay and reader logs ride along. - name: Mesh replay (4 relays) recipe: test mesh-replay --mesh test/mesh-replay/ring4.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay artifact: mesh-replay-ring4 - artifact-path: target/bench/nightly/mesh-replay + artifact-path: | + target/bench/nightly/mesh-replay + .tmp/harness/mesh-replay-*/*/*.log + .tmp/harness/mesh-replay-*/*/*.jsonl - name: Mesh replay (6 relays) recipe: test mesh-replay --mesh test/mesh-replay/ring6.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay artifact: mesh-replay-ring6 - artifact-path: target/bench/nightly/mesh-replay + artifact-path: | + target/bench/nightly/mesh-replay + .tmp/harness/mesh-replay-*/*/*.log + .tmp/harness/mesh-replay-*/*/*.jsonl steps: - name: Checkout diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index f0a3d81372..eb9b96798f 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -127,3 +127,11 @@ the base moves past it). Dropped: the aac `step` `av_lag_ms_p50` of 5.1 against 5, a one-night move near the threshold, and the Kotlin wrapper's Gradle plugin resolution failure, a one-off after two passing nights (re-check either if it repeats). + +The mesh replay's violations with no known entry on 2026-11-19 and 11-20 +came from debug-built relays starved for CPU on the 4-core runner, and from +a strand counted again at every later event; both fixed in +[moq-dev/moq#9236](https://github.com/moq-dev/moq/pull/9236). Its `shown` +rows (a relay still showing a broadcast after its source ended) appeared only +in those starved runs and did not reproduce locally. A failed replay job now +uploads its relay and reader logs, so read those if `shown` recurs. diff --git a/quest/m0/nightly/mesh-replay-violations.md b/quest/m0/nightly/mesh-replay-violations.md deleted file mode 100644 index 73da9a9acc..0000000000 --- a/quest/m0/nightly/mesh-replay-violations.md +++ /dev/null @@ -1,24 +0,0 @@ -# [M] The mesh replay's unknown violations are explained - -## Goal - -Nightly's 4- and 6-relay mesh replay jobs report no violation without a known -entry: each new one is fixed at its cause, or listed as known with the quest -that owns it. - -## Plan - -The replay jobs were added by -[moq-dev/moq#9153](https://github.com/moq-dev/moq/pull/9153) and first ran on -2026-11-19 (run [44562804083](https://github.com/moq-dev/moq/actions/runs/44562804083)). About 40 violations are -already listed as known and point at existing quests. These have no entry: - -- ring of 4: `end a's broadcasts: shown x64` -- ring of 6: `reopen link a-b: unfed (stuck) x74 ... ended: internal error` - -The 6-relay rolls that hung (`roll f to new`, `roll b to old`) belong to -[drain-exit](/quest/m0/nightly/drain-exit.md); re-check them once it lands. -Reproduce each with the replay's seed and fix in moq-net, one cause per -commit; if one turns out to be a known quest's case, add the entry instead. - -Public API: none expected. Wire: none expected. From a9c3178974267fb813fc5fd2a6d546a96cbacfc9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 12:13:46 -0800 Subject: [PATCH 4/8] quest: a stuck video source's arrival timestamps are whole microseconds Co-Authored-By: Claude Opus 5.5 --- .../test-flakes-3/publish-arrival-micros.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 quest/m0/test-flakes-3/publish-arrival-micros.md diff --git a/quest/m0/test-flakes-3/publish-arrival-micros.md b/quest/m0/test-flakes-3/publish-arrival-micros.md new file mode 100644 index 0000000000..b94d79c066 --- /dev/null +++ b/quest/m0/test-flakes-3/publish-arrival-micros.md @@ -0,0 +1,21 @@ +# [XS] A stuck video source's arrival timestamps are whole microseconds + +## Goal + +`@moq/publish`'s `falls back to arrival time when the source clock is stuck` +(`js/publish/src/video/processor.test.ts`) passes every run, and the frames +the fallback stamps carry whole-microsecond timestamps. + +## Plan + +Seen 2026-11-20 on `main` (9828a20cab): `bun test src/video/processor.test.ts` +in `js/publish` failed 3 of 5 runs, with a delta of `10000.000000000233` +against `10000`. The arrival comes from `msg.at - performance.timeOrigin`, two +Unix-scale millisecond floats, so the microseconds `MediaClock.capture(at).asMicros()` +returns carry float error that depends on the wall clock. A `VideoFrame` +timestamp is an integer, so a browser truncates it, and a frame can land 1µs +early. Round to whole microseconds where the processor stamps a frame (the +worker path and the `