From 76564a8f722fa18746f60793438544da2d2b25e3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 03:51:11 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m0/test-flakes-3/py-finish-publisher-ack.md Quest-Claim: 143e3975225bea5d From 119e4d22762aea330f2137d7b94ef2408d98f034 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 06:12:00 -0800 Subject: [PATCH 2/4] test(interop): a finish row names the reader or publisher that failed it The harness withholds a finish row's acknowledgement when any reader fails, including one that printed its marker and then exited non-zero, and every cell then reported the same generic detail. That made a reader failure look like a hung publisher. Each cell now reports its own failure, or the one that failed its row, and the python client dumps every thread when timeout sends SIGTERM, so a real hang shows where it waited. Co-Authored-By: Claude Opus 5.5 --- test/interop/clients/python/interop.py | 5 +++ test/interop/interop.sh | 48 +++++++++++++++++--------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/test/interop/clients/python/interop.py b/test/interop/clients/python/interop.py index e759872bfb3..330f47d6808 100644 --- a/test/interop/clients/python/interop.py +++ b/test/interop/clients/python/interop.py @@ -23,8 +23,10 @@ import argparse import asyncio import contextlib +import faulthandler import json import math +import signal import struct import sys from datetime import timedelta @@ -368,6 +370,9 @@ def main() -> None: if args.role == "subscribe" and not args.finish and (not args.audio_out or not args.audio_ms): parser.error("subscribe needs --audio-out and --audio-ms") + # The harness's `timeout` sends SIGTERM to a client that hangs: dump every thread first, so + # the log shows where it waited, then end as SIGTERM would. + faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) # Warnings and errors from the Rust layer, such as a publication outside the grant. moq.log_level("warn") try: diff --git a/test/interop/interop.sh b/test/interop/interop.sh index eb2b2e68e62..b17de1e9cf7 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1773,7 +1773,9 @@ run_finishes() { broadcast="$name-$$-$RANDOM/fixture.hang" grant="${broadcast%/*}/**" ack="$LOGS/$name-ack" - local pids=() fds=() names=() all_ready=1 all_clean=1 publisher_ok=1 + # why[i] is reader i's own failure; row is what failed the whole row, which every + # cell without a failure of its own reports, so a withheld acknowledgement names its cause. + local pids=() fds=() names=() why=() failed=() row="" status late=0 started=$SECONDS echo "=== finished media: $pub ===" @@ -1805,14 +1807,15 @@ run_finishes() { names+=("$sub") done for i in "${!pids[@]}"; do - if ! finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$'; then all_ready=0; fi + finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$' || why[i]="never ready" done - if [[ "$all_ready" -eq 1 ]]; then + pid="" + if [[ ${#why[@]} -eq 0 ]]; then harness_spawn "$name-pub" "$LOGS/$name-pub.log" run_finish_publisher "$pub" "$broadcast" "$(token_url --publish "$grant")" "$ack" pid="$HARNESS_PID" - if ! finish_ready "$pid" "$LOGS/$name-pub.log" '^finish ready linger_us='; then all_ready=0; fi - else pid=""; fi - if [[ "$all_ready" -eq 1 ]]; then + finish_ready "$pid" "$LOGS/$name-pub.log" '^finish ready linger_us=' || row="publisher never ready" + fi + if [[ -z "$row" && ${#why[@]} -eq 0 ]]; then for reader_fd in "${fds[@]}"; do printf 'read\n' >&"$reader_fd"; done if [[ "$late" -eq 1 ]]; then harness_spawn "$name-gst" "$LOGS/$name-gst.log" run_finish_gst "$broadcast" "$(token_url --subscribe "$grant")" "$LOGS/$name-gst.out" @@ -1821,24 +1824,37 @@ run_finishes() { names+=(gst) fi for i in "${!pids[@]}"; do - harness_wait "${pids[$i]}" || all_clean=0 - if ! grep -qx "$(finish_marker "${names[$i]}")" "$LOGS/$name-${names[$i]}.log"; then all_clean=0; fi + status=0 + harness_wait "${pids[$i]}" || status=$? + # A reader that printed its marker can still fail on the way out, such as a + # session drain that times out, and that withholds the acknowledgement too. + if [[ "$status" -ne 0 ]]; then + why[i]="exited $status" + elif ! grep -qx "$(finish_marker "${names[$i]}")" "$LOGS/$name-${names[$i]}.log"; then + why[i]="no finish marker" + fi done - else all_clean=0; fi - if [[ "$all_clean" -eq 1 ]]; then + fi + for i in "${!why[@]}"; do failed+=("${names[$i]}"); done + [[ ${#failed[@]} -eq 0 ]] || row="reader failed: ${failed[*]}" + if [[ -z "$row" ]]; then printf 'complete\n' >&"$ack_fd" exec {ack_fd}>&- - harness_wait "$pid" || publisher_ok=0 - grep -qx 'finish acknowledged' "$LOGS/$name-pub.log" || publisher_ok=0 + status=0 + harness_wait "$pid" || status=$? + if [[ "$status" -ne 0 ]]; then + row="publisher exited $status after the acknowledgement" + elif ! grep -qx 'finish acknowledged' "$LOGS/$name-pub.log"; then + row="publisher did not print finish acknowledged" + fi else - publisher_ok=0 [[ -z "$pid" ]] || harness_reap "$pid" exec {ack_fd}>&- fi for i in "${!pids[@]}"; do sub="${names[$i]}" local cell=(case=finish publisher="$pub" subscriber="$sub" seconds=$((SECONDS - started)) log="$LOGS/$name-$sub.log") - if [[ "$all_ready" -eq 1 && "$publisher_ok" -eq 1 ]] && grep -qx "$(finish_marker "$sub")" "$LOGS/$name-$sub.log"; then + if [[ -z "$row" ]]; then if [[ "$sub" == gst ]]; then echo " PASS finish $pub -> $sub (late tail; end N/A, moqsrc keeps watching)" record pass "${cell[@]}" detail="late tail only; end N/A: moqsrc keeps watching a finished broadcast" @@ -1847,8 +1863,8 @@ run_finishes() { record pass "${cell[@]}" fi else - record fail "${cell[@]}" detail="missing finish tail/EOF/withdrawal/health or acknowledgement" - echo " FAIL finish $pub -> $sub" + record fail "${cell[@]}" detail="${why[$i]:-$row}" + echo " FAIL finish $pub -> $sub (${why[$i]:-$row})" overall=1 cat "$LOGS/$name-$sub.log" "$LOGS/$name-pub.log" 2>/dev/null || true harness_reap "${pids[$i]}" From 286a206a3b4ea62f6eb7170453cd5f98bc5b95ec Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 06:39:40 -0800 Subject: [PATCH 3/4] test(interop): the gst finish reader's max delay covers its late join A finished track's last group ages out its max delay after it ends (#8638), so moqsrc's 1 s default dropped the finished tail whenever gst-launch took longer than that to start. The harness then withheld the publisher's acknowledgement, and the python row looked like a hung publisher. gst now grants its whole read phase, like the other finish readers. Co-Authored-By: Claude Opus 5.5 --- .../test-flakes-3/py-finish-publisher-ack.md | 20 ------------- test/interop/README.md | 8 +++++- test/interop/interop.sh | 28 +++++++++++-------- 3 files changed, 23 insertions(+), 33 deletions(-) delete mode 100644 quest/m0/test-flakes-3/py-finish-publisher-ack.md diff --git a/quest/m0/test-flakes-3/py-finish-publisher-ack.md b/quest/m0/test-flakes-3/py-finish-publisher-ack.md deleted file mode 100644 index cc041e252c8..00000000000 --- a/quest/m0/test-flakes-3/py-finish-publisher-ack.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] The python finish publisher acknowledges every run - -## Goal - -`just test interop --all --finish` passes its `python -> *` row: the python -publisher prints `finish acknowledged` once the harness writes `complete`. - -## Plan - -Seen once on 2026-11-17 while validating -[moq-dev/moq#9052](https://github.com/moq-dev/moq/pull/9052): every reader of the -python publisher passed (`health=ok`), the relay refused nothing, yet the -publisher (`test/interop/clients/python/interop.py`, the `finish_ack` branch) -printed nothing after `finish ready linger_us=` and its session stayed open -until the harness reaped it about two minutes later. A rerun of the row passed. -So it hung after announcing `.health`: in `acknowledge()` reading -the ack FIFO, or in `client.session.state()`. Trace which, under load, and fix -that cause rather than the harness timeout. - -Public API: none. Wire: none. diff --git a/test/interop/README.md b/test/interop/README.md index 543e1a9d357..7e2c9c026a9 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -108,7 +108,13 @@ GStreamer checks the late tail alone. `moqsrc` keeps watching a finished broadcast, since an announcement means online/offline, so it has no end, withdrawal, or session to report. It joins once the broadcast has finished and requires exactly ten nonempty video buffers; its cell passes with the end check -marked N/A. A pinned moq-transport pass records the case as skipped: +marked N/A. A finished track's last group ages out its max delay after it ends, +so like the other finish readers it subscribes with a max delay of its whole +read phase: how late `gst-launch` starts is not the delivery under test. + +The runner releases a publisher's teardown only once every reader exited +cleanly with its marker, so one failed reader fails the whole row. Each failed +cell names its own cause, or the reader or publisher that failed its row. A pinned moq-transport pass records the case as skipped: moq-transport is a compatibility layer, and a finished broadcast's late tail between our own clients is not planned work. diff --git a/test/interop/interop.sh b/test/interop/interop.sh index b17de1e9cf7..bdd8c62a066 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1689,13 +1689,17 @@ run_finish_publisher() { # leaving a WebSocket fallback that won a slow race would move it to a second session mid-read. # shellcheck disable=SC2329 # invoked indirectly via harness_spawn run_finish_gst() { - local broadcast="$1" url="$2" out="$3" version pid deadline count + local broadcast="$1" url="$2" out="$3" version pid deadline budget count version=$(cell_version gst) # The read phase's deadline, as the other readers get; SECONDS counts whole seconds. deadline=$((SECONDS + ${TIMEOUT%.*} * 2)) [[ "$TIMEOUT" != *.* ]] || deadline=$((deadline + 2)) + # A finished track's last group ages out its max delay after it ends, so moqsrc's 1 s default + # drops the tail whenever gst-launch takes longer than that to start. Like the other finish + # readers it grants the whole read phase: how late it joins is not the delivery under test. + budget=$(awk -v t="$TIMEOUT" 'BEGIN { print int(t * 2000) }') GST_PLUGIN_PATH_1_0="$GST_PLUGIN_DIR" GST_REGISTRY_1_0="$HARNESS_RUN/gst-run-registry.bin" \ - gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" ${version:+connect-version="$version"} connect-websocket-enabled=false \ + gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" max-delay="$budget" ${version:+connect-version="$version"} connect-websocket-enabled=false \ s.video_0 ! identity name=frames silent=false ! fakesink >"$out" 2>&1 & pid=$! count=0 @@ -1773,9 +1777,9 @@ run_finishes() { broadcast="$name-$$-$RANDOM/fixture.hang" grant="${broadcast%/*}/**" ack="$LOGS/$name-ack" - # why[i] is reader i's own failure; row is what failed the whole row, which every + # causes[i] is reader i's own failure; row is what failed the whole row, which every # cell without a failure of its own reports, so a withheld acknowledgement names its cause. - local pids=() fds=() names=() why=() failed=() row="" status + local pids=() fds=() names=() causes=() failed=() row="" status late=0 started=$SECONDS echo "=== finished media: $pub ===" @@ -1807,15 +1811,15 @@ run_finishes() { names+=("$sub") done for i in "${!pids[@]}"; do - finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$' || why[i]="never ready" + finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$' || causes[i]="never ready" done pid="" - if [[ ${#why[@]} -eq 0 ]]; then + if [[ ${#causes[@]} -eq 0 ]]; then harness_spawn "$name-pub" "$LOGS/$name-pub.log" run_finish_publisher "$pub" "$broadcast" "$(token_url --publish "$grant")" "$ack" pid="$HARNESS_PID" finish_ready "$pid" "$LOGS/$name-pub.log" '^finish ready linger_us=' || row="publisher never ready" fi - if [[ -z "$row" && ${#why[@]} -eq 0 ]]; then + if [[ -z "$row" && ${#causes[@]} -eq 0 ]]; then for reader_fd in "${fds[@]}"; do printf 'read\n' >&"$reader_fd"; done if [[ "$late" -eq 1 ]]; then harness_spawn "$name-gst" "$LOGS/$name-gst.log" run_finish_gst "$broadcast" "$(token_url --subscribe "$grant")" "$LOGS/$name-gst.out" @@ -1829,13 +1833,13 @@ run_finishes() { # A reader that printed its marker can still fail on the way out, such as a # session drain that times out, and that withholds the acknowledgement too. if [[ "$status" -ne 0 ]]; then - why[i]="exited $status" + causes[i]="exited $status" elif ! grep -qx "$(finish_marker "${names[$i]}")" "$LOGS/$name-${names[$i]}.log"; then - why[i]="no finish marker" + causes[i]="no finish marker" fi done fi - for i in "${!why[@]}"; do failed+=("${names[$i]}"); done + for i in "${!causes[@]}"; do failed+=("${names[$i]}"); done [[ ${#failed[@]} -eq 0 ]] || row="reader failed: ${failed[*]}" if [[ -z "$row" ]]; then printf 'complete\n' >&"$ack_fd" @@ -1863,8 +1867,8 @@ run_finishes() { record pass "${cell[@]}" fi else - record fail "${cell[@]}" detail="${why[$i]:-$row}" - echo " FAIL finish $pub -> $sub (${why[$i]:-$row})" + record fail "${cell[@]}" detail="${causes[$i]:-$row}" + echo " FAIL finish $pub -> $sub (${causes[$i]:-$row})" overall=1 cat "$LOGS/$name-$sub.log" "$LOGS/$name-pub.log" 2>/dev/null || true harness_reap "${pids[$i]}" From cd2334950b08e5931201e9358913db39f774e876 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 11:50:15 -0800 Subject: [PATCH 4/4] test(interop): gst's deadline and max delay share one window, and a gated gst still gets its failed cell Co-Authored-By: Claude Opus 5.5 --- test/interop/README.md | 15 ++++++++------- test/interop/interop.sh | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/test/interop/README.md b/test/interop/README.md index 7e2c9c026a9..2ce1c035f52 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -110,13 +110,14 @@ withdrawal, or session to report. It joins once the broadcast has finished and requires exactly ten nonempty video buffers; its cell passes with the end check marked N/A. A finished track's last group ages out its max delay after it ends, so like the other finish readers it subscribes with a max delay of its whole -read phase: how late `gst-launch` starts is not the delivery under test. - -The runner releases a publisher's teardown only once every reader exited -cleanly with its marker, so one failed reader fails the whole row. Each failed -cell names its own cause, or the reader or publisher that failed its row. A pinned moq-transport pass records the case as skipped: -moq-transport is a compatibility layer, and a finished broadcast's late tail -between our own clients is not planned work. +read phase: how late `gst-launch` starts is not the delivery under test. A +pinned moq-transport pass records the case as skipped: moq-transport is a +compatibility layer, and a finished broadcast's late tail between our own +clients is not planned work. + +The runner releases a publisher's teardown only once every reader it started +exited cleanly with its marker, so one failed reader fails the whole row. Each +failed cell names its own cause, or the reader or publisher that failed its row. Every client closes its session on the way out: publishers stop when their input ends (the browser on SIGTERM), and subscribers close once they have their frame. diff --git a/test/interop/interop.sh b/test/interop/interop.sh index bdd8c62a066..0530cc4e64d 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1689,17 +1689,18 @@ run_finish_publisher() { # leaving a WebSocket fallback that won a slow race would move it to a second session mid-read. # shellcheck disable=SC2329 # invoked indirectly via harness_spawn run_finish_gst() { - local broadcast="$1" url="$2" out="$3" version pid deadline budget count + local broadcast="$1" url="$2" out="$3" version pid window deadline count version=$(cell_version gst) - # The read phase's deadline, as the other readers get; SECONDS counts whole seconds. - deadline=$((SECONDS + ${TIMEOUT%.*} * 2)) - [[ "$TIMEOUT" != *.* ]] || deadline=$((deadline + 2)) + # The read phase, as the other readers get; SECONDS counts whole seconds, so a fractional + # timeout rounds up. + window=$((${TIMEOUT%.*} * 2)) + [[ "$TIMEOUT" != *.* ]] || window=$((window + 2)) + deadline=$((SECONDS + window)) # A finished track's last group ages out its max delay after it ends, so moqsrc's 1 s default # drops the tail whenever gst-launch takes longer than that to start. Like the other finish # readers it grants the whole read phase: how late it joins is not the delivery under test. - budget=$(awk -v t="$TIMEOUT" 'BEGIN { print int(t * 2000) }') GST_PLUGIN_PATH_1_0="$GST_PLUGIN_DIR" GST_REGISTRY_1_0="$HARNESS_RUN/gst-run-registry.bin" \ - gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" max-delay="$budget" ${version:+connect-version="$version"} connect-websocket-enabled=false \ + gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" max-delay="$((window * 1000))" ${version:+connect-version="$version"} connect-websocket-enabled=false \ s.video_0 ! identity name=frames silent=false ! fakesink >"$out" 2>&1 & pid=$! count=0 @@ -1838,6 +1839,11 @@ run_finishes() { causes[i]="no finish marker" fi done + elif [[ "$late" -eq 1 ]]; then + # gst never started, but its cell still fails with the row's cause. + pids+=("") + fds+=("") + names+=(gst) fi for i in "${!causes[@]}"; do failed+=("${names[$i]}"); done [[ ${#failed[@]} -eq 0 ]] || row="reader failed: ${failed[*]}"