diff --git a/quest/m0/test-flakes-3/py-finish-publisher-ack.md b/quest/m0/test-flakes-3/py-finish-publisher-ack.md deleted file mode 100644 index cc041e252c..0000000000 --- a/quest/m0/test-flakes-3/py-finish-publisher-ack.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] The python finish publisher acknowledges every run - -## Goal - -`just test interop --all --finish` passes its `python -> *` row: the python -publisher prints `finish acknowledged` once the harness writes `complete`. - -## Plan - -Seen once on 2026-11-17 while validating -[moq-dev/moq#9052](https://github.com/moq-dev/moq/pull/9052): every reader of the -python publisher passed (`health=ok`), the relay refused nothing, yet the -publisher (`test/interop/clients/python/interop.py`, the `finish_ack` branch) -printed nothing after `finish ready linger_us=` and its session stayed open -until the harness reaped it about two minutes later. A rerun of the row passed. -So it hung after announcing `.health`: in `acknowledge()` reading -the ack FIFO, or in `client.session.state()`. Trace which, under load, and fix -that cause rather than the harness timeout. - -Public API: none. Wire: none. diff --git a/test/interop/README.md b/test/interop/README.md index 543e1a9d35..2ce1c035f5 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -108,9 +108,16 @@ GStreamer checks the late tail alone. `moqsrc` keeps watching a finished broadcast, since an announcement means online/offline, so it has no end, withdrawal, or session to report. It joins once the broadcast has finished and requires exactly ten nonempty video buffers; its cell passes with the end check -marked N/A. A pinned moq-transport pass records the case as skipped: -moq-transport is a compatibility layer, and a finished broadcast's late tail -between our own clients is not planned work. +marked N/A. A finished track's last group ages out its max delay after it ends, +so like the other finish readers it subscribes with a max delay of its whole +read phase: how late `gst-launch` starts is not the delivery under test. A +pinned moq-transport pass records the case as skipped: moq-transport is a +compatibility layer, and a finished broadcast's late tail between our own +clients is not planned work. + +The runner releases a publisher's teardown only once every reader it started +exited cleanly with its marker, so one failed reader fails the whole row. Each +failed cell names its own cause, or the reader or publisher that failed its row. Every client closes its session on the way out: publishers stop when their input ends (the browser on SIGTERM), and subscribers close once they have their frame. diff --git a/test/interop/clients/python/interop.py b/test/interop/clients/python/interop.py index e759872bfb..330f47d680 100644 --- a/test/interop/clients/python/interop.py +++ b/test/interop/clients/python/interop.py @@ -23,8 +23,10 @@ import argparse import asyncio import contextlib +import faulthandler import json import math +import signal import struct import sys from datetime import timedelta @@ -368,6 +370,9 @@ def main() -> None: if args.role == "subscribe" and not args.finish and (not args.audio_out or not args.audio_ms): parser.error("subscribe needs --audio-out and --audio-ms") + # The harness's `timeout` sends SIGTERM to a client that hangs: dump every thread first, so + # the log shows where it waited, then end as SIGTERM would. + faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) # Warnings and errors from the Rust layer, such as a publication outside the grant. moq.log_level("warn") try: diff --git a/test/interop/interop.sh b/test/interop/interop.sh index eb2b2e68e6..0530cc4e64 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -1689,13 +1689,18 @@ run_finish_publisher() { # leaving a WebSocket fallback that won a slow race would move it to a second session mid-read. # shellcheck disable=SC2329 # invoked indirectly via harness_spawn run_finish_gst() { - local broadcast="$1" url="$2" out="$3" version pid deadline count + local broadcast="$1" url="$2" out="$3" version pid window deadline count version=$(cell_version gst) - # The read phase's deadline, as the other readers get; SECONDS counts whole seconds. - deadline=$((SECONDS + ${TIMEOUT%.*} * 2)) - [[ "$TIMEOUT" != *.* ]] || deadline=$((deadline + 2)) + # The read phase, as the other readers get; SECONDS counts whole seconds, so a fractional + # timeout rounds up. + window=$((${TIMEOUT%.*} * 2)) + [[ "$TIMEOUT" != *.* ]] || window=$((window + 2)) + deadline=$((SECONDS + window)) + # A finished track's last group ages out its max delay after it ends, so moqsrc's 1 s default + # drops the tail whenever gst-launch takes longer than that to start. Like the other finish + # readers it grants the whole read phase: how late it joins is not the delivery under test. GST_PLUGIN_PATH_1_0="$GST_PLUGIN_DIR" GST_REGISTRY_1_0="$HARNESS_RUN/gst-run-registry.bin" \ - gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" ${version:+connect-version="$version"} connect-websocket-enabled=false \ + gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" max-delay="$((window * 1000))" ${version:+connect-version="$version"} connect-websocket-enabled=false \ s.video_0 ! identity name=frames silent=false ! fakesink >"$out" 2>&1 & pid=$! count=0 @@ -1773,7 +1778,9 @@ run_finishes() { broadcast="$name-$$-$RANDOM/fixture.hang" grant="${broadcast%/*}/**" ack="$LOGS/$name-ack" - local pids=() fds=() names=() all_ready=1 all_clean=1 publisher_ok=1 + # causes[i] is reader i's own failure; row is what failed the whole row, which every + # cell without a failure of its own reports, so a withheld acknowledgement names its cause. + local pids=() fds=() names=() causes=() failed=() row="" status late=0 started=$SECONDS echo "=== finished media: $pub ===" @@ -1805,14 +1812,15 @@ run_finishes() { names+=("$sub") done for i in "${!pids[@]}"; do - if ! finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$'; then all_ready=0; fi + finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$' || causes[i]="never ready" done - if [[ "$all_ready" -eq 1 ]]; then + pid="" + if [[ ${#causes[@]} -eq 0 ]]; then harness_spawn "$name-pub" "$LOGS/$name-pub.log" run_finish_publisher "$pub" "$broadcast" "$(token_url --publish "$grant")" "$ack" pid="$HARNESS_PID" - if ! finish_ready "$pid" "$LOGS/$name-pub.log" '^finish ready linger_us='; then all_ready=0; fi - else pid=""; fi - if [[ "$all_ready" -eq 1 ]]; then + finish_ready "$pid" "$LOGS/$name-pub.log" '^finish ready linger_us=' || row="publisher never ready" + fi + if [[ -z "$row" && ${#causes[@]} -eq 0 ]]; then for reader_fd in "${fds[@]}"; do printf 'read\n' >&"$reader_fd"; done if [[ "$late" -eq 1 ]]; then harness_spawn "$name-gst" "$LOGS/$name-gst.log" run_finish_gst "$broadcast" "$(token_url --subscribe "$grant")" "$LOGS/$name-gst.out" @@ -1821,24 +1829,42 @@ run_finishes() { names+=(gst) fi for i in "${!pids[@]}"; do - harness_wait "${pids[$i]}" || all_clean=0 - if ! grep -qx "$(finish_marker "${names[$i]}")" "$LOGS/$name-${names[$i]}.log"; then all_clean=0; fi + status=0 + harness_wait "${pids[$i]}" || status=$? + # A reader that printed its marker can still fail on the way out, such as a + # session drain that times out, and that withholds the acknowledgement too. + if [[ "$status" -ne 0 ]]; then + causes[i]="exited $status" + elif ! grep -qx "$(finish_marker "${names[$i]}")" "$LOGS/$name-${names[$i]}.log"; then + causes[i]="no finish marker" + fi done - else all_clean=0; fi - if [[ "$all_clean" -eq 1 ]]; then + elif [[ "$late" -eq 1 ]]; then + # gst never started, but its cell still fails with the row's cause. + pids+=("") + fds+=("") + names+=(gst) + fi + for i in "${!causes[@]}"; do failed+=("${names[$i]}"); done + [[ ${#failed[@]} -eq 0 ]] || row="reader failed: ${failed[*]}" + if [[ -z "$row" ]]; then printf 'complete\n' >&"$ack_fd" exec {ack_fd}>&- - harness_wait "$pid" || publisher_ok=0 - grep -qx 'finish acknowledged' "$LOGS/$name-pub.log" || publisher_ok=0 + status=0 + harness_wait "$pid" || status=$? + if [[ "$status" -ne 0 ]]; then + row="publisher exited $status after the acknowledgement" + elif ! grep -qx 'finish acknowledged' "$LOGS/$name-pub.log"; then + row="publisher did not print finish acknowledged" + fi else - publisher_ok=0 [[ -z "$pid" ]] || harness_reap "$pid" exec {ack_fd}>&- fi for i in "${!pids[@]}"; do sub="${names[$i]}" local cell=(case=finish publisher="$pub" subscriber="$sub" seconds=$((SECONDS - started)) log="$LOGS/$name-$sub.log") - if [[ "$all_ready" -eq 1 && "$publisher_ok" -eq 1 ]] && grep -qx "$(finish_marker "$sub")" "$LOGS/$name-$sub.log"; then + if [[ -z "$row" ]]; then if [[ "$sub" == gst ]]; then echo " PASS finish $pub -> $sub (late tail; end N/A, moqsrc keeps watching)" record pass "${cell[@]}" detail="late tail only; end N/A: moqsrc keeps watching a finished broadcast" @@ -1847,8 +1873,8 @@ run_finishes() { record pass "${cell[@]}" fi else - record fail "${cell[@]}" detail="missing finish tail/EOF/withdrawal/health or acknowledgement" - echo " FAIL finish $pub -> $sub" + record fail "${cell[@]}" detail="${causes[$i]:-$row}" + echo " FAIL finish $pub -> $sub (${causes[$i]:-$row})" overall=1 cat "$LOGS/$name-$sub.log" "$LOGS/$name-pub.log" 2>/dev/null || true harness_reap "${pids[$i]}"