Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions quest/m0/test-flakes-3/py-finish-publisher-ack.md

This file was deleted.

13 changes: 10 additions & 3 deletions test/interop/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,16 @@ GStreamer checks the late tail alone. `moqsrc` keeps watching a finished
broadcast, since an announcement means online/offline, so it has no end,
withdrawal, or session to report. It joins once the broadcast has finished and
requires exactly ten nonempty video buffers; its cell passes with the end check
marked N/A. A pinned moq-transport pass records the case as skipped:
moq-transport is a compatibility layer, and a finished broadcast's late tail
between our own clients is not planned work.
marked N/A. A finished track's last group ages out its max delay after it ends,
so like the other finish readers it subscribes with a max delay of its whole
read phase: how late `gst-launch` starts is not the delivery under test. A
pinned moq-transport pass records the case as skipped: moq-transport is a
compatibility layer, and a finished broadcast's late tail between our own
clients is not planned work.

The runner releases a publisher's teardown only once every reader it started
exited cleanly with its marker, so one failed reader fails the whole row. Each
failed cell names its own cause, or the reader or publisher that failed its row.

Every client closes its session on the way out: publishers stop when their input
ends (the browser on SIGTERM), and subscribers close once they have their frame.
Expand Down
5 changes: 5 additions & 0 deletions test/interop/clients/python/interop.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
import argparse
import asyncio
import contextlib
import faulthandler
import json
import math
import signal
import struct
import sys
from datetime import timedelta
Expand Down Expand Up @@ -368,6 +370,9 @@ def main() -> None:
if args.role == "subscribe" and not args.finish and (not args.audio_out or not args.audio_ms):
parser.error("subscribe needs --audio-out and --audio-ms")

# The harness's `timeout` sends SIGTERM to a client that hangs: dump every thread first, so
# the log shows where it waited, then end as SIGTERM would.
faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
# Warnings and errors from the Rust layer, such as a publication outside the grant.
moq.log_level("warn")
try:
Expand Down
68 changes: 47 additions & 21 deletions test/interop/interop.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1689,13 +1689,18 @@ run_finish_publisher() {
# leaving a WebSocket fallback that won a slow race would move it to a second session mid-read.
# shellcheck disable=SC2329 # invoked indirectly via harness_spawn
run_finish_gst() {
local broadcast="$1" url="$2" out="$3" version pid deadline count
local broadcast="$1" url="$2" out="$3" version pid window deadline count
version=$(cell_version gst)
# The read phase's deadline, as the other readers get; SECONDS counts whole seconds.
deadline=$((SECONDS + ${TIMEOUT%.*} * 2))
[[ "$TIMEOUT" != *.* ]] || deadline=$((deadline + 2))
# The read phase, as the other readers get; SECONDS counts whole seconds, so a fractional
# timeout rounds up.
window=$((${TIMEOUT%.*} * 2))
[[ "$TIMEOUT" != *.* ]] || window=$((window + 2))
deadline=$((SECONDS + window))
# A finished track's last group ages out its max delay after it ends, so moqsrc's 1 s default
# drops the tail whenever gst-launch takes longer than that to start. Like the other finish
# readers it grants the whole read phase: how late it joins is not the delivery under test.
GST_PLUGIN_PATH_1_0="$GST_PLUGIN_DIR" GST_REGISTRY_1_0="$HARNESS_RUN/gst-run-registry.bin" \
gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" ${version:+connect-version="$version"} connect-websocket-enabled=false \
gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" max-delay="$((window * 1000))" ${version:+connect-version="$version"} connect-websocket-enabled=false \
s.video_0 ! identity name=frames silent=false ! fakesink >"$out" 2>&1 &
pid=$!
count=0
Expand Down Expand Up @@ -1773,7 +1778,9 @@ run_finishes() {
broadcast="$name-$$-$RANDOM/fixture.hang"
grant="${broadcast%/*}/**"
ack="$LOGS/$name-ack"
local pids=() fds=() names=() all_ready=1 all_clean=1 publisher_ok=1
# causes[i] is reader i's own failure; row is what failed the whole row, which every
# cell without a failure of its own reports, so a withheld acknowledgement names its cause.
local pids=() fds=() names=() causes=() failed=() row="" status
late=0
started=$SECONDS
echo "=== finished media: $pub ==="
Expand Down Expand Up @@ -1805,14 +1812,15 @@ run_finishes() {
names+=("$sub")
done
for i in "${!pids[@]}"; do
if ! finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$'; then all_ready=0; fi
finish_ready "${pids[$i]}" "$LOGS/$name-${names[$i]}.log" '^finish reader ready$' || causes[i]="never ready"
done
if [[ "$all_ready" -eq 1 ]]; then
pid=""
if [[ ${#causes[@]} -eq 0 ]]; then
harness_spawn "$name-pub" "$LOGS/$name-pub.log" run_finish_publisher "$pub" "$broadcast" "$(token_url --publish "$grant")" "$ack"
pid="$HARNESS_PID"
if ! finish_ready "$pid" "$LOGS/$name-pub.log" '^finish ready linger_us='; then all_ready=0; fi
else pid=""; fi
if [[ "$all_ready" -eq 1 ]]; then
finish_ready "$pid" "$LOGS/$name-pub.log" '^finish ready linger_us=' || row="publisher never ready"
fi
if [[ -z "$row" && ${#causes[@]} -eq 0 ]]; then
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
for reader_fd in "${fds[@]}"; do printf 'read\n' >&"$reader_fd"; done
if [[ "$late" -eq 1 ]]; then
harness_spawn "$name-gst" "$LOGS/$name-gst.log" run_finish_gst "$broadcast" "$(token_url --subscribe "$grant")" "$LOGS/$name-gst.out"
Expand All @@ -1821,24 +1829,42 @@ run_finishes() {
names+=(gst)
fi
for i in "${!pids[@]}"; do
harness_wait "${pids[$i]}" || all_clean=0
if ! grep -qx "$(finish_marker "${names[$i]}")" "$LOGS/$name-${names[$i]}.log"; then all_clean=0; fi
status=0
harness_wait "${pids[$i]}" || status=$?
# A reader that printed its marker can still fail on the way out, such as a
# session drain that times out, and that withholds the acknowledgement too.
if [[ "$status" -ne 0 ]]; then
causes[i]="exited $status"
elif ! grep -qx "$(finish_marker "${names[$i]}")" "$LOGS/$name-${names[$i]}.log"; then
causes[i]="no finish marker"
fi
done
else all_clean=0; fi
if [[ "$all_clean" -eq 1 ]]; then
elif [[ "$late" -eq 1 ]]; then
# gst never started, but its cell still fails with the row's cause.
pids+=("")
fds+=("")
names+=(gst)
fi
for i in "${!causes[@]}"; do failed+=("${names[$i]}"); done
[[ ${#failed[@]} -eq 0 ]] || row="reader failed: ${failed[*]}"
if [[ -z "$row" ]]; then
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
printf 'complete\n' >&"$ack_fd"
exec {ack_fd}>&-
harness_wait "$pid" || publisher_ok=0
grep -qx 'finish acknowledged' "$LOGS/$name-pub.log" || publisher_ok=0
status=0
harness_wait "$pid" || status=$?
if [[ "$status" -ne 0 ]]; then
row="publisher exited $status after the acknowledgement"
elif ! grep -qx 'finish acknowledged' "$LOGS/$name-pub.log"; then
row="publisher did not print finish acknowledged"
fi
else
publisher_ok=0
[[ -z "$pid" ]] || harness_reap "$pid"
exec {ack_fd}>&-
fi
for i in "${!pids[@]}"; do
sub="${names[$i]}"
local cell=(case=finish publisher="$pub" subscriber="$sub" seconds=$((SECONDS - started)) log="$LOGS/$name-$sub.log")
if [[ "$all_ready" -eq 1 && "$publisher_ok" -eq 1 ]] && grep -qx "$(finish_marker "$sub")" "$LOGS/$name-$sub.log"; then
if [[ -z "$row" ]]; then
if [[ "$sub" == gst ]]; then
echo " PASS finish $pub -> $sub (late tail; end N/A, moqsrc keeps watching)"
record pass "${cell[@]}" detail="late tail only; end N/A: moqsrc keeps watching a finished broadcast"
Expand All @@ -1847,8 +1873,8 @@ run_finishes() {
record pass "${cell[@]}"
fi
else
record fail "${cell[@]}" detail="missing finish tail/EOF/withdrawal/health or acknowledgement"
echo " FAIL finish $pub -> $sub"
record fail "${cell[@]}" detail="${causes[$i]:-$row}"
echo " FAIL finish $pub -> $sub (${causes[$i]:-$row})"
overall=1
cat "$LOGS/$name-$sub.log" "$LOGS/$name-pub.log" 2>/dev/null || true
harness_reap "${pids[$i]}"
Expand Down