From 1f3a432a4282211549927371004de102777f02a1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 17:58:07 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m0/nightly/obs-audio-candidate.md Quest-Claim: ac7120a81acb7405 From e6175aca6646773a1a178a5961dc5992e6ad2399 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 18:25:26 -0700 Subject: [PATCH 2/4] test(obs): observe the rejected candidate by identity Wait for the selected candidate failure while its weak handle is still alive, then verify that attempt retired and the original active track continues playback. A new catalog-created attempt no longer makes the assertion racy. Co-Authored-By: Codex --- cpp/obs/test/moq-source-test.cpp | 74 +++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/cpp/obs/test/moq-source-test.cpp b/cpp/obs/test/moq-source-test.cpp index bb75410cbc6..3419e1e9d0e 100644 --- a/cpp/obs/test/moq-source-test.cpp +++ b/cpp/obs/test/moq-source-test.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -111,25 +112,41 @@ uint32_t g_obs_sample_rate = 48000; // as a leak. std::atomic g_live_allocs{0}; -// Candidate resolve and subscribe failures, logged under the source lock as the candidate -// is released, so taking that lock afterwards observes the release. +// Watch one attempt, not whichever replacement a later catalog snapshot may start. +// The logger runs under the source lock, before the failed attempt is released. +struct CandidateFailure { + std::weak_ptr track; + bool failed = false; +}; std::mutex g_candidate_mutex; std::condition_variable g_candidate_failed; -int g_candidate_failures = 0; +CandidateFailure g_candidate; } // namespace extern "C" { // Deliberately silent: stdio locking would create happens-before edges between -// the test thread and the source's worker and mask the races we're after. Only a -// candidate's resolve or subscribe failure synchronizes, so a scenario can wait on it. +// the test thread and the source's worker and mask the races we're after. Match +// the watched role while its attempt is alive; playback identity checks below +// distinguish an active-track error from the candidate's failure. void blog(int, const char *format, ...) { - if (!strstr(format, "Failed to resolve") && !strstr(format, "Failed to subscribe") && - !strstr(format, "Audio track error")) + const char *kind = nullptr; + if (strstr(format, "Audio track error") || strstr(format, "Failed to subscribe to Audio track")) { + kind = "Audio"; + } else if (strstr(format, "Failed to subscribe to %s track")) { + va_list args; + va_start(args, format); + kind = va_arg(args, const char *); + va_end(args); + } + if (!kind) return; std::lock_guard lock(g_candidate_mutex); - g_candidate_failures++; + auto candidate = g_candidate.track.lock(); + if (!candidate || strcmp(candidate->kind, kind) != 0) + return; + g_candidate.failed = true; g_candidate_failed.notify_all(); } @@ -1065,10 +1082,11 @@ int main() Publisher publisher(relay, "obs/test"); void *source = createSource(); CHECK(publisher.WriteUntil([] { return g_output_audio > 0; })); - int failures = 0; + auto *ctx = static_cast(source); + std::weak_ptr active; { - std::lock_guard lock(g_candidate_mutex); - failures = g_candidate_failures; + std::lock_guard lock(ctx->mutex); + active = ctx->connection->audio; } moq::AudioInit init{}; init.format = moq::AudioFormat::kOpus; @@ -1077,16 +1095,27 @@ int main() moq::MediaTrackProducer::audio(publisher.broadcast, moq::MediaTarget::kNamed{std::string("!replacement")}, init), "replacement audio"); + std::weak_ptr candidate; + CHECK(publisher.WriteUntil([&] { + std::lock_guard lock(ctx->mutex); + const auto &pending = ctx->connection->pending_audio; + if (!pending || !pending->audio_consumer) + return false; + candidate = pending; + std::lock_guard failure_lock(g_candidate_mutex); + g_candidate = {candidate, false}; + return true; + })); CHECK(publisher.WriteUntil([&] { TestOk(replacement->write_frame(moq::Frame{BAD_OPUS, 1'000'000'000}), "replacement frame"); TestOk(replacement->cut(), "replacement audio cut"); std::lock_guard lock(g_candidate_mutex); - return g_candidate_failures > failures; + return g_candidate.failed && candidate.expired(); })); - auto *ctx = static_cast(source); { std::lock_guard lock(ctx->mutex); - CHECK(ctx->connection->audio && !ctx->connection->pending_audio); + auto playing = active.lock(); + CHECK(playing && ctx->connection->audio == playing); } const int audio = g_output_audio; CHECK(publisher.WriteUntil([&] { return g_output_audio >= audio + 3; })); @@ -1186,15 +1215,16 @@ int main() snapshot->video.clear(); snapshot->video.emplace("!unavailable", config); auto *ctx = static_cast(source); - int failures = 0; - { - std::lock_guard lock(g_candidate_mutex); - failures = g_candidate_failures; - } + std::weak_ptr candidate; + std::weak_ptr active; { std::lock_guard lock(ctx->mutex); + active = ctx->connection->video; moq_source_subscribe_video(ctx, ctx->connection, *snapshot); CHECK(ctx->connection->pending_video != nullptr); + candidate = ctx->connection->pending_video; + std::lock_guard failure_lock(g_candidate_mutex); + g_candidate = {candidate, false}; } if (hidden) { // An unannounced sibling stays pending, without interrupting current playback. @@ -1207,11 +1237,13 @@ int main() } { std::unique_lock lock(g_candidate_mutex); - g_candidate_failed.wait(lock, [&] { return g_candidate_failures > failures; }); + g_candidate_failed.wait(lock, [&] { return g_candidate.failed; }); } { std::lock_guard lock(ctx->mutex); - CHECK(ctx->connection && !ctx->connection->pending_video); + auto playing = active.lock(); + CHECK(ctx->connection && playing && ctx->connection->video == playing); + CHECK(candidate.expired()); } const int frames = g_output_frames; CHECK(publisher.WriteUntil([&] { return g_output_frames >= frames + 3; })); From 812cb6bd3ba22bc53b8185a9eb0c868d30f79a63 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 18:39:01 -0700 Subject: [PATCH 3/4] quest: complete the OBS candidate oracle investigation Co-Authored-By: Codex --- quest/m0/nightly/README.md | 1 - quest/m0/nightly/obs-audio-candidate.md | 27 ------------------------- 2 files changed, 28 deletions(-) delete mode 100644 quest/m0/nightly/obs-audio-candidate.md diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 42a09a3cad7..dfbeb1131da 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -34,5 +34,4 @@ Public API: none. Wire: none. ## Required - [Windows builds moq-relay tests](/quest/m0/nightly/windows-relay-expect.md) - an unfulfilled lint expectation stops failing the Windows build -- [OBS audio-candidate test](/quest/m0/nightly/obs-audio-candidate.md) - the failed audio candidate check passes under TSan and load, fixed at its reproduced cause - [Feature shapes per crate](/quest/m0/nightly/hls-poll-closed-gate.md) - moq-hls `Broadcaster::poll_closed` is gated on `server`, and the gate checks each crate's feature shapes in isolation diff --git a/quest/m0/nightly/obs-audio-candidate.md b/quest/m0/nightly/obs-audio-candidate.md deleted file mode 100644 index 4ec248b3176..00000000000 --- a/quest/m0/nightly/obs-audio-candidate.md +++ /dev/null @@ -1,27 +0,0 @@ -# [S] OBS audio-candidate test passes under TSan and load - -## Goal - -`cpp/obs/test/moq-source-test.cpp`'s "failed audio candidate preserves -playback" passes under OBS TSan every night and under load. On 2026-10-25 its check -`ctx->connection->audio && !ctx->connection->pending_audio` failed with no -TSan report. [moq-dev/moq#6711](https://github.com/moq-dev/moq/pull/6711) also -saw it fail intermittently under load, on its base commit too. - -## Plan - -The cause is not known yet. It is not the counter racing the clear: the -"Audio track error" log that bumps `g_candidate_failures` and the -`pending_audio` clear both run under `ctx->mutex`, which the check then takes. -But the counter is global, also bumped by any "Failed to resolve" or "Failed -to subscribe", and the check also needs the active `audio` to survive. -Reproduce first, under TSan or under CPU pressure or `SIGSTOP`/`SIGCONT` -pauses as [moq-dev/moq#6737](https://github.com/moq-dev/moq/pull/6737) did. -Find which conjunct fails and which track and error bumped the counter, then -fix that cause. If the wait is too broad, make it wait on the replacement's -own failure. Do not add a retry or a sleep. - -Planned 2026-10-25 (maintainer) from the triage of Nightly run -[40192013303](https://github.com/moq-dev/moq/actions/runs/40192013303). - -Public API: none. Wire: none. From 3694ca30d8a8a24cb3b0ed9b912271576146fb8c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 22:37:05 -0700 Subject: [PATCH 4/4] test(obs): bound the video candidate failure wait Co-Authored-By: Claude Opus 5.5 --- cpp/obs/test/moq-source-test.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cpp/obs/test/moq-source-test.cpp b/cpp/obs/test/moq-source-test.cpp index 3419e1e9d0e..fd5bf8d0740 100644 --- a/cpp/obs/test/moq-source-test.cpp +++ b/cpp/obs/test/moq-source-test.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -119,7 +118,6 @@ struct CandidateFailure { bool failed = false; }; std::mutex g_candidate_mutex; -std::condition_variable g_candidate_failed; CandidateFailure g_candidate; } // namespace @@ -147,7 +145,6 @@ void blog(int, const char *format, ...) if (!candidate || strcmp(candidate->kind, kind) != 0) return; g_candidate.failed = true; - g_candidate_failed.notify_all(); } void *bmalloc(size_t size) @@ -1235,10 +1232,10 @@ int main() route.epoch = "01900000-0000-7000-8000-000000000002"; TestOk(hidden->announce(route), "announce mismatched sibling"); } - { - std::unique_lock lock(g_candidate_mutex); - g_candidate_failed.wait(lock, [&] { return g_candidate.failed; }); - } + CHECK(WaitFor([] { + std::lock_guard lock(g_candidate_mutex); + return g_candidate.failed; + })); { std::lock_guard lock(ctx->mutex); auto playing = active.lock();