diff --git a/cpp/obs/src/moq-source.cpp b/cpp/obs/src/moq-source.cpp index e656fa88d5e..91733783cbb 100644 --- a/cpp/obs/src/moq-source.cpp +++ b/cpp/obs/src/moq-source.cpp @@ -611,18 +611,15 @@ moq_source_ranked_audio(const std::unordered_map &rendi }); } -// The ranked renditions to try in turn: the enabled ones. With none enabled, a consumer -// should not select a rendition but holds the one it has, so only those `held` accepts. -template +// The ranked renditions to try in turn: the enabled ones, since a consumer must not select a +// disabled rendition. +template static std::vector *> -moq_source_candidates(std::vector *> ranked, Held held) +moq_source_candidates(std::vector *> ranked) { - const bool any_enabled = - std::any_of(ranked.begin(), ranked.end(), [](const auto *entry) { return entry->second.enabled; }); - ranked.erase( - std::remove_if(ranked.begin(), ranked.end(), - [&](const auto *entry) { return any_enabled ? !entry->second.enabled : !held(*entry); }), - ranked.end()); + ranked.erase(std::remove_if(ranked.begin(), ranked.end(), + [](const auto *entry) { return !entry->second.enabled; }), + ranked.end()); return ranked; } @@ -976,10 +973,7 @@ static void moq_source_subscribe_track(struct moq_source *ctx, const std::shared static void moq_source_subscribe_video(struct moq_source *ctx, const std::shared_ptr &conn, const moq::Catalog &catalog) { - const auto candidates = moq_source_candidates(moq_source_ranked_video(catalog.video), [&](const auto &entry) { - return moq_source_same_video(conn->video.get(), entry.first, entry.second) || - moq_source_same_video(conn->pending_video.get(), entry.first, entry.second); - }); + const auto candidates = moq_source_candidates(moq_source_ranked_video(catalog.video)); if (candidates.empty()) { LOG_INFO("Catalog has no enabled video rendition; audio only"); moq_source_clear_video_locked(ctx); @@ -1021,10 +1015,7 @@ static void moq_source_subscribe_video(struct moq_source *ctx, const std::shared static void moq_source_subscribe_audio(struct moq_source *ctx, const std::shared_ptr &conn, const moq::Catalog &catalog) { - const auto candidates = moq_source_candidates(moq_source_ranked_audio(catalog.audio), [&](const auto &entry) { - return moq_source_same_audio(conn->audio.get(), entry.first, entry.second) || - moq_source_same_audio(conn->pending_audio.get(), entry.first, entry.second); - }); + const auto candidates = moq_source_candidates(moq_source_ranked_audio(catalog.audio)); if (candidates.empty()) { LOG_INFO("Catalog has no enabled audio rendition; video only"); moq_source_clear_audio_locked(ctx); diff --git a/cpp/obs/test/moq-source-test.cpp b/cpp/obs/test/moq-source-test.cpp index 16ad9730569..91a11c9690d 100644 --- a/cpp/obs/test/moq-source-test.cpp +++ b/cpp/obs/test/moq-source-test.cpp @@ -1187,24 +1187,12 @@ int main() std::vector{"f-stereo", "d-mono", "e-slow", "c-low", "b-unknown-bitrate", "a-disabled"})); - // Only enabled renditions are candidates while any is, even one the source holds. - const auto all = [](const auto &) { - return true; - }; - const auto none = [](const auto &) { - return false; - }; - CHECK((names(moq_source_candidates(moq_source_ranked_audio(audios), all)) == + // Only enabled renditions are candidates. + CHECK((names(moq_source_candidates(moq_source_ranked_audio(audios))) == std::vector{"f-stereo", "d-mono", "e-slow", "c-low", "b-unknown-bitrate"})); - // With every rendition disabled, only a held one remains, and nothing when none is held. for (auto &entry : audios) entry.second.enabled = false; - const auto held = [](const auto &entry) { - return entry.first == "c-low"; - }; - CHECK((names(moq_source_candidates(moq_source_ranked_audio(audios), held)) == - std::vector{"c-low"})); - CHECK(moq_source_candidates(moq_source_ranked_audio(audios), none).empty()); + CHECK(moq_source_candidates(moq_source_ranked_audio(audios)).empty()); } report("renditions rank like hang"); @@ -1308,8 +1296,8 @@ int main() } report("disabled and undecodable renditions fall through"); - // With every rendition disabled, the source holds the one it plays; with none of them - // held, the section is handled like an absent one. + // With every rendition disabled, the section is handled like an absent one, even the + // rendition the source plays. { TestRelay relay; reset(relay.Url()); @@ -1318,29 +1306,10 @@ int main() CHECK(publisher.WriteUntil([] { return g_output_frames > 0 && g_output_audio > 0; })); auto snapshot = CatalogSnapshot(publisher); auto *ctx = static_cast(source); - std::weak_ptr video; - std::weak_ptr audio; for (auto &entry : snapshot.video) entry.second.enabled = false; for (auto &entry : snapshot.audio) entry.second.enabled = false; - { - std::lock_guard lock(ctx->mutex); - const auto &conn = ctx->connection; - video = conn->video; - audio = conn->audio; - moq_source_subscribe_video(ctx, conn, snapshot); - moq_source_subscribe_audio(ctx, conn, snapshot); - CHECK(conn->video && conn->video == video.lock() && !conn->pending_video); - CHECK(conn->audio && conn->audio == audio.lock() && !conn->pending_audio); - } - const int frames = g_output_frames; - CHECK(publisher.WriteUntil([&] { return g_output_frames >= frames + 3; })); - - // A disabled rendition the source does not play is not held. - auto &config = snapshot.video.begin()->second; - config.coded = moq::Dimensions{(config.coded ? config.coded->width : 0) + 16, 16}; - snapshot.audio.begin()->second.sample_rate += 1; const int blanks = g_blank_calls; { std::lock_guard lock(ctx->mutex); @@ -1353,7 +1322,7 @@ int main() CHECK(g_blank_calls > blanks); destroySource(source); } - report("all-disabled section holds the playing rendition"); + report("an all-disabled section clears the playing rendition"); // A rendition's stream instance is part of what names it, for the active and the pending // track alike: a sibling publisher restarting behind a stable catalog resubscribes. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 0d4c40af414..65f0f46f21a 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -23989,7 +23989,7 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqaudioconsumer_subscribe() != - 42847) { + 37967) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqaudioencoder_new() != 52616) { @@ -24075,7 +24075,7 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqvideoconsumer_subscribe() != - 59752) { + 7811) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_constructor_moqvideoencoder_new() != 22531) { diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 74f33f1bb20..8172d1dd97a 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -277,11 +277,7 @@ audio-device dependencies: cargo install moq-cli --no-default-features --features "quic,websocket,play,openh264" ``` -`moq play` skips disabled renditions when it picks one. If no enabled rendition -matches its name and codec filters, it waits for a later catalog. Disabling the -current rendition switches to an enabled one of its kind that matches the -filters; with none, it stays subscribed, so re-enabling it (an unmute) plays at -once. +`moq play` skips disabled renditions. If none are enabled, it waits for a later catalog. Disabling the current rendition stops it and selects an enabled replacement. ## Capture diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 51011a50dc1..4d89b85551b 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -185,9 +185,4 @@ To decrypt a capture, set `quic-keylog=true` on either element along with `SSLKEYLOGFILE`; the variable alone writes nothing, and the flag without it fails the start. Existing Unix log files must be private (mode 0600); files accessible to group or others are refused. -`moqsrc` initially selects only enabled renditions. When a selected rendition is -disabled, it keeps reading on the same pad while no rendition of its kind is -enabled, so media arriving before the re-enable catalog update still plays. An -enabled alternative stops the disabled reader but keeps its pad idle without EOS; -enabling it again resumes on that pad. Removing a rendition drains its track and -ends its pad. +`moqsrc` creates source pads only for enabled renditions. Disabling a rendition stops its media and ends its pad with EOS; enabling it again adds a new pad, which starts at the live edge like any added rendition. Removing a rendition drains its track and ends its pad. diff --git a/doc/bin/obs.md b/doc/bin/obs.md index b91e8ea8992..d16f18be7cb 100644 --- a/doc/bin/obs.md +++ b/doc/bin/obs.md @@ -17,7 +17,7 @@ reconnects keep the same instance. A stop/start, or an OBS auto-reconnect after the client gives up, republishes the broadcast under a new instance, which viewers see as a new stream. -MoQ Source plays the best enabled audio and video rendition it can decode and place. It follows catalog changes while keeping the current audio and video playing until a replacement produces decoded output. A decoder setup refusal tries the next ranked rendition, including when a supported codec name carries a description the decoder cannot use. A later peer or decode failure leaves the current rendition playing. Disabling every rendition of a media kind keeps the one already playing; removing the media kind from the catalog clears it. It also follows a restarted or replaced publisher, including a sibling broadcast the catalog points at, playing the new one from its first keyframe. +MoQ Source plays the best enabled audio and video rendition it can decode and place. It follows catalog changes while keeping the current audio and video playing until a replacement produces decoded output. A decoder setup refusal tries the next ranked rendition, including when a supported codec name carries a description the decoder cannot use. A later peer or decode failure leaves the current rendition playing. Disabling or removing every rendition of a media kind clears it. It also follows a restarted or replaced publisher, including a sibling broadcast the catalog points at, playing the new one from its first keyframe. The source plays Opus, AAC-LC (mono or stereo), and PCM audio. A channel count names its speakers by the WAVE convention: 1, 2, 3 (2.1), 6 (5.1), and 8 (7.1) play on the matching OBS layout, while quad and 5.0 are upmixed to 5.1 and 6.1 to 7.1, which OBS lacks. Other counts are refused. diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 9c44549f6ef..cfd03a5d28c 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -103,13 +103,20 @@ observe(arrival, media): origin = arrival; interval = 0; peak = delay; return index = floor((arrival - origin) / RESAMPLE) + resumed = ended; ended = false if index <= interval: peak = max(peak, delay); return fold(bucket of peak) - fold(nothing) for each elapsed empty interval, at most IDLE_MAX + if resumed: # a declared silence folds nothing + origin = arrival; index = 0 + else: + fold(nothing) for each elapsed empty interval, at most IDLE_MAX interval = index; peak = delay +end(): # a declared end precedes the next observation + if not first observation: ended = true + fold(bucket): # prior starts at 1 rest = 1 - FORGET * (1 - prior) # what the decayed evidence leaves fresh = min(FRESH_MAX, rest) if bucket else 0 @@ -474,6 +481,25 @@ keeps decaying at the base rate. Only the weight the prior regained is replaced quickly, the same way it is at cold start, since it stands for no evidence at all. +### Declared silence + +A publisher that stops sending on purpose, such as a muted push-to-talk +speaker, marks the stop with a declared end. The silence after it says +nothing about the path, so relaxing toward the prior would only make a +listener who sat through a long mute come back over-buffered. `end()` tells +the estimator that a declared end lies between the previous observation and +the next. The next measured observation then folds the pending interval as +usual and starts a fresh interval at its own arrival instead of folding the +intervals in between. If it lands in the pending interval it is merged as +usual, since there is nothing to skip. Either way the end is spent, so a +later gap the publisher did not declare still relaxes the estimate. The +reference history, the histogram, and the prior are untouched, and an end +before the first observation, or under a fixed target, does nothing. + +The receiver calls `end()` before observing the first frame from a group later +than the declared end's own: the stopped encoder's final packets may follow the +marker in its group, and they belong to the speech before the silence, not after it. + ## Fixed targets An explicit delay **fixes the target and disables adaptation**. It is taken @@ -552,7 +578,8 @@ directly, so neither is graded against the other. `arrival` and `media` are parallel arrays in **delivery order**, the order the receiver observes them, so `media` is deliberately out of order in the `reorder` and `backlog` cases. `advertised` and `delay` are present only where the case uses -them. `target` lists `[frame index, target]` at every frame where the target +them, as is `ends`: the indices, ascending, of the frames observed right after a +[declared end](#declared-silence), each of which calls `end()` before it is observed. `target` lists `[frame index, target]` at every frame where the target differs from the previous frame's, starting at frame 0, so an implementation folds the trace one frame at a time and compares its target at each listed index. Tolerance is 1e-6 ms, which the `f64` rules above make generous. @@ -566,6 +593,7 @@ index. Tolerance is 1e-6 ms, which the `f64` rules above make generous. | `reorder` | A run delivered after a newer one is measured against the frames before it, so the target covers waiting for it | | `backlog` | A backlog delivered newest first behind the live edge is not measured, so its media span never reaches the target | | `idle` | A 30 s gap relaxes the histogram toward the prior | +| `silence` | A declared 60 s silence and a declared pause leave the target where it was, and a later undeclared gap still relaxes it | | `tunein` | A stale frame then the live edge 14.5 s later leaves the target near the frame duration. This is the #3517 regression | | `spike` | A recurring stall raises the target, and it decays once the stalls stop | | `straggler` | One late interval after the prior is spent leaves the target where it was | diff --git a/doc/concept/audio-jitter/corpus.ts b/doc/concept/audio-jitter/corpus.ts index 6af6626f491..299328f8170 100644 --- a/doc/concept/audio-jitter/corpus.ts +++ b/doc/concept/audio-jitter/corpus.ts @@ -24,6 +24,8 @@ export interface Case { arrival: number[]; /** Each frame's container timestamp in milliseconds, in the same order. */ media: number[]; + /** Indices of the frames observed right after a declared end, in ascending order. */ + ends?: number[]; /** `[frame index, target]` wherever the target differs from the previous frame's. */ target: [number, number][]; } @@ -87,14 +89,19 @@ function paced(frames: number, frame: number, base: number): Trace { } /** Replay a trace through the reference and keep only the frames where the target moved. */ -function run(trace: Trace, config: { frame: number; advertised?: number; delay?: number }): [number, number][] { +function run( + trace: Trace & { ends?: number[] }, + config: { frame: number; advertised?: number; delay?: number }, +): [number, number][] { if (config.delay !== undefined) return [[0, config.delay]]; const estimator = new Estimator({ frame: config.frame, advertised: config.advertised }); const target: [number, number][] = []; let previous: number | undefined; + const ends = new Set(trace.ends); for (let i = 0; i < trace.arrival.length; i++) { + if (ends.has(i)) estimator.end(); estimator.observe({ arrival: trace.arrival[i], media: trace.media[i] }); const value = estimator.target; if (value !== previous) target.push([i, value]); @@ -171,6 +178,13 @@ const traces: Omit[] = [ frame: FRAME, ...gap(paced(1000, FRAME, 50), 500, 30_000), }, + { + name: "silence", + about: "Even audio, a declared end and 60 s of silence, 5 s more, a declared 100 ms pause inside one interval, 5 s more, then an undeclared 30 s gap. Unlike `idle`, a declared silence is not evidence, so the target holds through both; the undeclared gap still relaxes it, since an end covers only the silence it declares.", + frame: FRAME, + ...gap(gap(gap(paced(1500, FRAME, 50), 500, 60_000), 760, 100), 1000, 30_000), + ends: [500, 760], + }, { name: "tunein", about: "One frame from a stale group, then the live edge 14.5 s later, then 30 s of even audio. The #3517 branch reported 14.56 s here; the target must stay near the frame duration.", diff --git a/doc/concept/audio-jitter/reference.ts b/doc/concept/audio-jitter/reference.ts index f2d92ee387f..ecd7a2fd3aa 100644 --- a/doc/concept/audio-jitter/reference.ts +++ b/doc/concept/audio-jitter/reference.ts @@ -73,6 +73,8 @@ export class Estimator { #interval = 0; /** The largest delay seen in that interval. */ #peak = 0; + /** A declared end precedes the next measured observation, so the silence before it is not idle. */ + #ended = false; #histogram = prior(); /** How much of the histogram is still the prior's rather than evidence. */ @@ -119,6 +121,8 @@ export class Estimator { } const interval = Math.floor((arrival - this.#origin) / RESAMPLE); + const resumed = this.#ended; + this.#ended = false; if (interval <= this.#interval) { this.#peak = Math.max(this.#peak, delay); return; @@ -126,15 +130,27 @@ export class Estimator { this.#fold(Math.min(BUCKETS - 1, Math.floor(this.#peak / BUCKET))); - const idle = Math.min(interval - this.#interval - 1, IDLE_MAX); - for (let i = 0; i < idle; i++) { - this.#fold(undefined); + if (resumed) { + // The publisher declared the silence, so it is not evidence of anything: the + // estimate keeps what it had and a fresh interval starts at this arrival. + this.#origin = arrival; + this.#interval = 0; + } else { + const idle = Math.min(interval - this.#interval - 1, IDLE_MAX); + for (let i = 0; i < idle; i++) { + this.#fold(undefined); + } + this.#interval = interval; } - - this.#interval = interval; this.#peak = delay; } + /** A declared end lies between the previous observation and the next. */ + end(): void { + // Before any observation there is no interval for the silence to stretch. + if (this.#origin !== undefined) this.#ended = true; + } + /** The target the receiver holds, in milliseconds. */ get target(): number { let cumulative = 0; diff --git a/doc/concept/audio-jitter/silence.json b/doc/concept/audio-jitter/silence.json new file mode 100644 index 00000000000..60df4c38167 --- /dev/null +++ b/doc/concept/audio-jitter/silence.json @@ -0,0 +1 @@ +{"name":"silence","about":"Even audio, a declared end and 60 s of silence, 5 s more, a declared 100 ms pause inside one interval, 5 s more, then an undeclared 30 s gap. Unlike `idle`, a declared silence is not evidence, so the target holds through both; the undeclared gap still relaxes it, since an end covers only the silence it declares.","frame":20,"arrival":[50,70,90,110,130,150,170,190,210,230,250,270,290,310,330,350,370,390,410,430,450,470,490,510,530,550,570,590,610,630,650,670,690,710,730,750,770,790,810,830,850,870,890,910,930,950,970,990,1010,1030,1050,1070,1090,1110,1130,1150,1170,1190,1210,1230,1250,1270,1290,1310,1330,1350,1370,1390,1410,1430,1450,1470,1490,1510,1530,1550,1570,1590,1610,1630,1650,1670,1690,1710,1730,1750,1770,1790,1810,1830,1850,1870,1890,1910,1930,1950,1970,1990,2010,2030,2050,2070,2090,2110,2130,2150,2170,2190,2210,2230,2250,2270,2290,2310,2330,2350,2370,2390,2410,2430,2450,2470,2490,2510,2530,2550,2570,2590,2610,2630,2650,2670,2690,2710,2730,2750,2770,2790,2810,2830,2850,2870,2890,2910,2930,2950,2970,2990,3010,3030,3050,3070,3090,3110,3130,3150,3170,3190,3210,3230,3250,3270,3290,3310,3330,3350,3370,3390,3410,3430,3450,3470,3490,3510,3530,3550,3570,3590,3610,3630,3650,3670,3690,3710,3730,3750,3770,3790,3810,3830,3850,3870,3890,3910,3930,3950,3970,3990,4010,4030,4050,4070,4090,4110,4130,4150,4170,4190,4210,4230,4250,4270,4290,4310,4330,4350,4370,4390,4410,4430,4450,4470,4490,4510,4530,4550,4570,4590,4610,4630,4650,4670,4690,4710,4730,4750,4770,4790,4810,4830,4850,4870,4890,4910,4930,4950,4970,4990,5010,5030,5050,5070,5090,5110,5130,5150,5170,5190,5210,5230,5250,5270,5290,5310,5330,5350,5370,5390,5410,5430,5450,5470,5490,5510,5530,5550,5570,5590,5610,5630,5650,5670,5690,5710,5730,5750,5770,5790,5810,5830,5850,5870,5890,5910,5930,5950,5970,5990,6010,6030,6050,6070,6090,6110,6130,6150,6170,6190,6210,6230,6250,6270,6290,6310,6330,6350,6370,6390,6410,6430,6450,6470,6490,6510,6530,6550,6570,6590,6610,6630,6650,6670,6690,6710,6730,6750,6770,6790,6810,6830,6850,6870,6890,6910,6930,6950,6970,6990,7010,7030,7050,7070,7090,7110,7130,7150,7170,7190,7210,7230,7250,7270,7290,7310,7330,7350,7370,7390,7410,7430,7450,7470,7490,7510,7530,7550,7570,7590,7610,7630,7650,7670,7690,7710,7730,7750,7770,7790,7810,7830,7850,7870,7890,7910,7930,7950,7970,7990,8010,8030,8050,8070,8090,8110,8130,8150,8170,8190,8210,8230,8250,8270,8290,8310,8330,8350,8370,8390,8410,8430,8450,8470,8490,8510,8530,8550,8570,8590,8610,8630,8650,8670,8690,8710,8730,8750,8770,8790,8810,8830,8850,8870,8890,8910,8930,8950,8970,8990,9010,9030,9050,9070,9090,9110,9130,9150,9170,9190,9210,9230,9250,9270,9290,9310,9330,9350,9370,9390,9410,9430,9450,9470,9490,9510,9530,9550,9570,9590,9610,9630,9650,9670,9690,9710,9730,9750,9770,9790,9810,9830,9850,9870,9890,9910,9930,9950,9970,9990,10010,10030,70050,70070,70090,70110,70130,70150,70170,70190,70210,70230,70250,70270,70290,70310,70330,70350,70370,70390,70410,70430,70450,70470,70490,70510,70530,70550,70570,70590,70610,70630,70650,70670,70690,70710,70730,70750,70770,70790,70810,70830,70850,70870,70890,70910,70930,70950,70970,70990,71010,71030,71050,71070,71090,71110,71130,71150,71170,71190,71210,71230,71250,71270,71290,71310,71330,71350,71370,71390,71410,71430,71450,71470,71490,71510,71530,71550,71570,71590,71610,71630,71650,71670,71690,71710,71730,71750,71770,71790,71810,71830,71850,71870,71890,71910,71930,71950,71970,71990,72010,72030,72050,72070,72090,72110,72130,72150,72170,72190,72210,72230,72250,72270,72290,72310,72330,72350,72370,72390,72410,72430,72450,72470,72490,72510,72530,72550,72570,72590,72610,72630,72650,72670,72690,72710,72730,72750,72770,72790,72810,72830,72850,72870,72890,72910,72930,72950,72970,72990,73010,73030,73050,73070,73090,73110,73130,73150,73170,73190,73210,73230,73250,73270,73290,73310,73330,73350,73370,73390,73410,73430,73450,73470,73490,73510,73530,73550,73570,73590,73610,73630,73650,73670,73690,73710,73730,73750,73770,73790,73810,73830,73850,73870,73890,73910,73930,73950,73970,73990,74010,74030,74050,74070,74090,74110,74130,74150,74170,74190,74210,74230,74250,74270,74290,74310,74330,74350,74370,74390,74410,74430,74450,74470,74490,74510,74530,74550,74570,74590,74610,74630,74650,74670,74690,74710,74730,74750,74770,74790,74810,74830,74850,74870,74890,74910,74930,74950,74970,74990,75010,75030,75050,75070,75090,75110,75130,75150,75170,75190,75210,75230,75350,75370,75390,75410,75430,75450,75470,75490,75510,75530,75550,75570,75590,75610,75630,75650,75670,75690,75710,75730,75750,75770,75790,75810,75830,75850,75870,75890,75910,75930,75950,75970,75990,76010,76030,76050,76070,76090,76110,76130,76150,76170,76190,76210,76230,76250,76270,76290,76310,76330,76350,76370,76390,76410,76430,76450,76470,76490,76510,76530,76550,76570,76590,76610,76630,76650,76670,76690,76710,76730,76750,76770,76790,76810,76830,76850,76870,76890,76910,76930,76950,76970,76990,77010,77030,77050,77070,77090,77110,77130,77150,77170,77190,77210,77230,77250,77270,77290,77310,77330,77350,77370,77390,77410,77430,77450,77470,77490,77510,77530,77550,77570,77590,77610,77630,77650,77670,77690,77710,77730,77750,77770,77790,77810,77830,77850,77870,77890,77910,77930,77950,77970,77990,78010,78030,78050,78070,78090,78110,78130,78150,78170,78190,78210,78230,78250,78270,78290,78310,78330,78350,78370,78390,78410,78430,78450,78470,78490,78510,78530,78550,78570,78590,78610,78630,78650,78670,78690,78710,78730,78750,78770,78790,78810,78830,78850,78870,78890,78910,78930,78950,78970,78990,79010,79030,79050,79070,79090,79110,79130,79150,79170,79190,79210,79230,79250,79270,79290,79310,79330,79350,79370,79390,79410,79430,79450,79470,79490,79510,79530,79550,79570,79590,79610,79630,79650,79670,79690,79710,79730,79750,79770,79790,79810,79830,79850,79870,79890,79910,79930,79950,79970,79990,80010,80030,80050,80070,80090,80110,80130,110150,110170,110190,110210,110230,110250,110270,110290,110310,110330,110350,110370,110390,110410,110430,110450,110470,110490,110510,110530,110550,110570,110590,110610,110630,110650,110670,110690,110710,110730,110750,110770,110790,110810,110830,110850,110870,110890,110910,110930,110950,110970,110990,111010,111030,111050,111070,111090,111110,111130,111150,111170,111190,111210,111230,111250,111270,111290,111310,111330,111350,111370,111390,111410,111430,111450,111470,111490,111510,111530,111550,111570,111590,111610,111630,111650,111670,111690,111710,111730,111750,111770,111790,111810,111830,111850,111870,111890,111910,111930,111950,111970,111990,112010,112030,112050,112070,112090,112110,112130,112150,112170,112190,112210,112230,112250,112270,112290,112310,112330,112350,112370,112390,112410,112430,112450,112470,112490,112510,112530,112550,112570,112590,112610,112630,112650,112670,112690,112710,112730,112750,112770,112790,112810,112830,112850,112870,112890,112910,112930,112950,112970,112990,113010,113030,113050,113070,113090,113110,113130,113150,113170,113190,113210,113230,113250,113270,113290,113310,113330,113350,113370,113390,113410,113430,113450,113470,113490,113510,113530,113550,113570,113590,113610,113630,113650,113670,113690,113710,113730,113750,113770,113790,113810,113830,113850,113870,113890,113910,113930,113950,113970,113990,114010,114030,114050,114070,114090,114110,114130,114150,114170,114190,114210,114230,114250,114270,114290,114310,114330,114350,114370,114390,114410,114430,114450,114470,114490,114510,114530,114550,114570,114590,114610,114630,114650,114670,114690,114710,114730,114750,114770,114790,114810,114830,114850,114870,114890,114910,114930,114950,114970,114990,115010,115030,115050,115070,115090,115110,115130,115150,115170,115190,115210,115230,115250,115270,115290,115310,115330,115350,115370,115390,115410,115430,115450,115470,115490,115510,115530,115550,115570,115590,115610,115630,115650,115670,115690,115710,115730,115750,115770,115790,115810,115830,115850,115870,115890,115910,115930,115950,115970,115990,116010,116030,116050,116070,116090,116110,116130,116150,116170,116190,116210,116230,116250,116270,116290,116310,116330,116350,116370,116390,116410,116430,116450,116470,116490,116510,116530,116550,116570,116590,116610,116630,116650,116670,116690,116710,116730,116750,116770,116790,116810,116830,116850,116870,116890,116910,116930,116950,116970,116990,117010,117030,117050,117070,117090,117110,117130,117150,117170,117190,117210,117230,117250,117270,117290,117310,117330,117350,117370,117390,117410,117430,117450,117470,117490,117510,117530,117550,117570,117590,117610,117630,117650,117670,117690,117710,117730,117750,117770,117790,117810,117830,117850,117870,117890,117910,117930,117950,117970,117990,118010,118030,118050,118070,118090,118110,118130,118150,118170,118190,118210,118230,118250,118270,118290,118310,118330,118350,118370,118390,118410,118430,118450,118470,118490,118510,118530,118550,118570,118590,118610,118630,118650,118670,118690,118710,118730,118750,118770,118790,118810,118830,118850,118870,118890,118910,118930,118950,118970,118990,119010,119030,119050,119070,119090,119110,119130,119150,119170,119190,119210,119230,119250,119270,119290,119310,119330,119350,119370,119390,119410,119430,119450,119470,119490,119510,119530,119550,119570,119590,119610,119630,119650,119670,119690,119710,119730,119750,119770,119790,119810,119830,119850,119870,119890,119910,119930,119950,119970,119990,120010,120030,120050,120070,120090,120110,120130],"media":[0,20,40,60,80,100,120,140,160,180,200,220,240,260,280,300,320,340,360,380,400,420,440,460,480,500,520,540,560,580,600,620,640,660,680,700,720,740,760,780,800,820,840,860,880,900,920,940,960,980,1000,1020,1040,1060,1080,1100,1120,1140,1160,1180,1200,1220,1240,1260,1280,1300,1320,1340,1360,1380,1400,1420,1440,1460,1480,1500,1520,1540,1560,1580,1600,1620,1640,1660,1680,1700,1720,1740,1760,1780,1800,1820,1840,1860,1880,1900,1920,1940,1960,1980,2000,2020,2040,2060,2080,2100,2120,2140,2160,2180,2200,2220,2240,2260,2280,2300,2320,2340,2360,2380,2400,2420,2440,2460,2480,2500,2520,2540,2560,2580,2600,2620,2640,2660,2680,2700,2720,2740,2760,2780,2800,2820,2840,2860,2880,2900,2920,2940,2960,2980,3000,3020,3040,3060,3080,3100,3120,3140,3160,3180,3200,3220,3240,3260,3280,3300,3320,3340,3360,3380,3400,3420,3440,3460,3480,3500,3520,3540,3560,3580,3600,3620,3640,3660,3680,3700,3720,3740,3760,3780,3800,3820,3840,3860,3880,3900,3920,3940,3960,3980,4000,4020,4040,4060,4080,4100,4120,4140,4160,4180,4200,4220,4240,4260,4280,4300,4320,4340,4360,4380,4400,4420,4440,4460,4480,4500,4520,4540,4560,4580,4600,4620,4640,4660,4680,4700,4720,4740,4760,4780,4800,4820,4840,4860,4880,4900,4920,4940,4960,4980,5000,5020,5040,5060,5080,5100,5120,5140,5160,5180,5200,5220,5240,5260,5280,5300,5320,5340,5360,5380,5400,5420,5440,5460,5480,5500,5520,5540,5560,5580,5600,5620,5640,5660,5680,5700,5720,5740,5760,5780,5800,5820,5840,5860,5880,5900,5920,5940,5960,5980,6000,6020,6040,6060,6080,6100,6120,6140,6160,6180,6200,6220,6240,6260,6280,6300,6320,6340,6360,6380,6400,6420,6440,6460,6480,6500,6520,6540,6560,6580,6600,6620,6640,6660,6680,6700,6720,6740,6760,6780,6800,6820,6840,6860,6880,6900,6920,6940,6960,6980,7000,7020,7040,7060,7080,7100,7120,7140,7160,7180,7200,7220,7240,7260,7280,7300,7320,7340,7360,7380,7400,7420,7440,7460,7480,7500,7520,7540,7560,7580,7600,7620,7640,7660,7680,7700,7720,7740,7760,7780,7800,7820,7840,7860,7880,7900,7920,7940,7960,7980,8000,8020,8040,8060,8080,8100,8120,8140,8160,8180,8200,8220,8240,8260,8280,8300,8320,8340,8360,8380,8400,8420,8440,8460,8480,8500,8520,8540,8560,8580,8600,8620,8640,8660,8680,8700,8720,8740,8760,8780,8800,8820,8840,8860,8880,8900,8920,8940,8960,8980,9000,9020,9040,9060,9080,9100,9120,9140,9160,9180,9200,9220,9240,9260,9280,9300,9320,9340,9360,9380,9400,9420,9440,9460,9480,9500,9520,9540,9560,9580,9600,9620,9640,9660,9680,9700,9720,9740,9760,9780,9800,9820,9840,9860,9880,9900,9920,9940,9960,9980,70000,70020,70040,70060,70080,70100,70120,70140,70160,70180,70200,70220,70240,70260,70280,70300,70320,70340,70360,70380,70400,70420,70440,70460,70480,70500,70520,70540,70560,70580,70600,70620,70640,70660,70680,70700,70720,70740,70760,70780,70800,70820,70840,70860,70880,70900,70920,70940,70960,70980,71000,71020,71040,71060,71080,71100,71120,71140,71160,71180,71200,71220,71240,71260,71280,71300,71320,71340,71360,71380,71400,71420,71440,71460,71480,71500,71520,71540,71560,71580,71600,71620,71640,71660,71680,71700,71720,71740,71760,71780,71800,71820,71840,71860,71880,71900,71920,71940,71960,71980,72000,72020,72040,72060,72080,72100,72120,72140,72160,72180,72200,72220,72240,72260,72280,72300,72320,72340,72360,72380,72400,72420,72440,72460,72480,72500,72520,72540,72560,72580,72600,72620,72640,72660,72680,72700,72720,72740,72760,72780,72800,72820,72840,72860,72880,72900,72920,72940,72960,72980,73000,73020,73040,73060,73080,73100,73120,73140,73160,73180,73200,73220,73240,73260,73280,73300,73320,73340,73360,73380,73400,73420,73440,73460,73480,73500,73520,73540,73560,73580,73600,73620,73640,73660,73680,73700,73720,73740,73760,73780,73800,73820,73840,73860,73880,73900,73920,73940,73960,73980,74000,74020,74040,74060,74080,74100,74120,74140,74160,74180,74200,74220,74240,74260,74280,74300,74320,74340,74360,74380,74400,74420,74440,74460,74480,74500,74520,74540,74560,74580,74600,74620,74640,74660,74680,74700,74720,74740,74760,74780,74800,74820,74840,74860,74880,74900,74920,74940,74960,74980,75000,75020,75040,75060,75080,75100,75120,75140,75160,75180,75300,75320,75340,75360,75380,75400,75420,75440,75460,75480,75500,75520,75540,75560,75580,75600,75620,75640,75660,75680,75700,75720,75740,75760,75780,75800,75820,75840,75860,75880,75900,75920,75940,75960,75980,76000,76020,76040,76060,76080,76100,76120,76140,76160,76180,76200,76220,76240,76260,76280,76300,76320,76340,76360,76380,76400,76420,76440,76460,76480,76500,76520,76540,76560,76580,76600,76620,76640,76660,76680,76700,76720,76740,76760,76780,76800,76820,76840,76860,76880,76900,76920,76940,76960,76980,77000,77020,77040,77060,77080,77100,77120,77140,77160,77180,77200,77220,77240,77260,77280,77300,77320,77340,77360,77380,77400,77420,77440,77460,77480,77500,77520,77540,77560,77580,77600,77620,77640,77660,77680,77700,77720,77740,77760,77780,77800,77820,77840,77860,77880,77900,77920,77940,77960,77980,78000,78020,78040,78060,78080,78100,78120,78140,78160,78180,78200,78220,78240,78260,78280,78300,78320,78340,78360,78380,78400,78420,78440,78460,78480,78500,78520,78540,78560,78580,78600,78620,78640,78660,78680,78700,78720,78740,78760,78780,78800,78820,78840,78860,78880,78900,78920,78940,78960,78980,79000,79020,79040,79060,79080,79100,79120,79140,79160,79180,79200,79220,79240,79260,79280,79300,79320,79340,79360,79380,79400,79420,79440,79460,79480,79500,79520,79540,79560,79580,79600,79620,79640,79660,79680,79700,79720,79740,79760,79780,79800,79820,79840,79860,79880,79900,79920,79940,79960,79980,80000,80020,80040,80060,80080,110100,110120,110140,110160,110180,110200,110220,110240,110260,110280,110300,110320,110340,110360,110380,110400,110420,110440,110460,110480,110500,110520,110540,110560,110580,110600,110620,110640,110660,110680,110700,110720,110740,110760,110780,110800,110820,110840,110860,110880,110900,110920,110940,110960,110980,111000,111020,111040,111060,111080,111100,111120,111140,111160,111180,111200,111220,111240,111260,111280,111300,111320,111340,111360,111380,111400,111420,111440,111460,111480,111500,111520,111540,111560,111580,111600,111620,111640,111660,111680,111700,111720,111740,111760,111780,111800,111820,111840,111860,111880,111900,111920,111940,111960,111980,112000,112020,112040,112060,112080,112100,112120,112140,112160,112180,112200,112220,112240,112260,112280,112300,112320,112340,112360,112380,112400,112420,112440,112460,112480,112500,112520,112540,112560,112580,112600,112620,112640,112660,112680,112700,112720,112740,112760,112780,112800,112820,112840,112860,112880,112900,112920,112940,112960,112980,113000,113020,113040,113060,113080,113100,113120,113140,113160,113180,113200,113220,113240,113260,113280,113300,113320,113340,113360,113380,113400,113420,113440,113460,113480,113500,113520,113540,113560,113580,113600,113620,113640,113660,113680,113700,113720,113740,113760,113780,113800,113820,113840,113860,113880,113900,113920,113940,113960,113980,114000,114020,114040,114060,114080,114100,114120,114140,114160,114180,114200,114220,114240,114260,114280,114300,114320,114340,114360,114380,114400,114420,114440,114460,114480,114500,114520,114540,114560,114580,114600,114620,114640,114660,114680,114700,114720,114740,114760,114780,114800,114820,114840,114860,114880,114900,114920,114940,114960,114980,115000,115020,115040,115060,115080,115100,115120,115140,115160,115180,115200,115220,115240,115260,115280,115300,115320,115340,115360,115380,115400,115420,115440,115460,115480,115500,115520,115540,115560,115580,115600,115620,115640,115660,115680,115700,115720,115740,115760,115780,115800,115820,115840,115860,115880,115900,115920,115940,115960,115980,116000,116020,116040,116060,116080,116100,116120,116140,116160,116180,116200,116220,116240,116260,116280,116300,116320,116340,116360,116380,116400,116420,116440,116460,116480,116500,116520,116540,116560,116580,116600,116620,116640,116660,116680,116700,116720,116740,116760,116780,116800,116820,116840,116860,116880,116900,116920,116940,116960,116980,117000,117020,117040,117060,117080,117100,117120,117140,117160,117180,117200,117220,117240,117260,117280,117300,117320,117340,117360,117380,117400,117420,117440,117460,117480,117500,117520,117540,117560,117580,117600,117620,117640,117660,117680,117700,117720,117740,117760,117780,117800,117820,117840,117860,117880,117900,117920,117940,117960,117980,118000,118020,118040,118060,118080,118100,118120,118140,118160,118180,118200,118220,118240,118260,118280,118300,118320,118340,118360,118380,118400,118420,118440,118460,118480,118500,118520,118540,118560,118580,118600,118620,118640,118660,118680,118700,118720,118740,118760,118780,118800,118820,118840,118860,118880,118900,118920,118940,118960,118980,119000,119020,119040,119060,119080,119100,119120,119140,119160,119180,119200,119220,119240,119260,119280,119300,119320,119340,119360,119380,119400,119420,119440,119460,119480,119500,119520,119540,119560,119580,119600,119620,119640,119660,119680,119700,119720,119740,119760,119780,119800,119820,119840,119860,119880,119900,119920,119940,119960,119980,120000,120020,120040,120060,120080],"ends":[500,760],"target":[[0,120],[75,100],[175,80],[225,60],[250,40],[1000,100],[1070,80],[1120,60],[1145,40]]} diff --git a/doc/concept/hang.md b/doc/concept/hang.md index c8ec33f8394..2962d673e7a 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -70,7 +70,7 @@ A few things the catalog can express beyond decoder config: - **Reorder depth.** A video rendition may declare `reorder`, the most frames that can precede one in decode order and follow it in presentation, as H.264 `max_num_reorder_frames` counts them; `0` declares none, which is not the same as absent. The Rust H.264 and H.265 importers copy it from the SPS when the SPS declares one (progressive H.264 only), and `@moq/publish` declares `0`. The MPEG-TS, Matroska, and FLV exporters then derive each group's decode times from that group's frames alone, so exports that joined at different groups agree, and they end the export on a frame reordered deeper than declared or a depth the codec level cannot reach. Without it they infer decode times from the timestamps they have read, as before. A publisher never changes it once declared. - **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, an AV1 track's av1C, or Matroska, but not yet from H.264 or H.265 in fMP4 or FLV. Nothing in `@moq/watch` reads it yet. - **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. Nothing in `@moq/watch` renders them yet. -- **Disabled renditions.** An audio or video rendition with `enabled: false` has no new frames coming, and a player should not select it. A subscriber may still get audio retained from before the disable, as when it subscribed a moment earlier. A player may keep a disabled rendition selected, and subscribed, while no enabled alternative exists, so a re-enable plays without a resubscribe; `@moq/watch` does. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. +- **Disabled renditions.** An audio or video rendition with `enabled: false` has no new frames coming, and a player must not select it. A subscriber may still get audio retained from before the disable, as when it subscribed a moment earlier. Disabling a rendition in `@moq/publish`, as hiding video does, keeps it in the catalog this way rather than removing it, so it is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. A muted `@moq/publish` microphone is not disabled: it stays enabled and sends nothing, ending each utterance with a discontinuity marker, so listeners stay subscribed and play the gap as intentional silence. - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. - **Join interval.** The root `media.join.interval` promises that every audio and video track starts a group at least that often: the longest GOP any of them publishes. It is fixed for the broadcast, and the [HLS gateway](/bin/hls) numbers every rendition's segments on the grid it fixes, segments the smallest whole multiple of it at least a second long, from each group's first timestamp, so renditions share segment numbers. A publisher that knows its GOP declares it; Rust publishers (`moq-mux` and everything built on it) without one measure each track over its first second or group, whichever is longer, and `@moq/publish` declares its keyframe interval. A broadcast without it gets no HLS playlists. - **Clock.** The optional `media.clock` maps PTS zero to wall time so every media track shares one fixed epoch after timescale conversion. Publish independent sources as sibling broadcasts under one prefix, such as `robot/front.hang` and `robot/rear.hang`; each broadcast's audio and video entries are renditions of that source. In one Rust publisher, pass the same `moq_mux::Clock` to each `catalog::Config::with_clock`; JS publishers on one page already share the page clock. Consumers discover the siblings by prefix and align their timestamps through the advertised wall mapping; `@moq/watch` plays one broadcast at a time. diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index ee0d58cddfc..10ee8b867f6 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -119,7 +119,7 @@ inside. The header is the reference: each function carries a doc comment. - **Stats.** `moq_session_stats()` fills `moq_connection_stats` with `rtt_us`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`). Each has a `_valid` flag, false when the transport does not report it, which is not the same as zero. `moq_session_snapshot()` adds the negotiated `protocol`. A failed dial or dropped connection reaches the session's `on_state` with its `failure` code, and `moq_error()` gives the reason during that callback. Hold the handle returned by `moq_session_recv_bandwidth()` to request receive estimates. `moq_bandwidth_consumer_peek()` reads the estimate and `moq_bandwidth_consumer_changed()` waits for a change. `moq_bandwidth_consumer_close()` releases demand; the last consumer stops PROBE. Stats snapshots do not request it. - **Viewers.** `moq_session_viewers()` returns a handle that divides the receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `moq_viewers_watch()` returns a viewer claiming up to `max_bps`, the bitrate of the largest rendition you would switch to; `moq_viewer_rate()` reads its share in bits per second, with `present` false while there is no estimate (keep the current rendition), and `moq_viewer_update()` changes the ceiling. Holding a viewer requests the estimate like a `moq_session_recv_bandwidth()` consumer, and `moq_viewer_close()` releases the claim. The session sends the summed ceilings as its PROBE target; `moq_viewers_set_target()` with `present` true overrides that, a `0` target sends none, and `present` false follows the viewers again. `moq_viewers_watch()`, `moq_viewer_update()`, and `moq_viewers_set_target()` return `MOQ_ERROR_OUT_OF_RANGE`, changing nothing, when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. `moq_viewers_close()` releases the registry handle; viewers watched through it stay until each is closed. - **Demand.** A watcher from `moq_publish_track_demand()` (or the media, video, and audio variants) calls back `MOQ_DEMAND_WANTED` or `MOQ_DEMAND_UNWANTED`, again whenever the wanted group moves. `moq_publish_demand_wanted()` reads that group: start a new one once it passes your largest. Set `refresh` in `moq_track_info` to have a joining subscriber ask for a new group once the newest has grown past `refresh` times its first frame. A producer publishes everything you write, subscribed or not, so a late subscriber can fetch the recent past. Capturing only while the watcher reports `MOQ_DEMAND_WANTED` is an opt-in that saves the encode but leaves nothing from before the first subscriber. -- **Disabled renditions.** A rendition with `enabled: false` has no frames coming until its publisher re-enables it. Query the latest catalog with `moq_consume_audio_enabled` or `moq_consume_video_enabled` before selecting a rendition and opening a decoder. Keep one already playing while every rendition of its kind is disabled: the convenience decoders accept a disabled configuration, and a re-enable, such as an unmute, then plays without a resubscribe. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. +- **Disabled renditions.** Audio and video consumers and decoders reject a rendition with `enabled: false`. Query the latest catalog with `moq_consume_audio_enabled` or `moq_consume_video_enabled` before selecting a rendition and opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. ## Reference diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index 56f5d8c7fe3..62b42ad3d5e 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -208,7 +208,7 @@ future completed, which is the executor thread. ## Disabled renditions -A rendition with `enabled: false` has no frames coming until its publisher re-enables it. Select an enabled rendition from the latest catalog before opening a decoder. Keep one already playing while every rendition of its kind is disabled: the convenience decoders accept a disabled configuration, and a re-enable, such as an unmute, then plays without a resubscribe. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq::Error::kUnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. +Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq::Error::kUnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. ## Bandwidth estimates diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index 23552e5633c..c5df8857bf0 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -159,7 +159,7 @@ shows which module holds each. - **Authorization.** On `moq-lite-07-wip` (opt in with `ConnectOptions(versions: ...)`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), listen to `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. - **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. - **Errors.** Read diagnostic payloads from `reason` and structured protocol failures from `details`. -- **Disabled renditions.** A rendition with `enabled: false` has no frames coming until its publisher re-enables it. Select an enabled rendition from the latest catalog before opening a decoder. Keep one already playing while every rendition of its kind is disabled: the convenience decoders accept a disabled configuration, and a re-enable, such as an unmute, then plays without a resubscribe. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `UnsupportedConfigMoqException` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. +- **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `UnsupportedConfigMoqException` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** A bare `Subscription()`, passing no subscription, and `defaultSubscription()` all mean moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once. `Delay` defaults both bounds to zero: pass `max: null` for no subscriber limit, or `min: null` to leave the target latency unstated. Decoders take the same `Delay`. - **Track demand.** Obtain a dynamic track request’s `demand()` before accepting it to watch subscribers through the handoff to its producer. The watch does not keep the track alive and ends when the track closes or the request is rejected. A producer publishes everything you write, subscribed or not, so a late subscriber can fetch the recent past. Capturing only while a producer's `demand()` reports a subscriber is an opt-in that saves the encode but leaves nothing from before the first one. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index b595ded6eb4..f364497668d 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -137,7 +137,7 @@ shows which module holds each. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig.Versions`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `Session().Grants()` returns a `GrantConsumer`: `Grant()` is the `Grant` the relay sent back, nil until it answers and while a client session reconnects, and `Changed(ctx)` waits for the next one. `Close()` it when done. - **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. - **Errors.** Read diagnostic payloads from `Reason` and structured protocol failures from `Details`. -- **Disabled renditions.** A rendition with `enabled: false` has no frames coming until its publisher re-enables it. Select an enabled rendition from the latest catalog before opening a decoder. Keep one already playing while every rendition of its kind is disabled: the convenience decoders accept a disabled configuration, and a re-enable, such as an unmute, then plays without a resubscribe. Among the video renditions you can decode, prefer the highest `Preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq.ErrUnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. +- **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `Preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq.ErrUnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** A zero `moq.Subscription{}`, a nil `*Subscription`, and `moq.DefaultSubscription()` all mean moq-net's defaults, whose zero `Delay` starts at the newest group and skips a superseded one at once. Set `Delay.MaxUnlimited` for no subscriber limit, and `Delay.MinUnstated` to leave the target latency unstated. Decoders take the same `moq.Delay`. - **Track demand.** Obtain a dynamic track request’s `Demand()` before accepting it to watch subscribers through the handoff to its producer. The watch does not keep the track alive and ends when the track closes or the request is rejected. A producer publishes everything you write, subscribed or not, so a late subscriber can fetch the recent past. Capturing only while a producer's `Demand()` reports a subscriber is an opt-in that saves the encode but leaves nothing from before the first one. diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index aeff8755be2..5b4d8e84856 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -31,7 +31,7 @@ WebCodecs, writes the catalog, and publishes a hang broadcast. | `url`, `name` | Relay URL (with `?jwt=` if needed) and broadcast name. | | `versions` | Protocol versions to offer, most preferred first, comma-separated like `moq-transport-18,moq-lite-06`. Every stable version by default. | | `source` | `camera`, `screen`, or `file`. | -| `muted`, `invisible` | Stop publishing audio or video. By default the microphone and camera stay open, so unmuting or unhiding is instant and the browser's capture indicators stay lit. | +| `muted`, `invisible` | Stop sending audio, or stop publishing video. A muted rendition stays in the catalog and its listeners stay subscribed, so the next word reaches them at once, as push-to-talk needs; they play the pause as comfort noise. By default the microphone and camera stay open, so unmuting or unhiding is instant and the browser's capture indicators stay lit. | | `capture` | What a muted or invisible track does with its device: `hold` (default) keeps it open, `release` closes it so its capture indicator goes dark, and resuming reopens it. A screen share or file stops once both tracks are off. Clearing `source` or removing the element releases everything either way. | | `preview` | What the nested element shows: the raw `source` (default), a decoded copy of the `encoded` stream to see what viewers get, or `none`. | | `backend` | How a `` preview is drawn; see [@moq/video](/lib/js/video). `auto` (default), `webgpu`, or `2d`. | @@ -82,9 +82,12 @@ audio still keeps 30 seconds, since raw history needs a bound. The catalog and HLS record tracks keep `@moq/net`'s zero default, since a viewer needs only their newest group. Capture stays open. Disabling an audio encoder keeps what it retained: a subscriber still gets that history, and it expires on its own. Players skip a -disabled rendition, so that history reaches subscribers that are already held -on it or that subscribe directly. The codec stays -configured across the disable, so re-enabling resumes at once. +disabled rendition, so that history reaches only subscribers that ask for the +track directly. The codec stays configured across the disable, so re-enabling +resumes at once. To pause audio that comes back, such as push-to-talk, set the +encoder's `muted` instead: it sends nothing, keeps the rendition enabled, and +marks where each utterance ends, so listeners stay subscribed and treat the gap +as intentional silence. Pass `onDemand: false` to `Publish.Broadcast` to encode audio continuously too. `onDemand: true` also gates video, which then loses its unsubscribed past. diff --git a/doc/lib/js/room.md b/doc/lib/js/room.md index e97f4a7f367..e3c8c5dbd77 100644 --- a/doc/lib/js/room.md +++ b/doc/lib/js/room.md @@ -41,7 +41,9 @@ const room = new Room({ connection, identity: Path.from("alice") }); the camera's hd and sd renditions encode even when nobody in the room watches, so a member who arrives late can fetch the last few seconds. While `microphoneEnabled` is on, the microphone keeps raw audio and encodes it once -someone listens. Turn either off to stop that capture. +someone listens. Turn either off to stop that capture. Turning the microphone +off mutes it rather than removing it, so listeners stay subscribed and hear +the next word at once. On a public prefix, skip the token and dial that path directly. The conferencing demo at [`demo/web`](https://github.com/moq-dev/moq/tree/main/demo/web) (`meet.html`) diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index b2c524ba5ba..11786fe0fef 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -113,7 +113,7 @@ shows which module holds each. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig(versions = ...)`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), collect `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. Ending the collection cancels the consumer. - **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. - **Errors.** Read diagnostic payloads from `reason` and structured protocol failures from `details`. -- **Disabled renditions.** A rendition with `enabled: false` has no frames coming until its publisher re-enables it. Select an enabled rendition from the latest catalog before opening a decoder. Keep one already playing while every rendition of its kind is disabled: the convenience decoders accept a disabled configuration, and a re-enable, such as an unmute, then plays without a resubscribe. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `MoqException.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. +- **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `MoqException.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** `Subscription.default()` returns moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once; passing no subscription means the same. `copy` it to change a field; building a `Subscription(...)` must name its `Delay(min, max)`, where a null `max` sets no subscriber limit and a null `min` leaves the target latency unstated. Decoders take the same `Delay`. - **Track demand.** Obtain a dynamic track request’s `demand()` before accepting it to watch subscribers through the handoff to its producer. The watch does not keep the track alive and ends when the track closes or the request is rejected. A producer publishes everything you write, subscribed or not, so a late subscriber can fetch the recent past. Capturing only while a producer's `demand()` reports a subscriber is an opt-in that saves the encode but leaves nothing from before the first one. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 3d2fd2640a7..a9578d94f9b 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -129,7 +129,7 @@ shows which module holds each. - **Authorization.** On `moq-lite-07-wip` (opt in with `versions=`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `async for grant in session.grants()` yields the `Grant` the relay sent back and each change, `None` until it answers and while a client session reconnects. `grants().grant()` reads it once; cancelling or leaving an `async with` releases the wait. - **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. - **Errors.** Read diagnostic payloads from `reason` and structured protocol failures from `details`. -- **Disabled renditions.** A rendition with `enabled: false` has no frames coming until its publisher re-enables it. Select an enabled rendition from the latest catalog before opening a decoder. Keep one already playing while every rendition of its kind is disabled: the convenience decoders accept a disabled configuration, and a re-enable, such as an unmute, then plays without a resubscribe. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq.Error.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. +- **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq.Error.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** A bare `moq.Subscription()`, passing no subscription, and `moq.Subscription.default()` all mean moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once. Pass `delay=moq.Delay(max=None)` for no subscriber limit, or `min=None` to leave the target latency unstated. Decoders take the same `moq.Delay`. - **Track demand.** Obtain a dynamic track request’s `demand()` before accepting it to watch subscribers through the handoff to its producer. The watch does not keep the track alive and ends when the track closes or the request is rejected. A producer publishes everything you write, subscribed or not, so a late subscriber can fetch the recent past. Capturing only while a producer's `demand()` reports a subscriber is an opt-in that saves the encode but leaves nothing from before the first one. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 4cf0183aee2..726940de7e6 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -111,7 +111,7 @@ shows which module holds each. - **Authorization.** On `moq-lite-07-wip` (opt in with `Client(versions:)`) and on moq-transport draft-18+ with the [MoQ Auth extension](/draft/moq-auth), `for try await grant in session.grants()` yields the `Grant` the relay sent back and each change, `nil` until it answers and while a client session reconnects. Cancelling the iterating task cancels the native wait. - **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. - **Errors.** Diagnostic cases use the `reason:` argument label; match their associated value to inspect it. Protocol errors carry `details`. -- **Disabled renditions.** A rendition with `enabled: false` has no frames coming until its publisher re-enables it. Select an enabled rendition from the latest catalog before opening a decoder. Keep one already playing while every rendition of its kind is disabled: the convenience decoders accept a disabled configuration, and a re-enable, such as an unmute, then plays without a resubscribe. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `MoqError.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. +- **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `MoqError.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** `Subscription.default` holds moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once; passing no subscription means the same. Copy it to change a field; building a `Subscription(...)` must name its `Delay(min:max:)`, where a `nil` `max` sets no subscriber limit and a `nil` `min` leaves the target latency unstated. Decoders take the same `Delay`. - **Track demand.** Obtain a dynamic track request’s `demand()` before accepting it to watch subscribers through the handoff to its producer. The watch does not keep the track alive and ends when the track closes or the request is rejected. A producer publishes everything you write, subscribed or not, so a late subscriber can fetch the recent past. Capturing only while a producer's `demand()` reports a subscriber is an opt-in that saves the encode but leaves nothing from before the first one. diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 11a905b72f8..fa80b68944b 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -829,8 +829,7 @@ For example, with video at 20 millisecond intervals and audio at 15 written as v ### enabled {#field-enabled} The `enabled` field says whether a consumer may select an audio or video rendition; a text rendition does not carry it. -When `enabled` is false, no frames are coming, and a consumer SHOULD NOT select the rendition. -A consumer MAY keep a rendition it already selected while that rendition is disabled and no enabled one it can decode remains, so a re-enable needs no new subscription. +When `enabled` is false, no frames are coming, and a consumer MUST NOT select the rendition. If absent, `enabled` defaults to true; a publisher SHOULD only write it when false. Earlier versions defined a video `stalled` field instead. @@ -1148,7 +1147,7 @@ This document has no IANA actions. - Added the optional `delay` rendition field: how far a rendition's timestamps trail those other renditions already wrote, never lowered once advertised and never subtracted across renditions. - Recommended namespaced keys for application root sections. - Added the optional `delay` field to `json` and `binary` track entries, measured only from payloads stamped with their capture time on the broadcast clock. -- Replaced the video `stalled` field with an optional `enabled` field on audio and video renditions. A consumer SHOULD NOT select a disabled rendition, MAY keep one it already selected while no enabled alternative remains, and ignores `stalled`. A consumer that predates `enabled` keeps selecting a disabled rendition. +- Replaced the video `stalled` field with an optional `enabled` field on audio and video renditions. A consumer MUST NOT select a disabled rendition and ignores `stalled`. A consumer that predates `enabled` keeps selecting a disabled rendition. - Audio `sampleRate` and `numberOfChannels` describe the decoded output, so HE-AAC names its SBR rate and parametric stereo names two channels. - Added an optional `track` field to video, audio, text, `json`, and `binary` entries naming the track in its broadcast; the entry's key is its ID and names the track when `track` is absent. A consumer MUST reject a catalog using one ID for both a video and an audio rendition. - Defined video `displayAspectWidth` and `displayAspectHeight` as the whole picture's display aspect, not the sample aspect. diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index c84397adf74..1ca49041eb7 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -61,8 +61,7 @@ const AudioConfigFields = z.object({ bitrate: z.optional(u53Schema), // Whether this rendition may be selected. When false, no frames are coming and a consumer - // should not select it, though one may keep it selected while no enabled alternative exists. - // Default: true, so publishers only write it when false. + // must not select it. Default: true, so publishers only write it when false. enabled: z.optional(z.boolean()), // The maximum delay between a frame being ready and the publisher flushing it, in whole diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index adbb69f3bbc..a04b81d3aad 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -178,8 +178,7 @@ export const VideoConfigSchema = z.object({ bitrate: z.optional(u53Schema), // Whether this rendition may be selected. When false, no frames are coming and a consumer - // should not select it, though one may keep it selected while no enabled alternative exists. - // Default: true, so publishers only write it when false. + // must not select it. Default: true, so publishers only write it when false. enabled: z.optional(z.boolean()), // Higher is preferred: a player selects among the highest preference it can decode, then by diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 159acea7fb6..d5090a4762a 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -856,6 +856,51 @@ test("Consumer measures arrivals as they land", async () => { } }); +test("Consumer does not relax its measurement across a declared silence", async () => { + const clock = new Manual(); + const installed = clock.install(); + // Two identical paths, except that one publisher declares its silence and the other just stops. + const declared = new Track.Producer("declared"); + const undeclared = new Track.Producer("undeclared"); + const options = { format: new LegacyFormat("audio"), delay: { max: 5000 as Milli } }; + const quiet = new Consumer(declared.subscribe(), options); + const lapsed = new Consumer(undeclared.subscribe(), options); + const producers = [new LegacyProducer(declared, options.format), new LegacyProducer(undeclared, options.format)]; + const payload = new Uint8Array([1]); + let media = 0; + const send = async (count: number) => { + for (let i = 0; i < count; i++) { + for (const producer of producers) producer.encode(payload, (media * 1000) as Micro, true); + media += 520; + await clock.advance(Nano.fromMilli(Milli(520))); + } + }; + + try { + // Arrivals exactly as fast as their media until the cold-start prior is spent. + await send(40); + const settled = quiet.spread.peek(); + expect(lapsed.spread.peek()).toBe(settled); + + // A minute of silence on both, declared on one. + producers[0].discontinuity(); + media += 60_000; + await clock.advance(Nano.fromMilli(Milli(60_000))); + await send(2); + + expect(quiet.spread.peek()).toBe(settled); + expect(lapsed.spread.peek()).toBeGreaterThan(settled); + } finally { + for (const track of [declared, undeclared]) { + track.finish(); + track.close(); + } + quiet.close(); + lapsed.close(); + installed.close(); + } +}); + // --- Ordering --- test("Consumer delivers groups in sequence order regardless of arrival order", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 50c8e5afbca..9ee90a2fef3 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -221,6 +221,11 @@ export class Consumer { * floor. Starts at the predecessor's estimate, or the cold-start estimate without one. */ readonly spread: Getter; + // The newest group opened by a declared end, until a later group's first frame tells the jitter + // estimator the silence is over. Not at the marker itself: the stopped encoder's final packets may + // follow it in its own group, and they belong to the speech before the silence. + #declared?: bigint; + #mediaSequence?: bigint; #signals = new Effect(); #closed = new Once(); @@ -378,6 +383,13 @@ export class Consumer { } } + // Note an end the publisher declared in `group`, which a later group's first frame resumes from. + #declare(group: Group): void { + const sequence = group.consumer.sequence; + if (this.#mediaSequence !== undefined && sequence < this.#mediaSequence) return; + if (this.#declared === undefined || sequence > this.#declared) this.#declared = sequence; + } + // Decode one wire frame into `group`, charging what it buffers. A payload that fails to decode sets // the group's fault, which next() raises once the frames ahead of it are read. #decode(group: Group, wire: Moq.Group.Frame): void { @@ -401,6 +413,7 @@ export class Consumer { return; } group.bare = decoded.length === 0; + if (group.bare) this.#declare(group); } // Decoded payloads are views of the wire payload, which stays allocated until the last of them is @@ -412,8 +425,17 @@ export class Consumer { // Observed here, before any gap handling or age budget: a target derived from // what survives the budget would only ever confirm the budget it was cut to. if (!marker) { + if (this.#mediaSequence === undefined || group.consumer.sequence > this.#mediaSequence) + this.#mediaSequence = group.consumer.sequence; + if (this.#declared !== undefined && group.consumer.sequence > this.#declared) { + this.#jitter.end(); + this.#declared = undefined; + } this.#jitter.observe(arrival, Milli.fromMicro(sample.timestamp)); this.#spread.set(this.#jitter.measured); + } else if (!group.media) { + // Only a marker ahead of any media declares an end; one after it is a frame's endpoint. + this.#declare(group); } const frame: ReceivedFrame = { arrival, diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts index 4e2c8c32bcc..c346af8d378 100644 --- a/js/hang/src/container/jitter.test.ts +++ b/js/hang/src/container/jitter.test.ts @@ -16,6 +16,7 @@ interface Case { delay?: number; arrival: number[]; media: number[]; + ends?: number[]; target: [number, number][]; } @@ -41,6 +42,7 @@ describe("audio jitter conformance", () => { "idle", "paced", "reorder", + "silence", "spike", "straggler", "tunein", @@ -51,8 +53,10 @@ describe("audio jitter conformance", () => { it(c.name, () => { const jitter = new Jitter(); const expected = new Map(c.target); + const ends = new Set(c.ends); let current: number | undefined; for (let i = 0; i < c.arrival.length; i++) { + if (ends.has(i)) jitter.end(); jitter.observe(c.arrival[i] as Milli, c.media[i] as Milli); const value = target(jitter, c); const listed = expected.get(i); @@ -84,6 +88,17 @@ describe("Jitter", () => { expect(jitter.measured).toBe(20 as Milli); }); + it("ignores an end before any observation", () => { + // A cold estimator has no interval for a silence to stretch, so the end must not linger and + // excuse the first unexplained gap. + const jitter = new Jitter(); + jitter.end(); + for (let i = 0; i < 500; i++) jitter.observe((50 + i * 20) as Milli, (i * 20) as Milli); + const settled = jitter.measured; + jitter.observe((50 + 500 * 20 + 30_000) as Milli, (500 * 20 + 30_000) as Milli); + expect(jitter.measured).toBeGreaterThan(settled); + }); + it("does not follow the absolute delay", () => { // Only the spread of arrivals matters, not how far away the sender is. const near = new Jitter(); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts index 0759bf70b6d..257beda9c2d 100644 --- a/js/hang/src/container/jitter.ts +++ b/js/hang/src/container/jitter.ts @@ -61,6 +61,8 @@ export class Jitter { #interval = 0; /** The largest delay seen in that interval. */ #peak = 0; + /** A declared end precedes the next measured observation, so the silence before it is not idle. */ + #ended = false; #histogram = Float64Array.from(PRIOR); /** How much of the histogram is still the prior's rather than evidence. */ @@ -93,6 +95,8 @@ export class Jitter { } const interval = Math.floor((arrival - this.#origin) / RESAMPLE); + const resumed = this.#ended; + this.#ended = false; if (interval <= this.#interval) { this.#peak = Math.max(this.#peak, delay); return; @@ -100,14 +104,30 @@ export class Jitter { this.#fold(Math.min(BUCKETS - 1, Math.floor(this.#peak / BUCKET))); - // An interval with no arrival relaxes the estimate toward the prior at the same rate. - const idle = Math.min(interval - this.#interval - 1, IDLE_MAX); - for (let i = 0; i < idle; i++) this.#fold(undefined); - - this.#interval = interval; + if (resumed) { + // A silence the publisher declared says nothing about the path, so the estimate keeps + // what it had and a fresh interval starts here. + this.#origin = arrival; + this.#interval = 0; + } else { + // An interval with no arrival relaxes the estimate toward the prior at the same rate. + const idle = Math.min(interval - this.#interval - 1, IDLE_MAX); + for (let i = 0; i < idle; i++) this.#fold(undefined); + this.#interval = interval; + } this.#peak = delay; } + /** + * A declared end lies between the previous observation and the next, so the silence it declares + * does not relax the estimate the way an unexplained gap does. Call it before observing the first + * frame after the end, not at the marker: the stopped encoder's final packets may still follow it. + */ + end(): void { + // Before any observation there is no interval for the silence to stretch. + if (this.#origin !== undefined) this.#ended = true; + } + /** * A new estimator that keeps what this one learned but measures arrivals afresh, for a track that * replaces this one on the same path. Its frames are measured against their own fastest, never diff --git a/js/publish/MIGRATE.md b/js/publish/MIGRATE.md index bbe065a4282..b3ec5506455 100644 --- a/js/publish/MIGRATE.md +++ b/js/publish/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Muting suppresses audio instead of disabling the rendition ([#7942](https://github.com/moq-dev/moq/pull/7942)) + +`` no longer marks the audio rendition `enabled: false`: it sends nothing, the rendition stays enabled in the catalog, and listeners stay subscribed, so the next word reaches them at once. +Each muted span ends with a discontinuity marker, which players treat as intentional silence. +The same is available on `Audio.Encoder` as the new `muted` knob; keep `enabled: false` for a stop the rendition does not come back from soon. +A viewer that read `enabled: false` in the catalog as "muted" no longer sees one; publish that state in your own catalog section or data track. + ### The capture graph renders to no output device ([#7881](https://github.com/moq-dev/moq/pull/7881)) `Audio.Capture`'s and `Audio.Encoder`'s `root` now sit in an `AudioContext` built with `sinkId: { type: "none" }`, which shortens the bursts capture hands the encoder. diff --git a/js/publish/api/index.api.md b/js/publish/api/index.api.md index 9ecb9d909da..c1005dd60ac 100644 --- a/js/publish/api/index.api.md +++ b/js/publish/api/index.api.md @@ -42,6 +42,7 @@ export namespace Audio { codec: Signals.Signal; groupDuration: Signals.Signal; readonly in: Signals.Readonlys; + muted: Signals.Signal; readonly name: string; readonly out: Signals.Readonlys; volume: Signals.Signal; @@ -62,6 +63,7 @@ export namespace Audio { export type EncoderProps = Signals.Inputs & { codec?: Audio.Codec | Signals.Signal; groupDuration?: Clock.Milli | Signals.Signal; + muted?: boolean | Signals.Signal; volume?: number | Signals.Signal; }; export interface Format { diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index f31e1f2ffe5..578b318b146 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -96,14 +96,15 @@ describe("toEncoderConfig", () => { }); }); -// Like Chrome's Opus encoder, it holds the newest chunks until later input pushes them out, and -// stamps each chunk from the first input's timestamp plus the audio encoded since, so a jump in -// input timestamps never reaches the output. -class LaggingAudioEncoder { - static instances: LaggingAudioEncoder[] = []; +// Like Chrome's Opus encoder, it emits one chunk per input and stamps each from the first input's +// timestamp plus the audio encoded since, so a jump in input timestamps never reaches the output. +// `LAG` holds that many of the newest chunks until later input pushes them out, a slower codec than +// any browser measured, for the tests that need one. +class FakeAudioEncoder { + static instances: FakeAudioEncoder[] = []; static probes: AudioEncoderConfig[] = []; static description: ((config: AudioEncoderConfig) => Uint8Array) | undefined; - static readonly LAG = 2; + static LAG = 0; // Called on configure; the encoder publishes its pipeline synchronously right after. static onConfigure: ((config: AudioEncoderConfig) => void) | undefined; @@ -122,7 +123,7 @@ class LaggingAudioEncoder { constructor(init: AudioEncoderInit) { this.#output = init.output; this.#error = init.error; - LaggingAudioEncoder.instances.push(this); + FakeAudioEncoder.instances.push(this); } fail(error: Error): void { @@ -136,7 +137,7 @@ class LaggingAudioEncoder { this.#fresh = true; queueMicrotask(() => { this.#fresh = false; - if (!this.#probe) LaggingAudioEncoder.onConfigure?.(config); + if (!this.#probe) FakeAudioEncoder.onConfigure?.(config); }); } @@ -145,7 +146,7 @@ class LaggingAudioEncoder { this.#base ??= data.timestamp; this.#held.push({ timestamp: this.#base + this.#encoded, duration }); this.#encoded += duration; - while (this.#held.length > LaggingAudioEncoder.LAG) { + while (this.#held.length > FakeAudioEncoder.LAG) { this.#emit(this.#held.shift() as { timestamp: number; duration: number }); } } @@ -153,7 +154,7 @@ class LaggingAudioEncoder { #emit({ timestamp, duration }: { timestamp: number; duration: number }): void { const chunk = { type: "key", timestamp, duration, byteLength: 1, copyTo: (dest: Uint8Array) => dest.set([1]) }; const config = this.#config; - const description = config && LaggingAudioEncoder.description?.(config); + const description = config && FakeAudioEncoder.description?.(config); const metadata = config && description ? { @@ -171,9 +172,9 @@ class LaggingAudioEncoder { async flush(): Promise { if (this.#fresh) { this.#probe = true; - if (this.#config) LaggingAudioEncoder.probes.push(structuredClone(this.#config)); - const index = LaggingAudioEncoder.instances.indexOf(this); - if (index !== -1) LaggingAudioEncoder.instances.splice(index, 1); + if (this.#config) FakeAudioEncoder.probes.push(structuredClone(this.#config)); + const index = FakeAudioEncoder.instances.indexOf(this); + if (index !== -1) FakeAudioEncoder.instances.splice(index, 1); } for (const chunk of this.#held.splice(0)) this.#emit(chunk); } @@ -206,12 +207,13 @@ class FakeAudioData { } function installFakeWebCodecs() { - LaggingAudioEncoder.instances = []; - LaggingAudioEncoder.probes = []; - LaggingAudioEncoder.description = undefined; + FakeAudioEncoder.instances = []; + FakeAudioEncoder.probes = []; + FakeAudioEncoder.LAG = 0; + FakeAudioEncoder.description = undefined; const names = ["AudioEncoder", "AudioDecoder", "AudioData"] as const; const originals = names.map((name) => Object.getOwnPropertyDescriptor(globalThis, name)); - const fakes = [LaggingAudioEncoder, class {}, FakeAudioData]; + const fakes = [FakeAudioEncoder, class {}, FakeAudioData]; names.forEach((name, i) => { Object.defineProperty(globalThis, name, { configurable: true, writable: true, value: fakes[i] }); }); @@ -284,8 +286,8 @@ function fakeCapture(feed: Feed) { // Resolves on the next AudioEncoder configure; the encoder publishes its pipeline synchronously after. function configured(): Promise { return new Promise((resolve) => { - LaggingAudioEncoder.onConfigure = (config) => { - LaggingAudioEncoder.onConfigure = undefined; + FakeAudioEncoder.onConfigure = (config) => { + FakeAudioEncoder.onConfigure = undefined; resolve(config); }; }); @@ -371,6 +373,7 @@ async function setup(order = new Order(), codec?: Codec, groupDuration?: Milli) // not the encoder's gap-blind one, or they trail the next gap's marker and are dropped as pre-gap. test("an on-demand gap marks where submitted audio ends and drops the chunks held across it", async () => { using _webcodecs = installFakeWebCodecs(); + FakeAudioEncoder.LAG = 2; using env = await setup(); const { track, rendition, feed, written, writes } = env; @@ -407,8 +410,8 @@ test("an on-demand gap marks where submitted audio ends and drops the chunks hel // A pause with a subscriber attached breaks the timeline just as losing demand does: a subscriber // that stays across it, or joins during it, must not read the audio before it as live. The codec -// keeps its configuration, so the audio it still held is flushed out before the marker. -test("disabling with a subscriber attached flushes the held audio, then marks where it ends", async () => { +// keeps its configuration and is not flushed. +test("disabling with a subscriber attached marks where the submitted audio ends", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(); const { enabled, feed, written } = env; @@ -421,14 +424,14 @@ test("disabling with a subscriber attached flushes the held audio, then marks wh await feed.drain(); }; - await push(4); // two written, two held + await push(4); enabled.set(false); await settle(); await push(2); // nothing publishing enabled.set(true); - await push(3); // the restarted codec, so one written and two held + await push(3); expect(written).toEqual([ [20_000, 1], @@ -437,14 +440,18 @@ test("disabling with a subscriber attached flushes the held audio, then marks wh [80_000, 1], [100_000, 0], [140_000, 1], + [160_000, 1], + [180_000, 1], ]); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.instances).toHaveLength(1); }); -// The codec has to restart across a capture gap to re-base its output clock, but the audio it still -// held predates the gap: it is flushed out first, and the marker follows it. -test("a capture gap flushes the held audio, then marks where it ends", async () => { +// A codec that still holds chunks at a capture gap stamps them, and the input after the gap, as one +// contiguous run. Stamping from what was submitted keeps each on its capture clock, and the marker +// waits behind the held chunks rather than dropping them as pre-gap. +test("a capture gap marks where the audio before it ends, after the chunks the codec holds", async () => { using _webcodecs = installFakeWebCodecs(); + FakeAudioEncoder.LAG = 2; using _now = manualAt(0); using env = await setup(); const { feed, written } = env; @@ -457,7 +464,7 @@ test("a capture gap flushes the held audio, then marks where it ends", async () }; await push(20_000, 4); // two written, two held - await push(200_000, 4); // the restarted codec, so two written and two held + await push(200_000, 4); // the held two, the marker, then two more written and two held expect(written).toEqual([ [20_000, 1], @@ -468,13 +475,14 @@ test("a capture gap flushes the held audio, then marks where it ends", async () [200_000, 1], [220_000, 1], ]); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.instances).toHaveLength(1); }); // A lost track already marked where its audio ends, so what the codec held for it must not open the // track that replaces it. test("a closed track's held audio stays off its replacement", async () => { using _webcodecs = installFakeWebCodecs(); + FakeAudioEncoder.LAG = 2; using _now = manualAt(0); using env = await setup(); const { track, rendition, feed } = env; @@ -542,7 +550,7 @@ test("re-enabling with an unchanged capture reuses the probe", async () => { const { encoder, enabled } = env; const before = encoder.out.catalog.peek(); - expect(LaggingAudioEncoder.probes).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); enabled.set(false); await settle(); @@ -551,7 +559,7 @@ test("re-enabling with an unchanged capture reuses the probe", async () => { enabled.set(true); await settle(2); expect(encoder.out.catalog.peek()).toEqual(before); - expect(LaggingAudioEncoder.probes).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); }); test("re-enabling with a changed sample rate probes again", async () => { @@ -568,7 +576,7 @@ test("re-enabling with a changed sample rate probes again", async () => { for (let config = await configuring; config.sampleRate !== 24_000; config = await configuring) { configuring = configured(); } - expect(LaggingAudioEncoder.probes.map((probe) => probe.sampleRate)).toEqual([48_000, 24_000]); + expect(FakeAudioEncoder.probes.map((probe) => probe.sampleRate)).toEqual([48_000, 24_000]); expect(encoder.out.catalog.peek()?.enabled).toBe(false); enabled.set(true); @@ -577,6 +585,71 @@ test("re-enabling with a changed sample rate probes again", async () => { expect(encoder.out.catalog.peek()?.enabled).toBeUndefined(); }); +// Push-to-talk: a mute sends nothing and ends the utterance with a marker, while the rendition stays +// enabled and its track stays the same, so a listener hears the next press with no catalog update. +test("muting sends nothing and marks where the audio ends, keeping the rendition enabled", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(); + const { encoder, feed, written, rendition } = env; + + const push = async (start: number, count: number) => { + for (let i = 0; i < count; i++) { + await feed.push({ timestamp: Micro(start + i * 20_000), channels: [new Float32Array(960)] }); + } + await feed.drain(); + }; + + const before = encoder.out.catalog.peek(); + expect(before?.enabled).toBeUndefined(); + await push(20_000, 2); + + encoder.muted.set(true); + await settle(); + expect(encoder.out.active.peek()).toBe(false); + await push(60_000, 3); // sent nowhere + expect(rendition.config.peek()).toEqual(before as never); + + encoder.muted.set(false); + await settle(); + expect(encoder.out.active.peek()).toBe(true); + await push(120_000, 2); + + expect(written).toEqual([ + [20_000, 1], + [40_000, 1], + [60_000, 0], + [120_000, 1], + [140_000, 1], + ]); + expect(rendition.config.peek()).toEqual(before as never); + expect(FakeAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); +}); + +// With `capture="release"` a mute closes the microphone, taking its format away. The rendition stays +// advertised, still enabled, through the mute and while the reopened microphone resolves. +test("a muted rendition keeps its config enabled while its capture is released", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(); + const { encoder, capture } = env; + + const before = encoder.out.catalog.peek(); + encoder.muted.set(true); + const format = capture.out.format.peek(); + capture.out.format.set(undefined as never); + await settle(); + expect(encoder.out.catalog.peek()).toEqual(before); + + encoder.muted.set(false); + await settle(); + expect(encoder.out.catalog.peek()).toEqual(before); + + capture.out.format.set(format); + await settle(10); + expect(encoder.out.catalog.peek()).toEqual(before); + expect(FakeAudioEncoder.probes).toHaveLength(1); +}); + // Chrome stamps encoder output from its first input, ignoring later jumps, so audio resumed after a // long mute must still carry its capture timestamps rather than trail them by the mute. test("audio resumed after a multi-second mute carries its capture timestamps", async () => { @@ -591,7 +664,7 @@ test("audio resumed after a multi-second mute carries its capture timestamps", a await feed.drain(); }; - await push(20_000, 4); // two written, two held + await push(20_000, 4); enabled.set(false); await settle(); @@ -599,7 +672,7 @@ test("audio resumed after a multi-second mute carries its capture timestamps", a // Capture keeps running through a five-second mute. await push(100_000, 250); enabled.set(true); - await push(5_100_000, 4); // two written, two held + await push(5_100_000, 4); expect(written).toEqual([ [20_000, 1], @@ -609,8 +682,10 @@ test("audio resumed after a multi-second mute carries its capture timestamps", a [100_000, 0], [5_100_000, 1], [5_120_000, 1], + [5_140_000, 1], + [5_160_000, 1], ]); - expect(LaggingAudioEncoder.probes).toHaveLength(1); + expect(FakeAudioEncoder.probes).toHaveLength(1); }); // Closing tears down the subscription and the pipeline, which both end the epoch; cleanups run @@ -623,21 +698,23 @@ test("closing with a subscriber attached marks the end once before the track clo for (let i = 0; i < 4; i++) { await feed.push({ timestamp: Micro(20_000 + i * 20_000), channels: [new Float32Array(960)] }); } - await feed.drain(); // two written, two held + await feed.drain(); encoder.close(); expect(written).toEqual([ [20_000, 1], [40_000, 1], + [60_000, 1], + [80_000, 1], [100_000, 0], ]); expect(track.closed.peek()).toBeDefined(); }); -// A push that completes several frames is still one continuous stream, so it must not restart the -// encoder and drop the chunks it holds. -test("a push completing several frames keeps the encoder running", async () => { +// A push that completes several frames is still one continuous stream, so it must not break the +// timeline between them. +test("a push completing several frames stays one stream", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(); const { feed, written } = env; @@ -653,6 +730,8 @@ test("a push completing several frames keeps the encoder running", async () => { [38_700, 1], [58_700, 1], [78_700, 1], + [98_700, 1], + [118_700, 1], ]); }); @@ -671,7 +750,7 @@ test("a group duration packs frames until the minimum and restarts after a break await feed.drain(); }; - await push(7); // five written, two held + await push(5); expect(groups).toEqual([5]); expect(appended[0].closed.peek()).toBeUndefined(); await push(2); // the next frame opens a new group and closes the first @@ -681,15 +760,16 @@ test("a group duration packs frames until the minimum and restarts after a break enabled.set(false); await settle(); + await push(2); // nothing publishing enabled.set(true); - await push(4); // the restarted codec, so two written and two held + await push(2); expect(written.map(([timestamp]) => timestamp)).toEqual([ - 20_000, 40_000, 60_000, 80_000, 100_000, 120_000, 140_000, 160_000, 180_000, 200_000, 200_000, 220_000, + 20_000, 40_000, 60_000, 80_000, 100_000, 120_000, 140_000, 160_000, 200_000, 220_000, ]); - // 120ms opens the next group, closing the five frames at 20-100ms. The break flushes 160-180ms - // into that group and ends it early, so resumed frames start a fresh group after the marker. - expect(groups).toEqual([5, 4, 1, 2]); + // 120ms opens the next group, closing the five frames at 20-100ms. The break ends that group + // early, so resumed frames start a fresh group after the marker. + expect(groups).toEqual([5, 2, 1, 2]); }); test("default audio groups carry twenty milliseconds", async () => { @@ -705,7 +785,7 @@ test("default audio groups carry twenty milliseconds", async () => { { mime: "opus", frameDuration: Milli(frameMs) }, groupMs === undefined ? undefined : Milli(groupMs), ); - for (let index = 0; index < packets * 2 + LaggingAudioEncoder.LAG; index++) { + for (let index = 0; index < packets * 2 + FakeAudioEncoder.LAG; index++) { await env.feed.push({ timestamp: Micro(20_000 + index * frameMs * 1000), channels: [new Float32Array(frameMs * 48)], @@ -750,6 +830,8 @@ test("the catalog bitrate only rises", async () => { // this one trailing it, which the catalog advertises as `delay` once a second of it has been measured. test("a rendition written behind another advertises delay", async () => { using _webcodecs = installFakeWebCodecs(); + // Each push writes the frame captured two before it, so the sibling stays ahead. + FakeAudioEncoder.LAG = 2; const clock = manualAt(200); try { @@ -760,8 +842,7 @@ test("a rendition written behind another advertises delay", async () => { expect(encoder.out.catalog.peek()?.delay).toBeUndefined(); // A sibling that stays 100ms ahead of the frames written, with a clock origin that keeps - // timestamps nonnegative. The encoder holds two frames, so each push writes the frame captured - // two before it. + // timestamps nonnegative. const sibling = new Estimator(); const start = Nano.toMicro(Nano(now())) - 100_000; const push = async (index: number) => { @@ -853,7 +934,7 @@ test("a failed audio encoder keeps its last catalog config disabled", async () = using _now = manualAt(0); using _error = spyOn(console, "error").mockImplementation(() => {}); using env = await setup(); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected live audio encoder"); const failure = new Error("audio encoder failed"); live.fail(failure); @@ -862,7 +943,7 @@ test("a failed audio encoder keeps its last catalog config disabled", async () = expect(env.encoder.out.error.peek()).toBe(failure); expect(env.encoder.out.catalog.peek()).toMatchObject({ codec: "opus", enabled: false }); expect(env.encoder.out.active.peek()).toBe(false); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.instances).toHaveLength(1); }); test("audio failure stays local through demand changes and capture format re-arms a fresh track", async () => { @@ -910,11 +991,11 @@ test("audio failure stays local through demand changes and capture format re-arm const siblingTrack = sibling.track.peek(); if (!siblingTrack) throw new Error("expected demanded sibling track"); expect(siblingTrack.appendGroup().sequence).toBe(0n); - await push(0, 1); - expect((await subscriber.largest())?.group).toBe(0n); + await push(0, 1); // a group per frame + expect((await subscriber.largest())?.group).toBe(2n); catalog = new Json.Snapshot.Consumer({ track: net.track(Broadcast.CATALOG_TRACK) }); expect((await catalog.latest())?.value.audio?.renditions.audio?.enabled).not.toBe(false); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected live audio encoder"); const failure = new Error("audio codec failed"); live.fail(failure); @@ -923,7 +1004,7 @@ test("audio failure stays local through demand changes and capture format re-arm expect(encoder.out.error.peek()).toBe(failure); expect((await catalog.latest())?.value.audio?.renditions.audio?.enabled).toBe(false); expect(encoder.out.active.peek()).toBe(false); - const instances = LaggingAudioEncoder.instances.length; + const instances = FakeAudioEncoder.instances.length; subscriber.close(); subscriber = net.track("audio").subscribe(); broadcast.audio("audio/new"); @@ -935,7 +1016,7 @@ test("audio failure stays local through demand changes and capture format re-arm expect(subscriber.closed.peek()).toBeDefined(); expect(encoder.out.error.peek()).toBe(failure); expect(encoder.out.catalog.peek()?.enabled).toBe(false); - expect(LaggingAudioEncoder.instances).toHaveLength(instances); + expect(FakeAudioEncoder.instances).toHaveLength(instances); expect(broadcast.net.peek()).toBe(net); expect(announced?.active.peek()?.instance).toBe(epoch); expect(siblingTrack.appendGroup().sequence).toBe(1n); @@ -953,7 +1034,7 @@ test("audio failure stays local through demand changes and capture format re-arm recovered = net.track(config.track).subscribe(); await settle(); await push(3, 2); - expect((await recovered.largest())?.group).toBe(0n); + expect((await recovered.largest())?.group).toBe(2n); expect(encoder.out.active.peek()).toBe(true); expect(recovered.closed.peek()).toBeUndefined(); expect(broadcast.net.peek()).toBe(net); @@ -983,7 +1064,7 @@ test("an audio failure without demand still recovers under a fresh track name", await settle(); const net = broadcast.net.peek(); if (!net) throw new Error("expected network broadcast"); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected audio encoder without demand"); const failure = new Error("idle audio encoder failed"); live.fail(failure); @@ -1028,7 +1109,7 @@ test.each(["failure first", "config first"])( await settle(); const net = broadcast.net.peek(); if (!net) throw new Error("expected network broadcast"); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected idle audio encoder"); const reconfiguring = configured(); if (order === "config first") encoder.codec.set({ mime: "opus", bitrate: 64_000 }); @@ -1084,7 +1165,7 @@ test.each([undefined, false, true])( }; await push(4); - expect(encoder.out.stats.peek().frames).toBe(onDemand === false ? 2 : 0); + expect(encoder.out.stats.peek().frames).toBe(onDemand === false ? 4 : 0); late = net.track("audio").subscribe({ delay: { max: Milli(10_000) } }); await settle(); expect(encoder.out.active.peek()).toBe(true); @@ -1155,7 +1236,7 @@ test("a backlog encode refusal aborts its rendition when demand arrives", async await settle(); await feed.push({ timestamp: Micro(0), channels: [new Float32Array(960)] }); await feed.drain(); - const live = LaggingAudioEncoder.instances.at(-1); + const live = FakeAudioEncoder.instances.at(-1); if (!live) throw new Error("expected configured encoder"); const failure = new Error("codec refused buffered input"); using _encode = spyOn(live, "encode").mockImplementation(() => { @@ -1203,13 +1284,14 @@ test("default audio replays the last second after three seconds without a subscr if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(); - expect(encoder.out.stats.peek().frames).toBe(50 - LaggingAudioEncoder.LAG); + expect(encoder.out.stats.peek().frames).toBe(50); for (let index = 150; index < 153; index++) { await feed.push({ timestamp: Micro(index * 20_000), channels: [new Float32Array(960)] }); } await feed.drain(); + // The subscription's one second reaches back from the newest frame, 152, to 102. const reader = late.ordered(); - for (let index = 100; index < 151; index++) { + for (let index = 102; index < 153; index++) { const group = await reader.nextGroup(); if (!group) throw new Error("expected buffered group"); const frame = await group.readFrame(); @@ -1324,7 +1406,7 @@ test("expired raw audio ends the epoch before the replay resumes", async () => { test("Opus description is probed with the exact live config before advertising", async () => { using _codecs = installFakeWebCodecs(); - LaggingAudioEncoder.description = (config) => new TextEncoder().encode(JSON.stringify(config)); + FakeAudioEncoder.description = (config) => new TextEncoder().encode(JSON.stringify(config)); using env = await setup(new Order(), { mime: "opus", frameDuration: Milli(2.5), @@ -1334,14 +1416,14 @@ test("Opus description is probed with the exact live config before advertising", bitrate: 64000, }); expect(env.encoder.out.catalog.peek()?.description).toBe(Buffer.from(JSON.stringify(env.config)).toString("hex")); - expect(LaggingAudioEncoder.probes).toEqual([env.config]); - expect(LaggingAudioEncoder.instances).toHaveLength(1); + expect(FakeAudioEncoder.probes).toEqual([env.config]); + expect(FakeAudioEncoder.instances).toHaveLength(1); expect(env.written).toHaveLength(0); const configuring = configured(); env.capture.in.source.set({ track: {}, kind: "voice" } as never); const voice = await configuring; expect(voice.opus).toMatchObject({ application: "voip", signal: "voice" }); - expect(LaggingAudioEncoder.probes.at(-1)).toEqual(voice); + expect(FakeAudioEncoder.probes.at(-1)).toEqual(voice); expect(env.encoder.out.catalog.peek()?.description).toBe(Buffer.from(JSON.stringify(voice)).toString("hex")); }); @@ -1385,7 +1467,7 @@ test.each([false, true])("audio retained before a disable is encoded on demand ( expect(encoder.out.stats.peek().frames).toBe(0); return; } - // The codec is flushed after the history, since nothing follows it while disabled. + // The history is encoded and its marker follows at once, since nothing else is owed while disabled. expect(encoder.out.stats.peek().frames).toBe(4); const reader = late.ordered(); for (let index = 0; index < 4; index++) { @@ -1404,7 +1486,7 @@ test.each([false, true])("audio retained before a disable is encoded on demand ( }); // Re-enabled before anyone wanted it, the retained audio stays ahead of the break: all of it is -// encoded and flushed before the marker, then the resumed audio starts a fresh epoch. +// encoded before the marker, then the resumed audio starts a fresh epoch. test("audio retained across a disable and re-enable is served before the break", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1443,8 +1525,8 @@ test("audio retained across a disable and re-enable is served before the break", if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(20); - // The four retained frames are flushed before the marker, then the restarted codec holds two. - expect(encoder.out.stats.peek().frames).toBe(5); + // The four retained frames, then the marker, then the three resumed. + expect(encoder.out.stats.peek().frames).toBe(7); expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); const reader = late.ordered(); const read: [number, number][] = []; @@ -1473,8 +1555,8 @@ test("audio retained across a disable and re-enable is served before the break", } }); -// A replay submits the whole backlog at once, so restarting the codec at a gap inside it would drop -// every frame of that drain still queued in a real codec. The gap flushes them out first instead. +// A replay submits the whole backlog at once, so a gap inside it must break the timeline without +// dropping any frame of that drain still owed by the codec. test("a capture gap inside replayed audio keeps every frame on both sides of it", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1507,8 +1589,8 @@ test("a capture gap inside replayed audio keeps every frame on both sides of it" if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(20); - // Every frame before the gap, then the restarted codec holds two of the four after it. - expect(encoder.out.stats.peek().frames).toBe(6); + // Every frame on both sides of the gap. + expect(encoder.out.stats.peek().frames).toBe(8); expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); const reader = late.ordered(); const read: [number, number][] = []; @@ -1538,8 +1620,8 @@ test("a capture gap inside replayed audio keeps every frame on both sides of it" } }); -// A zero max age retains nothing, not even the frame waiting behind a gap's flush, so the gap must not -// depend on the backlog to be noticed, or every frame after it would wait forever. +// A zero max age retains nothing, so a gap must not depend on the backlog to be noticed, or every +// frame after it would wait forever. test("a capture gap without retention resumes the audio after it", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1570,12 +1652,11 @@ test("a capture gap without retention resumes the audio after it", async () => { subscriber = net.track("audio").subscribe(); await settle(); - await push(0, 4); // two written, two held - // The first frame after the gap waits behind the flush, where nothing is retained. + await push(0, 4); await push(200_000, 4); expect(encode.mock.calls.map(([, timestamp]) => timestamp)).toEqual( - [0, 20_000, 40_000, 60_000, 220_000].map((t) => Micro(t)), + [0, 20_000, 40_000, 60_000, 200_000, 220_000, 240_000, 260_000].map((t) => Micro(t)), ); expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); } finally { @@ -1588,7 +1669,7 @@ test("a capture gap without retention resumes the audio after it", async () => { } }); -// Two disables before anyone listens leave two breaks in the backlog: each flushes the audio before it. +// Two disables before anyone listens leave two breaks in the backlog: each follows the audio before it. test("every disable retained before demand gets its own break", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); @@ -1676,10 +1757,9 @@ test("a break after expired history keeps the resumed audio flowing", async () = if (!net) throw new Error("expected network broadcast"); late = net.track("audio").subscribe({ delay: { max: Milli(1000) } }); await settle(20); - // The codec holds two of the three resumed frames. - expect(encoder.out.stats.peek().frames).toBe(1); - await push(2_060_000, 2); expect(encoder.out.stats.peek().frames).toBe(3); + await push(2_060_000, 2); + expect(encoder.out.stats.peek().frames).toBe(5); } finally { late?.close(); encoder.close(); @@ -1689,8 +1769,8 @@ test("a break after expired history keeps the resumed audio flowing", async () = }); // Demand that arrives after a re-enable but before any resumed frame drains the retained audio first, -// so the break still flushes it rather than cutting it off with the codec mid-flight. -test("demand between a re-enable and the resumed audio still flushes the break", async () => { +// so the break follows it rather than cutting it off with the codec mid-flight. +test("demand between a re-enable and the resumed audio still breaks after the retained audio", async () => { using _webcodecs = installFakeWebCodecs(); const clock = new Manual(); const installed = clock.install(); @@ -1729,8 +1809,7 @@ test("demand between a re-enable and the resumed audio still flushes the break", expect(discontinuity.mock.calls.map(([end]) => end)).toEqual([Micro(80_000)]); await push(200_000, 3); - // The restarted codec holds two of the three resumed frames. - expect(encoder.out.stats.peek().frames).toBe(5); + expect(encoder.out.stats.peek().frames).toBe(7); expect(discontinuity).toHaveBeenCalledTimes(1); } finally { discontinuity.mockRestore(); diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index c1203ad7bd7..61722066205 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -91,11 +91,12 @@ export interface Stats { /** Signals the encoder reads. */ export type EncoderInput = { - // Whether to encode this rendition. Defaults to true. When false it stops taking capture, ending the + // Whether to serve this rendition. Defaults to true. When false it stops taking capture, ending the // epoch, but stays in the catalog with `enabled: false` and its last config, and stays registered so a // subscriber still gets an idle track. Audio retained before the disable stays too: a subscriber // still gets it encoded, and it expires on the broadcast's `maxAge`. The codec stays configured, so - // re-enabling resumes at once. + // re-enabling resumes at once. For a pause the rendition comes back from, such as push-to-talk, use + // `muted` instead, which costs subscribers no catalog update. enabled: Getter; // The broadcast to register the rendition on. Undefined resolves the config but has nowhere to publish. @@ -116,6 +117,9 @@ export type EncoderProps = Inputs & { // User tuning knobs. Seed a value or wire a Signal; also live-editable via the matching field. volume?: number | Signal; + // Send no audio while true, keeping the rendition enabled and its subscribers attached. Defaults to false. + muted?: boolean | Signal; + // Codec selection plus encoder settings. Defaults to "opus". codec?: Codec | Signal; @@ -135,9 +139,9 @@ export type EncoderOutput = { // browser supports `sinkId`, so nothing connected to its destination is heard. Undefined for a // source that isn't a track. root: Signal; - // True while encoding capture: enabled with a captured format and a track, which also takes a - // subscriber unless the broadcast's `onDemand` is false. Retained audio encoded for a subscriber - // while disabled does not count. + // True while encoding capture: enabled and unmuted with a captured format and a track, which also + // takes a subscriber unless the broadcast's `onDemand` is false. Retained audio encoded for a + // subscriber while paused does not count. active: Signal; // Cumulative output totals (frames, bytes) measured while serving, and the target bitrate. stats: Signal; @@ -168,6 +172,12 @@ export class Encoder { /** Linear gain applied before encoding, where 1 is unity. Each change ramps over 50 ms, so a mute is silent once it passes. */ volume: Signal; + /** + * Whether to send nothing, as push-to-talk does between presses. Unlike `enabled`, the rendition + * stays enabled in the catalog and its subscribers stay attached, so the next audio reaches them + * at once; each muted span is a declared end, which a player fills as intentional silence. + */ + muted: Signal; /** The live-editable codec selection plus its encoder settings. */ codec: Signal; /** @@ -261,6 +271,9 @@ export class Encoder { // config as disabled until the re-enabled capture resolves a new one. A capture that never reopens // leaves it disabled, which is accurate: no frames are coming. #paused = false; + // Whether it was muted since a config last resolved, so it keeps advertising its last config, + // enabled, while a released capture is closed and until the reopened one resolves. + #muted = false; constructor(name: string, props?: EncoderProps) { // `source` moved to Audio.Capture, which renditions share. TypeScript catches this, but a @@ -278,6 +291,7 @@ export class Encoder { bandwidth: getter(props?.bandwidth), }; this.volume = Signal.from(props?.volume ?? 1); + this.muted = Signal.from(props?.muted ?? false); this.codec = Signal.from(props?.codec ?? "opus"); this.groupDuration = Signal.from(props?.groupDuration ?? Milli(20)); @@ -463,7 +477,7 @@ export class Encoder { }); effect.run((effect) => { - const enabled = effect.get(this.in.enabled); + const enabled = effect.get(this.in.enabled) && !effect.get(this.muted); const capture = effect.get(this.in.capture); const format = capture ? effect.get(capture.out.format) : undefined; const track = effect.get(rendition.track); @@ -473,7 +487,7 @@ export class Encoder { const onDemand = effect.get(broadcast.in.onDemand) ?? true; const demanded = !!track && effect.get(track.demand().wanted) !== undefined; - // Not gated on `enabled`: a disabled rendition takes no capture, but still encodes what it retained. + // Not gated on `enabled` or `muted`: a paused rendition takes no capture, but still encodes what it retained. this.#encoding = !!format && !!track && !fatal && (!onDemand || demanded); effect.set(this.#out.active, enabled && this.#encoding, false); try { @@ -490,7 +504,7 @@ export class Encoder { // encoder's share is honest. Wait for a bitrate so we never claim 0. The grant is ignored: // following it for Opus is out of scope. effect.run((effect) => { - const enabled = effect.get(this.in.enabled); + const enabled = effect.get(this.in.enabled) && !effect.get(this.muted); const track = effect.get(rendition.track); const allocator = effect.get(this.in.bandwidth); if (!enabled || !track || !allocator || effect.get(this.#out.error)) return; @@ -608,21 +622,23 @@ export class Encoder { effect.cleanup(() => this.#failures.update((n) => n - 1)); } - // Publish the probed config before the first frame reaches the track. A - // disabled rendition keeps its last config, since muting also releases the capture, and stays - // disabled after re-enabling until the reopened capture reports its format. + // Publish the probed config before the first frame reaches the track. A disabled rendition keeps its + // last config, since a disable may also release the capture, and stays disabled after re-enabling + // until the reopened capture reports its format. A muted one keeps it enabled throughout. #runCatalog(effect: Effect): void { const estimate = effect.get(this.#estimate); const enabled = effect.get(this.in.enabled) && !effect.get(this.#out.error); const slot = this.#current(effect); if (!enabled) this.#paused = true; + if (effect.get(this.muted)) this.#muted = true; const config = enabled ? effect.get(this.#config)?.catalog : undefined; if (!config) { - const last = this.#paused ? slot.last : undefined; - effect.set(this.#out.catalog, last && { ...last, ...estimate, enabled: false }); + const last = this.#paused || this.#muted ? slot.last : undefined; + effect.set(this.#out.catalog, last && { ...last, ...estimate, ...(this.#paused && { enabled: false }) }); return; } this.#paused = false; + this.#muted = false; // The catalog bitrate is the most this track has been configured for, so lowering the knob // retunes the encoder without lowering what the catalog advertises. @@ -646,15 +662,19 @@ export class Encoder { return opus; } - // End the epoch with a discontinuity marker (see Container.Legacy.Producer.discontinuity) where the - // submitted media ends, so a subscriber that stays across the break, or joins during it, doesn't - // read the audio before it as live. Called whenever the track goes away or the pipeline stops. - #cut(): void { - const end = this.#next; - this.#next = undefined; - const live = this.#live; - if (end === undefined || !live || live.track.closed.peek() !== undefined) return; + // End the epoch with a discontinuity marker (see Container.Legacy.Producer.discontinuity) at `end`, + // by default where the submitted media ends, so a subscriber that stays across the break, or joins + // during it, doesn't read the audio before it as live. A marker at or below the newest one is + // already covered by it. + #cut(end = this.#next): void { + if (end === undefined) return; + if (this.#next !== undefined && this.#next <= end) this.#next = undefined; + if (this.#floor !== undefined && end <= this.#floor) return; + // The epoch ends here even when no track is left to mark, so whatever the codec still owes for + // it stays off the next track. this.#floor = end; + const live = this.#live; + if (!live || live.track.closed.peek() !== undefined) return; live.producer.discontinuity(end); live.start = undefined; } @@ -692,11 +712,31 @@ export class Encoder { let resampler = createResampler(); let framer = createFramer(resolved, config.sampleRate); + // What the codec still owes, in submission order: each input's timestamp, or a break whose + // marker goes down once every chunk before it is out. Chunks are stamped from here, never + // by the codec: Chrome and Firefox stamp input after a gap as if it were contiguous. + // Every browser emits one chunk per input, in order, so a codec that holds a chunk only + // delays its stamp, and a break needs no flush. + const owed: ({ timestamp: Micro } | { cut: Micro })[] = []; + // Write the marker of every break the codec has caught up with. + const settle = () => { + for (let next = owed[0]; next && "cut" in next; next = owed[0]) { + owed.shift(); + this.#cut(next.cut); + } + }; + const encoder = new AudioEncoder({ output: (frame) => { if (frame.type !== "key") { throw new Error("only key frames are supported"); } + const input = owed.shift(); + if (!input || "cut" in input) { + this.#abort(new Error("the audio encoder emitted a chunk for no input")); + return; + } + const timestamp = input.timestamp; this.#out.stats.update((stats) => ({ ...stats, @@ -705,21 +745,20 @@ export class Encoder { })); const live = this.#live; - if (!live) return; - const timestamp = frame.timestamp as Micro; - if (this.#floor !== undefined && timestamp < this.#floor) return; - - // Every audio frame decodes on its own, so any of them can open a group: the - // first one at or past the minimum after the open group's start. Frames forward - // as they are written rather than waiting for the group to fill. A dropped - // group leaves a gap for the codec's PLC. - const keyframe = live.start === undefined || timestamp - live.start >= this.#groupDuration; - if (keyframe) live.start = timestamp; - live.producer.encode(frame, timestamp, keyframe); - const history = this.#history !== undefined && timestamp < this.#history; - if (!history && this.#estimator.write(frame.timestamp, order, keyframe)) { - this.#estimate.set({ ...this.#estimator.estimate }); + if (live && (this.#floor === undefined || timestamp >= this.#floor)) { + // Every audio frame decodes on its own, so any of them can open a group: the + // first one at or past the minimum after the open group's start. Frames forward + // as they are written rather than waiting for the group to fill. A dropped + // group leaves a gap for the codec's PLC. + const keyframe = live.start === undefined || timestamp - live.start >= this.#groupDuration; + if (keyframe) live.start = timestamp; + live.producer.encode(frame, timestamp, keyframe); + const history = this.#history !== undefined && timestamp < this.#history; + if (!history && this.#estimator.write(timestamp, order, keyframe)) { + this.#estimate.set({ ...this.#estimator.estimate }); + } } + settle(); }, error: (err) => { if (!aborted.aborted) this.#abort(err); @@ -738,15 +777,13 @@ export class Encoder { return; } - // Where the next frame starts if it continues the last one encoded. + // Where the next frame starts if it continues the last one submitted, unset after a break. let contiguous: Micro | undefined; - // Disabled: capture is dropped, while the retained audio stays reachable. + // Disabled or muted: capture is dropped, while the retained audio stays reachable. let paused = false; - // A disable left retained audio, so the timeline breaks after it rather than at once. + // A pause left retained audio, so the timeline breaks after it rather than at once. let breakPending = false; - // The codec is flushing before a break, so nothing new reaches it until the marker is down. - let flushing = false; const backlog = new Backlog(retention, config.sampleRate); const deadline = new Signal(undefined); @@ -763,18 +800,22 @@ export class Encoder { ); }); - // Whether a gap (a lost track, or a capture discontinuity) separates `data` from the last - // frame the codec took. Chrome stamps encoder output from the first input's timestamp - // plus the samples encoded since, ignoring any later jump, so across a gap the output - // would trail the capture clock by the gap, and the next gap's marker would then sit - // ahead of everything encoded after it. So a gap restarts the codec, which re-bases it. - // Timestamps round to whole microseconds, so a chunk's timestamp and the marker placed - // at the next frame's start agree exactly. - const gap = (data: AudioFrame) => contiguous !== undefined && Math.round(data.timestamp) !== contiguous; + // End the epoch where the submitted audio ends, once the codec has emitted all of it: a + // marker placed while chunks are still owed would drop them as pre-gap. + const cut = () => { + if (contiguous === undefined) return; + owed.push({ cut: contiguous }); + contiguous = undefined; + settle(); + }; - // Encode one frame, which must not follow a gap: the gap's flush and restart come first. + // Encode one frame, breaking the timeline first if it does not continue the last one. + // Timestamps round to whole microseconds, so a chunk's timestamp and the marker placed at + // the next frame's start agree exactly. const submit = (data: AudioFrame) => { const timestamp = Math.round(data.timestamp) as Micro; + if (contiguous !== undefined && timestamp !== contiguous) cut(); + const joinedLength = data.channels.reduce((total, channel) => total + channel.length, 0); const joined = new Float32Array(joinedLength); @@ -793,6 +834,7 @@ export class Encoder { transfer: [joined.buffer], }); + owed.push({ timestamp }); encoder.encode(frame); frame.close(); contiguous = Math.round( @@ -800,64 +842,29 @@ export class Encoder { ) as Micro; this.#next = contiguous; }; - // The next frame starts a new epoch, so it starts the codec afresh too. - const restart = () => { - encoder.reset(); - encoder.configure(encoderConfig); - contiguous = undefined; - }; - // End the epoch where the submitted audio ends, once the codec has emitted all of it: a - // marker placed while chunks are still in flight would drop them as pre-gap. - const cutAfterFlush = () => { - if (flushing) return; - // Nothing reached the codec since it last started, so nothing is in flight. - if (contiguous === undefined) return this.#cut(); - // The epoch already ended without a flush (a lost track, or expired history), so what - // the codec holds sits below that marker and has nowhere to go. - if (this.#next === undefined) return restart(); - flushing = true; - encoder - .flush() - .then(() => { - if (aborted.aborted) return; - flushing = false; - this.#cut(); - restart(); - if (this.#encoding) pipeline.drain(); - }) - .catch((error) => { - if (!aborted.aborted) this.#abort(error); - }); - }; const pipeline: Pipeline = { channelCount: config.numberOfChannels, drain: () => { - if (flushing) return; backlog.expire(clock()); - // Expired history is a real gap, unlike a loss of demand: end the epoch where - // the encoded audio stops, so its last frame isn't stretched across the gap. - // What the codec still holds is older than the expired audio, so it is dropped. - if (backlog.takeEvicted()) this.#cut(); + // Expired history is a real gap, unlike a loss of demand: end the epoch where the + // encoded audio stops, so its last frame isn't stretched across the gap. + if (backlog.takeEvicted()) cut(); for (;;) { const frame = backlog.peek(); if (!frame) break; - // A disable or a gap broke the timeline here; the rest follows once the marker - // is down. A cut with nothing in flight lands at once. - if (backlog.takeCut() || gap(frame)) { - cutAfterFlush(); - if (flushing) break; - } + // A pause broke the timeline here. + if (backlog.takeCut()) cut(); backlog.shift(); submit(frame); this.#history = this.#next; } deadline.set(backlog.deadline); - // The audio retained before a disable is all encoded, so the break follows it, even - // when re-enabled since: resumed audio then waits behind the flush. + // The audio retained before a pause is all encoded, so the break follows it, even + // when resumed since. if (breakPending && backlog.empty) { breakPending = false; - cutAfterFlush(); + cut(); } }, push: (captured: AudioFrame) => { @@ -867,28 +874,28 @@ export class Encoder { const frames = framer.push(input); for (const [i, data] of frames.entries()) { if (!this.#live) continue; - const cut = breakPending; + const pending = breakPending; breakPending = false; - const live = this.#encoding && !flushing && backlog.empty; - // Started here rather than from the backlog, which may not keep the frame: a - // zero max age retains nothing. - if (live && (cut || gap(data))) cutAfterFlush(); - if (live && !flushing) { + if (this.#encoding && backlog.empty) { + if (pending) cut(); submit(data); continue; } - // It waits for a subscriber, or behind the flush before a break. + // It waits for a subscriber. Started here rather than from the backlog, which + // may not keep the frame: a zero max age retains nothing. const end = frames[i + 1]?.timestamp ?? framer.next; - backlog.push(data, end, clock(), cut && !live); + backlog.push(data, end, clock(), pending); } deadline.set(backlog.deadline); }, }; - // A disable stops taking capture but keeps the codec and retained audio, so a subscriber - // still gets that history and re-enabling resumes without reconfiguring. + // A disable or a mute stops taking capture but keeps the codec and retained audio, so a + // subscriber still gets that history and resuming needs no reconfigure. Neither flushes: + // the codec emits what it owes on its own, and a flush would send its lookahead tail at + // once, a frame ahead of its media. effect.run((toggle) => { - if (toggle.get(this.in.enabled)) { + if (toggle.get(this.in.enabled) && !toggle.get(this.muted)) { paused = false; return; } @@ -896,7 +903,7 @@ export class Encoder { // The capture resumes on a later timeline, so drop the partial frame and filter state. resampler = createResampler(); framer = createFramer(resolved, config.sampleRate); - if (backlog.empty) cutAfterFlush(); + if (backlog.empty) cut(); else breakPending = true; }); diff --git a/js/publish/src/element.ts b/js/publish/src/element.ts index b45048cf90f..997339c5ec6 100644 --- a/js/publish/src/element.ts +++ b/js/publish/src/element.ts @@ -196,7 +196,8 @@ export default class MoqPublish extends HTMLElement { // The preview element, either a