Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions js/watch/src/audio/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,8 @@ async function play(initial: Delay, buffered = false, cmaf = false) {
context: decoder.out.context,
stalled: () => decoder.out.stalled.peek(),
stats: () => decoder.out.stats.peek(),
timestamp: () => decoder.out.timestamp.peek(),
feedback: () => decoder.feedback(Moq.Path.from("watched")),
arrivals: () => decoder.arrivals(),
error: decoder.out.error,
play,
Expand Down Expand Up @@ -1000,6 +1002,93 @@ for (const cmaf of [false, true]) {
});
}

describe("Decoder stalls", () => {
// Play a quantum, so the rendition is playing, then end it the given way and run the ring dry.
async function stopAfterPlaying(stop: (playback: Awaited<ReturnType<typeof play>>) => void) {
const playback = await play(Milli(100));
try {
await playback.play(12);
const [ring] = playback.rings();
expect(ring.read([new Float32Array(128), new Float32Array(128)])).toBe(128);
await microtasks();
expect(playback.stalled()).toBe(false);

stop(playback);
await microtasks();
drain(ring);
await microtasks();
return { stalls: playback.stats()?.stalls ?? 0, feedback: playback.feedback().audio?.stalls ?? 0 };
} finally {
playback.close();
}
}

it("counts a playing rendition that runs dry", async () => {
expect(await stopAfterPlaying(() => {})).toEqual({ stalls: 1, feedback: 1 });
});

it("counts nothing once the rendition leaves the catalog", async () => {
expect(await stopAfterPlaying((playback) => playback.remove())).toEqual({ stalls: 0, feedback: 0 });
});

it("counts nothing once the rendition fails to decode", async () => {
const counts = await stopAfterPlaying(() => codecErrors.at(-1)?.(new DOMException("bad", "EncodingError")));
expect(counts).toEqual({ stalls: 0, feedback: 0 });
});

it("counts nothing after a switch to instant", async () => {
expect(await stopAfterPlaying((playback) => playback.delay.set("instant"))).toEqual({ stalls: 0, feedback: 0 });
});

it("counts nothing while the decoder is disabled", async () => {
expect(await stopAfterPlaying((playback) => playback.decoding.set(false))).toEqual({ stalls: 0, feedback: 0 });
});

it("treats the return from instant as a startup", async () => {
const playback = await play(Milli(100));
try {
await playback.play(12);
const [ring] = playback.rings();
ring.read([new Float32Array(128), new Float32Array(128)]);
await microtasks();
playback.delay.set("instant");
await microtasks();
// The worklet keeps rendering the flushed ring, which reports it stalled.
ring.read([new Float32Array(128), new Float32Array(128)]);
await microtasks();
expect(playback.stalled()).toBe(true);
// The ring waits for the returning rendition's first audio, which is no stall.
playback.delay.set(Milli(100));
await microtasks();
await playback.play(12);
ring.read([new Float32Array(128), new Float32Array(128)]);
await microtasks();
expect(playback.stats()?.stalls ?? 0).toBe(0);
} finally {
playback.close();
}
});

it("clears the playhead and stall state with the ring that reported them", async () => {
const playback = await play(Milli(100));
try {
await playback.play(12);
const [ring] = playback.rings();
ring.read([new Float32Array(128), new Float32Array(128)]);
await microtasks();
expect(playback.timestamp()).toBeDefined();
expect(playback.stalled()).toBe(false);

playback.enabled.set(false);
await microtasks();
expect(playback.timestamp()).toBeUndefined();
expect(playback.stalled()).toBe(true);
} finally {
playback.close();
}
});
});

describe("fatal decoder errors", () => {
// A failure in the read loop, not a WebCodecs callback, must still reach `error` and release the
// subscription, instead of leaving the player apparently live.
Expand Down
126 changes: 69 additions & 57 deletions js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,18 @@ const EMPTY: Stats = { bytesReceived: 0, underruns: 0, late: 0, errors: 0, stall
// What a decode attempt reads, and where its counts go.
type Attempt = { config: DecoderConfig; failure: Signal<Error | undefined>; path: Moq.Path.Valid; key: Key };

// The newest container consumer, the path it reads, and its codec, whose estimate a replacement
// continues.
type Consumed = { path: Moq.Path.Valid; codec: string; consumer: Container.Consumer };

// What the decoder does with the selected rendition. Only a running rendition counts a stall.
type Phase =
| { kind: "stopped" }
// Decoding the rendition at `key` into the ring, which an underrun or a stall is counted against.
| { kind: "running"; key: Key; consumer?: Consumed }
// The source failed to decode, and stays failed until another source is selected.
| { kind: "failed"; failure: Failure };

/**
* Downloads audio from a track and emits it to an AudioContext.
*
Expand Down Expand Up @@ -164,10 +176,6 @@ export class Decoder {
// The container consumer's arrival estimate, unset while nothing is subscribed.
#measured = new Signal<Milli | undefined>(undefined);

// The newest container consumer, the path it reads, and its codec, whose estimate a replacement
// continues.
#consumer?: { path: Moq.Path.Valid; codec: string; consumer: Container.Consumer };

// The subscription's max delay: the shared budget, raised to the estimator's ceiling in "auto".
#subscribeMaxDelay = new Signal<Milli>(Milli.zero);
// The container consumer's delay: the playout delay to start at and the shared budget. Its warmup
Expand All @@ -187,14 +195,11 @@ export class Decoder {
#arrivals = new ArrivalProducer();

#signals = new Effect();
#failed?: Failure;
#phase = new Signal<Phase>({ kind: "stopped" });
readonly #running: Computed<boolean>;

// What each rendition delivered, for the feedback a viewer reports.
#tally = new Tally();
// The rendition decoding into the ring, which an underrun or a stall is counted against.
#playing?: Key;
// Whether audio played since the decoder was enabled, so the wait for the first is no stall.
#played = false;

// The catalog fields that require a replacement subscription or decoder.
readonly #identity: Computed<PlaybackIdentity | undefined>;
Expand Down Expand Up @@ -239,6 +244,7 @@ export class Decoder {
const config = effect.get(this.source.out.config);
return config ? playbackIdentity(config) : undefined;
});
this.#running = this.#signals.computed((effect) => effect.get(this.#phase).kind === "running");
this.#idle = this.#signals.computed((effect) => {
if (effect.get(this.source.out.config) !== undefined) return false;
const ring = effect.get(this.#ring);
Expand Down Expand Up @@ -281,25 +287,26 @@ export class Decoder {
}));
}

// Count each stall once audio has played, and how long it lasted. A disabled decoder waits for
// audio again: resuming is a startup, not a stall.
// Count each stall once audio has played, and how long it lasted. Each run of the decoder waits
// for its first audio, which is a startup, not a stall; a rendition swap keeps the run going.
#runStalls(effect: Effect): void {
if (!effect.get(this.in.enabled)) {
this.#played = false;
return;
}
if (!effect.get(this.#out.stalled)) {
this.#played = true;
return;
}
const key = this.#playing;
if (!this.#played || !key) return;

const end = this.#tally.stall(key);
this.#count(undefined, {}, (stats) => ({ stalls: stats.stalls + 1 }));
effect.cleanup(() => {
const duration = end();
this.#count(undefined, {}, (stats) => ({ stalled: Milli(stats.stalled + duration) }));
if (!effect.get(this.#running)) return;

let played = false;
effect.run((inner) => {
if (!inner.get(this.#out.stalled)) {
played = true;
return;
}
const phase = this.#phase.peek();
if (!played || phase.kind !== "running") return;

const end = this.#tally.stall(phase.key);
this.#count(undefined, {}, (stats) => ({ stalls: stats.stalls + 1 }));
inner.cleanup(() => {
const duration = end();
this.#count(undefined, {}, (stats) => ({ stalled: Milli(stats.stalled + duration) }));
});
});
}

Expand Down Expand Up @@ -395,23 +402,27 @@ export class Decoder {
if (end !== undefined) ring.end((end - this.#terminal.delay) as Micro);
effect.set(this.#ring, ring);

// Mirror ring state (timestamp/stalled) onto our public signals.
// Mirror ring state (timestamp/stalled) onto our public signals, until the ring closes.
effect.run((inner) => {
const ts = Milli.fromMicro(inner.get(ring.timestamp));
this.#out.timestamp.set(ts);
this.#trimDecodeBuffered(ts);
});
effect.cleanup(() => this.#out.timestamp.set(undefined));
effect.run((inner) => {
this.#out.stalled.set(inner.get(ring.stalled) && !inner.get(ring.silent));
});
effect.cleanup(() => this.#out.stalled.set(true));
// Each ring counts from zero, so add only what this one counted to the cumulative stats.
let counted = 0;
effect.run((inner) => {
const underruns = inner.get(ring.underruns);
const added = underruns - counted;
counted = underruns;
if (added <= 0) return;
this.#count(this.#playing, { underruns: added }, (stats) => ({ underruns: stats.underruns + added }));
const phase = this.#phase.peek();
const key = phase.kind === "running" ? phase.key : undefined;
this.#count(key, { underruns: added }, (stats) => ({ underruns: stats.underruns + added }));
});

effect.set(this.#out.root, worklet);
Expand Down Expand Up @@ -465,55 +476,52 @@ export class Decoder {
// Honor a per-rendition `broadcast` override: subscribe on the resolved source
// broadcast instead of the catalog's own broadcast. A stopped decoder resolves it only to
// tell whether a failure's source is still selected.
const phase = this.#phase.peek();
const active =
broadcast && track && identity && (running || this.#failed)
broadcast && track && identity && (running || phase.kind === "failed")
? broadcast.relativeBroadcast(effect, identity.broadcast, identity.instance)
: undefined;
const key = identity && JSON.stringify(identity);

// A failure belongs to its source: selecting another clears it, even while muted or paused.
const failed = this.#failed;
const selected = broadcast && failed ? effect.get(broadcast.out.active)?.closed : undefined;
if (failed && failureMoved(failed, { broadcast: selected, media: active?.closed, track, identity: key })) {
this.#failed = undefined;
this.#out.error.set(undefined);
if (phase.kind === "failed") {
const selected = broadcast ? effect.get(broadcast.out.active)?.closed : undefined;
if (failureMoved(phase.failure, { broadcast: selected, media: active?.closed, track, identity: key })) {
this.#phase.set({ kind: "stopped" });
this.#out.error.set(undefined);
}
}

if (!running || !broadcast || !track || !identity) {
// A source that already failed waits for another.
if (!running || !broadcast || !track || !identity || !active || !key || this.#phase.peek().kind === "failed") {
this.#retire();
return;
}

const config = identity.decoder;

if (!active || !key) {
this.#retire();
return;
}

// This source already failed; wait for another.
if (this.#failed) {
this.#retire();
return;
}
// Closing this effect releases both its container reader and track subscription.
const failure = new Signal<Error | undefined>(undefined);
const rendition: Key = { path: broadcast.in.name.peek(), id: track };
this.#playing = rendition;
// A swap keeps the outgoing consumer, whose estimate the replacement may continue.
this.#phase.set({
kind: "running",
key: rendition,
consumer: phase.kind === "running" ? phase.consumer : undefined,
});
let attempt: Effect | undefined;
effect.run((inner) => {
const error = inner.get(failure);
if (!error) return;
this.#count(rendition, { errors: 1 }, (stats) => ({ errors: stats.errors + 1 }));
this.#failed = {
this.#out.error.set(error);
attempt?.close();
this.#retire({
broadcast: broadcast.out.active.peek()?.closed,
media: active.closed,
track,
identity: key,
};
this.#out.error.set(error);
attempt?.close();
this.#retire();
});
});

// Another broadcast (a new name, or a republish) brings its own timeline, which a ring and a
Expand Down Expand Up @@ -795,7 +803,9 @@ export class Decoder {
format: Container.Format,
{ config, path }: { config: DecoderConfig; path: Moq.Path.Valid },
): { consumer: Container.Consumer; read: Signal<Queued> } {
const outgoing = this.#consumer;
const phase = this.#phase.peek();
if (phase.kind !== "running") throw new Error("audio: a consumer opened outside a running rendition");
const outgoing = phase.consumer;
const continues = outgoing?.path === path;
const consumer = new Container.Consumer(sub, {
format,
Expand All @@ -804,7 +814,7 @@ export class Decoder {
predecessor: continues ? outgoing.consumer : undefined,
});
effect.cleanup(() => consumer.close());
this.#consumer = { path, codec: config.codec, consumer };
this.#phase.set({ ...phase, consumer: { path, codec: config.codec, consumer } });

// Opus states its duration only per packet, so the same codec continuing keeps the outgoing
// one until the first packet refines it. Anything else starts from the codec's own constant.
Expand Down Expand Up @@ -843,9 +853,11 @@ export class Decoder {
return { consumer, read };
}

// No consumer replaces the outgoing one, so its estimate stops holding the buffer open.
#retire(): void {
this.#consumer = undefined;
// Leave the running phase, failed or stopped; a stop keeps an earlier failure. No consumer
// replaces the outgoing one, so its estimate stops holding the buffer open.
#retire(failure?: Failure): void {
if (failure) this.#phase.set({ kind: "failed", failure });
else if (this.#phase.peek().kind === "running") this.#phase.set({ kind: "stopped" });
this.#measured.set(undefined);
// Its arrivals stop holding the clock's reference.
this.#track.reset();
Expand Down
26 changes: 26 additions & 0 deletions js/watch/src/sync.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,32 @@ describe("buffered startup", () => {
installed.close();
}
});
// A video track's startup span opens at the first picture of its timeline, which a live stream
// played long ago. Turning lookahead on mid-stream must not anchor playback back there.
it("keeps playing when lookahead turns on mid-stream", async () => {
const clock = new Manual();
const installed = clock.install();
const lookahead = new Signal(Milli.zero);
const sync = new Sync({ delay: Milli(100), lookahead });
const span = new Signal<{ start: Milli; end: Milli } | null>({ start: Milli(0), end: Milli(200) });
const video = sync.track({ target: new Signal(Milli(100)), startup: span });
try {
await clock.advance(Nano.zero);
video.received(Milli(0));
await clock.advance(Nano.fromMilli(Milli(10_000)));
video.received(Milli(10_000));
expect(sync.now()).toBe(Milli(9900));

lookahead.set(Milli(1000));
await clock.advance(Nano.zero);
expect(sync.playing.peek()).toBe(true);
expect(sync.now()).toBe(Milli(9900));
} finally {
video.close();
sync.close();
installed.close();
}
});
it("removes a disabled track from the floor and leaves live pacing unchanged", async () => {
const clock = new Manual();
const installed = clock.install();
Expand Down
Loading