diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 548844290e3..a3519673654 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -49,7 +49,10 @@ for a permission, device, or settings change instead of prompting again. The video encoder follows its share of the connection's send-rate estimate, so several publishers on one connection do not each target the whole uplink. Codec, resolution, framerate, and bitrate are tunable through -`el.video.config`, and `el.video.cut()` asks for an extra keyframe. Size, +`el.video.config`, and `el.video.cut()` asks for an extra keyframe. A viewer +who joins once a group's deltas outweigh twice its keyframe gets a fresh one, +so `keyframeInterval` is only the longest a group runs. Through a moq-lite +relay, only the join that opens the relay's upstream subscription counts. Size, bitrate, and framerate change in place, while a codec or audio format change keeps the rendition's catalog ID under a fresh `track` name that viewers follow through the catalog. For diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 9dd00515ea9..3259aa4080b 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -27,7 +27,13 @@ hardware-only, AV1 decodes through NVDEC and MediaCodec, and VP8 and VP9 decode in software through the opt-in `vpx` feature. Capture is pull-driven: `encode::publish_capture` advertises the track up -front and opens the camera only while someone subscribes. Backends that can +front and opens the camera only while someone subscribes. A viewer who joins +once the current group's deltas outweigh twice its keyframe gets a fresh +keyframe, so the GOP is only the longest a group runs. Through a moq-lite +relay, only the join that opens the relay's upstream subscription counts. A +build with the `v4l2` feature keeps every `Auto` or `Hardware` selection to its +GOP, whichever encoder it lands on, since a V4L2 driver may refuse the request. +Backends that can change bitrate live follow the connection's send estimate. Where the platform allows it, frames stay on the GPU between capture, decode, encode, and render. diff --git a/js/publish/MIGRATE.md b/js/publish/MIGRATE.md index 0e66059ea83..f32a6a2a47f 100644 --- a/js/publish/MIGRATE.md +++ b/js/publish/MIGRATE.md @@ -4,6 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Encoder.cut()` is not rate limited, and video tracks advertise refresh (#6048) + +`Encoder.cut()` no longer spaces keyframes 500 ms apart: every call forces one on the next frame, so frequent calls produce all-keyframe video. +Debounce your own calls if you relied on the spacing. +Every `Broadcast.video()` track now advertises a refresh ratio of 2, so a producer registered there must open a group with a keyframe whenever `track.demand().wanted` passes the largest group. +The bundled `Encoder` already does; a custom producer has to as well. + ### A disabled rendition stays in the catalog (#4915) `enabled: false` keeps the rendition in the catalog with `enabled: false` instead of removing it, and viewers skip it. diff --git a/js/publish/src/broadcast.ts b/js/publish/src/broadcast.ts index 40c60d031f6..80a7d2c6368 100644 --- a/js/publish/src/broadcast.ts +++ b/js/publish/src/broadcast.ts @@ -10,6 +10,10 @@ import { errors } from "./internal"; import { Baseline } from "./jitter"; import { type Kind, Rendition } from "./rendition"; +// The refresh ratio video tracks advertise: a joiner forces a keyframe once a group's deltas +// outweigh twice its keyframe. Matches moq-video's capture `REFRESH`; see it for the measurements. +const VIDEO_REFRESH = 2; + // Signals the broadcast reads. Whoever owns the backing Signal (the element, or another component // whose output is wired in, e.g. a Video.Capture's `display`) does the writing. export type BroadcastInput = { @@ -133,7 +137,12 @@ export class Broadcast { this.#signals.run(this.#run.bind(this)); } - /** Register a video rendition under a full track name (e.g. `"video/hd"`). Throws if the name is taken. */ + /** + * Register a video rendition under a full track name (e.g. `"video/hd"`). Throws if the name is taken. + * + * Its track advertises a refresh ratio, so its producer opens a group with a keyframe whenever + * `track.demand().wanted` passes the largest group, as the video `Encoder` does. + */ video(name: string): Rendition { return this.#register(name, "video"); } @@ -413,9 +422,10 @@ export class Broadcast { continue; } if (!entry) { + const info = Container.trackInfo({ maxAge, priority: Catalog.PRIORITY[rendition.kind] }); const track = broadcast.createTrack( name, - Container.trackInfo({ maxAge, priority: Catalog.PRIORITY[rendition.kind] }), + rendition.kind === "video" ? { ...info, refresh: VIDEO_REFRESH } : info, ); entry = { track, owner: rendition }; tracks.set(rendition.name, entry); diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 4f9378dc929..b8ca9649e98 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -849,8 +849,30 @@ test("a re-probe swaps the catalog entry in one update", async () => { } }); -test("cut forces a keyframe, coalescing requests and spacing them at least 500ms apart", async () => { +// Drive a live encoder over a recording `VideoEncoder` that outputs every frame it is handed, so the +// track holds real groups: a keyframe of 100 bytes, each delta 60. `run` sends one frame per call +// at `millis` of media time and reads back which frames were forced to be keyframes. +async function recordKeyframes( + info: Partial, + run: (ctx: { + encoder: Encoder; + track: Moq.Track.Producer; + send: (millis: number) => Promise; + }) => Promise, +): Promise { const keys: number[] = []; + + class Frame { + readonly timestamp: number; + constructor(timestamp: number) { + this.timestamp = timestamp; + } + clone(): Frame { + return new Frame(this.timestamp); + } + close(): void {} + } + class RecordingVideoEncoder { state: CodecState = "unconfigured"; #output: VideoEncoderInit["output"]; @@ -869,9 +891,13 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms this.#codec = config.codec; } - // Records the test's frames only, not the probe's. + // Records the test's frames only, not the probe's, and outputs each a task later as WebCodecs does. encode(frame: VideoFrame, options?: VideoEncoderEncodeOptions): void { - if (options?.keyFrame && frame instanceof Frame) keys.push(frame.timestamp / 1000); + if (!(frame instanceof Frame)) return; + const key = !!options?.keyFrame; + if (key) keys.push(frame.timestamp / 1000); + const chunk = { type: key ? "key" : "delta", timestamp: frame.timestamp, byteLength: key ? 100 : 60 }; + queueMicrotask(() => this.#output({ ...chunk, copyTo: () => {} } as never, {} as never)); } // Only the probe flushes; report the configured codec, as Chrome does. @@ -894,17 +920,6 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms const videoFrame = Object.getOwnPropertyDescriptor(globalThis, "VideoFrame"); Object.defineProperty(globalThis, "VideoFrame", { configurable: true, value: FakeVideoFrame, writable: true }); - class Frame { - readonly timestamp: number; - constructor(timestamp: number) { - this.timestamp = timestamp; - } - clone(): Frame { - return new Frame(this.timestamp); - } - close(): void {} - } - const { Fanout } = await import("../fanout"); let controller!: ReadableStreamDefaultController; const fanout = new Fanout( @@ -916,7 +931,7 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms { clone: (frame) => frame.clone(), release: (frame) => frame.close() }, ); - const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MILLI }); + const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MICRO, ...info }); const rendition = { config: new Signal(undefined), track: new Signal(track), @@ -928,35 +943,19 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms }; const encoder = new Encoder("video", { enabled: true, - broadcast: { video: () => rendition } as never, + broadcast: { video: () => rendition, baseline: new Baseline() } as never, capture: capture as never, }); - // One frame per 100ms of media time, after calling cut() when `cut` says so. - const send = async (millis: number, cut = false) => { - if (cut) encoder.cut(); + const send = async (millis: number) => { controller.enqueue(new Frame(millis * 1000) as unknown as VideoFrame); await new Promise((resolve) => setTimeout(resolve, 5)); }; try { await settle(); - - // The opening keyframe serves a request made before it. - await send(0, true); - await send(100); - // Several requests before one frame produce one keyframe. - encoder.cut(); - encoder.cut(); - await send(600, true); - await send(700); - // Too soon after the last: deferred until 500ms have passed, not dropped. - await send(800, true); - await send(1000); - await send(1100); - await send(1200); - - expect(keys).toEqual([0, 600, 1100]); + await run({ encoder, track, send }); + return keys; } finally { encoder.close(); fanout.close(); @@ -966,6 +965,50 @@ test("cut forces a keyframe, coalescing requests and spacing them at least 500ms if (videoFrame) Object.defineProperty(globalThis, "VideoFrame", videoFrame); else Reflect.deleteProperty(globalThis, "VideoFrame"); } +} + +test("cut forces a keyframe on the next frame, coalescing requests without spacing them", async () => { + const keys = await recordKeyframes({}, async ({ encoder, send }) => { + // The opening keyframe serves a request made before it. + encoder.cut(); + await send(0); + await send(100); + // Several requests before one frame produce one keyframe. + encoder.cut(); + encoder.cut(); + await send(200); + // The very next frame can be cut again. + encoder.cut(); + await send(300); + await send(400); + }); + expect(keys).toEqual([0, 200, 300]); +}); + +test("a join over the refresh ratio forces a keyframe on the next frame", async () => { + const keys = await recordKeyframes({ refresh: 2 }, async ({ track, send }) => { + const watching = track.subscribe(); + const joined: Moq.Track.Subscriber[] = []; + try { + await send(0); + await send(50); + // One 60 byte delta is under twice the 100 byte keyframe. + joined.push(track.subscribe()); + await send(100); + await send(150); + await send(200); + // Four deltas are over it, 200ms after the keyframe. + joined.push(track.subscribe()); + await send(250); + // A join after the fresh keyframe reads its group. + joined.push(track.subscribe()); + await send(300); + } finally { + watching.close(); + for (const subscriber of joined) subscriber.close(); + } + }); + expect(keys).toEqual([0, 250]); }); test("a failed video encoder disables only its rendition and recovers on a fresh track", async () => { diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index fbb092e4733..64a3d26893b 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -48,7 +48,8 @@ export interface Config { // When combined with maxPixels, the smaller resulting cap wins. maxScale?: number; - // The interval at which to insert keyframes. (default: 2000 milliseconds) + // The longest interval between keyframes. A joining subscriber forces one sooner once the + // group's deltas outweigh the track's refresh ratio. (default: 2000 milliseconds) keyframeInterval?: Milli; // If not provided, the encoder will use the best bitrate for the given width, height, and framerate. @@ -368,9 +369,17 @@ export class Encoder { if (!capture) return; const producer = new Container.Legacy.Producer(track, new Container.Legacy.Format("video")); + const demand = track.demand(); let lastKeyframe: Micro | undefined; let lastEncoded: Micro | undefined; + // The largest group demand asked for that a keyframe already answers. A join over the refresh + // ratio asks for one past the largest, which the next frame opens. + let answered: number | undefined; + const answer = () => { + const wanted = demand.wanted.peek(); + if (wanted !== undefined) answered = Math.max(answered ?? 0, wanted); + }; let tearingDown = false; effect.spawn(async () => { @@ -388,6 +397,8 @@ export class Encoder { })); producer.encode(frame, frame.timestamp as Micro, key); + // Including a keyframe the encoder placed on its own, which opened a group too. + if (key) answer(); if (this.#estimator.flush(frame.timestamp, baseline)) { this.#estimate.set({ ...this.#estimator.estimate }); } @@ -458,19 +469,23 @@ export class Encoder { const interval = config?.keyframeInterval ?? Milli.fromSecond(2 as Second); - // Force a keyframe if this is the first frame (no group yet), the GOP elapsed, or - // the caller asked for one and the last is old enough. + // Force a keyframe if this is the first frame (no group yet), the GOP elapsed, the + // caller asked for one, or a join wants a group past the largest. No spacing: the + // refresh ratio already rate-limits joins by the bytes a group holds. const since = lastKeyframe === undefined ? undefined : frame.timestamp - lastKeyframe; + const wanted = demand.wanted.peek(); const keyFrame = resized || since === undefined || since >= Micro.fromMilli(interval) || - (this.#cut && since >= Micro.fromMilli(MIN_CUT_INTERVAL)); + this.#cut || + (wanted !== undefined && (answered === undefined || wanted > answered)); if (keyFrame) { lastKeyframe = frame.timestamp as Micro; resized = false; - // Any keyframe serves an outstanding request. + // Any keyframe serves an outstanding request and every join before it. this.#cut = false; + answer(); } encoder.encode(frame, { keyFrame }); @@ -804,9 +819,10 @@ export class Encoder { * Request a keyframe, opening a new group at a frame no earlier than this call. * * For a resume, a recording cut, or a known tune-in moment; {@link Config.keyframeInterval} is the - * cadence. Requests coalesce into the next keyframe, and forced keyframes land at least 500ms - * apart so a caller in a loop cannot pin the encoder at all-keyframe. A request while not encoding - * is served by the keyframe every encode starts with. + * longest a group runs, and a joining subscriber already opens one through the track's refresh + * ratio. Requests between two frames coalesce into one keyframe on the next, with no spacing: a + * caller in a loop pins the encoder at all-keyframe. A request while not encoding is served by the + * keyframe every encode starts with. */ cut(): void { this.#cut = true; @@ -817,10 +833,6 @@ export class Encoder { } } -// The closest two requested keyframes may land. A keyframe costs several times a predicted frame, -// and this stays well under the default two-second GOP so a request still beats the cadence. -const MIN_CUT_INTERVAL = 500 as Milli; - // The source's nominal frame rate: what the capture device settled on, or what a frame stream // declared. Undefined when nothing reports one. function sourceFrameRate(source: Source): number | undefined { diff --git a/quest/m1/refresh/README.md b/quest/m1/refresh/README.md index 708e994f19d..5d96bf43ca9 100644 --- a/quest/m1/refresh/README.md +++ b/quest/m1/refresh/README.md @@ -64,7 +64,6 @@ Work owned by this README, after the children land: - [Lite wire](/quest/m1/refresh/lite.md) - lite-07 TRACK_INFO advertises the ratio and SUBSCRIBE_UPDATE carries the wanted group, so a join at any relay reaches the origin - [moq-json](/quest/m1/refresh/json.md) - snapshot and window groups roll on a qualifying join, reading the ratio from the track; `delta_ratio` and `op_ratio` become `deltas` and `ops` switches in every binding -- [Video](/quest/m1/refresh/video.md) - video publishers advertise refresh and force a keyframe on demand, keeping the encoder's max GOP ## Related diff --git a/quest/m1/refresh/video.md b/quest/m1/refresh/video.md deleted file mode 100644 index 1ad728911c4..00000000000 --- a/quest/m1/refresh/video.md +++ /dev/null @@ -1,44 +0,0 @@ -# [M] Video keyframes on demand - -## Goal - -Rust and browser video publishers advertise `track::Info::refresh` and -force a keyframe in a new group when `Demand::wanted` passes the largest -group. The encoder's configured max GOP stays as the upper bound. With no -joiners, GOPs run to the max; a joiner gets a fresh keyframe once the GOP's -bytes after its keyframe exceed the ratio. - -## Plan - -Decided 2026-10-16 (see the [epic](/quest/m1/refresh/README.md)). - -- Open-ended GOPs: a keyframe comes on a refresh or at the max GOP. The cap - stays as the bound behind hops that cannot forward requests, and against - decoder drift. -- Scope: advertise refresh only on backends whose keyframe forcing is known - before the track is created, because `track::Info` is immutable. That - covers `js/publish` through WebCodecs (`keyFrame: true`) and the - moq-video encoders that support it. V4L2 does not advertise, because some - drivers reveal that they refuse keyframe requests only after - `Control::new` has created the track. Passthrough importers (CMAF, TS, - RTMP, SRT) cannot force one either. Decided in the #5734 review: never - advertise a request the backend might refuse. -- Remove the 500 ms floor between keyframes from moq-video's `Cuts` and the - WebCodecs publisher, and let refresh-driven keyframes take over. The byte - ratio already bounds the rate, and the floor would let a high-bitrate - joiner download well past the ratio (maintainer, #5734 review). -- Open, settled with the maintainer before any code: where the ratio is - configured (an encoder option, a rendition option, or a fixed default - with no public field), its default, and whether it varies per rendition. - The publishers create their tracks internally, so this decides how - `track::Info::refresh` gets populated. Measure the bytes a joiner - downloads against the extra keyframes sent on a representative stream, - then decide. -- Docs: the encoder options in `doc/` that describe GOP length. - -Tests: a forced-keyframe test per backend that supports it, using simulated -time where the encoder allows, and a browser test for `js/publish`. Each -covers a ratio crossing within 500 ms of the previous keyframe. - -Public API: set by the open decision above, either an encoder or rendition -config field for the ratio, or none for a fixed default. Wire: none. diff --git a/rs/moq-video/MIGRATE.md b/rs/moq-video/MIGRATE.md index f4e25fefa9e..e8292cbf31f 100644 --- a/rs/moq-video/MIGRATE.md +++ b/rs/moq-video/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Control::cut` is not rate limited, and capture advertises refresh (#6048) + +`encode::Control::cut` no longer spaces keyframes 500 ms apart: every request forces one on the next frame, so frequent calls produce all-keyframe video. +Debounce your own calls if you relied on the spacing. +Capture tracks now advertise `refresh: Some(2)` when every backend the selection may open honors a cut, and the driver forces a keyframe when `Demand::wanted` passes the largest group. + ### OpenH264 is opt-in (#5771) The default features drop `openh264`, so a default build has no software H.264 encoder or decoder and a GPU-less host fails to open one. diff --git a/rs/moq-video/src/encode/backend/mod.rs b/rs/moq-video/src/encode/backend/mod.rs index ca0df34862b..ea9b000437e 100644 --- a/rs/moq-video/src/encode/backend/mod.rs +++ b/rs/moq-video/src/encode/backend/mod.rs @@ -167,11 +167,15 @@ pub const NAMES: &[&str] = &[ "openh264", ]; -/// A backend constructor: name, the codecs it can emit, and an opener. +/// A backend constructor: name, the codecs it can emit, an opener, and whether it can always cut. struct Candidate { name: &'static str, codecs: &'static [Codec], open: fn(&Config) -> Result, Error>, + /// Whether every encoder this opens honors a cut ([`Backend::can_cut`]), known before + /// opening one. V4L2 learns it from the driver at open, so it cannot promise. + #[cfg_attr(not(feature = "capture"), allow(dead_code))] + cuts: bool, #[cfg(target_os = "linux")] vulkan: bool, } @@ -184,6 +188,7 @@ const HARDWARE: &[Candidate] = &[ name: videotoolbox::NAME, codecs: &[Codec::H264, Codec::H265], open: videotoolbox::VideoToolbox::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -192,6 +197,7 @@ const HARDWARE: &[Candidate] = &[ name: mediafoundation::NAME, codecs: &[Codec::H264, Codec::H265], open: mediafoundation::MediaFoundation::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -200,6 +206,7 @@ const HARDWARE: &[Candidate] = &[ name: mediacodec::NAME, codecs: &[Codec::H264, Codec::H265], open: mediacodec::MediaCodec::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -208,6 +215,7 @@ const HARDWARE: &[Candidate] = &[ name: nvenc::NAME, codecs: &[Codec::H264, Codec::H265], open: nvenc::Nvenc::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: true, }, @@ -216,6 +224,7 @@ const HARDWARE: &[Candidate] = &[ name: vaapi::NAME, codecs: &[Codec::H264], open: vaapi::Vaapi::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -227,6 +236,7 @@ const HARDWARE: &[Candidate] = &[ name: v4l2::NAME, codecs: &[Codec::H264], open: v4l2::V4l2::open, + cuts: false, #[cfg(target_os = "linux")] vulkan: false, }, @@ -241,6 +251,7 @@ const SOFTWARE: &[Candidate] = &[ name: openh264::NAME, codecs: &[Codec::H264], open: openh264::Openh264::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -255,6 +266,7 @@ const NAMED_ONLY: &[Candidate] = &[ name: probe::NAME, codecs: &[Codec::H264], open: probe::Probe::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }, @@ -262,6 +274,7 @@ const NAMED_ONLY: &[Candidate] = &[ name: probe::NO_CUT, codecs: &[Codec::H264], open: probe::Probe::open_no_cut, + cuts: false, #[cfg(target_os = "linux")] vulkan: false, }, @@ -297,12 +310,24 @@ impl<'a> Attempt<'a> { /// Open the best encoder for `config.codec` + `config.kind`, trying candidates /// in priority order and falling back until one succeeds. pub(crate) fn open(config: &Config) -> Result, Error> { - let codec = config.codec; + select(attempts(config.codec, &config.kind), config) +} + +/// Whether whichever backend [`open`] picks for `codec` + `kind` honors a cut, answered +/// before opening one: every candidate it may fall to must promise it. +#[cfg_attr(not(feature = "capture"), allow(dead_code))] +pub(crate) fn always_cuts(codec: Codec, kind: &Kind) -> bool { + let attempts = attempts(codec, kind); + !attempts.is_empty() && attempts.iter().all(|attempt| attempt.candidate.cuts) +} + +/// The candidates [`open`] tries for `codec` + `kind`, in order. +fn attempts(codec: Codec, kind: &Kind) -> Vec> { let supports = move |c: &Candidate| c.codecs.contains(&codec); let hardware = HARDWARE.iter().filter(|c| supports(c)).map(Attempt::hardware); let software = SOFTWARE.iter().filter(|c| supports(c)).map(Attempt::software); - let attempts: Vec = match &config.kind { + match kind { Kind::Auto => hardware.chain(software).collect(), Kind::Hardware => hardware.collect(), Kind::Software => software.collect(), @@ -312,9 +337,7 @@ pub(crate) fn open(config: &Config) -> Result, Error> { .chain(SOFTWARE.iter().chain(NAMED_ONLY.iter()).map(Attempt::software)) .filter(|a| supports(a.candidate) && a.candidate.name == name) .collect(), - }; - - select(attempts, config) + } } /// Try `attempts` in order and return the first encoder that opens, warning when @@ -540,6 +563,7 @@ mod tests { name: "stub", codecs: &[Codec::H264], open: Stub::open, + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }; @@ -550,6 +574,7 @@ mod tests { name: "driverless", codecs: &[Codec::H264], open: |_| Err(Error::Codec(anyhow::anyhow!("driver libraries not found"))), + cuts: true, #[cfg(target_os = "linux")] vulkan: false, }; @@ -558,6 +583,23 @@ mod tests { Config::new(320, 240, crate::Rate::new(30, 1).unwrap()) } + /// Only a selection whose every candidate promises cuts may advertise them, and a + /// candidate's promise holds for the encoder it opens. + #[test] + fn always_cuts_needs_every_candidate_to_promise() { + let named = |name: &str| Kind::Named(name.to_owned()); + assert!(always_cuts(Codec::H264, &named(probe::NAME))); + assert!(!always_cuts(Codec::H264, &named(probe::NO_CUT))); + assert!(!always_cuts(Codec::H264, &named("v4l2")), "V4L2 learns it at open"); + assert!(!always_cuts(Codec::H264, &named("nonexistent")), "nothing to open"); + + let _probe = probe::exclusive(); + for candidate in NAMED_ONLY { + let backend = (candidate.open)(&config()).unwrap(); + assert_eq!(backend.can_cut(), candidate.cuts, "{}", candidate.name); + } + } + #[tracing_test::traced_test] #[test] fn falling_past_hardware_warns() { diff --git a/rs/moq-video/src/encode/cuts.rs b/rs/moq-video/src/encode/cuts.rs index 49f823c7f6c..65748f2f7e3 100644 --- a/rs/moq-video/src/encode/cuts.rs +++ b/rs/moq-video/src/encode/cuts.rs @@ -1,76 +1,50 @@ //! [`Cuts`]: which captured frames to force a keyframe on. -use std::time::Duration; - -use moq_net::Timestamp; - -/// The closest a forced keyframe may land after any other keyframe, in media time. +/// One encoder's view of the keyframe asks: decides which frames to cut, coalescing. +/// Built fresh for every encoder the capture opens. /// -/// A keyframe costs several times a predicted frame, so a caller asking in a loop -/// would otherwise pin the encoder at all-IDR and starve the rest of the uplink. -/// Well under the default two-second GOP, so a request still beats the cadence. -const MIN_INTERVAL: Duration = Duration::from_millis(500); - -/// One encoder's view of the keyframe requests: decides which frames to cut, -/// coalescing and rate limiting. Built fresh for every encoder the capture opens. +/// Two asks open a group. `Control::cut` requests arrive as a running +/// count, so any number between two frames read as one. A subscriber joining over the +/// track's refresh ratio arrives as the group +/// [`Demand::wanted`](moq_net::track::Demand::wanted) asks for once it passes the largest +/// written, which only grows, so joins while the keyframe is still in the encoder ask for the +/// same group and coalesce into it. /// -/// Requests arrive as a running count, so any number between two frames read as one. -/// Every keyframe the encoder produces counts, including its own GOP cadence: it -/// serves the requests made before its frame and restarts the spacing window. +/// There is no spacing between cuts: the refresh ratio already rate-limits joins by the bytes +/// a group holds, and a floor would let a joiner download well past that ratio. pub(super) struct Cuts { /// The request count already accounted for. - seen: u64, - /// The frame the latest unserved request was noticed at, or `None` when none waits. - /// A keyframe at or after it serves every request so far. - pending: Option, - /// The latest keyframe, forced or not, or `None` before this encoder's first frame. - last: Option, + requests: u64, + /// The largest wanted group already cut for, or `None` before any. + wanted: Option, + /// Whether this encoder produced its first frame. + opened: bool, } impl Cuts { /// Start from `requests`, the count already served by earlier encoders. pub fn new(requests: u64) -> Self { Self { - seen: requests, - pending: None, - last: None, + requests, + wanted: None, + opened: false, } } - /// Whether the frame at `timestamp` should be cut, given the running request - /// count, recording it if so. - pub fn due(&mut self, requests: u64, timestamp: Timestamp) -> bool { - if requests != self.seen { - self.seen = requests; - self.pending = Some(timestamp); - } + /// Whether the next frame should be cut, given the running request count and `wanted`, + /// the group demand asks for when it is past the largest written. + pub fn due(&mut self, requests: u64, wanted: Option) -> bool { + let asked = requests != self.requests || wanted > self.wanted; + self.requests = requests; + self.wanted = self.wanted.max(wanted); // A fresh encoder opens with a keyframe on every backend, which serves anything - // requested before it. - let Some(last) = self.last else { - self.last = Some(timestamp); - self.pending = None; - return false; - }; - - if self.pending.is_none() || Duration::from(timestamp).saturating_sub(Duration::from(last)) < MIN_INTERVAL { + // asked for before it. + if !self.opened { + self.opened = true; return false; } - - self.pending = None; - self.last = Some(timestamp); - true - } - - /// Record a keyframe the encoder produced for the frame at `timestamp`, whether - /// forced or on its own cadence. - pub fn keyframe(&mut self, timestamp: Timestamp) { - // A buffering encoder can hand back a keyframe older than the latest request, - // which is not a frame that request asked for. - if self.pending.is_some_and(|pending| timestamp >= pending) { - self.pending = None; - } - self.last = Some(self.last.map_or(timestamp, |last| last.max(timestamp))); + asked } } @@ -78,94 +52,55 @@ impl Cuts { mod tests { use super::*; - fn at(millis: u64) -> Timestamp { - Timestamp::from_millis(millis).unwrap() - } - #[test] - fn the_opening_keyframe_serves_earlier_requests() { + fn the_opening_keyframe_serves_earlier_asks() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(1, at(0))); - assert!(!cuts.due(1, at(1_000)), "the request was already served"); + assert!(!cuts.due(1, Some(4))); + assert!(!cuts.due(1, Some(4)), "the asks were already served"); } #[test] fn requests_before_a_frame_coalesce() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(cuts.due(10, at(1_000))); - assert!(!cuts.due(10, at(2_000)), "ten requests before one frame cut once"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(10, None)); + assert!(!cuts.due(10, None), "ten requests before one frame cut once"); } + /// Back-to-back requests each cut: the caller decides how often to ask. #[test] - fn a_request_too_soon_is_deferred_not_dropped() { + fn requests_are_not_spaced() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(!cuts.due(1, at(100))); - assert!(!cuts.due(1, at(499))); - assert!(cuts.due(1, at(500)), "held until the interval elapsed"); - } - - #[test] - fn a_caller_in_a_loop_cannot_force_all_idr() { - let mut cuts = Cuts::new(0); - - // Ten seconds at 30 fps with a request before every frame. - let cut = (0..300u64) - .filter(|frame| cuts.due(frame + 1, at(frame * 1000 / 30))) - .count(); - assert_eq!(cut, 19, "one forced keyframe per half second after the opening one"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(1, None)); + assert!(cuts.due(2, None), "a request one frame after a cut still cuts"); } #[test] fn a_reopened_encoder_ignores_requests_already_counted() { let mut cuts = Cuts::new(5); - assert!(!cuts.due(5, at(0))); - assert!(!cuts.due(5, at(1_000))); - } - - /// The encoder's own GOP keyframe lands while a request waits out the spacing - /// window, so the request is served and no extra keyframe is forced. - #[test] - fn a_cadence_keyframe_serves_a_waiting_request() { - let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(cuts.due(1, at(1_000)), "forced"); - cuts.keyframe(at(1_000)); - - assert!(!cuts.due(2, at(1_100)), "too soon after the forced keyframe"); - assert!(!cuts.due(2, at(1_200))); - cuts.keyframe(at(1_200)); - - // Well past the window, and nothing left to force. - assert!(!cuts.due(2, at(1_600))); - assert!(!cuts.due(2, at(3_000))); + assert!(!cuts.due(5, None)); + assert!(!cuts.due(5, None)); } - /// A cadence keyframe restarts the spacing window, so a request right after it - /// cannot put two keyframes a frame apart. + /// A join cuts once per wanted group: the frames still in the encoder ask for the same + /// group, and a group that lands drops the ask until another join raises it. #[test] - fn a_request_right_after_a_cadence_keyframe_waits_the_interval() { + fn a_wanted_group_cuts_once() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(!cuts.due(0, at(2_000))); - cuts.keyframe(at(2_000)); - - assert!(!cuts.due(1, at(2_033))); - assert!(!cuts.due(1, at(2_499))); - assert!(cuts.due(1, at(2_500)), "held until the interval elapsed"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(0, Some(1)), "a join past the largest group"); + assert!(!cuts.due(0, Some(1)), "the keyframe is still in the encoder"); + assert!(!cuts.due(0, None), "group 1 landed"); + assert!(cuts.due(0, Some(2)), "the next join"); } - /// A buffering encoder can return a keyframe for a frame read before the - /// request, which does not serve it. + /// A request and a join before the same frame cut it once. #[test] - fn a_keyframe_older_than_the_request_does_not_serve_it() { + fn a_request_and_a_join_coalesce() { let mut cuts = Cuts::new(0); - assert!(!cuts.due(0, at(0))); - assert!(!cuts.due(1, at(100)), "too soon"); - cuts.keyframe(at(50)); - - assert!(!cuts.due(1, at(549)), "the window restarts at the latest keyframe"); - assert!(cuts.due(1, at(550)), "the request is still owed a keyframe"); + assert!(!cuts.due(0, None)); + assert!(cuts.due(1, Some(1))); + assert!(!cuts.due(1, Some(1))); } } diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index c9bd11e4d03..3ea92a8eee9 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -437,7 +437,9 @@ impl Encoder { /// Rarely needed: the encoder opens groups automatically, so reach for this /// only when something outside the encoder needs a boundary at a specific /// frame. A source group boundary (transcode mirrors the source's groups) is - /// the usual reason; a scene change or a source switch is another. + /// the usual reason; a scene change, a source switch, or a subscriber joining + /// over [`track::Info::refresh`](moq_net::track::Info::refresh) are others. + /// Capture cuts for joiners itself. /// /// The request waits for the next [`encode`](Self::encode) rather than applying /// at once, so it is safe to call before the frame exists. Calling it repeatedly diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 274c1ed42f2..2d8ea2eb45f 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -5,8 +5,12 @@ //! nothing has been encoded yet), and the encoder itself only runs while a //! subscriber is watching. Capture opens its camera once at startup to learn //! the mode it negotiates, then keeps it closed between viewers. This mirrors -//! `moq-boy`, which pauses its emulator on `track::Producer::used()` / -//! `unused()`. +//! `moq-boy`, which pauses its emulator while +//! [`Demand::wanted`](moq_net::track::Demand::wanted) is `None`. +//! +//! Capture advertises [`track::Info::refresh`](moq_net::track::Info::refresh) when its encoder +//! is sure to honor a cut, and forces a keyframe whenever a joining subscriber asks for a group +//! past the largest. The encoder's GOP stays the upper bound between keyframes. use moq_mux::catalog::hang::CatalogExt; #[cfg(feature = "capture")] @@ -34,6 +38,17 @@ use super::encoder::Codec; #[cfg(feature = "capture")] const DEFAULT_FRAMERATE: Rate = Rate::integer(30); +/// The [`track::Info::refresh`](moq_net::track::Info::refresh) ratio a capture advertises: a +/// joiner forces a keyframe once a group's deltas outweigh twice its keyframe, so it downloads +/// at most about three keyframes' worth before it can decode. +/// +/// Measured on realtime CBR encodes (two-second GOPs at 2 to 4 Mbps), where a GOP's deltas run +/// three to five keyframes: one join a second halves the mean joiner download for 10 to 13% +/// more bitrate, and one every ten seconds costs about 1%. A ratio scales with the keyframe, +/// so it holds for every rendition. +#[cfg(feature = "capture")] +const REFRESH: u32 = 2; + /// Convert the probed rendition into the importer hint published before the first frame. fn rendition_hint(rendition: hang::catalog::VideoConfig) -> moq_mux::catalog::VideoHint { let mut hint = moq_mux::catalog::VideoHint::default(); @@ -78,6 +93,12 @@ enum Codecs { /// publishes into a catalog of the same shape. pub struct Producer { codecs: Codecs, + /// The published track, read for its largest group. + #[cfg_attr(not(feature = "capture"), allow(dead_code))] + track: moq_net::track::Producer, + /// The largest group published, as last read from `track`. + #[cfg_attr(not(feature = "capture"), allow(dead_code))] + largest: Option, _ext: std::marker::PhantomData E>, } @@ -123,6 +144,7 @@ impl Producer { reserved: moq_mux::catalog::Reserved, rendition: hang::catalog::VideoConfig, ) -> Result { + let published = track.clone(); let codecs = match &rendition.codec { hang::catalog::VideoCodec::H264(_) => Codecs::H264 { split: moq_mux::codec::h264::Split::new(), @@ -141,6 +163,8 @@ impl Producer { }; Ok(Self { codecs, + track: published, + largest: None, _ext: std::marker::PhantomData, }) } @@ -155,6 +179,18 @@ impl Producer { } } + /// The group a subscriber wants past the largest published, which the next frame should + /// open with a keyframe, or `None` while the largest serves. + #[cfg(feature = "capture")] + fn wanted(&mut self) -> Option { + let wanted = self.track.demand().wanted()?; + // Ready only when the largest moved since the last read. + if let std::task::Poll::Ready(Some(largest)) = self.track.poll_largest(&kio::Waiter::noop()) { + self.largest = Some(largest.group); + } + self.largest.is_none_or(|largest| wanted > largest).then_some(wanted) + } + /// Publish already-encoded frames, each at its own timestamp. Each frame is one /// whole access unit in the producer's codec framing. pub fn publish(&mut self, encoded: &[Encoded]) -> Result<(), Error> { @@ -316,7 +352,11 @@ impl Control { Codec::H264 => ".avc3", Codec::H265 => ".hev1", }; - let track = broadcast.unique_track(suffix, catalog.track_info(hang::catalog::PRIORITY.video))?; + // Advertised only when every backend the encoder may open honors a cut, since the info + // is fixed before the driver learns which one it got. + let refresh = super::backend::always_cuts(options.encode.codec, &options.encode.kind).then_some(REFRESH); + let info = catalog.track_info(hang::catalog::PRIORITY.video).with_refresh(refresh); + let track = broadcast.unique_track(suffix, info)?; let requests = kio::Producer::new(Requests::default()); let driver = Driver { track, @@ -330,10 +370,11 @@ impl Control { /// Request a keyframe, opening a new group at a frame no earlier than this call. /// /// For a resume, a recording cut, or a known tune-in moment; [`Config::gop`](super::Config::gop) - /// is the cadence. Requests coalesce into the next keyframe, whether forced or on that cadence, - /// and a forced one lands at least 500ms after any other, so a caller in a loop cannot pin the - /// encoder at all-IDR; a request that arrives too soon waits rather than being dropped. A - /// request while nothing is watching is served by the keyframe every fresh encoder opens with. + /// is the longest a group runs, and a joining subscriber already opens one through + /// [`track::Info::refresh`](moq_net::track::Info::refresh). Requests between two frames + /// coalesce into one keyframe on the next, with no spacing: a caller in a loop pins the + /// encoder at all-IDR. A request while nothing is watching is served by the keyframe every + /// fresh encoder opens with. /// /// # Errors /// @@ -714,7 +755,8 @@ async fn capture_loop( }; // A reopen can land on another backend than the probe did, so the controls // learn this one's answer before any request is counted against it. - let served = learn(requests, refusal(&mut encoder).await?); + let refused = refusal(&mut encoder).await?; + let served = learn(requests, refused); tracing::info!(encoder = encoder.name(), device = camera.label(), "capturing"); // A reopen can negotiate a different mode (a display resized while nothing was @@ -730,7 +772,7 @@ async fn capture_loop( // reopened camera starts optimistic again rather than inheriting the // backed-off rate from whatever the link was doing last time. let mut rate = Some((reservation.consumer(), rate::Control::new(rate::Policy::new(ceiling)))); - // Per encoder, so a reopen forgets the old encoder's last keyframe along with it. + // Per encoder, since a fresh one opens with a keyframe that serves every earlier ask. let mut forced = Cuts::new(served); loop { @@ -760,16 +802,15 @@ async fn capture_loop( let Some(mut frame) = frame else { break }; frame.timestamp = map_capture_timestamp(capture_epoch, frame.timestamp)?; let count = counted.read().count; - if forced.due(count, frame.timestamp) { + // A backend that cannot cut leaves every group to its cadence. + let wanted = refused.is_none().then(|| producer.wanted()).flatten(); + if forced.due(count, wanted) { // Never refused: `Control::cut` stops counting once this encoder is known to. encoder.cut().await?; } let Some(encoded) = wait_capture(demand, encoder.encode(frame)).await? else { break; }; - for unit in encoded.iter().filter(|unit| unit.keyframe) { - forced.keyframe(unit.timestamp); - } producer.publish(&encoded)?; } @@ -1523,8 +1564,38 @@ mod tests { } } - /// Drive cuts through the public control and read their group boundaries from the track. - async fn capture_cuts(cut_after_cadence: bool) -> Vec { + /// What the driver is fed next, one at a time. + enum Step { + /// A frame 10ms after the previous one, waited on until the first subscriber has it. + Frame(crate::frame::Surface), + /// A keyframe request through the public control. + Cut, + /// Another subscriber joining at the live edge, held until the end. + Join, + } + + /// A frame of seeded noise, which no encoder can predict from the one before it. + fn noise(seed: u32) -> crate::frame::Surface { + let mut state = seed.wrapping_mul(2_654_435_761) | 1; + let data = (0..320 * 240 * 3 / 2) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + 120 + (state as u8 >> 4) + }) + .collect(); + crate::frame::Surface::I420(crate::frame::I420 { + width: 320, + height: 240, + data, + color: None, + }) + } + + /// Drive the real driver through `steps`, returning the info the track advertised and the + /// payload size of every frame the first subscriber read, grouped as it read them. + async fn capture_groups(steps: Vec) -> (moq_net::track::Info, Vec>) { use moq_mux::container::Container as _; // Off Apple, encoding waits on an OS thread. Keep Tokio from auto-advancing @@ -1535,7 +1606,7 @@ mod tests { }); let started = tokio::time::Instant::now(); // Capture streams are !Send on macOS. - let groups = tokio::task::LocalSet::new() + let result = tokio::task::LocalSet::new() .run_until(async { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); @@ -1543,10 +1614,12 @@ mod tests { moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); let mut options = Capture::default(); options.encode.kind = encoder::Kind::Software; + // Room for noise, which rate control would otherwise skip frames of. + options.encode.bitrate = Some(moq_net::bandwidth::Rate::from_mbps(4)); let (control, driver) = Control::new(broadcast.clone(), catalog, options).unwrap(); - // Ten nominal fps makes the default two-second GOP twenty frames. Feed them - // 10ms apart in media time so cadence falls inside the 500ms cut spacing window. + // Ten nominal fps makes the default two-second GOP twenty frames, which every + // script below stays inside. let rate = crate::Rate::new(10, 1).unwrap(); let (_probe, probe) = Synthetic::open(crate::Size::new(320, 240), rate); let (camera, stream) = Synthetic::open(crate::Size::new(320, 240), rate); @@ -1555,28 +1628,30 @@ mod tests { moq_mux::catalog::Consumer::<()>::new(&consumer, moq_mux::catalog::CatalogFormat::Hang) .await .unwrap(); - let (mut subscriber, format) = loop { + let (name, mut subscriber, format) = loop { let snapshot = catalog.next().await.unwrap().expect("a catalog"); if let Some((name, rendition)) = snapshot.video.renditions.iter().next() { break ( + name.clone(), consumer.track(name).unwrap().subscribe(None).await.unwrap(), moq_mux::catalog::hang::Container::try_from(rendition).unwrap(), ); } }; + let info = subscriber.info().clone(); let (published, mut received) = tokio::sync::mpsc::unbounded_channel(); let collect = tokio::task::spawn_local(async move { let mut groups = Vec::new(); while let Some(mut group) = subscriber.recv_group().await.unwrap() { - let mut frames = 0; + let mut frames = Vec::new(); while let Some(batch) = kio::wait(|waiter| format.poll_read(&mut group, waiter)).await.unwrap() { for frame in batch { // A duration marker closes the group but is not an encoded picture. if format.end(&frame).is_none() { - frames += 1; + frames.push(frame.payload.len()); published.send(()).unwrap(); } } @@ -1586,36 +1661,97 @@ mod tests { groups }); - let timestamps = (0..=20).map(|frame| frame * 10).chain([201, 699, 700, 1200]); - for (frame, millis) in timestamps.enumerate() { - camera.push_native(surface(), us(millis * 1000)); - // Wait for delivery before requesting a cut or replacing the capture's latest slot. - received.recv().await.expect("the frame reached the subscriber"); - if frame == 0 || (frame == 20 && cut_after_cadence) { - control.cut().unwrap(); + let mut joined = Vec::new(); + let mut millis = 0; + for step in steps { + match step { + Step::Frame(surface) => { + camera.push_native(surface, us(millis * 1000)); + millis += 10; + // Wait for delivery before the next step, which also keeps the + // capture's latest slot from being replaced. + received.recv().await.expect("the frame reached the subscriber"); + } + Step::Cut => control.cut().unwrap(), + Step::Join => joined.push(consumer.track(&name).unwrap().subscribe(None).await.unwrap()), } } drop(control); run.await.unwrap().unwrap(); - collect.await.unwrap() + (info, collect.await.unwrap()) }) .await; assert_eq!(started.elapsed(), Duration::ZERO); drop(hold); clock.await.unwrap(); - groups + result } + /// Each request opens a group at the next frame, with no spacing between them. #[tokio::test(start_paused = true)] - async fn cadence_keyframe_serves_a_pending_capture_cut() { - // The keyframe at 200ms serves the pending cut. Passing 700ms must not force another. - assert_eq!(capture_cuts(false).await, [20, 5]); + async fn capture_cuts_land_on_the_next_frame() { + let steps = vec![ + Step::Frame(surface()), + Step::Cut, + Step::Frame(surface()), + Step::Frame(surface()), + Step::Cut, + Step::Frame(surface()), + Step::Frame(surface()), + ]; + let (_, groups) = capture_groups(steps).await; + let lengths: Vec<_> = groups.iter().map(Vec::len).collect(); + assert_eq!(lengths, [1, 2, 2]); } + /// A capture whose encoder always cuts advertises the refresh ratio, and a join opens a + /// group at the next frame only once the deltas outweigh it, well within 500ms of the + /// keyframe before. #[tokio::test(start_paused = true)] - async fn capture_cut_after_cadence_waits_for_spacing() { - // A new cut after 200ms waits through 699ms, opens at 700ms, and fires only once. - assert_eq!(capture_cuts(true).await, [20, 3, 2]); + async fn a_join_over_the_refresh_ratio_forces_a_keyframe() { + let mut steps = vec![ + Step::Frame(noise(0)), + Step::Frame(surface()), + Step::Frame(surface()), + // Two flat deltas cost next to nothing against a noise keyframe. + Step::Join, + Step::Frame(surface()), + ]; + steps.extend((1..=10).map(|seed| Step::Frame(noise(seed)))); + steps.extend([Step::Join, Step::Frame(surface()), Step::Frame(surface())]); + let (info, groups) = capture_groups(steps).await; + assert_eq!(info.refresh, Some(REFRESH)); + + let lengths: Vec<_> = groups.iter().map(Vec::len).collect(); + assert_eq!(lengths, [14, 2], "only the second join cut, at the frame after it"); + let ratio = usize::try_from(REFRESH).unwrap(); + let first = &groups[0]; + assert!( + first[1..3].iter().sum::() <= ratio * first[0], + "the first join was under the ratio: {first:?}" + ); + assert!( + first[1..].iter().sum::() > ratio * first[0], + "the second join was over the ratio: {first:?}" + ); + } + + /// Refresh is advertised only when every backend the encoder may open honors a cut, since + /// the track info is fixed before the driver learns which one it got. + #[test] + fn a_backend_that_may_refuse_cuts_advertises_no_refresh() { + for (kind, refresh) in [ + (encoder::Kind::Software, Some(REFRESH)), + (encoder::Kind::Named(probe::NO_CUT.into()), None), + ] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = + moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut options = Capture::default(); + options.encode.kind = kind; + let (_control, driver) = Control::new(broadcast.clone(), catalog, options).unwrap(); + assert_eq!(driver.track.info().refresh, refresh); + } } /// Drop the controls while the driver sits in its `opens`th open, which never finishes.