From aea97892f7e19f65b21588b13bfba5cafedc742f Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Tue, 13 Oct 2026 14:00:23 -0600 Subject: [PATCH 1/5] feat(hang): add projection mod to video --- rs/hang/src/catalog/root.rs | 13 +++ rs/hang/src/catalog/video/mod.rs | 14 +++ rs/hang/src/catalog/video/projection.rs | 132 ++++++++++++++++++++++++ rs/hang/src/error.rs | 4 + 4 files changed, 163 insertions(+) create mode 100644 rs/hang/src/catalog/video/projection.rs diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 3489106df68..49fd1879798 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -150,6 +150,9 @@ impl Catalog { max: MAX_RENDITIONS, }); } + if let Some(projection) = &self.video.projection { + projection.check()?; + } Ok(()) } @@ -402,6 +405,16 @@ mod test { assert_eq!(encoded, output, "encode mismatch"); } + #[test] + fn check_rejects_bad_fov() { + let catalog = + Catalog::<()>::from_str(r#"{"video":{"renditions":{},"projection":{"kind":"equirectangular","fov":400}}}"#) + .unwrap(); + assert!(matches!( + catalog.check_renditions(), + Err(crate::Error::InvalidFieldOfView(_)) + )); + } #[test] fn rendition_with_broadcast_override() { // Decode a catalog where one rendition references a track in a sibling broadcast, diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index 8d3de493c9e..088ec863eac 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -2,12 +2,14 @@ mod av1; mod codec; mod h264; mod h265; +mod projection; mod vp9; pub use av1::*; pub use codec::*; pub use h264::*; pub use h265::*; +pub use projection::*; pub use vp9::*; use std::collections::{BTreeMap, btree_map}; @@ -54,6 +56,10 @@ pub struct Video { /// Default: false #[serde(default)] pub flip: Option, + + /// The type of projection for video + #[serde(default)] + pub projection: Option, } /// Catalog properties shared by every video rendition. @@ -68,11 +74,16 @@ pub struct VideoProperties { /// Flip horizontally after rotation, or clear the explicit value when absent. pub flip: Option, + + pub projection: Option, } impl VideoProperties { fn normalized(mut self) -> crate::Result { self.rotation = self.rotation.map(normalize_video_rotation).transpose()?; + if let Some(projection) = &self.projection { + projection.check()?; + } Ok(self) } } @@ -138,6 +149,7 @@ impl Video { self.display = properties.display; self.rotation = properties.rotation; self.flip = properties.flip; + self.projection = properties.projection; Ok(()) } } @@ -501,6 +513,7 @@ mod test { display: None, rotation: Some(f64::NAN), flip: None, + projection: None, }), Err(crate::Error::InvalidVideoRotation) )); @@ -537,6 +550,7 @@ mod test { }), rotation: Some(315.0), flip: None, + projection: None, }) .unwrap(); diff --git a/rs/hang/src/catalog/video/projection.rs b/rs/hang/src/catalog/video/projection.rs new file mode 100644 index 00000000000..c0f0a61d33b --- /dev/null +++ b/rs/hang/src/catalog/video/projection.rs @@ -0,0 +1,132 @@ +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; + +/// How the coded picture maps onto the viewer's field of view. +#[derive(Debug, Clone, PartialEq, Default)] +pub enum Projection { + /// A flat picture + #[default] + Rectilinear, + + /// Viewer is in the middle of wrapped sphere + Equirectangular { + /// Degrees of longitude the picture spans, in range 0..360 + fov: Option, + }, + + Unknown(UnknownProjection), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct UnknownProjection(serde_json::Map); + +impl UnknownProjection { + pub fn kind(&self) -> Option<&str> { + self.0.get("kind").and_then(serde_json::Value::as_str) + } + + pub fn fields(&self) -> &serde_json::Map { + &self.0 + } +} + +impl Projection { + /// Check that projection is in range 0..360. + pub(crate) fn check(&self) -> crate::Result<()> { + if let Self::Equirectangular { fov: Some(fov) } = self + && !(fov.is_finite() && *fov > 0.0 && *fov <= 360.0) + { + return Err(crate::Error::InvalidFieldOfView(*fov)); + } + Ok(()) + } +} + +#[serde_with::skip_serializing_none] +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase", deny_unknown_fields)] +enum Known { + Rectilinear {}, + Equirectangular { + #[serde(default)] + fov: Option, + }, +} + +impl Serialize for Projection { + fn serialize(&self, serializer: S) -> Result { + let known = match self { + Self::Rectilinear => Known::Rectilinear {}, + Self::Equirectangular { fov } => Known::Equirectangular { fov: *fov }, + Self::Unknown(unknown) => return unknown.0.serialize(serializer), + }; + known.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Projection { + fn deserialize>(deserializer: D) -> Result { + let object = serde_json::Map::deserialize(deserializer)?; + + match object.get("kind").and_then(serde_json::Value::as_str) { + Some("rectilinear" | "equirectangular") => { + let known = Known::deserialize(serde_json::Value::Object(object)).map_err(de::Error::custom)?; + Ok(match known { + Known::Rectilinear {} => Self::Rectilinear, + Known::Equirectangular { fov } => Self::Equirectangular { fov }, + }) + } + _ => Ok(Self::Unknown(UnknownProjection(object))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips() { + for (json, projection) in [ + (r#"{"kind":"rectilinear"}"#, Projection::Rectilinear), + ( + r#"{"kind":"equirectangular"}"#, + Projection::Equirectangular { fov: None }, + ), + ( + r#"{"kind":"equirectangular","fov":180.0}"#, + Projection::Equirectangular { fov: Some(180.0) }, + ), + ] { + let parsed: Projection = serde_json::from_str(json).unwrap(); + assert_eq!(parsed, projection); + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + } + } + + #[test] + fn unknown_round_trips() { + let json = r#"{"face":"front","kind":"cubemap"}"#; + let parsed: Projection = serde_json::from_str(json).unwrap(); + let Projection::Unknown(unknown) = &parsed else { + panic!("expected unknown: {parsed:?}"); + }; + assert_eq!(unknown.kind(), Some("cubemap")); + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + } + + #[test] + fn malformed_known_errors() { + serde_json::from_str::(r#"{"kind":"equirectangular","fov":"wide"}"#).unwrap_err(); + serde_json::from_str::(r#"{"kind":"rectilinear","fov":90}"#).unwrap_err(); + } + + #[test] + fn fov_must_be_in_range() { + for fov in [0.0, -90.0, 360.5, f64::NAN, f64::INFINITY] { + Projection::Equirectangular { fov: Some(fov) }.check().unwrap_err(); + } + for fov in [None, Some(0.5), Some(180.0), Some(360.0)] { + Projection::Equirectangular { fov }.check().unwrap(); + } + } +} diff --git a/rs/hang/src/error.rs b/rs/hang/src/error.rs index 03d42933d37..a238aec0c09 100644 --- a/rs/hang/src/error.rs +++ b/rs/hang/src/error.rs @@ -72,6 +72,10 @@ pub enum Error { /// The shared video rotation is not a finite number. #[error("video rotation must be finite")] InvalidVideoRotation, + + /// The projected video's field of view is outside (0, 360] degrees. + #[error("video field of view {0} must be in (0, 360] degrees")] + InvalidFieldOfView(f64), } /// A Result type alias for hang operations. From 8b2c28b27c8e1b3cb58bb806b08319e42a49035d Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Wed, 14 Oct 2026 14:24:30 -0600 Subject: [PATCH 2/5] feat(hang): add projection + audio layout to js --- js/hang/src/catalog/audio.test.ts | 18 +++++++++++ js/hang/src/catalog/audio.ts | 20 +++++++++--- js/hang/src/catalog/video.test.ts | 53 +++++++++++++++++++++++++++++++ js/hang/src/catalog/video.ts | 49 +++++++++++++++++++++++++++- 4 files changed, 134 insertions(+), 6 deletions(-) diff --git a/js/hang/src/catalog/audio.test.ts b/js/hang/src/catalog/audio.test.ts index e81523d1d84..6938005ea2c 100644 --- a/js/hang/src/catalog/audio.test.ts +++ b/js/hang/src/catalog/audio.test.ts @@ -49,3 +49,21 @@ test("audio warmup round trips as optional integer milliseconds", () => { expect(() => AudioConfigSchema.parse({ ...base, warmup })).toThrow(); } }); + +test("ambisonic layout needs a perfect-square channel count of at least 4", () => { + const base = { codec: "opus", container: { kind: "legacy" }, sampleRate: 48_000, layout: "ambisonic" }; + for (const numberOfChannels of [4, 9, 16]) { + expect(AudioConfigSchema.parse({ ...base, numberOfChannels }).layout).toBe("ambisonic"); + } + for (const numberOfChannels of [1, 2, 6, 8]) { + expect(() => AudioConfigSchema.parse({ ...base, numberOfChannels })).toThrow(); + } +}); + +test("audio layout is optional and keeps unknown values verbatim", () => { + const base = { codec: "opus", container: { kind: "legacy" }, sampleRate: 48_000, numberOfChannels: 6 }; + expect(AudioConfigSchema.parse(base).layout).toBeUndefined(); + // An unknown layout puts no constraint on the channel count. + expect(AudioConfigSchema.parse({ ...base, layout: "surround-5.1" }).layout).toBe("surround-5.1"); + expect(() => AudioConfigSchema.parse({ ...base, layout: 1 })).toThrow(); +}); diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index c40d4bd49d7..85b6fd2c67d 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -9,11 +9,7 @@ const TrackSchema = z.object({ name: z.string(), }); -/** - * Schema for a single audio rendition's decoder config. - * Mirrors WebCodecs AudioDecoderConfig (https://w3c.github.io/webcodecs/#audio-decoder-config). - */ -export const AudioConfigSchema = z.object({ +const AudioConfigFields = z.object({ // Optional reference to another broadcast that publishes this track, expressed // relative to the broadcast that served this catalog (e.g. "./source"). // If unset, the track lives in the same broadcast as the catalog. @@ -78,8 +74,22 @@ export const AudioConfigSchema = z.object({ z.transform((value) => (value === 0 ? undefined : value)), ), ), + + // "ambisonic" is AmbiX; the order comes from numberOfChannels. + layout: z.optional(z.union([z.literal("ambisonic"), z.string()])), }); +/** + * Schema for a single audio rendition's decoder config. + * Mirrors WebCodecs AudioDecoderConfig (https://w3c.github.io/webcodecs/#audio-decoder-config). + */ +export const AudioConfigSchema = AudioConfigFields.check( + z.refine( + (c) => c.layout !== "ambisonic" || (c.numberOfChannels >= 4 && Number.isInteger(Math.sqrt(c.numberOfChannels))), + { message: "ambisonic needs a perfect-square channel count >= 4" }, + ), +); + /** Schema for the catalog audio section: a map of track name to rendition config. */ export const AudioSchema = z.union([ z.object({ diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 69db8fc0a13..8cba108e06f 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -137,3 +137,56 @@ test("video warmup round trips as optional integer milliseconds", () => { expect(() => VideoConfigSchema.parse({ ...base, warmup })).toThrow(); } }); + +const section = (fields: object) => ({ renditions: {}, ...fields }); + +test("projection parses the known kinds", () => { + for (const [json, projection] of [ + ['{"kind":"rectilinear"}', { kind: "rectilinear" }], + ['{"kind":"equirectangular"}', { kind: "equirectangular" }], + ['{"kind":"equirectangular","fov":180.0}', { kind: "equirectangular", fov: 180 }], + ] as const) { + expect(VideoSchema.parse(section({ projection: JSON.parse(json) })).projection).toEqual(projection); + } +}); + +test("projection is optional", () => { + expect(VideoSchema.parse(section({})).projection).toBeUndefined(); +}); + +test("unknown projection is preserved instead of throwing", () => { + const raw = JSON.parse('{"face":"front","kind":"cubemap"}'); + expect(VideoSchema.parse(section({ projection: raw })).projection).toEqual({ kind: "unknown", raw }); +}); + +test("malformed known projection throws", () => { + for (const json of ['{"kind":"equirectangular","fov":"wide"}', '{"kind":"rectilinear","fov":90}']) { + expect(() => VideoSchema.parse(section({ projection: JSON.parse(json) }))).toThrow(); + } +}); + +test("projection fov must be in (0, 360]", () => { + for (const fov of [0, -90, 360.5, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => VideoSchema.parse(section({ projection: { kind: "equirectangular", fov } }))).toThrow(); + } + for (const fov of [0.5, 180, 360]) { + expect(VideoSchema.parse(section({ projection: { kind: "equirectangular", fov } })).projection).toEqual({ + kind: "equirectangular", + fov, + }); + } +}); + +test("stereo parses known layouts and keeps unknown ones verbatim", () => { + for (const stereo of ["mono", "side-by-side", "top-bottom", "interleaved"]) { + expect(VideoSchema.parse(section({ stereo })).stereo).toBe(stereo); + } + expect(VideoSchema.parse(section({})).stereo).toBeUndefined(); + expect(() => VideoSchema.parse(section({ stereo: 2 }))).toThrow(); +}); + +test("legacy array catalogs have no projection or stereo", () => { + const parsed = VideoSchema.parse([{ track: { name: "v" }, config: rendition() }]); + expect(parsed.projection).toBeUndefined(); + expect(parsed.stereo).toBeUndefined(); +}); diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index 570d5a31a9b..82c38d89ed4 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -90,8 +90,41 @@ export const VideoConfigSchema = z.object({ ), }); +// projection kinds this build knows how to render. +const KNOWN_PROJECTIONS = ["rectilinear", "equirectangular"]; + +// projection this build does not recognized, not to be rendered +export const UnknownProjectionSchema = z.pipe( + z.looseObject({ + kind: z.string().check( + z.refine((kind) => !KNOWN_PROJECTIONS.includes(kind), { + message: "recognized projection kind must match its own schema", + }), + ), + }), + z.transform((raw) => ({ kind: "unknown" as const, raw })), +); + +// How the coded picture maps onto the viewer's FOV. +export const ProjectionSchema = z.union([ + z.discriminatedUnion("kind", [ + // A flat picture + z.strictObject({ kind: z.literal("rectilinear") }), + // Longitude across the width and latitude down the height, wrapped onto a sphere. + z.strictObject({ + kind: z.literal("equirectangular"), + // Degrees of longitude the picture spans from 0..360. + fov: z.optional(z.number().check(z.positive(), z.lte(360))), + }), + ]), + UnknownProjectionSchema, +]); + +// How the viewer's pictures share each frame. +export const StereoSchema = z.union([z.enum(["mono", "side-by-side", "top-bottom"]), z.string()]); + /** - * Schema for the catalog video section: renditions plus display size, rotation, and flip. + * Schema for the catalog video section: renditions plus display size, rotation, flip, projection, and stereo. * Renditions mirror WebCodecs VideoDecoderConfig (https://w3c.github.io/webcodecs/#video-decoder-config). */ export const VideoSchema = z.union([ @@ -116,6 +149,14 @@ export const VideoSchema = z.union([ // If true, the decoder will flip the video horizontally // Default: false flip: z.optional(z.boolean()), + + // How the picture maps onto the viewer's field of view. + // Default: rectilinear + projection: z.optional(ProjectionSchema), + + // How the two eyes' pictures share each frame. + // Default: mono + stereo: z.optional(StereoSchema), }), // Backwards compatibility: transform old array of {track, config} to new object format z.pipe( @@ -135,6 +176,8 @@ export const VideoSchema = z.union([ : undefined, rotation: undefined, flip: undefined, + projection: undefined, + stereo: undefined, }; }), ), @@ -144,6 +187,10 @@ export const VideoSchema = z.union([ export type Video = z.infer; /** Decoder config for a single video rendition. */ export type VideoConfig = z.infer; +/** How the coded picture maps onto the viewer's FOV. */ +export type Projection = z.infer; +/** How the two eyes' pictures share each frame. */ +export type Stereo = z.infer; /** * Rank video renditions best first: largest picture, then highest bitrate, then name. From 510a337a0074240f85173a6f8b1f86a18b3f43c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 13:41:44 -0700 Subject: [PATCH 3/5] feat(hang): per-rendition projection, stereo, and audio layout Move projection to each video rendition beside color, add stereo (mono/left-right/top-bottom) in Rust and JS, and add the audio layout in Rust. Unknown values are preserved and treated as absent; unknown fields on a known projection are ignored. Restore the cross-kind duplicate rendition check and the color tests the earlier merge dropped. Document the fields in the hang draft and doc/concept. Co-Authored-By: Claude Opus 5.5 --- doc/concept/hang.md | 1 + drafts/draft-lcurley-moq-hang.md | 35 ++++- js/hang/src/catalog/audio.test.ts | 8 +- js/hang/src/catalog/audio.ts | 166 +++++++++++++----------- js/hang/src/catalog/index.ts | 4 +- js/hang/src/catalog/video.test.ts | 92 ++++++++----- js/hang/src/catalog/video.ts | 93 +++++++------ rs/hang/src/catalog/audio/layout.rs | 84 ++++++++++++ rs/hang/src/catalog/audio/mod.rs | 10 ++ rs/hang/src/catalog/root.rs | 62 +++++++-- rs/hang/src/catalog/video/mod.rs | 29 +++-- rs/hang/src/catalog/video/projection.rs | 62 ++++++--- rs/hang/src/catalog/video/stereo.rs | 72 ++++++++++ rs/hang/src/error.rs | 6 +- 14 files changed, 509 insertions(+), 215 deletions(-) create mode 100644 rs/hang/src/catalog/audio/layout.rs create mode 100644 rs/hang/src/catalog/video/stereo.rs diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 293ee3d6ac6..2ff655fb8fb 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -68,6 +68,7 @@ A few things the catalog can express beyond decoder config: - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. - **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, an AV1 track's av1C, or Matroska, but not yet from H.264 or H.265 in fMP4 or FLV. Nothing in `@moq/watch` reads it yet. +- **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. Nothing in `@moq/watch` renders them yet. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no frames coming, and a player must not select it. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. - **Archive.** A broadcast may advertise an `archive` entry mapping each track, the catalog included, to its own timeline track (a small index of the track's spans: timing plus group and frame positions) and fixed nominal record duration, plus the replay MoQ path, object-store URL, and format version when recorded. Media records end at the first group boundary at or beyond a 4s target; sparse timelines such as the catalog declare zero. Tracks commit and expire independently. An untimed track has no timing to index, so its timeline declares no duration and holds one record per group: it can be recorded and replayed, but not turned into HLS or kept in a DVR. The timelines are what let the [HLS gateway](/bin/hls) build playlists without subscribing to media. A reader that cannot decode the `archive` entry, such as one from a different version, ignores it and loses the timelines and replay, but live media still plays. diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 6d441557ac3..576b331cf17 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -251,6 +251,8 @@ type VideoDecoderConfigExtensions = { "displayAspectHeight": number | undefined, "preference": number | undefined, "color": Color | undefined, + "projection": Projection | undefined, + "stereo": string | undefined, } ~~~ @@ -296,6 +298,25 @@ type Chromaticity = { "x": number, "y": number } A publisher MUST omit a field its source leaves unspecified rather than write a default, and SHOULD prefer the value signalled in the elementary stream over a container's when they disagree. `color` is not part of the track's identity ({{identity}}). +`projection` maps the decoded picture onto the viewer's field of view: + +~~~ +type Projection = + { "kind": "rectilinear" } | + { "kind": "equirectangular", "fov": number | undefined } +~~~ + +`rectilinear`, the default when absent, is a flat picture. +`equirectangular` maps longitude linearly across the width and latitude linearly down the height, spanning all 180 degrees of latitude around a viewer at the centre. +Its `fov` is the degrees of longitude the picture spans, centred straight ahead: greater than 0 and at most 360, defaulting to 360, so VR180 is 180. + +`stereo` says how each eye's view is packed into the frame. +`mono`, the default when absent, shows the whole frame to both eyes. +`left-right` carries the left eye's view in the left half and the right eye's in the right half, and `top-bottom` carries the left eye's view in the top half and the right eye's in the bottom half. + +A consumer MUST treat an unrecognized `projection` kind or `stereo` value as absent, and MUST reject a recognized `projection` kind with a malformed field, such as a `fov` out of range. +Neither field is part of the track's identity. + For example: ~~~ @@ -370,7 +391,18 @@ See the [WebCodecs specification](https://www.w3.org/TR/webcodecs/#audio-decoder Any field carrying raw bytes, notably `description`, is a hex string ({{binary}}). The `sampleRate` and `numberOfChannels` fields describe the decoded output: for HE-AAC, the SBR rate rather than the core's, and two channels under parametric stereo. -In addition to the WebCodecs fields, each rendition MAY carry the common rendition fields ({{common}}). +In addition to the WebCodecs fields, each rendition MAY carry the common rendition fields ({{common}}) plus: + +~~~ +type AudioDecoderConfigExtensions = { + "layout": string | undefined, +} +~~~ + +`layout` says what the decoded channels carry; when absent, they follow the codec's own channel order. +`ambisonic` is full-sphere ambisonics in the AmbiX convention: ACN channel order with SN3D normalization, of the order N for which `numberOfChannels` is (N+1)^2. +A consumer MUST reject `ambisonic` with fewer than 4 channels or a count that is not a square, and MUST treat an unrecognized `layout` as absent. +`layout` is not part of the track's identity. ### PCM {#audio-pcm} @@ -1358,6 +1390,7 @@ This document has no IANA actions. - Added an optional `track` field to video, audio, text, `json`, and `binary` entries naming the track in its broadcast; the entry's key is its ID and names the track when `track` is absent. A consumer MUST reject a catalog using one ID for both a video and an audio rendition. - Defined video `displayAspectWidth` and `displayAspectHeight` as the whole picture's display aspect, not the sample aspect. - Added the optional video `color` field: H.273 primaries, transfer, matrix, and range, plus the SMPTE ST 2086 mastering display and CTA-861.3 content light levels. +- Added the optional video `projection` (rectilinear, or equirectangular with a longitude `fov`) and `stereo` (mono, left-right, or top-bottom) fields and the optional audio `layout` field (ambisonic). ## moq-hang-03 {:numbered="false"} diff --git a/js/hang/src/catalog/audio.test.ts b/js/hang/src/catalog/audio.test.ts index af4ba3ec683..6514eee8871 100644 --- a/js/hang/src/catalog/audio.test.ts +++ b/js/hang/src/catalog/audio.test.ts @@ -86,12 +86,12 @@ test("audio warmup round trips as optional integer milliseconds", () => { } }); -test("ambisonic layout needs a perfect-square channel count of at least 4", () => { +test("ambisonic layout needs a square channel count of at least 4", () => { const base = { codec: "opus", container: { kind: "legacy" }, sampleRate: 48_000, layout: "ambisonic" }; - for (const numberOfChannels of [4, 9, 16]) { + for (const numberOfChannels of [4, 9, 16, 225]) { expect(AudioConfigSchema.parse({ ...base, numberOfChannels }).layout).toBe("ambisonic"); } - for (const numberOfChannels of [1, 2, 6, 8]) { + for (const numberOfChannels of [0, 1, 2, 6, 8]) { expect(() => AudioConfigSchema.parse({ ...base, numberOfChannels })).toThrow(); } }); @@ -100,6 +100,6 @@ test("audio layout is optional and keeps unknown values verbatim", () => { const base = { codec: "opus", container: { kind: "legacy" }, sampleRate: 48_000, numberOfChannels: 6 }; expect(AudioConfigSchema.parse(base).layout).toBeUndefined(); // An unknown layout puts no constraint on the channel count. - expect(AudioConfigSchema.parse({ ...base, layout: "surround-5.1" }).layout).toBe("surround-5.1"); + expect(AudioConfigSchema.parse({ ...base, layout: "surround" }).layout).toBe("surround"); expect(() => AudioConfigSchema.parse({ ...base, layout: 1 })).toThrow(); }); diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index 876a0513c7f..9407708f22c 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -10,89 +10,97 @@ const TrackSchema = z.object({ name: z.string(), }); -const AudioConfigFields = z.object({ - // Optional reference to another broadcast that publishes this track, expressed - // relative to the broadcast that served this catalog (e.g. "./source"). - // If unset, the track lives in the same broadcast as the catalog. - broadcast: z.optional(RelativeBroadcastSchema), - // The track's name in its broadcast, when it differs from this entry's ID (its key in the - // catalog). Absent means the ID is the track name. Resolve it with `track`. - track: z.optional(z.string()), - - // Human-readable rendition name for track pickers. - label: z.optional(z.string()), - - // Registered WebCodecs codec string, or Hang's "pcm" extension for - // interleaved little-endian IEEE-754 binary32 samples. - codec: z.string(), - - // The container format, used to decode the timestamp and more. - container: ContainerSchema, - - // The description is used for some codecs. - // If provided, we can initialize the decoder based on the catalog alone. - // Otherwise, the initialization information is in-band. - description: z.optional(hexSchema), - - // The sample rate of the audio in Hz - sampleRate: u53Schema, - - // The number of channels in the audio - numberOfChannels: u53Schema, - - // The maximum bitrate of the audio in bits per second. - // TODO: Support up to Number.MAX_SAFE_INTEGER - bitrate: z.optional(u53Schema), - - // Whether this rendition may be selected. When false, no frames are coming and a consumer - // must not select it. Default: true, so publishers only write it when false. - enabled: z.optional(z.boolean()), - - // The maximum delay between a frame being ready and the publisher flushing it, in whole - // milliseconds rounded up. The player's jitter buffer should be larger than this value. - // If not provided, the player should assume each frame is flushed immediately. - // - // This is measured at the publisher (encoder latency, packet packing, reordering), - // never on the network a consumer sees. It only ever grows over the life of a stream. - // - // NOTE: The audio "frame" duration depends on the codec, sample rate, etc. - // ex: AAC often uses 1024 samples per frame, so at 44100Hz, this would be 1024/44100 = 24ms - jitter: z.optional( - z.pipe( - u53Schema, - z.transform((value) => (value === 0 ? undefined : value)), - ), - ), - - // After a non-continuous join, decode from the group start, present at start plus warmup, - // and join that many milliseconds earlier. - warmup: z.optional(u53Schema), - - // How far this rendition's frames reach the transport behind the broadcast's earliest - // rendition, in whole milliseconds rounded up. A player holds `delay + jitter` for it and never - // subtracts one rendition's `delay` from another's. Absent on the earliest rendition. It only - // ever grows over the life of a stream. - delay: z.optional( - z.pipe( - u53Schema, - z.transform((value) => (value === 0 ? undefined : value)), - ), - ), - - // "ambisonic" is AmbiX; the order comes from numberOfChannels. - layout: z.optional(z.union([z.literal("ambisonic"), z.string()])), -}); - /** * Schema for a single audio rendition's decoder config. * Mirrors WebCodecs AudioDecoderConfig (https://w3c.github.io/webcodecs/#audio-decoder-config). */ -export const AudioConfigSchema = AudioConfigFields.check( - z.refine( - (c) => c.layout !== "ambisonic" || (c.numberOfChannels >= 4 && Number.isInteger(Math.sqrt(c.numberOfChannels))), - { message: "ambisonic needs a perfect-square channel count >= 4" }, - ), -); +export const AudioConfigSchema = z + .object({ + // Optional reference to another broadcast that publishes this track, expressed + // relative to the broadcast that served this catalog (e.g. "./source"). + // If unset, the track lives in the same broadcast as the catalog. + broadcast: z.optional(RelativeBroadcastSchema), + // The track's name in its broadcast, when it differs from this entry's ID (its key in the + // catalog). Absent means the ID is the track name. Resolve it with `track`. + track: z.optional(z.string()), + + // Human-readable rendition name for track pickers. + label: z.optional(z.string()), + + // Registered WebCodecs codec string, or Hang's "pcm" extension for + // interleaved little-endian IEEE-754 binary32 samples. + codec: z.string(), + + // The container format, used to decode the timestamp and more. + container: ContainerSchema, + + // The description is used for some codecs. + // If provided, we can initialize the decoder based on the catalog alone. + // Otherwise, the initialization information is in-band. + description: z.optional(hexSchema), + + // The sample rate of the audio in Hz + sampleRate: u53Schema, + + // The number of channels in the audio + numberOfChannels: u53Schema, + + // What the decoded channels carry. Absent means the codec's own channel order. "ambisonic" is + // full-sphere AmbiX (ACN order, SN3D normalization) whose order follows from numberOfChannels, + // a square of at least 4. Typed as a plain string so an unrecognized layout survives a + // republish; a consumer treats one as absent. + layout: z.optional(z.string()), + + // The maximum bitrate of the audio in bits per second. + // TODO: Support up to Number.MAX_SAFE_INTEGER + bitrate: z.optional(u53Schema), + + // Whether this rendition may be selected. When false, no frames are coming and a consumer + // must not select it. Default: true, so publishers only write it when false. + enabled: z.optional(z.boolean()), + + // The maximum delay between a frame being ready and the publisher flushing it, in whole + // milliseconds rounded up. The player's jitter buffer should be larger than this value. + // If not provided, the player should assume each frame is flushed immediately. + // + // This is measured at the publisher (encoder latency, packet packing, reordering), + // never on the network a consumer sees. It only ever grows over the life of a stream. + // + // NOTE: The audio "frame" duration depends on the codec, sample rate, etc. + // ex: AAC often uses 1024 samples per frame, so at 44100Hz, this would be 1024/44100 = 24ms + jitter: z.optional( + z.pipe( + u53Schema, + z.transform((value) => (value === 0 ? undefined : value)), + ), + ), + + // After a non-continuous join, decode from the group start, present at start plus warmup, + // and join that many milliseconds earlier. + warmup: z.optional(u53Schema), + + // How far this rendition's frames reach the transport behind the broadcast's earliest + // rendition, in whole milliseconds rounded up. A player holds `delay + jitter` for it and never + // subtracts one rendition's `delay` from another's. Absent on the earliest rendition. It only + // ever grows over the life of a stream. + delay: z.optional( + z.pipe( + u53Schema, + z.transform((value) => (value === 0 ? undefined : value)), + ), + ), + }) + .check( + z.refine((c) => c.layout !== "ambisonic" || isAmbisonic(c.numberOfChannels), { + message: "ambisonic audio needs a square channel count of at least 4", + }), + ); + +// (order + 1)² channels for an order of at least 1. +function isAmbisonic(channels: number): boolean { + const order = Math.round(Math.sqrt(channels)); + return order >= 2 && order * order === channels; +} /** Schema for the catalog audio section: a map of track name to rendition config. */ export const AudioSchema = z.union([ diff --git a/js/hang/src/catalog/index.ts b/js/hang/src/catalog/index.ts index f79292cd9f1..9f943c39121 100644 --- a/js/hang/src/catalog/index.ts +++ b/js/hang/src/catalog/index.ts @@ -37,5 +37,5 @@ export * from "./root"; export * from "./stats"; export * from "./text"; export * from "./track"; -export type { Color, VideoConfig } from "./video"; -export { ColorSchema, VideoConfigSchema, VideoSchema } from "./video"; +export type { Color, Projection, VideoConfig } from "./video"; +export { ColorSchema, ProjectionSchema, VideoConfigSchema, VideoSchema } from "./video"; diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index dcced6eed22..6457969a265 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -167,55 +167,75 @@ test("video warmup round trips as optional integer milliseconds", () => { } }); -const section = (fields: object) => ({ renditions: {}, ...fields }); - -test("projection parses the known kinds", () => { - for (const [json, projection] of [ - ['{"kind":"rectilinear"}', { kind: "rectilinear" }], - ['{"kind":"equirectangular"}', { kind: "equirectangular" }], - ['{"kind":"equirectangular","fov":180.0}', { kind: "equirectangular", fov: 180 }], +test("color parses the H.273 code points and HDR10 metadata a Rust publisher writes", () => { + const color = { + primaries: 9, + transfer: 16, + matrix: 9, + range: "limited", + mastering: { + red: { x: 0.708, y: 0.292 }, + green: { x: 0.17, y: 0.797 }, + blue: { x: 0.131, y: 0.046 }, + white: { x: 0.3127, y: 0.329 }, + maxLuminance: 1000, + minLuminance: 0.005, + }, + contentLight: { maxCll: 1000, maxFall: 400 }, + }; + const config = VideoConfigSchema.parse({ codec: "hvc1.2.4.L93.B0", container: { kind: "legacy" }, color }); + expect(config.color).toEqual(color as typeof config.color); + expect(Catalog.ColorSchema.parse(color)).toEqual(config.color as Catalog.Color); + expect(VideoConfigSchema.parse({ codec: "vp8", container: { kind: "legacy" } }).color).toBeUndefined(); +}); + +test("color refuses out-of-range code points and light levels", () => { + const parse = (color: unknown) => + VideoConfigSchema.safeParse({ codec: "vp8", container: { kind: "legacy" }, color }).success; + expect(parse({ primaries: 256 })).toBe(false); + expect(parse({ contentLight: { maxCll: 65536, maxFall: 0 } })).toBe(false); + expect(parse({ matrix: 0 })).toBe(true); + expect(parse({ range: "full" })).toBe(true); + expect(parse({ range: true })).toBe(false); +}); + +const vr = (fields: object) => VideoConfigSchema.parse({ codec: "vp8", container: { kind: "legacy" }, ...fields }); + +test("projection parses the known kinds and is optional", () => { + expect(vr({}).projection).toBeUndefined(); + for (const projection of [ + { kind: "rectilinear" }, + { kind: "equirectangular" }, + { kind: "equirectangular", fov: 180 }, ] as const) { - expect(VideoSchema.parse(section({ projection: JSON.parse(json) })).projection).toEqual(projection); + expect(vr({ projection }).projection).toEqual(projection); } + expect(Catalog.ProjectionSchema.parse({ kind: "rectilinear" })).toEqual({ kind: "rectilinear" }); }); -test("projection is optional", () => { - expect(VideoSchema.parse(section({})).projection).toBeUndefined(); -}); - -test("unknown projection is preserved instead of throwing", () => { - const raw = JSON.parse('{"face":"front","kind":"cubemap"}'); - expect(VideoSchema.parse(section({ projection: raw })).projection).toEqual({ kind: "unknown", raw }); +test("projection keeps an unknown kind verbatim and ignores unknown fields", () => { + const raw = { face: "front", kind: "cubemap" }; + expect(vr({ projection: raw }).projection).toEqual({ kind: "unknown", raw }); + // A newer publisher may add a parameter to a known kind; this reader must not refuse it. + expect(vr({ projection: { kind: "rectilinear", fov: 90 } }).projection).toEqual({ kind: "rectilinear" }); }); -test("malformed known projection throws", () => { - for (const json of ['{"kind":"equirectangular","fov":"wide"}', '{"kind":"rectilinear","fov":90}']) { - expect(() => VideoSchema.parse(section({ projection: JSON.parse(json) }))).toThrow(); - } -}); - -test("projection fov must be in (0, 360]", () => { - for (const fov of [0, -90, 360.5, Number.NaN, Number.POSITIVE_INFINITY]) { - expect(() => VideoSchema.parse(section({ projection: { kind: "equirectangular", fov } }))).toThrow(); +test("projection refuses a malformed known kind", () => { + for (const fov of ["wide", 0, -90, 360.5, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => vr({ projection: { kind: "equirectangular", fov } })).toThrow(); } for (const fov of [0.5, 180, 360]) { - expect(VideoSchema.parse(section({ projection: { kind: "equirectangular", fov } })).projection).toEqual({ + expect(vr({ projection: { kind: "equirectangular", fov } }).projection).toEqual({ kind: "equirectangular", fov, }); } }); -test("stereo parses known layouts and keeps unknown ones verbatim", () => { - for (const stereo of ["mono", "side-by-side", "top-bottom", "interleaved"]) { - expect(VideoSchema.parse(section({ stereo })).stereo).toBe(stereo); +test("stereo is optional and keeps unknown packings verbatim", () => { + expect(vr({}).stereo).toBeUndefined(); + for (const stereo of ["mono", "left-right", "top-bottom", "multiview"]) { + expect(vr({ stereo }).stereo).toBe(stereo); } - expect(VideoSchema.parse(section({})).stereo).toBeUndefined(); - expect(() => VideoSchema.parse(section({ stereo: 2 }))).toThrow(); -}); - -test("legacy array catalogs have no projection or stereo", () => { - const parsed = VideoSchema.parse([{ track: { name: "v" }, config: rendition() }]); - expect(parsed.projection).toBeUndefined(); - expect(parsed.stereo).toBeUndefined(); + expect(() => vr({ stereo: 2 })).toThrow(); }); diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index 4750ef0474d..f16f035cc99 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -47,6 +47,42 @@ export const ColorSchema = z.object({ /** The colour a rendition's pixels carry: H.273 code points plus static HDR metadata. */ export type Color = z.infer; +/** The projection kinds this build knows. */ +const KNOWN_PROJECTIONS = ["rectilinear", "equirectangular"]; + +/** + * Schema for how a rendition's picture maps onto the viewer's field of view. Absent means rectilinear. + * + * An unrecognized kind parses as `{ kind: "unknown", raw }` instead of throwing, and a consumer treats + * it as absent. A recognized kind only parses through its own schema, so a malformed one throws. + */ +export const ProjectionSchema = z.union([ + z.discriminatedUnion("kind", [ + // A flat picture. + z.object({ kind: z.literal("rectilinear") }), + // Longitude linear across the width and latitude down the height, the full 180 degrees, + // surrounding a viewer at its centre. + z.object({ + kind: z.literal("equirectangular"), + // The degrees of longitude the picture spans, centred straight ahead. Absent means 360; VR180 is 180. + fov: z.optional(z.number().check(z.positive(), z.lte(360))), + }), + ]), + z.pipe( + z.looseObject({ + kind: z.string().check( + z.refine((kind) => !KNOWN_PROJECTIONS.includes(kind), { + message: "recognized projection kind must match its own schema", + }), + ), + }), + z.transform((raw) => ({ kind: "unknown" as const, raw })), + ), +]); + +/** How a rendition's picture maps onto the viewer's field of view. */ +export type Projection = z.infer; + /** Schema for a single video rendition's decoder config. Mirrors WebCodecs VideoDecoderConfig. */ export const VideoConfigSchema = z.object({ // Optional reference to another broadcast that publishes this track, expressed @@ -84,6 +120,14 @@ export const VideoConfigSchema = z.object({ // The colour the decoded pixels carry, so a renderer need not guess or reparse the bitstream. color: z.optional(ColorSchema), + // How the picture maps onto the viewer's field of view. Absent means rectilinear. + projection: z.optional(ProjectionSchema), + + // How each eye's view is packed into the frame: "mono", "left-right" (left eye in the left + // half), or "top-bottom" (left eye in the top half). Absent means mono. Typed as a plain string + // so an unrecognized packing survives a republish; a consumer treats one as absent. + stereo: z.optional(z.string()), + // The maximum frame rate of the video in frames per second. framerate: z.optional(z.number()), @@ -138,41 +182,8 @@ export const VideoConfigSchema = z.object({ ), }); -// projection kinds this build knows how to render. -const KNOWN_PROJECTIONS = ["rectilinear", "equirectangular"]; - -// projection this build does not recognized, not to be rendered -export const UnknownProjectionSchema = z.pipe( - z.looseObject({ - kind: z.string().check( - z.refine((kind) => !KNOWN_PROJECTIONS.includes(kind), { - message: "recognized projection kind must match its own schema", - }), - ), - }), - z.transform((raw) => ({ kind: "unknown" as const, raw })), -); - -// How the coded picture maps onto the viewer's FOV. -export const ProjectionSchema = z.union([ - z.discriminatedUnion("kind", [ - // A flat picture - z.strictObject({ kind: z.literal("rectilinear") }), - // Longitude across the width and latitude down the height, wrapped onto a sphere. - z.strictObject({ - kind: z.literal("equirectangular"), - // Degrees of longitude the picture spans from 0..360. - fov: z.optional(z.number().check(z.positive(), z.lte(360))), - }), - ]), - UnknownProjectionSchema, -]); - -// How the viewer's pictures share each frame. -export const StereoSchema = z.union([z.enum(["mono", "side-by-side", "top-bottom"]), z.string()]); - /** - * Schema for the catalog video section: renditions plus display size, rotation, flip, projection, and stereo. + * Schema for the catalog video section: renditions plus display size, rotation, and flip. * Renditions mirror WebCodecs VideoDecoderConfig (https://w3c.github.io/webcodecs/#video-decoder-config). */ export const VideoSchema = z.union([ @@ -197,14 +208,6 @@ export const VideoSchema = z.union([ // If true, the decoder will flip the video horizontally // Default: false flip: z.optional(z.boolean()), - - // How the picture maps onto the viewer's field of view. - // Default: rectilinear - projection: z.optional(ProjectionSchema), - - // How the two eyes' pictures share each frame. - // Default: mono - stereo: z.optional(StereoSchema), }), // Backwards compatibility: transform old array of {track, config} to new object format z.pipe( @@ -224,8 +227,6 @@ export const VideoSchema = z.union([ : undefined, rotation: undefined, flip: undefined, - projection: undefined, - stereo: undefined, }; }), ), @@ -235,10 +236,6 @@ export const VideoSchema = z.union([ export type Video = z.infer; /** Decoder config for a single video rendition. */ export type VideoConfig = z.infer; -/** How the coded picture maps onto the viewer's FOV. */ -export type Projection = z.infer; -/** How the two eyes' pictures share each frame. */ -export type Stereo = z.infer; /** Rank video best first: enabled, then highest preference, largest picture, highest bitrate, and name. */ export function ranked(renditions: Record): [string, VideoConfig][] { diff --git a/rs/hang/src/catalog/audio/layout.rs b/rs/hang/src/catalog/audio/layout.rs new file mode 100644 index 00000000000..82a7201e60f --- /dev/null +++ b/rs/hang/src/catalog/audio/layout.rs @@ -0,0 +1,84 @@ +use std::str::FromStr; + +use derive_more::Display; + +/// What a rendition's decoded channels carry, beyond the codec's own channel order. +/// +/// Absent means the codec's channel order. An unrecognized value decodes to +/// [`AudioLayout::Unknown`] instead of failing, and a consumer treats it as absent. +#[derive(Debug, Clone, PartialEq, Eq, Display)] +#[non_exhaustive] +pub enum AudioLayout { + /// Full-sphere ambisonics in AmbiX: ACN channel order with SN3D normalization. + /// + /// The order follows from the channel count, `(order + 1)²`, so the count is a square of at least 4. + #[display("ambisonic")] + Ambisonic, + + /// A layout this build does not recognize, preserved verbatim. + #[display("{_0}")] + Unknown(String), +} + +impl AudioLayout { + /// Refuse a channel count this layout cannot carry. + pub(crate) fn check(&self, channel_count: u32) -> crate::Result<()> { + if *self == Self::Ambisonic { + let order = channel_count.isqrt(); + if order < 2 || order * order != channel_count { + return Err(crate::Error::InvalidAmbisonicChannels(channel_count)); + } + } + Ok(()) + } +} + +impl FromStr for AudioLayout { + type Err = crate::Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "ambisonic" => Self::Ambisonic, + _ => Self::Unknown(s.to_string()), + }) + } +} + +#[cfg(test)] +mod test { + use crate::catalog::{AudioCodec, AudioConfig}; + + use super::*; + + #[test] + fn round_trips() { + for (layout, json) in [ + (AudioLayout::Ambisonic, "ambisonic"), + (AudioLayout::Unknown("surround".to_string()), "surround"), + ] { + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 4); + config.layout = Some(layout.clone()); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["layout"], json); + assert_eq!( + serde_json::from_value::(encoded).unwrap().layout, + Some(layout) + ); + } + } + + #[test] + fn ambisonic_needs_a_square_channel_count() { + for channels in [4, 9, 16, 225] { + AudioLayout::Ambisonic.check(channels).unwrap(); + } + for channels in [0, 1, 2, 6, 8] { + assert!(matches!( + AudioLayout::Ambisonic.check(channels), + Err(crate::Error::InvalidAmbisonicChannels(c)) if c == channels + )); + } + // An unknown layout puts no constraint on the channel count. + AudioLayout::Unknown("surround".to_string()).check(6).unwrap(); + } +} diff --git a/rs/hang/src/catalog/audio/mod.rs b/rs/hang/src/catalog/audio/mod.rs index c11c985154e..d36c5491c91 100644 --- a/rs/hang/src/catalog/audio/mod.rs +++ b/rs/hang/src/catalog/audio/mod.rs @@ -1,8 +1,10 @@ mod aac; mod codec; +mod layout; pub use aac::*; pub use codec::*; +pub use layout::*; use std::collections::{BTreeMap, btree_map}; @@ -127,6 +129,13 @@ pub struct AudioConfig { #[serde(rename = "numberOfChannels")] pub channel_count: u32, + /// What the decoded channels carry, or the codec's own channel order when absent. + /// + /// Live state, like video `color`. + #[serde(default)] + #[serde_as(as = "Option")] + pub layout: Option, + /// The maximum bitrate of the audio track in bits per second. #[serde(default)] pub bitrate: Option, @@ -199,6 +208,7 @@ impl AudioConfig { codec: codec.into(), sample_rate, channel_count, + layout: None, bitrate: None, enabled: true, description: None, diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index a47ff29b360..4fc866506aa 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -267,8 +267,8 @@ impl Catalog<()> { } impl Catalog { - /// Refuse a catalog with more renditions than a consumer can hold, or with an ID naming both - /// a video and an audio rendition. + /// Refuse a catalog with more renditions than a consumer can hold, with an ID naming both + /// a video and an audio rendition, or with a projection or audio layout out of range. pub fn check_renditions(&self) -> Result<()> { let count = self.video.renditions.len() + self.audio.renditions.len() + self.text.renditions.len(); if count > MAX_RENDITIONS { @@ -277,8 +277,23 @@ impl Catalog { max: MAX_RENDITIONS, }); } - if let Some(projection) = &self.video.projection { - projection.check()?; + if let Some(id) = self + .video + .renditions + .keys() + .find(|id| self.audio.renditions.contains_key(*id)) + { + return Err(crate::Error::DuplicateRendition(id.clone())); + } + for config in self.video.renditions.values() { + if let Some(projection) = &config.projection { + projection.check()?; + } + } + for config in self.audio.renditions.values() { + if let Some(layout) = &config.layout { + layout.check(config.channel_count)?; + } } Ok(()) } @@ -490,6 +505,32 @@ mod test { )); } + #[test] + fn spatial_fields_out_of_range_are_refused() { + use crate::catalog::{AudioLayout, Projection, VideoCodec, VideoConfig}; + + let mut catalog = Catalog::<()>::from_str( + r#"{"audio": {"renditions": {"amb": {"codec": "opus", "sampleRate": 48000, "numberOfChannels": 6, "layout": "ambisonic", "container": {"kind": "legacy"}}}}}"#, + ) + .unwrap(); + assert!(matches!( + catalog.check_renditions(), + Err(crate::Error::InvalidAmbisonicChannels(6)) + )); + catalog.audio.renditions.get_mut("amb").unwrap().channel_count = 4; + assert_eq!(catalog.audio.renditions["amb"].layout, Some(AudioLayout::Ambisonic)); + catalog.check_renditions().unwrap(); + + // A publisher building the catalog by hand is caught before it writes the bad value. + let mut config = VideoConfig::new(VideoCodec::VP8); + config.projection = Some(Projection::Equirectangular { fov: Some(400.0) }); + catalog.video.insert("vr", config).unwrap(); + assert!(matches!( + catalog.check_renditions(), + Err(crate::Error::InvalidFieldOfView(_)) + )); + } + #[test] fn simple() { let mut encoded = r#"{ @@ -605,6 +646,8 @@ mod test { display_aspect_width: None, display_aspect_height: None, color: None, + projection: None, + stereo: None, bitrate: None, enabled: true, preference: 0, @@ -627,6 +670,7 @@ mod test { codec: Opus, sample_rate: 48_000, channel_count: 2, + layout: None, bitrate: None, enabled: true, description: None, @@ -648,16 +692,6 @@ mod test { assert_eq!(encoded, output, "encode mismatch"); } - #[test] - fn check_rejects_bad_fov() { - let catalog = - Catalog::<()>::from_str(r#"{"video":{"renditions":{},"projection":{"kind":"equirectangular","fov":400}}}"#) - .unwrap(); - assert!(matches!( - catalog.check_renditions(), - Err(crate::Error::InvalidFieldOfView(_)) - )); - } #[test] fn rendition_with_broadcast_override() { // Decode a catalog where one rendition references a track in a sibling broadcast, diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index 7a5ae175f09..1a4feba1fe9 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -4,6 +4,7 @@ mod color; mod h264; mod h265; mod projection; +mod stereo; mod vp9; pub use av1::*; @@ -12,6 +13,7 @@ pub use color::*; pub use h264::*; pub use h265::*; pub use projection::*; +pub use stereo::*; pub use vp9::*; use std::collections::{BTreeMap, btree_map}; @@ -58,10 +60,6 @@ pub struct Video { /// Default: false #[serde(default)] pub flip: Option, - - /// The type of projection for video - #[serde(default)] - pub projection: Option, } /// Catalog properties shared by every video rendition. @@ -76,16 +74,11 @@ pub struct VideoProperties { /// Flip horizontally after rotation, or clear the explicit value when absent. pub flip: Option, - - pub projection: Option, } impl VideoProperties { fn normalized(mut self) -> crate::Result { self.rotation = self.rotation.map(normalize_video_rotation).transpose()?; - if let Some(projection) = &self.projection { - projection.check()?; - } Ok(self) } } @@ -153,7 +146,6 @@ impl Video { self.display = properties.display; self.rotation = properties.rotation; self.flip = properties.flip; - self.projection = properties.projection; Ok(()) } } @@ -253,6 +245,19 @@ pub struct VideoConfig { #[serde(default)] pub color: Option, + /// How the picture maps onto the viewer's field of view. Absent means rectilinear. + /// + /// Live state, like [`color`](Self::color). + #[serde(default)] + pub projection: Option, + + /// How each eye's view is packed into the frame. Absent means mono. + /// + /// Live state, like [`color`](Self::color). + #[serde(default)] + #[serde_as(as = "Option")] + pub stereo: Option, + /// The maximum bitrate of the video track, if known. #[serde(default)] pub bitrate: Option, @@ -342,6 +347,8 @@ impl VideoConfig { display_aspect_width: None, display_aspect_height: None, color: None, + projection: None, + stereo: None, bitrate: None, enabled: true, preference: 0, @@ -590,7 +597,6 @@ mod test { display: None, rotation: Some(f64::NAN), flip: None, - projection: None, }), Err(crate::Error::InvalidVideoRotation) )); @@ -627,7 +633,6 @@ mod test { }), rotation: Some(315.0), flip: None, - projection: None, }) .unwrap(); diff --git a/rs/hang/src/catalog/video/projection.rs b/rs/hang/src/catalog/video/projection.rs index c0f0a61d33b..e15eb138ef0 100644 --- a/rs/hang/src/catalog/video/projection.rs +++ b/rs/hang/src/catalog/video/projection.rs @@ -1,39 +1,51 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; -/// How the coded picture maps onto the viewer's field of view. +/// How a rendition's decoded picture maps onto the viewer's field of view. +/// +/// Absent means [`Rectilinear`](Self::Rectilinear). An unrecognized `kind` decodes to +/// [`Projection::Unknown`] instead of failing, and a consumer treats it as absent. #[derive(Debug, Clone, PartialEq, Default)] +#[non_exhaustive] pub enum Projection { - /// A flat picture + /// A flat picture. #[default] Rectilinear, - /// Viewer is in the middle of wrapped sphere + /// Longitude linear across the width and latitude down the height, the full 180 degrees, + /// surrounding a viewer at its centre. Equirectangular { - /// Degrees of longitude the picture spans, in range 0..360 + /// The degrees of longitude the picture spans, centred straight ahead, in (0, 360]. + /// Absent means 360; VR180 is 180. fov: Option, }, + /// A projection this build does not recognize, preserved verbatim. Unknown(UnknownProjection), } +/// The raw JSON of a projection whose `kind` is not recognized. +/// +/// Kept intact so a relay or transcoder that reparses and republishes a catalog round-trips it. #[derive(Debug, Clone, PartialEq)] pub struct UnknownProjection(serde_json::Map); impl UnknownProjection { + /// The `kind` as it appeared on the wire, or `None` if it was absent or not a string. pub fn kind(&self) -> Option<&str> { self.0.get("kind").and_then(serde_json::Value::as_str) } + /// The full JSON object, including `kind`. pub fn fields(&self) -> &serde_json::Map { &self.0 } } impl Projection { - /// Check that projection is in range 0..360. + /// Refuse a field of view outside (0, 360] degrees. pub(crate) fn check(&self) -> crate::Result<()> { if let Self::Equirectangular { fov: Some(fov) } = self - && !(fov.is_finite() && *fov > 0.0 && *fov <= 360.0) + && !(*fov > 0.0 && *fov <= 360.0) { return Err(crate::Error::InvalidFieldOfView(*fov)); } @@ -41,11 +53,12 @@ impl Projection { } } +/// The projections this build knows, split out so the tagged representation stays derived. #[serde_with::skip_serializing_none] #[derive(Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "camelCase", deny_unknown_fields)] +#[serde(tag = "kind", rename_all = "camelCase")] enum Known { - Rectilinear {}, + Rectilinear, Equirectangular { #[serde(default)] fov: Option, @@ -55,7 +68,7 @@ enum Known { impl Serialize for Projection { fn serialize(&self, serializer: S) -> Result { let known = match self { - Self::Rectilinear => Known::Rectilinear {}, + Self::Rectilinear => Known::Rectilinear, Self::Equirectangular { fov } => Known::Equirectangular { fov: *fov }, Self::Unknown(unknown) => return unknown.0.serialize(serializer), }; @@ -67,16 +80,19 @@ impl<'de> Deserialize<'de> for Projection { fn deserialize>(deserializer: D) -> Result { let object = serde_json::Map::deserialize(deserializer)?; - match object.get("kind").and_then(serde_json::Value::as_str) { + // Only route kinds we know to the derived enum, so a malformed known projection is still + // a hard error instead of silently becoming Unknown. + let projection = match object.get("kind").and_then(serde_json::Value::as_str) { Some("rectilinear" | "equirectangular") => { - let known = Known::deserialize(serde_json::Value::Object(object)).map_err(de::Error::custom)?; - Ok(match known { - Known::Rectilinear {} => Self::Rectilinear, + match Known::deserialize(serde_json::Value::Object(object)).map_err(de::Error::custom)? { + Known::Rectilinear => Self::Rectilinear, Known::Equirectangular { fov } => Self::Equirectangular { fov }, - }) + } } - _ => Ok(Self::Unknown(UnknownProjection(object))), - } + _ => Self::Unknown(UnknownProjection(object)), + }; + projection.check().map_err(de::Error::custom)?; + Ok(projection) } } @@ -115,9 +131,19 @@ mod tests { } #[test] - fn malformed_known_errors() { + fn unknown_fields_are_ignored() { + // A newer publisher may add a parameter to a known kind; this reader must not refuse it. + let parsed: Projection = serde_json::from_str(r#"{"kind":"rectilinear","fov":90}"#).unwrap(); + assert_eq!(parsed, Projection::Rectilinear); + } + + #[test] + fn malformed_known_is_refused() { serde_json::from_str::(r#"{"kind":"equirectangular","fov":"wide"}"#).unwrap_err(); - serde_json::from_str::(r#"{"kind":"rectilinear","fov":90}"#).unwrap_err(); + for fov in ["0", "-90", "360.5"] { + let json = format!(r#"{{"kind":"equirectangular","fov":{fov}}}"#); + serde_json::from_str::(&json).unwrap_err(); + } } #[test] diff --git a/rs/hang/src/catalog/video/stereo.rs b/rs/hang/src/catalog/video/stereo.rs new file mode 100644 index 00000000000..8009d901167 --- /dev/null +++ b/rs/hang/src/catalog/video/stereo.rs @@ -0,0 +1,72 @@ +use std::str::FromStr; + +use derive_more::Display; + +/// How a rendition packs each eye's view into its frames. +/// +/// Absent means [`Mono`](Self::Mono). An unrecognized value decodes to +/// [`Stereo::Unknown`] instead of failing, and a consumer treats it as absent. +#[derive(Debug, Clone, PartialEq, Eq, Display)] +#[non_exhaustive] +pub enum Stereo { + /// One view, shown to both eyes. + #[display("mono")] + Mono, + + /// The left eye's view in the left half of each frame, the right eye's in the right half. + #[display("left-right")] + LeftRight, + + /// The left eye's view in the top half of each frame, the right eye's in the bottom half. + #[display("top-bottom")] + TopBottom, + + /// A packing this build does not recognize, preserved verbatim. + #[display("{_0}")] + Unknown(String), +} + +impl FromStr for Stereo { + type Err = crate::Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "mono" => Self::Mono, + "left-right" => Self::LeftRight, + "top-bottom" => Self::TopBottom, + _ => Self::Unknown(s.to_string()), + }) + } +} + +#[cfg(test)] +mod test { + use crate::catalog::{VideoCodec, VideoConfig}; + + use super::*; + + #[test] + fn round_trips() { + for (stereo, json) in [ + (Stereo::Mono, "mono"), + (Stereo::LeftRight, "left-right"), + (Stereo::TopBottom, "top-bottom"), + (Stereo::Unknown("multiview".to_string()), "multiview"), + ] { + let mut config = VideoConfig::new(VideoCodec::VP8); + config.stereo = Some(stereo.clone()); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["stereo"], json); + assert_eq!( + serde_json::from_value::(encoded).unwrap().stereo, + Some(stereo) + ); + } + } + + #[test] + fn non_string_is_refused() { + let wire = serde_json::json!({ "codec": "vp8", "stereo": 2 }); + serde_json::from_value::(wire).unwrap_err(); + } +} diff --git a/rs/hang/src/error.rs b/rs/hang/src/error.rs index cbd72169a3c..eaeda3bbf8f 100644 --- a/rs/hang/src/error.rs +++ b/rs/hang/src/error.rs @@ -86,9 +86,13 @@ pub enum Error { #[error("video rotation must be finite")] InvalidVideoRotation, - /// The projected video's field of view is outside (0, 360] degrees. + /// An equirectangular video's field of view is outside (0, 360] degrees. #[error("video field of view {0} must be in (0, 360] degrees")] InvalidFieldOfView(f64), + + /// An ambisonic audio rendition's channel count is not a square of at least 4. + #[error("ambisonic audio needs a square channel count of at least 4, not {0}")] + InvalidAmbisonicChannels(u32), } /// A Result type alias for hang operations. From 3999384e61a592b23d91f3a059f985f4c072007e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 13:48:21 -0700 Subject: [PATCH 4/5] refactor(hang): keep the JS audio schema hunk small Co-Authored-By: Claude Opus 5.5 --- js/hang/src/catalog/audio.ts | 162 +++++++++++++++++------------------ 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index 9407708f22c..4c7451700bb 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -10,91 +10,91 @@ const TrackSchema = z.object({ name: z.string(), }); +const AudioConfigFields = z.object({ + // Optional reference to another broadcast that publishes this track, expressed + // relative to the broadcast that served this catalog (e.g. "./source"). + // If unset, the track lives in the same broadcast as the catalog. + broadcast: z.optional(RelativeBroadcastSchema), + // The track's name in its broadcast, when it differs from this entry's ID (its key in the + // catalog). Absent means the ID is the track name. Resolve it with `track`. + track: z.optional(z.string()), + + // Human-readable rendition name for track pickers. + label: z.optional(z.string()), + + // Registered WebCodecs codec string, or Hang's "pcm" extension for + // interleaved little-endian IEEE-754 binary32 samples. + codec: z.string(), + + // The container format, used to decode the timestamp and more. + container: ContainerSchema, + + // The description is used for some codecs. + // If provided, we can initialize the decoder based on the catalog alone. + // Otherwise, the initialization information is in-band. + description: z.optional(hexSchema), + + // The sample rate of the audio in Hz + sampleRate: u53Schema, + + // The number of channels in the audio + numberOfChannels: u53Schema, + + // What the decoded channels carry. Absent means the codec's own channel order. "ambisonic" is + // full-sphere AmbiX (ACN order, SN3D normalization) whose order follows from numberOfChannels, + // a square of at least 4. Typed as a plain string so an unrecognized layout survives a + // republish; a consumer treats one as absent. + layout: z.optional(z.string()), + + // The maximum bitrate of the audio in bits per second. + // TODO: Support up to Number.MAX_SAFE_INTEGER + bitrate: z.optional(u53Schema), + + // Whether this rendition may be selected. When false, no frames are coming and a consumer + // must not select it. Default: true, so publishers only write it when false. + enabled: z.optional(z.boolean()), + + // The maximum delay between a frame being ready and the publisher flushing it, in whole + // milliseconds rounded up. The player's jitter buffer should be larger than this value. + // If not provided, the player should assume each frame is flushed immediately. + // + // This is measured at the publisher (encoder latency, packet packing, reordering), + // never on the network a consumer sees. It only ever grows over the life of a stream. + // + // NOTE: The audio "frame" duration depends on the codec, sample rate, etc. + // ex: AAC often uses 1024 samples per frame, so at 44100Hz, this would be 1024/44100 = 24ms + jitter: z.optional( + z.pipe( + u53Schema, + z.transform((value) => (value === 0 ? undefined : value)), + ), + ), + + // After a non-continuous join, decode from the group start, present at start plus warmup, + // and join that many milliseconds earlier. + warmup: z.optional(u53Schema), + + // How far this rendition's frames reach the transport behind the broadcast's earliest + // rendition, in whole milliseconds rounded up. A player holds `delay + jitter` for it and never + // subtracts one rendition's `delay` from another's. Absent on the earliest rendition. It only + // ever grows over the life of a stream. + delay: z.optional( + z.pipe( + u53Schema, + z.transform((value) => (value === 0 ? undefined : value)), + ), + ), +}); + /** * Schema for a single audio rendition's decoder config. * Mirrors WebCodecs AudioDecoderConfig (https://w3c.github.io/webcodecs/#audio-decoder-config). */ -export const AudioConfigSchema = z - .object({ - // Optional reference to another broadcast that publishes this track, expressed - // relative to the broadcast that served this catalog (e.g. "./source"). - // If unset, the track lives in the same broadcast as the catalog. - broadcast: z.optional(RelativeBroadcastSchema), - // The track's name in its broadcast, when it differs from this entry's ID (its key in the - // catalog). Absent means the ID is the track name. Resolve it with `track`. - track: z.optional(z.string()), - - // Human-readable rendition name for track pickers. - label: z.optional(z.string()), - - // Registered WebCodecs codec string, or Hang's "pcm" extension for - // interleaved little-endian IEEE-754 binary32 samples. - codec: z.string(), - - // The container format, used to decode the timestamp and more. - container: ContainerSchema, - - // The description is used for some codecs. - // If provided, we can initialize the decoder based on the catalog alone. - // Otherwise, the initialization information is in-band. - description: z.optional(hexSchema), - - // The sample rate of the audio in Hz - sampleRate: u53Schema, - - // The number of channels in the audio - numberOfChannels: u53Schema, - - // What the decoded channels carry. Absent means the codec's own channel order. "ambisonic" is - // full-sphere AmbiX (ACN order, SN3D normalization) whose order follows from numberOfChannels, - // a square of at least 4. Typed as a plain string so an unrecognized layout survives a - // republish; a consumer treats one as absent. - layout: z.optional(z.string()), - - // The maximum bitrate of the audio in bits per second. - // TODO: Support up to Number.MAX_SAFE_INTEGER - bitrate: z.optional(u53Schema), - - // Whether this rendition may be selected. When false, no frames are coming and a consumer - // must not select it. Default: true, so publishers only write it when false. - enabled: z.optional(z.boolean()), - - // The maximum delay between a frame being ready and the publisher flushing it, in whole - // milliseconds rounded up. The player's jitter buffer should be larger than this value. - // If not provided, the player should assume each frame is flushed immediately. - // - // This is measured at the publisher (encoder latency, packet packing, reordering), - // never on the network a consumer sees. It only ever grows over the life of a stream. - // - // NOTE: The audio "frame" duration depends on the codec, sample rate, etc. - // ex: AAC often uses 1024 samples per frame, so at 44100Hz, this would be 1024/44100 = 24ms - jitter: z.optional( - z.pipe( - u53Schema, - z.transform((value) => (value === 0 ? undefined : value)), - ), - ), - - // After a non-continuous join, decode from the group start, present at start plus warmup, - // and join that many milliseconds earlier. - warmup: z.optional(u53Schema), - - // How far this rendition's frames reach the transport behind the broadcast's earliest - // rendition, in whole milliseconds rounded up. A player holds `delay + jitter` for it and never - // subtracts one rendition's `delay` from another's. Absent on the earliest rendition. It only - // ever grows over the life of a stream. - delay: z.optional( - z.pipe( - u53Schema, - z.transform((value) => (value === 0 ? undefined : value)), - ), - ), - }) - .check( - z.refine((c) => c.layout !== "ambisonic" || isAmbisonic(c.numberOfChannels), { - message: "ambisonic audio needs a square channel count of at least 4", - }), - ); +export const AudioConfigSchema = AudioConfigFields.check( + z.refine((c) => c.layout !== "ambisonic" || isAmbisonic(c.numberOfChannels), { + message: "ambisonic audio needs a square channel count of at least 4", + }), +); // (order + 1)² channels for an order of at least 1. function isAmbisonic(channels: number): boolean { From eadbcbc74717a61701be060881049550d923dcb7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 13:49:31 -0700 Subject: [PATCH 5/5] fix(hang): refuse a projection without a string kind Co-Authored-By: Claude Opus 5.5 --- js/hang/src/catalog/video.test.ts | 2 ++ rs/hang/src/catalog/video/projection.rs | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 6457969a265..db73a89841d 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -221,6 +221,8 @@ test("projection keeps an unknown kind verbatim and ignores unknown fields", () }); test("projection refuses a malformed known kind", () => { + expect(() => vr({ projection: { fov: 180 } })).toThrow(); + expect(() => vr({ projection: { kind: 5 } })).toThrow(); for (const fov of ["wide", 0, -90, 360.5, Number.NaN, Number.POSITIVE_INFINITY]) { expect(() => vr({ projection: { kind: "equirectangular", fov } })).toThrow(); } diff --git a/rs/hang/src/catalog/video/projection.rs b/rs/hang/src/catalog/video/projection.rs index e15eb138ef0..5a038995632 100644 --- a/rs/hang/src/catalog/video/projection.rs +++ b/rs/hang/src/catalog/video/projection.rs @@ -30,9 +30,13 @@ pub enum Projection { pub struct UnknownProjection(serde_json::Map); impl UnknownProjection { - /// The `kind` as it appeared on the wire, or `None` if it was absent or not a string. - pub fn kind(&self) -> Option<&str> { - self.0.get("kind").and_then(serde_json::Value::as_str) + /// The `kind` as it appeared on the wire. + pub fn kind(&self) -> &str { + // Deserialize refuses a projection without a string kind, so this is never empty. + self.0 + .get("kind") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() } /// The full JSON object, including `kind`. @@ -80,10 +84,14 @@ impl<'de> Deserialize<'de> for Projection { fn deserialize>(deserializer: D) -> Result { let object = serde_json::Map::deserialize(deserializer)?; + let Some(kind) = object.get("kind").and_then(serde_json::Value::as_str) else { + return Err(de::Error::custom("projection kind must be a string")); + }; + // Only route kinds we know to the derived enum, so a malformed known projection is still // a hard error instead of silently becoming Unknown. - let projection = match object.get("kind").and_then(serde_json::Value::as_str) { - Some("rectilinear" | "equirectangular") => { + let projection = match kind { + "rectilinear" | "equirectangular" => { match Known::deserialize(serde_json::Value::Object(object)).map_err(de::Error::custom)? { Known::Rectilinear => Self::Rectilinear, Known::Equirectangular { fov } => Self::Equirectangular { fov }, @@ -126,7 +134,7 @@ mod tests { let Projection::Unknown(unknown) = &parsed else { panic!("expected unknown: {parsed:?}"); }; - assert_eq!(unknown.kind(), Some("cubemap")); + assert_eq!(unknown.kind(), "cubemap"); assert_eq!(serde_json::to_string(&parsed).unwrap(), json); } @@ -139,6 +147,8 @@ mod tests { #[test] fn malformed_known_is_refused() { + serde_json::from_str::(r#"{"fov":180}"#).unwrap_err(); + serde_json::from_str::(r#"{"kind":5}"#).unwrap_err(); serde_json::from_str::(r#"{"kind":"equirectangular","fov":"wide"}"#).unwrap_err(); for fov in ["0", "-90", "360.5"] { let json = format!(r#"{{"kind":"equirectangular","fov":{fov}}}"#);