From bc754329d1027a57f669531ec40d0bee6ccadf39 Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Thu, 22 Oct 2026 10:19:48 -0600 Subject: [PATCH] feat(demo): add VR demo + associated spatial code --- .github/workflows/interop.yml | 9 ++ demo/vr/justfile | 4 + demo/vr/package.json | 21 +++ demo/vr/src/audio.ts | 110 ++++++++++++++++ demo/vr/src/index.html | 28 ++++ demo/vr/src/index.ts | 56 ++++++++ demo/vr/src/scene.ts | 164 ++++++++++++++++++++++++ demo/vr/src/vite-env.d.ts | 9 ++ demo/vr/tsconfig.json | 6 + demo/vr/vite.config.ts | 13 ++ doc/bin/cli.md | 12 ++ package.json | 1 + rs/hang/src/catalog/audio/layout.rs | 2 +- rs/hang/src/catalog/video/projection.rs | 2 +- rs/moq-cli/src/args.rs | 141 ++++++++++++++++++++ rs/moq-cli/src/main.rs | 3 +- rs/moq-mux/src/catalog/mod.rs | 2 + rs/moq-mux/src/catalog/producer.rs | 75 +++++++++++ rs/moq-mux/src/catalog/spatial.rs | 43 +++++++ rs/moq-mux/src/catalog/tracks.rs | 17 ++- test/README.md | 1 + test/interop/clients/js/vr.ts | 153 ++++++++++++++++++++++ test/justfile | 4 + test/vr/relay.toml | 15 +++ test/vr/run.sh | 33 +++++ 25 files changed, 920 insertions(+), 4 deletions(-) create mode 100644 demo/vr/justfile create mode 100644 demo/vr/package.json create mode 100644 demo/vr/src/audio.ts create mode 100644 demo/vr/src/index.html create mode 100644 demo/vr/src/index.ts create mode 100644 demo/vr/src/scene.ts create mode 100644 demo/vr/src/vite-env.d.ts create mode 100644 demo/vr/tsconfig.json create mode 100644 demo/vr/vite.config.ts create mode 100644 rs/moq-mux/src/catalog/spatial.rs create mode 100644 test/interop/clients/js/vr.ts create mode 100644 test/vr/relay.toml create mode 100755 test/vr/run.sh diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index e7c1765731a..cdb3a6ebf96 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -23,6 +23,8 @@ on: - "test/interop/**" - "test/max-age/**" - "test/portrait/**" + - "test/vr/**" + - "demo/vr/**" - "rs/moq-tokio/tests/integration/max_age.rs" # The interop job also depends on the recipe that invokes it and the # workspace membership that links the JS clients to source. @@ -123,6 +125,13 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + - name: VR projection + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test vr + shell: bash -leo pipefail {0} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + - name: Interop negative control if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just test interop-negative diff --git a/demo/vr/justfile b/demo/vr/justfile new file mode 100644 index 00000000000..408f18912bf --- /dev/null +++ b/demo/vr/justfile @@ -0,0 +1,4 @@ +set shell := ["bash", "-euo", "pipefail", "-c"] +serve url = "http://localhost:4443": + bun install + VITE_RELAY_URL="{{url}}" bun --bun vite diff --git a/demo/vr/package.json b/demo/vr/package.json new file mode 100644 index 00000000000..0e85d3464cc --- /dev/null +++ b/demo/vr/package.json @@ -0,0 +1,21 @@ +{ + "name": "@moq/demo-vr", + "private": true, + "type": "module", + "description": "Media over QUIC - VR demo", + "license": "(MIT OR Apache-2.0)", + "scripts": { + "dev": "vite", + "build": "vite build", + "check": "tsc --noEmit" + }, + "dependencies": { + "@moq/watch": "workspace:^", + "three": "^0.186.1" + }, + "devDependencies": { + "@types/three": "^0.186.0", + "typescript": "7.0.2", + "vite": "^8.3.4" + } +} diff --git a/demo/vr/src/audio.ts b/demo/vr/src/audio.ts new file mode 100644 index 00000000000..6a8ab69c749 --- /dev/null +++ b/demo/vr/src/audio.ts @@ -0,0 +1,110 @@ +import * as THREE from "three"; + +export type Placement = + | { kind: "ambisonic" } + | { kind: "point"; position: THREE.Vector3 } + | { kind: "head" }; + +// Virtual speakers at the corners of a cube in ambisonic axes (W, Y, Z, X) +const SPEAKERS = [-1, 1].flatMap((x) => + [-1, 1].flatMap((y) => [-1, 1].map((z) => [x, y, z].map((v) => v / Math.sqrt(3)))), +); + +const MAX_RE = 1 / Math.sqrt(3); + +// Renders audio graph around a viewer's head +export class Spatial { + #context: BaseAudioContext; + #nodes: AudioNode[] = []; + #tracksPosition: boolean; + + constructor(root: AudioNode, placement: Placement) { + this.#context = root.context; + const destination = this.#context.destination; + + switch (placement.kind) { + case "head": + root.connect(destination); + this.#tracksPosition = false; + break; + case "point": { + const panner = this.#panner(placement.position, 1); + root.connect(panner).connect(destination); + this.#tracksPosition = true; + break; + } + case "ambisonic": + this.#ambisonic(root, destination); + // The field is infinitely far, so only the head's rotation moves it. + this.#tracksPosition = false; + break; + } + this.#nodes.push(root); + } + + // Decodes out to fixed speakers around the scene + #ambisonic(root: AudioNode, destination: AudioNode) { + if (root.channelCount !== 4) + throw new Error(`first-order ambisonics needs 4 channels, got ${root.channelCount}`); + + const split = new ChannelSplitterNode(this.#context, { numberOfOutputs: 4 }); + root.connect(split); + this.#nodes.push(split); + + for (const [x, y, z] of SPEAKERS) { + // ACN order is W, Y, Z, X. + const weights = [1, 3 * MAX_RE * y, 3 * MAX_RE * z, 3 * MAX_RE * x].map((w) => w / SPEAKERS.length); + // Ambisonic axes to WebAudio + const panner = this.#panner(new THREE.Vector3(-y, z, -x), 0); + panner.connect(destination); + + weights.forEach((weight, channel) => { + const gain = new GainNode(this.#context, { gain: weight }); + split.connect(gain, channel).connect(panner); + this.#nodes.push(gain); + }); + } + } + + #panner(position: THREE.Vector3, rolloffFactor: number): PannerNode { + const panner = new PannerNode(this.#context, { + panningModel: "HRTF", + distanceModel: "inverse", + refDistance: 1, + rolloffFactor, + positionX: position.x, + positionY: position.y, + positionZ: position.z, + }); + this.#nodes.push(panner); + return panner; + } + + + // Move the audio with the transform of viewer's head + pose(head: THREE.Matrix4) { + const position = new THREE.Vector3(); + const rotation = new THREE.Quaternion(); + head.decompose(position, rotation, new THREE.Vector3()); + const forward = new THREE.Vector3(0, 0, -1).applyQuaternion(rotation); + const up = new THREE.Vector3(0, 1, 0).applyQuaternion(rotation); + + const listener = this.#context.listener; + const at = this.#context.currentTime; + const set = (param: AudioParam, value: number) => param.setTargetAtTime(value, at, 0.01); + if (!this.#tracksPosition) position.set(0, 0, 0); + set(listener.positionX, position.x); + set(listener.positionY, position.y); + set(listener.positionZ, position.z); + set(listener.forwardX, forward.x); + set(listener.forwardY, forward.y); + set(listener.forwardZ, forward.z); + set(listener.upX, up.x); + set(listener.upY, up.y); + set(listener.upZ, up.z); + } + + close() { + for (const node of this.#nodes) node.disconnect(); + } +} diff --git a/demo/vr/src/index.html b/demo/vr/src/index.html new file mode 100644 index 00000000000..cbaec435a6b --- /dev/null +++ b/demo/vr/src/index.html @@ -0,0 +1,28 @@ + + + + + + MoQ VR + + + + +
connecting…
+ + + diff --git a/demo/vr/src/index.ts b/demo/vr/src/index.ts new file mode 100644 index 00000000000..b070694c8d9 --- /dev/null +++ b/demo/vr/src/index.ts @@ -0,0 +1,56 @@ +import { Audio, Broadcast, Net, Signals, Sync, Video } from "@moq/watch"; +import { type Placement, Spatial } from "./audio"; +import { Scene } from "./scene"; + +const params = new URLSearchParams(location.search); +const url = new URL(params.get("url") ?? import.meta.env.VITE_RELAY_URL ?? "http://localhost:4443"); +const name = Net.Path.from(params.get("name") ?? "vr.hang"); + +const connection = new Net.Connection({ url }); +const broadcast = new Broadcast({ origin: connection.origin, name, enabled: true, announced: true }); +const audioSrc = new Audio.Source({ broadcast, supported: Audio.Decoder.supported }); +const videoSrc = new Video.Source({ broadcast, supported: Video.Decoder.supported, probe: connection.probe }); +const sync = new Sync({}); +const audio = new Audio.Decoder({ source: audioSrc, sync, enabled: true }); +const video = new Video.Decoder({ source: videoSrc, sync, enabled: true }); +const scene = new Scene(document.body); +const effect = new Signals.Effect(); + +// clone the frame once decoder gets it out +effect.run((e) => { + const frame = e.get(video.out.frame); + if (frame) scene.frame(frame.clone()); +}); + +const status = document.getElementById("status"); +effect.run((e) => { + if (!status) return; + const error = e.get(video.out.error) ?? e.get(audio.out.error); + status.textContent = error + ? `error: ${error.message}` + : `${e.get(connection.status)} · ${name} ${e.get(broadcast.out.status)}`; +}); + +effect.run((e) => { + const config = e.get(videoSrc.out.config); + if (config) scene.layout(config.projection, config.stereo); +}); +effect.run((e) => { + const root = e.get(audio.out.root); + const config = e.get(audioSrc.out.config); + if (!root || !config) return; + const projection = e.get(videoSrc.out.config)?.projection?.kind; + const placement: Placement = + config.layout === "ambisonic" + ? { kind: "ambisonic" } + : projection === "equirectangular" + ? { kind: "head" } + : { kind: "point", position: scene.speaker }; + const spatial = new Spatial(root, placement); + scene.listener = spatial; + + e.cleanup(() => { + spatial.close(); + scene.listener = undefined; + }); +}); diff --git a/demo/vr/src/scene.ts b/demo/vr/src/scene.ts new file mode 100644 index 00000000000..aba643e3aee --- /dev/null +++ b/demo/vr/src/scene.ts @@ -0,0 +1,164 @@ +import type { Hang } from "@moq/watch"; +import * as THREE from "three"; +import { VRButton } from "three/addons/webxr/VRButton.js"; +import type { Spatial } from "./audio"; + +type Projection = Hang.Catalog.Projection; + +const EYE = 1.6; +const SCREEN_HEIGHT = 1.8; +const SPHERE_RADIUS = 10; + +// Each eye's [u, v, width, height] in the frame. +const PACKING: Record = { + mono: [ + [0, 0, 1, 1], + [0, 0, 1, 1], + ], + "left-right": [ + [0, 0, 0.5, 1], + [0.5, 0, 0.5, 1], + ], + "top-bottom": [ + [0, 0.5, 1, 0.5], + [0, 0, 1, 0.5], + ], +}; + +// Paints decoded frames onto a screen or sphere according to projection +export class Scene { + readonly speaker = new THREE.Vector3(0, EYE, -2); + + // follow viewer's head per-frame + listener?: Spatial; + + #renderer: THREE.WebGLRenderer; + #scene = new THREE.Scene(); + #camera: THREE.PerspectiveCamera; + #texture = new THREE.VideoFrameTexture(); + #frame?: VideoFrame; + #eyes: THREE.Mesh[]; + + #projection?: Projection; + #stereo = "mono"; + #aspect = 16 / 9; + + constructor(parent: HTMLElement) { + this.#renderer = new THREE.WebGLRenderer({ antialias: true }); + this.#renderer.setPixelRatio(window.devicePixelRatio); + this.#renderer.setSize(window.innerWidth, window.innerHeight); + this.#renderer.xr.enabled = true; + this.#renderer.xr.setReferenceSpaceType("local-floor"); + parent.append(this.#renderer.domElement, VRButton.createButton(this.#renderer)); + + this.#camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 0.01, 100); + this.#camera.position.set(0, EYE, 0); + this.#camera.layers.enable(1); + + this.#scene.background = new THREE.Color(0x111111); + this.#texture.colorSpace = THREE.SRGBColorSpace; + + this.#eyes = [1, 2].map((layer) => { + const map = this.#texture.clone(); + const mesh = new THREE.Mesh( + new THREE.BufferGeometry(), + new THREE.MeshBasicMaterial({ map, toneMapped: false }), + ); + mesh.layers.set(layer); + mesh.visible = false; + this.#scene.add(mesh); + return mesh; + }); + this.#build(); + + window.addEventListener("resize", this.#resize); + this.#look(this.#renderer.domElement); + + this.#renderer.setAnimationLoop(() => { + this.#renderer.render(this.#scene, this.#camera); + const head = this.#renderer.xr.isPresenting ? this.#renderer.xr.getCamera() : this.#camera; + this.listener?.pose(head.matrixWorld); + }); + } + + frame(frame: VideoFrame) { + this.#texture.setFrame(frame); + for (const mesh of this.#eyes) { + if (mesh.material.map) mesh.material.map.needsUpdate = true; + mesh.visible = true; + } + this.#frame?.close(); + this.#frame = frame; + + const aspect = frame.displayWidth / frame.displayHeight; + if (aspect !== this.#aspect) { + this.#aspect = aspect; + this.#build(); + } + } + + // Layout for the picture according to projection and stereo + layout(projection?: Projection, stereo?: string) { + this.#projection = projection; + this.#stereo = stereo && stereo in PACKING ? stereo : "mono"; + this.#build(); + } + + #build() { + const packing = PACKING[this.#stereo]; + const projection = this.#projection?.kind === "equirectangular" ? this.#projection : undefined; + + for (const [eye, mesh] of this.#eyes.entries()) { + const [u, v, width, height] = packing[eye]; + mesh.material.map?.offset.set(u, v); + mesh.material.map?.repeat.set(width, height); + + mesh.geometry.dispose(); + if (projection) { + const fov = THREE.MathUtils.degToRad(projection.fov ?? 360); + // Centred on the seam opposite phi = 0, then mirrored and turned so the picture's centre + // faces -Z and its left half wraps to the viewer's left. + mesh.geometry = new THREE.SphereGeometry(SPHERE_RADIUS, 64, 32, Math.PI - fov / 2, fov).scale(-1, 1, 1); + mesh.position.set(0, EYE, 0); + mesh.rotation.y = -Math.PI / 2; + } else { + // The packed frame's aspect, unpacked to one eye's. + const aspect = (this.#aspect * width) / height; + mesh.geometry = new THREE.PlaneGeometry(SCREEN_HEIGHT * aspect, SCREEN_HEIGHT); + mesh.position.copy(this.speaker); + mesh.rotation.y = 0; + } + } + } + + #resize = () => { + this.#camera.aspect = window.innerWidth / window.innerHeight; + this.#camera.updateProjectionMatrix(); + this.#renderer.setSize(window.innerWidth, window.innerHeight); + }; + +// Looking around is dragging on desktop, on headset its tracked + #look(element: HTMLElement) { + const euler = new THREE.Euler(0, 0, 0, "YXZ"); + element.addEventListener("pointermove", (event) => { + if (event.buttons !== 1) return; + euler.setFromQuaternion(this.#camera.quaternion); + euler.y += event.movementX * 0.005; + euler.x = THREE.MathUtils.clamp(euler.x + event.movementY * 0.005, -Math.PI / 2, Math.PI / 2); + this.#camera.quaternion.setFromEuler(euler); + }); + } + + close() { + this.#renderer.setAnimationLoop(null); + window.removeEventListener("resize", this.#resize); + for (const mesh of this.#eyes) { + mesh.geometry.dispose(); + mesh.material.map?.dispose(); + mesh.material.dispose(); + } + this.#texture.dispose(); + this.#renderer.dispose(); + this.#frame?.close(); + } +} diff --git a/demo/vr/src/vite-env.d.ts b/demo/vr/src/vite-env.d.ts new file mode 100644 index 00000000000..4ef6324115a --- /dev/null +++ b/demo/vr/src/vite-env.d.ts @@ -0,0 +1,9 @@ +/// + +interface ImportMetaEnv { + readonly VITE_RELAY_URL?: string; +} + +interface ImportMeta { + readonly env: ImportMetaEnv; +} diff --git a/demo/vr/tsconfig.json b/demo/vr/tsconfig.json new file mode 100644 index 00000000000..47db9458530 --- /dev/null +++ b/demo/vr/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../js/tsconfig.json", + "compilerOptions": { "module": "esnext", "moduleResolution": "bundler", "outDir": "dist" }, + "include": ["src", "../../js/common/worklet.d.ts"] +} + diff --git a/demo/vr/vite.config.ts b/demo/vr/vite.config.ts new file mode 100644 index 00000000000..c40f79a9163 --- /dev/null +++ b/demo/vr/vite.config.ts @@ -0,0 +1,13 @@ +import { resolve } from "path"; +import { defineConfig } from "vite"; +import { crossOriginIsolation } from "../../js/common/vite-plugin-isolate"; +import { worklet } from "../../js/common/vite-plugin-worklet"; + +export default defineConfig({ + root: "src", + envDir: resolve(__dirname), + plugins: [worklet(), crossOriginIsolation()], + build: { target: "esnext" }, + server: { hmr: false }, +}); + diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 0d03ae49fa1..879940dd17f 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -122,6 +122,18 @@ program aboard, for what demultiplexing cannot carry, such as a scrambled service; `export ts --passthrough` writes it back out. FLV accepts the same legacy and enhanced-RTMP codecs as the [RTMP gateway](/bin/rtmp). +The stdin imports can mark the broadcast for VR, since none of their bitstreams +say so: `--projection equirectangular` for 360 video (`equirectangular:180` for +VR180), `--stereo left-right` or `top-bottom` for a picture that packs both eyes, +and `--audio-layout ambisonic` for AmbiX audio. An ambisonic track without a +square channel count of at least 4 is refused. + +```bash +ffmpeg -re -i vr180.mp4 -c copy -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer - | + moq --connect https://relay.example.com/anon --broadcast vr.hang \ + import --projection equirectangular:180 --stereo left-right fmp4 +``` + `import rtsp` pulls an IP camera's H.264 or H.265 video and AAC audio without re-encoding. RTP rides the RTSP connection, so run it on the camera's network and the camera needs no inbound port. Credentials go in the URL diff --git a/package.json b/package.json index c810a42f695..f39d2fe2bfe 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "demo/boy", "demo/web", "demo/carla/web", + "demo/vr", "js/moq-boy", "test/audio-quality/clients/js", "test/interop/clients/js", diff --git a/rs/hang/src/catalog/audio/layout.rs b/rs/hang/src/catalog/audio/layout.rs index 82a7201e60f..860b7f3a1ad 100644 --- a/rs/hang/src/catalog/audio/layout.rs +++ b/rs/hang/src/catalog/audio/layout.rs @@ -22,7 +22,7 @@ pub enum AudioLayout { impl AudioLayout { /// Refuse a channel count this layout cannot carry. - pub(crate) fn check(&self, channel_count: u32) -> crate::Result<()> { + pub fn check(&self, channel_count: u32) -> crate::Result<()> { if *self == Self::Ambisonic { let order = channel_count.isqrt(); if order < 2 || order * order != channel_count { diff --git a/rs/hang/src/catalog/video/projection.rs b/rs/hang/src/catalog/video/projection.rs index 5a038995632..2b5c634ec9a 100644 --- a/rs/hang/src/catalog/video/projection.rs +++ b/rs/hang/src/catalog/video/projection.rs @@ -47,7 +47,7 @@ impl UnknownProjection { impl Projection { /// Refuse a field of view outside (0, 360] degrees. - pub(crate) fn check(&self) -> crate::Result<()> { + pub fn check(&self) -> crate::Result<()> { if let Self::Equirectangular { fov: Some(fov) } = self && !(*fov > 0.0 && *fov <= 360.0) { diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 02fb827d2a9..cf6011fe904 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -322,6 +322,14 @@ impl Invocation { { anyhow::bail!("--stats names a track in the catalog the import builds, but {reason}"); } + if let Command::Import(import) = command + && import.spatial() != moq_mux::catalog::Spatial::default() + && import.source.stdin_format().is_none() + { + anyhow::bail!( + "--projection, --stereo, and --audio-layout stamp the catalog a stdin import builds, which this source does not" + ); + } let Command::Export(export) = command else { continue; }; @@ -763,6 +771,21 @@ pub struct Import { #[usage(long)] pub stats: bool, + /// How each video picture maps onto the viewer's view: `rectilinear`, or `equirectangular` + /// with an optional `:fov` in degrees (`equirectangular:180` for VR180). The stdin imports only. + #[usage(long)] + pub projection: Option, + + /// How each video picture packs both eyes: `mono`, `left-right`, or `top-bottom`. The stdin + /// imports only. + #[usage(long)] + pub stereo: Option, + + /// What each audio track's channels carry: `ambisonic` for AmbiX, which needs a square channel + /// count of at least 4. The stdin imports only. + #[usage(long)] + pub audio_layout: Option, + /// The single source feeding the Origin. #[usage(subcommand)] pub source: ImportSource, @@ -779,6 +802,71 @@ impl Import { } found } + + /// The spatial metadata the flags stamp onto each rendition. + pub fn spatial(&self) -> moq_mux::catalog::Spatial { + moq_mux::catalog::Spatial { + projection: self.projection.clone().map(|arg| arg.0), + stereo: self.stereo.clone().map(|arg| arg.0), + layout: self.audio_layout.clone().map(|arg| arg.0), + } + } +} + +/// An `import --projection` value. +#[derive(Clone, Debug, PartialEq)] +pub struct ProjectionArg(pub hang::catalog::Projection); + +impl std::str::FromStr for ProjectionArg { + type Err = String; + + fn from_str(arg: &str) -> Result { + use hang::catalog::Projection; + + let projection = match arg.split_once(':') { + None if arg == "rectilinear" => Projection::Rectilinear, + None if arg == "equirectangular" => Projection::Equirectangular { fov: None }, + Some(("equirectangular", fov)) => { + let fov = fov.parse().map_err(|_| format!("expected a field of view in degrees, got `{fov}`"))?; + Projection::Equirectangular { fov: Some(fov) } + } + _ => return Err(format!("expected `rectilinear` or `equirectangular[:fov]`, got `{arg}`")), + }; + projection.check().map_err(|err| err.to_string())?; + Ok(Self(projection)) + } +} + +/// An `import --stereo` value. +#[derive(Clone, Debug, PartialEq)] +pub struct StereoArg(pub hang::catalog::Stereo); + +impl std::str::FromStr for StereoArg { + type Err = String; + + fn from_str(arg: &str) -> Result { + match arg.parse() { + Ok(hang::catalog::Stereo::Unknown(_)) | Err(_) => { + Err(format!("expected `mono`, `left-right`, or `top-bottom`, got `{arg}`")) + } + Ok(stereo) => Ok(Self(stereo)), + } + } +} + +/// An `import --audio-layout` value. +#[derive(Clone, Debug, PartialEq)] +pub struct AudioLayoutArg(pub hang::catalog::AudioLayout); + +impl std::str::FromStr for AudioLayoutArg { + type Err = String; + + fn from_str(arg: &str) -> Result { + match arg.parse() { + Ok(hang::catalog::AudioLayout::Unknown(_)) | Err(_) => Err(format!("expected `ambisonic`, got `{arg}`")), + Ok(layout) => Ok(Self(layout)), + } + } } /// The single source feeding the Origin on an import. The container formats read @@ -1467,6 +1555,59 @@ mod tests { } } + /// The spatial flags parse into the catalog's types, refuse what it can't name, and only + /// stamp a catalog a stdin import builds. + #[test] + fn spatial_flags_parse_and_need_a_stdin_import() { + use hang::catalog::{AudioLayout, Projection, Stereo}; + + let import = |flags: &[&str], source: &[&str]| { + let argv = ["moq", "--connect", "http://relay", "--broadcast", "room", "import"]; + Invocation::try_parse_from(argv.iter().chain(flags).chain(source).copied()) + }; + let flags = [ + "--projection", + "equirectangular:180", + "--stereo", + "left-right", + "--audio-layout", + "ambisonic", + ]; + let invocation = import(&flags, &["fmp4"]).unwrap(); + invocation.validate().unwrap(); + let Command::Import(parsed) = &invocation.stages[0] else { + panic!("expected an import stage"); + }; + assert_eq!( + parsed.spatial(), + moq_mux::catalog::Spatial { + projection: Some(Projection::Equirectangular { fov: Some(180.0) }), + stereo: Some(Stereo::LeftRight), + layout: Some(AudioLayout::Ambisonic), + } + ); + + for value in ["rectilinear", "equirectangular"] { + import(&["--projection", value], &["avc3"]).unwrap().validate().unwrap(); + } + for flag in [ + ["--projection", "equirectangular:0"], + ["--projection", "equirectangular:361"], + ["--projection", "cubemap"], + ["--stereo", "interleaved"], + ["--audio-layout", "5.1"], + ] { + assert!(import(&flag, &["fmp4"]).is_err(), "{flag:?}"); + } + + let err = import(&["--stereo", "left-right"], &["hls", "http://origin/master.m3u8"]) + .unwrap() + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("--stereo"), "{err}"); + } + /// `--echo` reports what a stdout container export reads from a hang catalog, under one /// path segment, and the report sinks read no media to report on. #[test] diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 3b1f4da3d3c..9f423bad4d7 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -732,7 +732,8 @@ fn spawn_import( warn_if_missing_format(&name); let config = moq_mux::catalog::Config::default() .with_max_age(max_age) - .with_bandwidth(bandwidth.clone()); + .with_bandwidth(bandwidth.clone()) + .with_spatial(import.spatial()); let mut publish = if let ImportSource::Ts(TsImport { program: Some(TsProgram::All), .. diff --git a/rs/moq-mux/src/catalog/mod.rs b/rs/moq-mux/src/catalog/mod.rs index 4016f61b8e9..4b62bee5c46 100644 --- a/rs/moq-mux/src/catalog/mod.rs +++ b/rs/moq-mux/src/catalog/mod.rs @@ -31,6 +31,7 @@ mod estimate; mod format; mod producer; mod select; +mod spatial; mod stream; mod timebase; pub(crate) mod tracks; @@ -44,6 +45,7 @@ pub use estimate::{Estimate, Estimator}; pub use format::*; pub use producer::{Config, Guard, Producer}; pub use select::Select; +pub use spatial::Spatial; pub use stream::Stream; pub(crate) use timebase::Offset; pub use timebase::Timebase; diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 01c286a7638..bbd0d675260 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -292,6 +292,8 @@ pub struct Producer { baseline: super::estimate::Baseline, /// What each media track minted under this catalog has written. See [`stats`](Self::stats). sent: Arc>, + /// Stamped onto each rendition's config. See [`Config::with_spatial`]. + spatial: Arc, } // Manual Clone so a producer is cheaply clonable regardless of whether `E` is. @@ -306,6 +308,7 @@ impl Clone for Producer { bandwidth: self.bandwidth.clone(), baseline: self.baseline.clone(), sent: self.sent.clone(), + spatial: self.spatial.clone(), } } } @@ -324,6 +327,7 @@ pub struct Config { bandwidth: moq_net::bandwidth::Allocator, clock: Option, timeline: crate::timeline::Config, + spatial: super::Spatial, } impl Default for Config<()> { @@ -334,6 +338,7 @@ impl Default for Config<()> { bandwidth: moq_net::bandwidth::Allocator::unlimited(), clock: None, timeline: crate::timeline::Config::default(), + spatial: super::Spatial::default(), } } } @@ -350,6 +355,7 @@ impl Config { bandwidth: self.bandwidth, clock: self.clock, timeline: self.timeline, + spatial: self.spatial, } } @@ -401,6 +407,15 @@ impl Config { self.timeline = timeline; self } + + /// Stamp `spatial` onto every video and audio rendition whose config leaves those fields unset. + /// + /// A rendition it contradicts, such as an ambisonic layout on a stereo track, is refused when + /// its config is set. + pub fn with_spatial(mut self, spatial: super::Spatial) -> Self { + self.spatial = spatial; + self + } } impl Producer { @@ -474,6 +489,7 @@ impl Producer { bandwidth: config.bandwidth, baseline: Default::default(), sent: Default::default(), + spatial: Arc::new(config.spatial), }) } @@ -867,6 +883,11 @@ impl Producer { super::Estimator::with_broadcast(self.baseline.clone()) } + /// What each rendition's config is stamped with. See [`Config::with_spatial`]. + pub(crate) fn spatial(&self) -> &super::Spatial { + &self.spatial + } + /// The allocator passthrough tracks claim on. fMP4 writes groups by hand, so it reads this itself. pub(crate) fn bandwidth(&self) -> moq_net::bandwidth::Allocator { self.bandwidth.clone() @@ -3255,4 +3276,58 @@ mod test { v.estimate(jitter(2)).unwrap(); assert_eq!(sent(&catalog), Some(0), "the gated publish opened a window"); } + + #[test] + fn spatial_fills_only_what_a_rendition_leaves_unset() { + use hang::catalog::{AudioLayout, Projection, Stereo}; + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let spatial = super::super::Spatial { + projection: Some(Projection::Equirectangular { fov: Some(180.0) }), + stereo: Some(Stereo::LeftRight), + layout: Some(AudioLayout::Ambisonic), + }; + let catalog = Producer::new(&mut broadcast, Config::default().with_spatial(spatial)).unwrap(); + + let stamped = VideoConfig::new(hang::catalog::VideoCodec::VP8); + let mut declared = VideoConfig::new(hang::catalog::VideoCodec::VP8); + declared.stereo = Some(Stereo::TopBottom); + let audio = AudioConfig::new(AudioCodec::Opus, 48_000, 4); + + let mut keep = Vec::new(); + for (name, config) in [("stamped", stamped), ("declared", declared)] { + let track = broadcast.create_track(name, None).unwrap(); + let wire = super::super::hang::Container::try_from(&config).unwrap(); + keep.push(catalog.video(track, wire, config).unwrap()); + } + let track = broadcast.create_track("ambisonic", None).unwrap(); + let wire = super::super::hang::Container::try_from(&audio).unwrap(); + let _audio = catalog.audio(track, wire, audio).unwrap(); + + let snapshot = catalog.snapshot(); + let stamped = &snapshot.video.renditions["stamped"]; + assert_eq!(stamped.projection, Some(Projection::Equirectangular { fov: Some(180.0) })); + assert_eq!(stamped.stereo, Some(Stereo::LeftRight)); + assert_eq!(snapshot.video.renditions["declared"].stereo, Some(Stereo::TopBottom)); + assert_eq!(snapshot.audio.renditions["ambisonic"].layout, Some(AudioLayout::Ambisonic)); + } + + #[test] + fn spatial_refuses_a_rendition_it_contradicts() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let spatial = super::super::Spatial { + layout: Some(hang::catalog::AudioLayout::Ambisonic), + ..Default::default() + }; + let catalog = Producer::new(&mut broadcast, Config::default().with_spatial(spatial)).unwrap(); + + let stereo = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + let track = broadcast.create_track("stereo", None).unwrap(); + let wire = super::super::hang::Container::try_from(&stereo).unwrap(); + let Err(err) = catalog.audio(track, wire, stereo) else { + panic!("an ambisonic stereo rendition was accepted"); + }; + assert!(matches!(err, crate::Error::Hang(hang::Error::InvalidAmbisonicChannels(2)))); + assert!(catalog.snapshot().audio.renditions.is_empty()); + } } diff --git a/rs/moq-mux/src/catalog/spatial.rs b/rs/moq-mux/src/catalog/spatial.rs new file mode 100644 index 00000000000..a16f0a90994 --- /dev/null +++ b/rs/moq-mux/src/catalog/spatial.rs @@ -0,0 +1,43 @@ +use hang::catalog::{AudioLayout, Projection, Stereo}; + +/// How the broadcast's media fills the space around the viewer, stamped onto each video and +/// audio rendition whose own config leaves the field unset. +/// +/// The bitstreams the importers read carry none of this, so the publisher declares it. Set it +/// with [`Config::with_spatial`](super::Config::with_spatial). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Spatial { + /// How each video picture maps onto the viewer's field of view. + pub projection: Option, + + /// How each video picture packs the views of both eyes. + pub stereo: Option, + + /// What each audio rendition's decoded channels carry. + pub layout: Option, +} + +impl Spatial { + pub(super) fn video(&self, config: &mut hang::catalog::VideoConfig) -> crate::Result<()> { + if config.projection.is_none() { + config.projection.clone_from(&self.projection); + } + if config.stereo.is_none() { + config.stereo.clone_from(&self.stereo); + } + if let Some(projection) = &config.projection { + projection.check()?; + } + Ok(()) + } + + pub(super) fn audio(&self, config: &mut hang::catalog::AudioConfig) -> crate::Result<()> { + if config.layout.is_none() { + config.layout.clone_from(&self.layout); + } + if let Some(layout) = &config.layout { + layout.check(config.channel_count)?; + } + Ok(()) + } +} diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 70d6a478e18..d2a6b351a68 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -3,6 +3,7 @@ use std::time::Duration; use super::Estimate; use super::Producer; +use super::Spatial; use super::hang::{Catalog, CatalogExt}; /// A catalog config that can be published as a named rendition. @@ -109,6 +110,12 @@ pub trait RenditionConfig: Clone + Send + 'static { /// Replace the config's [`Estimate`] fields. Defaults to discarding them. fn set_estimate(&mut self, _estimate: Estimate) {} + + /// Fill the fields of [`Spatial`] this config leaves unset, refusing a result it can't carry. + /// Defaults to ignoring it. + fn spatial(&mut self, _spatial: &Spatial) -> crate::Result<()> { + Ok(()) + } } impl RenditionConfig for hang::catalog::JsonConfig { @@ -314,6 +321,9 @@ impl RenditionConfig for hang::catalog::VideoConfig { self.delay = estimate.delay; self.framerate = estimate.framerate; } + fn spatial(&mut self, spatial: &Spatial) -> crate::Result<()> { + spatial.video(self) + } } impl RenditionConfig for hang::catalog::AudioConfig { @@ -342,6 +352,9 @@ impl RenditionConfig for hang::catalog::AudioConfig { self.bitrate = estimate.bitrate; self.delay = estimate.delay; } + fn spatial(&mut self, spatial: &Spatial) -> crate::Result<()> { + spatial.audio(self) + } } impl RenditionConfig for hang::catalog::TextConfig { @@ -601,6 +614,7 @@ impl> Rendition { let resolved = Self::resolved(&supplied, &self.detected); self.check_decrease(&resolved)?; config.set_estimate(resolved.clone()); + config.spatial(self.catalog.spatial())?; { let mut guard = self.catalog.modify()?; let mut next = (*guard).clone(); @@ -714,10 +728,11 @@ impl> Rendition { } /// Write `config` into the catalog, leaving the caller to choose how the guard publishes it. - fn stage(&mut self, config: C) -> crate::Result> { + fn stage(&mut self, mut config: C) -> crate::Result> { if !self.present { return Err(crate::Error::NotPublished); } + config.spatial(self.catalog.spatial())?; self.check_decrease(&config.estimate())?; let mut guard = self.catalog.modify()?; let mut next = (*guard).clone(); diff --git a/test/README.md b/test/README.md index 8cc2ee20a38..1a990de5ea9 100644 --- a/test/README.md +++ b/test/README.md @@ -12,6 +12,7 @@ tests live in each language's own justfile. | [audio-quality](audio-quality/README.md) | `just test audio-quality` | browser and native audio playout stay within budget over an impaired path | | [drain](drain/README.md) | `just test drain` | a JS viewer migrates off a draining relay without a dropped group | | qmux | `just test qmux` | `@moq/qmux`, as client and as server, interoperates with the Rust qmux 0.6.3 release on every QMux draft | +| vr | `just test vr` | `demo/vr` paints a stereo 360 broadcast per eye facing forward, and its ambisonic decode follows the head | | video | `just test video` | the `@moq/video` renderer paints through WebGPU and falls back to Canvas2D after losing the GPU | | [json-properties](json-properties.md) | `just test json-properties` | both Snapshot codecs reconstruct, and both Window codecs emit the exact events for, random JSON sequences with replayable seeds | diff --git a/test/interop/clients/js/vr.ts b/test/interop/clients/js/vr.ts new file mode 100644 index 00000000000..c28afec5b75 --- /dev/null +++ b/test/interop/clients/js/vr.ts @@ -0,0 +1,153 @@ +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { check, finishTraces, launch, open, throwPageErrors } from "./harness"; + +const { values } = parseArgs({ options: { url: { type: "string" } } }); +if (!values.url) throw new Error("vr.ts requires --url"); +const run = process.env.MOQ_TEST_RUN; +if (!run) throw new Error("vr.ts runs under test/vr/run.sh, which builds the page into the run directory"); + +// Cross-origin isolated like the demo's vite server, so audio takes the SharedArrayBuffer path a deployment does. +const root = join(run, "vr-dist"); +const server = Bun.serve({ + port: 0, + async fetch(req) { + let path = new URL(req.url).pathname; + if (path === "/") path = "/index.html"; + const file = Bun.file(join(root, path)); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file, { + headers: { "Cross-Origin-Opener-Policy": "same-origin", "Cross-Origin-Embedder-Policy": "require-corp" }, + }); + }, +}); +const origin = `http://localhost:${server.port}`; + +const VIEWPORT = { width: 640, height: 360 }; +type Colour = "red" | "lime" | "blue" | "magenta" | "other"; + +// The colour at the centre of the page, read from a screenshot: WebGL discards its drawing buffer after +// each frame, so the canvas itself reads back blank. +async function centre(page: Page): Promise<{ colour: Colour; rgb: number[] }> { + const png = (await page.screenshot()).toString("base64"); + const rgb = await page.evaluate( + async ({ png, x, y }) => { + const image = await createImageBitmap(await (await fetch(`data:image/png;base64,${png}`)).blob()); + const canvas = new OffscreenCanvas(image.width, image.height); + const context = canvas.getContext("2d"); + if (!context) throw new Error("no 2d context"); + context.drawImage(image, 0, 0); + return Array.from(context.getImageData(x, y, 1, 1).data.slice(0, 3)); + }, + { png, x: VIEWPORT.width / 2, y: VIEWPORT.height / 2 }, + ); + const [r, g, b] = rgb; + const hi = (v: number) => v > 150; + const lo = (v: number) => v < 100; + const colour: Colour = + hi(r) && lo(g) && lo(b) + ? "red" + : lo(r) && hi(g) && lo(b) + ? "lime" + : lo(r) && lo(g) && hi(b) + ? "blue" + : hi(r) && lo(g) && hi(b) + ? "magenta" + : "other"; + return { colour, rgb }; +} + +async function waitForCentre(page: Page, want: Colour, timeout: number): Promise { + const deadline = Date.now() + timeout; + let last = await centre(page); + while (last.colour !== want && Date.now() < deadline) { + await page.waitForTimeout(250); + last = await centre(page); + } + check(last.colour === want, `centre is ${want}`, () => `got ${last.colour} rgb(${last.rgb.join(", ")})`); +} + +// Render plane waves through the ambisonic decoder offline and return each ear's RMS level. +async function ears(page: Page, direction: number[], yaw: number): Promise<{ L: number; R: number }> { + return page.evaluate( + async ({ direction, yaw }) => { + const { Spatial } = await import("/audio-check.js"); + const rate = 48_000; + const context = new OfflineAudioContext(2, rate, rate); + const buffer = context.createBuffer(4, rate, rate); + const [x, y, z] = direction; + // A 1 kHz plane wave encoded in ACN W, Y, Z, X with SN3D. + [1, y, z, x].forEach((gain, channel) => { + const data = buffer.getChannelData(channel); + for (let i = 0; i < data.length; i++) data[i] = 0.5 * gain * Math.sin((2 * Math.PI * 1000 * i) / rate); + }); + const discrete = { + channelCount: 4, + channelCountMode: "explicit", + channelInterpretation: "discrete", + } as const; + const source = new AudioBufferSourceNode(context, { buffer, ...discrete }); + const root = new GainNode(context, discrete); + source.connect(root); + const spatial = new Spatial(root, { kind: "ambisonic" }); + // The head yawed left by `yaw` about +Y, standing at the origin. + const head = { + decompose: ( + position: { set(x: number, y: number, z: number): void }, + rotation: { set(x: number, y: number, z: number, w: number): void }, + ) => { + position.set(0, 0, 0); + rotation.set(0, Math.sin(yaw / 2), 0, Math.cos(yaw / 2)); + }, + }; + spatial.pose(head); + source.start(); + const rendered = await context.startRendering(); + // Skip the first 100 ms, while the listener glides into place. + const rms = (channel: number) => { + const data = rendered.getChannelData(channel).subarray(rate / 10); + return Math.sqrt(data.reduce((sum, v) => sum + v * v, 0) / data.length); + }; + return { L: rms(0), R: rms(1) }; + }, + { direction, yaw }, + ); +} + +function louder(level: { L: number; R: number }, ear: "L" | "R" | "neither", assertion: string) { + const ratio = level.L / level.R; + const ok = ear === "L" ? ratio > 2 : ear === "R" ? ratio < 0.5 : ratio > 0.9 && ratio < 1.1; + check(ok, assertion, () => `L ${level.L.toFixed(3)} R ${level.R.toFixed(3)}`); +} + +const browser = await launch(["--use-angle=swiftshader", "--enable-unsafe-swiftshader"]); +let failed = true; +try { + const query = new URLSearchParams({ url: values.url, name: "vr.hang" }); + const [page, errors] = await open(browser, `${origin}/?${query}`, "vr", true, { viewport: VIEWPORT }); + + // Straight ahead is the left eye's centre: the catalog's stereo packing cropped the frame and the + // equirectangular projection faced its centre forward. + await waitForCentre(page, "red", 30_000); + + // Drag 90 degrees left (0.005 rad per pixel): a sphere wraps the left eye's left quarter there, + // where a flat screen would show only the background. + await page.mouse.move(VIEWPORT.width / 2, VIEWPORT.height / 2); + await page.mouse.down(); + await page.mouse.move(VIEWPORT.width / 2 + Math.PI / 2 / 0.005, VIEWPORT.height / 2, { steps: 20 }); + await page.mouse.up(); + await waitForCentre(page, "lime", 5_000); + + louder(await ears(page, [0, 1, 0], 0), "L", "a source on the left is louder in the left ear"); + louder(await ears(page, [0, 1, 0], Math.PI / 2), "neither", "turning left to face that source centres it"); + louder(await ears(page, [1, 0, 0], Math.PI / 2), "R", "a source ahead moves to the right ear after turning left"); + + throwPageErrors(errors); + console.log("vr: stereo equirectangular painted per eye, facing forward; ambisonics follow the head"); + failed = false; +} finally { + await finishTraces(failed); + await browser.close(); + server.stop(true); +} diff --git a/test/justfile b/test/justfile index fcd47b7be6e..1c9bb91223f 100644 --- a/test/justfile +++ b/test/justfile @@ -68,6 +68,10 @@ smoke *args: portrait: ./portrait/run.sh +# Paint a stereo 360 broadcast per eye in demo/vr, and turn its ambisonic decode with the head. +vr: + ./vr/run.sh + # Negative control: no publisher, every subscriber must time out (proves the # harness can actually report failure). diff --git a/test/vr/relay.toml b/test/vr/relay.toml new file mode 100644 index 00000000000..e9a52bad3ca --- /dev/null +++ b/test/vr/relay.toml @@ -0,0 +1,15 @@ +# Private loopback relay for the VR projection harness. +[log] +level = "info" + +[listen] +bind = "127.0.0.1:4443" +tls.generate = ["localhost", "127.0.0.1"] +# The CLI publisher negotiates lite-07; Chromium offers lite-06. +version = ["moq-lite-07-wip", "moq-lite-06"] + +[web.http] +listen = "127.0.0.1:4443" + +[auth] +public = "**" diff --git a/test/vr/run.sh b/test/vr/run.sh new file mode 100755 index 00000000000..cf3d915f80d --- /dev/null +++ b/test/vr/run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Publish a stereo 360 picture with VR catalog flags and check demo/vr paints each eye the right way round. +set -euo pipefail +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +harness_begin vr 'just test vr' +cd "$WORKSPACE" +cargo build --locked -p moq-relay -p moq-cli +bun install --frozen-lockfile +(cd demo/vr && bunx vite build --outDir "$HARNESS_RUN/vr-dist" --emptyOutDir) +# The ambisonic decoder alone, so the driver can render it offline without a stream. +bun build demo/vr/src/audio.ts --format esm --outfile "$HARNESS_RUN/vr-dist/audio-check.js" +(cd test/interop/clients/js && bunx playwright install chromium) +harness_port relay +URL="http://127.0.0.1:$HARNESS_PORT" +sed "s/4443/$HARNESS_PORT/g" "$DIR/relay.toml" >"$HARNESS_RUN/relay.toml" +harness_spawn relay "$HARNESS_RUN/relay.log" "$WORKSPACE/target/debug/moq-relay" "$HARNESS_RUN/relay.toml" +harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID" +# Left eye: blue, red straight ahead (u 0.5), lime 90 degrees left (u 0.25). Right eye: magenta, +# so a frame shown uncropped puts magenta straight ahead. +# shellcheck disable=SC2329 # invoked by harness_spawn +publish_vr() { + ffmpeg -hide_banner -loglevel error -re -f lavfi \ + -i "color=c=blue:s=960x480:r=15,drawbox=x=210:y=210:w=60:h=60:c=red:t=fill,drawbox=x=90:y=210:w=60:h=60:c=lime:t=fill,drawbox=x=480:y=0:w=480:h=480:c=magenta:t=fill" \ + -c:v libx264 -preset ultrafast -tune zerolatency -profile:v baseline -pix_fmt yuv420p -g 15 -f h264 - | + "$WORKSPACE/target/debug/moq" --connect "$URL" --connect-version moq-lite-07-wip --broadcast vr.hang \ + import --projection equirectangular --stereo left-right avc3 +} +harness_spawn publisher "$HARNESS_RUN/publisher.log" publish_vr +cd test/interop/clients/js +bun vr.ts --url "$URL"