From 4c2ba27464f5b41c951f21ad17624e4ec1d43a7b Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Fri, 13 Nov 2026 10:20:00 -0700 Subject: [PATCH 01/13] feat(demo): add hud + audio code to vr demo --- .github/workflows/interop.yml | 8 + .github/workflows/release-gate.yml | 2 + demo/vr/justfile | 4 + demo/vr/package.json | 22 ++ demo/vr/src/audio.ts | 123 ++++++++++ demo/vr/src/index.html | 29 +++ demo/vr/src/index.ts | 75 ++++++ demo/vr/src/scene.ts | 291 ++++++++++++++++++++++++ demo/vr/src/vite-env.d.ts | 9 + demo/vr/tsconfig.json | 5 + demo/vr/vite.config.ts | 11 + doc/bin/cli.md | 14 ++ package.json | 1 + rs/hang/src/catalog/audio/layout.rs | 2 +- rs/hang/src/catalog/video/projection.rs | 2 +- rs/moq-cli/src/args.rs | 141 ++++++++++++ rs/moq-cli/src/main.rs | 2 +- rs/moq-mux/src/catalog/mod.rs | 2 + rs/moq-mux/src/catalog/producer.rs | 75 ++++++ rs/moq-mux/src/catalog/spatial.rs | 43 ++++ rs/moq-mux/src/catalog/tracks.rs | 17 +- test/README.md | 1 + test/interop/clients/js/vr.ts | 150 ++++++++++++ test/justfile | 4 + test/vr/relay.toml | 15 ++ test/vr/run.sh | 36 +++ 26 files changed, 1080 insertions(+), 4 deletions(-) create mode 100644 demo/vr/justfile create mode 100644 demo/vr/package.json create mode 100644 demo/vr/src/audio.ts create mode 100644 demo/vr/src/index.html create mode 100644 demo/vr/src/index.ts create mode 100644 demo/vr/src/scene.ts create mode 100644 demo/vr/src/vite-env.d.ts create mode 100644 demo/vr/tsconfig.json create mode 100644 demo/vr/vite.config.ts create mode 100644 rs/moq-mux/src/catalog/spatial.rs create mode 100644 test/interop/clients/js/vr.ts create mode 100644 test/vr/relay.toml create mode 100755 test/vr/run.sh diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index c55c3a9c9a9..b2406aa83b1 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -160,6 +160,14 @@ jobs: HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs HARNESS_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl + - name: VR projection + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test vr + shell: bash -leo pipefail {0} + env: + HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs + HARNESS_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl + - name: Interop negative control id: negative if: ${{ !cancelled() && steps.setup.outcome == 'success' }} diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 9324e5f9190..9ae28ad624e 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -125,6 +125,8 @@ jobs: - "test/interop/**" - "test/max-age/**" - "test/portrait/**" + - "test/vr/**" + - "demo/vr/**" - "rs/moq-tokio/tests/integration/max_age.rs" - "test/datagram-tracks/**" - "rs/moq-tokio/tests/integration/datagram_tracks.rs" diff --git a/demo/vr/justfile b/demo/vr/justfile new file mode 100644 index 00000000000..408f18912bf --- /dev/null +++ b/demo/vr/justfile @@ -0,0 +1,4 @@ +set shell := ["bash", "-euo", "pipefail", "-c"] +serve url = "http://localhost:4443": + bun install + VITE_RELAY_URL="{{url}}" bun --bun vite diff --git a/demo/vr/package.json b/demo/vr/package.json new file mode 100644 index 00000000000..69858b6ebac --- /dev/null +++ b/demo/vr/package.json @@ -0,0 +1,22 @@ +{ + "name": "@moq/demo-vr", + "private": true, + "type": "module", + "description": "Media over QUIC - VR demo", + "license": "(MIT OR Apache-2.0)", + "scripts": { + "dev": "vite", + "build": "vite build", + "check": "tsc --noEmit" + }, + "dependencies": { + "@moq/watch": "workspace:^", + "three": "^0.186.1" + }, + "devDependencies": { + "@types/three": "^0.186.0", + "rolldown": "^1.2.16", + "typescript": "7.0.2", + "vite": "^8.3.5" + } +} diff --git a/demo/vr/src/audio.ts b/demo/vr/src/audio.ts new file mode 100644 index 00000000000..8338da08ba1 --- /dev/null +++ b/demo/vr/src/audio.ts @@ -0,0 +1,123 @@ +import * as THREE from "three"; + +export type Placement = { kind: "ambisonic" } | { kind: "point"; position: THREE.Vector3 } | { kind: "head" }; + +// Virtual speakers at the corners of a cube in ambisonic axes (W, Y, Z, X) +const SPEAKERS = [-1, 1].flatMap((x) => + [-1, 1].flatMap((y) => [-1, 1].map((z) => [x, y, z].map((v) => v / Math.sqrt(3)))), +); + +const MAX_RE = 1 / Math.sqrt(3); + +// Scratch for pose(), which runs every XR frame. +const POSE = { + position: new THREE.Vector3(), + rotation: new THREE.Quaternion(), + scale: new THREE.Vector3(), + forward: new THREE.Vector3(), + up: new THREE.Vector3(), +}; + +// Renders audio graph around a viewer's head +export class Spatial { + #context: BaseAudioContext; + #nodes: AudioNode[] = []; + #tracksPosition: boolean; + // Every placement mixes here, so muting is one param. + #out: GainNode; + + constructor(root: AudioNode, placement: Placement) { + this.#context = root.context; + this.#out = new GainNode(this.#context); + this.#out.connect(this.#context.destination); + this.#nodes.push(this.#out); + const destination = this.#out; + + switch (placement.kind) { + case "head": + root.connect(destination); + this.#tracksPosition = false; + break; + case "point": { + const panner = this.#panner(placement.position, 1); + root.connect(panner).connect(destination); + this.#tracksPosition = true; + break; + } + case "ambisonic": + this.#ambisonic(root, destination); + // The field is infinitely far, so only the head's rotation moves it. + this.#tracksPosition = false; + break; + } + this.#nodes.push(root); + } + + // Decodes out to fixed speakers around the scene + #ambisonic(root: AudioNode, destination: AudioNode) { + if (root.channelCount !== 4) + throw new Error(`first-order ambisonics needs 4 channels, got ${root.channelCount}`); + + const split = new ChannelSplitterNode(this.#context, { numberOfOutputs: 4 }); + root.connect(split); + this.#nodes.push(split); + + for (const [x, y, z] of SPEAKERS) { + // ACN order is W, Y, Z, X. + const weights = [1, 3 * MAX_RE * y, 3 * MAX_RE * z, 3 * MAX_RE * x].map((w) => w / SPEAKERS.length); + // Ambisonic axes to WebAudio + const panner = this.#panner(new THREE.Vector3(-y, z, -x), 0); + panner.connect(destination); + + weights.forEach((weight, channel) => { + const gain = new GainNode(this.#context, { gain: weight }); + split.connect(gain, channel).connect(panner); + this.#nodes.push(gain); + }); + } + } + + #panner(position: THREE.Vector3, rolloffFactor: number): PannerNode { + const panner = new PannerNode(this.#context, { + panningModel: "HRTF", + distanceModel: "inverse", + refDistance: 1, + rolloffFactor, + positionX: position.x, + positionY: position.y, + positionZ: position.z, + }); + this.#nodes.push(panner); + return panner; + } + + set muted(muted: boolean) { + this.#out.gain.setTargetAtTime(muted ? 0 : 1, this.#context.currentTime, 0.01); + } + + // Move the listener to the viewer's head, in the same space as the placement. + pose(head: THREE.Matrix4) { + const { position, rotation, scale, forward, up } = POSE; + head.decompose(position, rotation, scale); + forward.set(0, 0, -1).applyQuaternion(rotation); + up.set(0, 1, 0).applyQuaternion(rotation); + + const listener = this.#context.listener; + const at = this.#context.currentTime; + const set = (param: AudioParam, value: number) => param.setTargetAtTime(value, at, 0.01); + if (!this.#tracksPosition) position.set(0, 0, 0); + set(listener.positionX, position.x); + set(listener.positionY, position.y); + set(listener.positionZ, position.z); + set(listener.forwardX, forward.x); + set(listener.forwardY, forward.y); + set(listener.forwardZ, forward.z); + set(listener.upX, up.x); + set(listener.upY, up.y); + set(listener.upZ, up.z); + } + + close() { + for (const node of this.#nodes) node.disconnect(); + } +} diff --git a/demo/vr/src/index.html b/demo/vr/src/index.html new file mode 100644 index 00000000000..9ac55d45d39 --- /dev/null +++ b/demo/vr/src/index.html @@ -0,0 +1,29 @@ + + + + + + MoQ VR + + + + +
drag to look · R recentre · M mute
+ + + diff --git a/demo/vr/src/index.ts b/demo/vr/src/index.ts new file mode 100644 index 00000000000..064f02b39cb --- /dev/null +++ b/demo/vr/src/index.ts @@ -0,0 +1,75 @@ +import { Audio, Broadcast, Net, Signals, Sync, Video } from "@moq/watch"; +import { type Placement, Spatial } from "./audio"; +import { Scene } from "./scene"; + +const params = new URLSearchParams(location.search); +const url = new URL(params.get("url") ?? import.meta.env.VITE_RELAY_URL ?? "http://localhost:4443"); +const name = Net.Path.from(params.get("name") ?? "vr.hang"); + +const connection = new Net.Connection({ url }); +const broadcast = new Broadcast({ origin: connection.origin, name, enabled: true }); +const audioSrc = new Audio.Source({ broadcast, supported: Audio.Decoder.supported }); +const videoSrc = new Video.Source({ broadcast, supported: Video.Decoder.supported }); +const sync = new Sync({}); +const audio = new Audio.Decoder({ source: audioSrc, sync, enabled: true }); +const video = new Video.Decoder({ source: videoSrc, sync, enabled: true }); +const scene = new Scene(document.body); +const effect = new Signals.Effect(); + +// clone the frame once decoder gets it out +effect.run((e) => { + const frame = e.get(video.out.frame); + if (frame) scene.frame(frame.clone()); +}); + +effect.run((e) => { + const error = e.get(video.out.error) ?? e.get(audio.out.error); + const muted = e.get(scene.muted) ? " · muted" : ""; + const config = e.get(videoSrc.out.config); + const display = e.get(video.out.display); + const delay = e.get(sync.out.delay); + const fps = e.get(scene.fps); + // Stats change every frame, so sample them on the once-a-second fps tick instead. + const stats = video.out.stats.peek(); + + const projection = config?.projection; + const shape = projection?.kind === "equirectangular" ? `equirect ${projection.fov ?? 360}°` : "flat"; + const lines = [ + error + ? `error: ${error.message}` + : `${e.get(connection.status)} · ${name} ${e.get(broadcast.out.status)}${muted}`, + config + ? `${display?.width ?? "?"}×${display?.height ?? "?"} ${config.codec} · ${shape} · ${config.stereo ?? "mono"}` + : "no video", + `render ${fps.render} fps · video ${fps.video} fps`, + `buffer ${Math.round(delay.asMillis())} ms · late ${stats?.late ?? 0} · stalls ${stats?.stalls ?? 0}`, + ]; + scene.status(lines); +}); + +effect.run((e) => { + const config = e.get(videoSrc.out.config); + if (config) scene.layout(config.projection, config.stereo); +}); +effect.run((e) => { + const root = e.get(audio.out.root); + const config = e.get(audioSrc.out.config); + if (!root || !config) return; + const projection = e.get(videoSrc.out.config)?.projection?.kind; + const placement: Placement = + config.layout === "ambisonic" + ? { kind: "ambisonic" } + : projection === "equirectangular" + ? { kind: "head" } + : { kind: "point", position: scene.speaker }; + const spatial = new Spatial(root, placement); + scene.listener = spatial; + e.run((inner) => { + spatial.muted = inner.get(scene.muted); + }); + + e.cleanup(() => { + spatial.close(); + scene.listener = undefined; + }); +}); diff --git a/demo/vr/src/scene.ts b/demo/vr/src/scene.ts new file mode 100644 index 00000000000..fb427b861a9 --- /dev/null +++ b/demo/vr/src/scene.ts @@ -0,0 +1,291 @@ +import { type Hang, Signals } from "@moq/watch"; +import * as THREE from "three"; +import { VRButton } from "three/addons/webxr/VRButton.js"; +import type { Spatial } from "./audio"; + +type Projection = Hang.Catalog.Projection; + +const EYE = 1.6; +const SCREEN_HEIGHT = 1.8; +const SPHERE_RADIUS = 10; +const HUD_WIDTH = 0.9; +const HUD_LINES = 4; +const HUD_LINE = 56; +// Where the panel sits relative to the head: below the line of sight, at arm's length. +const HUD_OFFSET = new THREE.Vector3(0, -0.55, -1.2); +// How far the head turns from the panel before it swings back in front. +const HUD_FOLLOW = THREE.MathUtils.degToRad(35); +const UP = new THREE.Vector3(0, 1, 0); + +// Each eye's [u, v, width, height] in the frame. +const PACKING: Record = { + mono: [ + [0, 0, 1, 1], + [0, 0, 1, 1], + ], + "left-right": [ + [0, 0, 0.5, 1], + [0.5, 0, 0.5, 1], + ], + "top-bottom": [ + [0, 0.5, 1, 0.5], + [0, 0, 1, 0.5], + ], +}; + +// Paints decoded frames onto a screen or sphere according to projection +export class Scene { + // Screen centre in content space, shared as the point-audio source. + readonly speaker = new THREE.Vector3(0, EYE, -2); + + // Toggled by a controller squeeze or the M key. + readonly muted = new Signals.Signal(false); + + // Rendered and decoded frames per second, updated once a second. + readonly fps = new Signals.Signal({ render: 0, video: 0 }); + + // Posed in content space each frame, so audio turns with the picture on a recentre. + listener?: Spatial; + + #renderer: THREE.WebGLRenderer; + #scene = new THREE.Scene(); + #content = new THREE.Group(); + #camera: THREE.PerspectiveCamera; + #texture = new THREE.VideoFrameTexture(); + #frame?: VideoFrame; + #eyes: THREE.Mesh[]; + #hud: THREE.Mesh; + #hudCanvas = document.createElement("canvas"); + // The panel's current and target yaw; unset snaps it in front on the next frame. + #hudYaw?: number; + #hudTarget = 0; + #headPosition = new THREE.Vector3(); + #headDirection = new THREE.Vector3(); + #relative = new THREE.Matrix4(); + #renders = 0; + #frames = 0; + #since = performance.now(); + + #projection?: Projection; + #stereo = "mono"; + #aspect = 16 / 9; + + constructor(parent: HTMLElement) { + this.#renderer = new THREE.WebGLRenderer({ antialias: true }); + this.#renderer.setPixelRatio(window.devicePixelRatio); + this.#renderer.setSize(window.innerWidth, window.innerHeight); + this.#renderer.xr.enabled = true; + this.#renderer.xr.setReferenceSpaceType("local-floor"); + parent.append(this.#renderer.domElement, VRButton.createButton(this.#renderer)); + + this.#camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 0.01, 100); + this.#camera.position.set(0, EYE, 0); + this.#camera.layers.enable(1); + + this.#scene.background = new THREE.Color(0x111111); + this.#scene.add(this.#content); + this.#texture.colorSpace = THREE.SRGBColorSpace; + + this.#eyes = [1, 2].map((layer) => { + const map = this.#texture.clone(); + const mesh = new THREE.Mesh( + new THREE.BufferGeometry(), + new THREE.MeshBasicMaterial({ map, toneMapped: false }), + ); + mesh.layers.set(layer); + mesh.visible = false; + this.#content.add(mesh); + return mesh; + }); + this.#build(); + + this.#hudCanvas.width = 1024; + this.#hudCanvas.height = HUD_LINES * HUD_LINE + 32; + const hudMap = new THREE.CanvasTexture(this.#hudCanvas); + hudMap.colorSpace = THREE.SRGBColorSpace; + this.#hud = new THREE.Mesh( + new THREE.PlaneGeometry(HUD_WIDTH, (HUD_WIDTH * this.#hudCanvas.height) / this.#hudCanvas.width), + new THREE.MeshBasicMaterial({ map: hudMap, transparent: true, toneMapped: false }), + ); + // Outside the content, since it follows the head rather than the picture. + this.#scene.add(this.#hud); + this.#renderer.xr.addEventListener("sessionstart", () => { + this.#hudYaw = undefined; + }); + + // Trigger or pinch recentres; grip toggles mute. + for (const index of [0, 1]) { + const controller = this.#renderer.xr.getController(index); + controller.addEventListener("select", this.#recentre); + controller.addEventListener("squeeze", this.#toggleMute); + } + + window.addEventListener("resize", this.#resize); + window.addEventListener("keydown", this.#key); + this.#look(this.#renderer.domElement); + + this.#renderer.setAnimationLoop(() => { + this.#follow(); + this.#renderer.render(this.#scene, this.#camera); + this.#relative.copy(this.#content.matrixWorld).invert().multiply(this.#head().matrixWorld); + this.listener?.pose(this.#relative); + this.#renders++; + this.#count(); + }); + } + + frame(frame: VideoFrame) { + this.#frames++; + this.#texture.setFrame(frame); + for (const mesh of this.#eyes) { + if (mesh.material.map) mesh.material.map.needsUpdate = true; + mesh.visible = true; + } + this.#frame?.close(); + this.#frame = frame; + + const aspect = frame.displayWidth / frame.displayHeight; + if (aspect !== this.#aspect) { + this.#aspect = aspect; + this.#build(); + } + } + + // Layout for the picture according to projection and stereo + layout(projection?: Projection, stereo?: string) { + this.#projection = projection; + this.#stereo = stereo && stereo in PACKING ? stereo : "mono"; + this.#build(); + } + + // Up to HUD_LINES lines on the in-headset panel. + status(lines: string[]) { + const context = this.#hudCanvas.getContext("2d"); + if (!context) return; + const { width, height } = this.#hudCanvas; + context.clearRect(0, 0, width, height); + context.fillStyle = "rgba(0, 0, 0, 0.6)"; + context.beginPath(); + context.roundRect(0, 0, width, height, 24); + context.fill(); + context.fillStyle = "#fff"; + context.font = "40px monospace"; + context.textBaseline = "middle"; + for (const [index, line] of lines.slice(0, HUD_LINES).entries()) { + context.fillText(line, 32, 16 + HUD_LINE * (index + 0.5), width - 64); + } + if (this.#hud.material.map) this.#hud.material.map.needsUpdate = true; + } + + // Hold the panel still through a glance, then glide it back in front once the head turns away. + #follow() { + const head = this.#head(); + head.getWorldPosition(this.#headPosition); + head.getWorldDirection(this.#headDirection); + const yaw = Math.atan2(-this.#headDirection.x, -this.#headDirection.z); + const wrap = (angle: number) => Math.atan2(Math.sin(angle), Math.cos(angle)); + + if (this.#hudYaw === undefined) this.#hudYaw = this.#hudTarget = yaw; + if (Math.abs(wrap(yaw - this.#hudTarget)) > HUD_FOLLOW) this.#hudTarget = yaw; + this.#hudYaw += wrap(this.#hudTarget - this.#hudYaw) * 0.1; + + this.#hud.position.copy(HUD_OFFSET).applyAxisAngle(UP, this.#hudYaw).add(this.#headPosition); + // Turned to face the head, then tilted up toward the eyes. + this.#hud.rotation.set(-0.4, this.#hudYaw, 0, "YXZ"); + } + + #count() { + const now = performance.now(); + const elapsed = now - this.#since; + if (elapsed < 1000) return; + const rate = (count: number) => Math.round((count * 1000) / elapsed); + this.fps.set({ render: rate(this.#renders), video: rate(this.#frames) }); + this.#renders = 0; + this.#frames = 0; + this.#since = now; + } + + #head(): THREE.Camera { + return this.#renderer.xr.isPresenting ? this.#renderer.xr.getCamera() : this.#camera; + } + + // Turn the content to face wherever the viewer is looking. + #recentre = () => { + const head = this.#head(); + // TODO(human) + this.#content.updateMatrixWorld(); + }; + + #toggleMute = () => { + this.muted.set(!this.muted.peek()); + }; + + #key = (event: KeyboardEvent) => { + if (event.key === "r") this.#recentre(); + if (event.key === "m") this.#toggleMute(); + }; + + #build() { + const packing = PACKING[this.#stereo]; + const projection = this.#projection?.kind === "equirectangular" ? this.#projection : undefined; + + for (const [eye, mesh] of this.#eyes.entries()) { + const [u, v, width, height] = packing[eye]; + mesh.material.map?.offset.set(u, v); + mesh.material.map?.repeat.set(width, height); + + mesh.geometry.dispose(); + if (projection) { + const fov = THREE.MathUtils.degToRad(projection.fov ?? 360); + // Centred on the seam opposite phi = 0, then mirrored and turned so the picture's centre + // faces -Z and its left half wraps to the viewer's left. + mesh.geometry = new THREE.SphereGeometry(SPHERE_RADIUS, 64, 32, Math.PI - fov / 2, fov).scale(-1, 1, 1); + mesh.position.set(0, EYE, 0); + mesh.rotation.y = -Math.PI / 2; + } else { + // The packed frame's aspect, unpacked to one eye's. + const aspect = (this.#aspect * width) / height; + mesh.geometry = new THREE.PlaneGeometry(SCREEN_HEIGHT * aspect, SCREEN_HEIGHT); + mesh.position.copy(this.speaker); + mesh.rotation.y = 0; + } + } + } + + #resize = () => { + this.#camera.aspect = window.innerWidth / window.innerHeight; + this.#camera.updateProjectionMatrix(); + this.#renderer.setSize(window.innerWidth, window.innerHeight); + }; + + // Looking around is dragging on desktop; a headset tracks it. + #look(element: HTMLElement) { + const euler = new THREE.Euler(0, 0, 0, "YXZ"); + element.addEventListener("pointermove", (event) => { + if (event.buttons !== 1) return; + euler.setFromQuaternion(this.#camera.quaternion); + euler.y += event.movementX * 0.005; + euler.x = THREE.MathUtils.clamp(euler.x + event.movementY * 0.005, -Math.PI / 2, Math.PI / 2); + this.#camera.quaternion.setFromEuler(euler); + }); + } + + close() { + this.#renderer.setAnimationLoop(null); + window.removeEventListener("resize", this.#resize); + window.removeEventListener("keydown", this.#key); + for (const index of [0, 1]) { + const controller = this.#renderer.xr.getController(index); + controller.removeEventListener("select", this.#recentre); + controller.removeEventListener("squeeze", this.#toggleMute); + } + for (const mesh of [...this.#eyes, this.#hud]) { + mesh.geometry.dispose(); + mesh.material.map?.dispose(); + mesh.material.dispose(); + } + this.#texture.dispose(); + this.#renderer.dispose(); + this.#frame?.close(); + } +} diff --git a/demo/vr/src/vite-env.d.ts b/demo/vr/src/vite-env.d.ts new file mode 100644 index 00000000000..4ef6324115a --- /dev/null +++ b/demo/vr/src/vite-env.d.ts @@ -0,0 +1,9 @@ +/// + +interface ImportMetaEnv { + readonly VITE_RELAY_URL?: string; +} + +interface ImportMeta { + readonly env: ImportMetaEnv; +} diff --git a/demo/vr/tsconfig.json b/demo/vr/tsconfig.json new file mode 100644 index 00000000000..616947250ab --- /dev/null +++ b/demo/vr/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../js/tsconfig.json", + "compilerOptions": { "module": "esnext", "moduleResolution": "bundler", "outDir": "dist" }, + "include": ["src", "../../js/common/worklet.d.ts"] +} diff --git a/demo/vr/vite.config.ts b/demo/vr/vite.config.ts new file mode 100644 index 00000000000..1560c3f72bf --- /dev/null +++ b/demo/vr/vite.config.ts @@ -0,0 +1,11 @@ +import { resolve } from "path"; +import { defineConfig } from "vite"; +import { worklet } from "../../js/common/vite-plugin-worklet.ts"; + +export default defineConfig({ + root: "src", + envDir: resolve(import.meta.dirname), + plugins: [worklet()], + build: { target: "esnext" }, + server: { hmr: false }, +}); diff --git a/doc/bin/cli.md b/doc/bin/cli.md index eaa99666881..dfbe2576f36 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -108,6 +108,20 @@ MP4 parser. `import fmp4 --cmaf` publishes each fragment whole as [`cmaf`](/concept/hang#container) instead, keeping the source's exact sample timing. +The stdin imports can mark the broadcast for VR when the source does not: +`--projection equirectangular` for 360 video (`equirectangular:180` for VR180), +`--stereo left-right` or `top-bottom` for a picture that packs both eyes, and +`--audio-layout ambisonic` for AmbiX audio. fMP4 carrying Spherical Video V2 +boxes and Opus with channel mapping family 2 already say so, and win over the +flags, which matters mostly for bare bitstreams like `avc3`. An ambisonic track +without a square channel count of at least 4 is refused. + +```bash +ffmpeg -re -i vr180.mp4 -an -c:v copy -bsf:v h264_mp4toannexb -f h264 - | + moq --connect https://relay.example.com/anon --broadcast vr.hang \ + import --projection equirectangular:180 --stereo left-right avc3 +``` + `import rtsp` pulls an IP camera's H.264 or H.265 video and AAC audio without re-encoding. RTP rides the RTSP connection, so run it on the camera's network and the camera needs no inbound port. Credentials go in the URL diff --git a/package.json b/package.json index 60a83df9346..a09cfccd755 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "demo/boy", "demo/web", "demo/carla/web", + "demo/vr", "test/audio-quality/clients/js", "test/interop/clients/js", "test/interop/clients/js-native", diff --git a/rs/hang/src/catalog/audio/layout.rs b/rs/hang/src/catalog/audio/layout.rs index 82a7201e60f..860b7f3a1ad 100644 --- a/rs/hang/src/catalog/audio/layout.rs +++ b/rs/hang/src/catalog/audio/layout.rs @@ -22,7 +22,7 @@ pub enum AudioLayout { impl AudioLayout { /// Refuse a channel count this layout cannot carry. - pub(crate) fn check(&self, channel_count: u32) -> crate::Result<()> { + pub fn check(&self, channel_count: u32) -> crate::Result<()> { if *self == Self::Ambisonic { let order = channel_count.isqrt(); if order < 2 || order * order != channel_count { diff --git a/rs/hang/src/catalog/video/projection.rs b/rs/hang/src/catalog/video/projection.rs index 5a038995632..2b5c634ec9a 100644 --- a/rs/hang/src/catalog/video/projection.rs +++ b/rs/hang/src/catalog/video/projection.rs @@ -47,7 +47,7 @@ impl UnknownProjection { impl Projection { /// Refuse a field of view outside (0, 360] degrees. - pub(crate) fn check(&self) -> crate::Result<()> { + pub fn check(&self) -> crate::Result<()> { if let Self::Equirectangular { fov: Some(fov) } = self && !(*fov > 0.0 && *fov <= 360.0) { diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index ac0dcdf6bd6..5a12f91c897 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -360,6 +360,14 @@ impl Invocation { { anyhow::bail!("--linger keeps a cleanly ended broadcast up, but {reason}"); } + if let Command::Import(import) = command + && import.spatial() != moq_mux::catalog::Spatial::default() + && import.source.stdin_format().is_none() + { + anyhow::bail!( + "--projection, --stereo, and --audio-layout stamp the catalog a stdin import builds, which this source does not" + ); + } let Command::Export(export) = command else { continue; }; @@ -825,6 +833,21 @@ pub struct Import { #[usage(long, value_name = "PATH")] pub echo_path: Option, + /// How each video picture maps onto the viewer's view: `rectilinear`, or `equirectangular` + /// with an optional `:fov` in degrees (`equirectangular:180` for VR180). The stdin imports only. + #[usage(long)] + pub projection: Option, + + /// How each video picture packs both eyes: `mono`, `left-right`, or `top-bottom`. The stdin + /// imports only. + #[usage(long)] + pub stereo: Option, + + /// What each audio track's channels carry: `ambisonic` for AmbiX, which needs a square channel + /// count of at least 4. The stdin imports only. + #[usage(long)] + pub audio_layout: Option, + /// The single source feeding the Origin. #[usage(subcommand)] pub source: ImportSource, @@ -845,6 +868,71 @@ impl Import { } found } + + /// The spatial metadata the flags stamp onto each rendition. + pub fn spatial(&self) -> moq_mux::catalog::Spatial { + moq_mux::catalog::Spatial { + projection: self.projection.clone().map(|arg| arg.0), + stereo: self.stereo.clone().map(|arg| arg.0), + layout: self.audio_layout.clone().map(|arg| arg.0), + } + } +} + +/// An `import --projection` value. +#[derive(Clone, Debug, PartialEq)] +pub struct ProjectionArg(pub hang::catalog::Projection); + +impl std::str::FromStr for ProjectionArg { + type Err = String; + + fn from_str(arg: &str) -> Result { + use hang::catalog::Projection; + + let projection = match arg.split_once(':') { + None if arg == "rectilinear" => Projection::Rectilinear, + None if arg == "equirectangular" => Projection::Equirectangular { fov: None }, + Some(("equirectangular", fov)) => { + let fov = fov.parse().map_err(|_| format!("expected a field of view in degrees, got `{fov}`"))?; + Projection::Equirectangular { fov: Some(fov) } + } + _ => return Err(format!("expected `rectilinear` or `equirectangular[:fov]`, got `{arg}`")), + }; + projection.check().map_err(|err| err.to_string())?; + Ok(Self(projection)) + } +} + +/// An `import --stereo` value. +#[derive(Clone, Debug, PartialEq)] +pub struct StereoArg(pub hang::catalog::Stereo); + +impl std::str::FromStr for StereoArg { + type Err = String; + + fn from_str(arg: &str) -> Result { + match arg.parse() { + Ok(hang::catalog::Stereo::Unknown(_)) | Err(_) => { + Err(format!("expected `mono`, `left-right`, or `top-bottom`, got `{arg}`")) + } + Ok(stereo) => Ok(Self(stereo)), + } + } +} + +/// An `import --audio-layout` value. +#[derive(Clone, Debug, PartialEq)] +pub struct AudioLayoutArg(pub hang::catalog::AudioLayout); + +impl std::str::FromStr for AudioLayoutArg { + type Err = String; + + fn from_str(arg: &str) -> Result { + match arg.parse() { + Ok(hang::catalog::AudioLayout::Unknown(_)) | Err(_) => Err(format!("expected `ambisonic`, got `{arg}`")), + Ok(layout) => Ok(Self(layout)), + } + } } /// The single source feeding the Origin on an import. The container formats read @@ -1880,6 +1968,59 @@ mod tests { } } + /// The spatial flags parse into the catalog's types, refuse what it can't name, and only + /// stamp a catalog a stdin import builds. + #[test] + fn spatial_flags_parse_and_need_a_stdin_import() { + use hang::catalog::{AudioLayout, Projection, Stereo}; + + let import = |flags: &[&str], source: &[&str]| { + let argv = ["moq", "--connect", "http://relay", "--broadcast", "room", "import"]; + Invocation::try_parse_from(argv.iter().chain(flags).chain(source).copied()) + }; + let flags = [ + "--projection", + "equirectangular:180", + "--stereo", + "left-right", + "--audio-layout", + "ambisonic", + ]; + let invocation = import(&flags, &["fmp4"]).unwrap(); + invocation.validate().unwrap(); + let Command::Import(parsed) = &invocation.stages[0] else { + panic!("expected an import stage"); + }; + assert_eq!( + parsed.spatial(), + moq_mux::catalog::Spatial { + projection: Some(Projection::Equirectangular { fov: Some(180.0) }), + stereo: Some(Stereo::LeftRight), + layout: Some(AudioLayout::Ambisonic), + } + ); + + for value in ["rectilinear", "equirectangular"] { + import(&["--projection", value], &["avc3"]).unwrap().validate().unwrap(); + } + for flag in [ + ["--projection", "equirectangular:0"], + ["--projection", "equirectangular:361"], + ["--projection", "cubemap"], + ["--stereo", "interleaved"], + ["--audio-layout", "5.1"], + ] { + assert!(import(&flag, &["fmp4"]).is_err(), "{flag:?}"); + } + + let err = import(&["--stereo", "left-right"], &["hls", "http://origin/master.m3u8"]) + .unwrap() + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("--stereo"), "{err}"); + } + /// `--echo` reports what a stdout container export reads from a hang catalog, under one /// path segment, and the report sinks read no media to report on. #[test] diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 191ad854204..88f615bb613 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -779,7 +779,7 @@ fn spawn_import( if let Some(format) = import.source.stdin_format() { warn_if_missing_format(&name); - let config = ingest.catalog(); + let config = ingest.catalog().with_spatial(import.spatial()); let mut publish = if let ImportSource::Ts(TsImport { program: Some(TsProgram::All), .. diff --git a/rs/moq-mux/src/catalog/mod.rs b/rs/moq-mux/src/catalog/mod.rs index f19733d5786..76d102ecb73 100644 --- a/rs/moq-mux/src/catalog/mod.rs +++ b/rs/moq-mux/src/catalog/mod.rs @@ -32,6 +32,7 @@ mod format; pub(crate) mod interval; mod producer; mod select; +mod spatial; mod stream; mod timebase; pub(crate) mod tracks; @@ -50,6 +51,7 @@ pub use format::*; pub(crate) use producer::to_msf; pub use producer::{Config, Guard, Producer}; pub use select::Select; +pub use spatial::Spatial; pub use stream::Stream; pub(crate) use timebase::Offset; pub use timebase::Timebase; diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index c0f92185124..58c45062a61 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -373,6 +373,8 @@ pub struct Producer { order: super::estimate::Order, /// What each media track minted under this catalog has written. See [`stats`](Self::stats). sent: Arc>, + /// Stamped onto each rendition's config. See [`Config::with_spatial`]. + spatial: Arc, } // Manual Clone so a producer is cheaply clonable regardless of whether `E` is. @@ -386,6 +388,7 @@ impl Clone for Producer { bandwidth: self.bandwidth.clone(), order: self.order.clone(), sent: self.sent.clone(), + spatial: self.spatial.clone(), } } } @@ -403,6 +406,7 @@ pub struct Config { max_age: Option, bandwidth: moq_net::bandwidth::Allocator, clock: Option, + spatial: super::Spatial, } impl Default for Config<()> { @@ -412,6 +416,7 @@ impl Default for Config<()> { max_age: Some(hang::container::MAX_AGE), bandwidth: moq_net::bandwidth::Allocator::unlimited(), clock: None, + spatial: super::Spatial::default(), } } } @@ -427,6 +432,7 @@ impl Config { max_age: self.max_age, bandwidth: self.bandwidth, clock: self.clock, + spatial: self.spatial, } } @@ -473,6 +479,15 @@ impl Config { self.bandwidth = bandwidth; self } + + /// Stamp `spatial` onto every video and audio rendition whose config leaves those fields unset. + /// + /// A rendition it contradicts, such as an ambisonic layout on a stereo track, is refused when + /// its config is set. + pub fn with_spatial(mut self, spatial: super::Spatial) -> Self { + self.spatial = spatial; + self + } } impl Producer { @@ -532,6 +547,7 @@ impl Producer { bandwidth: config.bandwidth, order: Default::default(), sent: Default::default(), + spatial: Arc::new(config.spatial), }) } @@ -960,6 +976,11 @@ impl Producer { super::Estimator::with_order(self.order.clone()) } + /// What each rendition's config is stamped with. See [`Config::with_spatial`]. + pub(crate) fn spatial(&self) -> &super::Spatial { + &self.spatial + } + /// The allocator passthrough tracks claim on. fMP4 writes groups by hand, so it reads this itself. pub(crate) fn bandwidth(&self) -> moq_net::bandwidth::Allocator { self.bandwidth.clone() @@ -3681,4 +3702,58 @@ mod test { let track = &msf_wire(&catalog).tracks[0]; assert_eq!((track.bitrate, track.delay), (Some(1_000_000), None)); } + + #[test] + fn spatial_fills_only_what_a_rendition_leaves_unset() { + use hang::catalog::{AudioLayout, Projection, Stereo}; + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let spatial = super::super::Spatial { + projection: Some(Projection::Equirectangular { fov: Some(180.0) }), + stereo: Some(Stereo::LeftRight), + layout: Some(AudioLayout::Ambisonic), + }; + let catalog = Producer::new(&mut broadcast, Config::default().with_spatial(spatial)).unwrap(); + + let stamped = VideoConfig::new(hang::catalog::VideoCodec::VP8); + let mut declared = VideoConfig::new(hang::catalog::VideoCodec::VP8); + declared.stereo = Some(Stereo::TopBottom); + let audio = AudioConfig::new(AudioCodec::Opus, 48_000, 4); + + let mut keep = Vec::new(); + for (name, config) in [("stamped", stamped), ("declared", declared)] { + let track = broadcast.create_track(name, None).unwrap(); + let wire = super::super::hang::Container::try_from(&config).unwrap(); + keep.push(catalog.video(track, wire, config).unwrap()); + } + let track = broadcast.create_track("ambisonic", None).unwrap(); + let wire = super::super::hang::Container::try_from(&audio).unwrap(); + let _audio = catalog.audio(track, wire, audio).unwrap(); + + let snapshot = catalog.snapshot(); + let stamped = &snapshot.video.renditions["stamped"]; + assert_eq!(stamped.projection, Some(Projection::Equirectangular { fov: Some(180.0) })); + assert_eq!(stamped.stereo, Some(Stereo::LeftRight)); + assert_eq!(snapshot.video.renditions["declared"].stereo, Some(Stereo::TopBottom)); + assert_eq!(snapshot.audio.renditions["ambisonic"].layout, Some(AudioLayout::Ambisonic)); + } + + #[test] + fn spatial_refuses_a_rendition_it_contradicts() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let spatial = super::super::Spatial { + layout: Some(hang::catalog::AudioLayout::Ambisonic), + ..Default::default() + }; + let catalog = Producer::new(&mut broadcast, Config::default().with_spatial(spatial)).unwrap(); + + let stereo = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + let track = broadcast.create_track("stereo", None).unwrap(); + let wire = super::super::hang::Container::try_from(&stereo).unwrap(); + let Err(err) = catalog.audio(track, wire, stereo) else { + panic!("an ambisonic stereo rendition was accepted"); + }; + assert!(matches!(err, crate::Error::Hang(hang::Error::InvalidAmbisonicChannels(2)))); + assert!(catalog.snapshot().audio.renditions.is_empty()); + } } diff --git a/rs/moq-mux/src/catalog/spatial.rs b/rs/moq-mux/src/catalog/spatial.rs new file mode 100644 index 00000000000..a16f0a90994 --- /dev/null +++ b/rs/moq-mux/src/catalog/spatial.rs @@ -0,0 +1,43 @@ +use hang::catalog::{AudioLayout, Projection, Stereo}; + +/// How the broadcast's media fills the space around the viewer, stamped onto each video and +/// audio rendition whose own config leaves the field unset. +/// +/// The bitstreams the importers read carry none of this, so the publisher declares it. Set it +/// with [`Config::with_spatial`](super::Config::with_spatial). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Spatial { + /// How each video picture maps onto the viewer's field of view. + pub projection: Option, + + /// How each video picture packs the views of both eyes. + pub stereo: Option, + + /// What each audio rendition's decoded channels carry. + pub layout: Option, +} + +impl Spatial { + pub(super) fn video(&self, config: &mut hang::catalog::VideoConfig) -> crate::Result<()> { + if config.projection.is_none() { + config.projection.clone_from(&self.projection); + } + if config.stereo.is_none() { + config.stereo.clone_from(&self.stereo); + } + if let Some(projection) = &config.projection { + projection.check()?; + } + Ok(()) + } + + pub(super) fn audio(&self, config: &mut hang::catalog::AudioConfig) -> crate::Result<()> { + if config.layout.is_none() { + config.layout.clone_from(&self.layout); + } + if let Some(layout) = &config.layout { + layout.check(config.channel_count)?; + } + Ok(()) + } +} diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 945a2a80723..a9786ec957f 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -3,6 +3,7 @@ use std::time::Duration; use super::Estimate; use super::Producer; +use super::Spatial; use super::hang::{Catalog, CatalogExt}; /// A catalog config that can be published as a named rendition. @@ -111,6 +112,12 @@ pub trait RenditionConfig: Clone + Send + 'static { /// Replace the config's [`Estimate`] fields. Defaults to discarding them. fn set_estimate(&mut self, _estimate: Estimate) {} + + /// Fill the fields of [`Spatial`] this config leaves unset, refusing a result it can't carry. + /// Defaults to ignoring it. + fn spatial(&mut self, _spatial: &Spatial) -> crate::Result<()> { + Ok(()) + } } impl RenditionConfig for hang::catalog::JsonConfig { @@ -322,6 +329,9 @@ impl RenditionConfig for hang::catalog::VideoConfig { self.delay = estimate.delay; self.framerate = estimate.framerate; } + fn spatial(&mut self, spatial: &Spatial) -> crate::Result<()> { + spatial.video(self) + } } impl RenditionConfig for hang::catalog::AudioConfig { @@ -350,6 +360,9 @@ impl RenditionConfig for hang::catalog::AudioConfig { self.bitrate = estimate.bitrate; self.delay = estimate.delay; } + fn spatial(&mut self, spatial: &Spatial) -> crate::Result<()> { + spatial.audio(self) + } } impl RenditionConfig for hang::catalog::TextConfig { @@ -682,6 +695,7 @@ impl> Rendition { Self::check_decrease(&published, &resolved)?; } config.set_estimate(resolved.clone()); + config.spatial(self.catalog.spatial())?; { let mut guard = self.catalog.modify()?; write(&mut guard, &self.name, config)?; @@ -826,10 +840,11 @@ impl> Rendition { } /// Write `config` into the catalog, leaving the caller to choose how the guard publishes it. - fn stage(&mut self, config: C) -> crate::Result> { + fn stage(&mut self, mut config: C) -> crate::Result> { if !self.present { return Err(self.unpublished()); } + config.spatial(self.catalog.spatial())?; Self::check_decrease(&self.config()?.estimate(), &config.estimate())?; let mut guard = self.catalog.modify()?; write(&mut guard, &self.name, config)?; diff --git a/test/README.md b/test/README.md index 9f54a250896..81fd026b353 100644 --- a/test/README.md +++ b/test/README.md @@ -20,6 +20,7 @@ questions to a harness. For builds and CI selection, see the | [drain](drain/README.md) | `just test drain` | a JS viewer migrates off a draining relay without a dropped group | | [stream-credit](stream-credit/README.md) | `just test stream-credit` | a browser tab reads 300 live tracks past its ~100-stream WebTransport cap, the highest priority first, and nothing hangs | | qmux | `just test qmux` | `@moq/qmux`, as client and as server, interoperates with the Rust qmux 0.6.3 release on every QMux draft | +| vr | `just test vr` | `demo/vr` paints a stereo 360 broadcast per eye facing forward, and its ambisonic decode follows the head | | video | `just test video` | the `@moq/video` renderer paints through WebGPU and falls back to Canvas2D after losing the GPU | | [json-properties](json-properties.md) | `just test json-properties` | both Snapshot codecs reconstruct, and both Window codecs emit the exact events for, random JSON sequences with replayable seeds | diff --git a/test/interop/clients/js/vr.ts b/test/interop/clients/js/vr.ts new file mode 100644 index 00000000000..85fce6e0617 --- /dev/null +++ b/test/interop/clients/js/vr.ts @@ -0,0 +1,150 @@ +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { check, finishTraces, launch, open, throwPageErrors } from "./harness"; + +const { values } = parseArgs({ options: { url: { type: "string" } } }); +if (!values.url) throw new Error("vr.ts requires --url"); +const run = process.env.HARNESS_RUN_DIR; +if (!run) throw new Error("vr.ts runs under test/vr/run.sh, which builds the page into the run directory"); + +const root = join(run, "vr-dist"); +const server = Bun.serve({ + port: 0, + async fetch(req) { + let path = new URL(req.url).pathname; + if (path === "/") path = "/index.html"; + const file = Bun.file(join(root, path)); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file); + }, +}); +const origin = `http://localhost:${server.port}`; + +const VIEWPORT = { width: 640, height: 360 }; +type Colour = "red" | "lime" | "blue" | "magenta" | "other"; + +// The colour at the centre of the page, read from a screenshot: WebGL discards its drawing buffer after +// each frame, so the canvas itself reads back blank. +async function centre(page: Page): Promise<{ colour: Colour; rgb: number[] }> { + const png = (await page.screenshot()).toString("base64"); + const rgb = await page.evaluate( + async ({ png, x, y }) => { + const image = await createImageBitmap(await (await fetch(`data:image/png;base64,${png}`)).blob()); + const canvas = new OffscreenCanvas(image.width, image.height); + const context = canvas.getContext("2d"); + if (!context) throw new Error("no 2d context"); + context.drawImage(image, 0, 0); + return Array.from(context.getImageData(x, y, 1, 1).data.slice(0, 3)); + }, + { png, x: VIEWPORT.width / 2, y: VIEWPORT.height / 2 }, + ); + const [r, g, b] = rgb; + const hi = (v: number) => v > 150; + const lo = (v: number) => v < 100; + const colour: Colour = + hi(r) && lo(g) && lo(b) + ? "red" + : lo(r) && hi(g) && lo(b) + ? "lime" + : lo(r) && lo(g) && hi(b) + ? "blue" + : hi(r) && lo(g) && hi(b) + ? "magenta" + : "other"; + return { colour, rgb }; +} + +async function waitForCentre(page: Page, want: Colour, timeout: number): Promise { + const deadline = Date.now() + timeout; + let last = await centre(page); + while (last.colour !== want && Date.now() < deadline) { + await page.waitForTimeout(250); + last = await centre(page); + } + check(last.colour === want, `centre is ${want}`, () => `got ${last.colour} rgb(${last.rgb.join(", ")})`); +} + +// Render plane waves through the ambisonic decoder offline and return each ear's RMS level. +async function ears(page: Page, direction: number[], yaw: number): Promise<{ L: number; R: number }> { + return page.evaluate( + async ({ direction, yaw }) => { + const { Spatial } = await import("/audio-check.js"); + const rate = 48_000; + const context = new OfflineAudioContext(2, rate, rate); + const buffer = context.createBuffer(4, rate, rate); + const [x, y, z] = direction; + // A 1 kHz plane wave encoded in ACN W, Y, Z, X with SN3D. + [1, y, z, x].forEach((gain, channel) => { + const data = buffer.getChannelData(channel); + for (let i = 0; i < data.length; i++) data[i] = 0.5 * gain * Math.sin((2 * Math.PI * 1000 * i) / rate); + }); + const discrete = { + channelCount: 4, + channelCountMode: "explicit", + channelInterpretation: "discrete", + } as const; + const source = new AudioBufferSourceNode(context, { buffer, ...discrete }); + const root = new GainNode(context, discrete); + source.connect(root); + const spatial = new Spatial(root, { kind: "ambisonic" }); + // The head yawed left by `yaw` about +Y, standing at the origin. + const head = { + decompose: ( + position: { set(x: number, y: number, z: number): void }, + rotation: { set(x: number, y: number, z: number, w: number): void }, + ) => { + position.set(0, 0, 0); + rotation.set(0, Math.sin(yaw / 2), 0, Math.cos(yaw / 2)); + }, + }; + spatial.pose(head); + source.start(); + const rendered = await context.startRendering(); + // Skip the first 100 ms, while the listener glides into place. + const rms = (channel: number) => { + const data = rendered.getChannelData(channel).subarray(rate / 10); + return Math.sqrt(data.reduce((sum, v) => sum + v * v, 0) / data.length); + }; + return { L: rms(0), R: rms(1) }; + }, + { direction, yaw }, + ); +} + +function louder(level: { L: number; R: number }, ear: "L" | "R" | "neither", assertion: string) { + const ratio = level.L / level.R; + const ok = ear === "L" ? ratio > 2 : ear === "R" ? ratio < 0.5 : ratio > 0.9 && ratio < 1.1; + check(ok, assertion, () => `L ${level.L.toFixed(3)} R ${level.R.toFixed(3)}`); +} + +const browser = await launch(["--use-angle=swiftshader", "--enable-unsafe-swiftshader"]); +let failed = true; +try { + const query = new URLSearchParams({ url: values.url, name: "vr.hang" }); + const [page, errors] = await open(browser, `${origin}/?${query}`, "vr", true, { viewport: VIEWPORT }); + + // Straight ahead is the left eye's centre: the catalog's stereo packing cropped the frame and the + // equirectangular projection faced its centre forward. + await waitForCentre(page, "red", 30_000); + + // Drag 90 degrees left (0.005 rad per pixel): a sphere wraps the left eye's left quarter there, + // where a flat screen would show only the background. + await page.mouse.move(VIEWPORT.width / 2, VIEWPORT.height / 2); + await page.mouse.down(); + await page.mouse.move(VIEWPORT.width / 2 + Math.PI / 2 / 0.005, VIEWPORT.height / 2, { steps: 20 }); + await page.mouse.up(); + await waitForCentre(page, "lime", 5_000); + + louder(await ears(page, [0, 1, 0], 0), "L", "a source on the left is louder in the left ear"); + louder(await ears(page, [0, 1, 0], Math.PI / 2), "neither", "turning left to face that source centres it"); + louder(await ears(page, [1, 0, 0], Math.PI / 2), "R", "a source ahead moves to the right ear after turning left"); + + throwPageErrors(errors); + console.log("vr: stereo equirectangular painted per eye, facing forward; ambisonics follow the head"); + failed = false; +} finally { + await finishTraces(failed); + await browser.close(); + server.stop(true); +} diff --git a/test/justfile b/test/justfile index 070c538f528..97f250ff005 100644 --- a/test/justfile +++ b/test/justfile @@ -115,6 +115,10 @@ smoke *args: portrait: ./portrait/run.sh +# Paint a stereo 360 broadcast per eye in demo/vr, and turn its ambisonic decode with the head. +vr: + ./vr/run.sh + # Negative control: no publisher, every subscriber must time out (proves the # harness can actually report failure). diff --git a/test/vr/relay.toml b/test/vr/relay.toml new file mode 100644 index 00000000000..e9a52bad3ca --- /dev/null +++ b/test/vr/relay.toml @@ -0,0 +1,15 @@ +# Private loopback relay for the VR projection harness. +[log] +level = "info" + +[listen] +bind = "127.0.0.1:4443" +tls.generate = ["localhost", "127.0.0.1"] +# The CLI publisher negotiates lite-07; Chromium offers lite-06. +version = ["moq-lite-07-wip", "moq-lite-06"] + +[web.http] +listen = "127.0.0.1:4443" + +[auth] +public = "**" diff --git a/test/vr/run.sh b/test/vr/run.sh new file mode 100755 index 00000000000..9818475344c --- /dev/null +++ b/test/vr/run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Publish a stereo 360 picture with VR catalog flags and check demo/vr paints each eye the right way round. +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then + exec bash "$(dirname "$0")/../../sh/local.sh" bash "$0" "$@" +fi +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +harness_begin vr 'just test vr' +cd "$WORKSPACE" +cargo build --locked -p moq-relay -p moq-cli +uv run --no-project python sh/js/install.py frozen +(cd demo/vr && bunx vite build --outDir "$HARNESS_RUN/vr-dist" --emptyOutDir) +# The ambisonic decoder alone, so the driver can render it offline without a stream. +bun build demo/vr/src/audio.ts --format esm --outfile "$HARNESS_RUN/vr-dist/audio-check.js" +harness_port relay +URL="http://127.0.0.1:$HARNESS_PORT" +sed "s/4443/$HARNESS_PORT/g" "$DIR/relay.toml" >"$HARNESS_RUN/relay.toml" +harness_spawn relay "$HARNESS_RUN/relay.log" "$WORKSPACE/target/debug/moq-relay" "$HARNESS_RUN/relay.toml" +harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID" +# Left eye: blue, red straight ahead (u 0.5), lime 90 degrees left (u 0.25). Right eye: magenta, +# so a frame shown uncropped puts magenta straight ahead. +# shellcheck disable=SC2329 # invoked by harness_spawn +publish_vr() { + ffmpeg -hide_banner -loglevel error -re -f lavfi \ + -i "color=c=blue:s=960x480:r=15,drawbox=x=210:y=210:w=60:h=60:c=red:t=fill,drawbox=x=90:y=210:w=60:h=60:c=lime:t=fill,drawbox=x=480:y=0:w=480:h=480:c=magenta:t=fill" \ + -c:v libx264 -preset ultrafast -tune zerolatency -profile:v baseline -pix_fmt yuv420p -g 15 -f h264 - | + "$WORKSPACE/target/debug/moq" --connect "$URL" --connect-version moq-lite-07-wip --broadcast vr.hang \ + import --projection equirectangular --stereo left-right avc3 +} +harness_spawn publisher "$HARNESS_RUN/publisher.log" publish_vr +cd test/interop/clients/js +bun vr.ts --url "$URL" From 7beefa69655203f376aa4200cdb2ab2ea765961a Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Fri, 13 Nov 2026 10:22:37 -0700 Subject: [PATCH 02/13] chore(demo): lock the vr demo's dependencies Co-Authored-By: Claude Opus 5.5 --- bun.lock | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/bun.lock b/bun.lock index 2f51f8c5b49..51d9ab390de 100644 --- a/bun.lock +++ b/bun.lock @@ -72,6 +72,19 @@ "vite": "^8.3.6", }, }, + "demo/vr": { + "name": "@moq/demo-vr", + "dependencies": { + "@moq/watch": "workspace:^", + "three": "^0.186.1", + }, + "devDependencies": { + "@types/three": "^0.186.0", + "rolldown": "^1.2.16", + "typescript": "7.0.2", + "vite": "^8.3.5", + }, + }, "demo/web": { "name": "@moq/demo", "version": "0.1.1", @@ -685,6 +698,8 @@ "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + "@dimforge/rapier3d-compat": ["@dimforge/rapier3d-compat@0.12.0", "", {}, "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow=="], + "@discoveryjs/json-ext": ["@discoveryjs/json-ext@1.1.0", "", {}, "sha512-Xc3VhU02wqZ1HvHRJUwL09HkZSTvidqY5Ya0NXBSYOxAp+Ln9dcJr9fySI+CkONzP3PekQo9WdzCv0PGER/mOA=="], "@docsearch/css": ["@docsearch/css@3.8.2", "", {}, "sha512-y05ayQFyUmCXze79+56v/4HpycYF3uFqB78pLPrSV5ZKAlDuIAAJNhaRi8tTdRNXh05yxX/TyNnzD6LwSM89vQ=="], @@ -847,6 +862,8 @@ "@moq/demo-carla": ["@moq/demo-carla@workspace:demo/carla/web"], + "@moq/demo-vr": ["@moq/demo-vr@workspace:demo/vr"], + "@moq/drain-test": ["@moq/drain-test@workspace:test/drain"], "@moq/hang": ["@moq/hang@workspace:js/hang"], @@ -1069,6 +1086,8 @@ "@tybys/wasm-util": ["@tybys/wasm-util@0.10.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg=="], + "@tweenjs/tween.js": ["@tweenjs/tween.js@23.1.3", "", {}, "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA=="], + "@types/audioworklet": ["@types/audioworklet@0.0.100", "", {}, "sha512-mWt4Z5CWi0CupDnSYWJgx6dw61XUk9JSRGrM9F/YNrpOoY/0XG+sxP+8rAjQq80H4febOe8PrzGmmN/X3w8LsQ=="], "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], @@ -1115,14 +1134,20 @@ "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], + "@types/stats.js": ["@types/stats.js@0.17.4", "", {}, "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA=="], + "@types/supports-color": ["@types/supports-color@8.1.3", "", {}, "sha512-Hy6UMpxhE3j1tLpl27exp1XqHD7n8chAiNPzWfz16LPZoMMoSc4dzLl6w9qijkEb/r5O1ozdu1CWGA2L83ZeZg=="], "@types/text-table": ["@types/text-table@0.2.5", "", {}, "sha512-hcZhlNvMkQG/k1vcZ6yHOl6WAYftQ2MLfTHcYRZ2xYZFD8tGVnE3qFV0lj1smQeDSR7/yY0PyuUalauf33bJeA=="], + "@types/three": ["@types/three@0.186.0", "", { "dependencies": { "@dimforge/rapier3d-compat": "~0.12.0", "@tweenjs/tween.js": "~23.1.3", "@types/stats.js": "*", "@types/webxr": ">=0.5.17", "fflate": "~0.8.3", "meshoptimizer": "~1.1.1" } }, "sha512-mxYSBpDC+D0pLfSP6sW4WZTcT+nrtmZcimMqnVmy36Hte3XpeYSrvgg4TRdaM1GemGog1AWzI5qL2VoIfMXbJQ=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/web-bluetooth": ["@types/web-bluetooth@0.0.21", "", {}, "sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA=="], + "@types/webxr": ["@types/webxr@0.5.24", "", {}, "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg=="], + "@types/whatwg-mimetype": ["@types/whatwg-mimetype@3.0.2", "", {}, "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA=="], "@types/ws": ["@types/ws@8.18.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w=="], @@ -1393,6 +1418,8 @@ "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="], + "flat": ["flat@5.0.2", "", { "bin": { "flat": "cli.js" } }, "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ=="], "focus-trap": ["focus-trap@7.8.0", "", { "dependencies": { "tabbable": "^6.4.0" } }, "sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA=="], @@ -1577,6 +1604,8 @@ "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], + "meshoptimizer": ["meshoptimizer@1.1.1", "", {}, "sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g=="], + "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="], @@ -1895,6 +1924,8 @@ "text-table": ["text-table@0.2.0", "", {}, "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw=="], + "three": ["three@0.186.1", "", {}, "sha512-blFeqb49wRCSGUGj7gtpfnSGHy2lwDk94RhUmS1c/hTby70kvChbWpkJ4Pm1390LqzzvTmzgXKHPEafJwCb8jA=="], + "tinyexec": ["tinyexec@1.3.0", "", {}, "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ=="], "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], From 901d1e7973ee86a0a82287ecc3b2192c8a83a532 Mon Sep 17 00:00:00 2001 From: nuts_rice Date: Fri, 13 Nov 2026 10:44:27 -0700 Subject: [PATCH 03/13] feat(demo): add vr content recenter --- demo/vr/src/scene.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/demo/vr/src/scene.ts b/demo/vr/src/scene.ts index fb427b861a9..acc89c55f34 100644 --- a/demo/vr/src/scene.ts +++ b/demo/vr/src/scene.ts @@ -212,7 +212,11 @@ export class Scene { // Turn the content to face wherever the viewer is looking. #recentre = () => { const head = this.#head(); - // TODO(human) + head.getWorldPosition(this.#headPosition); + head.getWorldDirection(this.#headDirection); + const yaw = Math.atan2(-this.#headDirection.x, -this.#headDirection.z); + this.#content.rotation.set(0, yaw, 0); +this.#content.position.set(this.#headPosition.x, 0, this.#headPosition.z); this.#content.updateMatrixWorld(); }; From 7341acd0705b67cf3a527333b736b471ad32b6e4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 16:57:46 -0800 Subject: [PATCH 04/13] refactor(demo): drop the VR demo's catalog flags, which import already fills #7308 fills projection, stereo, and the ambisonic layout from MP4 st3d/sv3d, Matroska, and Opus family 2, so the CLI flags and moq_mux::catalog::Spatial duplicate it. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 14 --- rs/hang/src/catalog/audio/layout.rs | 2 +- rs/hang/src/catalog/video/projection.rs | 2 +- rs/moq-cli/src/args.rs | 141 ------------------------ rs/moq-cli/src/main.rs | 2 +- rs/moq-mux/src/catalog/mod.rs | 2 - rs/moq-mux/src/catalog/producer.rs | 75 ------------- rs/moq-mux/src/catalog/spatial.rs | 43 -------- rs/moq-mux/src/catalog/tracks.rs | 17 +-- 9 files changed, 4 insertions(+), 294 deletions(-) delete mode 100644 rs/moq-mux/src/catalog/spatial.rs diff --git a/doc/bin/cli.md b/doc/bin/cli.md index dfbe2576f36..eaa99666881 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -108,20 +108,6 @@ MP4 parser. `import fmp4 --cmaf` publishes each fragment whole as [`cmaf`](/concept/hang#container) instead, keeping the source's exact sample timing. -The stdin imports can mark the broadcast for VR when the source does not: -`--projection equirectangular` for 360 video (`equirectangular:180` for VR180), -`--stereo left-right` or `top-bottom` for a picture that packs both eyes, and -`--audio-layout ambisonic` for AmbiX audio. fMP4 carrying Spherical Video V2 -boxes and Opus with channel mapping family 2 already say so, and win over the -flags, which matters mostly for bare bitstreams like `avc3`. An ambisonic track -without a square channel count of at least 4 is refused. - -```bash -ffmpeg -re -i vr180.mp4 -an -c:v copy -bsf:v h264_mp4toannexb -f h264 - | - moq --connect https://relay.example.com/anon --broadcast vr.hang \ - import --projection equirectangular:180 --stereo left-right avc3 -``` - `import rtsp` pulls an IP camera's H.264 or H.265 video and AAC audio without re-encoding. RTP rides the RTSP connection, so run it on the camera's network and the camera needs no inbound port. Credentials go in the URL diff --git a/rs/hang/src/catalog/audio/layout.rs b/rs/hang/src/catalog/audio/layout.rs index 860b7f3a1ad..82a7201e60f 100644 --- a/rs/hang/src/catalog/audio/layout.rs +++ b/rs/hang/src/catalog/audio/layout.rs @@ -22,7 +22,7 @@ pub enum AudioLayout { impl AudioLayout { /// Refuse a channel count this layout cannot carry. - pub fn check(&self, channel_count: u32) -> crate::Result<()> { + pub(crate) fn check(&self, channel_count: u32) -> crate::Result<()> { if *self == Self::Ambisonic { let order = channel_count.isqrt(); if order < 2 || order * order != channel_count { diff --git a/rs/hang/src/catalog/video/projection.rs b/rs/hang/src/catalog/video/projection.rs index 2b5c634ec9a..5a038995632 100644 --- a/rs/hang/src/catalog/video/projection.rs +++ b/rs/hang/src/catalog/video/projection.rs @@ -47,7 +47,7 @@ impl UnknownProjection { impl Projection { /// Refuse a field of view outside (0, 360] degrees. - pub fn check(&self) -> crate::Result<()> { + pub(crate) fn check(&self) -> crate::Result<()> { if let Self::Equirectangular { fov: Some(fov) } = self && !(*fov > 0.0 && *fov <= 360.0) { diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 5a12f91c897..ac0dcdf6bd6 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -360,14 +360,6 @@ impl Invocation { { anyhow::bail!("--linger keeps a cleanly ended broadcast up, but {reason}"); } - if let Command::Import(import) = command - && import.spatial() != moq_mux::catalog::Spatial::default() - && import.source.stdin_format().is_none() - { - anyhow::bail!( - "--projection, --stereo, and --audio-layout stamp the catalog a stdin import builds, which this source does not" - ); - } let Command::Export(export) = command else { continue; }; @@ -833,21 +825,6 @@ pub struct Import { #[usage(long, value_name = "PATH")] pub echo_path: Option, - /// How each video picture maps onto the viewer's view: `rectilinear`, or `equirectangular` - /// with an optional `:fov` in degrees (`equirectangular:180` for VR180). The stdin imports only. - #[usage(long)] - pub projection: Option, - - /// How each video picture packs both eyes: `mono`, `left-right`, or `top-bottom`. The stdin - /// imports only. - #[usage(long)] - pub stereo: Option, - - /// What each audio track's channels carry: `ambisonic` for AmbiX, which needs a square channel - /// count of at least 4. The stdin imports only. - #[usage(long)] - pub audio_layout: Option, - /// The single source feeding the Origin. #[usage(subcommand)] pub source: ImportSource, @@ -868,71 +845,6 @@ impl Import { } found } - - /// The spatial metadata the flags stamp onto each rendition. - pub fn spatial(&self) -> moq_mux::catalog::Spatial { - moq_mux::catalog::Spatial { - projection: self.projection.clone().map(|arg| arg.0), - stereo: self.stereo.clone().map(|arg| arg.0), - layout: self.audio_layout.clone().map(|arg| arg.0), - } - } -} - -/// An `import --projection` value. -#[derive(Clone, Debug, PartialEq)] -pub struct ProjectionArg(pub hang::catalog::Projection); - -impl std::str::FromStr for ProjectionArg { - type Err = String; - - fn from_str(arg: &str) -> Result { - use hang::catalog::Projection; - - let projection = match arg.split_once(':') { - None if arg == "rectilinear" => Projection::Rectilinear, - None if arg == "equirectangular" => Projection::Equirectangular { fov: None }, - Some(("equirectangular", fov)) => { - let fov = fov.parse().map_err(|_| format!("expected a field of view in degrees, got `{fov}`"))?; - Projection::Equirectangular { fov: Some(fov) } - } - _ => return Err(format!("expected `rectilinear` or `equirectangular[:fov]`, got `{arg}`")), - }; - projection.check().map_err(|err| err.to_string())?; - Ok(Self(projection)) - } -} - -/// An `import --stereo` value. -#[derive(Clone, Debug, PartialEq)] -pub struct StereoArg(pub hang::catalog::Stereo); - -impl std::str::FromStr for StereoArg { - type Err = String; - - fn from_str(arg: &str) -> Result { - match arg.parse() { - Ok(hang::catalog::Stereo::Unknown(_)) | Err(_) => { - Err(format!("expected `mono`, `left-right`, or `top-bottom`, got `{arg}`")) - } - Ok(stereo) => Ok(Self(stereo)), - } - } -} - -/// An `import --audio-layout` value. -#[derive(Clone, Debug, PartialEq)] -pub struct AudioLayoutArg(pub hang::catalog::AudioLayout); - -impl std::str::FromStr for AudioLayoutArg { - type Err = String; - - fn from_str(arg: &str) -> Result { - match arg.parse() { - Ok(hang::catalog::AudioLayout::Unknown(_)) | Err(_) => Err(format!("expected `ambisonic`, got `{arg}`")), - Ok(layout) => Ok(Self(layout)), - } - } } /// The single source feeding the Origin on an import. The container formats read @@ -1968,59 +1880,6 @@ mod tests { } } - /// The spatial flags parse into the catalog's types, refuse what it can't name, and only - /// stamp a catalog a stdin import builds. - #[test] - fn spatial_flags_parse_and_need_a_stdin_import() { - use hang::catalog::{AudioLayout, Projection, Stereo}; - - let import = |flags: &[&str], source: &[&str]| { - let argv = ["moq", "--connect", "http://relay", "--broadcast", "room", "import"]; - Invocation::try_parse_from(argv.iter().chain(flags).chain(source).copied()) - }; - let flags = [ - "--projection", - "equirectangular:180", - "--stereo", - "left-right", - "--audio-layout", - "ambisonic", - ]; - let invocation = import(&flags, &["fmp4"]).unwrap(); - invocation.validate().unwrap(); - let Command::Import(parsed) = &invocation.stages[0] else { - panic!("expected an import stage"); - }; - assert_eq!( - parsed.spatial(), - moq_mux::catalog::Spatial { - projection: Some(Projection::Equirectangular { fov: Some(180.0) }), - stereo: Some(Stereo::LeftRight), - layout: Some(AudioLayout::Ambisonic), - } - ); - - for value in ["rectilinear", "equirectangular"] { - import(&["--projection", value], &["avc3"]).unwrap().validate().unwrap(); - } - for flag in [ - ["--projection", "equirectangular:0"], - ["--projection", "equirectangular:361"], - ["--projection", "cubemap"], - ["--stereo", "interleaved"], - ["--audio-layout", "5.1"], - ] { - assert!(import(&flag, &["fmp4"]).is_err(), "{flag:?}"); - } - - let err = import(&["--stereo", "left-right"], &["hls", "http://origin/master.m3u8"]) - .unwrap() - .validate() - .unwrap_err() - .to_string(); - assert!(err.contains("--stereo"), "{err}"); - } - /// `--echo` reports what a stdout container export reads from a hang catalog, under one /// path segment, and the report sinks read no media to report on. #[test] diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 88f615bb613..191ad854204 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -779,7 +779,7 @@ fn spawn_import( if let Some(format) = import.source.stdin_format() { warn_if_missing_format(&name); - let config = ingest.catalog().with_spatial(import.spatial()); + let config = ingest.catalog(); let mut publish = if let ImportSource::Ts(TsImport { program: Some(TsProgram::All), .. diff --git a/rs/moq-mux/src/catalog/mod.rs b/rs/moq-mux/src/catalog/mod.rs index 76d102ecb73..f19733d5786 100644 --- a/rs/moq-mux/src/catalog/mod.rs +++ b/rs/moq-mux/src/catalog/mod.rs @@ -32,7 +32,6 @@ mod format; pub(crate) mod interval; mod producer; mod select; -mod spatial; mod stream; mod timebase; pub(crate) mod tracks; @@ -51,7 +50,6 @@ pub use format::*; pub(crate) use producer::to_msf; pub use producer::{Config, Guard, Producer}; pub use select::Select; -pub use spatial::Spatial; pub use stream::Stream; pub(crate) use timebase::Offset; pub use timebase::Timebase; diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 58c45062a61..c0f92185124 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -373,8 +373,6 @@ pub struct Producer { order: super::estimate::Order, /// What each media track minted under this catalog has written. See [`stats`](Self::stats). sent: Arc>, - /// Stamped onto each rendition's config. See [`Config::with_spatial`]. - spatial: Arc, } // Manual Clone so a producer is cheaply clonable regardless of whether `E` is. @@ -388,7 +386,6 @@ impl Clone for Producer { bandwidth: self.bandwidth.clone(), order: self.order.clone(), sent: self.sent.clone(), - spatial: self.spatial.clone(), } } } @@ -406,7 +403,6 @@ pub struct Config { max_age: Option, bandwidth: moq_net::bandwidth::Allocator, clock: Option, - spatial: super::Spatial, } impl Default for Config<()> { @@ -416,7 +412,6 @@ impl Default for Config<()> { max_age: Some(hang::container::MAX_AGE), bandwidth: moq_net::bandwidth::Allocator::unlimited(), clock: None, - spatial: super::Spatial::default(), } } } @@ -432,7 +427,6 @@ impl Config { max_age: self.max_age, bandwidth: self.bandwidth, clock: self.clock, - spatial: self.spatial, } } @@ -479,15 +473,6 @@ impl Config { self.bandwidth = bandwidth; self } - - /// Stamp `spatial` onto every video and audio rendition whose config leaves those fields unset. - /// - /// A rendition it contradicts, such as an ambisonic layout on a stereo track, is refused when - /// its config is set. - pub fn with_spatial(mut self, spatial: super::Spatial) -> Self { - self.spatial = spatial; - self - } } impl Producer { @@ -547,7 +532,6 @@ impl Producer { bandwidth: config.bandwidth, order: Default::default(), sent: Default::default(), - spatial: Arc::new(config.spatial), }) } @@ -976,11 +960,6 @@ impl Producer { super::Estimator::with_order(self.order.clone()) } - /// What each rendition's config is stamped with. See [`Config::with_spatial`]. - pub(crate) fn spatial(&self) -> &super::Spatial { - &self.spatial - } - /// The allocator passthrough tracks claim on. fMP4 writes groups by hand, so it reads this itself. pub(crate) fn bandwidth(&self) -> moq_net::bandwidth::Allocator { self.bandwidth.clone() @@ -3702,58 +3681,4 @@ mod test { let track = &msf_wire(&catalog).tracks[0]; assert_eq!((track.bitrate, track.delay), (Some(1_000_000), None)); } - - #[test] - fn spatial_fills_only_what_a_rendition_leaves_unset() { - use hang::catalog::{AudioLayout, Projection, Stereo}; - - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let spatial = super::super::Spatial { - projection: Some(Projection::Equirectangular { fov: Some(180.0) }), - stereo: Some(Stereo::LeftRight), - layout: Some(AudioLayout::Ambisonic), - }; - let catalog = Producer::new(&mut broadcast, Config::default().with_spatial(spatial)).unwrap(); - - let stamped = VideoConfig::new(hang::catalog::VideoCodec::VP8); - let mut declared = VideoConfig::new(hang::catalog::VideoCodec::VP8); - declared.stereo = Some(Stereo::TopBottom); - let audio = AudioConfig::new(AudioCodec::Opus, 48_000, 4); - - let mut keep = Vec::new(); - for (name, config) in [("stamped", stamped), ("declared", declared)] { - let track = broadcast.create_track(name, None).unwrap(); - let wire = super::super::hang::Container::try_from(&config).unwrap(); - keep.push(catalog.video(track, wire, config).unwrap()); - } - let track = broadcast.create_track("ambisonic", None).unwrap(); - let wire = super::super::hang::Container::try_from(&audio).unwrap(); - let _audio = catalog.audio(track, wire, audio).unwrap(); - - let snapshot = catalog.snapshot(); - let stamped = &snapshot.video.renditions["stamped"]; - assert_eq!(stamped.projection, Some(Projection::Equirectangular { fov: Some(180.0) })); - assert_eq!(stamped.stereo, Some(Stereo::LeftRight)); - assert_eq!(snapshot.video.renditions["declared"].stereo, Some(Stereo::TopBottom)); - assert_eq!(snapshot.audio.renditions["ambisonic"].layout, Some(AudioLayout::Ambisonic)); - } - - #[test] - fn spatial_refuses_a_rendition_it_contradicts() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let spatial = super::super::Spatial { - layout: Some(hang::catalog::AudioLayout::Ambisonic), - ..Default::default() - }; - let catalog = Producer::new(&mut broadcast, Config::default().with_spatial(spatial)).unwrap(); - - let stereo = AudioConfig::new(AudioCodec::Opus, 48_000, 2); - let track = broadcast.create_track("stereo", None).unwrap(); - let wire = super::super::hang::Container::try_from(&stereo).unwrap(); - let Err(err) = catalog.audio(track, wire, stereo) else { - panic!("an ambisonic stereo rendition was accepted"); - }; - assert!(matches!(err, crate::Error::Hang(hang::Error::InvalidAmbisonicChannels(2)))); - assert!(catalog.snapshot().audio.renditions.is_empty()); - } } diff --git a/rs/moq-mux/src/catalog/spatial.rs b/rs/moq-mux/src/catalog/spatial.rs deleted file mode 100644 index a16f0a90994..00000000000 --- a/rs/moq-mux/src/catalog/spatial.rs +++ /dev/null @@ -1,43 +0,0 @@ -use hang::catalog::{AudioLayout, Projection, Stereo}; - -/// How the broadcast's media fills the space around the viewer, stamped onto each video and -/// audio rendition whose own config leaves the field unset. -/// -/// The bitstreams the importers read carry none of this, so the publisher declares it. Set it -/// with [`Config::with_spatial`](super::Config::with_spatial). -#[derive(Clone, Debug, Default, PartialEq)] -pub struct Spatial { - /// How each video picture maps onto the viewer's field of view. - pub projection: Option, - - /// How each video picture packs the views of both eyes. - pub stereo: Option, - - /// What each audio rendition's decoded channels carry. - pub layout: Option, -} - -impl Spatial { - pub(super) fn video(&self, config: &mut hang::catalog::VideoConfig) -> crate::Result<()> { - if config.projection.is_none() { - config.projection.clone_from(&self.projection); - } - if config.stereo.is_none() { - config.stereo.clone_from(&self.stereo); - } - if let Some(projection) = &config.projection { - projection.check()?; - } - Ok(()) - } - - pub(super) fn audio(&self, config: &mut hang::catalog::AudioConfig) -> crate::Result<()> { - if config.layout.is_none() { - config.layout.clone_from(&self.layout); - } - if let Some(layout) = &config.layout { - layout.check(config.channel_count)?; - } - Ok(()) - } -} diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index a9786ec957f..945a2a80723 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -3,7 +3,6 @@ use std::time::Duration; use super::Estimate; use super::Producer; -use super::Spatial; use super::hang::{Catalog, CatalogExt}; /// A catalog config that can be published as a named rendition. @@ -112,12 +111,6 @@ pub trait RenditionConfig: Clone + Send + 'static { /// Replace the config's [`Estimate`] fields. Defaults to discarding them. fn set_estimate(&mut self, _estimate: Estimate) {} - - /// Fill the fields of [`Spatial`] this config leaves unset, refusing a result it can't carry. - /// Defaults to ignoring it. - fn spatial(&mut self, _spatial: &Spatial) -> crate::Result<()> { - Ok(()) - } } impl RenditionConfig for hang::catalog::JsonConfig { @@ -329,9 +322,6 @@ impl RenditionConfig for hang::catalog::VideoConfig { self.delay = estimate.delay; self.framerate = estimate.framerate; } - fn spatial(&mut self, spatial: &Spatial) -> crate::Result<()> { - spatial.video(self) - } } impl RenditionConfig for hang::catalog::AudioConfig { @@ -360,9 +350,6 @@ impl RenditionConfig for hang::catalog::AudioConfig { self.bitrate = estimate.bitrate; self.delay = estimate.delay; } - fn spatial(&mut self, spatial: &Spatial) -> crate::Result<()> { - spatial.audio(self) - } } impl RenditionConfig for hang::catalog::TextConfig { @@ -695,7 +682,6 @@ impl> Rendition { Self::check_decrease(&published, &resolved)?; } config.set_estimate(resolved.clone()); - config.spatial(self.catalog.spatial())?; { let mut guard = self.catalog.modify()?; write(&mut guard, &self.name, config)?; @@ -840,11 +826,10 @@ impl> Rendition { } /// Write `config` into the catalog, leaving the caller to choose how the guard publishes it. - fn stage(&mut self, mut config: C) -> crate::Result> { + fn stage(&mut self, config: C) -> crate::Result> { if !self.present { return Err(self.unpublished()); } - config.spatial(self.catalog.spatial())?; Self::check_decrease(&self.config()?.estimate(), &config.estimate())?; let mut guard = self.catalog.modify()?; write(&mut guard, &self.name, config)?; From 0dfb627fba401cd771e1cf90483570c6bdf5c17f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:04:18 -0800 Subject: [PATCH 05/13] fix(watch): pass the OpusHead to WebCodecs so multichannel Opus decodes The decoder and its support probe dropped every Opus description, so Chromium refused anything past stereo: "description is required for multi-channel Opus". The catalog's description is the WebCodecs one, an OpusHead, which the Rust importers republish so decoders apply its channel mapping. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.ts | 20 +++++--------------- js/watch/src/audio/supported.ts | 30 ++++++++++++++---------------- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 07fcd4ef723..fb8df7e7ace 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -687,13 +687,8 @@ export class Decoder { this.#ring.peek()?.dropped(); }); - // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header. - const description = - config.codec === "opus" - ? undefined - : config.description - ? Util.Hex.toBytes(config.description) - : undefined; + // For Opus this is the OpusHead, whose channel mapping a stream past stereo cannot decode without. + const description = config.description ? Util.Hex.toBytes(config.description) : undefined; const decoderConfig: AudioDecoderConfig = { codec: config.codec, sampleRate: config.sampleRate, @@ -759,14 +754,9 @@ export class Decoder { .min(DELAY_MAX) .asMicros(), ); - // Opus in CMAF uses raw packets (not OGG-wrapped), so description must be omitted. - // The dOps box from the init segment is not a valid OGG Identification Header. - const description = - config.codec === "opus" - ? undefined - : config.description - ? Util.Hex.toBytes(config.description) - : init.description; + // The init segment's description stands in for a catalog without one. It never carries Opus's + // dOps, which is not the OpusHead WebCodecs takes. + const description = config.description ? Util.Hex.toBytes(config.description) : init.description; const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); diff --git a/js/watch/src/audio/supported.ts b/js/watch/src/audio/supported.ts index 81cd8cb0a80..d6054050b51 100644 --- a/js/watch/src/audio/supported.ts +++ b/js/watch/src/audio/supported.ts @@ -21,23 +21,21 @@ export async function supported(config: Catalog.AudioConfig): Promise { console.warn(`audio: opus advertised at ${config.sampleRate}Hz, which some browsers cannot decode`); } - // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header. + // For Opus this is the OpusHead; the init segment never yields Opus's dOps in its place. let description: Uint8Array | undefined; - if (config.codec !== "opus") { - if (config.description) { - description = Util.Hex.toBytes(config.description); - } else if (config.container.kind === "cmaf") { - try { - const Cmaf = await import("@moq/hang/cmaf"); - description = Cmaf.decodeInitSegment(base64ToBytes(config.container.init)).description; - } catch (err) { - // A malformed init segment means we can't extract the codec - // description, so we can't probe support reliably. Reject the - // track rather than letting isConfigSupported pass on a - // description-less config and then having decode() fail later. - console.warn(`audio: malformed CMAF init segment for codec ${config.codec}`, err); - return false; - } + if (config.description) { + description = Util.Hex.toBytes(config.description); + } else if (config.container.kind === "cmaf") { + try { + const Cmaf = await import("@moq/hang/cmaf"); + description = Cmaf.decodeInitSegment(base64ToBytes(config.container.init)).description; + } catch (err) { + // A malformed init segment means we can't extract the codec + // description, so we can't probe support reliably. Reject the + // track rather than letting isConfigSupported pass on a + // description-less config and then having decode() fail later. + console.warn(`audio: malformed CMAF init segment for codec ${config.codec}`, err); + return false; } } // Install the WebCodecs polyfill before probing, since playback would decode through it where From c94db0a865a68e0cd2e7be5d7f0b3a68157cb028 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:04:18 -0800 Subject: [PATCH 06/13] test(vr): import a VR180 ambisonic MP4 and check demo/vr lays it out by the catalog The harness writes st3d and sv3d into an ffmpeg-made MP4 with family 2 Opus, publishes it through moq import fmp4 and a relay, and asserts demo/vr reads projection, fov, stereo, and layout and applies each: the left eye ahead, its quarter 45 degrees left, nothing past the 180 degree edge, and a sound field that turns with the head. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/interop.yml | 4 +- demo/vr/src/audio.ts | 5 + demo/vr/src/index.ts | 3 + test/README.md | 2 +- test/interop/clients/js/vr.ts | 181 +++++++++++++++++++++------------- test/justfile | 2 +- test/vr/run.sh | 31 +++--- test/vr/spherical.py | 77 +++++++++++++++ 8 files changed, 222 insertions(+), 83 deletions(-) create mode 100644 test/vr/spherical.py diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index b2406aa83b1..a66cab4b0a0 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -160,7 +160,8 @@ jobs: HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs HARNESS_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl - - name: VR projection + - name: VR catalog fields + id: vr if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just test vr shell: bash -leo pipefail {0} @@ -244,6 +245,7 @@ jobs: "Interop (moq-transport)": "${{ steps.interop-ietf.outcome }}", "Released wire compatibility": "${{ steps.wire-compat.outcome }}", "Portrait transcode": "${{ steps.portrait.outcome }}", + "VR catalog fields": "${{ steps.vr.outcome }}", "Interop negative control": "${{ steps.negative.outcome }}", "Media output and lifecycle": "${{ steps.media.outcome }}", "Media track identity changes": "${{ steps.media-rename.outcome }}" diff --git a/demo/vr/src/audio.ts b/demo/vr/src/audio.ts index 8338da08ba1..47b785aebd0 100644 --- a/demo/vr/src/audio.ts +++ b/demo/vr/src/audio.ts @@ -91,6 +91,11 @@ export class Spatial { return panner; } + // The mix every placement feeds, for meters and the harness. + get output(): AudioNode { + return this.#out; + } + set muted(muted: boolean) { this.#out.gain.setTargetAtTime(muted ? 0 : 1, this.#context.currentTime, 0.01); } diff --git a/demo/vr/src/index.ts b/demo/vr/src/index.ts index 064f02b39cb..db538ff05d0 100644 --- a/demo/vr/src/index.ts +++ b/demo/vr/src/index.ts @@ -16,6 +16,9 @@ const video = new Video.Decoder({ source: videoSrc, sync, enabled: true }); const scene = new Scene(document.body); const effect = new Signals.Effect(); +// The console and test/vr read the catalog and the scene through this. +Object.assign(window, { vr: { audio: audioSrc, video: videoSrc, scene } }); + // clone the frame once decoder gets it out effect.run((e) => { const frame = e.get(video.out.frame); diff --git a/test/README.md b/test/README.md index 81fd026b353..ab9dc98c219 100644 --- a/test/README.md +++ b/test/README.md @@ -20,7 +20,7 @@ questions to a harness. For builds and CI selection, see the | [drain](drain/README.md) | `just test drain` | a JS viewer migrates off a draining relay without a dropped group | | [stream-credit](stream-credit/README.md) | `just test stream-credit` | a browser tab reads 300 live tracks past its ~100-stream WebTransport cap, the highest priority first, and nothing hangs | | qmux | `just test qmux` | `@moq/qmux`, as client and as server, interoperates with the Rust qmux 0.6.3 release on every QMux draft | -| vr | `just test vr` | `demo/vr` paints a stereo 360 broadcast per eye facing forward, and its ambisonic decode follows the head | +| vr | `just test vr` | a VR180 left-right MP4 with ambisonic Opus keeps its catalog projection, stereo, and layout through import and a relay, and `demo/vr` lays out each eye and turns the sound field with the head | | video | `just test video` | the `@moq/video` renderer paints through WebGPU and falls back to Canvas2D after losing the GPU | | [json-properties](json-properties.md) | `just test json-properties` | both Snapshot codecs reconstruct, and both Window codecs emit the exact events for, random JSON sequences with replayable seeds | diff --git a/test/interop/clients/js/vr.ts b/test/interop/clients/js/vr.ts index 85fce6e0617..dded439305d 100644 --- a/test/interop/clients/js/vr.ts +++ b/test/interop/clients/js/vr.ts @@ -22,7 +22,16 @@ const server = Bun.serve({ const origin = `http://localhost:${server.port}`; const VIEWPORT = { width: 640, height: 360 }; -type Colour = "red" | "lime" | "blue" | "magenta" | "other"; +// Radians the demo turns the camera per dragged pixel. +const DRAG = 0.005; +type Colour = "red" | "lime" | "blue" | "magenta" | "dark" | "other"; + +// What demo/vr puts on `window.vr`, as far as this reads it. +type Demo = { + video: { out: { config: { peek(): unknown } } }; + audio: { out: { config: { peek(): unknown } } }; + scene: { listener?: { output: AudioNode } }; +}; // The colour at the centre of the page, read from a screenshot: WebGL discards its drawing buffer after // each frame, so the canvas itself reads back blank. @@ -43,15 +52,17 @@ async function centre(page: Page): Promise<{ colour: Colour; rgb: number[] }> { const hi = (v: number) => v > 150; const lo = (v: number) => v < 100; const colour: Colour = - hi(r) && lo(g) && lo(b) - ? "red" - : lo(r) && hi(g) && lo(b) - ? "lime" - : lo(r) && lo(g) && hi(b) - ? "blue" - : hi(r) && lo(g) && hi(b) - ? "magenta" - : "other"; + r < 40 && g < 40 && b < 40 + ? "dark" + : hi(r) && lo(g) && lo(b) + ? "red" + : lo(r) && hi(g) && lo(b) + ? "lime" + : lo(r) && lo(g) && hi(b) + ? "blue" + : hi(r) && lo(g) && hi(b) + ? "magenta" + : "other"; return { colour, rgb }; } @@ -65,83 +76,115 @@ async function waitForCentre(page: Page, want: Colour, timeout: number): Promise check(last.colour === want, `centre is ${want}`, () => `got ${last.colour} rgb(${last.rgb.join(", ")})`); } -// Render plane waves through the ambisonic decoder offline and return each ear's RMS level. -async function ears(page: Page, direction: number[], yaw: number): Promise<{ L: number; R: number }> { - return page.evaluate( - async ({ direction, yaw }) => { - const { Spatial } = await import("/audio-check.js"); - const rate = 48_000; - const context = new OfflineAudioContext(2, rate, rate); - const buffer = context.createBuffer(4, rate, rate); - const [x, y, z] = direction; - // A 1 kHz plane wave encoded in ACN W, Y, Z, X with SN3D. - [1, y, z, x].forEach((gain, channel) => { - const data = buffer.getChannelData(channel); - for (let i = 0; i < data.length; i++) data[i] = 0.5 * gain * Math.sin((2 * Math.PI * 1000 * i) / rate); - }); - const discrete = { - channelCount: 4, - channelCountMode: "explicit", - channelInterpretation: "discrete", - } as const; - const source = new AudioBufferSourceNode(context, { buffer, ...discrete }); - const root = new GainNode(context, discrete); - source.connect(root); - const spatial = new Spatial(root, { kind: "ambisonic" }); - // The head yawed left by `yaw` about +Y, standing at the origin. - const head = { - decompose: ( - position: { set(x: number, y: number, z: number): void }, - rotation: { set(x: number, y: number, z: number, w: number): void }, - ) => { - position.set(0, 0, 0); - rotation.set(0, Math.sin(yaw / 2), 0, Math.cos(yaw / 2)); - }, - }; - spatial.pose(head); - source.start(); - const rendered = await context.startRendering(); - // Skip the first 100 ms, while the listener glides into place. - const rms = (channel: number) => { - const data = rendered.getChannelData(channel).subarray(rate / 10); - return Math.sqrt(data.reduce((sum, v) => sum + v * v, 0) / data.length); - }; - return { L: rms(0), R: rms(1) }; - }, - { direction, yaw }, - ); +// Turn the view left (right when negative) by dragging, as a desktop viewer looks around. The drag +// starts at the far edge so it stays on the page, where every pointer move lands. +async function turnLeft(page: Page, radians: number): Promise { + const { width, height } = VIEWPORT; + const distance = radians / DRAG; + check(Math.abs(distance) < width - 2, "the turn fits one drag", () => `${distance} px`); + const start = distance > 0 ? 1 : width - 1; + await page.mouse.move(start, height / 2); + await page.mouse.down(); + await page.mouse.move(start + distance, height / 2, { steps: 20 }); + await page.mouse.up(); +} + +// Each ear's RMS level at the demo's spatial mix, averaged over half a second of live playback. +async function ears(page: Page): Promise<{ L: number; R: number }> { + return page.evaluate(async () => { + const output = (window as unknown as { vr: Demo }).vr.scene.listener?.output; + if (!output) throw new Error("the demo has no spatial audio"); + const context = output.context; + const split = new ChannelSplitterNode(context, { numberOfOutputs: 2 }); + output.connect(split); + const meters = [0, 1].map((channel) => { + const analyser = new AnalyserNode(context, { fftSize: 2048 }); + split.connect(analyser, channel); + return analyser; + }); + const sums = [0, 0]; + const data = new Float32Array(2048); + for (let i = 0; i < 10; i++) { + await new Promise((resolve) => setTimeout(resolve, 50)); + for (const [channel, meter] of meters.entries()) { + meter.getFloatTimeDomainData(data); + sums[channel] += data.reduce((sum, v) => sum + v * v, 0) / data.length; + } + } + output.disconnect(split); + const [L, R] = sums.map((sum) => Math.sqrt(sum / 10)); + return { L, R }; + }); } -function louder(level: { L: number; R: number }, ear: "L" | "R" | "neither", assertion: string) { +function louder(level: { L: number; R: number }, ear: "L" | "neither", assertion: string) { const ratio = level.L / level.R; - const ok = ear === "L" ? ratio > 2 : ear === "R" ? ratio < 0.5 : ratio > 0.9 && ratio < 1.1; + const ok = level.L + level.R > 0.01 && (ear === "L" ? ratio > 2 : ratio > 0.8 && ratio < 1.25); check(ok, assertion, () => `L ${level.L.toFixed(3)} R ${level.R.toFixed(3)}`); } -const browser = await launch(["--use-angle=swiftshader", "--enable-unsafe-swiftshader"]); +const browser = await launch(["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"]); let failed = true; try { const query = new URLSearchParams({ url: values.url, name: "vr.hang" }); const [page, errors] = await open(browser, `${origin}/?${query}`, "vr", true, { viewport: VIEWPORT }); - // Straight ahead is the left eye's centre: the catalog's stereo packing cropped the frame and the + // The catalog the demo read carries what the importer found in the MP4's sv3d, st3d, and dOps. + await page.waitForFunction( + () => { + const { vr } = window as unknown as { vr: Demo }; + return vr.video.out.config.peek() !== undefined && vr.audio.out.config.peek() !== undefined; + }, + undefined, + { timeout: 30_000 }, + ); + const catalog = await page.evaluate(() => { + const { vr } = window as unknown as { vr: Demo }; + const video = vr.video.out.config.peek() as { projection?: unknown; stereo?: unknown }; + const audio = vr.audio.out.config.peek() as { layout?: unknown; numberOfChannels?: unknown }; + return { + projection: video.projection, + stereo: video.stereo, + layout: audio.layout, + channels: audio.numberOfChannels, + }; + }); + const want = { + projection: { kind: "equirectangular", fov: 180 }, + stereo: "left-right", + layout: "ambisonic", + channels: 4, + }; + check( + Bun.deepEquals(catalog, want), + "the catalog carries the fixture's VR fields", + () => JSON.stringify(catalog), + ); + + // Straight ahead is the left eye's centre: the stereo packing cropped the frame and the // equirectangular projection faced its centre forward. await waitForCentre(page, "red", 30_000); - // Drag 90 degrees left (0.005 rad per pixel): a sphere wraps the left eye's left quarter there, - // where a flat screen would show only the background. - await page.mouse.move(VIEWPORT.width / 2, VIEWPORT.height / 2); - await page.mouse.down(); - await page.mouse.move(VIEWPORT.width / 2 + Math.PI / 2 / 0.005, VIEWPORT.height / 2, { steps: 20 }); - await page.mouse.up(); + // The left eye's left quarter lies 45 degrees left over a 180 degree field. + await turnLeft(page, Math.PI / 4); await waitForCentre(page, "lime", 5_000); - louder(await ears(page, [0, 1, 0], 0), "L", "a source on the left is louder in the left ear"); - louder(await ears(page, [0, 1, 0], Math.PI / 2), "neither", "turning left to face that source centres it"); - louder(await ears(page, [1, 0, 0], Math.PI / 2), "R", "a source ahead moves to the right ear after turning left"); + // 120 degrees left is past the field's edge: empty scene. A full sphere would show the frame's blue. + await turnLeft(page, (Math.PI * 75) / 180); + await waitForCentre(page, "dark", 5_000); + + // Face forward again. The drags were gestures, so the audio graph is running. The fixture's sound + // is a plane wave from the left: the left ear hears it, and turning to face it centres it. + await turnLeft(page, (-Math.PI * 120) / 180); + const ahead = await ears(page); + louder(ahead, "L", "a source on the left is louder in the left ear"); + await turnLeft(page, Math.PI / 2); + const facing = await ears(page); + louder(facing, "neither", "turning left to face that source centres it"); throwPageErrors(errors); - console.log("vr: stereo equirectangular painted per eye, facing forward; ambisonics follow the head"); + const level = ({ L, R }: { L: number; R: number }) => `L ${L.toFixed(3)} R ${R.toFixed(3)}`; + console.log(`vr: ${JSON.stringify(catalog)} laid out; ahead ${level(ahead)}, facing ${level(facing)}`); failed = false; } finally { await finishTraces(failed); diff --git a/test/justfile b/test/justfile index 97f250ff005..7023141a463 100644 --- a/test/justfile +++ b/test/justfile @@ -115,7 +115,7 @@ smoke *args: portrait: ./portrait/run.sh -# Paint a stereo 360 broadcast per eye in demo/vr, and turn its ambisonic decode with the head. +# Import a VR180 MP4 with ambisonic Opus and check demo/vr lays it out by the catalog's VR fields. vr: ./vr/run.sh diff --git a/test/vr/run.sh b/test/vr/run.sh index 9818475344c..62f6ce6ccbc 100755 --- a/test/vr/run.sh +++ b/test/vr/run.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Publish a stereo 360 picture with VR catalog flags and check demo/vr paints each eye the right way round. +# Import a VR180 left-right MP4 with ambisonic Opus, publish it through a relay, and check demo/vr +# reads the catalog's projection, stereo, and layout and lays out the picture and sound by them. set -euo pipefail if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then @@ -14,23 +15,31 @@ cd "$WORKSPACE" cargo build --locked -p moq-relay -p moq-cli uv run --no-project python sh/js/install.py frozen (cd demo/vr && bunx vite build --outDir "$HARNESS_RUN/vr-dist" --emptyOutDir) -# The ambisonic decoder alone, so the driver can render it offline without a stream. -bun build demo/vr/src/audio.ts --format esm --outfile "$HARNESS_RUN/vr-dist/audio-check.js" + +# Each eye is 480x480 over 180 degrees. Left eye: blue, red straight ahead (u 0.5), lime 45 degrees +# left (u 0.25). Right eye: magenta, so an uncropped frame puts magenta ahead. The sound is a plane +# wave from the left in first-order AmbiX (ACN W, Y, Z, X), so W and Y carry the same tone. +ffmpeg -hide_banner -loglevel error -f lavfi \ + -i "color=c=blue:s=960x480:r=15:d=2,drawbox=x=210:y=210:w=60:h=60:c=red:t=fill,drawbox=x=90:y=210:w=60:h=60:c=lime:t=fill,drawbox=x=480:y=0:w=480:h=480:c=magenta:t=fill" \ + -f lavfi -i "aevalsrc=exprs='0.5*sin(2*PI*1000*t)|0.5*sin(2*PI*1000*t)|0|0':s=48000:d=2" \ + -c:v libx264 -preset ultrafast -profile:v baseline -pix_fmt yuv420p -g 15 \ + -c:a libopus -mapping_family 2 -b:a 128k "$HARNESS_RUN/flat.mp4" +uv run --no-project python -I "$DIR/spherical.py" "$HARNESS_RUN/flat.mp4" "$HARNESS_RUN/vr180.mp4" + harness_port relay URL="http://127.0.0.1:$HARNESS_PORT" sed "s/4443/$HARNESS_PORT/g" "$DIR/relay.toml" >"$HARNESS_RUN/relay.toml" harness_spawn relay "$HARNESS_RUN/relay.log" "$WORKSPACE/target/debug/moq-relay" "$HARNESS_RUN/relay.toml" harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID" -# Left eye: blue, red straight ahead (u 0.5), lime 90 degrees left (u 0.25). Right eye: magenta, -# so a frame shown uncropped puts magenta straight ahead. +# ffmpeg carries st3d and sv3d into each fragmented init segment only past -strict unofficial. # shellcheck disable=SC2329 # invoked by harness_spawn publish_vr() { - ffmpeg -hide_banner -loglevel error -re -f lavfi \ - -i "color=c=blue:s=960x480:r=15,drawbox=x=210:y=210:w=60:h=60:c=red:t=fill,drawbox=x=90:y=210:w=60:h=60:c=lime:t=fill,drawbox=x=480:y=0:w=480:h=480:c=magenta:t=fill" \ - -c:v libx264 -preset ultrafast -tune zerolatency -profile:v baseline -pix_fmt yuv420p -g 15 -f h264 - | - "$WORKSPACE/target/debug/moq" --connect "$URL" --connect-version moq-lite-07-wip --broadcast vr.hang \ - import --projection equirectangular --stereo left-right avc3 + ffmpeg -hide_banner -loglevel error -re -stream_loop -1 -i "$HARNESS_RUN/vr180.mp4" -c copy -strict unofficial \ + -movflags cmaf+frag_keyframe+empty_moov+default_base_moof -f mp4 - | + "$WORKSPACE/target/debug/moq" --connect "$URL" --connect-version moq-lite-07-wip --broadcast vr.hang import fmp4 } harness_spawn publisher "$HARNESS_RUN/publisher.log" publish_vr cd test/interop/clients/js -bun vr.ts --url "$URL" +# SwiftShader stands in for a GPU. ANGLE asks its Vulkan for a Wayland surface, which it lacks, +# whenever a Wayland session is visible, so a desktop run would have no WebGL. +env -u WAYLAND_DISPLAY bun vr.ts --url "$URL" diff --git a/test/vr/spherical.py b/test/vr/spherical.py new file mode 100644 index 00000000000..0027d2e41fa --- /dev/null +++ b/test/vr/spherical.py @@ -0,0 +1,77 @@ +"""Mark an MP4's video as VR180 left-right with Spherical Video V2 boxes. + +ffmpeg reads and remuxes `st3d` and `sv3d` but cannot create them, so this appends both to the +`avc1` sample entry and grows every enclosing box. The `moov` must follow `mdat`, as ffmpeg writes +it without `+faststart`, so no chunk offset moves. + +Usage: python -I spherical.py in.mp4 out.mp4 +""" + +import struct +import sys + +CONTAINERS = {b"moov", b"trak", b"mdia", b"minf", b"stbl"} + + +def box(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", 8 + len(body)) + kind + body + + +def full(kind: bytes, body: bytes) -> bytes: + return box(kind, b"\0\0\0\0" + body) + + +# Each equirectangular bound is the 0.32 fixed-point fraction of the sphere left uncovered: a +# quarter off each side leaves 180 degrees of longitude. +QUARTER = 1 << 30 +SPATIAL = full(b"st3d", bytes([2])) + box( # 2 is left-right + b"sv3d", + full(b"svhd", b"moq\0") + + box(b"proj", full(b"prhd", struct.pack(">iii", 0, 0, 0)) + full(b"equi", struct.pack(">IIII", 0, 0, QUARTER, QUARTER))), +) + + +def patch(data: bytearray, start: int, end: int) -> int: + """Append SPATIAL to the avc1 entry within [start, end), returning how much the range grew.""" + at = start + while at < end: + size, kind = struct.unpack(">I4s", data[at : at + 8]) + if size < 8: + raise ValueError(f"unsupported box size {size} for {kind!r}") + grown = 0 + if kind in CONTAINERS: + grown = patch(data, at + 8, at + size) + elif kind == b"stsd": + # A full box and an entry count precede the sample entries. + grown = patch(data, at + 16, at + size) + elif kind == b"avc1": + data[at + size : at + size] = SPATIAL + grown = len(SPATIAL) + if grown: + struct.pack_into(">I", data, at, size + grown) + return grown + at += size + return 0 + + +def main() -> None: + source, target = sys.argv[1:] + with open(source, "rb") as file: + data = bytearray(file.read()) + kinds, at = [], 0 + while at < len(data): + size, kind = struct.unpack(">I4s", data[at : at + 8]) + if size < 8: + raise ValueError(f"unsupported box size {size} for {kind!r}") + kinds.append(kind) + at += size + # The mdat must precede the moov, so growing the moov moves no sample. + if kinds.index(b"moov") < kinds.index(b"mdat"): + raise ValueError("moov precedes mdat; write the source without +faststart") + if not patch(data, 0, len(data)): + raise ValueError("no avc1 sample entry") + with open(target, "wb") as file: + file.write(data) + + +main() From 9ce8bbf080f25de045cfc5e7a9ade4af21c55b50 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:17:17 -0800 Subject: [PATCH 07/13] docs(demo): document the VR demo and serve it with just vr Also decode higher-order ambisonics at first order rather than refusing them, since the catalog allows any square channel count. Co-Authored-By: Claude Opus 5.5 --- demo/vr/justfile | 11 ++++++++--- demo/vr/src/audio.ts | 6 +++--- demo/vr/src/scene.ts | 2 +- doc/bin/demo.md | 27 ++++++++++++++++++++++++++- doc/concept/hang.md | 2 +- doc/setup/examples.md | 1 + justfile | 1 + test/interop/clients/js/vr.ts | 6 +----- test/vr/spherical.py | 9 ++++----- 9 files changed, 46 insertions(+), 19 deletions(-) diff --git a/demo/vr/justfile b/demo/vr/justfile index 408f18912bf..9a56ec58af9 100644 --- a/demo/vr/justfile +++ b/demo/vr/justfile @@ -1,4 +1,9 @@ set shell := ["bash", "-euo", "pipefail", "-c"] -serve url = "http://localhost:4443": - bun install - VITE_RELAY_URL="{{url}}" bun --bun vite + +# Serve the VR demo against a relay; the page takes ?url= and ?name= to override. +serve url="http://localhost:4443": install + VITE_RELAY_URL="{{ url }}" bun --bun vite + +# Install the workspace deps that vite and the demo's imports need. +install: + uv run --no-project python ../../sh/js/install.py update diff --git a/demo/vr/src/audio.ts b/demo/vr/src/audio.ts index 47b785aebd0..2927eb42bf8 100644 --- a/demo/vr/src/audio.ts +++ b/demo/vr/src/audio.ts @@ -53,10 +53,10 @@ export class Spatial { this.#nodes.push(root); } - // Decodes out to fixed speakers around the scene + // Decodes out to fixed speakers around the scene. A higher order plays at first order: the + // splitter's discrete downmix keeps ACN channels 0 to 3 and drops the rest. #ambisonic(root: AudioNode, destination: AudioNode) { - if (root.channelCount !== 4) - throw new Error(`first-order ambisonics needs 4 channels, got ${root.channelCount}`); + if (root.channelCount < 4) throw new Error(`ambisonics needs at least 4 channels, got ${root.channelCount}`); const split = new ChannelSplitterNode(this.#context, { numberOfOutputs: 4 }); root.connect(split); diff --git a/demo/vr/src/scene.ts b/demo/vr/src/scene.ts index acc89c55f34..5ef567e2c82 100644 --- a/demo/vr/src/scene.ts +++ b/demo/vr/src/scene.ts @@ -216,7 +216,7 @@ export class Scene { head.getWorldDirection(this.#headDirection); const yaw = Math.atan2(-this.#headDirection.x, -this.#headDirection.z); this.#content.rotation.set(0, yaw, 0); -this.#content.position.set(this.#headPosition.x, 0, this.#headPosition.z); + this.#content.position.set(this.#headPosition.x, 0, this.#headPosition.z); this.#content.updateMatrixWorld(); }; diff --git a/doc/bin/demo.md b/doc/bin/demo.md index 2d779d90fc9..e893d8c451e 100644 --- a/doc/bin/demo.md +++ b/doc/bin/demo.md @@ -1,6 +1,6 @@ --- title: Demos -description: The web demo, MoQ Boy, and CARLA +description: The web demo, MoQ Boy, CARLA, and VR --- # Demos @@ -69,3 +69,28 @@ What it demonstrates: - **Measured feedback.** Each truck's `status` track carries what the gate accepted and what the vehicle did, so the player draws requested against measured steering. Code: [`demo/carla`](https://github.com/moq-dev/moq/tree/main/demo/carla). + +## VR + +A WebXR player for 360 and VR180 video with spatial audio, in a headset or a +desktop tab. It lays a broadcast out by the catalog's +[spatial fields](/concept/hang#catalog): an equirectangular `projection` wraps +the picture on a sphere spanning its field of view, `stereo` crops each eye +from a packed frame, and an `ambisonic` audio layout decodes the sound field +around the viewer's head. Without them it shows a flat screen with the sound +coming from it. + +```bash +just relay # a local relay +ffmpeg -re -i vr180.mp4 -c copy -strict unofficial \ + -movflags cmaf+frag_keyframe+empty_moov+default_base_moof -f mp4 - | + moq --connect http://localhost:4443 --broadcast vr.hang import fmp4 +just vr serve # the player: Enter VR in a headset, or drag to look around +``` + +The importer reads the projection and stereo packing from an MP4's Spherical +Video V2 boxes, which ffmpeg writes only past `-strict unofficial`, and the +ambisonic layout from Opus channel mapping family 2. In the player, a trigger +or `R` recentres the picture where you look, and a grip or `M` mutes. + +Code: [`demo/vr`](https://github.com/moq-dev/moq/tree/main/demo/vr). diff --git a/doc/concept/hang.md b/doc/concept/hang.md index d4c0269ded6..fe1b0afc9ba 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -69,7 +69,7 @@ A few things the catalog can express beyond decoder config: - **Delay.** A rendition can also say how far its frames' timestamps trail those another rendition already wrote, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. A rendition alone measures none. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. - **Reorder depth.** A video rendition may declare `reorder`, the most frames that can precede one in decode order and follow it in presentation, as H.264 `max_num_reorder_frames` counts them; `0` declares none, which is not the same as absent. The Rust H.264 and H.265 importers copy it from the SPS when the SPS declares one (progressive H.264 only), and `@moq/publish` declares `0`. The MPEG-TS, Matroska, and FLV exporters then derive each group's decode times from that group's frames alone, so exports that joined at different groups agree, and they end the export on a frame reordered deeper than declared or a depth the codec level cannot reach. Without it they infer decode times from the timestamps they have read, as before. A publisher never changes it once declared. - **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, an AV1 track's av1C, or Matroska, but not yet from H.264 or H.265 in fMP4 or FLV. Nothing in `@moq/watch` reads it yet. -- **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. Nothing in `@moq/watch` renders them yet. +- **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. The Rust importers fill them from MP4 `sv3d` and `st3d`, Matroska, and Opus channel mapping family 2. `@moq/watch` plays the picture and channels as they are; the [VR demo](/bin/demo#vr) lays them out in a headset. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no new frames coming, and a player must not select it. A subscriber may still get audio retained from before the disable, as when it subscribed a moment earlier. Disabling a rendition in `@moq/publish`, as hiding video does, keeps it in the catalog this way rather than removing it, so it is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. A muted `@moq/publish` microphone is not disabled: it stays enabled and sends nothing, ending each utterance with a discontinuity marker, so listeners stay subscribed and play the gap as intentional silence. - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. - **Join interval.** The root `media.join.interval` promises that every audio and video track starts a group at least that often: the longest GOP any of them publishes. It is fixed for the broadcast, and the [HLS gateway](/bin/hls) numbers every rendition's segments on the grid it fixes, segments the smallest whole multiple of it at least a second long, from each group's first timestamp, so renditions share segment numbers. A publisher that knows its GOP declares it; Rust publishers (`moq-mux` and everything built on it) without one measure each track over its first second or group, whichever is longer, and `@moq/publish` declares its encoders' `join`. A broadcast without it gets no HLS playlists. diff --git a/doc/setup/examples.md b/doc/setup/examples.md index d8de784ac25..698c73ce6e7 100644 --- a/doc/setup/examples.md +++ b/doc/setup/examples.md @@ -18,6 +18,7 @@ migration guides. | Build a meeting with a roster and chat | [Rooms](/lib/js/room) | [`meet.ts`](https://github.com/moq-dev/moq/blob/main/demo/web/src/meet.ts) | | Send input back to a live application | [MoQ Boy](/bin/demo#moq-boy) | [`demo/boy`](https://github.com/moq-dev/moq/tree/main/demo/boy) | | Stream a simulator and send controls | [CARLA](/bin/demo#carla) | [`demo/carla`](https://github.com/moq-dev/moq/tree/main/demo/carla) | +| Play 360 or VR180 video with spatial audio in a headset | [VR](/bin/demo#vr) | [`demo/vr`](https://github.com/moq-dev/moq/tree/main/demo/vr) | | Bring an existing media tool | [OBS](/bin/obs), [GStreamer](/bin/gstreamer), or [gateways](/bin/#gateways) | [`demo/pub`](https://github.com/moq-dev/moq/tree/main/demo/pub), [`demo/sub`](https://github.com/moq-dev/moq/tree/main/demo/sub) | Run checkout examples inside the [Nix dev shell](/setup/dev). Each demo's diff --git a/justfile b/justfile index 59e7d4528ec..d8905f89bda 100644 --- a/justfile +++ b/justfile @@ -34,6 +34,7 @@ mod carla 'demo/carla' mod pub 'demo/pub' mod relay 'demo/relay' mod sub 'demo/sub' +mod vr 'demo/vr' mod web 'demo/web' # Run the demo by default. diff --git a/test/interop/clients/js/vr.ts b/test/interop/clients/js/vr.ts index dded439305d..6ad2166fa75 100644 --- a/test/interop/clients/js/vr.ts +++ b/test/interop/clients/js/vr.ts @@ -155,11 +155,7 @@ try { layout: "ambisonic", channels: 4, }; - check( - Bun.deepEquals(catalog, want), - "the catalog carries the fixture's VR fields", - () => JSON.stringify(catalog), - ); + check(Bun.deepEquals(catalog, want), "the catalog carries the fixture's VR fields", () => JSON.stringify(catalog)); // Straight ahead is the left eye's centre: the stereo packing cropped the frame and the // equirectangular projection faced its centre forward. diff --git a/test/vr/spherical.py b/test/vr/spherical.py index 0027d2e41fa..cba22dedb8e 100644 --- a/test/vr/spherical.py +++ b/test/vr/spherical.py @@ -24,11 +24,10 @@ def full(kind: bytes, body: bytes) -> bytes: # Each equirectangular bound is the 0.32 fixed-point fraction of the sphere left uncovered: a # quarter off each side leaves 180 degrees of longitude. QUARTER = 1 << 30 -SPATIAL = full(b"st3d", bytes([2])) + box( # 2 is left-right - b"sv3d", - full(b"svhd", b"moq\0") - + box(b"proj", full(b"prhd", struct.pack(">iii", 0, 0, 0)) + full(b"equi", struct.pack(">IIII", 0, 0, QUARTER, QUARTER))), -) +POSE = full(b"prhd", struct.pack(">iii", 0, 0, 0)) +EQUI = full(b"equi", struct.pack(">IIII", 0, 0, QUARTER, QUARTER)) +# Stereo mode 2 is left-right. +SPATIAL = full(b"st3d", bytes([2])) + box(b"sv3d", full(b"svhd", b"moq\0") + box(b"proj", POSE + EQUI)) def patch(data: bytearray, start: int, end: int) -> int: From 6ad30048fbe4bc247f493c201a3e47fd1ac5a3b0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:20:55 -0800 Subject: [PATCH 08/13] quest: follow up with a headset try of the VR demo Co-Authored-By: Claude Opus 5.5 --- quest/m1/vr-demo-try.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 quest/m1/vr-demo-try.md diff --git a/quest/m1/vr-demo-try.md b/quest/m1/vr-demo-try.md new file mode 100644 index 00000000000..4314db4f4ee --- /dev/null +++ b/quest/m1/vr-demo-try.md @@ -0,0 +1,19 @@ +# [XS] Try the VR demo in a headset and decide where it lives + +## Goal + +The maintainer has played `demo/vr` in a WebXR headset and decided whether +to promote it, such as hosting it beside the web demo at moq.dev with a VR180 +broadcast to watch. + +## Plan + +The harness (`just test vr`) proves the layout in headless Chromium on a +desktop view only; a headset's per-eye render, recentre, and HRTF audio have +had no human check. Run it with `just relay`, the `moq import fmp4` command in +`doc/bin/demo.md#vr`, and `just vr serve`, then open the page on the headset +(it needs a secure context, so serve it over HTTPS or tunnel to localhost). + +## Waiting + +- Maintainer: try it in a headset (an agent has none), then pick: (a) host it at moq.dev with a looping VR180 broadcast, recommended once it plays well; (b) keep it local-only as a reference consumer. Check by this quest's Plan recording the pick. From 7a88b4c6a7d92b3624c8cbb3511c9019f5d4c5bb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:27:48 -0800 Subject: [PATCH 09/13] fix(watch): keep the Opus pre-skip with Terminal once WebCodecs gets the OpusHead Chromium trims an OpusHead's pre-skip itself, again after every flush, so passing the header would trim the same samples twice. Hand WebCodecs a copy with the pre-skip zeroed; the channel mapping is what it needs. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/config.test.ts | 42 ++++++++++++++++++++++++++++++- js/watch/src/audio/config.ts | 20 +++++++++++++++ js/watch/src/audio/decoder.ts | 23 ++++++++++++----- js/watch/src/audio/supported.ts | 4 +-- 4 files changed, 80 insertions(+), 9 deletions(-) diff --git a/js/watch/src/audio/config.test.ts b/js/watch/src/audio/config.test.ts index fa9a5de101e..48294b41953 100644 --- a/js/watch/src/audio/config.test.ts +++ b/js/watch/src/audio/config.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test"; import { Duration, yieldNow } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import { Effect, Signal } from "@moq/signals"; -import { decoderConfig, frameDuration, packetDuration, playbackIdentity } from "./config"; +import { decoderConfig, decoderDescription, frameDuration, packetDuration, playbackIdentity } from "./config"; const flush = () => yieldNow(); @@ -90,3 +90,43 @@ test("an implicit CMAF duration falls through to the Opus TOC", () => { expect(packetDuration("opus", frame)).toEqual(Duration.millis(20)); expect(packetDuration("mp4a.40.2", frame)).toBeUndefined(); }); + +test("an Opus description reaches WebCodecs with its mapping and no pre-skip", () => { + // OpusHead, version 1, 4 channels, pre-skip 312, 48 kHz, gain 0, family 2 with 4 streams. + const head = new Uint8Array([ + ...new TextEncoder().encode("OpusHead"), + 1, + 4, + 0x38, + 0x01, + 0x80, + 0xbb, + 0, + 0, + 0, + 0, + 2, + 4, + 0, + 0, + 1, + 2, + 3, + ]); + const description = decoderDescription("opus", head); + expect(description?.slice(10, 12)).toEqual(new Uint8Array([0, 0])); + expect([...(description ?? [])].filter((_, index) => index < 10 || index > 11)).toEqual( + [...head].filter((_, index) => index < 10 || index > 11), + ); + // The catalog's bytes are left alone. + expect(head[10]).toBe(0x38); + + // dOps keeps its pre-skip at the same place after the version and channel count. + const dops = new Uint8Array([0, 2, 0x01, 0x38, 0, 0, 0xbb, 0x80, 0, 0, 0]); + expect(decoderDescription("opus", dops)?.slice(2, 4)).toEqual(new Uint8Array([0, 0])); + + expect(() => decoderDescription("opus", new Uint8Array([7, 1, 2]))).toThrow(); + expect(decoderDescription("opus", undefined)).toBeUndefined(); + const asc = new Uint8Array([0x12, 0x10]); + expect(decoderDescription("mp4a.40.2", asc)).toBe(asc); +}); diff --git a/js/watch/src/audio/config.ts b/js/watch/src/audio/config.ts index 531dd4b5da9..a493a30bfe8 100644 --- a/js/watch/src/audio/config.ts +++ b/js/watch/src/audio/config.ts @@ -51,6 +51,26 @@ export function playbackIdentity(config: Catalog.AudioConfig): PlaybackIdentity }; } +const OPUS_HEAD = new TextEncoder().encode("OpusHead"); + +/** + * The description to configure WebCodecs with: the catalog's, or the init segment's when it has none. + * + * An Opus stream past stereo cannot decode without its OpusHead's channel mapping, but Chromium then + * trims the header's pre-skip itself, again after every flush. The decoder's Terminal owns that trim, + * gap resumes included, so WebCodecs gets a copy with the pre-skip zeroed. + */ +export function decoderDescription(codec: string, description: Uint8Array | undefined): Uint8Array | undefined { + if (!description || codec !== "opus") return description; + // Validates the header, which may be an OpusHead, one without its signature, or a dOps. + Util.Opus.preSkip(description); + const signed = OPUS_HEAD.every((byte, index) => description[index] === byte); + const copy = description.slice(); + // Two bytes after the version and channel count, in either byte order. + copy.fill(0, (signed ? OPUS_HEAD.length : 0) + 2, (signed ? OPUS_HEAD.length : 0) + 4); + return copy; +} + /** * A frame's duration as a constant over the catalog's sample rate, for codecs that fix it. * diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index fb8df7e7ace..f38fa967a86 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -20,7 +20,14 @@ import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feed import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; import { DELAY_MAX, type Sync, type Track } from "../sync"; import { AudioBuffer } from "./buffer"; -import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; +import { + type DecoderConfig, + decoderDescription, + frameDuration, + type PlaybackIdentity, + packetDuration, + playbackIdentity, +} from "./config"; import { Handover } from "./handover"; import { bound, maxDelay, ringHold, target } from "./latency"; import { Order } from "./order"; @@ -687,8 +694,10 @@ export class Decoder { this.#ring.peek()?.dropped(); }); - // For Opus this is the OpusHead, whose channel mapping a stream past stereo cannot decode without. - const description = config.description ? Util.Hex.toBytes(config.description) : undefined; + const description = decoderDescription( + config.codec, + config.description ? Util.Hex.toBytes(config.description) : undefined, + ); const decoderConfig: AudioDecoderConfig = { codec: config.codec, sampleRate: config.sampleRate, @@ -754,9 +763,11 @@ export class Decoder { .min(DELAY_MAX) .asMicros(), ); - // The init segment's description stands in for a catalog without one. It never carries Opus's - // dOps, which is not the OpusHead WebCodecs takes. - const description = config.description ? Util.Hex.toBytes(config.description) : init.description; + // The init segment never yields Opus's dOps, which is not the OpusHead WebCodecs takes. + const description = decoderDescription( + config.codec, + config.description ? Util.Hex.toBytes(config.description) : init.description, + ); const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); diff --git a/js/watch/src/audio/supported.ts b/js/watch/src/audio/supported.ts index d6054050b51..2e59517ba10 100644 --- a/js/watch/src/audio/supported.ts +++ b/js/watch/src/audio/supported.ts @@ -3,7 +3,7 @@ import * as Util from "@moq/hang/util"; import { hosted } from "../assets"; import { base64ToBytes } from "../base64"; import { supportCacheKey } from "../support-probe"; -import { decoderConfigKey } from "./config"; +import { decoderConfigKey, decoderDescription } from "./config"; /** Whether the WebCodecs audio decoder can play this config. */ export async function supported(config: Catalog.AudioConfig): Promise { @@ -43,7 +43,7 @@ export async function supported(config: Catalog.AudioConfig): Promise { await Util.Libav.polyfill(hosted.url); const res = await AudioDecoder.isConfigSupported({ ...config, - description, + description: decoderDescription(config.codec, description), }); return res.supported ?? false; } From 0edcd21e981b300412267a965790c8b9e51ba993 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:35:55 -0800 Subject: [PATCH 10/13] chore(quest): claim quest/m1/vr-demo.md Quest-Claim: 6ac6c873ec08d5f2 From 9eb9be26373bcc78dac20b9061fea4672eaf0c5d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 17:36:06 -0800 Subject: [PATCH 11/13] quest: finish the VR demo Co-Authored-By: Claude Opus 5.5 --- quest/m1/vr-demo.md | 54 --------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 quest/m1/vr-demo.md diff --git a/quest/m1/vr-demo.md b/quest/m1/vr-demo.md deleted file mode 100644 index 8b69a945cce..00000000000 --- a/quest/m1/vr-demo.md +++ /dev/null @@ -1,54 +0,0 @@ -# [M] A VR demo reads the catalog's spatial fields, tested end to end - -## Goal - -`demo/vr` plays a broadcast in a WebXR headset (or a desktop tab) and picks -its layout from the hang catalog: video `projection` and its `fov` choose a -flat screen or an equirectangular sphere, `stereo` crops each eye from a -left-right or top-bottom frame, and an audio `layout` of `ambisonic` decodes -the sound field around the head. A harness imports a small VR180 left-right -fixture with first-order ambisonic Opus, publishes it through a relay, and -asserts the demo sees and applies the right projection, fov, stereo, and -layout. - -This gives the fields that -[moq-dev/moq#5543](https://github.com/moq-dev/moq/pull/5543) added and -[moq-dev/moq#7308](https://github.com/moq-dev/moq/pull/7308) fills on import -their first reader, so they ship with the cut instead of being deleted. - -## Plan - -Decided 2026-11-14 (maintainer): keep the fields and finish -[moq-dev/moq#6373](https://github.com/moq-dev/moq/pull/6373) so they are -exercised (rejected: deleting them before the cut, as the feature review's -`catalog-vr-fields` quest in -[moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764) recommended). -✅ m1, since a demo and a harness break nothing (rejected: m2, which holds -breaking work). - -- Build on @nuts-rice's commits from `nuts-rice/moq:webxr_vr_demo`, keeping - their authorship; the PR credits them and links #6373. Maintainer edits - are off on their fork, so the work lands from a moq-dev branch. -- Drop #6373's catalog plumbing that #7308 made redundant: - `moq_mux::catalog::Spatial`, its producer and track hunks, the `import - --projection/--stereo/--audio-layout` flags, and the `pub` on the hang - `check` helpers. The importers fill the fields from MP4 `st3d`/`sv3d`, - Matroska, and Opus family 2. Adding CLI or catalog API needs the - maintainer first. -- `@moq/watch` drops every Opus `description`, so Chromium refuses Opus past - stereo ("description is required for multi-channel Opus"). Passing the - catalog's OpusHead to both the support probe and the decoder, as the Rust - importers intend, is part of this quest, since the ambisonic decode needs - it. Chromium then trims the header's pre-skip itself, again after every - flush (measured: 648 of 960 frames), so WebCodecs gets a copy with the - pre-skip zeroed and the decoder's `Terminal` stays its one owner, gap - resumes included. -- The harness publishes the fixture through `moq import fmp4`, so it proves - the importer's side too, and runs nightly in `interop.yml` beside the - portrait harness, outside the merge gate. - -Public API: none (a behavior fix in `@moq/watch`). Wire: none. - -## Related - -- [fMP4 import fills the ambisonic layout](/quest/m5/mux-vr-ambisonic-fmp4.md) - the harness's fixture exercises that importer path From e0062126791a6138c69172f58c8a65761c2111c4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 20:41:06 -0800 Subject: [PATCH 12/13] quest: record the maintainer's pick to host the VR demo once it plays well Co-Authored-By: Claude Opus 5.5 --- quest/m1/vr-demo-try.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/quest/m1/vr-demo-try.md b/quest/m1/vr-demo-try.md index 4314db4f4ee..5c65cef0d0e 100644 --- a/quest/m1/vr-demo-try.md +++ b/quest/m1/vr-demo-try.md @@ -1,13 +1,18 @@ -# [XS] Try the VR demo in a headset and decide where it lives +# [XS] Try the VR demo in a headset, then host it at moq.dev ## Goal -The maintainer has played `demo/vr` in a WebXR headset and decided whether -to promote it, such as hosting it beside the web demo at moq.dev with a VR180 +The maintainer has played `demo/vr` in a WebXR headset, and once it plays +well there, it is hosted beside the web demo at moq.dev with a looping VR180 broadcast to watch. ## Plan +Decided 2026-11-14 (maintainer, pick round for +[moq-dev/moq#8781](https://github.com/moq-dev/moq/pull/8781)): ✅ host it at +moq.dev with a looping VR180 broadcast once it plays well in a headset +(rejected: keep it local-only as a reference consumer). + The harness (`just test vr`) proves the layout in headless Chromium on a desktop view only; a headset's per-eye render, recentre, and HRTF audio have had no human check. Run it with `just relay`, the `moq import fmp4` command in @@ -16,4 +21,4 @@ had no human check. Run it with `just relay`, the `moq import fmp4` command in ## Waiting -- Maintainer: try it in a headset (an agent has none), then pick: (a) host it at moq.dev with a looping VR180 broadcast, recommended once it plays well; (b) keep it local-only as a reference consumer. Check by this quest's Plan recording the pick. +- Maintainer: try it in a headset (an agent has none) and say whether it plays well enough to host. Check by this quest's Plan recording the result. From 85d3d88c1adf4be0501cb6a73ba14650f53b6b0c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 20:46:10 -0800 Subject: [PATCH 13/13] fix(demo,watch): pose the VR listener on Firefox, and hand WebCodecs only a signed OpusHead Firefox has no AudioListener params, so the throw ended three.js's render loop once audio attached; use setPosition and setOrientation there. A dOps or unsigned Opus header, which WebCodecs cannot read, stays dropped as before. Co-Authored-By: Claude Opus 5.5 --- demo/vr/src/audio.ts | 8 +++++++- js/watch/src/audio/config.test.ts | 7 +++---- js/watch/src/audio/config.ts | 11 ++++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/demo/vr/src/audio.ts b/demo/vr/src/audio.ts index 2927eb42bf8..45581f11e18 100644 --- a/demo/vr/src/audio.ts +++ b/demo/vr/src/audio.ts @@ -108,9 +108,15 @@ export class Spatial { up.set(0, 1, 0).applyQuaternion(rotation); const listener = this.#context.listener; + if (!this.#tracksPosition) position.set(0, 0, 0); + // Firefox has no listener AudioParams, only the older setters. + if (!listener.positionX) { + listener.setPosition(position.x, position.y, position.z); + listener.setOrientation(forward.x, forward.y, forward.z, up.x, up.y, up.z); + return; + } const at = this.#context.currentTime; const set = (param: AudioParam, value: number) => param.setTargetAtTime(value, at, 0.01); - if (!this.#tracksPosition) position.set(0, 0, 0); set(listener.positionX, position.x); set(listener.positionY, position.y); set(listener.positionZ, position.z); diff --git a/js/watch/src/audio/config.test.ts b/js/watch/src/audio/config.test.ts index 48294b41953..d7c3cb24b51 100644 --- a/js/watch/src/audio/config.test.ts +++ b/js/watch/src/audio/config.test.ts @@ -121,11 +121,10 @@ test("an Opus description reaches WebCodecs with its mapping and no pre-skip", ( // The catalog's bytes are left alone. expect(head[10]).toBe(0x38); - // dOps keeps its pre-skip at the same place after the version and channel count. + // WebCodecs reads only a signed OpusHead, so a dOps stays dropped, as a truncated header fails. const dops = new Uint8Array([0, 2, 0x01, 0x38, 0, 0, 0xbb, 0x80, 0, 0, 0]); - expect(decoderDescription("opus", dops)?.slice(2, 4)).toEqual(new Uint8Array([0, 0])); - - expect(() => decoderDescription("opus", new Uint8Array([7, 1, 2]))).toThrow(); + expect(decoderDescription("opus", dops)).toBeUndefined(); + expect(() => decoderDescription("opus", head.slice(0, 12))).toThrow(); expect(decoderDescription("opus", undefined)).toBeUndefined(); const asc = new Uint8Array([0x12, 0x10]); expect(decoderDescription("mp4a.40.2", asc)).toBe(asc); diff --git a/js/watch/src/audio/config.ts b/js/watch/src/audio/config.ts index a493a30bfe8..9e52ee077c2 100644 --- a/js/watch/src/audio/config.ts +++ b/js/watch/src/audio/config.ts @@ -58,16 +58,17 @@ const OPUS_HEAD = new TextEncoder().encode("OpusHead"); * * An Opus stream past stereo cannot decode without its OpusHead's channel mapping, but Chromium then * trims the header's pre-skip itself, again after every flush. The decoder's Terminal owns that trim, - * gap resumes included, so WebCodecs gets a copy with the pre-skip zeroed. + * gap resumes included, so WebCodecs gets a copy with the pre-skip zeroed. WebCodecs reads only a + * signed OpusHead, so a dOps or an unsigned header stays dropped, as before. */ export function decoderDescription(codec: string, description: Uint8Array | undefined): Uint8Array | undefined { if (!description || codec !== "opus") return description; - // Validates the header, which may be an OpusHead, one without its signature, or a dOps. + if (!OPUS_HEAD.every((byte, index) => description[index] === byte)) return undefined; + // Validates the header's length. Util.Opus.preSkip(description); - const signed = OPUS_HEAD.every((byte, index) => description[index] === byte); const copy = description.slice(); - // Two bytes after the version and channel count, in either byte order. - copy.fill(0, (signed ? OPUS_HEAD.length : 0) + 2, (signed ? OPUS_HEAD.length : 0) + 4); + // The two little-endian bytes after the version and channel count. + copy.fill(0, OPUS_HEAD.length + 2, OPUS_HEAD.length + 4); return copy; }