+
+
+
diff --git a/demo/vr/src/index.ts b/demo/vr/src/index.ts
new file mode 100644
index 00000000000..db538ff05d0
--- /dev/null
+++ b/demo/vr/src/index.ts
@@ -0,0 +1,78 @@
+import { Audio, Broadcast, Net, Signals, Sync, Video } from "@moq/watch";
+import { type Placement, Spatial } from "./audio";
+import { Scene } from "./scene";
+
+const params = new URLSearchParams(location.search);
+const url = new URL(params.get("url") ?? import.meta.env.VITE_RELAY_URL ?? "http://localhost:4443");
+const name = Net.Path.from(params.get("name") ?? "vr.hang");
+
+const connection = new Net.Connection({ url });
+const broadcast = new Broadcast({ origin: connection.origin, name, enabled: true });
+const audioSrc = new Audio.Source({ broadcast, supported: Audio.Decoder.supported });
+const videoSrc = new Video.Source({ broadcast, supported: Video.Decoder.supported });
+const sync = new Sync({});
+const audio = new Audio.Decoder({ source: audioSrc, sync, enabled: true });
+const video = new Video.Decoder({ source: videoSrc, sync, enabled: true });
+const scene = new Scene(document.body);
+const effect = new Signals.Effect();
+
+// The console and test/vr read the catalog and the scene through this.
+Object.assign(window, { vr: { audio: audioSrc, video: videoSrc, scene } });
+
+// clone the frame once decoder gets it out
+effect.run((e) => {
+ const frame = e.get(video.out.frame);
+ if (frame) scene.frame(frame.clone());
+});
+
+effect.run((e) => {
+ const error = e.get(video.out.error) ?? e.get(audio.out.error);
+ const muted = e.get(scene.muted) ? " · muted" : "";
+ const config = e.get(videoSrc.out.config);
+ const display = e.get(video.out.display);
+ const delay = e.get(sync.out.delay);
+ const fps = e.get(scene.fps);
+ // Stats change every frame, so sample them on the once-a-second fps tick instead.
+ const stats = video.out.stats.peek();
+
+ const projection = config?.projection;
+ const shape = projection?.kind === "equirectangular" ? `equirect ${projection.fov ?? 360}°` : "flat";
+ const lines = [
+ error
+ ? `error: ${error.message}`
+ : `${e.get(connection.status)} · ${name} ${e.get(broadcast.out.status)}${muted}`,
+ config
+ ? `${display?.width ?? "?"}×${display?.height ?? "?"} ${config.codec} · ${shape} · ${config.stereo ?? "mono"}`
+ : "no video",
+ `render ${fps.render} fps · video ${fps.video} fps`,
+ `buffer ${Math.round(delay.asMillis())} ms · late ${stats?.late ?? 0} · stalls ${stats?.stalls ?? 0}`,
+ ];
+ scene.status(lines);
+});
+
+effect.run((e) => {
+ const config = e.get(videoSrc.out.config);
+ if (config) scene.layout(config.projection, config.stereo);
+});
+effect.run((e) => {
+ const root = e.get(audio.out.root);
+ const config = e.get(audioSrc.out.config);
+ if (!root || !config) return;
+ const projection = e.get(videoSrc.out.config)?.projection?.kind;
+ const placement: Placement =
+ config.layout === "ambisonic"
+ ? { kind: "ambisonic" }
+ : projection === "equirectangular"
+ ? { kind: "head" }
+ : { kind: "point", position: scene.speaker };
+ const spatial = new Spatial(root, placement);
+ scene.listener = spatial;
+ e.run((inner) => {
+ spatial.muted = inner.get(scene.muted);
+ });
+
+ e.cleanup(() => {
+ spatial.close();
+ scene.listener = undefined;
+ });
+});
diff --git a/demo/vr/src/scene.ts b/demo/vr/src/scene.ts
new file mode 100644
index 00000000000..5ef567e2c82
--- /dev/null
+++ b/demo/vr/src/scene.ts
@@ -0,0 +1,295 @@
+import { type Hang, Signals } from "@moq/watch";
+import * as THREE from "three";
+import { VRButton } from "three/addons/webxr/VRButton.js";
+import type { Spatial } from "./audio";
+
+type Projection = Hang.Catalog.Projection;
+
+const EYE = 1.6;
+const SCREEN_HEIGHT = 1.8;
+const SPHERE_RADIUS = 10;
+const HUD_WIDTH = 0.9;
+const HUD_LINES = 4;
+const HUD_LINE = 56;
+// Where the panel sits relative to the head: below the line of sight, at arm's length.
+const HUD_OFFSET = new THREE.Vector3(0, -0.55, -1.2);
+// How far the head turns from the panel before it swings back in front.
+const HUD_FOLLOW = THREE.MathUtils.degToRad(35);
+const UP = new THREE.Vector3(0, 1, 0);
+
+// Each eye's [u, v, width, height] in the frame.
+const PACKING: Record = {
+ mono: [
+ [0, 0, 1, 1],
+ [0, 0, 1, 1],
+ ],
+ "left-right": [
+ [0, 0, 0.5, 1],
+ [0.5, 0, 0.5, 1],
+ ],
+ "top-bottom": [
+ [0, 0.5, 1, 0.5],
+ [0, 0, 1, 0.5],
+ ],
+};
+
+// Paints decoded frames onto a screen or sphere according to projection
+export class Scene {
+ // Screen centre in content space, shared as the point-audio source.
+ readonly speaker = new THREE.Vector3(0, EYE, -2);
+
+ // Toggled by a controller squeeze or the M key.
+ readonly muted = new Signals.Signal(false);
+
+ // Rendered and decoded frames per second, updated once a second.
+ readonly fps = new Signals.Signal({ render: 0, video: 0 });
+
+ // Posed in content space each frame, so audio turns with the picture on a recentre.
+ listener?: Spatial;
+
+ #renderer: THREE.WebGLRenderer;
+ #scene = new THREE.Scene();
+ #content = new THREE.Group();
+ #camera: THREE.PerspectiveCamera;
+ #texture = new THREE.VideoFrameTexture();
+ #frame?: VideoFrame;
+ #eyes: THREE.Mesh[];
+ #hud: THREE.Mesh;
+ #hudCanvas = document.createElement("canvas");
+ // The panel's current and target yaw; unset snaps it in front on the next frame.
+ #hudYaw?: number;
+ #hudTarget = 0;
+ #headPosition = new THREE.Vector3();
+ #headDirection = new THREE.Vector3();
+ #relative = new THREE.Matrix4();
+ #renders = 0;
+ #frames = 0;
+ #since = performance.now();
+
+ #projection?: Projection;
+ #stereo = "mono";
+ #aspect = 16 / 9;
+
+ constructor(parent: HTMLElement) {
+ this.#renderer = new THREE.WebGLRenderer({ antialias: true });
+ this.#renderer.setPixelRatio(window.devicePixelRatio);
+ this.#renderer.setSize(window.innerWidth, window.innerHeight);
+ this.#renderer.xr.enabled = true;
+ this.#renderer.xr.setReferenceSpaceType("local-floor");
+ parent.append(this.#renderer.domElement, VRButton.createButton(this.#renderer));
+
+ this.#camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 0.01, 100);
+ this.#camera.position.set(0, EYE, 0);
+ this.#camera.layers.enable(1);
+
+ this.#scene.background = new THREE.Color(0x111111);
+ this.#scene.add(this.#content);
+ this.#texture.colorSpace = THREE.SRGBColorSpace;
+
+ this.#eyes = [1, 2].map((layer) => {
+ const map = this.#texture.clone();
+ const mesh = new THREE.Mesh(
+ new THREE.BufferGeometry(),
+ new THREE.MeshBasicMaterial({ map, toneMapped: false }),
+ );
+ mesh.layers.set(layer);
+ mesh.visible = false;
+ this.#content.add(mesh);
+ return mesh;
+ });
+ this.#build();
+
+ this.#hudCanvas.width = 1024;
+ this.#hudCanvas.height = HUD_LINES * HUD_LINE + 32;
+ const hudMap = new THREE.CanvasTexture(this.#hudCanvas);
+ hudMap.colorSpace = THREE.SRGBColorSpace;
+ this.#hud = new THREE.Mesh(
+ new THREE.PlaneGeometry(HUD_WIDTH, (HUD_WIDTH * this.#hudCanvas.height) / this.#hudCanvas.width),
+ new THREE.MeshBasicMaterial({ map: hudMap, transparent: true, toneMapped: false }),
+ );
+ // Outside the content, since it follows the head rather than the picture.
+ this.#scene.add(this.#hud);
+ this.#renderer.xr.addEventListener("sessionstart", () => {
+ this.#hudYaw = undefined;
+ });
+
+ // Trigger or pinch recentres; grip toggles mute.
+ for (const index of [0, 1]) {
+ const controller = this.#renderer.xr.getController(index);
+ controller.addEventListener("select", this.#recentre);
+ controller.addEventListener("squeeze", this.#toggleMute);
+ }
+
+ window.addEventListener("resize", this.#resize);
+ window.addEventListener("keydown", this.#key);
+ this.#look(this.#renderer.domElement);
+
+ this.#renderer.setAnimationLoop(() => {
+ this.#follow();
+ this.#renderer.render(this.#scene, this.#camera);
+ this.#relative.copy(this.#content.matrixWorld).invert().multiply(this.#head().matrixWorld);
+ this.listener?.pose(this.#relative);
+ this.#renders++;
+ this.#count();
+ });
+ }
+
+ frame(frame: VideoFrame) {
+ this.#frames++;
+ this.#texture.setFrame(frame);
+ for (const mesh of this.#eyes) {
+ if (mesh.material.map) mesh.material.map.needsUpdate = true;
+ mesh.visible = true;
+ }
+ this.#frame?.close();
+ this.#frame = frame;
+
+ const aspect = frame.displayWidth / frame.displayHeight;
+ if (aspect !== this.#aspect) {
+ this.#aspect = aspect;
+ this.#build();
+ }
+ }
+
+ // Layout for the picture according to projection and stereo
+ layout(projection?: Projection, stereo?: string) {
+ this.#projection = projection;
+ this.#stereo = stereo && stereo in PACKING ? stereo : "mono";
+ this.#build();
+ }
+
+ // Up to HUD_LINES lines on the in-headset panel.
+ status(lines: string[]) {
+ const context = this.#hudCanvas.getContext("2d");
+ if (!context) return;
+ const { width, height } = this.#hudCanvas;
+ context.clearRect(0, 0, width, height);
+ context.fillStyle = "rgba(0, 0, 0, 0.6)";
+ context.beginPath();
+ context.roundRect(0, 0, width, height, 24);
+ context.fill();
+ context.fillStyle = "#fff";
+ context.font = "40px monospace";
+ context.textBaseline = "middle";
+ for (const [index, line] of lines.slice(0, HUD_LINES).entries()) {
+ context.fillText(line, 32, 16 + HUD_LINE * (index + 0.5), width - 64);
+ }
+ if (this.#hud.material.map) this.#hud.material.map.needsUpdate = true;
+ }
+
+ // Hold the panel still through a glance, then glide it back in front once the head turns away.
+ #follow() {
+ const head = this.#head();
+ head.getWorldPosition(this.#headPosition);
+ head.getWorldDirection(this.#headDirection);
+ const yaw = Math.atan2(-this.#headDirection.x, -this.#headDirection.z);
+ const wrap = (angle: number) => Math.atan2(Math.sin(angle), Math.cos(angle));
+
+ if (this.#hudYaw === undefined) this.#hudYaw = this.#hudTarget = yaw;
+ if (Math.abs(wrap(yaw - this.#hudTarget)) > HUD_FOLLOW) this.#hudTarget = yaw;
+ this.#hudYaw += wrap(this.#hudTarget - this.#hudYaw) * 0.1;
+
+ this.#hud.position.copy(HUD_OFFSET).applyAxisAngle(UP, this.#hudYaw).add(this.#headPosition);
+ // Turned to face the head, then tilted up toward the eyes.
+ this.#hud.rotation.set(-0.4, this.#hudYaw, 0, "YXZ");
+ }
+
+ #count() {
+ const now = performance.now();
+ const elapsed = now - this.#since;
+ if (elapsed < 1000) return;
+ const rate = (count: number) => Math.round((count * 1000) / elapsed);
+ this.fps.set({ render: rate(this.#renders), video: rate(this.#frames) });
+ this.#renders = 0;
+ this.#frames = 0;
+ this.#since = now;
+ }
+
+ #head(): THREE.Camera {
+ return this.#renderer.xr.isPresenting ? this.#renderer.xr.getCamera() : this.#camera;
+ }
+
+ // Turn the content to face wherever the viewer is looking.
+ #recentre = () => {
+ const head = this.#head();
+ head.getWorldPosition(this.#headPosition);
+ head.getWorldDirection(this.#headDirection);
+ const yaw = Math.atan2(-this.#headDirection.x, -this.#headDirection.z);
+ this.#content.rotation.set(0, yaw, 0);
+ this.#content.position.set(this.#headPosition.x, 0, this.#headPosition.z);
+ this.#content.updateMatrixWorld();
+ };
+
+ #toggleMute = () => {
+ this.muted.set(!this.muted.peek());
+ };
+
+ #key = (event: KeyboardEvent) => {
+ if (event.key === "r") this.#recentre();
+ if (event.key === "m") this.#toggleMute();
+ };
+
+ #build() {
+ const packing = PACKING[this.#stereo];
+ const projection = this.#projection?.kind === "equirectangular" ? this.#projection : undefined;
+
+ for (const [eye, mesh] of this.#eyes.entries()) {
+ const [u, v, width, height] = packing[eye];
+ mesh.material.map?.offset.set(u, v);
+ mesh.material.map?.repeat.set(width, height);
+
+ mesh.geometry.dispose();
+ if (projection) {
+ const fov = THREE.MathUtils.degToRad(projection.fov ?? 360);
+ // Centred on the seam opposite phi = 0, then mirrored and turned so the picture's centre
+ // faces -Z and its left half wraps to the viewer's left.
+ mesh.geometry = new THREE.SphereGeometry(SPHERE_RADIUS, 64, 32, Math.PI - fov / 2, fov).scale(-1, 1, 1);
+ mesh.position.set(0, EYE, 0);
+ mesh.rotation.y = -Math.PI / 2;
+ } else {
+ // The packed frame's aspect, unpacked to one eye's.
+ const aspect = (this.#aspect * width) / height;
+ mesh.geometry = new THREE.PlaneGeometry(SCREEN_HEIGHT * aspect, SCREEN_HEIGHT);
+ mesh.position.copy(this.speaker);
+ mesh.rotation.y = 0;
+ }
+ }
+ }
+
+ #resize = () => {
+ this.#camera.aspect = window.innerWidth / window.innerHeight;
+ this.#camera.updateProjectionMatrix();
+ this.#renderer.setSize(window.innerWidth, window.innerHeight);
+ };
+
+ // Looking around is dragging on desktop; a headset tracks it.
+ #look(element: HTMLElement) {
+ const euler = new THREE.Euler(0, 0, 0, "YXZ");
+ element.addEventListener("pointermove", (event) => {
+ if (event.buttons !== 1) return;
+ euler.setFromQuaternion(this.#camera.quaternion);
+ euler.y += event.movementX * 0.005;
+ euler.x = THREE.MathUtils.clamp(euler.x + event.movementY * 0.005, -Math.PI / 2, Math.PI / 2);
+ this.#camera.quaternion.setFromEuler(euler);
+ });
+ }
+
+ close() {
+ this.#renderer.setAnimationLoop(null);
+ window.removeEventListener("resize", this.#resize);
+ window.removeEventListener("keydown", this.#key);
+ for (const index of [0, 1]) {
+ const controller = this.#renderer.xr.getController(index);
+ controller.removeEventListener("select", this.#recentre);
+ controller.removeEventListener("squeeze", this.#toggleMute);
+ }
+ for (const mesh of [...this.#eyes, this.#hud]) {
+ mesh.geometry.dispose();
+ mesh.material.map?.dispose();
+ mesh.material.dispose();
+ }
+ this.#texture.dispose();
+ this.#renderer.dispose();
+ this.#frame?.close();
+ }
+}
diff --git a/demo/vr/src/vite-env.d.ts b/demo/vr/src/vite-env.d.ts
new file mode 100644
index 00000000000..4ef6324115a
--- /dev/null
+++ b/demo/vr/src/vite-env.d.ts
@@ -0,0 +1,9 @@
+///
+
+interface ImportMetaEnv {
+ readonly VITE_RELAY_URL?: string;
+}
+
+interface ImportMeta {
+ readonly env: ImportMetaEnv;
+}
diff --git a/demo/vr/tsconfig.json b/demo/vr/tsconfig.json
new file mode 100644
index 00000000000..616947250ab
--- /dev/null
+++ b/demo/vr/tsconfig.json
@@ -0,0 +1,5 @@
+{
+ "extends": "../../js/tsconfig.json",
+ "compilerOptions": { "module": "esnext", "moduleResolution": "bundler", "outDir": "dist" },
+ "include": ["src", "../../js/common/worklet.d.ts"]
+}
diff --git a/demo/vr/vite.config.ts b/demo/vr/vite.config.ts
new file mode 100644
index 00000000000..1560c3f72bf
--- /dev/null
+++ b/demo/vr/vite.config.ts
@@ -0,0 +1,11 @@
+import { resolve } from "path";
+import { defineConfig } from "vite";
+import { worklet } from "../../js/common/vite-plugin-worklet.ts";
+
+export default defineConfig({
+ root: "src",
+ envDir: resolve(import.meta.dirname),
+ plugins: [worklet()],
+ build: { target: "esnext" },
+ server: { hmr: false },
+});
diff --git a/doc/bin/demo.md b/doc/bin/demo.md
index 2d779d90fc9..e893d8c451e 100644
--- a/doc/bin/demo.md
+++ b/doc/bin/demo.md
@@ -1,6 +1,6 @@
---
title: Demos
-description: The web demo, MoQ Boy, and CARLA
+description: The web demo, MoQ Boy, CARLA, and VR
---
# Demos
@@ -69,3 +69,28 @@ What it demonstrates:
- **Measured feedback.** Each truck's `status` track carries what the gate accepted and what the vehicle did, so the player draws requested against measured steering.
Code: [`demo/carla`](https://github.com/moq-dev/moq/tree/main/demo/carla).
+
+## VR
+
+A WebXR player for 360 and VR180 video with spatial audio, in a headset or a
+desktop tab. It lays a broadcast out by the catalog's
+[spatial fields](/concept/hang#catalog): an equirectangular `projection` wraps
+the picture on a sphere spanning its field of view, `stereo` crops each eye
+from a packed frame, and an `ambisonic` audio layout decodes the sound field
+around the viewer's head. Without them it shows a flat screen with the sound
+coming from it.
+
+```bash
+just relay # a local relay
+ffmpeg -re -i vr180.mp4 -c copy -strict unofficial \
+ -movflags cmaf+frag_keyframe+empty_moov+default_base_moof -f mp4 - |
+ moq --connect http://localhost:4443 --broadcast vr.hang import fmp4
+just vr serve # the player: Enter VR in a headset, or drag to look around
+```
+
+The importer reads the projection and stereo packing from an MP4's Spherical
+Video V2 boxes, which ffmpeg writes only past `-strict unofficial`, and the
+ambisonic layout from Opus channel mapping family 2. In the player, a trigger
+or `R` recentres the picture where you look, and a grip or `M` mutes.
+
+Code: [`demo/vr`](https://github.com/moq-dev/moq/tree/main/demo/vr).
diff --git a/doc/concept/hang.md b/doc/concept/hang.md
index d4c0269ded6..fe1b0afc9ba 100644
--- a/doc/concept/hang.md
+++ b/doc/concept/hang.md
@@ -69,7 +69,7 @@ A few things the catalog can express beyond decoder config:
- **Delay.** A rendition can also say how far its frames' timestamps trail those another rendition already wrote, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. A rendition alone measures none. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's.
- **Reorder depth.** A video rendition may declare `reorder`, the most frames that can precede one in decode order and follow it in presentation, as H.264 `max_num_reorder_frames` counts them; `0` declares none, which is not the same as absent. The Rust H.264 and H.265 importers copy it from the SPS when the SPS declares one (progressive H.264 only), and `@moq/publish` declares `0`. The MPEG-TS, Matroska, and FLV exporters then derive each group's decode times from that group's frames alone, so exports that joined at different groups agree, and they end the export on a frame reordered deeper than declared or a depth the codec level cannot reach. Without it they infer decode times from the timestamps they have read, as before. A publisher never changes it once declared.
- **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, an AV1 track's av1C, or Matroska, but not yet from H.264 or H.265 in fMP4 or FLV. Nothing in `@moq/watch` reads it yet.
-- **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. Nothing in `@moq/watch` renders them yet.
+- **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. The Rust importers fill them from MP4 `sv3d` and `st3d`, Matroska, and Opus channel mapping family 2. `@moq/watch` plays the picture and channels as they are; the [VR demo](/bin/demo#vr) lays them out in a headset.
- **Disabled renditions.** An audio or video rendition with `enabled: false` has no new frames coming, and a player must not select it. A subscriber may still get audio retained from before the disable, as when it subscribed a moment earlier. Disabling a rendition in `@moq/publish`, as hiding video does, keeps it in the catalog this way rather than removing it, so it is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. A muted `@moq/publish` microphone is not disabled: it stays enabled and sends nothing, ending each utterance with a discontinuity marker, so listeners stay subscribed and play the gap as intentional silence.
- **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero.
- **Join interval.** The root `media.join.interval` promises that every audio and video track starts a group at least that often: the longest GOP any of them publishes. It is fixed for the broadcast, and the [HLS gateway](/bin/hls) numbers every rendition's segments on the grid it fixes, segments the smallest whole multiple of it at least a second long, from each group's first timestamp, so renditions share segment numbers. A publisher that knows its GOP declares it; Rust publishers (`moq-mux` and everything built on it) without one measure each track over its first second or group, whichever is longer, and `@moq/publish` declares its encoders' `join`. A broadcast without it gets no HLS playlists.
diff --git a/doc/setup/examples.md b/doc/setup/examples.md
index d8de784ac25..698c73ce6e7 100644
--- a/doc/setup/examples.md
+++ b/doc/setup/examples.md
@@ -18,6 +18,7 @@ migration guides.
| Build a meeting with a roster and chat | [Rooms](/lib/js/room) | [`meet.ts`](https://github.com/moq-dev/moq/blob/main/demo/web/src/meet.ts) |
| Send input back to a live application | [MoQ Boy](/bin/demo#moq-boy) | [`demo/boy`](https://github.com/moq-dev/moq/tree/main/demo/boy) |
| Stream a simulator and send controls | [CARLA](/bin/demo#carla) | [`demo/carla`](https://github.com/moq-dev/moq/tree/main/demo/carla) |
+| Play 360 or VR180 video with spatial audio in a headset | [VR](/bin/demo#vr) | [`demo/vr`](https://github.com/moq-dev/moq/tree/main/demo/vr) |
| Bring an existing media tool | [OBS](/bin/obs), [GStreamer](/bin/gstreamer), or [gateways](/bin/#gateways) | [`demo/pub`](https://github.com/moq-dev/moq/tree/main/demo/pub), [`demo/sub`](https://github.com/moq-dev/moq/tree/main/demo/sub) |
Run checkout examples inside the [Nix dev shell](/setup/dev). Each demo's
diff --git a/js/watch/src/audio/config.test.ts b/js/watch/src/audio/config.test.ts
index fa9a5de101e..d7c3cb24b51 100644
--- a/js/watch/src/audio/config.test.ts
+++ b/js/watch/src/audio/config.test.ts
@@ -2,7 +2,7 @@ import { expect, test } from "bun:test";
import { Duration, yieldNow } from "@moq/clock";
import * as Catalog from "@moq/hang/catalog";
import { Effect, Signal } from "@moq/signals";
-import { decoderConfig, frameDuration, packetDuration, playbackIdentity } from "./config";
+import { decoderConfig, decoderDescription, frameDuration, packetDuration, playbackIdentity } from "./config";
const flush = () => yieldNow();
@@ -90,3 +90,42 @@ test("an implicit CMAF duration falls through to the Opus TOC", () => {
expect(packetDuration("opus", frame)).toEqual(Duration.millis(20));
expect(packetDuration("mp4a.40.2", frame)).toBeUndefined();
});
+
+test("an Opus description reaches WebCodecs with its mapping and no pre-skip", () => {
+ // OpusHead, version 1, 4 channels, pre-skip 312, 48 kHz, gain 0, family 2 with 4 streams.
+ const head = new Uint8Array([
+ ...new TextEncoder().encode("OpusHead"),
+ 1,
+ 4,
+ 0x38,
+ 0x01,
+ 0x80,
+ 0xbb,
+ 0,
+ 0,
+ 0,
+ 0,
+ 2,
+ 4,
+ 0,
+ 0,
+ 1,
+ 2,
+ 3,
+ ]);
+ const description = decoderDescription("opus", head);
+ expect(description?.slice(10, 12)).toEqual(new Uint8Array([0, 0]));
+ expect([...(description ?? [])].filter((_, index) => index < 10 || index > 11)).toEqual(
+ [...head].filter((_, index) => index < 10 || index > 11),
+ );
+ // The catalog's bytes are left alone.
+ expect(head[10]).toBe(0x38);
+
+ // WebCodecs reads only a signed OpusHead, so a dOps stays dropped, as a truncated header fails.
+ const dops = new Uint8Array([0, 2, 0x01, 0x38, 0, 0, 0xbb, 0x80, 0, 0, 0]);
+ expect(decoderDescription("opus", dops)).toBeUndefined();
+ expect(() => decoderDescription("opus", head.slice(0, 12))).toThrow();
+ expect(decoderDescription("opus", undefined)).toBeUndefined();
+ const asc = new Uint8Array([0x12, 0x10]);
+ expect(decoderDescription("mp4a.40.2", asc)).toBe(asc);
+});
diff --git a/js/watch/src/audio/config.ts b/js/watch/src/audio/config.ts
index 531dd4b5da9..9e52ee077c2 100644
--- a/js/watch/src/audio/config.ts
+++ b/js/watch/src/audio/config.ts
@@ -51,6 +51,27 @@ export function playbackIdentity(config: Catalog.AudioConfig): PlaybackIdentity
};
}
+const OPUS_HEAD = new TextEncoder().encode("OpusHead");
+
+/**
+ * The description to configure WebCodecs with: the catalog's, or the init segment's when it has none.
+ *
+ * An Opus stream past stereo cannot decode without its OpusHead's channel mapping, but Chromium then
+ * trims the header's pre-skip itself, again after every flush. The decoder's Terminal owns that trim,
+ * gap resumes included, so WebCodecs gets a copy with the pre-skip zeroed. WebCodecs reads only a
+ * signed OpusHead, so a dOps or an unsigned header stays dropped, as before.
+ */
+export function decoderDescription(codec: string, description: Uint8Array | undefined): Uint8Array | undefined {
+ if (!description || codec !== "opus") return description;
+ if (!OPUS_HEAD.every((byte, index) => description[index] === byte)) return undefined;
+ // Validates the header's length.
+ Util.Opus.preSkip(description);
+ const copy = description.slice();
+ // The two little-endian bytes after the version and channel count.
+ copy.fill(0, OPUS_HEAD.length + 2, OPUS_HEAD.length + 4);
+ return copy;
+}
+
/**
* A frame's duration as a constant over the catalog's sample rate, for codecs that fix it.
*
diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts
index 07fcd4ef723..f38fa967a86 100644
--- a/js/watch/src/audio/decoder.ts
+++ b/js/watch/src/audio/decoder.ts
@@ -20,7 +20,14 @@ import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feed
import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media";
import { DELAY_MAX, type Sync, type Track } from "../sync";
import { AudioBuffer } from "./buffer";
-import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config";
+import {
+ type DecoderConfig,
+ decoderDescription,
+ frameDuration,
+ type PlaybackIdentity,
+ packetDuration,
+ playbackIdentity,
+} from "./config";
import { Handover } from "./handover";
import { bound, maxDelay, ringHold, target } from "./latency";
import { Order } from "./order";
@@ -687,13 +694,10 @@ export class Decoder {
this.#ring.peek()?.dropped();
});
- // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header.
- const description =
- config.codec === "opus"
- ? undefined
- : config.description
- ? Util.Hex.toBytes(config.description)
- : undefined;
+ const description = decoderDescription(
+ config.codec,
+ config.description ? Util.Hex.toBytes(config.description) : undefined,
+ );
const decoderConfig: AudioDecoderConfig = {
codec: config.codec,
sampleRate: config.sampleRate,
@@ -759,14 +763,11 @@ export class Decoder {
.min(DELAY_MAX)
.asMicros(),
);
- // Opus in CMAF uses raw packets (not OGG-wrapped), so description must be omitted.
- // The dOps box from the init segment is not a valid OGG Identification Header.
- const description =
- config.codec === "opus"
- ? undefined
- : config.description
- ? Util.Hex.toBytes(config.description)
- : init.description;
+ // The init segment never yields Opus's dOps, which is not the OpusHead WebCodecs takes.
+ const description = decoderDescription(
+ config.codec,
+ config.description ? Util.Hex.toBytes(config.description) : init.description,
+ );
const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path });
diff --git a/js/watch/src/audio/supported.ts b/js/watch/src/audio/supported.ts
index 81cd8cb0a80..2e59517ba10 100644
--- a/js/watch/src/audio/supported.ts
+++ b/js/watch/src/audio/supported.ts
@@ -3,7 +3,7 @@ import * as Util from "@moq/hang/util";
import { hosted } from "../assets";
import { base64ToBytes } from "../base64";
import { supportCacheKey } from "../support-probe";
-import { decoderConfigKey } from "./config";
+import { decoderConfigKey, decoderDescription } from "./config";
/** Whether the WebCodecs audio decoder can play this config. */
export async function supported(config: Catalog.AudioConfig): Promise {
@@ -21,23 +21,21 @@ export async function supported(config: Catalog.AudioConfig): Promise {
console.warn(`audio: opus advertised at ${config.sampleRate}Hz, which some browsers cannot decode`);
}
- // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header.
+ // For Opus this is the OpusHead; the init segment never yields Opus's dOps in its place.
let description: Uint8Array | undefined;
- if (config.codec !== "opus") {
- if (config.description) {
- description = Util.Hex.toBytes(config.description);
- } else if (config.container.kind === "cmaf") {
- try {
- const Cmaf = await import("@moq/hang/cmaf");
- description = Cmaf.decodeInitSegment(base64ToBytes(config.container.init)).description;
- } catch (err) {
- // A malformed init segment means we can't extract the codec
- // description, so we can't probe support reliably. Reject the
- // track rather than letting isConfigSupported pass on a
- // description-less config and then having decode() fail later.
- console.warn(`audio: malformed CMAF init segment for codec ${config.codec}`, err);
- return false;
- }
+ if (config.description) {
+ description = Util.Hex.toBytes(config.description);
+ } else if (config.container.kind === "cmaf") {
+ try {
+ const Cmaf = await import("@moq/hang/cmaf");
+ description = Cmaf.decodeInitSegment(base64ToBytes(config.container.init)).description;
+ } catch (err) {
+ // A malformed init segment means we can't extract the codec
+ // description, so we can't probe support reliably. Reject the
+ // track rather than letting isConfigSupported pass on a
+ // description-less config and then having decode() fail later.
+ console.warn(`audio: malformed CMAF init segment for codec ${config.codec}`, err);
+ return false;
}
}
// Install the WebCodecs polyfill before probing, since playback would decode through it where
@@ -45,7 +43,7 @@ export async function supported(config: Catalog.AudioConfig): Promise {
await Util.Libav.polyfill(hosted.url);
const res = await AudioDecoder.isConfigSupported({
...config,
- description,
+ description: decoderDescription(config.codec, description),
});
return res.supported ?? false;
}
diff --git a/justfile b/justfile
index 59e7d4528ec..d8905f89bda 100644
--- a/justfile
+++ b/justfile
@@ -34,6 +34,7 @@ mod carla 'demo/carla'
mod pub 'demo/pub'
mod relay 'demo/relay'
mod sub 'demo/sub'
+mod vr 'demo/vr'
mod web 'demo/web'
# Run the demo by default.
diff --git a/package.json b/package.json
index 60a83df9346..a09cfccd755 100644
--- a/package.json
+++ b/package.json
@@ -59,6 +59,7 @@
"demo/boy",
"demo/web",
"demo/carla/web",
+ "demo/vr",
"test/audio-quality/clients/js",
"test/interop/clients/js",
"test/interop/clients/js-native",
diff --git a/quest/m1/vr-demo-try.md b/quest/m1/vr-demo-try.md
new file mode 100644
index 00000000000..5c65cef0d0e
--- /dev/null
+++ b/quest/m1/vr-demo-try.md
@@ -0,0 +1,24 @@
+# [XS] Try the VR demo in a headset, then host it at moq.dev
+
+## Goal
+
+The maintainer has played `demo/vr` in a WebXR headset, and once it plays
+well there, it is hosted beside the web demo at moq.dev with a looping VR180
+broadcast to watch.
+
+## Plan
+
+Decided 2026-11-14 (maintainer, pick round for
+[moq-dev/moq#8781](https://github.com/moq-dev/moq/pull/8781)): ✅ host it at
+moq.dev with a looping VR180 broadcast once it plays well in a headset
+(rejected: keep it local-only as a reference consumer).
+
+The harness (`just test vr`) proves the layout in headless Chromium on a
+desktop view only; a headset's per-eye render, recentre, and HRTF audio have
+had no human check. Run it with `just relay`, the `moq import fmp4` command in
+`doc/bin/demo.md#vr`, and `just vr serve`, then open the page on the headset
+(it needs a secure context, so serve it over HTTPS or tunnel to localhost).
+
+## Waiting
+
+- Maintainer: try it in a headset (an agent has none) and say whether it plays well enough to host. Check by this quest's Plan recording the result.
diff --git a/quest/m1/vr-demo.md b/quest/m1/vr-demo.md
deleted file mode 100644
index 8b69a945cce..00000000000
--- a/quest/m1/vr-demo.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# [M] A VR demo reads the catalog's spatial fields, tested end to end
-
-## Goal
-
-`demo/vr` plays a broadcast in a WebXR headset (or a desktop tab) and picks
-its layout from the hang catalog: video `projection` and its `fov` choose a
-flat screen or an equirectangular sphere, `stereo` crops each eye from a
-left-right or top-bottom frame, and an audio `layout` of `ambisonic` decodes
-the sound field around the head. A harness imports a small VR180 left-right
-fixture with first-order ambisonic Opus, publishes it through a relay, and
-asserts the demo sees and applies the right projection, fov, stereo, and
-layout.
-
-This gives the fields that
-[moq-dev/moq#5543](https://github.com/moq-dev/moq/pull/5543) added and
-[moq-dev/moq#7308](https://github.com/moq-dev/moq/pull/7308) fills on import
-their first reader, so they ship with the cut instead of being deleted.
-
-## Plan
-
-Decided 2026-11-14 (maintainer): keep the fields and finish
-[moq-dev/moq#6373](https://github.com/moq-dev/moq/pull/6373) so they are
-exercised (rejected: deleting them before the cut, as the feature review's
-`catalog-vr-fields` quest in
-[moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764) recommended).
-✅ m1, since a demo and a harness break nothing (rejected: m2, which holds
-breaking work).
-
-- Build on @nuts-rice's commits from `nuts-rice/moq:webxr_vr_demo`, keeping
- their authorship; the PR credits them and links #6373. Maintainer edits
- are off on their fork, so the work lands from a moq-dev branch.
-- Drop #6373's catalog plumbing that #7308 made redundant:
- `moq_mux::catalog::Spatial`, its producer and track hunks, the `import
- --projection/--stereo/--audio-layout` flags, and the `pub` on the hang
- `check` helpers. The importers fill the fields from MP4 `st3d`/`sv3d`,
- Matroska, and Opus family 2. Adding CLI or catalog API needs the
- maintainer first.
-- `@moq/watch` drops every Opus `description`, so Chromium refuses Opus past
- stereo ("description is required for multi-channel Opus"). Passing the
- catalog's OpusHead to both the support probe and the decoder, as the Rust
- importers intend, is part of this quest, since the ambisonic decode needs
- it. Chromium then trims the header's pre-skip itself, again after every
- flush (measured: 648 of 960 frames), so WebCodecs gets a copy with the
- pre-skip zeroed and the decoder's `Terminal` stays its one owner, gap
- resumes included.
-- The harness publishes the fixture through `moq import fmp4`, so it proves
- the importer's side too, and runs nightly in `interop.yml` beside the
- portrait harness, outside the merge gate.
-
-Public API: none (a behavior fix in `@moq/watch`). Wire: none.
-
-## Related
-
-- [fMP4 import fills the ambisonic layout](/quest/m5/mux-vr-ambisonic-fmp4.md) - the harness's fixture exercises that importer path
diff --git a/test/README.md b/test/README.md
index 9f54a250896..ab9dc98c219 100644
--- a/test/README.md
+++ b/test/README.md
@@ -20,6 +20,7 @@ questions to a harness. For builds and CI selection, see the
| [drain](drain/README.md) | `just test drain` | a JS viewer migrates off a draining relay without a dropped group |
| [stream-credit](stream-credit/README.md) | `just test stream-credit` | a browser tab reads 300 live tracks past its ~100-stream WebTransport cap, the highest priority first, and nothing hangs |
| qmux | `just test qmux` | `@moq/qmux`, as client and as server, interoperates with the Rust qmux 0.6.3 release on every QMux draft |
+| vr | `just test vr` | a VR180 left-right MP4 with ambisonic Opus keeps its catalog projection, stereo, and layout through import and a relay, and `demo/vr` lays out each eye and turns the sound field with the head |
| video | `just test video` | the `@moq/video` renderer paints through WebGPU and falls back to Canvas2D after losing the GPU |
| [json-properties](json-properties.md) | `just test json-properties` | both Snapshot codecs reconstruct, and both Window codecs emit the exact events for, random JSON sequences with replayable seeds |
diff --git a/test/interop/clients/js/vr.ts b/test/interop/clients/js/vr.ts
new file mode 100644
index 00000000000..6ad2166fa75
--- /dev/null
+++ b/test/interop/clients/js/vr.ts
@@ -0,0 +1,189 @@
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import type { Page } from "playwright";
+import { check, finishTraces, launch, open, throwPageErrors } from "./harness";
+
+const { values } = parseArgs({ options: { url: { type: "string" } } });
+if (!values.url) throw new Error("vr.ts requires --url");
+const run = process.env.HARNESS_RUN_DIR;
+if (!run) throw new Error("vr.ts runs under test/vr/run.sh, which builds the page into the run directory");
+
+const root = join(run, "vr-dist");
+const server = Bun.serve({
+ port: 0,
+ async fetch(req) {
+ let path = new URL(req.url).pathname;
+ if (path === "/") path = "/index.html";
+ const file = Bun.file(join(root, path));
+ if (!(await file.exists())) return new Response("not found", { status: 404 });
+ return new Response(file);
+ },
+});
+const origin = `http://localhost:${server.port}`;
+
+const VIEWPORT = { width: 640, height: 360 };
+// Radians the demo turns the camera per dragged pixel.
+const DRAG = 0.005;
+type Colour = "red" | "lime" | "blue" | "magenta" | "dark" | "other";
+
+// What demo/vr puts on `window.vr`, as far as this reads it.
+type Demo = {
+ video: { out: { config: { peek(): unknown } } };
+ audio: { out: { config: { peek(): unknown } } };
+ scene: { listener?: { output: AudioNode } };
+};
+
+// The colour at the centre of the page, read from a screenshot: WebGL discards its drawing buffer after
+// each frame, so the canvas itself reads back blank.
+async function centre(page: Page): Promise<{ colour: Colour; rgb: number[] }> {
+ const png = (await page.screenshot()).toString("base64");
+ const rgb = await page.evaluate(
+ async ({ png, x, y }) => {
+ const image = await createImageBitmap(await (await fetch(`data:image/png;base64,${png}`)).blob());
+ const canvas = new OffscreenCanvas(image.width, image.height);
+ const context = canvas.getContext("2d");
+ if (!context) throw new Error("no 2d context");
+ context.drawImage(image, 0, 0);
+ return Array.from(context.getImageData(x, y, 1, 1).data.slice(0, 3));
+ },
+ { png, x: VIEWPORT.width / 2, y: VIEWPORT.height / 2 },
+ );
+ const [r, g, b] = rgb;
+ const hi = (v: number) => v > 150;
+ const lo = (v: number) => v < 100;
+ const colour: Colour =
+ r < 40 && g < 40 && b < 40
+ ? "dark"
+ : hi(r) && lo(g) && lo(b)
+ ? "red"
+ : lo(r) && hi(g) && lo(b)
+ ? "lime"
+ : lo(r) && lo(g) && hi(b)
+ ? "blue"
+ : hi(r) && lo(g) && hi(b)
+ ? "magenta"
+ : "other";
+ return { colour, rgb };
+}
+
+async function waitForCentre(page: Page, want: Colour, timeout: number): Promise {
+ const deadline = Date.now() + timeout;
+ let last = await centre(page);
+ while (last.colour !== want && Date.now() < deadline) {
+ await page.waitForTimeout(250);
+ last = await centre(page);
+ }
+ check(last.colour === want, `centre is ${want}`, () => `got ${last.colour} rgb(${last.rgb.join(", ")})`);
+}
+
+// Turn the view left (right when negative) by dragging, as a desktop viewer looks around. The drag
+// starts at the far edge so it stays on the page, where every pointer move lands.
+async function turnLeft(page: Page, radians: number): Promise {
+ const { width, height } = VIEWPORT;
+ const distance = radians / DRAG;
+ check(Math.abs(distance) < width - 2, "the turn fits one drag", () => `${distance} px`);
+ const start = distance > 0 ? 1 : width - 1;
+ await page.mouse.move(start, height / 2);
+ await page.mouse.down();
+ await page.mouse.move(start + distance, height / 2, { steps: 20 });
+ await page.mouse.up();
+}
+
+// Each ear's RMS level at the demo's spatial mix, averaged over half a second of live playback.
+async function ears(page: Page): Promise<{ L: number; R: number }> {
+ return page.evaluate(async () => {
+ const output = (window as unknown as { vr: Demo }).vr.scene.listener?.output;
+ if (!output) throw new Error("the demo has no spatial audio");
+ const context = output.context;
+ const split = new ChannelSplitterNode(context, { numberOfOutputs: 2 });
+ output.connect(split);
+ const meters = [0, 1].map((channel) => {
+ const analyser = new AnalyserNode(context, { fftSize: 2048 });
+ split.connect(analyser, channel);
+ return analyser;
+ });
+ const sums = [0, 0];
+ const data = new Float32Array(2048);
+ for (let i = 0; i < 10; i++) {
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ for (const [channel, meter] of meters.entries()) {
+ meter.getFloatTimeDomainData(data);
+ sums[channel] += data.reduce((sum, v) => sum + v * v, 0) / data.length;
+ }
+ }
+ output.disconnect(split);
+ const [L, R] = sums.map((sum) => Math.sqrt(sum / 10));
+ return { L, R };
+ });
+}
+
+function louder(level: { L: number; R: number }, ear: "L" | "neither", assertion: string) {
+ const ratio = level.L / level.R;
+ const ok = level.L + level.R > 0.01 && (ear === "L" ? ratio > 2 : ratio > 0.8 && ratio < 1.25);
+ check(ok, assertion, () => `L ${level.L.toFixed(3)} R ${level.R.toFixed(3)}`);
+}
+
+const browser = await launch(["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"]);
+let failed = true;
+try {
+ const query = new URLSearchParams({ url: values.url, name: "vr.hang" });
+ const [page, errors] = await open(browser, `${origin}/?${query}`, "vr", true, { viewport: VIEWPORT });
+
+ // The catalog the demo read carries what the importer found in the MP4's sv3d, st3d, and dOps.
+ await page.waitForFunction(
+ () => {
+ const { vr } = window as unknown as { vr: Demo };
+ return vr.video.out.config.peek() !== undefined && vr.audio.out.config.peek() !== undefined;
+ },
+ undefined,
+ { timeout: 30_000 },
+ );
+ const catalog = await page.evaluate(() => {
+ const { vr } = window as unknown as { vr: Demo };
+ const video = vr.video.out.config.peek() as { projection?: unknown; stereo?: unknown };
+ const audio = vr.audio.out.config.peek() as { layout?: unknown; numberOfChannels?: unknown };
+ return {
+ projection: video.projection,
+ stereo: video.stereo,
+ layout: audio.layout,
+ channels: audio.numberOfChannels,
+ };
+ });
+ const want = {
+ projection: { kind: "equirectangular", fov: 180 },
+ stereo: "left-right",
+ layout: "ambisonic",
+ channels: 4,
+ };
+ check(Bun.deepEquals(catalog, want), "the catalog carries the fixture's VR fields", () => JSON.stringify(catalog));
+
+ // Straight ahead is the left eye's centre: the stereo packing cropped the frame and the
+ // equirectangular projection faced its centre forward.
+ await waitForCentre(page, "red", 30_000);
+
+ // The left eye's left quarter lies 45 degrees left over a 180 degree field.
+ await turnLeft(page, Math.PI / 4);
+ await waitForCentre(page, "lime", 5_000);
+
+ // 120 degrees left is past the field's edge: empty scene. A full sphere would show the frame's blue.
+ await turnLeft(page, (Math.PI * 75) / 180);
+ await waitForCentre(page, "dark", 5_000);
+
+ // Face forward again. The drags were gestures, so the audio graph is running. The fixture's sound
+ // is a plane wave from the left: the left ear hears it, and turning to face it centres it.
+ await turnLeft(page, (-Math.PI * 120) / 180);
+ const ahead = await ears(page);
+ louder(ahead, "L", "a source on the left is louder in the left ear");
+ await turnLeft(page, Math.PI / 2);
+ const facing = await ears(page);
+ louder(facing, "neither", "turning left to face that source centres it");
+
+ throwPageErrors(errors);
+ const level = ({ L, R }: { L: number; R: number }) => `L ${L.toFixed(3)} R ${R.toFixed(3)}`;
+ console.log(`vr: ${JSON.stringify(catalog)} laid out; ahead ${level(ahead)}, facing ${level(facing)}`);
+ failed = false;
+} finally {
+ await finishTraces(failed);
+ await browser.close();
+ server.stop(true);
+}
diff --git a/test/justfile b/test/justfile
index 070c538f528..7023141a463 100644
--- a/test/justfile
+++ b/test/justfile
@@ -115,6 +115,10 @@ smoke *args:
portrait:
./portrait/run.sh
+# Import a VR180 MP4 with ambisonic Opus and check demo/vr lays it out by the catalog's VR fields.
+vr:
+ ./vr/run.sh
+
# Negative control: no publisher, every subscriber must time out (proves the
# harness can actually report failure).
diff --git a/test/vr/relay.toml b/test/vr/relay.toml
new file mode 100644
index 00000000000..e9a52bad3ca
--- /dev/null
+++ b/test/vr/relay.toml
@@ -0,0 +1,15 @@
+# Private loopback relay for the VR projection harness.
+[log]
+level = "info"
+
+[listen]
+bind = "127.0.0.1:4443"
+tls.generate = ["localhost", "127.0.0.1"]
+# The CLI publisher negotiates lite-07; Chromium offers lite-06.
+version = ["moq-lite-07-wip", "moq-lite-06"]
+
+[web.http]
+listen = "127.0.0.1:4443"
+
+[auth]
+public = "**"
diff --git a/test/vr/run.sh b/test/vr/run.sh
new file mode 100755
index 00000000000..62f6ce6ccbc
--- /dev/null
+++ b/test/vr/run.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Import a VR180 left-right MP4 with ambisonic Opus, publish it through a relay, and check demo/vr
+# reads the catalog's projection, stereo, and layout and lays out the picture and sound by them.
+set -euo pipefail
+
+if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then
+ exec bash "$(dirname "$0")/../../sh/local.sh" bash "$0" "$@"
+fi
+DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+WORKSPACE=$(cd "$DIR/../.." && pwd)
+# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh
+source "$DIR/../lib/harness.sh"
+harness_begin vr 'just test vr'
+cd "$WORKSPACE"
+cargo build --locked -p moq-relay -p moq-cli
+uv run --no-project python sh/js/install.py frozen
+(cd demo/vr && bunx vite build --outDir "$HARNESS_RUN/vr-dist" --emptyOutDir)
+
+# Each eye is 480x480 over 180 degrees. Left eye: blue, red straight ahead (u 0.5), lime 45 degrees
+# left (u 0.25). Right eye: magenta, so an uncropped frame puts magenta ahead. The sound is a plane
+# wave from the left in first-order AmbiX (ACN W, Y, Z, X), so W and Y carry the same tone.
+ffmpeg -hide_banner -loglevel error -f lavfi \
+ -i "color=c=blue:s=960x480:r=15:d=2,drawbox=x=210:y=210:w=60:h=60:c=red:t=fill,drawbox=x=90:y=210:w=60:h=60:c=lime:t=fill,drawbox=x=480:y=0:w=480:h=480:c=magenta:t=fill" \
+ -f lavfi -i "aevalsrc=exprs='0.5*sin(2*PI*1000*t)|0.5*sin(2*PI*1000*t)|0|0':s=48000:d=2" \
+ -c:v libx264 -preset ultrafast -profile:v baseline -pix_fmt yuv420p -g 15 \
+ -c:a libopus -mapping_family 2 -b:a 128k "$HARNESS_RUN/flat.mp4"
+uv run --no-project python -I "$DIR/spherical.py" "$HARNESS_RUN/flat.mp4" "$HARNESS_RUN/vr180.mp4"
+
+harness_port relay
+URL="http://127.0.0.1:$HARNESS_PORT"
+sed "s/4443/$HARNESS_PORT/g" "$DIR/relay.toml" >"$HARNESS_RUN/relay.toml"
+harness_spawn relay "$HARNESS_RUN/relay.log" "$WORKSPACE/target/debug/moq-relay" "$HARNESS_RUN/relay.toml"
+harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID"
+# ffmpeg carries st3d and sv3d into each fragmented init segment only past -strict unofficial.
+# shellcheck disable=SC2329 # invoked by harness_spawn
+publish_vr() {
+ ffmpeg -hide_banner -loglevel error -re -stream_loop -1 -i "$HARNESS_RUN/vr180.mp4" -c copy -strict unofficial \
+ -movflags cmaf+frag_keyframe+empty_moov+default_base_moof -f mp4 - |
+ "$WORKSPACE/target/debug/moq" --connect "$URL" --connect-version moq-lite-07-wip --broadcast vr.hang import fmp4
+}
+harness_spawn publisher "$HARNESS_RUN/publisher.log" publish_vr
+cd test/interop/clients/js
+# SwiftShader stands in for a GPU. ANGLE asks its Vulkan for a Wayland surface, which it lacks,
+# whenever a Wayland session is visible, so a desktop run would have no WebGL.
+env -u WAYLAND_DISPLAY bun vr.ts --url "$URL"
diff --git a/test/vr/spherical.py b/test/vr/spherical.py
new file mode 100644
index 00000000000..cba22dedb8e
--- /dev/null
+++ b/test/vr/spherical.py
@@ -0,0 +1,76 @@
+"""Mark an MP4's video as VR180 left-right with Spherical Video V2 boxes.
+
+ffmpeg reads and remuxes `st3d` and `sv3d` but cannot create them, so this appends both to the
+`avc1` sample entry and grows every enclosing box. The `moov` must follow `mdat`, as ffmpeg writes
+it without `+faststart`, so no chunk offset moves.
+
+Usage: python -I spherical.py in.mp4 out.mp4
+"""
+
+import struct
+import sys
+
+CONTAINERS = {b"moov", b"trak", b"mdia", b"minf", b"stbl"}
+
+
+def box(kind: bytes, body: bytes) -> bytes:
+ return struct.pack(">I", 8 + len(body)) + kind + body
+
+
+def full(kind: bytes, body: bytes) -> bytes:
+ return box(kind, b"\0\0\0\0" + body)
+
+
+# Each equirectangular bound is the 0.32 fixed-point fraction of the sphere left uncovered: a
+# quarter off each side leaves 180 degrees of longitude.
+QUARTER = 1 << 30
+POSE = full(b"prhd", struct.pack(">iii", 0, 0, 0))
+EQUI = full(b"equi", struct.pack(">IIII", 0, 0, QUARTER, QUARTER))
+# Stereo mode 2 is left-right.
+SPATIAL = full(b"st3d", bytes([2])) + box(b"sv3d", full(b"svhd", b"moq\0") + box(b"proj", POSE + EQUI))
+
+
+def patch(data: bytearray, start: int, end: int) -> int:
+ """Append SPATIAL to the avc1 entry within [start, end), returning how much the range grew."""
+ at = start
+ while at < end:
+ size, kind = struct.unpack(">I4s", data[at : at + 8])
+ if size < 8:
+ raise ValueError(f"unsupported box size {size} for {kind!r}")
+ grown = 0
+ if kind in CONTAINERS:
+ grown = patch(data, at + 8, at + size)
+ elif kind == b"stsd":
+ # A full box and an entry count precede the sample entries.
+ grown = patch(data, at + 16, at + size)
+ elif kind == b"avc1":
+ data[at + size : at + size] = SPATIAL
+ grown = len(SPATIAL)
+ if grown:
+ struct.pack_into(">I", data, at, size + grown)
+ return grown
+ at += size
+ return 0
+
+
+def main() -> None:
+ source, target = sys.argv[1:]
+ with open(source, "rb") as file:
+ data = bytearray(file.read())
+ kinds, at = [], 0
+ while at < len(data):
+ size, kind = struct.unpack(">I4s", data[at : at + 8])
+ if size < 8:
+ raise ValueError(f"unsupported box size {size} for {kind!r}")
+ kinds.append(kind)
+ at += size
+ # The mdat must precede the moov, so growing the moov moves no sample.
+ if kinds.index(b"moov") < kinds.index(b"mdat"):
+ raise ValueError("moov precedes mdat; write the source without +faststart")
+ if not patch(data, 0, len(data)):
+ raise ValueError("no avc1 sample entry")
+ with open(target, "wb") as file:
+ file.write(data)
+
+
+main()